From 65886363a46a72bebd08c1789eeb22ebbaf5c49b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 5 Jun 2019 12:10:08 +1000 Subject: [PATCH 01/18] DCD-386: Start configuration of DIY backup. --- aws_bitbucket_nfs_node.yml | 1 + group_vars/aws_node_local.yml | 14 +++-- .../diy_backup/molecule/default/playbook.yml | 12 +++++ .../molecule/default/tests/test_default.py | 4 ++ roles/diy_backup/tasks/main.yml | 6 +++ .../templates/bitbucket.diy-backup.vars.sh.j2 | 53 +++++++++++++++++++ 6 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 diff --git a/aws_bitbucket_nfs_node.yml b/aws_bitbucket_nfs_node.yml index 7227539..8e95593 100644 --- a/aws_bitbucket_nfs_node.yml +++ b/aws_bitbucket_nfs_node.yml @@ -12,3 +12,4 @@ - role: linux_common - role: aws_common - role: nfs_server + - role: diy_backup diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 5d0f330..d983341 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -80,6 +80,7 @@ atl_catalina_opts: "{{ lookup('env', 'ATL_CATALINA_OPTS') or '' }}" atl_proxy_name: "{{ lookup('env', 'ATL_PROXY_NAME') | lower }}" atl_proxy_port: "{{ lookup('env', 'ATL_TOMCAT_PROXYPORT') }}" atl_tomcat_port: "{{ lookup('env', 'ATL_TOMCAT_DEFAULTCONNECTORPORT') or '8080' }}" +atl_ssl_proxy: "{{ lookup('env', 'ATL_SSL_PROXY') or 'false' }}" atl_tomcat_acceptcount: "{{ lookup('env', 'ATL_TOMCAT_ACCEPTCOUNT') or '10' }}" atl_tomcat_connectiontimeout: "{{ lookup('env', 'ATL_TOMCAT_CONNECTIONTIMEOUT') or '20000' }}" @@ -93,6 +94,13 @@ atl_tomcat_scheme: "{{ lookup('env', 'ATL_TOMCAT_SCHEME') or 'http' }}" atl_tomcat_secure: "{{ lookup('env', 'ATL_TOMCAT_SECURE') or 'false' }}" atl_fileserver_host: "{{ lookup('env', 'ATL_FILESERVER_IP') }}" -atl_elasticsearch_endpoint: "{{ lookup('env', 'ATL_ELASTICSEARCH_ENDPOINT') }}" - -atl_ssl_proxy: "{{ lookup('env', 'ATL_SSL_PROXY') or 'false' }}" +atl_elasticsearch_host: "{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}" +atl_elasticsearch_endpoint: "http://{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}" +atl_nfs_server_device: "{{ lookup('env', 'ATL_NFS_SERVER_DEVICE') }}" +atl_nfs_disk_volume_type: "{{ lookup('env', 'ATL_NFS_DISK_VOLUME_TYPE') }}" +atl_nfs_disk_volume_iops: "{{ lookup('env', 'ATL_NFS_DISK_VOLUME_IOPS') }}" +atl_rds_instance_id: "{{ lookup('env', 'ATL_RDS_INSTANCE_ID') }}" +atl_rds_instance_class: "{{ lookup('env', 'ATL_RDS_INSTANCE_CLASS') }}" +atl_rds_multi_az: "{{ lookup('env', 'ATL_RDS_MULTI_AZ') }}" +atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}" +atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}" diff --git a/roles/diy_backup/molecule/default/playbook.yml b/roles/diy_backup/molecule/default/playbook.yml index d46bce6..88861a3 100644 --- a/roles/diy_backup/molecule/default/playbook.yml +++ b/roles/diy_backup/molecule/default/playbook.yml @@ -6,6 +6,18 @@ atl_product_family: "stash" atl_product_edition: "bitbucket" + atl_aws_iam_role: "atl_aws_iam_role" + atl_aws_stack_name: "atl_aws_stack_name" + atl_nfs_disk_volume_iops: "atl_nfs_disk_volume_iops" + atl_nfs_disk_volume_type: "atl_nfs_disk_volume_type" + atl_nfs_fs_type: "atl_nfs_fs_type" + atl_proxy_name: "atl_proxy_name" + atl_rds_instance_class: "atl_rds_instance_class" + atl_rds_instance_id: "atl_rds_instance_id" + atl_rds_multi_az: "atl_rds_multi_az" + atl_rds_security_group: "atl_rds_security_group" + atl_rds_subnet_group_name: "atl_rds_subnet_group_name" + pre_tasks: - name: Create base dir file: diff --git a/roles/diy_backup/molecule/default/tests/test_default.py b/roles/diy_backup/molecule/default/tests/test_default.py index cf881db..580f08b 100644 --- a/roles/diy_backup/molecule/default/tests/test_default.py +++ b/roles/diy_backup/molecule/default/tests/test_default.py @@ -10,3 +10,7 @@ def test_git_clone(host): f = host.file('/opt/atlassian/bitbucket-diy-backup') assert f.exists assert f.is_directory + +def test_diy_config(host): + f = host.file('/opt/atlassian/bitbucket-diy-backup/bitbucket.diy-backup.vars.sh') + assert f.exists diff --git a/roles/diy_backup/tasks/main.yml b/roles/diy_backup/tasks/main.yml index 8b066ec..c675375 100644 --- a/roles/diy_backup/tasks/main.yml +++ b/roles/diy_backup/tasks/main.yml @@ -6,3 +6,9 @@ repo: "{{ atl_diy_backup_repo }}" dest: "{{ atl_diy_backup_dir }}" version: "master" + +- name: Configure DIY backup for BB on AWS + template: + src: "bitbucket.diy-backup.vars.sh.j2" + dest: "{{ atl_diy_backup_dir }}/bitbucket.diy-backup.vars.sh" + mode: 0640 diff --git a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 new file mode 100644 index 0000000..32b06ff --- /dev/null +++ b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 @@ -0,0 +1,53 @@ +INSTANCE_NAME={{ atl_aws_stack_name }} + +AWS_INFO=$(curl -Lsf http://169.254.169.254/latest/dynamic/instance-identity/document) +AWS_ACCOUNT_ID=$(echo "${AWS_INFO}" | jq -r .accountId) +AWS_AVAILABILITY_ZONE=$(echo "${AWS_INFO}" | jq -r .availabilityZone) +AWS_REGION=$(echo "${AWS_INFO}" | jq -r .region) +AWS_EC2_INSTANCE_ID=$(echo "${AWS_INFO}" | jq -r .instanceId) + +BITBUCKET_VERBOSE_BACKUP=true +KEEP_BACKUPS=5 + +{% if atl_ssl_proxy is defined and atl_ssl_proxy %} +BITBUCKET_URL=https://{{ atl_proxy_name }} +{% else %} +BITBUCKET_URL=http://{{ atl_proxy_name }} +{% endif %} + +BITBUCKET_UID={{ atl_product_user }} +BITBUCKET_GID={{ atl_product_user }} + +BACKUP_ZERO_DOWNTIME=true + +BACKUP_ELASTICSEARCH_TYPE=amazon-es +BACKUP_ARCHIVE_TYPE=aws-snapshots + +BACKUP_DISK_TYPE=amazon-ebs +STANDBY_DISK_TYPE=none +EBS_VOLUME_MOUNT_POINT_AND_DEVICE_NAMES=({{ atl_shared_mountpoint }}:{{ atl_nfs_server_device }}) +HOME_DIRECTORY_MOUNT_POINT={{ atl_shared_mountpoint }} +RESTORE_DISK_VOLUME_TYPE={{ atl_nfs_disk_volume_type }} +RESTORE_DISK_IOPS={{ atl_nfs_disk_volume_iops }} +FILESYSTEM_TYPE={{ atl_nfs_fs_type }} + +BACKUP_DATABASE_TYPE=amazon-rds +RDS_INSTANCE_ID={{ atl_rds_instance_id }} +RESTORE_RDS_INSTANCE_CLASS={{ atl_rds_instance_class }} +RESTORE_RDS_MULTI_AZ={{ atl_rds_multi_az }} +RESTORE_RDS_SUBNET_GROUP_NAME={{ atl_rds_subnet_group_name }} +RESTORE_RDS_SECURITY_GROUP={{ atl_rds_security_group }} +DR_RDS_READ_REPLICA= + +BACKUP_DEST_AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID +BACKUP_DEST_REGION=$AWS_REGION +BACKUP_DEST_AWS_ROLE={{ atl_aws_iam_role }} + +ELASTICSEARCH_INDEX_NAME=bitbucket-search-v1 +ELASTICSEARCH_REPOSITORY_NAME=bitbucket-snapshots +ELASTICSEARCH_HOST={{ atl_elasticsearch_host }} +ELASTICSEARCH_S3_BUCKET= +ELASTICSEARCH_S3_BUCKET_REGION=us-east-1 +ELASTICSEARCH_SNAPSHOT_IAM_ROLE={{ atl_aws_iam_role }} + +STANDBY_JDBC_URL= From 76db370513f1d235c9a0f00d14edf54f8c60d18f Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 5 Jun 2019 13:03:59 +1000 Subject: [PATCH 02/18] DCD-386: Add backup wrapper and some fixes. --- roles/diy_backup/tasks/main.yml | 6 ++++++ .../templates/bitbucket.diy-backup.vars.sh.j2 | 4 +++- roles/diy_backup/templates/run-backup.j2 | 13 +++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 roles/diy_backup/templates/run-backup.j2 diff --git a/roles/diy_backup/tasks/main.yml b/roles/diy_backup/tasks/main.yml index c675375..3f1b1be 100644 --- a/roles/diy_backup/tasks/main.yml +++ b/roles/diy_backup/tasks/main.yml @@ -12,3 +12,9 @@ src: "bitbucket.diy-backup.vars.sh.j2" dest: "{{ atl_diy_backup_dir }}/bitbucket.diy-backup.vars.sh" mode: 0640 + +- name: Install backup wrapper script + template: + src: "run-backup.j2" + dest: "{{ atl_installation_base }}/bin/run-backup" + mode: 0750 diff --git a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 index 32b06ff..addfc1e 100644 --- a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 +++ b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 @@ -1,3 +1,5 @@ +# Created by Ansible + INSTANCE_NAME={{ atl_aws_stack_name }} AWS_INFO=$(curl -Lsf http://169.254.169.254/latest/dynamic/instance-identity/document) @@ -9,7 +11,7 @@ AWS_EC2_INSTANCE_ID=$(echo "${AWS_INFO}" | jq -r .instanceId) BITBUCKET_VERBOSE_BACKUP=true KEEP_BACKUPS=5 -{% if atl_ssl_proxy is defined and atl_ssl_proxy %} +{% if atl_ssl_proxy is defined and atl_ssl_proxy == 'true' %} BITBUCKET_URL=https://{{ atl_proxy_name }} {% else %} BITBUCKET_URL=http://{{ atl_proxy_name }} diff --git a/roles/diy_backup/templates/run-backup.j2 b/roles/diy_backup/templates/run-backup.j2 new file mode 100644 index 0000000..074b296 --- /dev/null +++ b/roles/diy_backup/templates/run-backup.j2 @@ -0,0 +1,13 @@ +#!/bin/bash + +cd {{ atl_diy_backup_repo }} + +if [[ -f .venv/bin/activate ]]; then + source .venv/bin/activate +else + virtualenv .venv + source .venv/bin/activate + pip install boto +fi + +./bitbucket.diy-backup.sh From 696b3b4a8badcd37f9098922d8d16c35d0cf73c5 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 5 Jun 2019 13:05:50 +1000 Subject: [PATCH 03/18] DCD-386: Fix path. --- roles/diy_backup/templates/run-backup.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/diy_backup/templates/run-backup.j2 b/roles/diy_backup/templates/run-backup.j2 index 074b296..403737e 100644 --- a/roles/diy_backup/templates/run-backup.j2 +++ b/roles/diy_backup/templates/run-backup.j2 @@ -1,6 +1,6 @@ #!/bin/bash -cd {{ atl_diy_backup_repo }} +cd {{ atl_diy_backup_dir }} if [[ -f .venv/bin/activate ]]; then source .venv/bin/activate From bee4939382f20730240c9135716771a57a4bd767 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 5 Jun 2019 14:20:19 +1000 Subject: [PATCH 04/18] DCD-386: Use the supplied backup bucket name. --- group_vars/aws_node_local.yml | 2 ++ roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index d983341..7ed87b9 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -96,6 +96,8 @@ atl_tomcat_secure: "{{ lookup('env', 'ATL_TOMCAT_SECURE') or 'false' }}" atl_fileserver_host: "{{ lookup('env', 'ATL_FILESERVER_IP') }}" atl_elasticsearch_host: "{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}" atl_elasticsearch_endpoint: "http://{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}" +atl_elasticsearch_s3_bucket: "{{ lookup('env', 'ATL_ELASTICSEARCH_S3_BUCKET') }}" + atl_nfs_server_device: "{{ lookup('env', 'ATL_NFS_SERVER_DEVICE') }}" atl_nfs_disk_volume_type: "{{ lookup('env', 'ATL_NFS_DISK_VOLUME_TYPE') }}" atl_nfs_disk_volume_iops: "{{ lookup('env', 'ATL_NFS_DISK_VOLUME_IOPS') }}" diff --git a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 index addfc1e..37217ba 100644 --- a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 +++ b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 @@ -48,7 +48,7 @@ BACKUP_DEST_AWS_ROLE={{ atl_aws_iam_role }} ELASTICSEARCH_INDEX_NAME=bitbucket-search-v1 ELASTICSEARCH_REPOSITORY_NAME=bitbucket-snapshots ELASTICSEARCH_HOST={{ atl_elasticsearch_host }} -ELASTICSEARCH_S3_BUCKET= +ELASTICSEARCH_S3_BUCKET={{ atl_elasticsearch_s3_bucket }} ELASTICSEARCH_S3_BUCKET_REGION=us-east-1 ELASTICSEARCH_SNAPSHOT_IAM_ROLE={{ atl_aws_iam_role }} From 345736920052405e58e1cd75b14d11341d7e4bec Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 5 Jun 2019 15:00:34 +1000 Subject: [PATCH 05/18] DCD-386: No need to archive snapshots. --- roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 index 37217ba..5511ca4 100644 --- a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 +++ b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 @@ -23,7 +23,7 @@ BITBUCKET_GID={{ atl_product_user }} BACKUP_ZERO_DOWNTIME=true BACKUP_ELASTICSEARCH_TYPE=amazon-es -BACKUP_ARCHIVE_TYPE=aws-snapshots +BACKUP_ARCHIVE_TYPE= BACKUP_DISK_TYPE=amazon-ebs STANDBY_DISK_TYPE=none From e84b0da43c1642ed37b7b5313353389cb9fdd066 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 6 Jun 2019 13:45:53 +1000 Subject: [PATCH 06/18] DCD-386: DIY backup fixes. --- group_vars/aws_node_local.yml | 1 + roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 7ed87b9..2208f82 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -43,6 +43,7 @@ atl_efs_id: "{{ lookup('env', 'ATL_EFS_ID') }}" atl_aws_stack_name: "{{ lookup('env', 'ATL_AWS_STACK_NAME') }}" atl_aws_region: "{{ lookup('env', 'ATL_AWS_REGION') }}" atl_aws_iam_role: "{{ lookup('env', 'ATL_AWS_IAM_ROLE') }}" +atl_aws_iam_role_arn: "{{ lookup('env', 'ATL_AWS_IAM_ROLE_ARN') }}" atl_db_engine: "{{ lookup('env', 'ATL_DB_ENGINE') }}" atl_db_host: "{{ lookup('env', 'ATL_DB_HOST') }}" diff --git a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 index 5511ca4..27e29e7 100644 --- a/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 +++ b/roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2 @@ -43,13 +43,13 @@ DR_RDS_READ_REPLICA= BACKUP_DEST_AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID BACKUP_DEST_REGION=$AWS_REGION -BACKUP_DEST_AWS_ROLE={{ atl_aws_iam_role }} +BACKUP_DEST_AWS_ROLE={{ atl_aws_iam_role_arn }} ELASTICSEARCH_INDEX_NAME=bitbucket-search-v1 ELASTICSEARCH_REPOSITORY_NAME=bitbucket-snapshots ELASTICSEARCH_HOST={{ atl_elasticsearch_host }} ELASTICSEARCH_S3_BUCKET={{ atl_elasticsearch_s3_bucket }} ELASTICSEARCH_S3_BUCKET_REGION=us-east-1 -ELASTICSEARCH_SNAPSHOT_IAM_ROLE={{ atl_aws_iam_role }} +ELASTICSEARCH_SNAPSHOT_IAM_ROLE={{ atl_aws_iam_role_arn }} STANDBY_JDBC_URL= From 56f4f8db3758c93499dc2a45502200bf7a243154 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Thu, 6 Jun 2019 15:37:37 +1000 Subject: [PATCH 07/18] DCD-386: The BB node mounts NFS deeper in the FS. --- aws_bitbucket_dc_node.yml | 4 ++++ roles/nfs_mount/defaults/main.yml | 4 ++++ roles/nfs_mount/tasks/main.yml | 4 ++-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 roles/nfs_mount/defaults/main.yml diff --git a/aws_bitbucket_dc_node.yml b/aws_bitbucket_dc_node.yml index 0f97654..3108bf8 100644 --- a/aws_bitbucket_dc_node.yml +++ b/aws_bitbucket_dc_node.yml @@ -8,6 +8,10 @@ atl_product_edition: "bitbucket" atl_product_user: "bitbucket" + atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared" + atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared" + + atl_startup_systemd_params: - "UMask=0027" - "LimitNOFILE=4096" diff --git a/roles/nfs_mount/defaults/main.yml b/roles/nfs_mount/defaults/main.yml new file mode 100644 index 0000000..6464ddd --- /dev/null +++ b/roles/nfs_mount/defaults/main.yml @@ -0,0 +1,4 @@ +--- + +atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}" +atl_nfs_target: "{{ atl_shared_mountpoint }}" diff --git a/roles/nfs_mount/tasks/main.yml b/roles/nfs_mount/tasks/main.yml index 19abff9..9e170e4 100644 --- a/roles/nfs_mount/tasks/main.yml +++ b/roles/nfs_mount/tasks/main.yml @@ -10,8 +10,8 @@ - name: Enable mountpoint in fstab mount: - path: "{{ atl_shared_mountpoint }}" - src: "{{ atl_fileserver_host }}:{{ atl_shared_mountpoint }}" + src: "{{ atl_fileserver_host }}:{{ atl_nfs_target }}" + path: "{{ atl_nfs_mountpoint }}" fstype: nfs opts: "rw,nfsvers=4.1,lookupcache=pos,noatime,intr,rsize=32768,wsize=32768,_netdev" state: mounted From 6955cf04dd87234e344e81fbb70b14bd41fb9dfe Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Fri, 7 Jun 2019 11:36:21 +1000 Subject: [PATCH 08/18] DCD-386: Create the shared mount point as BB expects it to exist. --- aws_bitbucket_nfs_node.yml | 3 +++ roles/nfs_server/tasks/main.yml | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/aws_bitbucket_nfs_node.yml b/aws_bitbucket_nfs_node.yml index 8e95593..e5fd0c8 100644 --- a/aws_bitbucket_nfs_node.yml +++ b/aws_bitbucket_nfs_node.yml @@ -8,6 +8,9 @@ atl_product_edition: "bitbucket" atl_product_user: "bitbucket" + atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared" + atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared" + roles: - role: linux_common - role: aws_common diff --git a/roles/nfs_server/tasks/main.yml b/roles/nfs_server/tasks/main.yml index 14aaeab..d032589 100644 --- a/roles/nfs_server/tasks/main.yml +++ b/roles/nfs_server/tasks/main.yml @@ -33,6 +33,15 @@ state: mounted +- name: Create the shared home as BB mounts this directly + file: + path: "{{ atl_shared_mountpoint }}/bitbucket/shared" + state: directory + owner: "{{ atl_product_user }}" + group: "{{ atl_product_user }}" + mode: 0750 + + - name: Create the NFS export file template: src: "media-atl.exports.j2" From 5976b7f7064003479b9274967a8cc8030ac3d09b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 09:43:26 +1000 Subject: [PATCH 09/18] DCD-386: Use the six module to paper over Python changes. --- bin/run-all-tests | 2 +- .../molecule/bitbucket_latest/tests/test_default.py | 2 +- .../molecule/confluence_latest/tests/test_default.py | 2 +- roles/product_install/molecule/default/tests/test_default.py | 2 +- .../molecule/jira_cached_with_downgrade/tests/test_default.py | 2 +- .../molecule/jira_cached_with_upgrade/tests/test_default.py | 2 +- .../molecule/jira_software_latest/tests/test_default.py | 2 +- .../molecule/jira_version_from_file/tests/test_default.py | 2 +- .../molecule/jira_version_latest/tests/test_default.py | 2 +- .../molecule/jira_version_override/tests/test_default.py | 2 +- roles/product_install/molecule/resources/tests/test_common.py | 2 +- .../product_install/molecule/servicedesk3/tests/test_default.py | 2 +- .../product_install/molecule/servicedesk4/tests/test_default.py | 2 +- .../molecule/servicedesk_latest/tests/test_default.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/run-all-tests b/bin/run-all-tests index a7077b5..b5dc601 100755 --- a/bin/run-all-tests +++ b/bin/run-all-tests @@ -5,7 +5,7 @@ set -e # Use Ansible from virtualenv if provided if [[ -f .venv/bin/ansible ]]; then source .venv/bin/activate - pip install molecule docker + pip install molecule docker six fi for role in `find roles/ -name molecule`; do diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index 18e4662..d83ada5 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import json import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/confluence_latest/tests/test_default.py b/roles/product_install/molecule/confluence_latest/tests/test_default.py index 9cf0634..72a5adf 100644 --- a/roles/product_install/molecule/confluence_latest/tests/test_default.py +++ b/roles/product_install/molecule/confluence_latest/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import json import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/default/tests/test_default.py b/roles/product_install/molecule/default/tests/test_default.py index dc659f1..82f1205 100644 --- a/roles/product_install/molecule/default/tests/test_default.py +++ b/roles/product_install/molecule/default/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py index 98cd175..e21a7a2 100644 --- a/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py index 4e96866..9e9cc48 100644 --- a/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py +++ b/roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/jira_software_latest/tests/test_default.py b/roles/product_install/molecule/jira_software_latest/tests/test_default.py index 5bb9a71..05e48cb 100644 --- a/roles/product_install/molecule/jira_software_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_software_latest/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py index 438dbc3..102cf44 100644 --- a/roles/product_install/molecule/jira_version_from_file/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_from_file/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/jira_version_latest/tests/test_default.py b/roles/product_install/molecule/jira_version_latest/tests/test_default.py index dc659f1..82f1205 100644 --- a/roles/product_install/molecule/jira_version_latest/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_latest/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/jira_version_override/tests/test_default.py b/roles/product_install/molecule/jira_version_override/tests/test_default.py index bf9d37d..a6407d7 100644 --- a/roles/product_install/molecule/jira_version_override/tests/test_default.py +++ b/roles/product_install/molecule/jira_version_override/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/resources/tests/test_common.py b/roles/product_install/molecule/resources/tests/test_common.py index bb85871..a63af19 100644 --- a/roles/product_install/molecule/resources/tests/test_common.py +++ b/roles/product_install/molecule/resources/tests/test_common.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/servicedesk3/tests/test_default.py b/roles/product_install/molecule/servicedesk3/tests/test_default.py index aaa6e3a..2ec583f 100644 --- a/roles/product_install/molecule/servicedesk3/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk3/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/servicedesk4/tests/test_default.py b/roles/product_install/molecule/servicedesk4/tests/test_default.py index a052f2d..dff1ffe 100644 --- a/roles/product_install/molecule/servicedesk4/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk4/tests/test_default.py @@ -1,5 +1,5 @@ import os -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner diff --git a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py index b595161..7168f06 100644 --- a/roles/product_install/molecule/servicedesk_latest/tests/test_default.py +++ b/roles/product_install/molecule/servicedesk_latest/tests/test_default.py @@ -1,6 +1,6 @@ import os import json -import urllib.request +from six.moves import urllib import testinfra.utils.ansible_runner From e6f1212afbeb33c89c629a2c4b9fcc7faf625b00 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 09:55:53 +1000 Subject: [PATCH 10/18] DCD-386: Cleanup some YAML whitespace. --- roles/product_startup/molecule/bitbucket/playbook.yml | 2 +- roles/product_startup/molecule/synchrony/playbook.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/product_startup/molecule/bitbucket/playbook.yml b/roles/product_startup/molecule/bitbucket/playbook.yml index 84d25ae..ccbb797 100644 --- a/roles/product_startup/molecule/bitbucket/playbook.yml +++ b/roles/product_startup/molecule/bitbucket/playbook.yml @@ -23,6 +23,6 @@ file: path: '/etc/systemd/system/' state: directory - + roles: - role: product_startup diff --git a/roles/product_startup/molecule/synchrony/playbook.yml b/roles/product_startup/molecule/synchrony/playbook.yml index d848712..0e19553 100644 --- a/roles/product_startup/molecule/synchrony/playbook.yml +++ b/roles/product_startup/molecule/synchrony/playbook.yml @@ -22,6 +22,6 @@ file: path: '/etc/systemd/system/' state: directory - + roles: - role: product_startup From 9aef39034da74fdf7a7794b52b61409ef8cf1a64 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 10:09:03 +1000 Subject: [PATCH 11/18] DCD-386: Fix bitbucket systemd test. --- roles/product_startup/molecule/bitbucket/tests/test_default.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/product_startup/molecule/bitbucket/tests/test_default.py b/roles/product_startup/molecule/bitbucket/tests/test_default.py index 1f9aeb0..d12ab59 100644 --- a/roles/product_startup/molecule/bitbucket/tests/test_default.py +++ b/roles/product_startup/molecule/bitbucket/tests/test_default.py @@ -11,4 +11,4 @@ def test_service_file(host): assert f.contains("^ExecStart=/opt/atlassian/bitbucket/current/bin/start-bitbucket.sh -fg --no-search$") assert f.contains("^UMask=0027$") assert f.contains("^LimitNOFILE=4096$") - assert f.contains("^Environment=BITBUCKET_HOME=/media/atl/bitbucket/shared$") + assert f.contains("^Environment=BITBUCKET_HOME=/media/atl/bitbucket$") From 74ed58bda21280b26fc56f83d2884cf55384aba7 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 10:30:49 +1000 Subject: [PATCH 12/18] DCD-386: Fix DIY backup test. --- roles/diy_backup/molecule/default/playbook.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/diy_backup/molecule/default/playbook.yml b/roles/diy_backup/molecule/default/playbook.yml index 88861a3..a5cecc3 100644 --- a/roles/diy_backup/molecule/default/playbook.yml +++ b/roles/diy_backup/molecule/default/playbook.yml @@ -21,7 +21,7 @@ pre_tasks: - name: Create base dir file: - path: '/opt/atlassian/' + path: '/opt/atlassian/bin' state: directory - name: Install git From 8826f7e4c2e6b7b0fbe4361ad01f626309532b9a Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 10:59:34 +1000 Subject: [PATCH 13/18] DCD-386: Consistent order for test runs. --- bin/run-all-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run-all-tests b/bin/run-all-tests index b5dc601..7489276 100755 --- a/bin/run-all-tests +++ b/bin/run-all-tests @@ -8,7 +8,7 @@ if [[ -f .venv/bin/ansible ]]; then pip install molecule docker six fi -for role in `find roles/ -name molecule`; do +for role in `find roles/ -name molecule | sort`; do pushd `dirname $role` molecule test --all popd From 9c29300c607362ddc0beeb238423d9f5fe79d2b3 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 01:12:46 +0000 Subject: [PATCH 14/18] DCD-386: Molecule may not set the v4 address, so force it. --- roles/aws_common/molecule/default/playbook.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/aws_common/molecule/default/playbook.yml b/roles/aws_common/molecule/default/playbook.yml index e2ed9b1..6ccfba2 100644 --- a/roles/aws_common/molecule/default/playbook.yml +++ b/roles/aws_common/molecule/default/playbook.yml @@ -3,6 +3,8 @@ hosts: all vars: ansible_ec2_local_ipv4: "1.1.1.1" + ansible_default_ipv4: + address: "9.9.9.9" ansible_ec2_instance_id: "NONE" roles: - role: aws_common From 0d6204a85cd8e7a535a8c5e4d3f1f17a81961916 Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 01:52:41 +0000 Subject: [PATCH 15/18] DCD-386: Provide a default for the product user UID. --- roles/linux_common/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 roles/linux_common/defaults/main.yml diff --git a/roles/linux_common/defaults/main.yml b/roles/linux_common/defaults/main.yml new file mode 100644 index 0000000..561baf2 --- /dev/null +++ b/roles/linux_common/defaults/main.yml @@ -0,0 +1,3 @@ +--- + +atl_product_user_uid: '2001' From f8641a00440ba817c7bf7677bf023ca76a80a79a Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 03:32:28 +0000 Subject: [PATCH 16/18] DCD-386: Tweaks to Bitbucket layout and tests. --- aws_bitbucket_dc_node.yml | 4 +++- group_vars/aws_node_local.yml | 2 +- .../molecule/bitbucket_latest/tests/test_default.py | 4 ++-- roles/product_startup/molecule/bitbucket/playbook.yml | 3 +++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/aws_bitbucket_dc_node.yml b/aws_bitbucket_dc_node.yml index 3108bf8..a7a6d6d 100644 --- a/aws_bitbucket_dc_node.yml +++ b/aws_bitbucket_dc_node.yml @@ -8,6 +8,8 @@ atl_product_edition: "bitbucket" atl_product_user: "bitbucket" + atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" + atl_product_home_shared: "{{ atl_product_home }}/shared" atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared" atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared" @@ -15,7 +17,7 @@ atl_startup_systemd_params: - "UMask=0027" - "LimitNOFILE=4096" - - "Environment=BITBUCKET_HOME={{ atl_product_home_shared }}" + - "Environment=BITBUCKET_HOME={{ atl_product_home }}" atl_startup_exec_options: - "-fg" - "--no-search" diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index 2208f82..bbfe960 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -23,7 +23,7 @@ atl_product_home: "{{ atl_home_base }}/{{ atl_product_family }}" atl_product_shared_home_map: confluence: "confluence/shared-home" jira: "jira/shared" - stash: "bitbucket" + stash: "bitbucket/shared" atl_product_home_shared: "{{ atl_shared_mountpoint }}/{{ atl_product_shared_home_map[atl_product_family] }}" atl_product_shared_plugins: "{{ atl_product_home_shared }}/plugins/installed-plugins" diff --git a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py index d83ada5..7157e10 100644 --- a/roles/product_install/molecule/bitbucket_latest/tests/test_default.py +++ b/roles/product_install/molecule/bitbucket_latest/tests/test_default.py @@ -8,7 +8,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all') def test_version_downloaded(host): - verfile = host.file('/media/atl/stash/shared/bitbucket.version') + verfile = host.file('/media/atl/bitbucket/shared/bitbucket.version') assert verfile.exists def test_symlink_created(host): @@ -21,7 +21,7 @@ def test_unpacked(host): assert verfile.exists def test_version_file_is_latest(host): - verfile = host.file('/media/atl/stash/shared/bitbucket.version') + verfile = host.file('/media/atl/bitbucket/shared/bitbucket.version') assert verfile.exists upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest") diff --git a/roles/product_startup/molecule/bitbucket/playbook.yml b/roles/product_startup/molecule/bitbucket/playbook.yml index ccbb797..2658f21 100644 --- a/roles/product_startup/molecule/bitbucket/playbook.yml +++ b/roles/product_startup/molecule/bitbucket/playbook.yml @@ -6,6 +6,9 @@ atl_product_family: "stash" atl_product_edition: "bitbucket" + atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" + atl_product_home_shared: "{{ atl_product_home }}/shared" + atl_startup_systemd_params: - "UMask=0027" - "LimitNOFILE=4096" From 7a5a8e003b212ba4768a4ec2f181bb9bf1dfcb3b Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Tue, 11 Jun 2019 05:20:55 +0000 Subject: [PATCH 17/18] DCD-386: Cleanup BB home vars. --- aws_bitbucket_dc_node.yml | 3 +-- roles/bitbucket_config/tasks/main.yml | 4 ++-- roles/product_install/molecule/bitbucket_latest/playbook.yml | 3 +++ roles/product_startup/molecule/bitbucket/playbook.yml | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/aws_bitbucket_dc_node.yml b/aws_bitbucket_dc_node.yml index a7a6d6d..d473ea5 100644 --- a/aws_bitbucket_dc_node.yml +++ b/aws_bitbucket_dc_node.yml @@ -9,11 +9,10 @@ atl_product_user: "bitbucket" atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" - atl_product_home_shared: "{{ atl_product_home }}/shared" + atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared" atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared" - atl_startup_systemd_params: - "UMask=0027" - "LimitNOFILE=4096" diff --git a/roles/bitbucket_config/tasks/main.yml b/roles/bitbucket_config/tasks/main.yml index 6a23bdb..cf6bcce 100644 --- a/roles/bitbucket_config/tasks/main.yml +++ b/roles/bitbucket_config/tasks/main.yml @@ -2,7 +2,7 @@ - name: Create Bitbucket shared dir if necessary file: - path: "{{ atl_product_home_shared }}/shared/" + path: "{{ atl_product_home_shared }}" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" mode: 0750 @@ -11,7 +11,7 @@ - name: Create Bitbucket config file template: src: bitbucket.properties.j2 - dest: "{{ atl_product_home_shared }}/shared/bitbucket.properties" + dest: "{{ atl_product_home_shared }}/bitbucket.properties" owner: "{{ atl_product_user }}" group: "{{ atl_product_user }}" diff --git a/roles/product_install/molecule/bitbucket_latest/playbook.yml b/roles/product_install/molecule/bitbucket_latest/playbook.yml index b9eb653..907928d 100644 --- a/roles/product_install/molecule/bitbucket_latest/playbook.yml +++ b/roles/product_install/molecule/bitbucket_latest/playbook.yml @@ -5,6 +5,9 @@ atl_product_family: "stash" atl_product_edition: "bitbucket" atl_product_user: "bitbucket" + + atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}" + roles: - role: linux_common - role: product_common diff --git a/roles/product_startup/molecule/bitbucket/playbook.yml b/roles/product_startup/molecule/bitbucket/playbook.yml index 2658f21..08520cb 100644 --- a/roles/product_startup/molecule/bitbucket/playbook.yml +++ b/roles/product_startup/molecule/bitbucket/playbook.yml @@ -12,7 +12,7 @@ atl_startup_systemd_params: - "UMask=0027" - "LimitNOFILE=4096" - - "Environment=BITBUCKET_HOME={{ atl_product_home_shared }}" + - "Environment=BITBUCKET_HOME={{ atl_product_home }}" atl_startup_exec_options: - "-fg" From 39a03184565fd75a673a383191e4ea1ed37c554d Mon Sep 17 00:00:00 2001 From: Steve Smith Date: Wed, 12 Jun 2019 10:01:42 +1000 Subject: [PATCH 18/18] DCD-386: Remove duplicated lookup. --- group_vars/aws_node_local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/group_vars/aws_node_local.yml b/group_vars/aws_node_local.yml index bbfe960..699745f 100644 --- a/group_vars/aws_node_local.yml +++ b/group_vars/aws_node_local.yml @@ -96,7 +96,7 @@ atl_tomcat_secure: "{{ lookup('env', 'ATL_TOMCAT_SECURE') or 'false' }}" atl_fileserver_host: "{{ lookup('env', 'ATL_FILESERVER_IP') }}" atl_elasticsearch_host: "{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}" -atl_elasticsearch_endpoint: "http://{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}" +atl_elasticsearch_endpoint: "http://{{ atl_elasticsearch_host }}" atl_elasticsearch_s3_bucket: "{{ lookup('env', 'ATL_ELASTICSEARCH_S3_BUCKET') }}" atl_nfs_server_device: "{{ lookup('env', 'ATL_NFS_SERVER_DEVICE') }}"