Merged in DCD-386-bitbucket-deployments (pull request #6)

DCD-386: Bitbucket deployments

Approved-by: Ben Partridge <bpartridge@atlassian.com>

aws_bitbucket_dc_node.yml

@@ -8,10 +8,15 @@
     atl_product_edition: "bitbucket"
     atl_product_user: "bitbucket"
 
+    atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
+
+    atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared"
+    atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared"
+
     atl_startup_systemd_params:
       - "UMask=0027"
       - "LimitNOFILE=4096"
-      - "Environment=BITBUCKET_HOME={{ atl_product_home_shared }}"
+      - "Environment=BITBUCKET_HOME={{ atl_product_home }}"
     atl_startup_exec_options:
       - "-fg"
       - "--no-search"

aws_bitbucket_nfs_node.yml

@@ -8,7 +8,11 @@
     atl_product_edition: "bitbucket"
     atl_product_user: "bitbucket"
 
+    atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}/bitbucket/shared"
+    atl_nfs_target: "{{ atl_shared_mountpoint }}/bitbucket/shared"
+
   roles:
     - role: linux_common
     - role: aws_common
     - role: nfs_server
+    - role: diy_backup

bin/run-all-tests

@@ -5,10 +5,10 @@ set -e
 # Use Ansible from virtualenv if provided
 if [[ -f .venv/bin/ansible ]]; then
     source .venv/bin/activate
-    pip install molecule docker
+    pip install molecule docker six
 fi
 
-for role in `find roles/ -name molecule`; do
+for role in `find roles/ -name molecule | sort`; do
     pushd `dirname $role`
     molecule test --all
     popd

group_vars/aws_node_local.yml

@@ -23,7 +23,7 @@ atl_product_home: "{{ atl_home_base }}/{{ atl_product_family }}"
 atl_product_shared_home_map:
   confluence: "confluence/shared-home"
   jira: "jira/shared"
-  stash: "bitbucket"
+  stash: "bitbucket/shared"
 atl_product_home_shared: "{{ atl_shared_mountpoint }}/{{ atl_product_shared_home_map[atl_product_family] }}"
 atl_product_shared_plugins: "{{ atl_product_home_shared }}/plugins/installed-plugins"
 
@@ -43,6 +43,7 @@ atl_efs_id: "{{ lookup('env', 'ATL_EFS_ID') }}"
 atl_aws_stack_name: "{{ lookup('env', 'ATL_AWS_STACK_NAME') }}"
 atl_aws_region: "{{ lookup('env', 'ATL_AWS_REGION') }}"
 atl_aws_iam_role: "{{ lookup('env', 'ATL_AWS_IAM_ROLE') }}"
+atl_aws_iam_role_arn: "{{ lookup('env', 'ATL_AWS_IAM_ROLE_ARN') }}"
 
 atl_db_engine: "{{ lookup('env', 'ATL_DB_ENGINE') }}"
 atl_db_host: "{{ lookup('env', 'ATL_DB_HOST') }}"
@@ -80,6 +81,7 @@ atl_catalina_opts: "{{ lookup('env', 'ATL_CATALINA_OPTS') or '' }}"
 atl_proxy_name:  "{{ lookup('env', 'ATL_PROXY_NAME') | lower }}"
 atl_proxy_port:  "{{ lookup('env', 'ATL_TOMCAT_PROXYPORT') }}"
 atl_tomcat_port: "{{ lookup('env', 'ATL_TOMCAT_DEFAULTCONNECTORPORT') or '8080' }}"
+atl_ssl_proxy: "{{ lookup('env', 'ATL_SSL_PROXY') or 'false' }}"
 
 atl_tomcat_acceptcount: "{{ lookup('env', 'ATL_TOMCAT_ACCEPTCOUNT') or '10' }}"
 atl_tomcat_connectiontimeout: "{{ lookup('env', 'ATL_TOMCAT_CONNECTIONTIMEOUT') or '20000' }}"
@@ -93,6 +95,15 @@ atl_tomcat_scheme: "{{ lookup('env', 'ATL_TOMCAT_SCHEME') or 'http' }}"
 atl_tomcat_secure: "{{ lookup('env', 'ATL_TOMCAT_SECURE') or 'false' }}"
 
 atl_fileserver_host: "{{ lookup('env', 'ATL_FILESERVER_IP') }}"
-atl_elasticsearch_endpoint: "{{ lookup('env', 'ATL_ELASTICSEARCH_ENDPOINT') }}"
+atl_elasticsearch_host: "{{ lookup('env', 'ATL_ELASTICSEARCH_HOST') }}"
+atl_elasticsearch_endpoint: "http://{{ atl_elasticsearch_host }}"
+atl_elasticsearch_s3_bucket: "{{ lookup('env', 'ATL_ELASTICSEARCH_S3_BUCKET') }}"
 
-atl_ssl_proxy: "{{ lookup('env', 'ATL_SSL_PROXY') or 'false' }}"
+atl_nfs_server_device: "{{ lookup('env', 'ATL_NFS_SERVER_DEVICE') }}"
+atl_nfs_disk_volume_type: "{{ lookup('env', 'ATL_NFS_DISK_VOLUME_TYPE') }}"
+atl_nfs_disk_volume_iops: "{{ lookup('env', 'ATL_NFS_DISK_VOLUME_IOPS') }}"
+atl_rds_instance_id: "{{ lookup('env', 'ATL_RDS_INSTANCE_ID') }}"
+atl_rds_instance_class: "{{ lookup('env', 'ATL_RDS_INSTANCE_CLASS') }}"
+atl_rds_multi_az: "{{ lookup('env', 'ATL_RDS_MULTI_AZ') }}"
+atl_rds_subnet_group_name: "{{ lookup('env', 'ATL_RDS_SUBNET_GROUP_NAME') }}"
+atl_rds_security_group: "{{ lookup('env', 'ATL_RDS_SECURITY_GROUP') }}"

roles/aws_common/molecule/default/playbook.yml

@@ -3,6 +3,8 @@
   hosts: all
   vars:
     ansible_ec2_local_ipv4: "1.1.1.1"
+    ansible_default_ipv4:
+      address: "9.9.9.9"
     ansible_ec2_instance_id: "NONE"
   roles:
     - role: aws_common

roles/bitbucket_config/tasks/main.yml

@@ -2,7 +2,7 @@
 
 - name: Create Bitbucket shared dir if necessary
   file:
-    path: "{{ atl_product_home_shared }}/shared/"
+    path: "{{ atl_product_home_shared }}"
     owner: "{{ atl_product_user }}"
     group: "{{ atl_product_user }}"
     mode: 0750
@@ -11,7 +11,7 @@
 - name: Create Bitbucket config file
   template:
     src: bitbucket.properties.j2
-    dest: "{{ atl_product_home_shared }}/shared/bitbucket.properties"
+    dest: "{{ atl_product_home_shared }}/bitbucket.properties"
     owner: "{{ atl_product_user }}"
     group: "{{ atl_product_user }}"
 

roles/diy_backup/molecule/default/playbook.yml

@@ -6,10 +6,22 @@
     atl_product_family: "stash"
     atl_product_edition: "bitbucket"
 
+    atl_aws_iam_role: "atl_aws_iam_role"
+    atl_aws_stack_name: "atl_aws_stack_name"
+    atl_nfs_disk_volume_iops: "atl_nfs_disk_volume_iops"
+    atl_nfs_disk_volume_type: "atl_nfs_disk_volume_type"
+    atl_nfs_fs_type: "atl_nfs_fs_type"
+    atl_proxy_name: "atl_proxy_name"
+    atl_rds_instance_class: "atl_rds_instance_class"
+    atl_rds_instance_id: "atl_rds_instance_id"
+    atl_rds_multi_az: "atl_rds_multi_az"
+    atl_rds_security_group: "atl_rds_security_group"
+    atl_rds_subnet_group_name: "atl_rds_subnet_group_name"
+
   pre_tasks:
     - name: Create base dir
       file:
-        path: '/opt/atlassian/'
+        path: '/opt/atlassian/bin'
         state: directory
 
     - name: Install git

roles/diy_backup/molecule/default/tests/test_default.py

@@ -10,3 +10,7 @@ def test_git_clone(host):
     f = host.file('/opt/atlassian/bitbucket-diy-backup')
     assert f.exists
     assert f.is_directory
+
+def test_diy_config(host):
+    f = host.file('/opt/atlassian/bitbucket-diy-backup/bitbucket.diy-backup.vars.sh')
+    assert f.exists

roles/diy_backup/tasks/main.yml

@@ -6,3 +6,15 @@
     repo: "{{ atl_diy_backup_repo }}"
     dest: "{{ atl_diy_backup_dir }}"
     version: "master"
+
+- name: Configure DIY backup for BB on AWS
+  template:
+    src: "bitbucket.diy-backup.vars.sh.j2"
+    dest: "{{ atl_diy_backup_dir }}/bitbucket.diy-backup.vars.sh"
+    mode: 0640
+
+- name: Install backup wrapper script
+  template:
+    src: "run-backup.j2"
+    dest: "{{ atl_installation_base }}/bin/run-backup"
+    mode: 0750

roles/diy_backup/templates/bitbucket.diy-backup.vars.sh.j2

@@ -0,0 +1,55 @@
+# Created by Ansible
+
+INSTANCE_NAME={{ atl_aws_stack_name }}
+
+AWS_INFO=$(curl -Lsf http://169.254.169.254/latest/dynamic/instance-identity/document)
+AWS_ACCOUNT_ID=$(echo "${AWS_INFO}" | jq -r .accountId)
+AWS_AVAILABILITY_ZONE=$(echo "${AWS_INFO}" | jq -r .availabilityZone)
+AWS_REGION=$(echo "${AWS_INFO}" | jq -r .region)
+AWS_EC2_INSTANCE_ID=$(echo "${AWS_INFO}" | jq -r .instanceId)
+
+BITBUCKET_VERBOSE_BACKUP=true
+KEEP_BACKUPS=5
+
+{% if atl_ssl_proxy is defined and atl_ssl_proxy == 'true' %}
+BITBUCKET_URL=https://{{ atl_proxy_name }}
+{% else %}
+BITBUCKET_URL=http://{{ atl_proxy_name }}
+{% endif %}
+
+BITBUCKET_UID={{ atl_product_user }}
+BITBUCKET_GID={{ atl_product_user }}
+
+BACKUP_ZERO_DOWNTIME=true
+
+BACKUP_ELASTICSEARCH_TYPE=amazon-es
+BACKUP_ARCHIVE_TYPE=
+
+BACKUP_DISK_TYPE=amazon-ebs
+STANDBY_DISK_TYPE=none
+EBS_VOLUME_MOUNT_POINT_AND_DEVICE_NAMES=({{ atl_shared_mountpoint }}:{{ atl_nfs_server_device }})
+HOME_DIRECTORY_MOUNT_POINT={{ atl_shared_mountpoint }}
+RESTORE_DISK_VOLUME_TYPE={{ atl_nfs_disk_volume_type }}
+RESTORE_DISK_IOPS={{ atl_nfs_disk_volume_iops }}
+FILESYSTEM_TYPE={{ atl_nfs_fs_type }}
+
+BACKUP_DATABASE_TYPE=amazon-rds
+RDS_INSTANCE_ID={{ atl_rds_instance_id }}
+RESTORE_RDS_INSTANCE_CLASS={{ atl_rds_instance_class }}
+RESTORE_RDS_MULTI_AZ={{ atl_rds_multi_az }}
+RESTORE_RDS_SUBNET_GROUP_NAME={{ atl_rds_subnet_group_name }}
+RESTORE_RDS_SECURITY_GROUP={{ atl_rds_security_group }}
+DR_RDS_READ_REPLICA=
+
+BACKUP_DEST_AWS_ACCOUNT_ID=$AWS_ACCOUNT_ID
+BACKUP_DEST_REGION=$AWS_REGION
+BACKUP_DEST_AWS_ROLE={{ atl_aws_iam_role_arn }}
+
+ELASTICSEARCH_INDEX_NAME=bitbucket-search-v1
+ELASTICSEARCH_REPOSITORY_NAME=bitbucket-snapshots
+ELASTICSEARCH_HOST={{ atl_elasticsearch_host }}
+ELASTICSEARCH_S3_BUCKET={{ atl_elasticsearch_s3_bucket }}
+ELASTICSEARCH_S3_BUCKET_REGION=us-east-1
+ELASTICSEARCH_SNAPSHOT_IAM_ROLE={{ atl_aws_iam_role_arn }}
+
+STANDBY_JDBC_URL=

roles/diy_backup/templates/run-backup.j2

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+cd {{ atl_diy_backup_dir }}
+
+if [[ -f .venv/bin/activate ]]; then
+    source .venv/bin/activate
+else
+    virtualenv .venv
+    source .venv/bin/activate
+    pip install boto
+fi
+
+./bitbucket.diy-backup.sh

roles/linux_common/defaults/main.yml

@@ -0,0 +1,3 @@
+---
+
+atl_product_user_uid: '2001'

roles/nfs_mount/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+
+atl_nfs_mountpoint: "{{ atl_shared_mountpoint }}"
+atl_nfs_target: "{{ atl_shared_mountpoint }}"

roles/nfs_mount/tasks/main.yml

@@ -10,8 +10,8 @@
 
 - name: Enable mountpoint in fstab
   mount:
-    path: "{{ atl_shared_mountpoint }}"
-    src: "{{ atl_fileserver_host }}:{{ atl_shared_mountpoint }}"
+    src: "{{ atl_fileserver_host }}:{{ atl_nfs_target }}"
+    path: "{{ atl_nfs_mountpoint }}"
     fstype: nfs
     opts: "rw,nfsvers=4.1,lookupcache=pos,noatime,intr,rsize=32768,wsize=32768,_netdev"
     state: mounted

roles/nfs_server/tasks/main.yml

@@ -33,6 +33,15 @@
     state: mounted
 
 
+- name: Create the shared home as BB mounts this directly
+  file:
+    path: "{{ atl_shared_mountpoint }}/bitbucket/shared"
+    state: directory
+    owner: "{{ atl_product_user }}"
+    group: "{{ atl_product_user }}"
+    mode: 0750
+
+
 - name: Create the NFS export file
   template:
     src: "media-atl.exports.j2"

roles/product_install/molecule/bitbucket_latest/playbook.yml

@@ -5,6 +5,9 @@
     atl_product_family: "stash"
     atl_product_edition: "bitbucket"
     atl_product_user: "bitbucket"
+
+    atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
+
   roles:
     - role: linux_common
     - role: product_common

roles/product_install/molecule/bitbucket_latest/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 import json
 
 import testinfra.utils.ansible_runner
@@ -8,7 +8,7 @@ testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
     os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
 
 def test_version_downloaded(host):
-    verfile = host.file('/media/atl/stash/shared/bitbucket.version')
+    verfile = host.file('/media/atl/bitbucket/shared/bitbucket.version')
     assert verfile.exists
 
 def test_symlink_created(host):
@@ -21,7 +21,7 @@ def test_unpacked(host):
     assert verfile.exists
 
 def test_version_file_is_latest(host):
-    verfile = host.file('/media/atl/stash/shared/bitbucket.version')
+    verfile = host.file('/media/atl/bitbucket/shared/bitbucket.version')
     assert verfile.exists
 
     upstream_fd = urllib.request.urlopen("https://marketplace.atlassian.com/rest/2/applications/bitbucket/versions/latest")

roles/product_install/molecule/confluence_latest/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 import json
 
 import testinfra.utils.ansible_runner

roles/product_install/molecule/default/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/jira_cached_with_downgrade/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/jira_cached_with_upgrade/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/jira_software_latest/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/jira_version_from_file/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/jira_version_latest/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/jira_version_override/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/resources/tests/test_common.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/servicedesk3/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/servicedesk4/tests/test_default.py

@@ -1,5 +1,5 @@
 import os
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_install/molecule/servicedesk_latest/tests/test_default.py

@@ -1,6 +1,6 @@
 import os
 import json
-import urllib.request
+from six.moves import urllib
 
 import testinfra.utils.ansible_runner
 

roles/product_startup/molecule/bitbucket/playbook.yml

@@ -6,10 +6,13 @@
     atl_product_family: "stash"
     atl_product_edition: "bitbucket"
 
+    atl_product_home: "{{ atl_shared_mountpoint }}/{{ atl_product_edition }}"
+    atl_product_home_shared: "{{ atl_product_home }}/shared"
+
     atl_startup_systemd_params:
       - "UMask=0027"
       - "LimitNOFILE=4096"
-      - "Environment=BITBUCKET_HOME={{ atl_product_home_shared }}"
+      - "Environment=BITBUCKET_HOME={{ atl_product_home }}"
 
     atl_startup_exec_options:
       - "-fg"
@@ -23,6 +26,6 @@
       file:
         path: '/etc/systemd/system/'
         state: directory
-    
+
   roles:
     - role: product_startup

roles/product_startup/molecule/bitbucket/tests/test_default.py

@@ -11,4 +11,4 @@ def test_service_file(host):
     assert f.contains("^ExecStart=/opt/atlassian/bitbucket/current/bin/start-bitbucket.sh -fg --no-search$")
     assert f.contains("^UMask=0027$")
     assert f.contains("^LimitNOFILE=4096$")
-    assert f.contains("^Environment=BITBUCKET_HOME=/media/atl/bitbucket/shared$")
+    assert f.contains("^Environment=BITBUCKET_HOME=/media/atl/bitbucket$")

roles/product_startup/molecule/synchrony/playbook.yml

@@ -22,6 +22,6 @@
       file:
         path: '/etc/systemd/system/'
         state: directory
-    
+
   roles:
     - role: product_startup