Merge remote-tracking branch 'origin/master' into 196-simple-webapp-config

summerisgone · summerisgone · commit cb256c163dca · 2020-05-19T01:32:41.000+05:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-16.04
     strategy:
       matrix:
-        UBUNTU_VERSION: ["18.04", "19.10"]
+        UBUNTU_VERSION: ["18.04", "20.04"]
     steps:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v1
@@ -98,7 +98,7 @@ jobs:
     runs-on: ubuntu-16.04
     strategy:
       matrix:
-        UBUNTU_VERSION: ["18.04", "19.10"]
+        UBUNTU_VERSION: ["18.04", "20.04"]
     steps:
       - uses: actions/checkout@v1
       - uses: actions/setup-python@v1
diff --git a/config.cfg b/config.cfg
@@ -49,7 +49,6 @@ reduce_mtu: 0
 # /etc/systemd/system/dnsmasq.service.d/100-CustomLimitations.conf
 adblock_lists:
  - "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts"
- - "https://hosts-file.net/ad_servers.txt"
 
 # Enable DNS encryption.
 # If 'false', 'dns_servers' should be specified below.
@@ -157,10 +156,14 @@ SSH_keys:
 cloud_providers:
   azure:
     size: Standard_B1S
-    image: 19.10-DAILY
+    image:
+      publisher: Canonical
+      offer: 0001-com-ubuntu-server-focal-daily
+      sku: 20_04-daily-lts
+      version: latest
   digitalocean:
     size: s-1vcpu-1gb
-    image: "ubuntu-19-10-x64"
+    image: "ubuntu-20-04-x64"
   ec2:
     # Change the encrypted flag to "false" to disable AWS volume encryption.
     encrypted: true
@@ -169,31 +172,31 @@ cloud_providers:
     use_existing_eip: false
     size: t2.micro
     image:
-      name: "ubuntu-eoan-19.10"
+      name: "ubuntu-focal-20.04"
       owner: "099720109477"
   gce:
     size: f1-micro
-    image: ubuntu-1910
+    image: ubuntu-2004-lts
     external_static_ip: false
   lightsail:
     size: nano_1_0
     image: ubuntu_18_04
   scaleway:
     size: DEV1-S
-    image: Ubuntu Bionic Beaver
+    image: Ubuntu 20.04 Focal Fossa
     arch: x86_64
   hetzner:
     server_type: cx11
-    image: ubuntu-18.04
+    image: ubuntu-20.04
   openstack:
     flavor_ram: ">=512"
     image:  Ubuntu-18.04
   cloudstack:
     size: Micro
-    image: Linux Ubuntu 19.10 64-bit
+    image: Linux Ubuntu 20.04 LTS 64-bit
     disk: 10
   vultr:
-    os: Ubuntu 19.10 x64
+    os: Ubuntu 20.04 x64
     size: 1024 MB RAM,25 GB SSD,1.00 TB BW
   local:
 
diff --git a/docs/client-linux-wireguard.md b/docs/client-linux-wireguard.md
@@ -49,3 +49,19 @@ sudo systemctl enable wg-quick@wg0
 ```
 
 If your Linux distribution does not use `systemd` you can bring up WireGuard with `sudo wg-quick up wg0`.
+
+## Using a DNS Search Domain
+
+As of the `v1.0.20200510` release of `wireguard-tools` WireGuard supports setting a DNS search domain. In your `wg0.conf` file a non-numeric entry on the `DNS` line will be used as a search domain. For example this:
+```
+DNS =  172.27.153.31, fd00::b:991f, mydomain.com
+```
+will cause your `/etc/resolv.conf` to contain:
+```
+search mydomain.com
+nameserver 172.27.153.31
+nameserver fd00::b:991f
+```
+If you're using the version of WireGuard included with Ubuntu as of 19.10 it might be from before this feature was added. To use the latest version of WireGuard add the PPA repository as shown above.
+
+Note that using the PPA repository on Ubuntu 20.04 LTS instead of the WireGuard modules shipped in the kernel package may cause the installation of about 40 additional packages in order to compile the kernel module.
diff --git a/docs/deploy-to-ubuntu.md b/docs/deploy-to-ubuntu.md
@@ -4,7 +4,7 @@ You can use Algo to configure a pre-existing server as an AlgoVPN rather than us
 
 To perform a local installation, install the Algo scripts following the normal installation instructions, then choose:
 ```
-Install to existing Ubuntu 18.04 or 19.10 server (for more advanced users)
+Install to existing Ubuntu 18.04 or 20.04 server (for more advanced users)
 ```
 Make sure your target server is running an unmodified copy of the operating system version specified. The target can be the same system where you've installed the Algo scripts, or a remote system that you are able to access as root via SSH without needing to enter the SSH key passphrase (such as when using `ssh-agent`).
 
diff --git a/docs/deploy-to-unsupported-cloud.md b/docs/deploy-to-unsupported-cloud.md
@@ -2,7 +2,7 @@
 
 Algo officially supports the [cloud providers listed here](https://github.com/trailofbits/algo/blob/master/README.md#deploy-the-algo-server). If you want to deploy Algo on another virtual hosting provider, that provider must support:
 
-1. the base operating system image that Algo uses (Ubuntu 18.04 or 19.10), and
+1. the base operating system image that Algo uses (Ubuntu 18.04 or 20.04), and
 2. a minimum of certain kernel modules required for the strongSwan IPsec server.
 
 Please see the [Required Kernel Modules](https://wiki.strongswan.org/projects/strongswan/wiki/KernelModules) documentation from strongSwan for a list of the specific required modules and a script to check for them. As a first step, we recommend running their shell script to determine initial compatibility with your new hosting provider.
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
@@ -223,6 +223,23 @@ The error is caused because Digital Ocean changed its API to treat the tag argum
 5. Finally run `doctl compute tag list` to make sure that the tag has been deleted
 6. Run algo as directed
 
+### Azure: No such file or directory: '/home/username/.azure/azureProfile.json'
+ 
+ ```
+ TASK [cloud-azure : Create AlgoVPN Server] *****************************************************************************************************************************************************************
+An exception occurred during task execution. To see the full traceback, use -vvv. 
+The error was: FileNotFoundError: [Errno 2] No such file or directory: '/home/ubuntu/.azure/azureProfile.json'
+fatal: [localhost]: FAILED! => {"changed": false, "module_stderr": "Traceback (most recent call last):
+File \"/usr/local/lib/python3.6/dist-packages/azure/cli/core/_session.py\", line 39, in load
+with codecs_open(self.filename, 'r', encoding=self._encoding) as f:
+File \"/usr/lib/python3.6/codecs.py\", line 897, in open\n    file = builtins.open(filename, mode, buffering)
+FileNotFoundError: [Errno 2] No such file or directory: '/home/ubuntu/.azure/azureProfile.json'
+", "module_stdout": "", "msg": "MODULE FAILURE
+See stdout/stderr for the exact error", "rc": 1}
+```
+
+It happens when your machine is not authenticated in the azure cloud, follow this [guide](https://trailofbits.github.io/algo/cloud-azure.html) to configure your environment
+
 ### Windows: The value of parameter linuxConfiguration.ssh.publicKeys.keyData is invalid
 
 You tried to deploy Algo from Windows and you received an error like this one:
@@ -266,7 +283,7 @@ TASK [wireguard : Generate public keys] ****************************************
 
 fatal: [localhost]: FAILED! => {"msg": "An unhandled exception occurred while running the lookup plugin 'file'. Error was a <class 'ansible.errors.AnsibleError'>, original message: could not locate file in lookup: configs/xxx.xxx.xxx.xxx/wireguard//private/dan"}
 ```
-This error is usually hit when using the local install option on a server that isn't Ubuntu 18.04. You should upgrade your server to Ubuntu 18.04. If this doesn't work, try removing `*.lock` files at /etc/wireguard/ as follows:
+This error is usually hit when using the local install option on a server that isn't Ubuntu 18.04 or later. You should upgrade your server to Ubuntu 18.04 or later. If this doesn't work, try removing `*.lock` files at /etc/wireguard/ as follows:
 
 ```ssh
 sudo rm -rf /etc/wireguard/*.lock
diff --git a/input.yml b/input.yml
@@ -21,7 +21,7 @@
       - { name: Scaleway, alias: scaleway}
       - { name: OpenStack (DreamCompute optimised), alias: openstack }
       - { name: CloudStack (Exoscale optimised), alias: cloudstack }
-      - { name: "Install to existing Ubuntu 18.04 or 19.10 server (for more advanced users)", alias: local }
+      - { name: "Install to existing Ubuntu 18.04 or 20.04 server (for more advanced users)", alias: local }
   vars_files:
     - config.cfg
 
diff --git a/roles/client/files/libstrongswan-relax-constraints.conf b/roles/client/files/libstrongswan-relax-constraints.conf
@@ -0,0 +1,5 @@
+libstrongswan {
+  x509 {
+    enforce_critical = no
+  }
+}
diff --git a/roles/client/tasks/main.yml b/roles/client/tasks/main.yml
@@ -53,6 +53,14 @@
   notify:
     - restart strongswan
 
+- name: Configure libstrongswan to relax CA constraints
+  copy:
+    src: libstrongswan-relax-constraints.conf
+    dest: "{{ configs_prefix }}/strongswan.d/relax-ca-constraints.conf"
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Setup the certificates and keys
   template:
     src: "{{ item.src }}"
diff --git a/roles/cloud-azure/files/deployment.json b/roles/cloud-azure/files/deployment.json
@@ -11,9 +11,18 @@
     "vmSize": {
       "type": "string"
     },
+    "imageReferencePublisher": {
+      "type": "string"
+    },
+    "imageReferenceOffer": {
+      "type": "string"
+    },
     "imageReferenceSku": {
       "type": "string"
     },
+    "imageReferenceVersion": {
+      "type": "string"
+    },
     "SshPort": {
       "type": "int"
     },
@@ -182,10 +191,10 @@
         },
         "storageProfile": {
           "imageReference": {
-            "publisher": "Canonical",
-            "offer": "UbuntuServer",
+            "publisher": "[parameters('imageReferencePublisher')]",
+            "offer": "[parameters('imageReferenceOffer')]",
             "sku": "[parameters('imageReferenceSku')]",
-            "version": "latest"
+            "version": "[parameters('imageReferenceVersion')]"
           },
           "osDisk": {
             "createOption": "FromImage"
diff --git a/roles/cloud-azure/tasks/main.yml b/roles/cloud-azure/tasks/main.yml
@@ -29,8 +29,14 @@
         value: "{{ wireguard_port }}"
       vmSize:
         value: "{{ cloud_providers.azure.size }}"
+      imageReferencePublisher:
+        value: "{{ cloud_providers.azure.image.publisher }}"
+      imageReferenceOffer:
+        value: "{{ cloud_providers.azure.image.offer }}"
       imageReferenceSku:
-        value: "{{ cloud_providers.azure.image }}"
+        value: "{{ cloud_providers.azure.image.sku }}"
+      imageReferenceVersion:
+        value: "{{ cloud_providers.azure.image.version }}"
       SshPort:
         value: "{{ ssh_port }}"
       UserData:
diff --git a/roles/common/tasks/facts.yml b/roles/common/tasks/facts.yml
@@ -6,7 +6,7 @@
 
 - name: Set facts
   set_fact:
-    CA_password: "{{ lookup('password', '/dev/null length=16 chars=ascii_letters,digits,_,@') }}"
+    CA_password: "{{ ca_password|default(lookup('password', '/dev/null length=16 chars=ascii_letters,digits,_,@')) }}"
     IP_subject_alt_name: "{{ IP_subject_alt_name }}"
 
 - name: Set IPv6 support as a fact
diff --git a/roles/common/templates/rules.v6.j2 b/roles/common/templates/rules.v6.j2
@@ -35,7 +35,7 @@ COMMIT
 -A PREROUTING --in-interface {{ ansible_default_ipv6['interface'] }} -p udp --dport {{ wireguard_port_avoid }} -j REDIRECT --to-port {{ wireguard_port_actual }}
 {% endif %}
 # Allow traffic from the VPN network to the outside world, and replies
--A POSTROUTING -s {{ subnets|join(',') }} -m policy --pol none --dir out -j SNAT --to {{ ipv6_egress_ip | ipaddr('address') }}
+-A POSTROUTING -s {{ subnets|join(',') }} -m policy --pol none --dir out {{ '-j SNAT --to ' + ipv6_egress_ip | ipaddr('address') if alternative_ingress_ip else '-j MASQUERADE' }}
 
 COMMIT
 
diff --git a/roles/dns/templates/adblock.sh.j2 b/roles/dns/templates/adblock.sh.j2
@@ -14,7 +14,7 @@ rm -f $BLOCKHOSTS
 echo 'Downloading hosts lists...'
 #Download and process the files needed to make the lists (enable/add more, if you want)
 for url in $BLOCKLIST_URLS; do
-  wget --timeout=2 --tries=3 -qO- "$url" | grep -Ev "(localhost)" | grep -Ew "(0.0.0.0|127.0.0.1)" | awk '{sub(/\r$/,"");print $2}'  >> "$TEMP"
+  wget --timeout=2 --tries=3 -qO- "$url" | grep -Ev "(localhost)" | grep -Ev "#" | sed -E "s/(0.0.0.0 |127.0.0.1 |255.255.255.255 )//" >> "$TEMP"
 done
 
 #Add black list, if non-empty
diff --git a/roles/strongswan/defaults/main.yml b/roles/strongswan/defaults/main.yml
@@ -3,6 +3,7 @@ ipsec_config_path: "configs/{{ IP_subject_alt_name }}/ipsec/"
 ipsec_pki_path: "{{ ipsec_config_path }}/.pki/"
 strongswan_shell: /usr/sbin/nologin
 strongswan_home: /var/lib/strongswan
+strongswan_service: "{{ 'strongswan-starter' if ansible_facts['distribution_version'] is version('20.04', '>=') else 'strongswan' }}"
 BetweenClients_DROP: true
 algo_ondemand_cellular: false
 algo_ondemand_wifi: false
diff --git a/roles/strongswan/handlers/main.yml b/roles/strongswan/handlers/main.yml
@@ -1,5 +1,5 @@
 - name: restart strongswan
-  service: name=strongswan state=restarted
+  service: name={{ strongswan_service }} state=restarted
 
 - name: daemon-reload
   systemd: daemon_reload=true
diff --git a/roles/strongswan/tasks/main.yml b/roles/strongswan/tasks/main.yml
@@ -24,7 +24,7 @@
 
 - name: strongSwan started
   service:
-    name: strongswan
+    name: "{{ strongswan_service }}"
     state: started
     enabled: true
 
diff --git a/roles/strongswan/tasks/ubuntu.yml b/roles/strongswan/tasks/ubuntu.yml
@@ -35,12 +35,12 @@
   service: name={{ item }} enabled=yes
   with_items:
     - apparmor
-    - strongswan
+    - "{{ strongswan_service }}"
     - netfilter-persistent
 
 - name: Ubuntu | Ensure that the strongswan service directory exists
   file:
-    path: /etc/systemd/system/strongswan.service.d/
+    path: /etc/systemd/system/{{ strongswan_service }}.service.d/
     state: directory
     mode: 0755
     owner: root
@@ -49,7 +49,7 @@
 - name: Ubuntu | Setup the cgroup limitations for the ipsec daemon
   template:
     src: 100-CustomLimitations.conf.j2
-    dest: /etc/systemd/system/strongswan.service.d/100-CustomLimitations.conf
+    dest: /etc/systemd/system/{{ strongswan_service }}.service.d/100-CustomLimitations.conf
   notify:
     - daemon-reload
     - restart strongswan
diff --git a/roles/wireguard/tasks/ubuntu.yml b/roles/wireguard/tasks/ubuntu.yml
@@ -1,12 +1,22 @@
 ---
-- name: WireGuard repository configured
-  apt_repository:
-    repo: ppa:wireguard/wireguard
-    state: present
-  register: result
-  until: result is succeeded
-  retries: 10
-  delay: 3
+- block:
+  - name: WireGuard repository configured
+    apt_repository:
+      repo: ppa:wireguard/wireguard
+      state: present
+    register: result
+    until: result is succeeded
+    retries: 10
+    delay: 3
+
+  - name: Configure unattended-upgrades
+    copy:
+      src: 50-wireguard-unattended-upgrades
+      dest: /etc/apt/apt.conf.d/50-wireguard-unattended-upgrades
+      owner: root
+      group: root
+      mode: 0644
+  when: ansible_facts['distribution_version'] is version('20.04', '<')
 
 - name: WireGuard installed
   apt:
@@ -19,14 +29,6 @@
     dest: /etc/wireguard/.reload-module-on-update
     state: touch
 
-- name: Configure unattended-upgrades
-  copy:
-    src: 50-wireguard-unattended-upgrades
-    dest: /etc/apt/apt.conf.d/50-wireguard-unattended-upgrades
-    owner: root
-    group: root
-    mode: 0644
-
 - name: Set OS specific facts
   set_fact:
     service_name: "wg-quick@{{ wireguard_interface }}"
diff --git a/tests/ca-password-fix.sh b/tests/ca-password-fix.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Refer PR #1774 for more details.
+
+set -ex
+
+DEPLOY_ARGS="provider=local server=10.0.8.100 ssh_user=ubuntu endpoint=10.0.8.100 ondemand_cellular=true ondemand_wifi=true ondemand_wifi_exclude=test dns_adblocking=true ssh_tunneling=true store_pki=true install_headers=false tests=true local_service_ip=172.16.0.1"
+
+CA_PASSWORD="test123"
+
+if [ "${DEPLOY}" == "docker" ]
+then
+  docker run -i -v $(pwd)/config.cfg:/algo/config.cfg -v ~/.ssh:/root/.ssh -v $(pwd)/configs:/algo/configs -e "DEPLOY_ARGS=${DEPLOY_ARGS}" local/algo /bin/sh -c "chown -R root: /root/.ssh && chmod -R 600 /root/.ssh && source .env/bin/activate && ansible-playbook main.yml -e \"${DEPLOY_ARGS}\" --skip-tags debug"
+else
+  ansible-playbook main.yml -e "${DEPLOY_ARGS} ca_password=${CA_PASSWORD}"
+fi
+
+CA_PASSWORD_OUT=$(grep ca_password: configs/localhost/.config.yml | awk '{print $2}' | xargs)
+
+if [ "$CA_PASSWORD" = "$CA_PASSWORD_OUT" ]; then
+  echo "ca_password tests(PR #1774) passed"
+else
+  echo "ca_password tests(PR #1774) failed"
+  exit 1
+fi
diff --git a/tests/pre-deploy.sh b/tests/pre-deploy.sh
@@ -25,6 +25,10 @@ lxc profile set default security.privileged true
 lxc profile show default
 lxc launch ubuntu:${UBUNTU_VERSION} algo
 
+if [[ ${UBUNTU_VERSION} == "20.04" ]]; then
+  lxc exec algo -- apt remove snapd --purge -y || true
+fi
+
 ip addr
 
 until dig A +short algo.lxd @10.0.8.1 | grep -vE '^$' > /dev/null; do

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +libstrongswan {
 +  x509 {
 +    enforce_critical = no
 +  }
 +}