From 9d8a5cc2b85465aa524c5d7f11941a8a0e163a63 Mon Sep 17 00:00:00 2001 From: Simon Leiner Date: Tue, 30 Aug 2022 02:45:07 +0200 Subject: [PATCH] Execute Vagrant cluster in CI (#57) --- .github/workflows/test.yml | 69 ++++++++++++++++++++++ vagrant/Vagrantfile | 14 ++--- vagrant/test_cluster.py | 114 +++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100755 vagrant/test_cluster.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..6db201a --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,69 @@ +--- +name: Test +"on": + pull_request: + push: + branches: + - master + +jobs: + vagrant: + name: Vagrant + runs-on: macos-12 + + env: + HOMEBREW_NO_INSTALL_CLEANUP: 1 + VAGRANT_CWD: ${{ github.workspace }}/vagrant + + steps: + - name: Check out the codebase + uses: actions/checkout@v2 + + - name: Install Ansible + run: brew install ansible + + - name: Install role dependencies + run: ansible-galaxy install -r collections/requirements.yml + + - name: Configure VirtualBox + run: >- + sudo mkdir -p /etc/vbox && + echo "* 192.168.30.0/24" | sudo tee -a /etc/vbox/networks.conf > /dev/null + + - name: Cache Vagrant boxes + uses: actions/cache@v3 + with: + path: | + ~/.vagrant.d/boxes + key: vagrant-boxes-${{ hashFiles('**/Vagrantfile') }} + restore-keys: | + vagrant-boxes + + - name: Create virtual machines + run: vagrant up + timeout-minutes: 10 + + - name: Provision cluster using Ansible + # Since Ansible sets up _all_ machines, it is sufficient to run it only + # once (i.e, for a single node - we are choosing control1 here) + run: vagrant provision control1 --provision-with ansible + timeout-minutes: 25 + + - name: Set up kubectl on the host + run: brew install kubectl && + mkdir -p ~/.kube && + vagrant ssh control1 --command "cat ~/.kube/config" > ~/.kube/config + + - name: Show cluster nodes + run: kubectl describe -A nodes + + - name: Show cluster pods + run: kubectl describe -A pods + + - name: Test cluster + run: $VAGRANT_CWD/test_cluster.py --verbose --locals + timeout-minutes: 5 + + - name: Destroy virtual machines + if: always() # do this even if a step before has failed + run: vagrant destroy --force diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index 0e9ac61..e019ac6 100755 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -3,12 +3,12 @@ Vagrant.configure("2") do |config| # General configuration - config.vm.box = "generic/ubuntu2110" + config.vm.box = "generic/ubuntu2204" config.vm.synced_folder ".", "/vagrant", disabled: true config.ssh.insert_key = false config.vm.provider :virtualbox do |v| - v.memory = 4096 + v.memory = 2048 v.cpus = 2 v.linked_clone = true end @@ -50,7 +50,7 @@ Vagrant.configure("2") do |config| "master" => ["control1", "control2", "control3"], "node" => ["node1", "node2"], "k3s_cluster:children" => ["master", "node"], - "k3s_cluster:vars" => {"k3s_version" => "v1.23.4+k3s1", + "k3s_cluster:vars" => {"k3s_version" => "v1.24.3+k3s1", "ansible_user" => "vagrant", "systemd_dir" => "/etc/systemd/system", "flannel_iface" => "eth1", @@ -58,9 +58,9 @@ Vagrant.configure("2") do |config| "k3s_token" => "supersecret", "extra_server_args" => "--node-ip={{ ansible_eth1.ipv4.address }} --flannel-iface={{ flannel_iface }} --no-deploy servicelb --no-deploy traefik", "extra_agent_args" => "--flannel-iface={{ flannel_iface }}", - "kube_vip_tag_version" => "v0.4.2", - "metal_lb_speaker_tag_version" => "v0.12.1", - "metal_lb_controller_tag_version" => "v0.12.1", + "kube_vip_tag_version" => "v0.5.0", + "metal_lb_speaker_tag_version" => "v0.13.4", + "metal_lb_controller_tag_version" => "v0.13.4", "metal_lb_ip_range" => "192.168.30.80-192.168.30.90", "retry_count" => "30"} } @@ -76,4 +76,4 @@ Vagrant.configure("2") do |config| } } end -end \ No newline at end of file +end diff --git a/vagrant/test_cluster.py b/vagrant/test_cluster.py new file mode 100755 index 0000000..f4e1921 --- /dev/null +++ b/vagrant/test_cluster.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# Perform a few tests on a cluster created with this playbook. +# To simplify test execution, the scripts does not depend on any third-party +# packages, only the Python standard library. + +import json +import subprocess +import unittest +from pathlib import Path +from time import sleep +from warnings import warn + + +VAGRANT_DIR = Path(__file__).parent.absolute() +PLAYBOOK_DIR = VAGRANT_DIR.parent.absolute() + + +class TestK3sCluster(unittest.TestCase): + def _kubectl(self, args: str, json_out: bool = True) -> dict | None: + cmd = "kubectl" + if json_out: + cmd += " -o json" + cmd += f" {args}" + + result = subprocess.run(cmd, capture_output=True, shell=True, check=True) + + if json_out: + return json.loads(result.stdout) + else: + return None + + def _curl(self, url: str) -> str: + options = [ + "--silent", # no progress info + "--show-error", # ... but errors should still be shown + "--fail", # set exit code on error + "--location", # follow redirects + ] + cmd = f'curl {" ".join(options)} "{url}"' + + result = subprocess.run(cmd, capture_output=True, shell=True, check=True) + output = result.stdout.decode("utf-8") + return output + + def _apply_manifest(self, manifest_file: Path) -> dict: + apply_result = self._kubectl( + f'apply --filename="{manifest_file}" --cascade="background"' + ) + self.addCleanup( + lambda: self._kubectl( + f'delete --filename="{manifest_file}"', + json_out=False, + ) + ) + return apply_result + + @staticmethod + def _retry(function, retries: int = 5, seconds_between_retries=1): + for retry in range(1, retries + 1): + try: + return function() + except Exception as exc: + if retry < retries: + sleep(seconds_between_retries) + continue + else: + raise exc + + def _get_load_balancer_ip( + self, + service: str, + namespace: str = "default", + ) -> str | None: + svc_description = self._kubectl( + f'get --namespace="{namespace}" service "{service}"' + ) + ip = svc_description["status"]["loadBalancer"]["ingress"][0]["ip"] + return ip + + def test_nodes_exist(self): + out = self._kubectl("get nodes") + node_names = {item["metadata"]["name"] for item in out["items"]} + self.assertEqual( + node_names, + {"control1", "control2", "control3", "node1", "node2"}, + ) + + def test_ip_address_pool_exists(self): + out = self._kubectl("get --all-namespaces IpAddressPool") + pools = out["items"] + self.assertGreater(len(pools), 0) + + def test_nginx_example_page(self): + # Deploy the manifests to the cluster + deployment = self._apply_manifest(PLAYBOOK_DIR / "example" / "deployment.yml") + service = self._apply_manifest(PLAYBOOK_DIR / "example" / "service.yml") + + # Assert that the dummy page is available + metallb_ip = self._retry( + lambda: self._get_load_balancer_ip(service["metadata"]["name"]) + ) + # Now that an IP address was assigned, let's reload the service description: + service = self._kubectl(f'get service "{service["metadata"]["name"]}"') + metallb_port = service["spec"]["ports"][0]["port"] + + response_body = self._retry( + lambda: self._curl(f"http://{metallb_ip}:{metallb_port}/") + ) + self.assertIn("Welcome to nginx!", response_body) + + +if __name__ == "__main__": + unittest.main()