diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index bd098d1..f1d9864 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -49,3 +49,24 @@ metal_lb_controller_tag_version: "v0.13.7" # metallb ip range for load balancer metal_lb_ip_range: "192.168.30.80-192.168.30.90" + +# Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes +# in your hosts.ini file. +# Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this. +# Most notably, your containers must be privileged, and must not have nesting set to true. +# Please note this script disables most of the security of lxc containers, with the trade off being that lxc +# containers are significantly more resource efficent compared to full VMs. +# Mixing and matching VMs and lxc containers is not supported, ymmv if you want to do this. +# I would only really recommend using this if you have partiularly low powered proxmox nodes where the overhead of +# VMs would use a significant portion of your available resources. +proxmox_lxc_configure: false +# the user that you would use to ssh into the host, for example if you run ssh some-user@my-proxmox-host, +# set this value to some-user +proxmox_lxc_ssh_user: root +# the unique proxmox ids for all of the containers in the cluster, both worker and master nodes +proxmox_lxc_ct_ids: + - 200 + - 201 + - 202 + - 203 + - 204 diff --git a/inventory/sample/hosts.ini b/inventory/sample/hosts.ini index b656847..7045423 100644 --- a/inventory/sample/hosts.ini +++ b/inventory/sample/hosts.ini @@ -7,6 +7,11 @@ 192.168.30.41 192.168.30.42 +# only required if proxmox_lxc_configure: true +# must contain all proxmox instances that have a master or worker node +# [proxmox] +# 192.168.30.43 + [k3s_cluster:children] master node diff --git a/roles/lxc/handlers/main.yml b/roles/lxc/handlers/main.yml new file mode 100644 index 0000000..1bba5c8 --- /dev/null +++ b/roles/lxc/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: reboot server + reboot: diff --git a/roles/lxc/tasks/main.yml b/roles/lxc/tasks/main.yml new file mode 100644 index 0000000..d47200a --- /dev/null +++ b/roles/lxc/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: configure rc.local for proxmox lxc containers + copy: + src: "{{ playbook_dir }}/scripts/rc.local" + dest: "/etc/rc.local" + mode: "u=rwx,g=rx,o=rx" + notify: reboot server diff --git a/roles/proxmox_lxc/handlers/main.yml b/roles/proxmox_lxc/handlers/main.yml new file mode 100644 index 0000000..9b99cb2 --- /dev/null +++ b/roles/proxmox_lxc/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: reboot containers + command: + "pct reboot {{ item }}" + loop: "{{ proxmox_lxc_filtered_ids }}" diff --git a/roles/proxmox_lxc/tasks/main.yml b/roles/proxmox_lxc/tasks/main.yml new file mode 100644 index 0000000..76d43a0 --- /dev/null +++ b/roles/proxmox_lxc/tasks/main.yml @@ -0,0 +1,50 @@ +--- +- name: check for container files that exist on this host + stat: + path: "/etc/pve/lxc/{{ item }}.conf" + loop: "{{ proxmox_lxc_ct_ids }}" + register: stat_results + +- name: filter out files that do not exist + set_fact: + proxmox_lxc_filtered_files: + '{{ stat_results.results | rejectattr("stat.exists", "false") | map(attribute="stat.path") }}' + +# used for the reboot handler +- name: get container ids from filtered files + set_fact: + proxmox_lxc_filtered_ids: + '{{ proxmox_lxc_filtered_files | map("split", "/") | map("last") | map("split", ".") | map("first") }}' + +# https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 +- name: Ensure lxc config has the right apparmor profile + lineinfile: + dest: "{{ item }}" + regexp: "^lxc.apparmor.profile" + line: "lxc.apparmor.profile: unconfined" + loop: "{{ proxmox_lxc_filtered_files }}" + notify: reboot containers + +- name: Ensure lxc config has the right cgroup + lineinfile: + dest: "{{ item }}" + regexp: "^lxc.cgroup.devices.allow" + line: "lxc.cgroup.devices.allow: a" + loop: "{{ proxmox_lxc_filtered_files }}" + notify: reboot containers + +- name: Ensure lxc config has the right cap drop + lineinfile: + dest: "{{ item }}" + regexp: "^lxc.cap.drop" + line: "lxc.cap.drop: " + loop: "{{ proxmox_lxc_filtered_files }}" + notify: reboot containers + +- name: Ensure lxc config has the right mounts + lineinfile: + dest: "{{ item }}" + regexp: "^lxc.mount.auto" + line: 'lxc.mount.auto: "proc:rw sys:rw"' + loop: "{{ proxmox_lxc_filtered_files }}" + notify: reboot containers diff --git a/scripts/rc.local b/scripts/rc.local new file mode 100644 index 0000000..daa6c77 --- /dev/null +++ b/scripts/rc.local @@ -0,0 +1,10 @@ +#!/bin/sh -e + +# Kubeadm 1.15 needs /dev/kmsg to be there, but it's not in lxc, but we can just use /dev/console instead +# see: https://github.com/kubernetes-sigs/kind/issues/662 +if [ ! -e /dev/kmsg ]; then + ln -s /dev/console /dev/kmsg +fi + +# https://medium.com/@kvaps/run-kubernetes-in-lxc-container-f04aa94b6c9c +mount --make-rshared / diff --git a/site.yml b/site.yml index 35f81df..f6f0b09 100644 --- a/site.yml +++ b/site.yml @@ -1,8 +1,18 @@ --- +- hosts: proxmox + gather_facts: true + become: yes + remote_user: "{{ proxmox_lxc_ssh_user }}" + roles: + - role: proxmox_lxc + when: proxmox_lxc_configure + - hosts: k3s_cluster gather_facts: yes roles: + - role: lxc + when: proxmox_lxc_configure - role: prereq become: true - role: download