add support for proxmox lxc containers (#209)

Co-authored-by: Adam Doussan <acdoussan@Adams-MacBook-Pro.local>

inventory/sample/group_vars/all.yml

@@ -49,3 +49,24 @@ metal_lb_controller_tag_version: "v0.13.7"
 
 # metallb ip range for load balancer
 metal_lb_ip_range: "192.168.30.80-192.168.30.90"
+
+# Only enable if your nodes are proxmox LXC nodes, make sure to configure your proxmox nodes
+# in your hosts.ini file.
+# Please read https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185 before using this.
+# Most notably, your containers must be privileged, and must not have nesting set to true.
+# Please note this script disables most of the security of lxc containers, with the trade off being that lxc
+# containers are significantly more resource efficent compared to full VMs.
+# Mixing and matching VMs and lxc containers is not supported, ymmv if you want to do this.
+# I would only really recommend using this if you have partiularly low powered proxmox nodes where the overhead of
+# VMs would use a significant portion of your available resources.
+proxmox_lxc_configure: false
+# the user that you would use to ssh into the host, for example if you run ssh some-user@my-proxmox-host,
+# set this value to some-user
+proxmox_lxc_ssh_user: root
+# the unique proxmox ids for all of the containers in the cluster, both worker and master nodes
+proxmox_lxc_ct_ids:
+  - 200
+  - 201
+  - 202
+  - 203
+  - 204

inventory/sample/hosts.ini

@@ -7,6 +7,11 @@
 192.168.30.41
 192.168.30.42
 
+# only required if proxmox_lxc_configure: true
+# must contain all proxmox instances that have a master or worker node
+# [proxmox]
+# 192.168.30.43
+
 [k3s_cluster:children]
 master
 node

roles/lxc/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: reboot server
+  reboot:

roles/lxc/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: configure rc.local for proxmox lxc containers
+  copy:
+    src: "{{ playbook_dir }}/scripts/rc.local"
+    dest: "/etc/rc.local"
+    mode: "u=rwx,g=rx,o=rx"
+  notify: reboot server

roles/proxmox_lxc/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: reboot containers
+  command:
+    "pct reboot {{ item }}"
+  loop: "{{ proxmox_lxc_filtered_ids }}"

roles/proxmox_lxc/tasks/main.yml

@@ -0,0 +1,50 @@
+---
+- name: check for container files that exist on this host
+  stat:
+    path: "/etc/pve/lxc/{{ item }}.conf"
+  loop: "{{ proxmox_lxc_ct_ids }}"
+  register: stat_results
+
+- name: filter out files that do not exist
+  set_fact:
+    proxmox_lxc_filtered_files:
+      '{{ stat_results.results | rejectattr("stat.exists", "false") | map(attribute="stat.path") }}'
+
+# used for the reboot handler
+- name: get container ids from filtered files
+  set_fact:
+    proxmox_lxc_filtered_ids:
+      '{{ proxmox_lxc_filtered_files | map("split", "/") | map("last") | map("split", ".") | map("first") }}'
+
+# https://gist.github.com/triangletodd/02f595cd4c0dc9aac5f7763ca2264185
+- name: Ensure lxc config has the right apparmor profile
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.apparmor.profile"
+    line: "lxc.apparmor.profile: unconfined"
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right cgroup
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cgroup.devices.allow"
+    line: "lxc.cgroup.devices.allow: a"
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right cap drop
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.cap.drop"
+    line: "lxc.cap.drop: "
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers
+
+- name: Ensure lxc config has the right mounts
+  lineinfile:
+    dest: "{{ item }}"
+    regexp: "^lxc.mount.auto"
+    line: 'lxc.mount.auto: "proc:rw sys:rw"'
+  loop: "{{ proxmox_lxc_filtered_files }}"
+  notify: reboot containers

scripts/rc.local

@@ -0,0 +1,10 @@
+#!/bin/sh -e
+
+# Kubeadm 1.15 needs /dev/kmsg to be there, but it's not in lxc, but we can just use /dev/console instead
+# see: https://github.com/kubernetes-sigs/kind/issues/662
+if [ ! -e /dev/kmsg ]; then
+    ln -s /dev/console /dev/kmsg
+fi
+
+# https://medium.com/@kvaps/run-kubernetes-in-lxc-container-f04aa94b6c9c
+mount --make-rshared /

site.yml

@@ -1,8 +1,18 @@
 ---
 
+- hosts: proxmox
+  gather_facts: true
+  become: yes
+  remote_user: "{{ proxmox_lxc_ssh_user }}"
+  roles:
+    - role: proxmox_lxc
+      when: proxmox_lxc_configure
+
 - hosts: k3s_cluster
   gather_facts: yes
   roles:
+    - role: lxc
+      when: proxmox_lxc_configure
     - role: prereq
       become: true
     - role: download