From e524e06abd6334e7b4d72a3b75f53c66651687bf Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 20:57:07 +0200 Subject: [PATCH 01/13] feat: initial setup for host plumbum --- inventory.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/inventory.yml b/inventory.yml index b1513b4..8ff62db 100644 --- a/inventory.yml +++ b/inventory.yml @@ -10,6 +10,7 @@ all: platon.n39.eu: plumbum.n39.eu: pottwal.n39.eu: + plumbum.n39.eu: radon.n39.eu: unicorn.n39.eu: k3s-c1.n39.eu: @@ -34,6 +35,7 @@ all: oganesson.n39.eu: plumbum.n39.eu: pottwal.n39.eu: + plumbum.n39.eu: radon.n39.eu: unicorn.n39.eu: k3s-c1.n39.eu: @@ -52,6 +54,7 @@ all: platon.n39.eu: plumbum.n39.eu: pottwal.n39.eu: + plumbum.n39.eu: radon.n39.eu: wittgenstein.n39.eu: unicorn.n39.eu: From 760f2d713762ff760c1225239239677d34eeea29 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Fri, 29 Sep 2023 09:38:36 +0200 Subject: [PATCH 02/13] initial plan for storage host role --- roles/k3s-nfs-host/tasks/main.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 roles/k3s-nfs-host/tasks/main.yml diff --git a/roles/k3s-nfs-host/tasks/main.yml b/roles/k3s-nfs-host/tasks/main.yml new file mode 100644 index 0000000..aa7929e --- /dev/null +++ b/roles/k3s-nfs-host/tasks/main.yml @@ -0,0 +1,25 @@ +# nfs server einrichten +# apt install nfs-kernel-server + +# mountpoint anlegen +# /srv/nfs +# sudo chown nobody:nogroup /srv/nfs +# sudo chmod 0777 /srv/nfs + +# modify /etc/exports# +# echo '/srv/nfs 10.0.0.0/24(rw,sync,no_subtree_check)' + +# restart nfs server + +# apply node label +- name: Apply labels to each worker nodes + k8s: + kubeconfig: "{{ ocp_ignition_file_path }}/auth/kubeconfig" + state: present + definition: + apiVersion: v1 + kind: Node + metadata: + name: {{ ansible_hostname }} + labels: + node-role.kubernetes.n39.eu/nfs-host: true \ No newline at end of file From f1fd49431d6acdfb300915f817eea768934cb251 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Fri, 29 Sep 2023 14:25:40 +0200 Subject: [PATCH 03/13] add storage device to host_vars --- host_vars/k3s-w1.n39.eu/vars.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/host_vars/k3s-w1.n39.eu/vars.yml b/host_vars/k3s-w1.n39.eu/vars.yml index cd6e098..c70331e 100644 --- a/host_vars/k3s-w1.n39.eu/vars.yml +++ b/host_vars/k3s-w1.n39.eu/vars.yml @@ -1,2 +1,3 @@ --- mac: "12:D8:CD:E4:69:2A" +nfs_host_storage_device: "/dev/sdb" From aff06a396fd3a7361e4e6b70464c2fbe3f0b9472 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Fri, 29 Sep 2023 14:26:07 +0200 Subject: [PATCH 04/13] add more ansible tasks to rolE --- roles/k3s-nfs-host/defaults/main.yml | 4 +++ roles/k3s-nfs-host/tasks/main.yml | 40 ++++++++++++++++++------- roles/k3s-nfs-host/templates/exports.j2 | 1 + 3 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 roles/k3s-nfs-host/defaults/main.yml create mode 100644 roles/k3s-nfs-host/templates/exports.j2 diff --git a/roles/k3s-nfs-host/defaults/main.yml b/roles/k3s-nfs-host/defaults/main.yml new file mode 100644 index 0000000..a5ed9b6 --- /dev/null +++ b/roles/k3s-nfs-host/defaults/main.yml @@ -0,0 +1,4 @@ +# Defaults for k3s-nfs-host +--- +nfs_host_mount_point: /srv/nfs +nfs_host_kubeconfig_path: "$HOME/.kube/config-k3s-netz39-tunnel" \ No newline at end of file diff --git a/roles/k3s-nfs-host/tasks/main.yml b/roles/k3s-nfs-host/tasks/main.yml index aa7929e..d60ff16 100644 --- a/roles/k3s-nfs-host/tasks/main.yml +++ b/roles/k3s-nfs-host/tasks/main.yml @@ -1,20 +1,38 @@ -# nfs server einrichten -# apt install nfs-kernel-server +--- -# mountpoint anlegen -# /srv/nfs -# sudo chown nobody:nogroup /srv/nfs -# sudo chmod 0777 /srv/nfs +# - name: Create a new ext4 primary partition +# community.general.parted: +# device: "{{ nfs_host_storage_device }}" +# number: 1 +# state: present +# fs_type: ext4 + +- name: Install nfs-kernel-server + ansible.builtin.apt: + state: present + name: + - nfs-kernel-server + +- name: ensure nfs mountpoint is present + ansible.builtin.file: + path: "{{ nfs_host_mount_point }}" + state: directory + owner: nobody + group: nogroup + mode: '0777' # modify /etc/exports# # echo '/srv/nfs 10.0.0.0/24(rw,sync,no_subtree_check)' - +- name: ensure exports file has k3s ips + ansible.builtin.template: + src: templates/exports.j2 + dest: "/etc/exports" # restart nfs server # apply node label -- name: Apply labels to each worker nodes - k8s: - kubeconfig: "{{ ocp_ignition_file_path }}/auth/kubeconfig" +- name: Apply labels to storage worker node + kubernetes.core.k8s: + kubeconfig: "{{ nfs_host_kubeconfig_path }}" state: present definition: apiVersion: v1 @@ -22,4 +40,4 @@ metadata: name: {{ ansible_hostname }} labels: - node-role.kubernetes.n39.eu/nfs-host: true \ No newline at end of file + node-role.kubernetes.n39.eu/nfs-host: true diff --git a/roles/k3s-nfs-host/templates/exports.j2 b/roles/k3s-nfs-host/templates/exports.j2 new file mode 100644 index 0000000..847b8de --- /dev/null +++ b/roles/k3s-nfs-host/templates/exports.j2 @@ -0,0 +1 @@ +{{ nfs_host_mount_point }} k3s-w[0-9]+.n39.eu(rw,sync,no_subtree_check) \ No newline at end of file From 213887052097b4f4eb3365d2cf50d6d1cffbfafe Mon Sep 17 00:00:00 2001 From: David Kilias Date: Fri, 29 Sep 2023 16:00:57 +0200 Subject: [PATCH 05/13] nfs server wird nicht im k3s laufen, labeling entfernt --- host_vars/k3s-w1.n39.eu/vars.yml | 1 - roles/k3s-nfs-host/defaults/main.yml | 4 ---- roles/nfs-host/defaults/main.yml | 4 ++++ roles/{k3s-nfs-host => nfs-host}/tasks/main.yml | 14 -------------- .../templates/exports.j2 | 0 5 files changed, 4 insertions(+), 19 deletions(-) delete mode 100644 roles/k3s-nfs-host/defaults/main.yml create mode 100644 roles/nfs-host/defaults/main.yml rename roles/{k3s-nfs-host => nfs-host}/tasks/main.yml (67%) rename roles/{k3s-nfs-host => nfs-host}/templates/exports.j2 (100%) diff --git a/host_vars/k3s-w1.n39.eu/vars.yml b/host_vars/k3s-w1.n39.eu/vars.yml index c70331e..cd6e098 100644 --- a/host_vars/k3s-w1.n39.eu/vars.yml +++ b/host_vars/k3s-w1.n39.eu/vars.yml @@ -1,3 +1,2 @@ --- mac: "12:D8:CD:E4:69:2A" -nfs_host_storage_device: "/dev/sdb" diff --git a/roles/k3s-nfs-host/defaults/main.yml b/roles/k3s-nfs-host/defaults/main.yml deleted file mode 100644 index a5ed9b6..0000000 --- a/roles/k3s-nfs-host/defaults/main.yml +++ /dev/null @@ -1,4 +0,0 @@ -# Defaults for k3s-nfs-host ---- -nfs_host_mount_point: /srv/nfs -nfs_host_kubeconfig_path: "$HOME/.kube/config-k3s-netz39-tunnel" \ No newline at end of file diff --git a/roles/nfs-host/defaults/main.yml b/roles/nfs-host/defaults/main.yml new file mode 100644 index 0000000..634be53 --- /dev/null +++ b/roles/nfs-host/defaults/main.yml @@ -0,0 +1,4 @@ +# Defaults for nfs-host +--- +nfs_host_mount_point: "/srv/nfs" +nfs_host_storage_device: "/dev/sdb" \ No newline at end of file diff --git a/roles/k3s-nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml similarity index 67% rename from roles/k3s-nfs-host/tasks/main.yml rename to roles/nfs-host/tasks/main.yml index d60ff16..9f67277 100644 --- a/roles/k3s-nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -1,5 +1,4 @@ --- - # - name: Create a new ext4 primary partition # community.general.parted: # device: "{{ nfs_host_storage_device }}" @@ -28,16 +27,3 @@ src: templates/exports.j2 dest: "/etc/exports" # restart nfs server - -# apply node label -- name: Apply labels to storage worker node - kubernetes.core.k8s: - kubeconfig: "{{ nfs_host_kubeconfig_path }}" - state: present - definition: - apiVersion: v1 - kind: Node - metadata: - name: {{ ansible_hostname }} - labels: - node-role.kubernetes.n39.eu/nfs-host: true diff --git a/roles/k3s-nfs-host/templates/exports.j2 b/roles/nfs-host/templates/exports.j2 similarity index 100% rename from roles/k3s-nfs-host/templates/exports.j2 rename to roles/nfs-host/templates/exports.j2 From aa54dcffdc979a6a7e25e74e12f1e74d365089f8 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 18:24:53 +0200 Subject: [PATCH 06/13] set access mode to 0000 to disable writes in mount point --- roles/nfs-host/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml index 9f67277..f98c2bb 100644 --- a/roles/nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -18,7 +18,7 @@ state: directory owner: nobody group: nogroup - mode: '0777' + mode: '0000' # modify /etc/exports# # echo '/srv/nfs 10.0.0.0/24(rw,sync,no_subtree_check)' From 3bdd93b16d98af2f725ce23e40f0ca53f27a73a0 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 18:45:38 +0200 Subject: [PATCH 07/13] feat: make nfs exports more configurable --- roles/nfs-host/defaults/main.yml | 6 +++++- roles/nfs-host/tasks/main.yml | 7 +++---- roles/nfs-host/templates/exports.j2 | 4 +++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/roles/nfs-host/defaults/main.yml b/roles/nfs-host/defaults/main.yml index 634be53..386ccf7 100644 --- a/roles/nfs-host/defaults/main.yml +++ b/roles/nfs-host/defaults/main.yml @@ -1,4 +1,8 @@ # Defaults for nfs-host --- -nfs_host_mount_point: "/srv/nfs" +nfs_host_exports: + - directory: "/srv/nfs" + hosts: "k3s-w[0-9]+.n39.eu" + options: rw,sync,no_subtree_check + nfs_host_storage_device: "/dev/sdb" \ No newline at end of file diff --git a/roles/nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml index f98c2bb..ceb1644 100644 --- a/roles/nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -14,15 +14,14 @@ - name: ensure nfs mountpoint is present ansible.builtin.file: - path: "{{ nfs_host_mount_point }}" + path: "{{ item.directory }}" state: directory owner: nobody group: nogroup mode: '0000' + loop: "{{ nfs_host_exports | dict2items }}" -# modify /etc/exports# -# echo '/srv/nfs 10.0.0.0/24(rw,sync,no_subtree_check)' -- name: ensure exports file has k3s ips +- name: template /etc/exports ansible.builtin.template: src: templates/exports.j2 dest: "/etc/exports" diff --git a/roles/nfs-host/templates/exports.j2 b/roles/nfs-host/templates/exports.j2 index 847b8de..87d2c20 100644 --- a/roles/nfs-host/templates/exports.j2 +++ b/roles/nfs-host/templates/exports.j2 @@ -1 +1,3 @@ -{{ nfs_host_mount_point }} k3s-w[0-9]+.n39.eu(rw,sync,no_subtree_check) \ No newline at end of file +{% for export in nfs_host_exports %} +{{ export.directory }} {{ export.hosts }}({{ export.options }}) +{% endfor %} \ No newline at end of file From ffa3b5d975bc4a441393fd3191bf40cf5f88552a Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 21:20:51 +0200 Subject: [PATCH 08/13] rework nfs-host role --- host_vars/plumbum.n39.eu/vars.yml | 5 +++++ roles/nfs-host/defaults/main.yml | 10 +++++----- roles/nfs-host/handlers/main.yml | 3 +++ roles/nfs-host/tasks/main.yml | 9 +++++++-- 4 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 roles/nfs-host/handlers/main.yml diff --git a/host_vars/plumbum.n39.eu/vars.yml b/host_vars/plumbum.n39.eu/vars.yml index c7eab39..1214d68 100644 --- a/host_vars/plumbum.n39.eu/vars.yml +++ b/host_vars/plumbum.n39.eu/vars.yml @@ -1,2 +1,7 @@ --- +server_admin: "admin+plumbum@netz39.de" mac: "32:A3:94:A0:23:77" +nfs_host_exports: + - directory: "/srv/nfs" + hosts: "k3s-w[0-9]+.n39.eu" + options: rw,sync,no_subtree_check \ No newline at end of file diff --git a/roles/nfs-host/defaults/main.yml b/roles/nfs-host/defaults/main.yml index 386ccf7..210a3be 100644 --- a/roles/nfs-host/defaults/main.yml +++ b/roles/nfs-host/defaults/main.yml @@ -1,8 +1,8 @@ # Defaults for nfs-host --- -nfs_host_exports: - - directory: "/srv/nfs" - hosts: "k3s-w[0-9]+.n39.eu" - options: rw,sync,no_subtree_check +nfs_host_exports: [] + # - directory: "/srv/nfs" + # hosts: "k3s-w[0-9]+.n39.eu" + # options: rw,sync,no_subtree_check -nfs_host_storage_device: "/dev/sdb" \ No newline at end of file +nfs_host_storage_device: "/dev/sdb" diff --git a/roles/nfs-host/handlers/main.yml b/roles/nfs-host/handlers/main.yml new file mode 100644 index 0000000..4cdcec5 --- /dev/null +++ b/roles/nfs-host/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: reload nfs + command: 'exportfs -ra' diff --git a/roles/nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml index ceb1644..7ff864c 100644 --- a/roles/nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -11,8 +11,9 @@ state: present name: - nfs-kernel-server + - nfs-common -- name: ensure nfs mountpoint is present +- name: ensure nfs mountpoints exist ansible.builtin.file: path: "{{ item.directory }}" state: directory @@ -25,4 +26,8 @@ ansible.builtin.template: src: templates/exports.j2 dest: "/etc/exports" -# restart nfs server + notify: reload nfs + +- name: Ensure nfs is running. + ansible.builtin.service: "name={{ nfs_server_daemon }} state=started enabled=yes" + when: nfs_host_exports|length From 9887e40fbabfc85d3e391eab69cab18999860bb8 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 21:23:32 +0200 Subject: [PATCH 09/13] add nfs-host role for plumbum.n39.eu --- host-plumbum.yml | 11 +++++++++++ host_vars/plumbum.n39.eu/vars.yml | 4 ---- main.yml | 3 +++ roles/nfs-host/tasks/main.yml | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) create mode 100644 host-plumbum.yml diff --git a/host-plumbum.yml b/host-plumbum.yml new file mode 100644 index 0000000..7170280 --- /dev/null +++ b/host-plumbum.yml @@ -0,0 +1,11 @@ +--- +- hosts: plumbum.n39.eu + become: true + + roles: + - role: nfs-host + vars: + nfs_host_exports: + - directory: "/srv/nfs" + hosts: "k3s-w[0-9]+.n39.eu" + options: rw,sync,no_subtree_check diff --git a/host_vars/plumbum.n39.eu/vars.yml b/host_vars/plumbum.n39.eu/vars.yml index 1214d68..af6228c 100644 --- a/host_vars/plumbum.n39.eu/vars.yml +++ b/host_vars/plumbum.n39.eu/vars.yml @@ -1,7 +1,3 @@ --- server_admin: "admin+plumbum@netz39.de" mac: "32:A3:94:A0:23:77" -nfs_host_exports: - - directory: "/srv/nfs" - hosts: "k3s-w[0-9]+.n39.eu" - options: rw,sync,no_subtree_check \ No newline at end of file diff --git a/main.yml b/main.yml index 785175b..5af6b86 100644 --- a/main.yml +++ b/main.yml @@ -39,3 +39,6 @@ - name: Hobbes specific setup import_playbook: host-hobbes.yml + +- name: Plumbum specific setup + import_playbook: host-plumbum.yml diff --git a/roles/nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml index 7ff864c..ebb9145 100644 --- a/roles/nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -20,7 +20,7 @@ owner: nobody group: nogroup mode: '0000' - loop: "{{ nfs_host_exports | dict2items }}" + with_items: "{{ nfs_host_exports }}" - name: template /etc/exports ansible.builtin.template: @@ -29,5 +29,5 @@ notify: reload nfs - name: Ensure nfs is running. - ansible.builtin.service: "name={{ nfs_server_daemon }} state=started enabled=yes" + ansible.builtin.service: "name=nfs-kernel-server state=started enabled=yes" when: nfs_host_exports|length From d0ee296f5f806198d977063dab77c8a52a71619a Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 22:28:13 +0200 Subject: [PATCH 10/13] install nfs-common on all k3s nodes --- group-k3s.yml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 group-k3s.yml diff --git a/group-k3s.yml b/group-k3s.yml new file mode 100644 index 0000000..e653d64 --- /dev/null +++ b/group-k3s.yml @@ -0,0 +1,9 @@ +--- +- hosts: k3s + become: true + + tasks: + - name: Ensure nfs-common is installed on k3s VMs + ansible.builtin.apt: + pkg: nfs-common + state: present \ No newline at end of file From 1bf1191484e413c9433724f6b2b2442ccab77846 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 22:28:29 +0200 Subject: [PATCH 11/13] fix permissions on nfs mount point --- roles/nfs-host/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml index ebb9145..d78ebe7 100644 --- a/roles/nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -19,7 +19,7 @@ state: directory owner: nobody group: nogroup - mode: '0000' + mode: '0777' with_items: "{{ nfs_host_exports }}" - name: template /etc/exports From 6fc1e804794b4d5a3c4a2f5b38959d4b091ee600 Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 22:35:34 +0200 Subject: [PATCH 12/13] fix: fix nfs exports hosts glob --- host-plumbum.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/host-plumbum.yml b/host-plumbum.yml index 7170280..d6d1a18 100644 --- a/host-plumbum.yml +++ b/host-plumbum.yml @@ -7,5 +7,5 @@ vars: nfs_host_exports: - directory: "/srv/nfs" - hosts: "k3s-w[0-9]+.n39.eu" + hosts: "*.n39.eu" options: rw,sync,no_subtree_check From 764e54f5304a2c68865b8e18e30ea56d5f4a84bf Mon Sep 17 00:00:00 2001 From: David Kilias Date: Mon, 9 Oct 2023 23:35:50 +0200 Subject: [PATCH 13/13] fix: correct order of operations in nfs-host role --- inventory.yml | 3 --- roles/nfs-host/tasks/main.yml | 24 ++++++++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/inventory.yml b/inventory.yml index 8ff62db..b1513b4 100644 --- a/inventory.yml +++ b/inventory.yml @@ -10,7 +10,6 @@ all: platon.n39.eu: plumbum.n39.eu: pottwal.n39.eu: - plumbum.n39.eu: radon.n39.eu: unicorn.n39.eu: k3s-c1.n39.eu: @@ -35,7 +34,6 @@ all: oganesson.n39.eu: plumbum.n39.eu: pottwal.n39.eu: - plumbum.n39.eu: radon.n39.eu: unicorn.n39.eu: k3s-c1.n39.eu: @@ -54,7 +52,6 @@ all: platon.n39.eu: plumbum.n39.eu: pottwal.n39.eu: - plumbum.n39.eu: radon.n39.eu: wittgenstein.n39.eu: unicorn.n39.eu: diff --git a/roles/nfs-host/tasks/main.yml b/roles/nfs-host/tasks/main.yml index d78ebe7..c1ac52d 100644 --- a/roles/nfs-host/tasks/main.yml +++ b/roles/nfs-host/tasks/main.yml @@ -1,17 +1,18 @@ --- -# - name: Create a new ext4 primary partition -# community.general.parted: -# device: "{{ nfs_host_storage_device }}" -# number: 1 -# state: present -# fs_type: ext4 - -- name: Install nfs-kernel-server +- name: Install required packages ansible.builtin.apt: state: present name: - nfs-kernel-server - nfs-common + - parted + +- name: Create a new ext4 primary partition + community.general.parted: + device: "{{ nfs_host_storage_device }}" + number: 1 + state: present + fs_type: ext4 - name: ensure nfs mountpoints exist ansible.builtin.file: @@ -22,6 +23,13 @@ mode: '0777' with_items: "{{ nfs_host_exports }}" +- name: Mount up device by label + ansible.posix.mount: + path: "{{ nfs_host_exports[0].directory }}" + src: /dev/sdb1 + fstype: ext4 + state: present + - name: template /etc/exports ansible.builtin.template: src: templates/exports.j2