commit 8b322f39bf43484d8feb8cb89321330996adae21 Author: Pål-Kristian Hamre Date: Tue Jul 15 21:31:41 2025 +0200 Initial commit. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..97d99c4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# --- Vagrant --- +# Ignore the .vagrant directory, which contains all the machine-specific state +# created by Vagrant. This is the most important rule for Vagrant projects. +.vagrant/ + +# Ignore Vagrant log files +vagrant.log + +# Ignore packaged Vagrant boxes +*.box + + +# --- Ansible --- +# Ignore Ansible log files +*.log + +# Ignore Ansible retry files, which are created when a playbook fails +*.retry + +# Ignore Vault password files. It's a good practice even if you aren't using Vault yet. +.vault_pass.txt +vault.yml + + +# --- Common IDE/Editor Files --- +# Visual Studio Code +.vscode/ +# JetBrains (IntelliJ, PyCharm, etc.) +.idea/ +# Vim +*.swp +*.swo +# Sublime Text +*.sublime-project +*.sublime-workspace + + +# --- Operating System Files --- +# macOS +.DS_Store +.AppleDouble +.LSOverride + +# Windows +Thumbs.db +ehthumbs.db +Desktop.ini + + +# --- Python --- +# Ignore Python virtual environments +venv/ +*.pyc +__pycache__/ + + +# --- SSH Keys --- +# Never commit private SSH keys +*.pem +id_rsa +id_ed25519 diff --git a/README.md b/README.md new file mode 100644 index 0000000..cc11220 --- /dev/null +++ b/README.md @@ -0,0 +1,114 @@ +# Vagrant & Ansible Kubernetes Cluster + +This project automates the setup of a high-availability (HA) Kubernetes cluster on a local machine using Vagrant for VM management and Ansible for provisioning. + +The final environment consists of: +* **3 Control Plane Nodes**: Providing a resilient control plane. +* **2 Worker Nodes**: For deploying applications. +* **Networking**: All nodes are connected to the host machine via libvirt's default network (`192.168.122.0/24`). +* **Provisioning**: The cluster is bootstrapped using `kubeadm` and uses Calico for the CNI. + +## Prerequisites + +Before you begin, ensure you have the following software installed on your host machine: + +* [Vagrant](https://www.vagrantup.com/downloads) +* A Vagrant provider, such as [libvirt](https://github.com/vagrant-libvirt/vagrant-libvirt). +* [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) (version 2.10 or newer). + +## Project Structure + +Your project directory should look like this: + +``` +. +├── Vagrantfile # Defines the virtual machines for Vagrant +├── ansible.cfg # Configuration for Ansible +├── cluster.yml # Ansible playbook to deploy Kubernetes +├── inventory.ini # Ansible inventory defining the cluster nodes +└── README.md # This file +``` + +## Setup Instructions + +Follow these steps to build and provision the entire cluster from scratch. + +### Step 1: Customize Configuration (Optional) + +The project is configured to work out-of-the-box for user `pkhamre`. If your setup is different, you may need to adjust the following files: + +1. **`Vagrantfile`**: + * `USERNAME`: Change this if you want to create a different user on the VMs. + * `PUBLIC_KEY_PATH`: Update this to the path of the SSH public key you want to grant access with. + +2. **`ansible.cfg`**: + * `remote_user`: Ensure this matches the `USERNAME` from the `Vagrantfile`. + * `private_key_file`: Ensure this points to the corresponding SSH private key for the public key specified in the `Vagrantfile`. + +3. **`inventory.ini`**: + * The IP addresses are hardcoded to match the `Vagrantfile`. If you change the IPs in `Vagrantfile`, you must update them here as well. + +### Step 2: Create the Virtual Machines + +With the configuration set, use Vagrant to create the five virtual machines defined in the `Vagrantfile`. This command will download the base OS image (if not already cached) and boot the VMs. + +```bash +vagrant up +``` + +This will create the following VMs with static IPs on the `192.168.122.0/24` network: +* `k8s-cp-1` (192.168.122.101) +* `k8s-cp-2` (192.168.122.102) +* `k8s-cp-3` (192.168.122.103) +* `k8s-worker-1` (192.168.122.111) +* `k8s-worker-2` (192.168.122.112) + +### Step 3: Deploy Kubernetes with Ansible + +Once the VMs are running, execute the Ansible playbook. Ansible will connect to each machine, install `containerd` and Kubernetes components, and bootstrap the cluster using `kubeadm`. + +```bash +ansible-playbook cluster.yml +``` + +The playbook will: +1. Install prerequisites on all nodes. +2. Initialize the first control plane node (`k8s-cp-1`). +3. Install the Calico CNI for pod networking. +4. Join the remaining control plane nodes. +5. Join the worker nodes. + +### Step 4: Verify the Cluster + +After the playbook completes, you can access the cluster and verify its status. + +1. SSH into the first control plane node: + ```bash + ssh pkhamre@192.168.122.101 + ``` + +2. Check the status of all nodes. The `kubectl` command-line tool is pre-configured for your user. + ```bash + kubectl get nodes -o wide + ``` + +You should see all 5 nodes in the `Ready` state. It may take a minute for all nodes to report as ready after the playbook finishes. + +``` +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +k8s-cp-1 Ready control-plane 5m12s v1.30.3 192.168.122.101 Ubuntu 24.04 LTS 6.8.0-31-generic containerd://1.7.13 +k8s-cp-2 Ready control-plane 4m2s v1.30.3 192.168.122.102 Ubuntu 24.04 LTS 6.8.0-31-generic containerd://1.7.13 +k8s-cp-3 Ready control-plane 3m56s v1.30.3 192.168.122.103 Ubuntu 24.04 LTS 6.8.0-31-generic containerd://1.7.13 +k8s-worker-1 Ready 2m45s v1.30.3 192.168.122.111 Ubuntu 24.04 LTS 6.8.0-31-generic containerd://1.7.13 +k8s-worker-2 Ready 2m40s v1.30.3 192.168.122.112 Ubuntu 24.04 LTS 6.8.0-31-generic containerd://1.7.13 +``` + +Congratulations! Your Kubernetes cluster is now ready. + +## Cleanup + +To tear down the cluster and delete all virtual machines and associated resources, run the following command from the project directory: + +```bash +vagrant destroy -f +``` \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..ff73e67 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,88 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +# --- USER CUSTOMIZATION --- +# Change the values below to your desired settings. + +# 1. The username for the new user account with sudo permissions. +USERNAME = "pkhamre" + +# 2. The absolute path to your SSH public key. +PUBLIC_KEY_PATH = File.expand_path("~/.ssh/id_ed25519.pub") + +# --- VM & CLUSTER CONFIGURATION --- + +# Base box for all VMs +VAGRANT_BOX = "cloud-image/ubuntu-24.04" +VAGRANT_BOX_VERSION = "20250704.0.0" + +# Predefined static IP addresses and configurations for each node +NODES = [ + { hostname: "k8s-cp-1", ip: "192.168.122.101", memory: 2048, cpus: 2 }, + { hostname: "k8s-cp-2", ip: "192.168.122.102", memory: 2048, cpus: 2 }, + { hostname: "k8s-cp-3", ip: "192.168.122.103", memory: 2048, cpus: 2 }, + { hostname: "k8s-worker-1", ip: "192.168.122.111", memory: 2048, cpus: 4 }, + { hostname: "k8s-worker-2", ip: "192.168.122.112", memory: 2048, cpus: 4 } +] + +Vagrant.configure("2") do |config| + config.vm.box = VAGRANT_BOX + config.vm.box_version = VAGRANT_BOX_VERSION + + # Verify that the specified SSH public key file exists before proceeding. + if !File.exist?(PUBLIC_KEY_PATH) + raise "SSH public key not found at path: #{PUBLIC_KEY_PATH}. Please update the PUBLIC_KEY_PATH variable in the Vagrantfile." + end + publicKey = File.read(PUBLIC_KEY_PATH).strip + + # --- DEFINE VMS FROM THE NODES LIST --- + NODES.each do |node_config| + config.vm.define node_config[:hostname] do |node| + node.vm.hostname = node_config[:hostname] + + # ** CORRECTED NETWORK CONFIGURATION ** + # Use 'private_network' to assign a static IP and 'libvirt__network_name' + # to connect to an existing libvirt virtual network. + node.vm.network "private_network", + ip: node_config[:ip], + libvirt__network_name: "default" + + node.vm.provider "libvirt" do |libvirt| + libvirt.memory = node_config[:memory] + libvirt.cpus = node_config[:cpus] + end + end + end + + # --- COMMON PROVISIONING SCRIPT --- + # This script runs on all nodes to create a user and set up SSH access. + config.vm.provision "shell", inline: <<-SHELL + echo ">>> Starting user and SSH configuration..." + + # Create the user with a home directory and add to the sudo group + if ! id -u #{USERNAME} >/dev/null 2>&1; then + echo ">>> Creating user '#{USERNAME}'" + useradd #{USERNAME} --create-home --shell /bin/bash --groups sudo + else + echo ">>> User '#{USERNAME}' already exists" + fi + + # Grant passwordless sudo to the new user + echo ">>> Configuring passwordless sudo for '#{USERNAME}'" + echo '#{USERNAME} ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/#{USERNAME} + chmod 0440 /etc/sudoers.d/#{USERNAME} + + # Set up SSH key-based authentication + echo ">>> Adding SSH public key for '#{USERNAME}'" + HOME_DIR=$(eval echo ~#{USERNAME}) + mkdir -p $HOME_DIR/.ssh + echo '#{publicKey}' > $HOME_DIR/.ssh/authorized_keys + + # Set correct permissions for the .ssh directory and authorized_keys file + chown -R #{USERNAME}:#{USERNAME} $HOME_DIR/.ssh + chmod 700 $HOME_DIR/.ssh + chmod 600 $HOME_DIR/.ssh/authorized_keys + + echo ">>> User configuration complete!" + SHELL +end diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..6f4bd4e --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,19 @@ +[defaults] +# Use the inventory file in the current directory +inventory = inventory.ini + +# The user to connect to the remote hosts with +remote_user = pkhamre + +# The private key to use for SSH authentication +private_key_file = ~/.ssh/id_ed25519 + +# Disable host key checking for simplicity with new Vagrant VMs +host_key_checking = False + +[privilege_escalation] +# Use sudo for privilege escalation (e.g., for installing packages) +become = true +become_method = sudo +become_user = root +become_ask_pass = false diff --git a/cluster.yml b/cluster.yml new file mode 100644 index 0000000..70bf6dd --- /dev/null +++ b/cluster.yml @@ -0,0 +1,181 @@ +--- +# +# Playbook to set up a Kubernetes cluster using kubeadm +# + +# ============================================================================== +# PHASE 1: Install prerequisites on all nodes +# ============================================================================== +- name: 1. Install prerequisites on all nodes + hosts: all + become: yes + tasks: + - name: Ensure kernel modules are loaded + modprobe: + name: "{{ item }}" + state: present + loop: + - overlay + - br_netfilter + + - name: Persist kernel modules across reboots + copy: + dest: /etc/modules-load.d/k8s.conf + content: | + overlay + br_netfilter + + - name: Set required sysctl parameters + sysctl: + name: "{{ item.key }}" + value: "{{ item.value }}" + sysctl_set: yes + state: present + reload: yes + loop: + - { key: 'net.bridge.bridge-nf-call-iptables', value: '1' } + - { key: 'net.ipv4.ip_forward', value: '1' } + - { key: 'net.bridge.bridge-nf-call-ip6tables', value: '1' } + + - name: Install required packages + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - containerd + state: present + update_cache: yes + + - name: Ensure /etc/containerd directory exists + file: + path: /etc/containerd + state: directory + mode: '0755' + + - name: Configure containerd and enable SystemdCgroup + shell: | + containerd config default | tee /etc/containerd/config.toml >/dev/null 2>&1 + sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml + args: + executable: /bin/bash + changed_when: true + + - name: Restart and enable containerd + systemd: + name: containerd + state: restarted + enabled: yes + + - name: Add Kubernetes v1.33 apt key + get_url: + url: https://pkgs.k8s.io/core:/stable:/v1.33/deb/Release.key + dest: /etc/apt/keyrings/kubernetes-apt-keyring.asc + mode: '0644' + force: true + + - name: Add Kubernetes v1.33 apt repository + apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.asc] https://pkgs.k8s.io/core:/stable:/v1.33/deb/ /" + state: present + filename: kubernetes + + - name: Install kubeadm, kubelet, and kubectl + apt: + name: + - kubelet + - kubeadm + - kubectl + state: present + update_cache: yes + + - name: Hold Kubernetes packages to prevent accidental updates + dpkg_selections: + name: "{{ item }}" + selection: hold + loop: + - kubelet + - kubeadm + - kubectl + +# ============================================================================== +# PHASE 2: Set up the first control plane +# ============================================================================== +- name: 2. Set up the first control plane + hosts: control_planes[0] + become: yes + tasks: + - name: Initialize the Kubernetes cluster + command: "kubeadm init --control-plane-endpoint={{ apiserver_endpoint }} --upload-certs --pod-network-cidr=192.168.0.0/16" + args: + creates: /etc/kubernetes/admin.conf + + - name: Create .kube directory for the user + file: + path: "/home/{{ ansible_user }}/.kube" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + + - name: Copy admin.conf to user's .kube directory + copy: + src: /etc/kubernetes/admin.conf + dest: "/home/{{ ansible_user }}/.kube/config" + remote_src: yes + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0644' + + - name: Install Calico CNI v3.30.2 + become: false # Run as the user with kubectl access + command: "kubectl apply -f https://raw.githubusercontent.com/projectcalico/calico/v3.30.2/manifests/calico.yaml" + args: + creates: /etc/cni/net.d/10-calico.conflist + + - name: Generate join command for other nodes + command: kubeadm token create --print-join-command + register: join_command_control_plane + + - name: Generate certificate key for joining control planes + command: kubeadm init phase upload-certs --upload-certs + register: cert_key + +# ============================================================================== +# PHASE 2.5: Store join commands as facts on localhost +# ============================================================================== +- name: 2.5. Store join commands on the Ansible controller + hosts: localhost + connection: local + gather_facts: no + tasks: + - name: Store the join commands as facts + set_fact: + # *** THIS IS THE FIX *** + # Add the --control-plane flag to the join command for control planes. + join_cp_command: "{{ hostvars[groups['control_planes'][0]]['join_command_control_plane']['stdout'] }} --control-plane --certificate-key {{ hostvars[groups['control_planes'][0]]['cert_key']['stdout_lines'][-1] }}" + join_worker_command: "{{ hostvars[groups['control_planes'][0]]['join_command_control_plane']['stdout'] }}" + +# ============================================================================== +# PHASE 3: Join other control planes to the cluster +# ============================================================================== +- name: 3. Join other control planes to the cluster + hosts: control_planes[1:] + become: yes + tasks: + - name: Join control plane to the cluster + command: "{{ hostvars['localhost']['join_cp_command'] }}" + args: + creates: /etc/kubernetes/kubelet.conf + +# ============================================================================== +# PHASE 4: Set up the worker nodes +# ============================================================================== +- name: 4. Join worker nodes to the cluster + hosts: workers + become: yes + tasks: + - name: Join worker node to the cluster + command: "{{ hostvars['localhost']['join_worker_command'] }}" + args: + creates: /etc/kubernetes/kubelet.conf diff --git a/inventory.ini b/inventory.ini new file mode 100644 index 0000000..b0895a3 --- /dev/null +++ b/inventory.ini @@ -0,0 +1,13 @@ +[control_planes] +192.168.122.101 +192.168.122.102 +192.168.122.103 + +[workers] +192.168.122.111 +192.168.122.112 + +[all:vars] +# The IP of the first control plane node. This will be the API endpoint for the cluster. +# This MUST match the first IP in the [control_planes] group. +apiserver_endpoint = "192.168.122.101"