===== OKD1 =====
==== VM spin-up ====
#!/bin/bash
virsh destroy okd-bootstrap
virsh undefine okd-bootstrap
virsh destroy okd-cp1
virsh undefine okd-cp1
virsh destroy okd-cp2
virsh undefine okd-cp2
virsh destroy okd-cp3
virsh undefine okd-cp3
rm -rf /var/lib/libvirt/images/*.ign
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-35
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-35
# 3. Préparation Ignition
cp bootstrap.ign /var/lib/libvirt/images/bootstrap.ign
chmod 777 /var/lib/libvirt/images/bootstrap.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/bootstrap.ign"
# 4. Installation
virt-install \
--name okd-bootstrap \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-35 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:07 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-36
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-36
# 3. Préparation Ignition
cp master.ign /var/lib/libvirt/images/master.ign
chmod 777 /var/lib/libvirt/images/master.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/master.ign"
# 4. Installation
virt-install \
--name okd-cp1 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-36 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:01 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-37
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-37
# 3. Préparation Ignition
cp master.ign /var/lib/libvirt/images/master.ign
chmod 777 /var/lib/libvirt/images/master.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/master.ign"
# 4. Installation
virt-install \
--name okd-cp2 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-37 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:02 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-38
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-38
# 3. Préparation Ignition
cp master.ign /var/lib/libvirt/images/master.ign
chmod 777 /var/lib/libvirt/images/master.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/master.ign"
# 4. Installation
virt-install \
--name okd-cp3 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-38 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:03 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
#!/bin/bash
virsh destroy okd-worker1
virsh undefine okd-worker1
virsh destroy okd-worker2
virsh undefine okd-worker2
virsh destroy okd-worker3
virsh undefine okd-worker3
virsh destroy okd-worker4
virsh undefine okd-worker4
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-39
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-43
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-39
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker1 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-39 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-43 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:04 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:04 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-40
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-44
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-40
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker2 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-40 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-44 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:05 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:05 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-41
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-45
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-41
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker3 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-41 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-45 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:06 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:06 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-42
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-46
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-42
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker4 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-42 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-46 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:09 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:09 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
==== Installation command ====
openshift-install create ignition-configs --dir=./install_dir
export KUBECONFIG=./install_dir/auth/kubeconfig
oc get nodes -o wide
openshift-install wait-for bootstrap-complete --dir=./install_dir
oc get csr | grep -i pending
oc get csr -o name | xargs oc adm certificate approve
oc patch ingresscontroller default -n openshift-ingress-operator --type=merge -p '{"spec":{"replicas": 4}}'
cat ./install_dir/auth/kubeadmin-password
kubeadmin
Once the installation completed you should get access to the OKD GUI via https://console-openshift-console.apps.okd.int.example.com/
==== Configure NFS client on worker nodes ====
oc create namespace nfs-csi && \
oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/rbac-csi-nfs.yaml && \
oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/csi-nfs-driverinfo.yaml && \
oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/csi-nfs-controller.yaml && \
oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/csi-nfs-node.yaml && \
oc get pods -n kube-system | grep nfs
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: nfs.csi.k8s.io
parameters:
server: 172.16.100.14
share: /mnt/pool1/nfs/okd
reclaimPolicy: Delete
volumeBindingMode: Immediate
mountOptions:
- nfsvers=4.1
oc apply -f storageclass-nfs.yaml
oc get storageclass
====Test NFS storage ====
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nfs-test-pvc
spec:
storageClassName: nfs
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
apiVersion: v1
kind: Pod
metadata:
name: test-pod
spec:
containers:
- name: test-pod
image: registry.access.redhat.com/ubi9/ubi
command: ["sh", "-c", "sleep 3600"]
volumeMounts:
- name: data
mountPath: /data
volumes:
- name: data
persistentVolumeClaim:
claimName: nfs-test-pvc
oc apply -f nfs-test-pvc.yaml
oc apply -f test-pod.yaml
oc exec -it nfs-test-pod -- sh
echo "NFS OKD OK" > /data/test.txt
exit
oc delete pod tets-pod
oc apply -f test-pod.yaml
oc exec -it nfs-test-pod -- cat /data/test.txt
NFS OKD OK
==== MACVLAN ====
This page documents how to configure a Macvlan secondary network on an OKD 4.5 cluster composed of 3 control planes and 4 workers.
The goal is to allow pods to attach to an external Layer‑2 network (VLAN 222) using a macvlan interface.
Configuration involves two main components:
A NetworkAttachmentDefinition (NAD) for Multus
A MachineConfig to create the VLAN interface on each worker node
====== 1. Overview ======
OpenShift/OKD manages the primary interface of each node using its internal SDN (OVN-Kubernetes in 4.x).
However, when a pod needs direct access to a physical VLAN (example: routed lab network, IoT segment, infrastructure VLAN), we must use Multus + Macvlan to create a secondary network.
Because OKD nodes are immutable, the required VLAN interface (enp8s0.222) must be created via a MachineConfig.
The workflow:
MachineConfig creates VLAN interface enp8s0.222
NAD defines a macvlan network using that interface
Pods reference the network through annotations
====== 2. NetworkAttachmentDefinition ======
This definition describes the macvlan secondary network.
===== File: macvlan-enp8s0-vlan222.yaml =====
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: macvlan-enp8s0-vlan222
namespace: default
spec:
config: |
{
"cniVersion": "0.3.1",
"name": "macvlan-enp8s0-vlan222",
"type": "macvlan",
"master": "enp8s0.222",
"mode": "bridge",
"ipam": {
"type": "static"
}
}
===== Explanation =====
^ Field ^ Description ^
| type: macvlan | Creates one macvlan interface per pod |
| master: enp8s0.222 | Parent interface (VLAN must already exist on the node) |
| mode: bridge | Allows pod ↔ pod traffic on the same worker |
| ipam: static | Pod IPs must be manually defined in annotations |
===== Example pod annotation =====
annotations:
k8s.v1.cni.cncf.io/networks: |
[{
"name": "macvlan-enp8s0-vlan222",
"ips": ["192.168.222.50/24"],
"gateway": "192.168.222.1"
}]
====== 3. MachineConfig (VLAN Creation) ======
OKD nodes are immutable; network interfaces cannot be managed with nmcli or traditional config files.
A MachineConfig is required to create the VLAN interface during node boot.
===== File: workers-enp8s0-vlan222-config.yaml =====
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
name: 99-enp8s0-vlan222
labels:
machineconfiguration.openshift.io/role: worker
spec:
config:
ignition:
version: 3.2.0
storage:
files:
- path: /usr/local/bin/create-vlan222.sh
mode: 0755
contents:
source: data:,%23%21%2Fbin%2Fbash%0Aip%20link%20add%20link%20enp8s0%20name%20enp8s0.222%20type%20vlan%20id%20222%20%7C%7C%20true%0Aip%20link%20set%20enp8s0.222%20up
systemd:
units:
- name: create-vlan222.service
enabled: true
contents: |
[Unit]
Description=Create VLAN 222 interface on enp8s0
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/create-vlan222.sh
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
===== Explanation =====
The MachineConfig performs two main actions:
==== 1. Creates script: /usr/local/bin/create-vlan222.sh ====
Decoded content:
#!/bin/bash
ip link add link enp8s0 name enp8s0.222 type vlan id 222 || true
ip link set enp8s0.222 up
Notes:
Creates VLAN 222 on interface enp8s0
Brings the interface UP
|| true prevents failure if the interface already exists
==== 2. Installs systemd service to run the script ====
Runs after the network is online to guarantee the parent interface exists.
====== 4. How it Works End-to-End ======
MachineConfig adds VLAN interface enp8s0.222 to all worker nodes
NAD defines a macvlan network using this interface
Pods request the network with a Multus annotation
Pods receive a secondary interface (net1) on VLAN 222
This provides Layer‑2 connectivity directly from the pod to the external network, bypassing the cluster SDN.
====== 5. Validation Steps ======
==== Check VLAN presence ====
jonathan@jonathan-VirtualBox:~/okd1/dokuwiki$ oc debug node/okd-worker1.okd.int.example.com
Starting pod/okd-worker1okdintclocheca-debug-fssrl ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
Pod IP: 172.16.100.154
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
sh-5.1# ip link show enp8s0.222
9: enp8s0.222@enp8s0: mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
link/ether 52:54:00:aa:bc:04 brd ff:ff:ff:ff:ff:ff
sh-5.1#
==== Check Multus attachment definitions ====
jonathan@jonathan-VirtualBox:~/okd1/dokuwiki$ oc get net-attach-def -n default
NAME AGE
macvlan-enp8s0 43h
macvlan-enp8s0-vlan222 17h
===== Monitoring MachineConfig Deployment =====
The following command is used to check the status of the MachineConfigPool for the worker nodes:
oc get mcp worker
This command displays the current state of the worker MachineConfigPool (MCP). It is used to monitor the rollout of a MachineConfig across all worker nodes.
It shows information such as:
* whether the workers are **updating**
* how many nodes are **ready**
* how many nodes still need to **reboot**
* whether the pool is fully **updated** and **in sync**
In summary: **this command is used to verify and monitor the deployment of MachineConfig changes on worker nodes.**
===== Example Pods Using the macvlan-enp8s0-vlan222 Network =====
The following examples show how to create simple pods that attach to the
**macvlan-enp8s0-vlan222** secondary network.
Each pod receives a static IP address on VLAN 222 using Multus annotations.
Both pods use the **nicolaka/netshoot** image, which provides useful networking tools for testing.
==== Pod 1: macvlan-pod1 ====
apiVersion: v1
kind: Pod
metadata:
name: macvlan-pod1
annotations:
k8s.v1.cni.cncf.io/networks: |
[{
"name": "macvlan-enp8s0-vlan222",
"ips": [ "10.194.22.200/24" ],
"gateway": ["10.194.22.1"]
}]
spec:
containers:
- name: test
image: nicolaka/netshoot
command: ["sleep","3600"]
securityContext:
privileged: true
==== Pod 2: macvlan-pod2 ====
apiVersion: v1
kind: Pod
metadata:
name: macvlan-pod2
annotations:
k8s.v1.cni.cncf.io/networks: |
[{
"name": "macvlan-enp8s0-vlan222",
"ips": [ "10.194.22.201/24" ],
"gateway": ["10.194.22.1"]
}]
spec:
containers:
- name: test
image: nicolaka/netshoot
command: ["sleep","3600"]
securityContext:
privileged: true
==== Verifying Connectivity ====
Once both pods are running, you can verify their network configuration:
oc exec -it macvlan-pod1 -- ip addr
oc exec -it macvlan-pod2 -- ip addr
Test L2/L3 communication:
oc exec -it macvlan-pod1 -- ping 10.194.22.201
oc exec -it macvlan-pod2 -- ping 10.194.22.200
\n
\n
===== Creating Local Users in OKD 4.21 Using HTPasswd =====
A fresh OKD installation only provides the ``kubeadmin`` user.
To allow local users to log in and deploy pods, configure an HTPasswd
identity provider.
----
==== Step 1: Create the htpasswd file ====
docker run --rm \
-v $(pwd):/work \
docker.io/httpd:2 \
htpasswd -cbB /work/users.htpasswd user1 MyPassword123
docker run --rm \
-v $(pwd):/work \
docker.io/httpd:2 \
htpasswd -bB /work/users.htpasswd user2 MyPassword123
docker run --rm \
-v $(pwd):/work \
docker.io/httpd:2 \
htpasswd -bB /work/users.htpasswd user3 MyPassword123
----
==== Step 2: Create the secret in the openshift-config namespace ====
oc create secret generic htpasswd-secret \
--from-file=htpasswd=users.htpasswd \
-n openshift-config
----
==== Step 3: Configure OAuth to use HTPasswd ====
Edit the OAuth configuration:
oc edit oauth cluster
Add:
spec:
identityProviders:
- name: local-users
mappingMethod: claim
type: HTPasswd
htpasswd:
fileData:
name: htpasswd-secret
Wait for the OAuth operator to restart.
----
==== Step 4: Test login ====
Log out of the Web Console and log in with:
* Username: ``myuser``
* Password: ``MyPassword123``
----
==== Step 5: Grant permissions to the new user ====
Allow user to create their own Projects:
oc adm policy add-cluster-role-to-user self-provisioner myuser
Or give admin access to a specific namespace:
oc adm policy add-role-to-user admin myuser -n mynamespace
The user can now create and run pods.