#!/bin/bash
virsh destroy okd-bootstrap
virsh undefine okd-bootstrap
virsh destroy okd-cp1
virsh undefine okd-cp1
virsh destroy okd-cp2
virsh undefine okd-cp2
virsh destroy okd-cp3
virsh undefine okd-cp3
rm -rf /var/lib/libvirt/images/*.ign
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-35
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-35
# 3. Préparation Ignition
cp bootstrap.ign /var/lib/libvirt/images/bootstrap.ign
chmod 777 /var/lib/libvirt/images/bootstrap.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/bootstrap.ign"
# 4. Installation
virt-install \
--name okd-bootstrap \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-35 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:07 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-36
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-36
# 3. Préparation Ignition
cp master.ign /var/lib/libvirt/images/master.ign
chmod 777 /var/lib/libvirt/images/master.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/master.ign"
# 4. Installation
virt-install \
--name okd-cp1 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-36 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:01 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-37
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-37
# 3. Préparation Ignition
cp master.ign /var/lib/libvirt/images/master.ign
chmod 777 /var/lib/libvirt/images/master.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/master.ign"
# 4. Installation
virt-install \
--name okd-cp2 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-37 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:02 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-38
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-38
# 3. Préparation Ignition
cp master.ign /var/lib/libvirt/images/master.ign
chmod 777 /var/lib/libvirt/images/master.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/master.ign"
# 4. Installation
virt-install \
--name okd-cp3 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-38 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:03 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
#!/bin/bash
virsh destroy okd-worker1
virsh undefine okd-worker1
virsh destroy okd-worker2
virsh undefine okd-worker2
virsh destroy okd-worker3
virsh undefine okd-worker3
virsh destroy okd-worker4
virsh undefine okd-worker4
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-39
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-43
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-39
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker1 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-39 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-43 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:04 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:04 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-40
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-44
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-40
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker2 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-40 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-44 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:05 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:05 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-41
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-45
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-41
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker3 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-41 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-45 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:06 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:06 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
########################################################################
# 1. Nettoyage et Copie de l'image
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-42
wipefs -a /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-46
qemu-img convert -O raw /mnt/images/linux/fcos.qcow2 /dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-42
# 3. Préparation Ignition
cp worker.ign /var/lib/libvirt/images/worker.ign
chmod 777 /var/lib/libvirt/images/worker.ign
IGNITION_DEVICE_ARG="--qemu-commandline=-fw_cfg name=opt/com.coreos/config,file=/var/lib/libvirt/images/worker.ign"
# 4. Installation
virt-install \
--name okd-worker4 \
--memory 16384 --vcpus 4 --cpu host-passthrough \
--machine q35 --os-variant=fedora-coreos-stable \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-42 \
--disk path=/dev/disk/by-path/ip-172.16.100.14:3260-iscsi-iqn.2005-10.org.freenas.ctl:vms-storage-lun-46 \
--network network=vlan219,model=virtio,mac=52:54:00:aa:bb:09 \
--network network=ovsbr0,portgroup=vlan-all,model=virtio,mac=52:54:00:aa:bc:09 \
--graphics none \
--import \
--noautoconsole \
"${IGNITION_DEVICE_ARG}"
openshift-install create ignition-configs --dir=./install_dir
export KUBECONFIG=./install_dir/auth/kubeconfig
oc get nodes -o wide
openshift-install wait-for bootstrap-complete --dir=./install_dir
oc get csr | grep -i pending
oc get csr -o name | xargs oc adm certificate approve
oc patch ingresscontroller default -n openshift-ingress-operator --type=merge -p '{"spec":{"replicas": 4}}'
cat ./install_dir/auth/kubeadmin-password
kubeadmin
<PASSWORD_REMOVED>
Once the installation completed you should get access to the OKD GUI via https://console-openshift-console.apps.okd.int.example.com/
oc create namespace nfs-csi && \ oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/rbac-csi-nfs.yaml && \ oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/csi-nfs-driverinfo.yaml && \ oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/csi-nfs-controller.yaml && \ oc apply -f https://raw.githubusercontent.com/kubernetes-csi/csi-driver-nfs/master/deploy/csi-nfs-node.yaml && \ oc get pods -n kube-system | grep nfs
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: nfs
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: nfs.csi.k8s.io
parameters:
server: 172.16.100.14
share: /mnt/pool1/nfs/okd
reclaimPolicy: Delete
volumeBindingMode: Immediate
mountOptions:
- nfsvers=4.1
oc apply -f storageclass-nfs.yaml oc get storageclass
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: nfs-test-pvc
spec:
storageClassName: nfs
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
apiVersion: v1
kind: Pod
metadata:
name: test-pod
spec:
containers:
- name: test-pod
image: registry.access.redhat.com/ubi9/ubi
command: ["sh", "-c", "sleep 3600"]
volumeMounts:
- name: data
mountPath: /data
volumes:
- name: data
persistentVolumeClaim:
claimName: nfs-test-pvc
oc apply -f nfs-test-pvc.yaml oc apply -f test-pod.yaml oc exec -it nfs-test-pod -- sh echo "NFS OKD OK" > /data/test.txt exit oc delete pod tets-pod oc apply -f test-pod.yaml oc exec -it nfs-test-pod -- cat /data/test.txt NFS OKD OK
This page documents how to configure a Macvlan secondary network on an OKD 4.5 cluster composed of 3 control planes and 4 workers. The goal is to allow pods to attach to an external Layer‑2 network (VLAN 222) using a macvlan interface. Configuration involves two main components:
A NetworkAttachmentDefinition (NAD) for Multus A MachineConfig to create the VLAN interface on each worker node
OpenShift/OKD manages the primary interface of each node using its internal SDN (OVN-Kubernetes in 4.x). However, when a pod needs direct access to a physical VLAN (example: routed lab network, IoT segment, infrastructure VLAN), we must use Multus + Macvlan to create a secondary network. Because OKD nodes are immutable, the required VLAN interface (enp8s0.222) must be created via a MachineConfig. The workflow:
MachineConfig creates VLAN interface enp8s0.222 NAD defines a macvlan network using that interface Pods reference the network through annotations
This definition describes the macvlan secondary network.
apiVersion: k8s.cni.cncf.io/v1
kind: NetworkAttachmentDefinition
metadata:
name: macvlan-enp8s0-vlan222
namespace: default
spec:
config: |
{
"cniVersion": "0.3.1",
"name": "macvlan-enp8s0-vlan222",
"type": "macvlan",
"master": "enp8s0.222",
"mode": "bridge",
"ipam": {
"type": "static"
}
}
| Field | Description |
|---|---|
| type: macvlan | Creates one macvlan interface per pod |
| master: enp8s0.222 | Parent interface (VLAN must already exist on the node) |
| mode: bridge | Allows pod ↔ pod traffic on the same worker |
| ipam: static | Pod IPs must be manually defined in annotations |
annotations:
k8s.v1.cni.cncf.io/networks: |
[{
"name": "macvlan-enp8s0-vlan222",
"ips": ["192.168.222.50/24"],
"gateway": "192.168.222.1"
}]
OKD nodes are immutable; network interfaces cannot be managed with nmcli or traditional config files. A MachineConfig is required to create the VLAN interface during node boot.
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
name: 99-enp8s0-vlan222
labels:
machineconfiguration.openshift.io/role: worker
spec:
config:
ignition:
version: 3.2.0
storage:
files:
- path: /usr/local/bin/create-vlan222.sh
mode: 0755
contents:
source: data:,%23%21%2Fbin%2Fbash%0Aip%20link%20add%20link%20enp8s0%20name%20enp8s0.222%20type%20vlan%20id%20222%20%7C%7C%20true%0Aip%20link%20set%20enp8s0.222%20up
systemd:
units:
- name: create-vlan222.service
enabled: true
contents: |
[Unit]
Description=Create VLAN 222 interface on enp8s0
After=network-online.target
Wants=network-online.target
[Service]
Type=oneshot
ExecStart=/usr/local/bin/create-vlan222.sh
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target
The MachineConfig performs two main actions:
Decoded content:
#!/bin/bash ip link add link enp8s0 name enp8s0.222 type vlan id 222 || true ip link set enp8s0.222 up
Notes: Creates VLAN 222 on interface enp8s0 Brings the interface UP
Runs after the network is online to guarantee the parent interface exists.
MachineConfig adds VLAN interface enp8s0.222 to all worker nodes NAD defines a macvlan network using this interface Pods request the network with a Multus annotation Pods receive a secondary interface (net1) on VLAN 222
This provides Layer‑2 connectivity directly from the pod to the external network, bypassing the cluster SDN.
jonathan@jonathan-VirtualBox:~/okd1/dokuwiki$ oc debug node/okd-worker1.okd.int.example.com
Starting pod/okd-worker1okdintclocheca-debug-fssrl ...
To use host binaries, run `chroot /host`. Instead, if you need to access host namespaces, run `nsenter -a -t 1`.
Pod IP: 172.16.100.154
All commands and output from this session will be recorded in container logs, including credentials and sensitive information passed through the command prompt.
If you don't see a command prompt, try pressing enter.
sh-5.1# ip link show enp8s0.222
9: enp8s0.222@enp8s0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
link/ether 52:54:00:aa:bc:04 brd ff:ff:ff:ff:ff:ff
sh-5.1#
jonathan@jonathan-VirtualBox:~/okd1/dokuwiki$ oc get net-attach-def -n default NAME AGE macvlan-enp8s0 43h macvlan-enp8s0-vlan222 17h
The following command is used to check the status of the MachineConfigPool for the worker nodes:
oc get mcp worker
This command displays the current state of the worker MachineConfigPool (MCP). It is used to monitor the rollout of a MachineConfig across all worker nodes.
It shows information such as:
In summary: this command is used to verify and monitor the deployment of MachineConfig changes on worker nodes.
The following examples show how to create simple pods that attach to the macvlan-enp8s0-vlan222 secondary network. Each pod receives a static IP address on VLAN 222 using Multus annotations.
Both pods use the nicolaka/netshoot image, which provides useful networking tools for testing.
apiVersion: v1 kind: Pod metadata: name: macvlan-pod1 annotations: k8s.v1.cni.cncf.io/networks: | [{ "name": "macvlan-enp8s0-vlan222", "ips": [ "10.194.22.200/24" ], "gateway": ["10.194.22.1"] }] spec: containers: - name: test image: nicolaka/netshoot command: ["sleep","3600"] securityContext: privileged: true
apiVersion: v1 kind: Pod metadata: name: macvlan-pod2 annotations: k8s.v1.cni.cncf.io/networks: | [{ "name": "macvlan-enp8s0-vlan222", "ips": [ "10.194.22.201/24" ], "gateway": ["10.194.22.1"] }] spec: containers: - name: test image: nicolaka/netshoot command: ["sleep","3600"] securityContext: privileged: true
Once both pods are running, you can verify their network configuration:
oc exec -it macvlan-pod1 -- ip addr oc exec -it macvlan-pod2 -- ip addr
Test L2/L3 communication:
oc exec -it macvlan-pod1 -- ping 10.194.22.201 oc exec -it macvlan-pod2 -- ping 10.194.22.200
\n \n
A fresh OKD installation only provides the ``kubeadmin`` user. To allow local users to log in and deploy pods, configure an HTPasswd identity provider.
docker run --rm \ -v $(pwd):/work \ docker.io/httpd:2 \ htpasswd -cbB /work/users.htpasswd user1 MyPassword123 docker run --rm \ -v $(pwd):/work \ docker.io/httpd:2 \ htpasswd -bB /work/users.htpasswd user2 MyPassword123 docker run --rm \ -v $(pwd):/work \ docker.io/httpd:2 \ htpasswd -bB /work/users.htpasswd user3 MyPassword123
oc create secret generic htpasswd-secret \ --from-file=htpasswd=users.htpasswd \ -n openshift-config
Edit the OAuth configuration:
oc edit oauth cluster
Add:
spec: identityProviders: - name: local-users mappingMethod: claim type: HTPasswd htpasswd: fileData: name: htpasswd-secret
Wait for the OAuth operator to restart.
Log out of the Web Console and log in with:
Allow user to create their own Projects:
oc adm policy add-cluster-role-to-user self-provisioner myuser
Or give admin access to a specific namespace:
oc adm policy add-role-to-user admin myuser -n mynamespace
The user can now create and run pods.