==== OKD ==== ====DHCP==== === dhcpd.conf === host okd-haproxy { hardware ethernet 52:54:01:d2:46:20; fixed-address 192.168.100.20; } host okd-cp1 { hardware ethernet 52:54:01:d2:46:21; fixed-address 192.168.100.20; } host okd-cp2 { hardware ethernet 52:54:01:d2:46:22; fixed-address 192.168.100.20; } host okd-cp3 { hardware ethernet 52:54:01:d2:46:23; fixed-address 192.168.100.20; } host okd-worker1 { hardware ethernet 52:54:01:d2:46:24; fixed-address 192.168.100.20; } host okd-worker2 { hardware ethernet 52:54:01:d2:46:25; fixed-address 192.168.100.20; } host okd-worker3 { hardware ethernet 52:54:01:d2:46:26; fixed-address 192.168.100.20; } host okd-worker4 { hardware ethernet 52:54:01:d2:46:27; fixed-address 192.168.100.20; } host okd-bootstrap { hardware ethernet 52:54:01:d2:46:28; fixed-address 192.168.100.29; } ==== BIND ==== === Zone === $TTL 604800 @ IN SOA ns2.okd.example.com. admin.example.com. ( 2025020501 ; Serial 604800 ; Refresh 86400 ; Retry 2419200 ; Expire 604800 ) ; Negative Cache TTL ; Nameservers NS ns2.example.com. NS ns3.example.com. haproxy1 IN A 192.168.100.20 ; Temp Bootstrap Node bootstrap IN A 192.168.100.29 ; Control Plane Nodes cp1 IN A 192.168.100.21 cp2 IN A 192.168.100.22 cp3 IN A 192.168.100.23 ; Worker Nodes worker1 IN A 192.168.100.24 worker2 IN A 192.168.100.25 worker3 IN A 192.168.100.26 worker4 IN A 192.168.100.27 ; OpenShift Internal - Load balancer api IN A 192.168.100.20 ; External API Load Balancer api-int IN A 1192.168.100.20 ; Internal API (Bootstrap Node or Load Balancer) *.apps IN A 192.168.100.20 ; Ingress Load Balancer console-openshift-console IN A 192.168.100.20 oauth-openshift IN A 192.168.100.20 ; ETCD Cluster etcd1 IN CNAME cp1 etcd2 IN CNAME cp2 etcd3 IN CNAME cp3 etcd-1 IN CNAME cp1 etcd-2 IN CNAME cp2 etcd-3 IN CNAME cp3 _etcd-server-ssl 86400 IN SRV 0 10 2380 192.168.100.21 _etcd-server-ssl 86400 IN SRV 0 10 2380 192.168.100.22 _etcd-server-ssl 86400 IN SRV 0 10 2380 192.168.100.23 ===Reverse Zone === $ORIGIN 20.194.10.in-addr.arpa. $TTL 86400 @ IN SOA ns2.example.com. admin.example.com. ( 2025020501 ; serial 21600 ; refresh after 6 hours 3600 ; retry after 1 hour 604800 ; expire after 1 week 86400 ) ; minimum TTL of 1 day ; Nameservers NS ns2.example.com. NS ns3.example.com. ; PTR Records 20 IN PTR haproxy.okd.example.com. ; 192.168.100.20 29 IN PTR bootstrap.okd.example.com. ; 192.168.100.29 21 IN PTR cp1.okd.example.com. ; 192.168.100.21 22 IN PTR cp2.okd.example.com. ; 192.168.100.22 23 IN PTR cp3.okd.example.com. ; 192.168.100.23 24 IN PTR worker1.okd.example.com. ; 192.168.100.24 25 IN PTR worker2.okd.example.com. ; 192.168.100.25 26 IN PTR worker3.okd.example.com. ; 192.168.100.26 27 IN PTR worker4.okd.example.com. ; 192.168.100.27 20 IN PTR api.lab.okd.example.com. ; 192.168.100.20 20 IN PTR api-int.okd.example.com. ; 192.168.100.20 ====haproxy==== === Installation === OS: Rocky Linux 9 dnf install epel-release dnf install haproxy git coreos-installer podman podman-compose -y setsebool -P haproxy_connect_any=1 systemctl enable haproxy systemctl start haproxy === /etc/haproxy/haproxy.cfg === # Global settings #--------------------------------------------------------------------- global maxconn 20000 log /dev/log local0 info chroot /var/lib/haproxy pidfile /var/run/haproxy.pid user haproxy group haproxy daemon # turn on stats unix socket stats socket /var/lib/haproxy/stats #--------------------------------------------------------------------- # common defaults that all the 'listen' and 'backend' sections will # use if not designated in their block #--------------------------------------------------------------------- defaults mode http log global option httplog option dontlognull option http-server-close option forwardfor except 127.0.0.0/8 option redispatch retries 3 timeout http-request 10s timeout queue 1m timeout connect 10s timeout client 300s timeout server 300s timeout http-keep-alive 10s timeout check 10s maxconn 20000 listen stats bind :9000 mode http stats enable stats uri / frontend okd_k8s_api_fe bind :6443 default_backend okd_k8s_api_be mode tcp option tcplog backend okd_k8s_api_be balance source mode tcp server okd-bootstrap 192.168.100.29:6443 check server okd-controlplane1 192.168.100.21:6443 check server okd-controlplane2 192.168.100.22:6443 check server okd-controlplane3 192.168.100.23:6443 check frontend okd_machine_config_server_fe bind :22623 default_backend okd_machine_config_server_be mode tcp option tcplog backend okd_machine_config_server_be balance source mode tcp server okd-bootstrap 192.168.100.29:22623 check server okd-controlplane1 192.168.100.21:22623 check server okd-controlplane2 192.168.100.22:22623 check server okd-controlplane3 192.168.100.23:22623 check frontend okd_http_ingress_traffic_fe bind :80 default_backend okd_http_ingress_traffic_be mode tcp option tcplog backend okd_http_ingress_traffic_be balance source mode tcp server okd-worker1 192.168.100.24:80 check server okd-worker2 192.168.100.25:80 check server okd-worker3 192.168.100.26:80 check server okd-worker4 192.168.100.27:80 check frontend okd_https_ingress_traffic_fe bind *:443 default_backend okd_https_ingress_traffic_be mode tcp option tcplog backend okd_https_ingress_traffic_be balance source mode tcp server okd-worker1 192.168.100.24:443 check server okd-worker2 192.168.100.25:443 check server okd-worker3 192.168.100.26:443 check server okd-worker4 192.168.100.27:443 check ====Setup Apache ==== Disable SELinux: /etc/selinux/config setenforce 0 mkdir -p /root/apache/files mkdir -p /root/apache/config Content of /root/apache/config/httpd.conf: ServerRoot "/usr/local/apache2" Listen 80 LoadModule mpm_event_module modules/mod_mpm_event.so LoadModule authn_file_module modules/mod_authn_file.so LoadModule authn_core_module modules/mod_authn_core.so LoadModule authz_host_module modules/mod_authz_host.so LoadModule authz_groupfile_module modules/mod_authz_groupfile.so LoadModule authz_user_module modules/mod_authz_user.so LoadModule authz_core_module modules/mod_authz_core.so LoadModule access_compat_module modules/mod_access_compat.so LoadModule auth_basic_module modules/mod_auth_basic.so LoadModule reqtimeout_module modules/mod_reqtimeout.so LoadModule filter_module modules/mod_filter.so LoadModule mime_module modules/mod_mime.so LoadModule log_config_module modules/mod_log_config.so LoadModule env_module modules/mod_env.so LoadModule headers_module modules/mod_headers.so LoadModule setenvif_module modules/mod_setenvif.so LoadModule version_module modules/mod_version.so LoadModule unixd_module modules/mod_unixd.so LoadModule status_module modules/mod_status.so LoadModule autoindex_module modules/mod_autoindex.so LoadModule dir_module modules/mod_dir.so LoadModule alias_module modules/mod_alias.so User www-data Group www-data ServerAdmin you@example.com AllowOverride none Require all denied DocumentRoot "/usr/local/apache2/htdocs" Options +Indexes Options Indexes FollowSymLinks AllowOverride None Require all granted DirectoryIndex index.html Require all denied ErrorLog /proc/self/fd/2 LogLevel warn LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined LogFormat "%h %l %u %t \"%r\" %>s %b" common LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio CustomLog /proc/self/fd/1 common ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" AllowOverride None Options None Require all granted RequestHeader unset Proxy early TypesConfig conf/mime.types AddType application/x-compress .Z AddType application/x-gzip .gz .tgz Include conf/extra/proxy-html.conf SSLRandomSeed startup builtin SSLRandomSeed connect builtin ==== OKD Installation ==== Do the following on the haproxy node which also act as helper(jumpbox) node. Go there (https://github.com/okd-project/okd/releases) and grab the following files: https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-4.17.0-okd-scos.0.tar.gz https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-install-linux-4.17.0-okd-scos.0.tar.gz Download these to file on your helper node: mkdir /root/okd cd /root/okd curl -LO https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-4.17.0-okd-scos.0.tar.gz curl -LO https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-install-linux-4.17.0-okd-scos.0.tar.gz tar -xvf openshift-install-linux-4.17.0-okd-scos.0.tar.gz tar -xvf openshift-client-linux-4.17.0-okd-scos.0.tar.gz mv kubectl oc openshift-install /usr/local/bin/ [root@haproxy okd]# oc version Client Version: 4.17.0-okd-scos.0 Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3 [root@haproxy okd]# openshift-install version openshift-install 4.17.0-okd-scos.0 built from commit e09aaf186494231de92bd80f02a43db764bd5e27 release image quay.io/okd/scos-release@sha256:a4c786872daa033c942a56ee437b52b1546abfe5f546754845b805a7cff6c0c4 WARNING Release Image Architecture not detected. Release Image Architecture is unknown release architecture unknown default architecture amd64 ssh-keygen mkdir /root/okd/install_dir Create this file: /root/okd/install_dir/file install-config.yaml: apiVersion: v1 baseDomain: cloche.ca metadata: name: okd compute:dnf - hyperthreading: Enabled name: worker replicas: 0 controlPlane: hyperthreading: Enabled name: master replicas: 3 networking: clusterNetwork: - cidr: 10.128.0.0/14 hostPrefix: 23 networkType: OVNKubernetes serviceNetwork: - 172.30.0.0/16 platform: none: {} fips: false pullSecret: '{"auths":{"fake":{"auth": ""}}}' sshKey: 'PUT YOUR SSH PUBLIC KEY HERE' We continue... cp /root/okd/install_dir/install-config.yaml /root/okd/install_dir/install-config.yaml.bak [root@haproxy ~]# cd /root/okd [root@haproxy okd]# openshift-install create manifests --dir=install_dir/ WARNING Release Image Architecture not detected. Release Image Architecture is unknown INFO Consuming Install Config from target directory WARNING Making control-plane schedulable by setting MastersSchedulable to true for Scheduler cluster settings INFO Manifests created in: install_dir/manifests and install_dir/openshift [root@haproxy okd]# [root@haproxy okd]# sed -i 's/mastersSchedulable: true/mastersSchedulable: False/' install_dir/manifests/cluster-scheduler-02-config.yml [root@haproxy okd]# cat install_dir/manifests/cluster-scheduler-02-config.yml apiVersion: config.openshift.io/v1 kind: Scheduler metadata: creationTimestamp: null name: cluster spec: mastersSchedulable: False policy: name: "" profileCustomizations: dynamicResourceAllocation: "" status: {} [root@haproxy okd]# [root@haproxy okd]# openshift-install create ignition-configs --dir=install_dir/ INFO Consuming Worker Machines from target directory INFO Consuming OpenShift Install (Manifests) from target directory INFO Consuming Master Machines from target directory INFO Consuming Common Manifests from target directory INFO Consuming Openshift Manifests from target directory INFO Ignition-Configs created in: install_dir and install_dir/auth [root@haproxy okd]# [root@haproxy okd]# openshift-install coreos print-stream-json | grep iso | grep x86_64 "location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso", "signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso.sig", [root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 765M 100 765M 0 0 10.8M 0 0:01:10 0:01:10 --:--:-- 10.9M [root@haproxy okd]# openshift-install coreos print-stream-json | grep metal | grep x86_64 "location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal4k.x86_64.raw.xz", "signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal4k.x86_64.raw.xz.sig", "location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz", "signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig", [root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 618M 100 618M 0 0 10.9M 0 0:00:56 0:00:56 --:--:-- 11.4M [root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 100 566 100 566 0 0 2963 0 --:--:-- --:--:-- --:--:-- 2963 [root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso bootstrap.iso [root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso master.iso [root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso worker.iso coreos-installer iso kargs modify \ -a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz" \ -a "coreos.inst.install_dev=/dev/sda" \ -a "coreos.inst.ignition_url=http://192.168.100.20:8000/bootstrap.ign" \ bootstrap.iso coreos-installer iso kargs modify \ -a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz" \ -a "coreos.inst.install_dev=/dev/sda" \ -a "coreos.inst.ignition_url=http://192.168.100.20:8000/master.ign" \ master.iso coreos-installer iso kargs modify \ -a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz" \ -a "coreos.inst.install_dev=/dev/sda" \ -a "coreos.inst.ignition_url=http://192.168.100.20:8000/worker.ign" \ worker.iso Move ignition files and OKD metal raw image to Apache web directory: [root@haproxy okd]# cp /root/okd/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz /root/okd/apache/files/ [root@haproxy okd]# cp /root/okd/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig /root/okd/apache/files/ [root@haproxy okd]# cp /root/okd/install_dir/master.ign /root/okd/apache/files/ [root@haproxy okd]# cp /root/okd/install_dir/worker.ign /root/okd/apache/files/ [root@haproxy okd]# cp /root/okd/install_dir/bootstrap.ign /root/okd/apache/files/ [root@haproxy okd]# ls -ls /root/okd/apache/files/ total 633372 268 -rw-r-----. 1 root root 273173 Feb 6 13:30 bootstrap.ign 633092 -rw-r--r--. 1 root root 648286096 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz 4 -rw-r--r--. 1 root root 566 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig 4 -rw-r-----. 1 root root 1715 Feb 6 13:30 master.ign 4 -rw-r-----. 1 root root 1715 Feb 6 13:30 worker.ign [root@haproxy okd]# chmod 744 -R /root/okd/ apache/ fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz master.iso README.md bootstrap.iso fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig openshift-client-linux-4.17.0-okd-scos.0.tar.gz worker.iso fedora-coreos-39.20231101.3.0-live.x86_64.iso install_dir/ openshift-install-linux-4.17.0-okd-scos.0.tar.gz [root@haproxy okd]# chmod 744 -R /root/okd/apache/files/ [root@haproxy okd]# ls -ls /root/okd/apache/files/ total 633372 268 -rwxr--r--. 1 root root 273173 Feb 6 13:30 bootstrap.ign 633092 -rwxr--r--. 1 root root 648286096 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz 4 -rwxr--r--. 1 root root 566 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig 4 -rwxr--r--. 1 root root 1715 Feb 6 13:30 master.ign 4 -rwxr--r--. 1 root root 1715 Feb 6 13:30 worker.ign In order to make all the workers available in haproxy: [root@haproxy okd]# oc get pods -n openshift-ingress -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES router-default-7c58f66c46-6gvk4 1/1 Running 0 112s 10.194.20.24 worker1.okd.cloche.ca router-default-7c58f66c46-l5hj5 1/1 Running 0 112s 10.194.20.25 worker2.okd.cloche.ca [root@haproxy okd]# oc scale --replicas=4 ingresscontroller/default -n openshift-ingress-operator ingresscontroller.operator.openshift.io/default scaled [root@haproxy okd]# oc get pods -n openshift-ingress -o wide --watch NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES router-default-7c58f66c46-6gvk4 1/1 Running 0 29m 10.194.20.24 worker1.okd.cloche.ca router-default-7c58f66c46-l5hj5 1/1 Running 0 29m 10.194.20.25 worker2.okd.cloche.ca router-default-7c58f66c46-n9q9q 1/1 Running 0 25m 10.194.20.27 worker4.okd.cloche.ca router-default-7c58f66c46-wbtbh 1/1 Running 0 25m 10.194.20.26 worker3.okd.cloche.ca