docker:okd
Table of Contents
OKD
DHCP
dhcpd.conf
host okd-haproxy {
hardware ethernet 52:54:01:d2:46:20;
fixed-address 192.168.100.20;
}
host okd-cp1 {
hardware ethernet 52:54:01:d2:46:21;
fixed-address 192.168.100.20;
}
host okd-cp2 {
hardware ethernet 52:54:01:d2:46:22;
fixed-address 192.168.100.20;
}
host okd-cp3 {
hardware ethernet 52:54:01:d2:46:23;
fixed-address 192.168.100.20;
}
host okd-worker1 {
hardware ethernet 52:54:01:d2:46:24;
fixed-address 192.168.100.20;
}
host okd-worker2 {
hardware ethernet 52:54:01:d2:46:25;
fixed-address 192.168.100.20;
}
host okd-worker3 {
hardware ethernet 52:54:01:d2:46:26;
fixed-address 192.168.100.20;
}
host okd-worker4 {
hardware ethernet 52:54:01:d2:46:27;
fixed-address 192.168.100.20;
}
host okd-bootstrap {
hardware ethernet 52:54:01:d2:46:28;
fixed-address 192.168.100.29;
}
BIND
Zone
$TTL 604800
@ IN SOA ns2.okd.example.com. admin.example.com. (
2025020501 ; Serial
604800 ; Refresh
86400 ; Retry
2419200 ; Expire
604800 ) ; Negative Cache TTL
; Nameservers
NS ns2.example.com.
NS ns3.example.com.
haproxy1 IN A 192.168.100.20
; Temp Bootstrap Node
bootstrap IN A 192.168.100.29
; Control Plane Nodes
cp1 IN A 192.168.100.21
cp2 IN A 192.168.100.22
cp3 IN A 192.168.100.23
; Worker Nodes
worker1 IN A 192.168.100.24
worker2 IN A 192.168.100.25
worker3 IN A 192.168.100.26
worker4 IN A 192.168.100.27
; OpenShift Internal - Load balancer
api IN A 192.168.100.20 ; External API Load Balancer
api-int IN A 1192.168.100.20 ; Internal API (Bootstrap Node or Load Balancer)
*.apps IN A 192.168.100.20 ; Ingress Load Balancer
console-openshift-console IN A 192.168.100.20
oauth-openshift IN A 192.168.100.20
; ETCD Cluster
etcd1 IN CNAME cp1
etcd2 IN CNAME cp2
etcd3 IN CNAME cp3
etcd-1 IN CNAME cp1
etcd-2 IN CNAME cp2
etcd-3 IN CNAME cp3
_etcd-server-ssl 86400 IN SRV 0 10 2380 192.168.100.21
_etcd-server-ssl 86400 IN SRV 0 10 2380 192.168.100.22
_etcd-server-ssl 86400 IN SRV 0 10 2380 192.168.100.23
Reverse Zone
$ORIGIN 20.194.10.in-addr.arpa.
$TTL 86400
@ IN SOA ns2.example.com. admin.example.com. (
2025020501 ; serial
21600 ; refresh after 6 hours
3600 ; retry after 1 hour
604800 ; expire after 1 week
86400 ) ; minimum TTL of 1 day
; Nameservers
NS ns2.example.com.
NS ns3.example.com.
; PTR Records
20 IN PTR haproxy.okd.example.com. ; 192.168.100.20
29 IN PTR bootstrap.okd.example.com. ; 192.168.100.29
21 IN PTR cp1.okd.example.com. ; 192.168.100.21
22 IN PTR cp2.okd.example.com. ; 192.168.100.22
23 IN PTR cp3.okd.example.com. ; 192.168.100.23
24 IN PTR worker1.okd.example.com. ; 192.168.100.24
25 IN PTR worker2.okd.example.com. ; 192.168.100.25
26 IN PTR worker3.okd.example.com. ; 192.168.100.26
27 IN PTR worker4.okd.example.com. ; 192.168.100.27
20 IN PTR api.lab.okd.example.com. ; 192.168.100.20
20 IN PTR api-int.okd.example.com. ; 192.168.100.20
haproxy
Installation
OS: Rocky Linux 9
dnf install epel-release dnf install haproxy git coreos-installer podman podman-compose -y setsebool -P haproxy_connect_any=1 systemctl enable haproxy systemctl start haproxy
/etc/haproxy/haproxy.cfg
# Global settings
#---------------------------------------------------------------------
global
maxconn 20000
log /dev/log local0 info
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 300s
timeout server 300s
timeout http-keep-alive 10s
timeout check 10s
maxconn 20000
listen stats
bind :9000
mode http
stats enable
stats uri /
frontend okd_k8s_api_fe
bind :6443
default_backend okd_k8s_api_be
mode tcp
option tcplog
backend okd_k8s_api_be
balance source
mode tcp
server okd-bootstrap 192.168.100.29:6443 check
server okd-controlplane1 192.168.100.21:6443 check
server okd-controlplane2 192.168.100.22:6443 check
server okd-controlplane3 192.168.100.23:6443 check
frontend okd_machine_config_server_fe
bind :22623
default_backend okd_machine_config_server_be
mode tcp
option tcplog
backend okd_machine_config_server_be
balance source
mode tcp
server okd-bootstrap 192.168.100.29:22623 check
server okd-controlplane1 192.168.100.21:22623 check
server okd-controlplane2 192.168.100.22:22623 check
server okd-controlplane3 192.168.100.23:22623 check
frontend okd_http_ingress_traffic_fe
bind :80
default_backend okd_http_ingress_traffic_be
mode tcp
option tcplog
backend okd_http_ingress_traffic_be
balance source
mode tcp
server okd-worker1 192.168.100.24:80 check
server okd-worker2 192.168.100.25:80 check
server okd-worker3 192.168.100.26:80 check
server okd-worker4 192.168.100.27:80 check
frontend okd_https_ingress_traffic_fe
bind *:443
default_backend okd_https_ingress_traffic_be
mode tcp
option tcplog
backend okd_https_ingress_traffic_be
balance source
mode tcp
server okd-worker1 192.168.100.24:443 check
server okd-worker2 192.168.100.25:443 check
server okd-worker3 192.168.100.26:443 check
server okd-worker4 192.168.100.27:443 check
Setup Apache
Disable SELinux: /etc/selinux/config setenforce 0
mkdir -p /root/apache/files mkdir -p /root/apache/config
Content of /root/apache/config/httpd.conf:
ServerRoot "/usr/local/apache2"
Listen 80
LoadModule mpm_event_module modules/mod_mpm_event.so
LoadModule authn_file_module modules/mod_authn_file.so
LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_user_module modules/mod_authz_user.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so
LoadModule reqtimeout_module modules/mod_reqtimeout.so
LoadModule filter_module modules/mod_filter.so
LoadModule mime_module modules/mod_mime.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule env_module modules/mod_env.so
LoadModule headers_module modules/mod_headers.so
LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so
LoadModule unixd_module modules/mod_unixd.so
LoadModule status_module modules/mod_status.so
LoadModule autoindex_module modules/mod_autoindex.so
<IfModule !mpm_prefork_module>
</IfModule>
<IfModule mpm_prefork_module>
</IfModule>
LoadModule dir_module modules/mod_dir.so
LoadModule alias_module modules/mod_alias.so
<IfModule unixd_module>
User www-data
Group www-data
</IfModule>
ServerAdmin you@example.com
<Directory />
AllowOverride none
Require all denied
</Directory>
DocumentRoot "/usr/local/apache2/htdocs"
<Directory "/usr/local/apache2/htdocs">
Options +Indexes
Options Indexes FollowSymLinks
AllowOverride None
Require all granted
</Directory>
<IfModule dir_module>
DirectoryIndex index.html
</IfModule>
<Files ".ht*">
Require all denied
</Files>
ErrorLog /proc/self/fd/2
LogLevel warn
<IfModule log_config_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%h %l %u %t \"%r\" %>s %b" common
<IfModule logio_module>
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
</IfModule>
CustomLog /proc/self/fd/1 common
</IfModule>
<IfModule alias_module>
ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/"
</IfModule>
<IfModule cgid_module>
</IfModule>
<Directory "/usr/local/apache2/cgi-bin">
AllowOverride None
Options None
Require all granted
</Directory>
<IfModule headers_module>
RequestHeader unset Proxy early
</IfModule>
<IfModule mime_module>
TypesConfig conf/mime.types
AddType application/x-compress .Z
AddType application/x-gzip .gz .tgz
</IfModule>
<IfModule proxy_html_module>
Include conf/extra/proxy-html.conf
</IfModule>
<IfModule ssl_module>
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
</IfModule>
OKD Installation
Do the following on the haproxy node which also act as helper(jumpbox) node. Go there (https://github.com/okd-project/okd/releases) and grab the following files: https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-4.17.0-okd-scos.0.tar.gz https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-install-linux-4.17.0-okd-scos.0.tar.gz Download these to file on your helper node:
mkdir /root/okd cd /root/okd curl -LO https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-4.17.0-okd-scos.0.tar.gz curl -LO https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-install-linux-4.17.0-okd-scos.0.tar.gz tar -xvf openshift-install-linux-4.17.0-okd-scos.0.tar.gz tar -xvf openshift-client-linux-4.17.0-okd-scos.0.tar.gz mv kubectl oc openshift-install /usr/local/bin/ [root@haproxy okd]# oc version Client Version: 4.17.0-okd-scos.0 Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3 [root@haproxy okd]# openshift-install version openshift-install 4.17.0-okd-scos.0 built from commit e09aaf186494231de92bd80f02a43db764bd5e27 release image quay.io/okd/scos-release@sha256:a4c786872daa033c942a56ee437b52b1546abfe5f546754845b805a7cff6c0c4 WARNING Release Image Architecture not detected. Release Image Architecture is unknown release architecture unknown default architecture amd64 ssh-keygen mkdir /root/okd/install_dir
Create this file: /root/okd/install_dir/file install-config.yaml:
apiVersion: v1
baseDomain: cloche.ca
metadata:
name: okd
compute:dnf
- hyperthreading: Enabled
name: worker
replicas: 0
controlPlane:
hyperthreading: Enabled
name: master
replicas: 3
networking:
clusterNetwork:
- cidr: 10.128.0.0/14
hostPrefix: 23
networkType: OVNKubernetes
serviceNetwork:
- 172.30.0.0/16
platform:
none: {}
fips: false
pullSecret: '{"auths":{"fake":{"auth": ""}}}'
sshKey: 'PUT YOUR SSH PUBLIC KEY HERE'
We continue…
cp /root/okd/install_dir/install-config.yaml /root/okd/install_dir/install-config.yaml.bak
[root@haproxy ~]# cd /root/okd
[root@haproxy okd]# openshift-install create manifests --dir=install_dir/
WARNING Release Image Architecture not detected. Release Image Architecture is unknown
INFO Consuming Install Config from target directory
WARNING Making control-plane schedulable by setting MastersSchedulable to true for Scheduler cluster settings
INFO Manifests created in: install_dir/manifests and install_dir/openshift
[root@haproxy okd]#
[root@haproxy okd]# sed -i 's/mastersSchedulable: true/mastersSchedulable: False/' install_dir/manifests/cluster-scheduler-02-config.yml
[root@haproxy okd]# cat install_dir/manifests/cluster-scheduler-02-config.yml
apiVersion: config.openshift.io/v1
kind: Scheduler
metadata:
creationTimestamp: null
name: cluster
spec:
mastersSchedulable: False
policy:
name: ""
profileCustomizations:
dynamicResourceAllocation: ""
status: {}
[root@haproxy okd]#
[root@haproxy okd]# openshift-install create ignition-configs --dir=install_dir/
INFO Consuming Worker Machines from target directory
INFO Consuming OpenShift Install (Manifests) from target directory
INFO Consuming Master Machines from target directory
INFO Consuming Common Manifests from target directory
INFO Consuming Openshift Manifests from target directory
INFO Ignition-Configs created in: install_dir and install_dir/auth
[root@haproxy okd]#
[root@haproxy okd]# openshift-install coreos print-stream-json | grep iso | grep x86_64
"location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso",
"signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso.sig",
[root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 765M 100 765M 0 0 10.8M 0 0:01:10 0:01:10 --:--:-- 10.9M
[root@haproxy okd]# openshift-install coreos print-stream-json | grep metal | grep x86_64
"location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal4k.x86_64.raw.xz",
"signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal4k.x86_64.raw.xz.sig",
"location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz",
"signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig",
[root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 618M 100 618M 0 0 10.9M 0 0:00:56 0:00:56 --:--:-- 11.4M
[root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 566 100 566 0 0 2963 0 --:--:-- --:--:-- --:--:-- 2963
[root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso bootstrap.iso
[root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso master.iso
[root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso worker.iso
coreos-installer iso kargs modify \
-a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz" \
-a "coreos.inst.install_dev=/dev/sda" \
-a "coreos.inst.ignition_url=http://192.168.100.20:8000/bootstrap.ign" \
bootstrap.iso
coreos-installer iso kargs modify \
-a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz" \
-a "coreos.inst.install_dev=/dev/sda" \
-a "coreos.inst.ignition_url=http://192.168.100.20:8000/master.ign" \
master.iso
coreos-installer iso kargs modify \
-a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz" \
-a "coreos.inst.install_dev=/dev/sda" \
-a "coreos.inst.ignition_url=http://192.168.100.20:8000/worker.ign" \
worker.iso
Move ignition files and OKD metal raw image to Apache web directory:
[root@haproxy okd]# cp /root/okd/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/install_dir/master.ign /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/install_dir/worker.ign /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/install_dir/bootstrap.ign /root/okd/apache/files/
[root@haproxy okd]# ls -ls /root/okd/apache/files/
total 633372
268 -rw-r-----. 1 root root 273173 Feb 6 13:30 bootstrap.ign
633092 -rw-r--r--. 1 root root 648286096 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz
4 -rw-r--r--. 1 root root 566 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig
4 -rw-r-----. 1 root root 1715 Feb 6 13:30 master.ign
4 -rw-r-----. 1 root root 1715 Feb 6 13:30 worker.ign
[root@haproxy okd]# chmod 744 -R /root/okd/
apache/ fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz master.iso README.md
bootstrap.iso fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig openshift-client-linux-4.17.0-okd-scos.0.tar.gz worker.iso
fedora-coreos-39.20231101.3.0-live.x86_64.iso install_dir/ openshift-install-linux-4.17.0-okd-scos.0.tar.gz
[root@haproxy okd]# chmod 744 -R /root/okd/apache/files/
[root@haproxy okd]# ls -ls /root/okd/apache/files/
total 633372
268 -rwxr--r--. 1 root root 273173 Feb 6 13:30 bootstrap.ign
633092 -rwxr--r--. 1 root root 648286096 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz
4 -rwxr--r--. 1 root root 566 Feb 6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig
4 -rwxr--r--. 1 root root 1715 Feb 6 13:30 master.ign
4 -rwxr--r--. 1 root root 1715 Feb 6 13:30 worker.ign
In order to make all the workers available in haproxy:
[root@haproxy okd]# oc get pods -n openshift-ingress -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES router-default-7c58f66c46-6gvk4 1/1 Running 0 112s 10.194.20.24 worker1.okd.cloche.ca <none> <none> router-default-7c58f66c46-l5hj5 1/1 Running 0 112s 10.194.20.25 worker2.okd.cloche.ca <none> <none> [root@haproxy okd]# oc scale --replicas=4 ingresscontroller/default -n openshift-ingress-operator ingresscontroller.operator.openshift.io/default scaled [root@haproxy okd]# oc get pods -n openshift-ingress -o wide --watch NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES router-default-7c58f66c46-6gvk4 1/1 Running 0 29m 10.194.20.24 worker1.okd.cloche.ca <none> <none> router-default-7c58f66c46-l5hj5 1/1 Running 0 29m 10.194.20.25 worker2.okd.cloche.ca <none> <none> router-default-7c58f66c46-n9q9q 1/1 Running 0 25m 10.194.20.27 worker4.okd.cloche.ca <none> <none> router-default-7c58f66c46-wbtbh 1/1 Running 0 25m 10.194.20.26 worker3.okd.cloche.ca <none> <none>
docker/okd.txt · Last modified: by jonathan