JoLeKing

This page is read only. You can view the source, but not change it. Ask your administrator if you think this is wrong.
==== OKD ====
====DHCP====
=== dhcpd.conf ===
<code>
  host okd-haproxy {
    hardware ethernet 52:54:01:d2:46:20;
    fixed-address 192.168.100.20;
  }
  host okd-cp1 {
    hardware ethernet 52:54:01:d2:46:21;
    fixed-address 192.168.100.20;
  }
  host okd-cp2 {
    hardware ethernet 52:54:01:d2:46:22;
    fixed-address 192.168.100.20;
  }
  host okd-cp3 {
    hardware ethernet 52:54:01:d2:46:23;
    fixed-address 192.168.100.20;
  }
  host okd-worker1 {
    hardware ethernet 52:54:01:d2:46:24;
    fixed-address 192.168.100.20;
  }
  host okd-worker2 {
    hardware ethernet 52:54:01:d2:46:25;
    fixed-address 192.168.100.20;
  }
  host okd-worker3 {
    hardware ethernet 52:54:01:d2:46:26;
    fixed-address 192.168.100.20;
  }
  host okd-worker4 {
    hardware ethernet 52:54:01:d2:46:27;
    fixed-address 192.168.100.20;
  }
  host okd-bootstrap {
    hardware ethernet 52:54:01:d2:46:28;
    fixed-address 192.168.100.29;
  }
</code>
==== BIND ====
=== Zone ===
<code>
$TTL 604800
@   IN  SOA ns2.okd.example.com. admin.example.com. (
            2025020501 ; Serial
            604800     ; Refresh
            86400      ; Retry
            2419200    ; Expire
            604800 )   ; Negative Cache TTL
; Nameservers
                NS	ns2.example.com.
                NS	ns3.example.com.
                                    
                       
haproxy1                IN      A       192.168.100.20
; Temp Bootstrap Node
bootstrap        IN       A      192.168.100.29
; Control Plane Nodes
cp1           IN	A       192.168.100.21
cp2           IN	A       192.168.100.22
cp3           IN	A       192.168.100.23
; Worker Nodes
worker1        IN      A           192.168.100.24
worker2        IN      A           192.168.100.25
worker3        IN      A           192.168.100.26
worker4        IN      A           192.168.100.27
; OpenShift Internal - Load balancer
api        IN    A              192.168.100.20    ; External API Load Balancer
api-int    IN    A              1192.168.100.20    ; Internal API (Bootstrap Node or Load Balancer)
*.apps     IN    A              192.168.100.20    ; Ingress Load Balancer
console-openshift-console	IN	A	192.168.100.20
oauth-openshift			IN	A	192.168.100.20
; ETCD Cluster
etcd1   IN    CNAME     cp1
etcd2   IN    CNAME     cp2
etcd3   IN    CNAME     cp3
etcd-1	IN    CNAME     cp1
etcd-2	IN    CNAME     cp2
etcd-3	IN    CNAME     cp3
_etcd-server-ssl	86400		IN	SRV	0	10	2380	192.168.100.21
_etcd-server-ssl	86400           IN      SRV     0       10      2380    192.168.100.22
_etcd-server-ssl	86400           IN      SRV     0       10      2380    192.168.100.23
</code>
===Reverse Zone ===
<code>
$ORIGIN 20.194.10.in-addr.arpa.
$TTL 86400
@     IN     SOA    ns2.example.com.     admin.example.com. (
                    2025020501 ; serial
                    21600      ; refresh after 6 hours
                    3600       ; retry after 1 hour
                    604800     ; expire after 1 week
                    86400 )    ; minimum TTL of 1 day
; Nameservers
                NS	ns2.example.com.
                NS	ns3.example.com.
; PTR Records
20  IN  PTR  haproxy.okd.example.com.  ; 192.168.100.20
29  IN  PTR  bootstrap.okd.example.com. ; 192.168.100.29
21  IN  PTR  cp1.okd.example.com. ; 192.168.100.21
22  IN  PTR  cp2.okd.example.com. ; 192.168.100.22
23  IN  PTR  cp3.okd.example.com. ; 192.168.100.23
24  IN  PTR  worker1.okd.example.com. ; 192.168.100.24
25  IN  PTR  worker2.okd.example.com. ; 192.168.100.25
26  IN  PTR  worker3.okd.example.com. ; 192.168.100.26
27  IN  PTR  worker4.okd.example.com. ; 192.168.100.27
20  IN	PTR  api.lab.okd.example.com. ; 192.168.100.20
20  IN	PTR  api-int.okd.example.com. ; 192.168.100.20
</code>

====haproxy====
=== Installation ===
OS: Rocky Linux 9
<code>
dnf install epel-release
dnf install haproxy git coreos-installer podman podman-compose -y
setsebool -P haproxy_connect_any=1
systemctl enable haproxy
systemctl start haproxy
</code>
=== /etc/haproxy/haproxy.cfg ===
<code>
# Global settings
#---------------------------------------------------------------------
global
    maxconn     20000
    log         /dev/log local0 info
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    user        haproxy
    group       haproxy
    daemon
    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          300s
    timeout server          300s
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 20000
listen stats
    bind :9000
    mode http
    stats enable
    stats uri /
frontend okd_k8s_api_fe
    bind :6443
    default_backend okd_k8s_api_be
    mode tcp
    option tcplog
backend okd_k8s_api_be
    balance source
    mode tcp
    server      okd-bootstrap 192.168.100.29:6443 check
    server      okd-controlplane1 192.168.100.21:6443 check
    server      okd-controlplane2 192.168.100.22:6443 check
    server      okd-controlplane3 192.168.100.23:6443 check
frontend okd_machine_config_server_fe
    bind :22623
    default_backend okd_machine_config_server_be
    mode tcp
    option tcplog
backend okd_machine_config_server_be
    balance source
    mode tcp
    server      okd-bootstrap 192.168.100.29:22623 check
    server      okd-controlplane1 192.168.100.21:22623 check
    server      okd-controlplane2 192.168.100.22:22623 check
    server      okd-controlplane3 192.168.100.23:22623 check
frontend okd_http_ingress_traffic_fe
    bind :80
    default_backend okd_http_ingress_traffic_be
    mode tcp
    option tcplog
backend okd_http_ingress_traffic_be
    balance source
    mode tcp
    server      okd-worker1 192.168.100.24:80 check
    server      okd-worker2 192.168.100.25:80 check
    server      okd-worker3 192.168.100.26:80 check
    server      okd-worker4 192.168.100.27:80 check
frontend okd_https_ingress_traffic_fe
    bind *:443
    default_backend okd_https_ingress_traffic_be
    mode tcp
    option tcplog
backend okd_https_ingress_traffic_be
    balance source
    mode tcp
    server      okd-worker1 192.168.100.24:443 check
    server      okd-worker2 192.168.100.25:443 check
    server      okd-worker3 192.168.100.26:443 check
    server      okd-worker4 192.168.100.27:443 check
</code>


====Setup Apache ====

Disable SELinux:
/etc/selinux/config
setenforce 0

mkdir -p /root/apache/files
mkdir -p /root/apache/config


Content of /root/apache/config/httpd.conf:
<code>
ServerRoot "/usr/local/apache2"

Listen 80

LoadModule mpm_event_module modules/mod_mpm_event.so

LoadModule authn_file_module modules/mod_authn_file.so

LoadModule authn_core_module modules/mod_authn_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule authz_groupfile_module modules/mod_authz_groupfile.so
LoadModule authz_user_module modules/mod_authz_user.so

LoadModule authz_core_module modules/mod_authz_core.so

LoadModule access_compat_module modules/mod_access_compat.so
LoadModule auth_basic_module modules/mod_auth_basic.so

LoadModule reqtimeout_module modules/mod_reqtimeout.so

LoadModule filter_module modules/mod_filter.so

LoadModule mime_module modules/mod_mime.so

LoadModule log_config_module modules/mod_log_config.so

LoadModule env_module modules/mod_env.so

LoadModule headers_module modules/mod_headers.so

LoadModule setenvif_module modules/mod_setenvif.so
LoadModule version_module modules/mod_version.so

LoadModule unixd_module modules/mod_unixd.so

LoadModule status_module modules/mod_status.so
LoadModule autoindex_module modules/mod_autoindex.so

<IfModule !mpm_prefork_module>

</IfModule>
<IfModule mpm_prefork_module>

</IfModule>

LoadModule dir_module modules/mod_dir.so

LoadModule alias_module modules/mod_alias.so

<IfModule unixd_module>

User www-data
Group www-data

</IfModule>

ServerAdmin you@example.com

<Directory />
    AllowOverride none
    Require all denied
</Directory>

DocumentRoot "/usr/local/apache2/htdocs"
<Directory "/usr/local/apache2/htdocs">
    Options +Indexes

    Options Indexes FollowSymLinks

    AllowOverride None

    Require all granted
</Directory>

<IfModule dir_module>
    DirectoryIndex index.html
</IfModule>

<Files ".ht*">
    Require all denied
</Files>

ErrorLog /proc/self/fd/2

LogLevel warn

<IfModule log_config_module>

    LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
    LogFormat "%h %l %u %t \"%r\" %>s %b" common

    <IfModule logio_module>

      LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
    </IfModule>

    CustomLog /proc/self/fd/1 common

</IfModule>

<IfModule alias_module>

    ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/"

</IfModule>

<IfModule cgid_module>

</IfModule>

<Directory "/usr/local/apache2/cgi-bin">
    AllowOverride None
    Options None
    Require all granted
</Directory>

<IfModule headers_module>

    RequestHeader unset Proxy early
</IfModule>

<IfModule mime_module>

    TypesConfig conf/mime.types

    AddType application/x-compress .Z
    AddType application/x-gzip .gz .tgz

</IfModule>

<IfModule proxy_html_module>
Include conf/extra/proxy-html.conf
</IfModule>

<IfModule ssl_module>
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
</IfModule>
</code>

==== OKD Installation ====
Do the following on the haproxy node which also act as helper(jumpbox) node.
Go there (https://github.com/okd-project/okd/releases) and grab the following files:
https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-4.17.0-okd-scos.0.tar.gz
https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-install-linux-4.17.0-okd-scos.0.tar.gz
Download these to file on your helper node:
<code>
mkdir /root/okd
cd /root/okd
curl -LO https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-client-linux-4.17.0-okd-scos.0.tar.gz
curl -LO https://github.com/okd-project/okd/releases/download/4.17.0-okd-scos.0/openshift-install-linux-4.17.0-okd-scos.0.tar.gz
tar -xvf openshift-install-linux-4.17.0-okd-scos.0.tar.gz
tar -xvf openshift-client-linux-4.17.0-okd-scos.0.tar.gz
mv kubectl oc openshift-install /usr/local/bin/
[root@haproxy okd]# oc version
Client Version: 4.17.0-okd-scos.0
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
[root@haproxy okd]# openshift-install version
openshift-install 4.17.0-okd-scos.0
built from commit e09aaf186494231de92bd80f02a43db764bd5e27
release image quay.io/okd/scos-release@sha256:a4c786872daa033c942a56ee437b52b1546abfe5f546754845b805a7cff6c0c4
WARNING Release Image Architecture not detected. Release Image Architecture is unknown 
release architecture unknown
default architecture amd64
ssh-keygen
mkdir /root/okd/install_dir

</code>

Create this file: /root/okd/install_dir/file install-config.yaml:
<code>
apiVersion: v1
baseDomain: cloche.ca
metadata:
  name: okd
compute:dnf 
- hyperthreading: Enabled
  name: worker
  replicas: 0
controlPlane:
  hyperthreading: Enabled
  name: master
  replicas: 3
networking:
  clusterNetwork:
  - cidr: 10.128.0.0/14 
    hostPrefix: 23 
  networkType: OVNKubernetes
  serviceNetwork: 
  - 172.30.0.0/16
platform:
  none: {}
fips: false
pullSecret: '{"auths":{"fake":{"auth": ""}}}'
sshKey: 'PUT YOUR SSH PUBLIC KEY HERE'
</code>
We continue...
<code>
cp /root/okd/install_dir/install-config.yaml /root/okd/install_dir/install-config.yaml.bak
[root@haproxy ~]# cd /root/okd
[root@haproxy okd]# openshift-install create manifests --dir=install_dir/
WARNING Release Image Architecture not detected. Release Image Architecture is unknown 
INFO Consuming Install Config from target directory 
WARNING Making control-plane schedulable by setting MastersSchedulable to true for Scheduler cluster settings 
INFO Manifests created in: install_dir/manifests and install_dir/openshift
[root@haproxy okd]#
[root@haproxy okd]# sed -i 's/mastersSchedulable: true/mastersSchedulable: False/' install_dir/manifests/cluster-scheduler-02-config.yml
[root@haproxy okd]# cat install_dir/manifests/cluster-scheduler-02-config.yml
apiVersion: config.openshift.io/v1
kind: Scheduler
metadata:
  creationTimestamp: null
  name: cluster
spec:
  mastersSchedulable: False
  policy:
    name: ""
  profileCustomizations:
    dynamicResourceAllocation: ""
status: {}
[root@haproxy okd]#
[root@haproxy okd]# openshift-install create ignition-configs --dir=install_dir/
INFO Consuming Worker Machines from target directory 
INFO Consuming OpenShift Install (Manifests) from target directory 
INFO Consuming Master Machines from target directory 
INFO Consuming Common Manifests from target directory 
INFO Consuming Openshift Manifests from target directory 
INFO Ignition-Configs created in: install_dir and install_dir/auth 
[root@haproxy okd]#
[root@haproxy okd]# openshift-install coreos print-stream-json | grep iso | grep x86_64
                                "location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso",
                                "signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso.sig",

[root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-live.x86_64.iso
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  765M  100  765M    0     0  10.8M      0  0:01:10  0:01:10 --:--:-- 10.9M

[root@haproxy okd]# openshift-install coreos print-stream-json | grep metal | grep x86_64
                                "location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal4k.x86_64.raw.xz",
                                "signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal4k.x86_64.raw.xz.sig",
                                "location": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz",
                                "signature": "https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig",

[root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100  618M  100  618M    0     0  10.9M      0  0:00:56  0:00:56 --:--:-- 11.4M
[root@haproxy okd]# curl -LO https://builds.coreos.fedoraproject.org/prod/streams/stable/builds/39.20231101.3.0/x86_64/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   566  100   566    0     0   2963      0 --:--:-- --:--:-- --:--:--  2963

[root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso bootstrap.iso
[root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso master.iso
[root@haproxy okd]# cp fedora-coreos-39.20231101.3.0-live.x86_64.iso worker.iso

coreos-installer iso kargs modify  \
-a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz"  \
-a "coreos.inst.install_dev=/dev/sda"  \
-a "coreos.inst.ignition_url=http://192.168.100.20:8000/bootstrap.ign"  \
bootstrap.iso
coreos-installer iso kargs modify  \
-a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz"  \
-a "coreos.inst.install_dev=/dev/sda"  \
-a "coreos.inst.ignition_url=http://192.168.100.20:8000/master.ign"  \
master.iso
coreos-installer iso kargs modify  \
-a "coreos.inst.image_url=http://192.168.100.20:8000/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz"  \
-a "coreos.inst.install_dev=/dev/sda"  \
-a "coreos.inst.ignition_url=http://192.168.100.20:8000/worker.ign"  \
worker.iso

</code>


Move ignition files and OKD metal raw image to Apache web directory:
<code>
[root@haproxy okd]# cp /root/okd/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/install_dir/master.ign /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/install_dir/worker.ign /root/okd/apache/files/
[root@haproxy okd]# cp /root/okd/install_dir/bootstrap.ign /root/okd/apache/files/
[root@haproxy okd]# ls -ls /root/okd/apache/files/
total 633372
   268 -rw-r-----. 1 root root    273173 Feb  6 13:30 bootstrap.ign
633092 -rw-r--r--. 1 root root 648286096 Feb  6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz
     4 -rw-r--r--. 1 root root       566 Feb  6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig
     4 -rw-r-----. 1 root root      1715 Feb  6 13:30 master.ign
     4 -rw-r-----. 1 root root      1715 Feb  6 13:30 worker.ign
[root@haproxy okd]# chmod 744 -R /root/okd/
apache/                                                fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz      master.iso                                             README.md
bootstrap.iso                                          fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig  openshift-client-linux-4.17.0-okd-scos.0.tar.gz        worker.iso
fedora-coreos-39.20231101.3.0-live.x86_64.iso          install_dir/                                           openshift-install-linux-4.17.0-okd-scos.0.tar.gz       
[root@haproxy okd]# chmod 744 -R /root/okd/apache/files/
[root@haproxy okd]# ls -ls /root/okd/apache/files/
total 633372
   268 -rwxr--r--. 1 root root    273173 Feb  6 13:30 bootstrap.ign
633092 -rwxr--r--. 1 root root 648286096 Feb  6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz
     4 -rwxr--r--. 1 root root       566 Feb  6 13:29 fedora-coreos-39.20231101.3.0-metal.x86_64.raw.xz.sig
     4 -rwxr--r--. 1 root root      1715 Feb  6 13:30 master.ign
     4 -rwxr--r--. 1 root root      1715 Feb  6 13:30 worker.ign
</code>


In order to make all the workers available in haproxy:
<code>
[root@haproxy okd]# oc get pods -n openshift-ingress -o wide 
NAME                              READY   STATUS    RESTARTS   AGE    IP             NODE                    NOMINATED NODE   READINESS GATES
router-default-7c58f66c46-6gvk4   1/1     Running   0          112s   10.194.20.24   worker1.okd.cloche.ca   <none>           <none>
router-default-7c58f66c46-l5hj5   1/1     Running   0          112s   10.194.20.25   worker2.okd.cloche.ca   <none>           <none>

[root@haproxy okd]# oc scale --replicas=4 ingresscontroller/default -n openshift-ingress-operator
ingresscontroller.operator.openshift.io/default scaled

[root@haproxy okd]# oc get pods -n openshift-ingress -o wide --watch
NAME                              READY   STATUS    RESTARTS   AGE   IP             NODE                    NOMINATED NODE   READINESS GATES
router-default-7c58f66c46-6gvk4   1/1     Running   0          29m   10.194.20.24   worker1.okd.cloche.ca   <none>           <none>
router-default-7c58f66c46-l5hj5   1/1     Running   0          29m   10.194.20.25   worker2.okd.cloche.ca   <none>           <none>
router-default-7c58f66c46-n9q9q   1/1     Running   0          25m   10.194.20.27   worker4.okd.cloche.ca   <none>           <none>
router-default-7c58f66c46-wbtbh   1/1     Running   0          25m   10.194.20.26   worker3.okd.cloche.ca   <none>           <none>

</code>