Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
<mxfile host="Electron">
<diagram name="Page-1" id="YXbzoI76saGIsczbTkJC">
<mxGraphModel dx="1182" dy="1185" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="1169" pageHeight="827" math="0" shadow="0">
<root>
<mxCell id="0" />
<mxCell id="1" parent="0" />
<mxCell id="B3RradtRmsC1N9M19CpR-1" parent="1" style="rounded=0;whiteSpace=wrap;html=1;align=left;verticalAlign=top;fontStyle=1" value="&lt;b&gt;&lt;font style=&quot;font-size: 18px;&quot;&gt;OpenShift Hub Cluster&lt;/font&gt;&lt;/b&gt;" vertex="1">
<mxGeometry height="280" width="470" x="30" y="80" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-2" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;fillColor=#e1d5e7;strokeColor=#9673a6;align=center;verticalAlign=bottom;fontStyle=1" value="&lt;font face=&quot;Arial, sans-serif&quot;&gt;&lt;span style=&quot;white-space-collapse: preserve;&quot;&gt;Hosted Control Plane of Tenant A&lt;/span&gt;&lt;/font&gt;" vertex="1">
<mxGeometry height="190" width="284" x="173" y="130" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-3" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;align=left;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.04,0,0,0,0],[0.08,0,0,0,0],[0.12,0,0,0,0],[0.15,0,0,0,0],[0.19,0,0,0,0],[0.23,0,0,0,0],[0.25,0,0,0,0],[0.25,1,0,0,0],[0.27,0,0,0,0],[0.31,0,0,0,0],[0.35,0,0,0,0],[0.38,0,0,0,0],[0.42,0,0,0,0],[0.46,0,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.54,0,0,0,0],[0.58,0,0,0,0],[0.62,0,0,0,0],[0.65,0,0,0,0],[0.69,0,0,0,0],[0.73,0,0,0,0],[0.75,0,0,0,0],[0.75,1,0,0,0],[0.77,0,0,0,0],[0.81,0,0,0,0],[0.85,0,0,0,0],[0.88,0,0,0,0],[0.92,0,0,0,0],[0.96,0,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];" value="&lt;font style=&quot;font-size: 16px;&quot;&gt;&lt;b&gt;Managment Network Segment&lt;/b&gt;&lt;/font&gt;" vertex="1">
<mxGeometry height="20" width="487" x="30" y="403" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-4" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;points=[[0,0,0,0,0],[0,0.25,0,0,0],[0,0.5,0,0,0],[0,0.75,0,0,0],[0,1,0,0,0],[0.02,1,0,0,0],[0.04,0,0,0,0],[0.04,1,0,0,0],[0.06,1,0,0,0],[0.08,0,0,0,0],[0.08,1,0,0,0],[0.1,1,0,0,0],[0.12,0,0,0,0],[0.12,1,0,0,0],[0.14,1,0,0,0],[0.15,0,0,0,0],[0.16,1,0,0,0],[0.18,1,0,0,0],[0.19,0,0,0,0],[0.2,1,0,0,0],[0.22,1,0,0,0],[0.23,0,0,0,0],[0.24,1,0,0,0],[0.25,0,0,0,0],[0.25,1,0,0,0],[0.27,0,0,0,0],[0.27,1,0,0,0],[0.29,1,0,0,0],[0.31,0,0,0,0],[0.31,1,0,0,0],[0.33,1,0,0,0],[0.35,0,0,0,0],[0.35,1,0,0,0],[0.37,1,0,0,0],[0.38,0,0,0,0],[0.39,1,0,0,0],[0.41,1,0,0,0],[0.42,0,0,0,0],[0.43,1,0,0,0],[0.45,1,0,0,0],[0.46,0,0,0,0],[0.47,1,0,0,0],[0.49,1,0,0,0],[0.5,0,0,0,0],[0.5,1,0,0,0],[0.51,1,0,0,0],[0.53,1,0,0,0],[0.54,0,0,0,0],[0.55,1,0,0,0],[0.57,1,0,0,0],[0.58,0,0,0,0],[0.59,1,0,0,0],[0.61,1,0,0,0],[0.62,0,0,0,0],[0.63,1,0,0,0],[0.65,0,0,0,0],[0.65,1,0,0,0],[0.67,1,0,0,0],[0.69,0,0,0,0],[0.69,1,0,0,0],[0.71,1,0,0,0],[0.73,0,0,0,0],[0.73,1,0,0,0],[0.75,0,0,0,0],[0.75,1,0,0,0],[0.76,1,0,0,0],[0.77,0,0,0,0],[0.78,1,0,0,0],[0.8,1,0,0,0],[0.81,0,0,0,0],[0.82,1,0,0,0],[0.84,1,0,0,0],[0.85,0,0,0,0],[0.86,1,0,0,0],[0.88,0,0,0,0],[0.88,1,0,0,0],[0.9,1,0,0,0],[0.92,0,0,0,0],[0.92,1,0,0,0],[0.94,1,0,0,0],[0.96,0,0,0,0],[0.96,1,0,0,0],[0.98,1,0,0,0],[1,0,0,0,0],[1,0.25,0,0,0],[1,0.5,0,0,0],[1,0.75,0,0,0],[1,1,0,0,0]];align=left;" value="&lt;b&gt;&lt;font style=&quot;font-size: 18px;&quot;&gt;Tenant A Network Segment&lt;/font&gt;&lt;/b&gt;" vertex="1">
<mxGeometry height="20" width="477" x="30" y="20" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-16" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;fillColor=#0050ef;fontColor=#ffffff;strokeColor=#001DBC;" value="Worker VM 1" vertex="1">
<mxGeometry height="30" width="120" x="183" y="170" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-17" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;fillColor=#0050ef;fontColor=#ffffff;strokeColor=#001DBC;" value="Worker VM 2" vertex="1">
<mxGeometry height="30" width="120" x="318" y="170" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-18" edge="1" parent="1" source="B3RradtRmsC1N9M19CpR-17" style="endArrow=oval;html=1;rounded=0;strokeWidth=5;fillColor=#e1d5e7;strokeColor=#9673a6;exitX=0.75;exitY=0;exitDx=0;exitDy=0;startArrow=oval;startFill=1;endFill=1;entryX=0.78;entryY=1;entryDx=0;entryDy=0;entryPerimeter=0;" target="B3RradtRmsC1N9M19CpR-4" value="">
<mxGeometry height="50" relative="1" width="50" as="geometry">
<mxPoint x="680" y="470" as="sourcePoint" />
<mxPoint x="627" y="180" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-20" edge="1" parent="1" source="B3RradtRmsC1N9M19CpR-16" style="endArrow=oval;html=1;rounded=0;strokeWidth=5;fillColor=#e1d5e7;strokeColor=#9673a6;startArrow=oval;startFill=1;endFill=1;entryX=0.53;entryY=1;entryDx=0;entryDy=0;entryPerimeter=0;exitX=0.75;exitY=0;exitDx=0;exitDy=0;" target="B3RradtRmsC1N9M19CpR-4" value="">
<mxGeometry height="50" relative="1" width="50" as="geometry">
<mxPoint x="420" y="510" as="sourcePoint" />
<mxPoint x="90" y="530" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-32" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;fillColor=#0050ef;fontColor=#ffffff;strokeColor=#001DBC;" value="Worker BareMetal" vertex="1">
<mxGeometry height="30" width="120" x="80" y="343" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-33" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;fillColor=#0050ef;fontColor=#ffffff;strokeColor=#001DBC;" value="Worker BareMetal" vertex="1">
<mxGeometry height="30" width="120" x="215" y="343" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-34" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;fillColor=#0050ef;fontColor=#ffffff;strokeColor=#001DBC;" value="Worker BareMetal" vertex="1">
<mxGeometry height="30" width="120" x="350" y="343" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-35" edge="1" parent="1" source="B3RradtRmsC1N9M19CpR-34" style="endArrow=oval;html=1;rounded=0;strokeWidth=5;fillColor=#d5e8d4;strokeColor=#82b366;exitX=0.5;exitY=1;exitDx=0;exitDy=0;startArrow=oval;startFill=1;endFill=1;entryX=0.77;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="B3RradtRmsC1N9M19CpR-3" value="">
<mxGeometry height="50" relative="1" width="50" as="geometry">
<mxPoint x="560" y="403" as="sourcePoint" />
<mxPoint x="520" y="533" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-36" edge="1" parent="1" source="B3RradtRmsC1N9M19CpR-33" style="endArrow=oval;html=1;rounded=0;strokeWidth=5;fillColor=#d5e8d4;strokeColor=#82b366;exitX=0.5;exitY=1;exitDx=0;exitDy=0;startArrow=oval;startFill=1;endFill=1;entryX=0.5;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="B3RradtRmsC1N9M19CpR-3" value="">
<mxGeometry height="50" relative="1" width="50" as="geometry">
<mxPoint x="420" y="453" as="sourcePoint" />
<mxPoint x="435" y="543" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-37" edge="1" parent="1" source="B3RradtRmsC1N9M19CpR-32" style="endArrow=oval;html=1;rounded=0;strokeWidth=5;fillColor=#d5e8d4;strokeColor=#82b366;exitX=0.5;exitY=1;exitDx=0;exitDy=0;startArrow=oval;startFill=1;endFill=1;entryX=0.23;entryY=0;entryDx=0;entryDy=0;entryPerimeter=0;" target="B3RradtRmsC1N9M19CpR-3" value="">
<mxGeometry height="50" relative="1" width="50" as="geometry">
<mxPoint x="220" y="433" as="sourcePoint" />
<mxPoint x="210" y="523" as="targetPoint" />
</mxGeometry>
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-48" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;" value="&lt;span id=&quot;docs-internal-guid-aa7dbe45-7fff-f4de-7713-e002c688d2cc&quot;&gt;&lt;span style=&quot;font-family: Arial, sans-serif; background-color: transparent; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-variant-position: normal; font-variant-emoji: normal; vertical-align: baseline; white-space-collapse: preserve;&quot;&gt;Ignition&lt;/span&gt;&lt;/span&gt;" vertex="1">
<mxGeometry height="30" width="120" x="318" y="210" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-49" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;" value="API Server" vertex="1">
<mxGeometry height="30" width="120" x="318" y="250" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-50" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;" value="OAuth" vertex="1">
<mxGeometry height="30" width="120" x="183" y="210" as="geometry" />
</mxCell>
<mxCell id="B3RradtRmsC1N9M19CpR-51" parent="1" style="rounded=0;whiteSpace=wrap;html=1;fontSize=12;" value="&lt;span id=&quot;docs-internal-guid-44e9e855-7fff-e21b-1846-7dce005654dc&quot;&gt;&lt;span style=&quot;font-family: Arial, sans-serif; background-color: transparent; font-variant-numeric: normal; font-variant-east-asian: normal; font-variant-alternates: normal; font-variant-position: normal; font-variant-emoji: normal; vertical-align: baseline; white-space-collapse: preserve;&quot;&gt;Konnectivity&lt;/span&gt;&lt;/span&gt;" vertex="1">
<mxGeometry height="30" width="120" x="183" y="250" as="geometry" />
</mxCell>
</root>
</mxGraphModel>
</diagram>
</mxfile>
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
global
log 127.0.0.1 local2
pidfile /var/run/haproxy.pid
maxconn 4000
daemon
defaults
mode http
log global
option dontlognull
option http-server-close
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000

listen api
bind *:6443
mode tcp
balance source
server ucs-blade-server-5 10.32.96.105:30918 check inter 1s
server ucs-blade-server-6 10.32.96.106:30918 check inter 1s
server ucs-blade-server-7 10.32.96.107:30918 check inter 1s
server ucs-blade-server-8 10.32.96.108:30918 check inter 1s
Original file line number Diff line number Diff line change
@@ -0,0 +1,230 @@
---
title: Hosted Control Plane and tenant networking
linktitle: Hosted Control Plane and tenant networking
description: Hosted Control Plane and tenant networking
tags: ['hcp','v4.21']
---
# Hosted Control Plane and tenant networking

Official documentation: Net yet available

Tested with:

|Component|Version|
|---|---|
|OpenShift|v4.21.9|
|OpenShift Virt|v4.21.0|

## ToDo's

* Add custom endpointpublishing strategy
* Find a solution for the nodeport chicken egg problem of external api load balancer
* Check WebUI bug: Ingress domain is wrong.

## Overview

Challenge: running an hosted cluster with in different tenant network segment/vlan without widely open access from tenant segment to managment segment.

Addtional requirement, the hub cluster should not have any address or network connection into the tenant network segment. It's only allowed to place virtual machines into the network segment.

![](overview.drawio){ page="Page-1" }

The worker nodes of the hosted cluster are quite easy to solve, just connected them into the tenant network segment (import, DHCP is required).

The hosted control plane compontents to expose into tenant network segment is more challenging. Following components have to concider:

* API Server
* OAuth
* Konnectivity
* Ignition

Here an list of possible exposing options for these components:

|Component/Service|Exposing strategy (`servicePublishingStrategy`)|Kubernetes Service type `LoadBalancer`|Ingress/Route|
|---|---|---|---|
|API Server|<li>LoadBalancer (Recommended, K8s Service Type Load Balancer)</li><li>NodePort* (not for production)</li>|✅|❌|
|OAuth|<li>Route/Ingress (default)</li><li>NodePort* (not for production)</li>|❌|✅|
|Konnectivity|<li>Route/Ingress (default)</li><li>LoadBalancer (K8s Service Type Load Balancer)</li><li>NodePort* (not for production)</li>|✅|✅|
|Ignition|<li>Route/Ingress (default)</li><li>NodePort* (not for production)</li>|✅|❌|

For our proof of concept we want to try following, exposing the components via:

* API Server: LoadBalancer
* OAuth: Router/Ingress: via a dedicted router shard.
* Konnectivity: via a dedicted router shard.
* Ignition: via a dedicted router shard.

## Exposing compontents via router/ingress shard

The idea with the dedicated router/ingress shared is to expose the router/ingress shard into the tenant network segment and only for the hosted cluster components.

In front of the router/ingress shared is an external load balancer (for example, f5 bigip, netscaler,..) with access into the managment network segment and expose the router shared into the tenant network segment.

## Proof of concept envrioment overview

![](overview.drawio){ page="Page-2" }

### Router between Mgmt and Tenant-A

[VyOS Router](https://vyos.io/) router & firewall. Do not allow Traffic between Mgmt and Tenant-A network except DNS and gateway. To provde direct internect connection.

??? example "VyOS config commands"

```shell
--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/vyos-router-2003.txt"
```

### Ingress Sharding

* [2.3.4. Ingress sharding in OpenShift Container Platform](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/ingress_and_load_balancing/configuring-ingress-cluster-traffic#nw-ingress-sharding-concept_configuring-ingress-cluster-traffic-ingress-controller)
* [3.1.3.8.1. Example load balancer configuration for user-provisioned clusters](https://docs.redhat.com/en/documentation/openshift_container_platform/4.21/html/installing_on_vmware_vsphere/user-provisioned-infrastructure)

???+ example "Ingress Controller"

```yaml
--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-controller-shard.yaml"
```

```shell
% oc get svc -n openshift-ingress router-nodeport-tenant-a
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
router-nodeport-tenant-a NodePort 172.30.141.209 <none> 80:32460/TCP,443:32488/TCP,1936:32095/TCP 106s
```

Ingress sharding load balancer is an RHEL 9 system with haproxy.

* Install HAProxy `dnf install haproxy`
* Configure selinux `setsebool -P haproxy_connect_any 1`
* Apply Example haproxy.conf (don't forget to update ports)
* Enabel and start haproxy `systemctl enable --now haproxy`

??? example "HAProxy config"

```shell
--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-shared-haproxy.conf"
```

Add DNS Records

```bind
konnectivity.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111
oauth.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111
ignition.tenant-a.coe.muc.redhat.com. IN A 192.168.203.111
```

### Start hosted control plane and nodepool

```yaml
apiVersion: hypershift.openshift.io/v1beta1
kind: HostedCluster
metadata:
name: 'tenant-a'
namespace: 'clusters'
labels:
"cluster.open-cluster-management.io/clusterset": 'default'
spec:
configuration:
ingress:
appsDomain: apps.tenant-a.coe.muc.redhat.com
domain: ''
loadBalancer:
platform:
type: ''
channel: fast-4.21
etcd:
managed:
storage:
persistentVolume:
size: 8Gi
type: PersistentVolume
managementType: Managed
release:
image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi
pullSecret:
name: pullsecret-cluster-tenant-a
sshKey:
name: sshkey-cluster-tenant-a
networking:
clusterNetwork:
- cidr: 10.132.0.0/14
serviceNetwork:
- cidr: 172.31.0.0/16
networkType: OVNKubernetes
controllerAvailabilityPolicy: SingleReplica
infrastructureAvailabilityPolicy: SingleReplica
platform:
type: KubeVirt
kubevirt:
baseDomainPassthrough: false
infraID: 'tenant-a'
services:
- service: APIServer
servicePublishingStrategy:
type: LoadBalancer
loadBalancer:
hostname: api.tenant-a.coe.muc.redhat.com
- service: OAuthServer
servicePublishingStrategy:
type: Route
route:
hostname: oauth.tenant-a.coe.muc.redhat.com
- service: OIDC
servicePublishingStrategy:
type: Route
- service: Konnectivity
servicePublishingStrategy:
type: Route
route:
hostname: konnectivity.tenant-a.coe.muc.redhat.com
- service: Ignition
servicePublishingStrategy:
type: Route
route:
hostname: ignition.tenant-a.coe.muc.redhat.com
```

```yaml
---
apiVersion: hypershift.openshift.io/v1beta1
kind: NodePool
metadata:
name: 'tenant-a'
namespace: 'clusters'
spec:
arch: amd64
clusterName: 'tenant-a'
replicas: 2
management:
autoRepair: false
upgradeType: Replace
platform:
type: KubeVirt
kubevirt:
compute:
cores: 2
memory: 8Gi
rootVolume:
type: Persistent
persistent:
size: 32Gi
additionalNetworks:
- name: default/cudn-localnet1-2003
attachDefaultNetwork: false
release:
image: quay.io/openshift-release-dev/ocp-release:4.21.11-multi
```

### Deploy external load balancer for ingress of hosted cluster

Ingress sharding load balancer is an RHEL 9 system with haproxy.

* Install HAProxy `dnf install haproxy`
* Configure selinux `setsebool -P haproxy_connect_any 1`
* Apply Example haproxy.conf (don't forget to update ports)
* Enabel and start haproxy `systemctl enable --now haproxy`

??? example "HAProxy config"

```shell
--8<-- "content/cluster-installation/hosted-control-plane/tenant-network/ingress-lb.conf"
```
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
apiVersion: operator.openshift.io/v1
kind: IngressController
metadata:
name: tenant-a
namespace: openshift-ingress-operator
spec:
domain: tenant-a.coe.muc.redhat.com

endpointPublishingStrategy:
type: NodePortService
namespaceSelector:
matchExpressions:
- key: kubernetes.io/metadata.name
operator: In
values:
- ingress-test
- clusters-tenant-a
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
global
log 127.0.0.1 local2
pidfile /var/run/haproxy.pid
maxconn 4000
daemon
defaults
mode http
log global
option dontlognull
option http-server-close
option redispatch
retries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 3000

listen ingress-router-443
bind *:443
mode tcp
balance source
server tenant-a-gngj5-mfwp6 192.168.203.101:30190 check inter 1s
server tenant-a-gngj5-rrbmv 192.168.203.102:30190 check inter 1s

listen ingress-router-80
bind *:80
mode tcp
balance source
server tenant-a-gngj5-mfwp6 192.168.203.101:30282 check inter 1s
server tenant-a-gngj5-rrbmv 192.168.203.102:30282 check inter 1s
Loading
Loading