core/oc-datacenter
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WatchDog Kube

Browse Source
This commit is contained in:
mr
2026-03-24 10:50:36 +01:00
parent a7ffede3e2
commit dab61463f0
14 changed files with 884 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
infrastructure/admiralty.go
View File

@@ -4,7 +4,9 @@ import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"time"
@@ -14,7 +16,6 @@ import (
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/tools"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
)
@@ -43,7 +44,9 @@ type admiraltyConsidersPayload struct {
// emitAdmiraltyConsiders publishes a PB_CONSIDERS back to OriginID with the result
// of the admiralty provisioning. secret is the base64-encoded kubeconfig; err is nil on success.
func emitAdmiraltyConsiders(executionsID, originID, secret string, provErr error) {
// When self is true the origin is the local peer: emits directly on CONSIDERS_EVENT
// instead of routing through PROPALGATION_EVENT.
func emitAdmiraltyConsiders(executionsID, originID, secret string, provErr error, self bool) {
var errStr *string
if provErr != nil {
s := provErr.Error()
@@ -55,6 +58,15 @@ func emitAdmiraltyConsiders(executionsID, originID, secret string, provErr error
Secret: secret,
Error: errStr,
})
if self {
go tools.NewNATSCaller().SetNATSPub(tools.CONSIDERS_EVENT, tools.NATSResponse{
FromApp: "oc-datacenter",
Datatype: tools.COMPUTE_RESOURCE,
Method: int(tools.CONSIDERS_EVENT),
Payload: payload,
})
return
}
b, _ := json.Marshal(&tools.PropalgationMessage{
DataType: tools.COMPUTE_RESOURCE.EnumIndex(),
Action: tools.PB_CONSIDERS,
@@ -83,41 +95,36 @@ func NewAdmiraltySetter(execIDS string) *AdmiraltySetter {
// InitializeAsSource is called on the peer that acts as the SOURCE cluster (compute provider).
// It creates the AdmiraltySource resource, generates a kubeconfig for the target peer,
// and publishes it on NATS so the target peer can complete its side of the setup.
func (s *AdmiraltySetter) InitializeAsSource(ctx context.Context, localPeerID string, destPeerID string, originID string) {
func (s *AdmiraltySetter) InitializeAsSource(ctx context.Context, localPeerID string, destPeerID string, originID string, self bool) error {
logger := oclib.GetLogger()
serv, err := tools.NewKubernetesService(conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData)
if err != nil {
logger.Error().Msg("InitializeAsSource: failed to create service: " + err.Error())
return
return errors.New("InitializeAsSource: failed to create service: " + err.Error())
}
// Create the AdmiraltySource resource on this cluster (inlined from CreateAdmiraltySource controller)
logger.Info().Msg("Creating AdmiraltySource ns-" + s.ExecutionsID)
_, err = serv.CreateAdmiraltySource(ctx, s.ExecutionsID)
if err != nil && !apierrors.IsAlreadyExists(err) {
logger.Error().Msg("InitializeAsSource: failed to create source: " + err.Error())
return
if err != nil && !strings.Contains(err.Error(), "already exists") {
return errors.New("InitializeAsSource: failed to create service: " + err.Error())
}
// Generate a service-account token for the namespace (inlined from GetAdmiraltyKubeconfig controller)
token, err := serv.GenerateToken(ctx, s.ExecutionsID, 3600)
if err != nil {
logger.Error().Msg("InitializeAsSource: failed to generate token for ns-" + s.ExecutionsID + ": " + err.Error())
return
return errors.New("InitializeAsSource: failed to generate token for ns-" + s.ExecutionsID + ": " + err.Error())
}
kubeconfig, err := buildHostKubeWithToken(token)
if err != nil {
logger.Error().Msg("InitializeAsSource: " + err.Error())
return
return errors.New("InitializeAsSource: " + err.Error())
}
b, err := json.Marshal(kubeconfig)
if err != nil {
logger.Error().Msg("InitializeAsSource: failed to marshal kubeconfig: " + err.Error())
return
return errors.New("InitializeAsSource: failed to marshal kubeconfig: " + err.Error())
}
encodedKubeconfig := base64.StdEncoding.EncodeToString(b)
kube := KubeconfigEvent{
@@ -128,14 +135,14 @@ func (s *AdmiraltySetter) InitializeAsSource(ctx context.Context, localPeerID st
OriginID: originID,
}
if destPeerID == localPeerID {
s.InitializeAsTarget(ctx, kube)
return
// Self case: source and target are the same cluster, no Admiralty target to configure.
emitAdmiraltyConsiders(s.ExecutionsID, originID, encodedKubeconfig, nil, true)
return nil
}
// Publish the kubeconfig on NATS so the target peer can proceed
payload, err := json.Marshal(kube)
if err != nil {
logger.Error().Msg("InitializeAsSource: failed to marshal kubeconfig event: " + err.Error())
return
return errors.New("InitializeAsSource: failed to marshal kubeconfig event: " + err.Error())
}
if b, err := json.Marshal(&tools.PropalgationMessage{
@@ -145,20 +152,22 @@ func (s *AdmiraltySetter) InitializeAsSource(ctx context.Context, localPeerID st
}); err == nil {
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
FromApp: "oc-datacenter",
Datatype: -1,
Datatype: tools.COMPUTE_RESOURCE,
User: "",
Method: int(tools.PROPALGATION_EVENT),
Payload: b,
})
}
logger.Info().Msg("InitializeAsSource: kubeconfig published for ns-" + s.ExecutionsID)
return nil
}
// InitializeAsTarget is called on the peer that acts as the TARGET cluster (scheduler).
// It waits for the kubeconfig published by the source peer via NATS, then creates
// the Secret, AdmiraltyTarget, and polls until the virtual node appears.
// kubeconfigCh must be obtained from RegisterKubeconfigWaiter before this goroutine starts.
func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj KubeconfigEvent) {
// self must be true when the origin peer is the local peer (direct CONSIDERS_EVENT emission).
func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj KubeconfigEvent, self bool) {
logger := oclib.GetLogger()
defer kubeconfigChannels.Delete(s.ExecutionsID)
@@ -174,17 +183,17 @@ func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj
// 1. Create the namespace
logger.Info().Msg("InitializeAsTarget: creating Namespace " + s.ExecutionsID)
if err := serv.CreateNamespace(ctx, s.ExecutionsID); err != nil && !apierrors.IsAlreadyExists(err) {
if err := serv.CreateNamespace(ctx, s.ExecutionsID); err != nil && !strings.Contains(err.Error(), "already exists") {
logger.Error().Msg("InitializeAsTarget: failed to create namespace: " + err.Error())
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err, self)
return
}
// 2. Create the ServiceAccount sa-{executionID}
logger.Info().Msg("InitializeAsTarget: creating ServiceAccount sa-" + s.ExecutionsID)
if err := serv.CreateServiceAccount(ctx, s.ExecutionsID); err != nil && !apierrors.IsAlreadyExists(err) {
if err := serv.CreateServiceAccount(ctx, s.ExecutionsID); err != nil && !strings.Contains(err.Error(), "already exists") {
logger.Error().Msg("InitializeAsTarget: failed to create service account: " + err.Error())
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err, self)
return
}
@@ -204,18 +213,18 @@ func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj
{"get", "create", "update"},
{"get"},
{"patch"}},
); err != nil && !apierrors.IsAlreadyExists(err) {
); err != nil && !strings.Contains(err.Error(), "already exists") {
logger.Error().Msg("InitializeAsTarget: failed to create role: " + err.Error())
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err, self)
return
}
// 4. Create the RoleBinding
rbName := "rb-" + s.ExecutionsID
logger.Info().Msg("InitializeAsTarget: creating RoleBinding " + rbName)
if err := serv.CreateRoleBinding(ctx, s.ExecutionsID, rbName, roleName); err != nil && !apierrors.IsAlreadyExists(err) {
if err := serv.CreateRoleBinding(ctx, s.ExecutionsID, rbName, roleName); err != nil && !strings.Contains(err.Error(), "already exists") {
logger.Error().Msg("InitializeAsTarget: failed to create role binding: " + err.Error())
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err, self)
return
}
@@ -223,7 +232,7 @@ func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj
logger.Info().Msg("InitializeAsTarget: creating Secret ns-" + s.ExecutionsID)
if _, err := serv.CreateKubeconfigSecret(ctx, kubeconfigData, s.ExecutionsID, kubeconfigObj.SourcePeerID); err != nil {
logger.Error().Msg("InitializeAsTarget: failed to create kubeconfig secret: " + err.Error())
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err, self)
return
}
@@ -235,14 +244,14 @@ func (s *AdmiraltySetter) InitializeAsTarget(ctx context.Context, kubeconfigObj
if err == nil {
err = fmt.Errorf("CreateAdmiraltyTarget returned nil response")
}
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, "", err, self)
return
}
// Poll until the virtual node appears (inlined from GetNodeReady controller)
logger.Info().Msg("InitializeAsTarget: waiting for virtual node ns-" + s.ExecutionsID)
s.waitForNode(ctx, serv, kubeconfigObj.SourcePeerID)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, kubeconfigData, nil)
emitAdmiraltyConsiders(s.ExecutionsID, kubeconfigObj.OriginID, kubeconfigData, nil, self)
}
// waitForNode polls GetOneNode until the Admiralty virtual node appears on this cluster.
@@ -325,7 +334,11 @@ func buildHostKubeWithToken(token string) (*models.KubeConfigValue, error) {
if len(token) == 0 {
return nil, fmt.Errorf("buildHostKubeWithToken: empty token")
}
encodedCA := base64.StdEncoding.EncodeToString([]byte(conf.GetConfig().KubeCA))
apiHost := conf.GetConfig().KubeExternalHost
if apiHost == "" {
apiHost = conf.GetConfig().KubeHost
}
encodedCA := conf.GetConfig().KubeCA
return &models.KubeConfigValue{
APIVersion: "v1",
CurrentContext: "default",
@@ -334,7 +347,7 @@ func buildHostKubeWithToken(token string) (*models.KubeConfigValue, error) {
Clusters: []models.KubeconfigNamedCluster{{
Name: "default",
Cluster: models.KubeconfigCluster{
Server: "https://" + conf.GetConfig().KubeHost + ":6443",
Server: "https://" + apiHost + ":6443",
CertificateAuthorityData: encodedCA,
},
}},