Oc-Datacenter Allowed Resource And Prepull Images For Efficient process
This commit is contained in:
219
infrastructure/storage/minio.go
Normal file
219
infrastructure/storage/minio.go
Normal file
@@ -0,0 +1,219 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"oc-datacenter/conf"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"github.com/minio/madmin-go/v4"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/rest"
|
||||
|
||||
"github.com/necmettindev/randomstring"
|
||||
)
|
||||
|
||||
type MinioService struct {
|
||||
Url string
|
||||
RootKey string
|
||||
RootSecret string
|
||||
MinioAdminClient *madmin.AdminClient
|
||||
}
|
||||
|
||||
type StatementEntry struct {
|
||||
Effect string `json:"Effect"`
|
||||
Action []string `json:"Action"`
|
||||
Resource string `json:"Resource"`
|
||||
}
|
||||
|
||||
type PolicyDocument struct {
|
||||
Version string `json:"Version"`
|
||||
Statement []StatementEntry `json:"Statement"`
|
||||
}
|
||||
|
||||
func NewMinioService(url string) *MinioService {
|
||||
return &MinioService{
|
||||
Url: url,
|
||||
RootKey: conf.GetConfig().MinioRootKey,
|
||||
RootSecret: conf.GetConfig().MinioRootSecret,
|
||||
}
|
||||
}
|
||||
|
||||
func (m *MinioService) CreateClient() error {
|
||||
cred := credentials.NewStaticV4(m.RootKey, m.RootSecret, "")
|
||||
cli, err := madmin.NewWithOptions(m.Url, &madmin.Options{Creds: cred, Secure: false}) // Maybe in the future we should use the secure option ?
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m.MinioAdminClient = cli
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MinioService) CreateCredentials(executionId string) (string, string, error) {
|
||||
|
||||
policy := PolicyDocument{
|
||||
Version: "2012-10-17",
|
||||
Statement: []StatementEntry{
|
||||
{
|
||||
Effect: "Allow",
|
||||
Action: []string{"s3:GetObject", "s3:PutObject"},
|
||||
Resource: "arn:aws:s3:::" + executionId + "/*",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
p, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
randAccess, randSecret := getRandomCreds()
|
||||
|
||||
req := madmin.AddServiceAccountReq{
|
||||
Policy: p,
|
||||
TargetUser: m.RootKey,
|
||||
AccessKey: randAccess,
|
||||
SecretKey: randSecret,
|
||||
}
|
||||
|
||||
res, err := m.MinioAdminClient.AddServiceAccount(context.Background(), req)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
|
||||
return res.AccessKey, res.SecretKey, nil
|
||||
|
||||
}
|
||||
|
||||
func getRandomCreds() (string, string) {
|
||||
opts := randomstring.GenerationOptions{
|
||||
Length: 20,
|
||||
}
|
||||
|
||||
a, _ := randomstring.GenerateString(opts)
|
||||
|
||||
opts.Length = 40
|
||||
s, _ := randomstring.GenerateString(opts)
|
||||
|
||||
return a, s
|
||||
|
||||
}
|
||||
func (m *MinioService) CreateMinioConfigMap(minioID string, executionId string, url string) error {
|
||||
config, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
clientset, err := kubernetes.NewForConfig(config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configMap := &v1.ConfigMap{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: minioID + "artifact-repository",
|
||||
Namespace: executionId,
|
||||
},
|
||||
Data: map[string]string{
|
||||
minioID + "s3-local": fmt.Sprintf(`
|
||||
s3:
|
||||
bucket: %s
|
||||
endpoint: %s
|
||||
insecure: true
|
||||
accessKeySecret:
|
||||
name: %s-secret-s3
|
||||
key: accesskey
|
||||
secretKeySecret:
|
||||
name: %s-secret-s3
|
||||
key: secretkey
|
||||
`, minioID+"-"+executionId, url, minioID, minioID),
|
||||
},
|
||||
}
|
||||
|
||||
existing, err := clientset.CoreV1().
|
||||
ConfigMaps(executionId).
|
||||
Get(context.Background(), minioID+"artifact-repository", metav1.GetOptions{})
|
||||
|
||||
if err == nil {
|
||||
// Update
|
||||
existing.Data = configMap.Data
|
||||
_, err = clientset.CoreV1().
|
||||
ConfigMaps(executionId).
|
||||
Update(context.Background(), existing, metav1.UpdateOptions{})
|
||||
} else {
|
||||
// Create
|
||||
_, err = clientset.CoreV1().
|
||||
ConfigMaps(executionId).
|
||||
Create(context.Background(), configMap, metav1.CreateOptions{})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *MinioService) CreateBucket(minioID string, executionId string) error {
|
||||
l := oclib.GetLogger()
|
||||
cred := credentials.NewStaticV4(m.RootKey, m.RootSecret, "")
|
||||
client, err := minio.New(m.Url, &minio.Options{
|
||||
Creds: cred,
|
||||
Secure: false,
|
||||
})
|
||||
if err != nil {
|
||||
l.Error().Msg("Error when creating the minio client for the data plane")
|
||||
return err
|
||||
}
|
||||
|
||||
err = client.MakeBucket(context.Background(), minioID+"-"+executionId, minio.MakeBucketOptions{})
|
||||
if err != nil {
|
||||
l.Error().Msg("Error when creating the bucket for namespace " + executionId)
|
||||
return err
|
||||
}
|
||||
|
||||
l.Info().Msg("Created the bucket " + minioID + "-" + executionId + " on " + m.Url + " minio")
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteCredentials revokes a scoped Minio service account by its access key.
|
||||
func (m *MinioService) DeleteCredentials(accessKey string) error {
|
||||
if err := m.MinioAdminClient.DeleteServiceAccount(context.Background(), accessKey); err != nil {
|
||||
return fmt.Errorf("DeleteCredentials: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteBucket removes the execution bucket from Minio.
|
||||
func (m *MinioService) DeleteBucket(minioID, executionId string) error {
|
||||
l := oclib.GetLogger()
|
||||
cred := credentials.NewStaticV4(m.RootKey, m.RootSecret, "")
|
||||
client, err := minio.New(m.Url, &minio.Options{Creds: cred, Secure: false})
|
||||
if err != nil {
|
||||
l.Error().Msg("Error when creating minio client for bucket deletion")
|
||||
return err
|
||||
}
|
||||
bucketName := minioID + "-" + executionId
|
||||
if err := client.RemoveBucket(context.Background(), bucketName); err != nil {
|
||||
l.Error().Msg("Error when deleting bucket " + bucketName)
|
||||
return err
|
||||
}
|
||||
l.Info().Msg("Deleted bucket " + bucketName + " on " + m.Url)
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteMinioConfigMap removes the artifact-repository ConfigMap from the execution namespace.
|
||||
func (m *MinioService) DeleteMinioConfigMap(minioID, executionId string) error {
|
||||
cfg, err := rest.InClusterConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clientset, err := kubernetes.NewForConfig(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return clientset.CoreV1().ConfigMaps(executionId).Delete(
|
||||
context.Background(), minioID+"artifact-repository", metav1.DeleteOptions{},
|
||||
)
|
||||
}
|
||||
362
infrastructure/storage/minio_setter.go
Normal file
362
infrastructure/storage/minio_setter.go
Normal file
@@ -0,0 +1,362 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
|
||||
"oc-datacenter/conf"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/dbs"
|
||||
bookingmodel "cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/live"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// MinioCredentialEvent is the NATS payload used to transfer Minio credentials between peers.
|
||||
//
|
||||
// Two-phase protocol over PROPALGATION_EVENT (Action = PB_MINIO_CONFIG):
|
||||
// - Phase 1 – role assignment (Access == ""):
|
||||
// oc-discovery routes this to the SOURCE peer (Minio host) → InitializeAsSource.
|
||||
// - Phase 2 – credential delivery (Access != ""):
|
||||
// oc-discovery routes this to the TARGET peer (compute host) → InitializeAsTarget.
|
||||
type MinioCredentialEvent struct {
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
MinioID string `json:"minio_id"`
|
||||
Access string `json:"access"`
|
||||
Secret string `json:"secret"`
|
||||
SourcePeerID string `json:"source_peer_id"`
|
||||
DestPeerID string `json:"dest_peer_id"`
|
||||
URL string `json:"url"`
|
||||
// OriginID is the peer that initiated the provisioning request.
|
||||
// The PB_CONSIDERS response is routed back to this peer.
|
||||
OriginID string `json:"origin_id"`
|
||||
}
|
||||
|
||||
// minioConsidersPayload is the PB_CONSIDERS payload emitted after minio provisioning.
|
||||
type minioConsidersPayload struct {
|
||||
OriginID string `json:"origin_id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
Secret string `json:"secret,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// MinioSetter carries the execution context for a Minio credential provisioning.
|
||||
type MinioSetter struct {
|
||||
ExecutionsID string // used as both the bucket name and the K8s namespace suffix
|
||||
MinioID string // ID of the Minio storage resource
|
||||
}
|
||||
|
||||
func NewMinioSetter(execID, minioID string) *MinioSetter {
|
||||
return &MinioSetter{ExecutionsID: execID, MinioID: minioID}
|
||||
}
|
||||
|
||||
// emitConsiders publishes a PB_CONSIDERS back to OriginID with the result of
|
||||
// the minio provisioning. secret is the provisioned credential; err is nil on success.
|
||||
// When self is true the origin is the local peer: emits directly on CONSIDERS_EVENT
|
||||
// instead of routing through PROPALGATION_EVENT.
|
||||
func (m *MinioSetter) emitConsiders(executionsID, originID, secret string, provErr error, self bool) {
|
||||
fmt.Println("emitConsiders !")
|
||||
var errStr *string
|
||||
if provErr != nil {
|
||||
s := provErr.Error()
|
||||
errStr = &s
|
||||
}
|
||||
payload, _ := json.Marshal(minioConsidersPayload{
|
||||
OriginID: originID,
|
||||
ExecutionsID: executionsID,
|
||||
Secret: secret,
|
||||
Error: errStr,
|
||||
})
|
||||
if self {
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.CONSIDERS_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: tools.STORAGE_RESOURCE,
|
||||
Method: int(tools.CONSIDERS_EVENT),
|
||||
Payload: payload,
|
||||
})
|
||||
return
|
||||
}
|
||||
b, _ := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: tools.STORAGE_RESOURCE.EnumIndex(),
|
||||
Action: tools.PB_CONSIDERS,
|
||||
Payload: payload,
|
||||
})
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: -1,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
|
||||
// InitializeAsSource is called on the peer that hosts the Minio instance.
|
||||
//
|
||||
// It:
|
||||
// 1. Looks up the live-storage endpoint URL for MinioID.
|
||||
// 2. Creates a scoped service account (access + secret limited to the execution bucket).
|
||||
// 3. Creates the execution bucket.
|
||||
// 4. If source and dest are the same peer, calls InitializeAsTarget directly.
|
||||
// Otherwise, publishes a MinioCredentialEvent via NATS (Phase 2) so that
|
||||
// oc-discovery can route the credentials to the compute peer.
|
||||
func (m *MinioSetter) InitializeAsSource(ctx context.Context, localPeerID, destPeerID, originID string, self bool) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
url, err := m.loadMinioURL(localPeerID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
service := NewMinioService(url)
|
||||
if err := service.CreateClient(); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to create admin client: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
access, secret, err := service.CreateCredentials(m.ExecutionsID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to create service account: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := service.CreateBucket(m.MinioID, m.ExecutionsID); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to create bucket: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.InitializeAsSource: bucket and service account ready for " + m.ExecutionsID)
|
||||
|
||||
event := MinioCredentialEvent{
|
||||
ExecutionsID: m.ExecutionsID,
|
||||
MinioID: m.MinioID,
|
||||
Access: access,
|
||||
Secret: secret,
|
||||
SourcePeerID: localPeerID,
|
||||
DestPeerID: destPeerID,
|
||||
OriginID: originID,
|
||||
}
|
||||
|
||||
if destPeerID == localPeerID {
|
||||
// Same peer: store the secret locally without going through NATS.
|
||||
m.InitializeAsTarget(ctx, event, true)
|
||||
return
|
||||
}
|
||||
|
||||
// Cross-peer: publish credentials (Phase 2) so oc-discovery routes them to the compute peer.
|
||||
payload, err := json.Marshal(event)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsSource: failed to marshal credential event: " + err.Error())
|
||||
return
|
||||
}
|
||||
if b, err := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: -1,
|
||||
Action: tools.PB_MINIO_CONFIG,
|
||||
Payload: payload,
|
||||
}); err == nil {
|
||||
go tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-datacenter",
|
||||
Datatype: tools.STORAGE_RESOURCE,
|
||||
User: "",
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
logger.Info().Msg("MinioSetter.InitializeAsSource: credentials published via NATS for " + m.ExecutionsID)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// InitializeAsTarget is called on the peer that runs the compute workload.
|
||||
//
|
||||
// It stores the Minio credentials received from the source peer (via NATS or directly)
|
||||
// as a Kubernetes secret inside the execution namespace, making them available to pods.
|
||||
// self must be true when the origin peer is the local peer (direct CONSIDERS_EVENT emission).
|
||||
func (m *MinioSetter) InitializeAsTarget(ctx context.Context, event MinioCredentialEvent, self bool) {
|
||||
fmt.Println("InitializeAsTarget is Self :", self)
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
k, err := tools.NewKubernetesService(
|
||||
conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsTarget: failed to create k8s service: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if err := k.CreateSecret(ctx, event.MinioID, event.ExecutionsID, event.Access, event.Secret); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsTarget: failed to create k8s secret: " + err.Error())
|
||||
m.emitConsiders(event.ExecutionsID, event.OriginID, "", err, self)
|
||||
return
|
||||
}
|
||||
|
||||
if err := NewMinioService(event.URL).CreateMinioConfigMap(event.MinioID, event.ExecutionsID, event.URL); err != nil {
|
||||
logger.Error().Msg("MinioSetter.InitializeAsTarget: failed to create config map: " + err.Error())
|
||||
m.emitConsiders(event.ExecutionsID, event.OriginID, "", err, self)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.InitializeAsTarget: Minio credentials stored in namespace " + event.ExecutionsID)
|
||||
m.emitConsiders(event.ExecutionsID, event.OriginID, event.Secret, nil, self)
|
||||
}
|
||||
|
||||
// MinioDeleteEvent is the NATS payload used to tear down Minio resources.
|
||||
// It mirrors MinioCredentialEvent but carries the access key for revocation.
|
||||
type MinioDeleteEvent struct {
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
MinioID string `json:"minio_id"`
|
||||
Access string `json:"access"` // service account access key to revoke on the Minio host
|
||||
SourcePeerID string `json:"source_peer_id"`
|
||||
DestPeerID string `json:"dest_peer_id"`
|
||||
OriginID string `json:"origin_id"`
|
||||
}
|
||||
|
||||
// TeardownAsTarget is called on the peer that runs the compute workload.
|
||||
// It reads the stored access key from the K8s secret, then removes both the secret
|
||||
// and the artifact-repository ConfigMap from the execution namespace.
|
||||
// For same-peer deployments it calls TeardownAsSource directly; otherwise it
|
||||
// publishes a MinioDeleteEvent via NATS (PB_DELETE) so oc-discovery routes it to
|
||||
// the Minio host peer.
|
||||
func (m *MinioSetter) TeardownAsTarget(ctx context.Context, event MinioDeleteEvent) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
k, err := tools.NewKubernetesService(
|
||||
conf.GetConfig().KubeHost+":"+conf.GetConfig().KubePort,
|
||||
conf.GetConfig().KubeCA, conf.GetConfig().KubeCert, conf.GetConfig().KubeData,
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsTarget: failed to create k8s service: " + err.Error())
|
||||
m.emitConsiders(event.ExecutionsID, event.OriginID, "", err, event.SourcePeerID == event.DestPeerID)
|
||||
return
|
||||
}
|
||||
|
||||
// Read the access key from the K8s secret before deleting it.
|
||||
accessKey := event.Access
|
||||
if accessKey == "" {
|
||||
if secret, err := k.Set.CoreV1().Secrets(event.ExecutionsID).Get(
|
||||
ctx, event.MinioID+"-secret-s3", metav1.GetOptions{},
|
||||
); err == nil {
|
||||
accessKey = string(secret.Data["access-key"])
|
||||
}
|
||||
}
|
||||
|
||||
// Delete K8s credentials secret.
|
||||
if err := k.Set.CoreV1().Secrets(event.ExecutionsID).Delete(
|
||||
ctx, event.MinioID+"-secret-s3", metav1.DeleteOptions{},
|
||||
); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsTarget: failed to delete secret: " + err.Error())
|
||||
}
|
||||
|
||||
// Delete artifact-repository ConfigMap.
|
||||
if err := NewMinioService("").DeleteMinioConfigMap(event.MinioID, event.ExecutionsID); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsTarget: failed to delete configmap: " + err.Error())
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.TeardownAsTarget: K8s resources removed for " + event.ExecutionsID)
|
||||
|
||||
// For same-peer deployments the source cleanup runs directly here so the
|
||||
// caller (REMOVE_EXECUTION handler) doesn't have to distinguish roles.
|
||||
if event.SourcePeerID == event.DestPeerID {
|
||||
event.Access = accessKey
|
||||
m.TeardownAsSource(ctx, event)
|
||||
}
|
||||
}
|
||||
|
||||
// TeardownAsSource is called on the peer that hosts the Minio instance.
|
||||
// It revokes the scoped service account and removes the execution bucket.
|
||||
func (m *MinioSetter) TeardownAsSource(ctx context.Context, event MinioDeleteEvent) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
url, err := m.loadMinioURL(event.SourcePeerID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
svc := NewMinioService(url)
|
||||
if err := svc.CreateClient(); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: failed to create admin client: " + err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
if event.Access != "" {
|
||||
if err := svc.DeleteCredentials(event.Access); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: failed to delete service account: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if err := svc.DeleteBucket(event.MinioID, event.ExecutionsID); err != nil {
|
||||
logger.Error().Msg("MinioSetter.TeardownAsSource: failed to delete bucket: " + err.Error())
|
||||
}
|
||||
|
||||
logger.Info().Msg("MinioSetter.TeardownAsSource: Minio resources removed for " + event.ExecutionsID)
|
||||
}
|
||||
|
||||
// loadMinioURL searches through all live storages accessible by peerID to find
|
||||
// the one that references MinioID, and returns its endpoint URL.
|
||||
func (m *MinioSetter) loadMinioURL(peerID string) (string, error) {
|
||||
res := oclib.NewRequest(oclib.LibDataEnum(oclib.LIVE_STORAGE), "", peerID, []string{}, nil).LoadAll(false)
|
||||
if res.Err != "" {
|
||||
return "", fmt.Errorf("loadMinioURL: failed to load live storages: %s", res.Err)
|
||||
}
|
||||
for _, dbo := range res.Data {
|
||||
l := dbo.(*live.LiveStorage)
|
||||
if slices.Contains(l.ResourcesID, m.MinioID) {
|
||||
return l.Source, nil
|
||||
}
|
||||
|
||||
}
|
||||
return "", fmt.Errorf("loadMinioURL: no live storage found for minio ID %s", m.MinioID)
|
||||
}
|
||||
|
||||
// teardownMinioForExecution tears down all Minio configuration for the execution:
|
||||
// - storage bookings where this peer is the compute target → TeardownAsTarget
|
||||
// - storage bookings where this peer is the Minio source → TeardownAsSource
|
||||
func (m *MinioSetter) TeardownForExecution(ctx context.Context, localPeerID string) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
res := oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), "", localPeerID, []string{}, nil).
|
||||
Search(&dbs.Filters{
|
||||
And: map[string][]dbs.Filter{
|
||||
"executions_id": {{Operator: dbs.EQUAL.String(), Value: m.ExecutionsID}},
|
||||
"resource_type": {{Operator: dbs.EQUAL.String(), Value: tools.LIVE_STORAGE.EnumIndex()}},
|
||||
},
|
||||
}, "", false)
|
||||
|
||||
if res.Err != "" || len(res.Data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, dbo := range res.Data {
|
||||
b, ok := dbo.(*bookingmodel.Booking)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if b.DestPeerID == localPeerID {
|
||||
// This peer is the compute target: tear down K8s secret + configmap.
|
||||
logger.Info().Msgf("InfraTeardown: Minio target teardown exec=%s storage=%s", m.ExecutionsID, b.ResourceID)
|
||||
event := MinioDeleteEvent{
|
||||
ExecutionsID: m.ExecutionsID,
|
||||
MinioID: b.ResourceID,
|
||||
SourcePeerID: b.DestPeerID,
|
||||
DestPeerID: localPeerID,
|
||||
OriginID: "",
|
||||
}
|
||||
m.TeardownAsTarget(ctx, event)
|
||||
} else {
|
||||
// This peer is the Minio source: revoke SA + remove execution bucket.
|
||||
logger.Info().Msgf("InfraTeardown: Minio source teardown exec=%s storage=%s", m.ExecutionsID, b.ResourceID)
|
||||
event := MinioDeleteEvent{
|
||||
ExecutionsID: m.ExecutionsID,
|
||||
MinioID: b.ResourceID,
|
||||
SourcePeerID: localPeerID,
|
||||
DestPeerID: b.DestPeerID,
|
||||
OriginID: "",
|
||||
}
|
||||
m.TeardownAsSource(ctx, event)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"oc-datacenter/conf"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/dbs"
|
||||
bookingmodel "cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/live"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
)
|
||||
@@ -44,13 +46,22 @@ func ClaimName(storageName, executionsID string) string {
|
||||
type PVCSetter struct {
|
||||
ExecutionsID string
|
||||
StorageID string
|
||||
// ClaimSuffix overrides ExecutionsID as the suffix in ClaimName when non-empty.
|
||||
// Used when the PVC namespace differs from the claim name suffix (Admiralty target).
|
||||
ClaimSuffix string
|
||||
}
|
||||
|
||||
func NewPVCSetter(execID, storageID string) *PVCSetter {
|
||||
return &PVCSetter{ExecutionsID: execID, StorageID: storageID}
|
||||
}
|
||||
|
||||
func emitConsiders(executionsID, originID string, provErr error, self bool) {
|
||||
// NewPVCSetterWithClaimSuffix creates a PVCSetter where the claim name suffix
|
||||
// differs from the execution namespace (e.g. Admiralty target provisioning).
|
||||
func NewPVCSetterWithClaimSuffix(storageID, claimSuffix string) *PVCSetter {
|
||||
return &PVCSetter{StorageID: storageID, ClaimSuffix: claimSuffix}
|
||||
}
|
||||
|
||||
func (p *PVCSetter) emitConsiders(executionsID, originID string, provErr error, self bool) {
|
||||
type pvcConsidersPayload struct {
|
||||
OriginID string `json:"origin_id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
@@ -96,7 +107,7 @@ func (p *PVCSetter) InitializeAsSource(ctx context.Context, event PVCProvisionEv
|
||||
sizeStr, err := p.loadStorageSize(event.SourcePeerID)
|
||||
if err != nil {
|
||||
logger.Error().Msg("PVCSetter.InitializeAsSource: " + err.Error())
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, err, self)
|
||||
p.emitConsiders(event.ExecutionsID, event.OriginID, err, self)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -106,19 +117,23 @@ func (p *PVCSetter) InitializeAsSource(ctx context.Context, event PVCProvisionEv
|
||||
)
|
||||
if err != nil {
|
||||
logger.Error().Msg("PVCSetter.InitializeAsSource: failed to create k8s service: " + err.Error())
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, err, self)
|
||||
p.emitConsiders(event.ExecutionsID, event.OriginID, err, self)
|
||||
return
|
||||
}
|
||||
|
||||
claimName := ClaimName(event.StorageName, event.ExecutionsID)
|
||||
claimSuffix := event.ExecutionsID
|
||||
if p.ClaimSuffix != "" {
|
||||
claimSuffix = p.ClaimSuffix
|
||||
}
|
||||
claimName := ClaimName(event.StorageName, claimSuffix)
|
||||
if err := k.CreatePVC(ctx, claimName, event.ExecutionsID, sizeStr); err != nil {
|
||||
logger.Error().Msg("PVCSetter.InitializeAsSource: failed to create PVC: " + err.Error())
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, err, self)
|
||||
p.emitConsiders(event.ExecutionsID, event.OriginID, err, self)
|
||||
return
|
||||
}
|
||||
|
||||
logger.Info().Msg("PVCSetter.InitializeAsSource: PVC " + claimName + " created in " + event.ExecutionsID)
|
||||
emitConsiders(event.ExecutionsID, event.OriginID, nil, self)
|
||||
p.emitConsiders(event.ExecutionsID, event.OriginID, nil, self)
|
||||
}
|
||||
|
||||
// TeardownAsSource deletes the PVC from the execution namespace.
|
||||
@@ -172,3 +187,44 @@ func (p *PVCSetter) loadStorageSize(peerID string) (string, error) {
|
||||
}
|
||||
return "10Gi", nil
|
||||
}
|
||||
|
||||
// teardownPVCForExecution deletes all local PVCs provisioned for the execution.
|
||||
// It searches LIVE_STORAGE bookings and resolves the storage name via the live storage.
|
||||
func (p *PVCSetter) TeardownForExecution(ctx context.Context, localPeerID string) {
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
res := oclib.NewRequest(oclib.LibDataEnum(oclib.BOOKING), "", localPeerID, []string{}, nil).
|
||||
Search(&dbs.Filters{
|
||||
And: map[string][]dbs.Filter{
|
||||
"executions_id": {{Operator: dbs.EQUAL.String(), Value: p.ExecutionsID}},
|
||||
"resource_type": {{Operator: dbs.EQUAL.String(), Value: tools.LIVE_STORAGE.EnumIndex()}},
|
||||
},
|
||||
}, "", false)
|
||||
|
||||
if res.Err != "" || len(res.Data) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for _, dbo := range res.Data {
|
||||
b, ok := dbo.(*bookingmodel.Booking)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// Resolve storage name from live storage to compute the claim name.
|
||||
storageName := ResolveStorageName(b.ResourceID, localPeerID)
|
||||
if storageName == "" {
|
||||
continue
|
||||
}
|
||||
logger.Info().Msgf("InfraTeardown: PVC teardown exec=%s storage=%s", p.ExecutionsID, b.ResourceID)
|
||||
event := PVCDeleteEvent{
|
||||
ExecutionsID: p.ExecutionsID,
|
||||
StorageID: b.ResourceID,
|
||||
StorageName: storageName,
|
||||
SourcePeerID: localPeerID,
|
||||
DestPeerID: b.DestPeerID,
|
||||
OriginID: "",
|
||||
}
|
||||
p.StorageID = b.ResourceID
|
||||
p.TeardownAsSource(ctx, event)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user