Scheduling Node
This commit is contained in:
343
infrastructure/check.go
Normal file
343
infrastructure/check.go
Normal file
@@ -0,0 +1,343 @@
|
||||
package infrastructure
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking/planner"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Slot availability check
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
checkWindowHours = 5 // how far ahead to scan for a free slot (hours)
|
||||
checkStepMin = 15 // time increment per scan step (minutes)
|
||||
// asapBuffer is the minimum lead time added to time.Now() for as_possible
|
||||
// and WHEN_POSSIBLE bookings. It absorbs NATS propagation + p2p stream
|
||||
// latency so the ExpectedStartDate never arrives already in the past at
|
||||
// the destination peer.
|
||||
asapBuffer = 2 * time.Minute
|
||||
)
|
||||
|
||||
// CheckResult holds the outcome of a slot availability check.
|
||||
type CheckResult struct {
|
||||
Available bool `json:"available"`
|
||||
Start time.Time `json:"start"`
|
||||
End *time.Time `json:"end,omitempty"`
|
||||
// NextSlot is the nearest free slot found within checkWindowHours when
|
||||
// the requested slot is unavailable, or the preferred (conflict-free) slot
|
||||
// when running in preemption mode.
|
||||
NextSlot *time.Time `json:"next_slot,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
// Preemptible is true when the check was run in preemption mode.
|
||||
Preemptible bool `json:"preemptible,omitempty"`
|
||||
// SchedulingID is the session identifier the client must supply to Schedule
|
||||
// in order to confirm the draft bookings created during this Check session.
|
||||
SchedulingID string `json:"scheduling_id,omitempty"`
|
||||
}
|
||||
|
||||
// bookingResource is the minimum info needed to verify a resource against the
|
||||
// planner cache.
|
||||
type bookingResource struct {
|
||||
id string // resource MongoDB _id
|
||||
peerPID string // peer public PeerID (PID) — PlannerCache key
|
||||
instanceID string // resolved from WorkflowSchedule.SelectedInstances
|
||||
}
|
||||
|
||||
// Check verifies that all booking-relevant resources (storage and compute) of
|
||||
// the given workflow have capacity for the requested time slot.
|
||||
//
|
||||
// - asap=true → ignore ws.Start, begin searching from time.Now()
|
||||
// - preemption → always return Available=true but populate Warnings with
|
||||
// conflicts and NextSlot with the nearest conflict-free alternative
|
||||
func (ws *WorkflowSchedule) Check(wfID string, asap bool, preemption bool, request *tools.APIRequest) (*CheckResult, error) {
|
||||
// 1. Load workflow
|
||||
obj, code, err := workflow.NewAccessor(request).LoadOne(wfID)
|
||||
if code != 200 || err != nil {
|
||||
msg := "could not load workflow " + wfID
|
||||
if err != nil {
|
||||
msg += ": " + err.Error()
|
||||
}
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
wf := obj.(*workflow.Workflow)
|
||||
|
||||
// 2. Resolve start
|
||||
start := ws.Start
|
||||
if asap || start.IsZero() {
|
||||
start = time.Now().Add(asapBuffer)
|
||||
}
|
||||
|
||||
// 3. Resolve end – use explicit end/duration or estimate via Planify
|
||||
end := ws.End
|
||||
if end == nil {
|
||||
if ws.DurationS > 0 {
|
||||
e := start.Add(time.Duration(ws.DurationS * float64(time.Second)))
|
||||
end = &e
|
||||
} else {
|
||||
_, longest, _, _, planErr := wf.Planify(
|
||||
start, nil,
|
||||
ws.SelectedInstances, ws.SelectedPartnerships,
|
||||
ws.SelectedBuyings, ws.SelectedStrategies,
|
||||
int(ws.BookingMode), request,
|
||||
)
|
||||
if planErr == nil && longest > 0 {
|
||||
e := start.Add(time.Duration(longest) * time.Second)
|
||||
end = &e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Extract booking-relevant (storage + compute) resources from the graph,
|
||||
// resolving the selected instance for each resource.
|
||||
checkables := collectBookingResources(wf, ws.SelectedInstances)
|
||||
// 5. Check every resource against its peer's planner
|
||||
unavailable, warnings := checkResourceAvailability(checkables, start, end)
|
||||
result := &CheckResult{
|
||||
Start: start,
|
||||
End: end,
|
||||
Warnings: warnings,
|
||||
}
|
||||
|
||||
// 6. Preemption mode: mark as schedulable regardless of conflicts, but
|
||||
// surface warnings and the nearest conflict-free alternative.
|
||||
if preemption {
|
||||
result.Available = true
|
||||
result.Preemptible = true
|
||||
if len(unavailable) > 0 {
|
||||
result.NextSlot = findNextSlot(checkables, start, end, checkWindowHours)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 7. All resources are free
|
||||
if len(unavailable) == 0 {
|
||||
result.Available = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 8. Slot unavailable – locate the nearest free slot within the window
|
||||
result.Available = false
|
||||
result.NextSlot = findNextSlot(checkables, start, end, checkWindowHours)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// collectBookingResources returns unique storage and compute resources from the
|
||||
// workflow graph. For each resource the selected instance ID is resolved from
|
||||
// selectedInstances (the scheduler's SelectedInstances ConfigItem) so the planner
|
||||
// check targets the exact instance chosen by the user.
|
||||
func collectBookingResources(wf *workflow.Workflow, selectedInstances workflow.ConfigItem) map[string]bookingResource {
|
||||
if wf.Graph == nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
result := map[string]bookingResource{}
|
||||
|
||||
// Resolve MongoDB peer _id (DID) → public PeerID (PID) used as PlannerCache key.
|
||||
peerAccess := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil)
|
||||
didToPID := map[string]string{}
|
||||
resolvePID := func(did string) string {
|
||||
if pid, ok := didToPID[did]; ok {
|
||||
return pid
|
||||
}
|
||||
if data := peerAccess.LoadOne(did); data.Data != nil {
|
||||
if p := data.ToPeer(); p != nil {
|
||||
didToPID[did] = p.PeerID
|
||||
return p.PeerID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
resolveInstanceID := func(res interface {
|
||||
GetID() string
|
||||
GetCreatorID() string
|
||||
}) string {
|
||||
idx := selectedInstances.Get(res.GetID())
|
||||
switch r := res.(type) {
|
||||
case *resources.StorageResource:
|
||||
if inst := r.GetSelectedInstance(idx); inst != nil {
|
||||
return inst.GetID()
|
||||
}
|
||||
case *resources.ComputeResource:
|
||||
if inst := r.GetSelectedInstance(idx); inst != nil {
|
||||
return inst.GetID()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsStorage) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
id := res.GetID()
|
||||
if seen[id] {
|
||||
continue
|
||||
}
|
||||
pid := resolvePID(res.GetCreatorID())
|
||||
if pid == "" {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
result[pid] = bookingResource{
|
||||
id: id,
|
||||
peerPID: pid,
|
||||
instanceID: resolveInstanceID(res),
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsCompute) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
id := res.GetID()
|
||||
if seen[id] {
|
||||
continue
|
||||
}
|
||||
pid := resolvePID(res.GetCreatorID())
|
||||
if pid == "" {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
result[pid] = bookingResource{
|
||||
id: id,
|
||||
peerPID: pid,
|
||||
instanceID: resolveInstanceID(res),
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// checkResourceAvailability returns the IDs of unavailable resources and
|
||||
// human-readable warning messages.
|
||||
func checkResourceAvailability(res map[string]bookingResource, start time.Time, end *time.Time) (unavailable []string, warnings []string) {
|
||||
for _, r := range res {
|
||||
plannerMu.RLock()
|
||||
entry := PlannerCache[r.peerPID]
|
||||
plannerMu.RUnlock()
|
||||
if entry == nil || entry.Planner == nil {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"peer %s planner not in cache for resource %s – assuming available", r.peerPID, r.id))
|
||||
continue
|
||||
}
|
||||
if !checkInstance(entry.Planner, r.id, r.instanceID, start, end) {
|
||||
unavailable = append(unavailable, r.id)
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"resource %s is not available in [%s – %s]",
|
||||
r.id, start.Format(time.RFC3339), formatOptTime(end)))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// checkInstance checks availability for the specific instance resolved by the
|
||||
// scheduler. When instanceID is empty (no instance selected / none resolvable),
|
||||
// it falls back to checking all instances known in the planner and returns true
|
||||
// if any one has remaining capacity. Returns true when no capacity is recorded.
|
||||
func checkInstance(p *planner.Planner, resourceID string, instanceID string, start time.Time, end *time.Time) bool {
|
||||
if instanceID != "" {
|
||||
return p.Check(resourceID, instanceID, nil, start, end)
|
||||
}
|
||||
// Fallback: accept if any known instance has free capacity
|
||||
caps, ok := p.Capacities[resourceID]
|
||||
if !ok || len(caps) == 0 {
|
||||
return true // no recorded usage → assume free
|
||||
}
|
||||
for id := range caps {
|
||||
if p.Check(resourceID, id, nil, start, end) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// findNextSlot scans forward from 'from' in checkStepMin increments for up to
|
||||
// windowH hours and returns the first candidate start time at which all
|
||||
// resources are simultaneously free.
|
||||
func findNextSlot(resources map[string]bookingResource, from time.Time, originalEnd *time.Time, windowH int) *time.Time {
|
||||
duration := time.Hour
|
||||
if originalEnd != nil {
|
||||
if d := originalEnd.Sub(from); d > 0 {
|
||||
duration = d
|
||||
}
|
||||
}
|
||||
step := time.Duration(checkStepMin) * time.Minute
|
||||
limit := from.Add(time.Duration(windowH) * time.Hour)
|
||||
for t := from.Add(step); t.Before(limit); t = t.Add(step) {
|
||||
e := t.Add(duration)
|
||||
if unavail, _ := checkResourceAvailability(resources, t, &e); len(unavail) == 0 {
|
||||
return &t
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatOptTime(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "open"
|
||||
}
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// GetWorkflowPeerIDs loads the workflow and returns the deduplicated list of
|
||||
// creator peer IDs for all its storage and compute resources.
|
||||
// These are the peers whose planners must be watched by a check stream.
|
||||
func GetWorkflowPeerIDs(wfID string, request *tools.APIRequest) ([]string, error) {
|
||||
obj, code, err := workflow.NewAccessor(request).LoadOne(wfID)
|
||||
if code != 200 || err != nil {
|
||||
msg := "could not load workflow " + wfID
|
||||
if err != nil {
|
||||
msg += ": " + err.Error()
|
||||
}
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
wf := obj.(*workflow.Workflow)
|
||||
if wf.Graph == nil {
|
||||
return nil, nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var peerIDs []string
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsStorage) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
if id := res.GetCreatorID(); id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
peerIDs = append(peerIDs, id)
|
||||
}
|
||||
}
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsCompute) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
if id := res.GetCreatorID(); id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
peerIDs = append(peerIDs, id)
|
||||
}
|
||||
}
|
||||
realPeersID := []string{}
|
||||
access := oclib.NewRequestAdmin(oclib.LibDataEnum(tools.PEER), nil)
|
||||
for _, id := range peerIDs {
|
||||
if data := access.LoadOne(id); data.Data != nil {
|
||||
realPeersID = append(realPeersID, data.ToPeer().PeerID)
|
||||
}
|
||||
}
|
||||
return realPeersID, nil
|
||||
}
|
||||
197
infrastructure/considers.go
Normal file
197
infrastructure/considers.go
Normal file
@@ -0,0 +1,197 @@
|
||||
package infrastructure
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/models/common/enum"
|
||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
"oc-scheduler/infrastructure/scheduling"
|
||||
)
|
||||
|
||||
type executionConsidersPayload struct {
|
||||
ID string `json:"id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
ExecutionID string `json:"execution_id"`
|
||||
PeerIDs []string `json:"peer_ids"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Per-execution mutex map (replaces the global stateMu)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
var execLocksMu sync.RWMutex
|
||||
var execLocks = map[string]*sync.Mutex{} // executionID → per-execution mutex
|
||||
|
||||
// RegisterExecLock creates a mutex entry for the execution. Called when a new execution draft is persisted.
|
||||
func RegisterExecLock(executionID string) {
|
||||
execLocksMu.Lock()
|
||||
execLocks[executionID] = &sync.Mutex{}
|
||||
execLocksMu.Unlock()
|
||||
}
|
||||
|
||||
// UnregisterExecLock removes the mutex entry. Called on unschedule and execution deletion.
|
||||
func UnregisterExecLock(executionID string) {
|
||||
execLocksMu.Lock()
|
||||
delete(execLocks, executionID)
|
||||
execLocksMu.Unlock()
|
||||
}
|
||||
|
||||
// applyConsidersLocal applies the considers update directly for a confirmed
|
||||
// booking or purchase (bypasses NATS since updateExecutionState resolves the
|
||||
// execution from the resource itself).
|
||||
func applyConsidersLocal(id string, dt tools.DataType) {
|
||||
payload, err := json.Marshal(&executionConsidersPayload{ID: id})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
updateExecutionState(payload, dt)
|
||||
}
|
||||
|
||||
// EmitConsidersExecution broadcasts a Considers / WORKFLOW_EXECUTION message to all
|
||||
// storage and compute peers of wf once the execution has transitioned to SCHEDULED.
|
||||
// Each receiving peer will use it to confirm (IsDraft=false) their local drafts.
|
||||
func EmitConsidersExecution(exec *workflow_execution.WorkflowExecution, wf *workflow.Workflow) {
|
||||
if wf == nil || wf.Graph == nil {
|
||||
return
|
||||
}
|
||||
peerIDs, err := GetWorkflowPeerIDs(wf.GetID(), &tools.APIRequest{Admin: true})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(peerIDs) == 0 {
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(executionConsidersPayload{
|
||||
ID: exec.GetID(),
|
||||
ExecutionID: exec.GetID(),
|
||||
ExecutionsID: exec.ExecutionsID,
|
||||
PeerIDs: peerIDs})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(tools.PropalgationMessage{
|
||||
DataType: int(tools.WORKFLOW_EXECUTION),
|
||||
Action: tools.PB_CONSIDERS,
|
||||
Payload: payload,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: tools.WORKFLOW_EXECUTION,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
|
||||
// updateExecutionState sets BookingsState[id]=true (dt==BOOKING) or
|
||||
// PurchasesState[id]=true (dt==PURCHASE_RESOURCE) on the target execution.
|
||||
// payload must be JSON-encoded {"id":"...", "execution_id":"..."}.
|
||||
func updateExecutionState(payload []byte, dt tools.DataType) {
|
||||
var data executionConsidersPayload
|
||||
if err := json.Unmarshal(payload, &data); err != nil || data.ID == "" {
|
||||
return
|
||||
}
|
||||
schdata := oclib.NewRequestAdmin(oclib.LibDataEnum(dt), nil).LoadOne(data.ID)
|
||||
if schdata.Data == nil {
|
||||
return
|
||||
}
|
||||
sch := scheduling.ToSchedulerObject(dt, schdata.Data)
|
||||
if sch == nil {
|
||||
return
|
||||
}
|
||||
execID := sch.GetExecutionId()
|
||||
|
||||
execLocksMu.RLock()
|
||||
mu := execLocks[execID]
|
||||
execLocksMu.RUnlock()
|
||||
if mu == nil {
|
||||
fmt.Printf("updateExecutionState: no lock for execution %s, skipping\n", execID)
|
||||
return
|
||||
}
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
res, _, err := workflow_execution.NewAccessor(adminReq).LoadOne(execID)
|
||||
if err != nil || res == nil {
|
||||
fmt.Printf("updateExecutionState: could not load execution %s: %v\n", data.ExecutionID, err)
|
||||
return
|
||||
}
|
||||
|
||||
exec := res.(*workflow_execution.WorkflowExecution)
|
||||
fmt.Println("sch.GetExecutionId()", data.ID, exec.BookingsState)
|
||||
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
if exec.BookingsState == nil {
|
||||
exec.BookingsState = map[string]bool{}
|
||||
}
|
||||
exec.BookingsState[data.ID] = true
|
||||
fmt.Println("sch.GetExecutionId()", data.ID)
|
||||
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
if exec.PurchasesState == nil {
|
||||
exec.PurchasesState = map[string]bool{}
|
||||
}
|
||||
exec.PurchasesState[data.ID] = true
|
||||
}
|
||||
allConfirmed := true
|
||||
for _, st := range exec.BookingsState {
|
||||
if !st {
|
||||
allConfirmed = false
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, st := range exec.PurchasesState {
|
||||
if !st {
|
||||
allConfirmed = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allConfirmed {
|
||||
exec.State = enum.SCHEDULED
|
||||
exec.IsDraft = false
|
||||
}
|
||||
if _, _, err := utils.GenericRawUpdateOne(exec, exec.GetID(), workflow_execution.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Printf("updateExecutionState: could not update execution %s: %v\n", sch.GetExecutionId(), err)
|
||||
return
|
||||
}
|
||||
if allConfirmed {
|
||||
// Confirm the order and notify all peers that execution is scheduled.
|
||||
go confirmSessionOrder(exec.ExecutionsID, adminReq)
|
||||
obj, _, err := workflow.NewAccessor(adminReq).LoadOne(exec.WorkflowID)
|
||||
if err == nil && obj != nil {
|
||||
go EmitConsidersExecution(exec, obj.(*workflow.Workflow))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// confirmExecutionDrafts is called when a Considers/WORKFLOW_EXECUTION message
|
||||
// is received from oc-discovery, meaning the originating peer has confirmed the
|
||||
// execution as SCHEDULED. For every booking and purchase ID listed in the
|
||||
// execution's states, we confirm the local draft (IsDraft=false).
|
||||
func confirmExecutionDrafts(payload []byte) {
|
||||
var data executionConsidersPayload
|
||||
if err := json.Unmarshal(payload, &data); err != nil {
|
||||
fmt.Printf("confirmExecutionDrafts: could not parse payload: %v\n", err)
|
||||
return
|
||||
}
|
||||
access := oclib.NewRequestAdmin(oclib.LibDataEnum(tools.WORKFLOW_EXECUTION), nil)
|
||||
d := access.LoadOne(data.ExecutionID)
|
||||
if exec := d.ToWorkflowExecution(); exec != nil {
|
||||
for id := range exec.BookingsState {
|
||||
go confirmResource(id, tools.BOOKING)
|
||||
}
|
||||
for id := range exec.PurchasesState {
|
||||
go confirmResource(id, tools.PURCHASE_RESOURCE)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,155 +5,18 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"oc-scheduler/conf"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/config"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking/planner"
|
||||
"cloud.o-forge.io/core/oc-lib/models/common/enum"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources/purchase_resource"
|
||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow/graph"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
const plannerTTL = 24 * time.Hour
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Planner cache — protected by plannerMu
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// plannerEntry wraps a planner snapshot with refresh-ownership tracking.
|
||||
// At most one check session may be the "refresh owner" of a given peer's
|
||||
// planner at a time: it emits PB_PLANNER to request a fresh snapshot from
|
||||
// oc-discovery and, on close (clean or forced), emits PB_CLOSE_PLANNER to
|
||||
// release the stream. Any subsequent session that needs the same peer's
|
||||
// planner will see Refreshing=true and skip the duplicate request.
|
||||
type plannerEntry struct {
|
||||
Planner *planner.Planner
|
||||
Refreshing bool // true while a PB_PLANNER request is in flight
|
||||
RefreshOwner string // session UUID that initiated the current refresh
|
||||
}
|
||||
|
||||
var plannerMu sync.RWMutex
|
||||
var PlannerCache = map[string]*plannerEntry{}
|
||||
var plannerAddedAt = map[string]time.Time{} // peerID → first-seen timestamp
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subscriber registries — one keyed by peerID, one by workflowID
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
var subsMu sync.RWMutex
|
||||
var plannerSubs = map[string][]chan struct{}{} // peerID → notification channels
|
||||
var workflowSubs = map[string][]chan struct{}{} // workflowID → notification channels
|
||||
|
||||
// SubscribePlannerUpdates registers interest in planner changes for the given
|
||||
// peer IDs. The returned channel receives one struct{} (non-blocking) each time
|
||||
// any of those planners is updated. Call cancel to unregister.
|
||||
func SubscribePlannerUpdates(peerIDs []string) (<-chan struct{}, func()) {
|
||||
return subscribe(&subsMu, plannerSubs, peerIDs)
|
||||
}
|
||||
|
||||
// SubscribeWorkflowUpdates registers interest in workflow modifications for the
|
||||
// given workflow ID. The returned channel is signalled when the workflow changes
|
||||
// (peer list may have grown or shrunk). Call cancel to unregister.
|
||||
func SubscribeWorkflowUpdates(wfID string) (<-chan struct{}, func()) {
|
||||
ch, cancel := subscribe(&subsMu, workflowSubs, []string{wfID})
|
||||
return ch, cancel
|
||||
}
|
||||
|
||||
// subscribe is the generic helper used by both registries.
|
||||
func subscribe(mu *sync.RWMutex, registry map[string][]chan struct{}, keys []string) (<-chan struct{}, func()) {
|
||||
ch := make(chan struct{}, 1)
|
||||
mu.Lock()
|
||||
for _, k := range keys {
|
||||
registry[k] = append(registry[k], ch)
|
||||
}
|
||||
mu.Unlock()
|
||||
cancel := func() {
|
||||
mu.Lock()
|
||||
for _, k := range keys {
|
||||
subs := registry[k]
|
||||
for i, s := range subs {
|
||||
if s == ch {
|
||||
registry[k] = append(subs[:i], subs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
return ch, cancel
|
||||
}
|
||||
|
||||
func notifyPlannerWatchers(peerID string) {
|
||||
notify(&subsMu, plannerSubs, peerID)
|
||||
}
|
||||
|
||||
func notifyWorkflowWatchers(wfID string) {
|
||||
notify(&subsMu, workflowSubs, wfID)
|
||||
}
|
||||
|
||||
func notify(mu *sync.RWMutex, registry map[string][]chan struct{}, key string) {
|
||||
mu.RLock()
|
||||
subs := registry[key]
|
||||
mu.RUnlock()
|
||||
for _, ch := range subs {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// storePlanner inserts or updates the planner snapshot for peerID.
|
||||
// On first insertion it schedules an automatic eviction after plannerTTL.
|
||||
// Existing refresh-ownership state (Refreshing / RefreshOwner) is preserved
|
||||
// so that an in-flight request is not inadvertently reset.
|
||||
// All subscribers interested in this peer are notified.
|
||||
func storePlanner(peerID string, p *planner.Planner) {
|
||||
plannerMu.Lock()
|
||||
entry := PlannerCache[peerID]
|
||||
isNew := entry == nil
|
||||
if isNew {
|
||||
entry = &plannerEntry{}
|
||||
PlannerCache[peerID] = entry
|
||||
plannerAddedAt[peerID] = time.Now()
|
||||
go evictAfter(peerID, plannerTTL)
|
||||
}
|
||||
entry.Planner = p
|
||||
plannerMu.Unlock()
|
||||
notifyPlannerWatchers(peerID)
|
||||
}
|
||||
|
||||
// evictAfter waits ttl from first insertion then deletes the cache entry and
|
||||
// emits PB_CLOSE_PLANNER so oc-discovery stops streaming for this peer.
|
||||
// This is the only path that actually removes an entry from PlannerCache;
|
||||
// session close (ReleaseRefreshOwnership) only resets ownership state.
|
||||
func evictAfter(peerID string, ttl time.Duration) {
|
||||
time.Sleep(ttl)
|
||||
plannerMu.Lock()
|
||||
_, exists := PlannerCache[peerID]
|
||||
if exists {
|
||||
delete(PlannerCache, peerID)
|
||||
delete(plannerAddedAt, peerID)
|
||||
}
|
||||
plannerMu.Unlock()
|
||||
if exists {
|
||||
EmitNATS(peerID, tools.PropalgationMessage{Action: tools.PB_CLOSE_PLANNER})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NATS emission
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -174,275 +37,48 @@ func EmitNATS(peerID string, message tools.PropalgationMessage) {
|
||||
})
|
||||
}
|
||||
|
||||
type executionConsidersPayload struct {
|
||||
ID string `json:"id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
ExecutionID string `json:"execution_id"`
|
||||
PeerIDs []string `json:"peer_ids"`
|
||||
}
|
||||
|
||||
// emitConsiders broadcasts a PROPALGATION_EVENT with the Considers action,
|
||||
// carrying the stored resource ID and its datatype (BOOKING or PURCHASE_RESOURCE).
|
||||
func emitConsiders(id string, executionID string, dt tools.DataType) {
|
||||
access := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.WORKFLOW_EXECUTION), nil)
|
||||
data := access.LoadOne(executionID)
|
||||
if data.ToWorkflowExecution() != nil {
|
||||
exec := data.ToWorkflowExecution()
|
||||
if peers, err := GetWorkflowPeerIDs(exec.WorkflowID, &tools.APIRequest{Admin: true}); err == nil {
|
||||
payload, _ := json.Marshal(&executionConsidersPayload{
|
||||
ID: id,
|
||||
ExecutionsID: exec.ExecutionsID,
|
||||
ExecutionID: executionID,
|
||||
PeerIDs: peers,
|
||||
})
|
||||
b, _ := json.Marshal(tools.PropalgationMessage{
|
||||
DataType: int(dt),
|
||||
Action: tools.PB_CONSIDERS,
|
||||
Payload: payload,
|
||||
})
|
||||
tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: dt,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// EmitConsidersExecution broadcasts a Considers / WORKFLOW_EXECUTION message to all
|
||||
// storage and compute peers of wf once the execution has transitioned to SCHEDULED.
|
||||
// Each receiving peer will use it to confirm (IsDraft=false) their local drafts.
|
||||
func EmitConsidersExecution(exec *workflow_execution.WorkflowExecution, wf *workflow.Workflow) {
|
||||
if wf == nil || wf.Graph == nil {
|
||||
return
|
||||
}
|
||||
peerIDs, err := GetWorkflowPeerIDs(wf.GetID(), &tools.APIRequest{Admin: true})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if len(peerIDs) == 0 {
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(executionConsidersPayload{
|
||||
ID: exec.GetID(),
|
||||
ExecutionID: exec.GetID(),
|
||||
ExecutionsID: exec.ExecutionsID,
|
||||
PeerIDs: peerIDs})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
b, err := json.Marshal(tools.PropalgationMessage{
|
||||
DataType: int(tools.WORKFLOW_EXECUTION),
|
||||
Action: tools.PB_CONSIDERS,
|
||||
Payload: payload,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: tools.WORKFLOW_EXECUTION,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
|
||||
// updateExecutionState sets BookingsState[id]=true (dt==BOOKING) or
|
||||
// PurchasesState[id]=true (dt==PURCHASE_RESOURCE) on the target execution.
|
||||
// payload must be JSON-encoded {"id":"...", "execution_id":"..."}.
|
||||
func updateExecutionState(payload []byte, dt tools.DataType) {
|
||||
var data executionConsidersPayload
|
||||
if err := json.Unmarshal(payload, &data); err != nil || data.ID == "" || data.ExecutionID == "" {
|
||||
return
|
||||
}
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
res, _, err := workflow_execution.NewAccessor(adminReq).LoadOne(data.ExecutionID)
|
||||
if err != nil || res == nil {
|
||||
fmt.Printf("updateExecutionState: could not load execution %s: %v\n", data.ExecutionID, err)
|
||||
return
|
||||
}
|
||||
exec := res.(*workflow_execution.WorkflowExecution)
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
if exec.BookingsState == nil {
|
||||
exec.BookingsState = map[string]bool{}
|
||||
}
|
||||
exec.BookingsState[data.ID] = true
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
if exec.PurchasesState == nil {
|
||||
exec.PurchasesState = map[string]bool{}
|
||||
}
|
||||
exec.PurchasesState[data.ID] = true
|
||||
}
|
||||
found := true
|
||||
for _, st := range exec.BookingsState {
|
||||
if !st {
|
||||
found = false
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, st := range exec.PurchasesState {
|
||||
if !st {
|
||||
found = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if found {
|
||||
exec.State = enum.SCHEDULED
|
||||
}
|
||||
if _, _, err := utils.GenericRawUpdateOne(exec, data.ExecutionID, workflow_execution.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Printf("updateExecutionState: could not update execution %s: %v\n", data.ExecutionID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// confirmExecutionDrafts is called when a Considers/WORKFLOW_EXECUTION message
|
||||
// is received from oc-discovery, meaning the originating peer has confirmed the
|
||||
// execution as SCHEDULED. For every booking and purchase ID listed in the
|
||||
// execution's states, we confirm the local draft (IsDraft=false).
|
||||
func confirmExecutionDrafts(payload []byte) {
|
||||
var data executionConsidersPayload
|
||||
if err := json.Unmarshal(payload, &data); err != nil {
|
||||
fmt.Printf("confirmExecutionDrafts: could not parse payload: %v\n", err)
|
||||
return
|
||||
}
|
||||
access := oclib.NewRequestAdmin(oclib.LibDataEnum(tools.WORKFLOW_EXECUTION), nil)
|
||||
d := access.LoadOne(data.ExecutionID)
|
||||
if exec := d.ToWorkflowExecution(); exec != nil {
|
||||
for id := range exec.BookingsState {
|
||||
go confirmResource(id, tools.BOOKING)
|
||||
}
|
||||
for id := range exec.PurchasesState {
|
||||
go confirmResource(id, tools.PURCHASE_RESOURCE)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NATS listeners
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func ListenNATS() {
|
||||
tools.NewNATSCaller().ListenNats(map[tools.NATSMethod]func(tools.NATSResponse){
|
||||
// Receive planner snapshots pushed by oc-discovery and cache them.
|
||||
// Considers messages:
|
||||
// BOOKING / PURCHASE_RESOURCE → mark the individual resource as
|
||||
// considered in the target WorkflowExecution (BookingsState / PurchasesState).
|
||||
// WORKFLOW_EXECUTION → the execution reached SCHEDULED; confirm all
|
||||
// local draft bookings and purchases listed in its states.
|
||||
tools.PLANNER_EXECUTION: func(resp tools.NATSResponse) {
|
||||
m := map[string]interface{}{}
|
||||
p := planner.Planner{}
|
||||
if err := json.Unmarshal(resp.Payload, &m); err != nil {
|
||||
return
|
||||
}
|
||||
if err := json.Unmarshal(resp.Payload, &p); err != nil {
|
||||
return
|
||||
}
|
||||
storePlanner(fmt.Sprintf("%v", m["peer_id"]), &p)
|
||||
},
|
||||
tools.PROPALGATION_EVENT: func(resp tools.NATSResponse) {
|
||||
if resp.FromApp != "oc-discovery" {
|
||||
return
|
||||
}
|
||||
var prop tools.PropalgationMessage
|
||||
if err := json.Unmarshal(resp.Payload, &prop); err != nil {
|
||||
return
|
||||
}
|
||||
switch prop.Action {
|
||||
case tools.PB_CONSIDERS:
|
||||
switch tools.DataType(prop.DataType) {
|
||||
case tools.BOOKING, tools.PURCHASE_RESOURCE:
|
||||
updateExecutionState(prop.Payload, tools.DataType(prop.DataType))
|
||||
case tools.WORKFLOW_EXECUTION:
|
||||
confirmExecutionDrafts(prop.Payload)
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
// Incoming resource creation events:
|
||||
// - WORKFLOW → refresh peer planner entries and notify CheckStream watchers.
|
||||
// - BOOKING → if destined for us, validate, store as draft, start 10-min
|
||||
// expiry timer, and emit a "considers_booking" response.
|
||||
// - PURCHASE → if destined for us, store as draft, start 10-min expiry
|
||||
// timer, and emit a "considers_purchase" response.
|
||||
tools.REMOVE_RESOURCE: func(resp tools.NATSResponse) {
|
||||
switch resp.Datatype {
|
||||
case tools.WORKFLOW:
|
||||
wf := workflow.Workflow{}
|
||||
if err := json.Unmarshal(resp.Payload, &wf); err != nil {
|
||||
return
|
||||
}
|
||||
notifyWorkflowWatchers(wf.GetID())
|
||||
}
|
||||
},
|
||||
tools.CREATE_RESOURCE: func(resp tools.NATSResponse) {
|
||||
switch resp.Datatype {
|
||||
case tools.WORKFLOW:
|
||||
wf := workflow.Workflow{}
|
||||
if err := json.Unmarshal(resp.Payload, &wf); err != nil {
|
||||
return
|
||||
}
|
||||
broadcastPlanner(&wf)
|
||||
notifyWorkflowWatchers(wf.GetID())
|
||||
case tools.BOOKING:
|
||||
var bk booking.Booking
|
||||
if err := json.Unmarshal(resp.Payload, &bk); err != nil {
|
||||
return
|
||||
}
|
||||
self, err := oclib.GetMySelf()
|
||||
if err != nil || self == nil || bk.DestPeerID != self.GetID() {
|
||||
return
|
||||
}
|
||||
// Reject bookings whose start date is already in the past.
|
||||
if !bk.ExpectedStartDate.IsZero() && bk.ExpectedStartDate.Before(time.Now()) {
|
||||
fmt.Println("ListenNATS: booking start date is in the past, discarding")
|
||||
return
|
||||
}
|
||||
// Verify the slot is free in our planner (if we have one).
|
||||
plannerMu.RLock()
|
||||
selfEntry := PlannerCache[self.PeerID]
|
||||
plannerMu.RUnlock()
|
||||
if selfEntry != nil && selfEntry.Planner != nil && !checkInstance(selfEntry.Planner, bk.ResourceID, bk.InstanceID, bk.ExpectedStartDate, bk.ExpectedEndDate) {
|
||||
fmt.Println("ListenNATS: booking conflicts with local planner, discarding")
|
||||
return
|
||||
}
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
bk.IsDraft = true
|
||||
stored, _, err := booking.NewAccessor(adminReq).StoreOne(&bk)
|
||||
if err != nil {
|
||||
fmt.Println("ListenNATS: could not store booking:", err)
|
||||
return
|
||||
}
|
||||
storedID := stored.GetID()
|
||||
go refreshSelfPlanner(self.PeerID, adminReq)
|
||||
time.AfterFunc(10*time.Minute, func() { draftTimeout(storedID, tools.BOOKING) })
|
||||
go emitConsiders(storedID, stored.(*booking.Booking).ExecutionID, tools.BOOKING)
|
||||
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
var pr purchase_resource.PurchaseResource
|
||||
if err := json.Unmarshal(resp.Payload, &pr); err != nil {
|
||||
return
|
||||
}
|
||||
self, err := oclib.GetMySelf()
|
||||
if err != nil || self == nil || pr.DestPeerID != self.GetID() {
|
||||
return
|
||||
}
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
pr.IsDraft = true
|
||||
stored, _, err := purchase_resource.NewAccessor(adminReq).StoreOne(&pr)
|
||||
if err != nil {
|
||||
fmt.Println("ListenNATS: could not store purchase:", err)
|
||||
return
|
||||
}
|
||||
storedID := stored.GetID()
|
||||
time.AfterFunc(10*time.Minute, func() { draftTimeout(storedID, tools.PURCHASE_RESOURCE) })
|
||||
go emitConsiders(storedID, stored.(*purchase_resource.PurchaseResource).ExecutionID, tools.PURCHASE_RESOURCE)
|
||||
}
|
||||
},
|
||||
tools.PLANNER_EXECUTION: handlePlannerExecution,
|
||||
tools.PROPALGATION_EVENT: handlePropagationEvent,
|
||||
tools.REMOVE_RESOURCE: handleRemoveResource,
|
||||
tools.CREATE_RESOURCE: handleCreateResource,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Confirm channels
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// ListenConfirm opens a direct NATS connection and subscribes to the hardcoded
|
||||
// "confirm_booking" and "confirm_purchase" subjects. It reconnects automatically
|
||||
// if the connection is lost.
|
||||
func ListenConfirm() {
|
||||
natsURL := config.GetConfig().NATSUrl
|
||||
if natsURL == "" {
|
||||
fmt.Println("ListenConfirm: NATS_SERVER not set, skipping confirm listeners")
|
||||
return
|
||||
}
|
||||
for {
|
||||
nc, err := nats.Connect(natsURL)
|
||||
if err != nil {
|
||||
fmt.Println("ListenConfirm: could not connect to NATS:", err)
|
||||
time.Sleep(time.Minute)
|
||||
continue
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go listenConfirmChannel(nc, "confirm_booking", tools.BOOKING, &wg)
|
||||
go listenConfirmChannel(nc, "confirm_purchase", tools.PURCHASE_RESOURCE, &wg)
|
||||
wg.Wait()
|
||||
nc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Draft timeout
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -474,254 +110,9 @@ func draftTimeout(id string, dt tools.DataType) {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Confirm channels
|
||||
// Kubernetes namespace helper
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// confirmResource sets IsDraft=false for a booking or purchase resource.
|
||||
// For bookings it also advances State to SCHEDULED and refreshes the local planner.
|
||||
func confirmResource(id string, dt tools.DataType) {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
res, _, err := booking.NewAccessor(adminReq).LoadOne(id)
|
||||
if err != nil || res == nil {
|
||||
fmt.Printf("confirmResource: could not load booking %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
bk := res.(*booking.Booking)
|
||||
bk.IsDraft = false
|
||||
bk.State = enum.SCHEDULED
|
||||
if _, _, err := utils.GenericRawUpdateOne(bk, id, booking.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Printf("confirmResource: could not confirm booking %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
createNamespace(bk.ExecutionsID) // create Namespace locally
|
||||
self, err := oclib.GetMySelf()
|
||||
if err == nil && self != nil {
|
||||
go refreshSelfPlanner(self.PeerID, adminReq)
|
||||
}
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
res, _, err := purchase_resource.NewAccessor(adminReq).LoadOne(id)
|
||||
if err != nil || res == nil {
|
||||
fmt.Printf("confirmResource: could not load purchase %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
pr := res.(*purchase_resource.PurchaseResource)
|
||||
pr.IsDraft = false
|
||||
if _, _, err := utils.GenericRawUpdateOne(pr, id, purchase_resource.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Printf("confirmResource: could not confirm purchase %s: %v\n", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// listenConfirmChannel subscribes to a NATS subject and calls confirmResource
|
||||
// for each message received. The message body is expected to be the plain
|
||||
// resource ID (UTF-8 string).
|
||||
func listenConfirmChannel(nc *nats.Conn, subject string, dt tools.DataType, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
ch := make(chan *nats.Msg, 64)
|
||||
sub, err := nc.ChanSubscribe(subject, ch)
|
||||
if err != nil {
|
||||
fmt.Printf("listenConfirmChannel: could not subscribe to %s: %v\n", subject, err)
|
||||
return
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
for msg := range ch {
|
||||
confirmResource(string(msg.Data), dt)
|
||||
}
|
||||
}
|
||||
|
||||
// ListenConfirm opens a direct NATS connection and subscribes to the hardcoded
|
||||
// "confirm_booking" and "confirm_purchase" subjects. It reconnects automatically
|
||||
// if the connection is lost.
|
||||
func ListenConfirm() {
|
||||
natsURL := config.GetConfig().NATSUrl
|
||||
if natsURL == "" {
|
||||
fmt.Println("ListenConfirm: NATS_SERVER not set, skipping confirm listeners")
|
||||
return
|
||||
}
|
||||
for {
|
||||
nc, err := nats.Connect(natsURL)
|
||||
if err != nil {
|
||||
fmt.Println("ListenConfirm: could not connect to NATS:", err)
|
||||
time.Sleep(time.Minute)
|
||||
continue
|
||||
}
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go listenConfirmChannel(nc, "confirm_booking", tools.BOOKING, &wg)
|
||||
go listenConfirmChannel(nc, "confirm_purchase", tools.PURCHASE_RESOURCE, &wg)
|
||||
wg.Wait()
|
||||
nc.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Self-planner initialisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// InitSelfPlanner bootstraps our own planner entry at startup.
|
||||
// It waits (with 15-second retries) for our peer record to be present in the
|
||||
// database before generating the first planner snapshot and broadcasting it
|
||||
// on PB_PLANNER. This handles the race between oc-scheduler starting before
|
||||
// oc-peer has fully registered our node.
|
||||
func InitSelfPlanner() {
|
||||
for {
|
||||
self, err := oclib.GetMySelf()
|
||||
if err != nil || self == nil {
|
||||
fmt.Println("InitSelfPlanner: self peer not found yet, retrying in 15s...")
|
||||
time.Sleep(15 * time.Second)
|
||||
continue
|
||||
}
|
||||
refreshSelfPlanner(self.PeerID, &tools.APIRequest{Admin: true})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Self-planner refresh
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// refreshSelfPlanner regenerates the local planner from the current state of
|
||||
// the booking DB, stores it in PlannerCache under our own node UUID, and
|
||||
// broadcasts it on PROPALGATION_EVENT / PB_PLANNER so all listeners (including
|
||||
// oc-discovery) are kept in sync.
|
||||
//
|
||||
// It should be called whenever a booking for our own peer is created, whether
|
||||
// by direct DB insertion (self-peer routing) or upon receiving a CREATE_RESOURCE
|
||||
// BOOKING message from oc-discovery.
|
||||
func refreshSelfPlanner(peerID string, request *tools.APIRequest) {
|
||||
p, err := planner.GenerateShallow(request)
|
||||
if err != nil {
|
||||
fmt.Println("refreshSelfPlanner: could not generate planner:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update the local cache and notify any waiting CheckStream goroutines.
|
||||
storePlanner(peerID, p)
|
||||
|
||||
// Broadcast the updated planner so remote peers (and oc-discovery) can
|
||||
// refresh their view of our availability.
|
||||
type plannerWithPeer struct {
|
||||
PeerID string `json:"peer_id"`
|
||||
*planner.Planner
|
||||
}
|
||||
plannerPayload, err := json.Marshal(plannerWithPeer{PeerID: peerID, Planner: p})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
EmitNATS(peerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_PLANNER,
|
||||
Payload: plannerPayload,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Planner broadcast
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// RequestPlannerRefresh asks oc-discovery for a fresh planner snapshot for
|
||||
// each peer in peerIDs. Only the first session to request a given peer becomes
|
||||
// its "refresh owner": subsequent sessions see Refreshing=true and skip the
|
||||
// duplicate PB_PLANNER emission. Returns the subset of peerIDs for which this
|
||||
// session claimed ownership (needed to release on close).
|
||||
func RequestPlannerRefresh(peerIDs []string, sessionID string) []string {
|
||||
var owned []string
|
||||
for _, peerID := range peerIDs {
|
||||
plannerMu.Lock()
|
||||
entry := PlannerCache[peerID]
|
||||
if entry == nil {
|
||||
entry = &plannerEntry{}
|
||||
PlannerCache[peerID] = entry
|
||||
plannerAddedAt[peerID] = time.Now()
|
||||
go evictAfter(peerID, plannerTTL)
|
||||
}
|
||||
shouldRequest := !entry.Refreshing
|
||||
if shouldRequest {
|
||||
entry.Refreshing = true
|
||||
entry.RefreshOwner = sessionID
|
||||
}
|
||||
plannerMu.Unlock()
|
||||
if shouldRequest {
|
||||
owned = append(owned, peerID)
|
||||
payload, _ := json.Marshal(map[string]any{"peer_id": peerID})
|
||||
EmitNATS(peerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_PLANNER,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
return owned
|
||||
}
|
||||
|
||||
// ReleaseRefreshOwnership is called when a check session closes (clean or
|
||||
// forced). For each peer this session owns, it resets the refresh state and
|
||||
// emits PB_CLOSE_PLANNER so oc-discovery stops the planner stream.
|
||||
// The planner data itself stays in the cache until TTL eviction.
|
||||
func ReleaseRefreshOwnership(peerIDs []string, sessionID string) {
|
||||
for _, peerID := range peerIDs {
|
||||
plannerMu.Lock()
|
||||
if entry := PlannerCache[peerID]; entry != nil && entry.RefreshOwner == sessionID {
|
||||
entry.Refreshing = false
|
||||
entry.RefreshOwner = ""
|
||||
}
|
||||
plannerMu.Unlock()
|
||||
payload, _ := json.Marshal(map[string]any{"peer_id": peerID})
|
||||
EmitNATS(peerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_CLOSE_PLANNER,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// broadcastPlanner iterates the storage and compute peers of the given workflow
|
||||
// and, for each peer not yet in the cache, emits a PB_PLANNER propagation so
|
||||
// downstream consumers (oc-discovery, other schedulers) refresh their state.
|
||||
func broadcastPlanner(wf *workflow.Workflow) {
|
||||
if wf.Graph == nil {
|
||||
return
|
||||
}
|
||||
items := []graph.GraphItem{}
|
||||
items = append(items, wf.GetGraphItems(wf.Graph.IsStorage)...)
|
||||
items = append(items, wf.GetGraphItems(wf.Graph.IsCompute)...)
|
||||
|
||||
seen := []string{}
|
||||
for _, item := range items {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
creatorID := res.GetCreatorID()
|
||||
if slices.Contains(seen, creatorID) {
|
||||
continue
|
||||
}
|
||||
|
||||
data := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil).LoadOne(creatorID)
|
||||
p := data.ToPeer()
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
plannerMu.RLock()
|
||||
cached := PlannerCache[p.PeerID]
|
||||
plannerMu.RUnlock()
|
||||
|
||||
// Only request if no snapshot and no refresh already in flight.
|
||||
if cached == nil || (cached.Planner == nil && !cached.Refreshing) {
|
||||
payload, err := json.Marshal(map[string]interface{}{"peer_id": p.PeerID})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
seen = append(seen, creatorID)
|
||||
EmitNATS(p.PeerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_PLANNER,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createNamespace(ns string) error {
|
||||
/*
|
||||
* This function is used to create a namespace.
|
||||
|
||||
274
infrastructure/nats_handlers.go
Normal file
274
infrastructure/nats_handlers.go
Normal file
@@ -0,0 +1,274 @@
|
||||
package infrastructure
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking/planner"
|
||||
"cloud.o-forge.io/core/oc-lib/models/common/enum"
|
||||
"cloud.o-forge.io/core/oc-lib/models/peer"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources/purchase_resource"
|
||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
func handlePlannerExecution(resp tools.NATSResponse) {
|
||||
m := map[string]interface{}{}
|
||||
p := planner.Planner{}
|
||||
if err := json.Unmarshal(resp.Payload, &m); err != nil {
|
||||
return
|
||||
}
|
||||
if err := json.Unmarshal(resp.Payload, &p); err != nil {
|
||||
return
|
||||
}
|
||||
storePlanner(fmt.Sprintf("%v", m["peer_id"]), &p)
|
||||
}
|
||||
|
||||
func handlePropagationEvent(resp tools.NATSResponse) {
|
||||
if resp.FromApp != "oc-discovery" {
|
||||
return
|
||||
}
|
||||
var prop tools.PropalgationMessage
|
||||
if err := json.Unmarshal(resp.Payload, &prop); err != nil {
|
||||
return
|
||||
}
|
||||
switch prop.Action {
|
||||
case tools.PB_CONSIDERS:
|
||||
fmt.Println("PB_CONSIDERS")
|
||||
switch tools.DataType(prop.DataType) {
|
||||
case tools.BOOKING, tools.PURCHASE_RESOURCE:
|
||||
fmt.Println("updateExecutionState", tools.DataType(prop.DataType))
|
||||
updateExecutionState(prop.Payload, tools.DataType(prop.DataType))
|
||||
case tools.WORKFLOW_EXECUTION:
|
||||
confirmExecutionDrafts(prop.Payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleRemoveResource(resp tools.NATSResponse) {
|
||||
switch resp.Datatype {
|
||||
case tools.WORKFLOW:
|
||||
wf := workflow.Workflow{}
|
||||
if err := json.Unmarshal(resp.Payload, &wf); err != nil {
|
||||
return
|
||||
}
|
||||
notifyWorkflowWatchers(wf.GetID())
|
||||
case tools.BOOKING:
|
||||
var p removeResourcePayload
|
||||
if err := json.Unmarshal(resp.Payload, &p); err != nil {
|
||||
return
|
||||
}
|
||||
self, err := oclib.GetMySelf()
|
||||
if err != nil || self == nil {
|
||||
return
|
||||
}
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
res, _, loadErr := booking.NewAccessor(adminReq).LoadOne(p.ID)
|
||||
if loadErr != nil || res == nil {
|
||||
return
|
||||
}
|
||||
existing := res.(*booking.Booking)
|
||||
if existing.SchedulerPeerID != p.SchedulerPeerID || existing.ExecutionsID != p.ExecutionsID {
|
||||
fmt.Println("ListenNATS REMOVE_RESOURCE booking: auth mismatch, ignoring", p.ID)
|
||||
return
|
||||
}
|
||||
booking.NewAccessor(adminReq).DeleteOne(p.ID)
|
||||
go refreshSelfPlanner(self.PeerID, adminReq)
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
var p removeResourcePayload
|
||||
if err := json.Unmarshal(resp.Payload, &p); err != nil {
|
||||
return
|
||||
}
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
res, _, loadErr := purchase_resource.NewAccessor(adminReq).LoadOne(p.ID)
|
||||
if loadErr != nil || res == nil {
|
||||
return
|
||||
}
|
||||
existing := res.(*purchase_resource.PurchaseResource)
|
||||
if existing.SchedulerPeerID != p.SchedulerPeerID || existing.ExecutionsID != p.ExecutionsID {
|
||||
fmt.Println("ListenNATS REMOVE_RESOURCE purchase: auth mismatch, ignoring", p.ID)
|
||||
return
|
||||
}
|
||||
purchase_resource.NewAccessor(adminReq).DeleteOne(p.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func handleCreateBooking(bk *booking.Booking, self *peer.Peer, adminReq *tools.APIRequest) {
|
||||
// Upsert: if a booking with this ID already exists, verify auth and update.
|
||||
if existing, _, loadErr := booking.NewAccessor(adminReq).LoadOne(bk.GetID()); loadErr == nil && existing != nil {
|
||||
prev := existing.(*booking.Booking)
|
||||
if prev.SchedulerPeerID != bk.SchedulerPeerID || prev.ExecutionsID != bk.ExecutionsID {
|
||||
fmt.Println("ListenNATS CREATE_RESOURCE booking upsert: auth mismatch, ignoring", bk.GetID())
|
||||
return
|
||||
}
|
||||
if !prev.IsDrafted() && bk.IsDraft {
|
||||
// Already confirmed, refuse downgrade.
|
||||
return
|
||||
}
|
||||
// Expired check only on confirmation (IsDraft→false).
|
||||
if !bk.IsDraft && !prev.ExpectedStartDate.IsZero() && prev.ExpectedStartDate.Before(time.Now()) {
|
||||
fmt.Println("ListenNATS CREATE_RESOURCE booking: expired, deleting", bk.GetID())
|
||||
booking.NewAccessor(adminReq).DeleteOne(bk.GetID())
|
||||
return
|
||||
}
|
||||
if _, _, err := utils.GenericRawUpdateOne(bk, bk.GetID(), booking.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Println("ListenNATS CREATE_RESOURCE booking update failed:", err)
|
||||
return
|
||||
}
|
||||
go refreshSelfPlanner(self.PeerID, adminReq)
|
||||
if !bk.IsDraft {
|
||||
go applyConsidersLocal(bk.GetID(), tools.BOOKING)
|
||||
}
|
||||
return
|
||||
}
|
||||
// New booking: standard create flow.
|
||||
if !bk.ExpectedStartDate.IsZero() && bk.ExpectedStartDate.Before(time.Now()) {
|
||||
fmt.Println("ListenNATS: booking start date is in the past, discarding")
|
||||
return
|
||||
}
|
||||
plannerMu.RLock()
|
||||
selfEntry := PlannerCache[self.PeerID]
|
||||
plannerMu.RUnlock()
|
||||
if selfEntry != nil && selfEntry.Planner != nil && !checkInstance(selfEntry.Planner, bk.ResourceID, bk.InstanceID, bk.ExpectedStartDate, bk.ExpectedEndDate) {
|
||||
fmt.Println("ListenNATS: booking conflicts with local planner, discarding")
|
||||
return
|
||||
}
|
||||
bk.IsDraft = true
|
||||
stored, _, err := booking.NewAccessor(adminReq).StoreOne(bk)
|
||||
if err != nil {
|
||||
fmt.Println("ListenNATS: could not store booking:", err)
|
||||
return
|
||||
}
|
||||
storedID := stored.GetID()
|
||||
go refreshSelfPlanner(self.PeerID, adminReq)
|
||||
time.AfterFunc(10*time.Minute, func() { draftTimeout(storedID, tools.BOOKING) })
|
||||
}
|
||||
|
||||
func handleCreatePurchase(pr *purchase_resource.PurchaseResource, self *peer.Peer, adminReq *tools.APIRequest) {
|
||||
if pr.DestPeerID != self.GetID() {
|
||||
return
|
||||
}
|
||||
// Upsert: if a purchase with this ID already exists, verify auth and update.
|
||||
if existing, _, loadErr := purchase_resource.NewAccessor(adminReq).LoadOne(pr.GetID()); loadErr == nil && existing != nil {
|
||||
prev := existing.(*purchase_resource.PurchaseResource)
|
||||
if prev.SchedulerPeerID != pr.SchedulerPeerID || prev.ExecutionsID != pr.ExecutionsID {
|
||||
fmt.Println("ListenNATS CREATE_RESOURCE purchase upsert: auth mismatch, ignoring", pr.GetID())
|
||||
return
|
||||
}
|
||||
if !prev.IsDrafted() && pr.IsDraft {
|
||||
return
|
||||
}
|
||||
if _, _, err := utils.GenericRawUpdateOne(pr, pr.GetID(), purchase_resource.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Println("ListenNATS CREATE_RESOURCE purchase update failed:", err)
|
||||
return
|
||||
}
|
||||
if !pr.IsDraft {
|
||||
go applyConsidersLocal(pr.GetID(), tools.PURCHASE_RESOURCE)
|
||||
}
|
||||
return
|
||||
}
|
||||
// New purchase: standard create flow.
|
||||
pr.IsDraft = true
|
||||
stored, _, err := purchase_resource.NewAccessor(adminReq).StoreOne(pr)
|
||||
if err != nil {
|
||||
fmt.Println("ListenNATS: could not store purchase:", err)
|
||||
return
|
||||
}
|
||||
storedID := stored.GetID()
|
||||
time.AfterFunc(10*time.Minute, func() { draftTimeout(storedID, tools.PURCHASE_RESOURCE) })
|
||||
}
|
||||
|
||||
func handleCreateResource(resp tools.NATSResponse) {
|
||||
switch resp.Datatype {
|
||||
case tools.WORKFLOW:
|
||||
wf := workflow.Workflow{}
|
||||
if err := json.Unmarshal(resp.Payload, &wf); err != nil {
|
||||
return
|
||||
}
|
||||
broadcastPlanner(&wf)
|
||||
notifyWorkflowWatchers(wf.GetID())
|
||||
case tools.BOOKING:
|
||||
var bk booking.Booking
|
||||
if err := json.Unmarshal(resp.Payload, &bk); err != nil {
|
||||
return
|
||||
}
|
||||
self, err := oclib.GetMySelf()
|
||||
/*if err != nil || self == nil || bk.DestPeerID != self.GetID() {
|
||||
return
|
||||
}*/
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
_ = err
|
||||
handleCreateBooking(&bk, self, adminReq)
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
var pr purchase_resource.PurchaseResource
|
||||
if err := json.Unmarshal(resp.Payload, &pr); err != nil {
|
||||
return
|
||||
}
|
||||
self, err := oclib.GetMySelf()
|
||||
if err != nil || self == nil {
|
||||
return
|
||||
}
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
handleCreatePurchase(&pr, self, adminReq)
|
||||
}
|
||||
}
|
||||
|
||||
// confirmResource sets IsDraft=false for a booking or purchase resource.
|
||||
// For bookings it also advances State to SCHEDULED and refreshes the local planner.
|
||||
func confirmResource(id string, dt tools.DataType) {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
res, _, err := booking.NewAccessor(adminReq).LoadOne(id)
|
||||
if err != nil || res == nil {
|
||||
fmt.Printf("confirmResource: could not load booking %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
bk := res.(*booking.Booking)
|
||||
bk.IsDraft = false
|
||||
bk.State = enum.SCHEDULED
|
||||
if _, _, err := utils.GenericRawUpdateOne(bk, id, booking.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Printf("confirmResource: could not confirm booking %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
createNamespace(bk.ExecutionsID) // create Namespace locally
|
||||
self, err := oclib.GetMySelf()
|
||||
if err == nil && self != nil {
|
||||
go refreshSelfPlanner(self.PeerID, adminReq)
|
||||
}
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
res, _, err := purchase_resource.NewAccessor(adminReq).LoadOne(id)
|
||||
if err != nil || res == nil {
|
||||
fmt.Printf("confirmResource: could not load purchase %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
pr := res.(*purchase_resource.PurchaseResource)
|
||||
pr.IsDraft = false
|
||||
if _, _, err := utils.GenericRawUpdateOne(pr, id, purchase_resource.NewAccessor(adminReq)); err != nil {
|
||||
fmt.Printf("confirmResource: could not confirm purchase %s: %v\n", id, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// listenConfirmChannel subscribes to a NATS subject and calls confirmResource
|
||||
// for each message received. The message body is expected to be the plain
|
||||
// resource ID (UTF-8 string).
|
||||
func listenConfirmChannel(nc *nats.Conn, subject string, dt tools.DataType, wg *sync.WaitGroup) {
|
||||
defer wg.Done()
|
||||
ch := make(chan *nats.Msg, 64)
|
||||
sub, err := nc.ChanSubscribe(subject, ch)
|
||||
if err != nil {
|
||||
fmt.Printf("listenConfirmChannel: could not subscribe to %s: %v\n", subject, err)
|
||||
return
|
||||
}
|
||||
defer sub.Unsubscribe()
|
||||
for msg := range ch {
|
||||
confirmResource(string(msg.Data), dt)
|
||||
}
|
||||
}
|
||||
353
infrastructure/planner.go
Normal file
353
infrastructure/planner.go
Normal file
@@ -0,0 +1,353 @@
|
||||
package infrastructure
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking/planner"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow/graph"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
)
|
||||
|
||||
const plannerTTL = 24 * time.Hour
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Planner cache — protected by plannerMu
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// plannerEntry wraps a planner snapshot with refresh-ownership tracking.
|
||||
// At most one check session may be the "refresh owner" of a given peer's
|
||||
// planner at a time: it emits PB_PLANNER to request a fresh snapshot from
|
||||
// oc-discovery and, on close (clean or forced), emits PB_CLOSE_PLANNER to
|
||||
// release the stream. Any subsequent session that needs the same peer's
|
||||
// planner will see Refreshing=true and skip the duplicate request.
|
||||
type plannerEntry struct {
|
||||
Planner *planner.Planner
|
||||
Refreshing bool // true while a PB_PLANNER request is in flight
|
||||
RefreshOwner string // session UUID that initiated the current refresh
|
||||
}
|
||||
|
||||
var plannerMu sync.RWMutex
|
||||
var PlannerCache = map[string]*plannerEntry{}
|
||||
var plannerAddedAt = map[string]time.Time{} // peerID → first-seen timestamp
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Subscriber registries — one keyed by peerID, one by workflowID
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
var subsMu sync.RWMutex
|
||||
var plannerSubs = map[string][]chan string{} // peerID → channels (deliver peerID)
|
||||
var workflowSubs = map[string][]chan struct{}{} // workflowID → notification channels
|
||||
|
||||
// subscribePlanners registers interest in planner changes for the given peer IDs.
|
||||
// The returned channel receives the peerID string (non-blocking) each time any
|
||||
// of those planners is updated. Call cancel to unregister.
|
||||
func subscribePlanners(peerIDs []string) (<-chan string, func()) {
|
||||
ch := make(chan string, 1)
|
||||
subsMu.Lock()
|
||||
for _, k := range peerIDs {
|
||||
plannerSubs[k] = append(plannerSubs[k], ch)
|
||||
}
|
||||
subsMu.Unlock()
|
||||
cancel := func() {
|
||||
subsMu.Lock()
|
||||
for _, k := range peerIDs {
|
||||
subs := plannerSubs[k]
|
||||
for i, s := range subs {
|
||||
if s == ch {
|
||||
plannerSubs[k] = append(subs[:i], subs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
subsMu.Unlock()
|
||||
}
|
||||
return ch, cancel
|
||||
}
|
||||
|
||||
// SubscribePlannerUpdates registers interest in planner changes for the given
|
||||
// peer IDs. The returned channel receives the peerID string (non-blocking) each
|
||||
// time any of those planners is updated. Call cancel to unregister.
|
||||
func SubscribePlannerUpdates(peerIDs []string) (<-chan string, func()) {
|
||||
return subscribePlanners(peerIDs)
|
||||
}
|
||||
|
||||
// SubscribeWorkflowUpdates registers interest in workflow modifications for the
|
||||
// given workflow ID. The returned channel is signalled when the workflow changes
|
||||
// (peer list may have grown or shrunk). Call cancel to unregister.
|
||||
func SubscribeWorkflowUpdates(wfID string) (<-chan struct{}, func()) {
|
||||
ch, cancel := subscribe(&subsMu, workflowSubs, []string{wfID})
|
||||
return ch, cancel
|
||||
}
|
||||
|
||||
// subscribe is the generic helper used by the workflow registry.
|
||||
func subscribe(mu *sync.RWMutex, registry map[string][]chan struct{}, keys []string) (<-chan struct{}, func()) {
|
||||
ch := make(chan struct{}, 1)
|
||||
mu.Lock()
|
||||
for _, k := range keys {
|
||||
registry[k] = append(registry[k], ch)
|
||||
}
|
||||
mu.Unlock()
|
||||
cancel := func() {
|
||||
mu.Lock()
|
||||
for _, k := range keys {
|
||||
subs := registry[k]
|
||||
for i, s := range subs {
|
||||
if s == ch {
|
||||
registry[k] = append(subs[:i], subs[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
mu.Unlock()
|
||||
}
|
||||
return ch, cancel
|
||||
}
|
||||
|
||||
func notifyPlannerWatchers(peerID string) {
|
||||
subsMu.RLock()
|
||||
subs := plannerSubs[peerID]
|
||||
subsMu.RUnlock()
|
||||
for _, ch := range subs {
|
||||
select {
|
||||
case ch <- peerID:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func notifyWorkflowWatchers(wfID string) {
|
||||
notify(&subsMu, workflowSubs, wfID)
|
||||
}
|
||||
|
||||
func notify(mu *sync.RWMutex, registry map[string][]chan struct{}, key string) {
|
||||
mu.RLock()
|
||||
subs := registry[key]
|
||||
mu.RUnlock()
|
||||
for _, ch := range subs {
|
||||
select {
|
||||
case ch <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cache helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// storePlanner inserts or updates the planner snapshot for peerID.
|
||||
// On first insertion it schedules an automatic eviction after plannerTTL.
|
||||
// Existing refresh-ownership state (Refreshing / RefreshOwner) is preserved
|
||||
// so that an in-flight request is not inadvertently reset.
|
||||
// All subscribers interested in this peer are notified.
|
||||
func storePlanner(peerID string, p *planner.Planner) {
|
||||
plannerMu.Lock()
|
||||
entry := PlannerCache[peerID]
|
||||
isNew := entry == nil
|
||||
if isNew {
|
||||
entry = &plannerEntry{}
|
||||
PlannerCache[peerID] = entry
|
||||
plannerAddedAt[peerID] = time.Now()
|
||||
go evictAfter(peerID, plannerTTL)
|
||||
}
|
||||
entry.Planner = p
|
||||
plannerMu.Unlock()
|
||||
notifyPlannerWatchers(peerID)
|
||||
}
|
||||
|
||||
// evictAfter waits ttl from first insertion then deletes the cache entry and
|
||||
// emits PB_CLOSE_PLANNER so oc-discovery stops streaming for this peer.
|
||||
// This is the only path that actually removes an entry from PlannerCache;
|
||||
// session close (ReleaseRefreshOwnership) only resets ownership state.
|
||||
func evictAfter(peerID string, ttl time.Duration) {
|
||||
time.Sleep(ttl)
|
||||
plannerMu.Lock()
|
||||
_, exists := PlannerCache[peerID]
|
||||
if exists {
|
||||
delete(PlannerCache, peerID)
|
||||
delete(plannerAddedAt, peerID)
|
||||
}
|
||||
plannerMu.Unlock()
|
||||
if exists {
|
||||
EmitNATS(peerID, tools.PropalgationMessage{Action: tools.PB_CLOSE_PLANNER})
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Planner refresh / broadcast
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// RequestPlannerRefresh asks oc-discovery for a fresh planner snapshot for
|
||||
// each peer in peerIDs. Only the first session to request a given peer becomes
|
||||
// its "refresh owner": subsequent sessions see Refreshing=true and skip the
|
||||
// duplicate PB_PLANNER emission. Returns the subset of peerIDs for which this
|
||||
// session claimed ownership (needed to release on close).
|
||||
func RequestPlannerRefresh(peerIDs []string, executionsID string) []string {
|
||||
var owned []string
|
||||
for _, peerID := range peerIDs {
|
||||
plannerMu.Lock()
|
||||
entry := PlannerCache[peerID]
|
||||
if entry == nil {
|
||||
entry = &plannerEntry{}
|
||||
PlannerCache[peerID] = entry
|
||||
plannerAddedAt[peerID] = time.Now()
|
||||
go evictAfter(peerID, plannerTTL)
|
||||
}
|
||||
shouldRequest := !entry.Refreshing
|
||||
if shouldRequest {
|
||||
entry.Refreshing = true
|
||||
entry.RefreshOwner = executionsID
|
||||
}
|
||||
plannerMu.Unlock()
|
||||
if shouldRequest {
|
||||
owned = append(owned, peerID)
|
||||
if p, err := oclib.GetMySelf(); err == nil && p != nil && p.PeerID == peerID {
|
||||
// Self peer: generate and cache the planner directly without
|
||||
// going through NATS / oc-discovery.
|
||||
go refreshSelfPlanner(peerID, &tools.APIRequest{Admin: true})
|
||||
} else {
|
||||
payload, _ := json.Marshal(map[string]any{"peer_id": peerID})
|
||||
fmt.Println("PB_PLANNER", peerID)
|
||||
EmitNATS(peerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_PLANNER,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return owned
|
||||
}
|
||||
|
||||
// ReleaseRefreshOwnership is called when a check session closes (clean or
|
||||
// forced). For each peer this session owns, it resets the refresh state and
|
||||
// emits PB_CLOSE_PLANNER so oc-discovery stops the planner stream.
|
||||
// The planner data itself stays in the cache until TTL eviction.
|
||||
func ReleaseRefreshOwnership(peerIDs []string, executionsID string) {
|
||||
for _, peerID := range peerIDs {
|
||||
plannerMu.Lock()
|
||||
if entry := PlannerCache[peerID]; entry != nil && entry.RefreshOwner == executionsID {
|
||||
entry.Refreshing = false
|
||||
entry.RefreshOwner = ""
|
||||
}
|
||||
plannerMu.Unlock()
|
||||
payload, _ := json.Marshal(map[string]any{"peer_id": peerID})
|
||||
EmitNATS(peerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_CLOSE_PLANNER,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// broadcastPlanner iterates the storage and compute peers of the given workflow
|
||||
// and, for each peer not yet in the cache, emits a PB_PLANNER propagation so
|
||||
// downstream consumers (oc-discovery, other schedulers) refresh their state.
|
||||
func broadcastPlanner(wf *workflow.Workflow) {
|
||||
if wf.Graph == nil {
|
||||
return
|
||||
}
|
||||
items := []graph.GraphItem{}
|
||||
items = append(items, wf.GetGraphItems(wf.Graph.IsStorage)...)
|
||||
items = append(items, wf.GetGraphItems(wf.Graph.IsCompute)...)
|
||||
|
||||
seen := []string{}
|
||||
for _, item := range items {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
creatorID := res.GetCreatorID()
|
||||
if slices.Contains(seen, creatorID) {
|
||||
continue
|
||||
}
|
||||
|
||||
data := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil).LoadOne(creatorID)
|
||||
p := data.ToPeer()
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
plannerMu.RLock()
|
||||
cached := PlannerCache[p.PeerID]
|
||||
plannerMu.RUnlock()
|
||||
|
||||
// Only request if no snapshot and no refresh already in flight.
|
||||
if cached == nil || (cached.Planner == nil && !cached.Refreshing) {
|
||||
payload, err := json.Marshal(map[string]interface{}{"peer_id": p.PeerID})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
seen = append(seen, creatorID)
|
||||
EmitNATS(p.PeerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_PLANNER,
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Self-planner initialisation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// InitSelfPlanner bootstraps our own planner entry at startup.
|
||||
// It waits (with 15-second retries) for our peer record to be present in the
|
||||
// database before generating the first planner snapshot and broadcasting it
|
||||
// on PB_PLANNER. This handles the race between oc-scheduler starting before
|
||||
// oc-peer has fully registered our node.
|
||||
func InitSelfPlanner() {
|
||||
for {
|
||||
self, err := oclib.GetMySelf()
|
||||
if err != nil || self == nil {
|
||||
fmt.Println("InitSelfPlanner: self peer not found yet, retrying in 15s...")
|
||||
time.Sleep(15 * time.Second)
|
||||
continue
|
||||
}
|
||||
refreshSelfPlanner(self.PeerID, &tools.APIRequest{Admin: true})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Self-planner refresh
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// refreshSelfPlanner regenerates the local planner from the current state of
|
||||
// the booking DB, stores it in PlannerCache under our own node UUID, and
|
||||
// broadcasts it on PROPALGATION_EVENT / PB_PLANNER so all listeners (including
|
||||
// oc-discovery) are kept in sync.
|
||||
//
|
||||
// It should be called whenever a booking for our own peer is created, whether
|
||||
// by direct DB insertion (self-peer routing) or upon receiving a CREATE_RESOURCE
|
||||
// BOOKING message from oc-discovery.
|
||||
func refreshSelfPlanner(peerID string, request *tools.APIRequest) {
|
||||
p, err := planner.GenerateShallow(request)
|
||||
if err != nil {
|
||||
fmt.Println("refreshSelfPlanner: could not generate planner:", err)
|
||||
return
|
||||
}
|
||||
|
||||
// Update the local cache and notify any waiting CheckStream goroutines.
|
||||
storePlanner(peerID, p)
|
||||
|
||||
// Broadcast the updated planner so remote peers (and oc-discovery) can
|
||||
// refresh their view of our availability.
|
||||
type plannerWithPeer struct {
|
||||
PeerID string `json:"peer_id"`
|
||||
*planner.Planner
|
||||
}
|
||||
plannerPayload, err := json.Marshal(plannerWithPeer{PeerID: peerID, Planner: p})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
EmitNATS(peerID, tools.PropalgationMessage{
|
||||
Action: tools.PB_PLANNER,
|
||||
Payload: plannerPayload,
|
||||
})
|
||||
}
|
||||
@@ -4,18 +4,17 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"oc-scheduler/infrastructure/scheduling"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/models/bill"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking/planner"
|
||||
"cloud.o-forge.io/core/oc-lib/models/common/enum"
|
||||
"cloud.o-forge.io/core/oc-lib/models/common/pricing"
|
||||
"cloud.o-forge.io/core/oc-lib/models/order"
|
||||
"cloud.o-forge.io/core/oc-lib/models/peer"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources/purchase_resource"
|
||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow"
|
||||
@@ -48,6 +47,9 @@ type WorkflowSchedule struct {
|
||||
SelectedStrategies workflow.ConfigItem `json:"selected_strategies"`
|
||||
|
||||
SelectedBillingStrategy pricing.BillingStrategy `json:"selected_billing_strategy"`
|
||||
|
||||
// Confirm, when true, triggers Schedule() to confirm the drafts held by this session.
|
||||
Confirm bool `json:"confirm,omitempty"`
|
||||
}
|
||||
|
||||
// TODO PREEMPTION !
|
||||
@@ -67,7 +69,7 @@ ne pourra se lancé que SI il n'existe pas d'exécution se lançant durant la p
|
||||
func NewScheduler(mode int, start string, end string, durationInS float64, cron string) *WorkflowSchedule {
|
||||
ws := &WorkflowSchedule{
|
||||
UUID: uuid.New().String(),
|
||||
Start: time.Now(),
|
||||
Start: time.Now().Add(asapBuffer),
|
||||
BookingMode: booking.BookingMode(mode),
|
||||
DurationS: durationInS,
|
||||
Cron: cron,
|
||||
@@ -84,21 +86,18 @@ func NewScheduler(mode int, start string, end string, durationInS float64, cron
|
||||
return ws
|
||||
}
|
||||
|
||||
func (ws *WorkflowSchedule) GetBuyAndBook(wfID string, request *tools.APIRequest) (bool, *workflow.Workflow, []*workflow_execution.WorkflowExecution, []*purchase_resource.PurchaseResource, []*booking.Booking, error) {
|
||||
if request.Caller == nil && request.Caller.URLS == nil && request.Caller.URLS[tools.BOOKING] == nil || request.Caller.URLS[tools.BOOKING][tools.GET] == "" {
|
||||
return false, nil, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, errors.New("no caller defined")
|
||||
}
|
||||
func (ws *WorkflowSchedule) GetBuyAndBook(wfID string, request *tools.APIRequest) (bool, *workflow.Workflow, []*workflow_execution.WorkflowExecution, []scheduling.SchedulerObject, []scheduling.SchedulerObject, error) {
|
||||
access := workflow.NewAccessor(request)
|
||||
res, code, err := access.LoadOne(wfID)
|
||||
if code != 200 {
|
||||
return false, nil, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, errors.New("could not load the workflow with id: " + err.Error())
|
||||
return false, nil, []*workflow_execution.WorkflowExecution{}, []scheduling.SchedulerObject{}, []scheduling.SchedulerObject{}, errors.New("could not load the workflow with id: " + err.Error())
|
||||
}
|
||||
wf := res.(*workflow.Workflow)
|
||||
isPreemptible, longest, priceds, wf, err := wf.Planify(ws.Start, ws.End,
|
||||
ws.SelectedInstances, ws.SelectedPartnerships, ws.SelectedBuyings, ws.SelectedStrategies,
|
||||
int(ws.BookingMode), request)
|
||||
if err != nil {
|
||||
return false, wf, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, err
|
||||
return false, wf, []*workflow_execution.WorkflowExecution{}, []scheduling.SchedulerObject{}, []scheduling.SchedulerObject{}, err
|
||||
}
|
||||
ws.DurationS = longest
|
||||
ws.Message = "We estimate that the workflow will start at " + ws.Start.String() + " and last " + fmt.Sprintf("%v", ws.DurationS) + " seconds."
|
||||
@@ -107,101 +106,94 @@ func (ws *WorkflowSchedule) GetBuyAndBook(wfID string, request *tools.APIRequest
|
||||
}
|
||||
execs, err := ws.GetExecutions(wf, isPreemptible)
|
||||
if err != nil {
|
||||
return false, wf, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, err
|
||||
return false, wf, []*workflow_execution.WorkflowExecution{}, []scheduling.SchedulerObject{}, []scheduling.SchedulerObject{}, err
|
||||
}
|
||||
purchased := []*purchase_resource.PurchaseResource{}
|
||||
bookings := []*booking.Booking{}
|
||||
purchased := []scheduling.SchedulerObject{}
|
||||
bookings := []scheduling.SchedulerObject{}
|
||||
for _, exec := range execs {
|
||||
purchased = append(purchased, exec.Buy(ws.SelectedBillingStrategy, ws.UUID, wfID, priceds)...)
|
||||
bookings = append(bookings, exec.Book(ws.UUID, wfID, priceds)...)
|
||||
for _, obj := range exec.Buy(ws.SelectedBillingStrategy, ws.UUID, wfID, priceds) {
|
||||
purchased = append(purchased, scheduling.ToSchedulerObject(tools.PURCHASE_RESOURCE, obj))
|
||||
}
|
||||
for _, obj := range exec.Book(ws.UUID, wfID, priceds) {
|
||||
bookings = append(bookings, scheduling.ToSchedulerObject(tools.BOOKING, obj))
|
||||
}
|
||||
}
|
||||
return true, wf, execs, purchased, bookings, nil
|
||||
}
|
||||
|
||||
func (ws *WorkflowSchedule) GenerateOrder(purchases []*purchase_resource.PurchaseResource, bookings []*booking.Booking, request *tools.APIRequest) error {
|
||||
// GenerateOrder creates a draft order (+ draft bill) for the given purchases and bookings.
|
||||
// Returns the created order ID and any error.
|
||||
func (ws *WorkflowSchedule) GenerateOrder(purchases []scheduling.SchedulerObject, bookings []scheduling.SchedulerObject, executionsID string, request *tools.APIRequest) (string, error) {
|
||||
newOrder := &order.Order{
|
||||
AbstractObject: utils.AbstractObject{
|
||||
Name: "order_" + request.PeerID + "_" + time.Now().UTC().Format("2006-01-02T15:04:05"),
|
||||
IsDraft: true,
|
||||
},
|
||||
ExecutionsID: ws.UUID,
|
||||
Purchases: purchases,
|
||||
Bookings: bookings,
|
||||
ExecutionsID: executionsID,
|
||||
Purchases: []*purchase_resource.PurchaseResource{},
|
||||
Bookings: []*booking.Booking{},
|
||||
Status: enum.PENDING,
|
||||
}
|
||||
if res, _, err := order.NewAccessor(request).StoreOne(newOrder); err == nil {
|
||||
if _, err := bill.DraftFirstBill(res.(*order.Order), request); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
} else {
|
||||
return err
|
||||
for _, purch := range purchases {
|
||||
newOrder.Purchases = append(
|
||||
newOrder.Purchases, scheduling.FromSchedulerObject(tools.PURCHASE_RESOURCE, purch).(*purchase_resource.PurchaseResource))
|
||||
}
|
||||
for _, b := range bookings {
|
||||
newOrder.Bookings = append(
|
||||
newOrder.Bookings, scheduling.FromSchedulerObject(tools.BOOKING, b).(*booking.Booking))
|
||||
}
|
||||
res, _, err := order.NewAccessor(request).StoreOne(newOrder)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err := bill.DraftFirstBill(res.(*order.Order), request); err != nil {
|
||||
return res.GetID(), err
|
||||
}
|
||||
return res.GetID(), nil
|
||||
}
|
||||
|
||||
func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*WorkflowSchedule, *workflow.Workflow, []*workflow_execution.WorkflowExecution, error) {
|
||||
if request == nil {
|
||||
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no request found")
|
||||
}
|
||||
c := request.Caller
|
||||
if c == nil || c.URLS == nil || c.URLS[tools.BOOKING] == nil {
|
||||
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no caller defined")
|
||||
}
|
||||
methods := c.URLS[tools.BOOKING]
|
||||
if _, ok := methods[tools.GET]; !ok {
|
||||
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no path found")
|
||||
}
|
||||
ok, wf, executions, purchases, bookings, err := ws.GetBuyAndBook(wfID, request)
|
||||
ws.WorkflowExecution = executions
|
||||
if !ok || err != nil {
|
||||
return ws, nil, executions, errors.New("could not book the workflow : " + fmt.Sprintf("%v", err))
|
||||
}
|
||||
ws.Workflow = wf
|
||||
|
||||
// Resolve our own peer MongoDB-ID once; used to decide local vs NATS routing.
|
||||
selfID, _ := oclib.GetMySelf()
|
||||
|
||||
errCh := make(chan error, len(purchases))
|
||||
for _, purchase := range purchases {
|
||||
purchase.IsDraft = true
|
||||
go propagateResource(purchase, purchase.DestPeerID, tools.PURCHASE_RESOURCE, selfID, request, errCh)
|
||||
}
|
||||
for i := 0; i < len(purchases); i++ {
|
||||
if err := <-errCh; err != nil {
|
||||
return ws, wf, executions, errors.New("could not propagate purchase: " + fmt.Sprintf("%v", err))
|
||||
// If the client provides a scheduling_id from a Check session, confirm the
|
||||
// pre-created drafts (bookings/purchases). Executions already exist as drafts
|
||||
// and will be confirmed later by the considers mechanism.
|
||||
if ws.UUID != "" {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
|
||||
// Obsolescence check: abort if any session execution's start date has passed.
|
||||
executions := loadSessionExecs(ws.UUID)
|
||||
for _, exec := range executions {
|
||||
if !exec.ExecDate.IsZero() && exec.ExecDate.Before(time.Now()) {
|
||||
return ws, nil, nil, fmt.Errorf("execution %s is obsolete (start date in the past)", exec.GetID())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
errCh = make(chan error, len(bookings))
|
||||
for _, bk := range bookings {
|
||||
bk.IsDraft = true
|
||||
go propagateResource(bk, bk.DestPeerID, tools.BOOKING, selfID, request, errCh)
|
||||
}
|
||||
for i := 0; i < len(bookings); i++ {
|
||||
if err := <-errCh; err != nil {
|
||||
return ws, wf, executions, errors.New("could not propagate booking: " + fmt.Sprintf("%v", err))
|
||||
if err := ConfirmSession(ws.UUID, selfID, request); err != nil {
|
||||
return ws, nil, []*workflow_execution.WorkflowExecution{}, fmt.Errorf("confirm session failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := ws.GenerateOrder(purchases, bookings, request); err != nil {
|
||||
return ws, wf, executions, err
|
||||
}
|
||||
|
||||
fmt.Println("Schedules")
|
||||
for _, exec := range executions {
|
||||
err := exec.PurgeDraft(request)
|
||||
if err != nil {
|
||||
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("purge draft" + fmt.Sprintf("%v", err))
|
||||
for _, exec := range executions {
|
||||
go WatchExecDeadline(exec.GetID(), exec.ExecDate, selfID, request)
|
||||
}
|
||||
exec.StoreDraftDefault()
|
||||
utils.GenericStoreOne(exec, workflow_execution.NewAccessor(request))
|
||||
go EmitConsidersExecution(exec, wf)
|
||||
|
||||
obj, _, _ := workflow.NewAccessor(request).LoadOne(wfID)
|
||||
if obj == nil {
|
||||
return ws, nil, executions, nil
|
||||
}
|
||||
wf := obj.(*workflow.Workflow)
|
||||
ws.Workflow = wf
|
||||
ws.WorkflowExecution = executions
|
||||
wf.GetAccessor(adminReq).UpdateOne(wf.Serialize(wf), wf.GetID())
|
||||
return ws, wf, executions, nil
|
||||
}
|
||||
fmt.Println("Schedules")
|
||||
|
||||
wf.GetAccessor(&tools.APIRequest{Admin: true}).UpdateOne(wf.Serialize(wf), wf.GetID())
|
||||
|
||||
return ws, wf, executions, nil
|
||||
// Schedule must be called from a Check session (ws.UUID set above).
|
||||
// Direct scheduling without a prior Check session is not supported.
|
||||
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no scheduling session: use the Check stream first")
|
||||
}
|
||||
|
||||
// propagateResource routes a purchase or booking to its destination:
|
||||
@@ -210,14 +202,12 @@ func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*
|
||||
// - Otherwise a NATS CREATE_RESOURCE message is emitted so the destination
|
||||
// peer can process it asynchronously.
|
||||
//
|
||||
// The caller is responsible for setting obj.IsDraft = true before calling.
|
||||
// The caller is responsible for setting obj.IsDraft before calling.
|
||||
func propagateResource(obj utils.DBObject, destPeerID string, dt tools.DataType, selfMongoID *peer.Peer, request *tools.APIRequest, errCh chan error) {
|
||||
if selfMongoID == nil {
|
||||
return
|
||||
} // booking or purchase
|
||||
if destPeerID == selfMongoID.GetID() {
|
||||
if _, _, err := obj.GetAccessor(request).StoreOne(obj); err != nil {
|
||||
errCh <- fmt.Errorf("could not store %s locally: %w", dt.String(), err)
|
||||
stored := oclib.NewRequestAdmin(oclib.LibDataEnum(dt), nil).StoreOne(obj.Serialize(obj))
|
||||
if stored.Err != "" || stored.Data == nil {
|
||||
errCh <- fmt.Errorf("could not store %s locally: %s", dt.String(), stored.Err)
|
||||
return
|
||||
}
|
||||
// The planner tracks booking time-slots only; purchases do not affect it.
|
||||
@@ -227,17 +217,32 @@ func propagateResource(obj utils.DBObject, destPeerID string, dt tools.DataType,
|
||||
errCh <- nil
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(obj)
|
||||
m := obj.Serialize(obj)
|
||||
if m["dest_peer_id"] != nil {
|
||||
if data := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil).LoadOne(fmt.Sprintf("%v", m["dest_peer_id"])); data.Data != nil {
|
||||
m["peer_id"] = data.Data.(*peer.Peer).PeerID
|
||||
}
|
||||
} else {
|
||||
fmt.Println("NO DEST ID")
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
errCh <- fmt.Errorf("could not serialize %s: %w", dt.String(), err)
|
||||
return
|
||||
}
|
||||
tools.NewNATSCaller().SetNATSPub(tools.CREATE_RESOURCE, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: dt,
|
||||
Method: int(tools.CREATE_RESOURCE),
|
||||
if b, err := json.Marshal(&tools.PropalgationMessage{
|
||||
DataType: dt.EnumIndex(),
|
||||
Action: tools.PB_CREATE,
|
||||
Payload: payload,
|
||||
})
|
||||
}); err == nil {
|
||||
tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: dt,
|
||||
Method: int(tools.PROPALGATION_EVENT),
|
||||
Payload: b,
|
||||
})
|
||||
}
|
||||
errCh <- nil
|
||||
}
|
||||
|
||||
@@ -335,303 +340,3 @@ type Schedule struct {
|
||||
* TODO : LARGEST GRAIN PLANIFYING THE WORKFLOW WHEN OPTION IS SET
|
||||
* SET PROTECTION BORDER TIME
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Slot availability check
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
checkWindowHours = 5 // how far ahead to scan for a free slot (hours)
|
||||
checkStepMin = 15 // time increment per scan step (minutes)
|
||||
)
|
||||
|
||||
// CheckResult holds the outcome of a slot availability check.
|
||||
type CheckResult struct {
|
||||
Available bool `json:"available"`
|
||||
Start time.Time `json:"start"`
|
||||
End *time.Time `json:"end,omitempty"`
|
||||
// NextSlot is the nearest free slot found within checkWindowHours when
|
||||
// the requested slot is unavailable, or the preferred (conflict-free) slot
|
||||
// when running in preemption mode.
|
||||
NextSlot *time.Time `json:"next_slot,omitempty"`
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
// Preemptible is true when the check was run in preemption mode.
|
||||
Preemptible bool `json:"preemptible,omitempty"`
|
||||
}
|
||||
|
||||
// bookingResource is the minimum info needed to verify a resource against the
|
||||
// planner cache.
|
||||
type bookingResource struct {
|
||||
id string
|
||||
peerID string
|
||||
instanceID string // resolved from WorkflowSchedule.SelectedInstances
|
||||
}
|
||||
|
||||
// Check verifies that all booking-relevant resources (storage and compute) of
|
||||
// the given workflow have capacity for the requested time slot.
|
||||
//
|
||||
// - asap=true → ignore ws.Start, begin searching from time.Now()
|
||||
// - preemption → always return Available=true but populate Warnings with
|
||||
// conflicts and NextSlot with the nearest conflict-free alternative
|
||||
func (ws *WorkflowSchedule) Check(wfID string, asap bool, preemption bool, request *tools.APIRequest) (*CheckResult, error) {
|
||||
// 1. Load workflow
|
||||
obj, code, err := workflow.NewAccessor(request).LoadOne(wfID)
|
||||
if code != 200 || err != nil {
|
||||
msg := "could not load workflow " + wfID
|
||||
if err != nil {
|
||||
msg += ": " + err.Error()
|
||||
}
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
wf := obj.(*workflow.Workflow)
|
||||
|
||||
// 2. Resolve start
|
||||
start := ws.Start
|
||||
if asap || start.IsZero() {
|
||||
start = time.Now()
|
||||
}
|
||||
|
||||
// 3. Resolve end – use explicit end/duration or estimate via Planify
|
||||
end := ws.End
|
||||
if end == nil {
|
||||
if ws.DurationS > 0 {
|
||||
e := start.Add(time.Duration(ws.DurationS * float64(time.Second)))
|
||||
end = &e
|
||||
} else {
|
||||
_, longest, _, _, planErr := wf.Planify(
|
||||
start, nil,
|
||||
ws.SelectedInstances, ws.SelectedPartnerships,
|
||||
ws.SelectedBuyings, ws.SelectedStrategies,
|
||||
int(ws.BookingMode), request,
|
||||
)
|
||||
if planErr == nil && longest > 0 {
|
||||
e := start.Add(time.Duration(longest) * time.Second)
|
||||
end = &e
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Extract booking-relevant (storage + compute) resources from the graph,
|
||||
// resolving the selected instance for each resource.
|
||||
checkables := collectBookingResources(wf, ws.SelectedInstances)
|
||||
fmt.Println(checkables)
|
||||
// 5. Check every resource against its peer's planner
|
||||
unavailable, warnings := checkResourceAvailability(checkables, start, end)
|
||||
fmt.Println(unavailable, warnings)
|
||||
result := &CheckResult{
|
||||
Start: start,
|
||||
End: end,
|
||||
Warnings: warnings,
|
||||
}
|
||||
|
||||
// 6. Preemption mode: mark as schedulable regardless of conflicts, but
|
||||
// surface warnings and the nearest conflict-free alternative.
|
||||
if preemption {
|
||||
result.Available = true
|
||||
result.Preemptible = true
|
||||
if len(unavailable) > 0 {
|
||||
result.NextSlot = findNextSlot(checkables, start, end, checkWindowHours)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 7. All resources are free
|
||||
if len(unavailable) == 0 {
|
||||
result.Available = true
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 8. Slot unavailable – locate the nearest free slot within the window
|
||||
result.Available = false
|
||||
result.NextSlot = findNextSlot(checkables, start, end, checkWindowHours)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// collectBookingResources returns unique storage and compute resources from the
|
||||
// workflow graph. For each resource the selected instance ID is resolved from
|
||||
// selectedInstances (the scheduler's SelectedInstances ConfigItem) so the planner
|
||||
// check targets the exact instance chosen by the user.
|
||||
func collectBookingResources(wf *workflow.Workflow, selectedInstances workflow.ConfigItem) []bookingResource {
|
||||
if wf.Graph == nil {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var result []bookingResource
|
||||
|
||||
resolveInstanceID := func(res interface {
|
||||
GetID() string
|
||||
GetCreatorID() string
|
||||
}) string {
|
||||
idx := selectedInstances.Get(res.GetID())
|
||||
switch r := res.(type) {
|
||||
case *resources.StorageResource:
|
||||
if inst := r.GetSelectedInstance(idx); inst != nil {
|
||||
return inst.GetID()
|
||||
}
|
||||
case *resources.ComputeResource:
|
||||
if inst := r.GetSelectedInstance(idx); inst != nil {
|
||||
return inst.GetID()
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsStorage) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
id, peerID := res.GetID(), res.GetCreatorID()
|
||||
if peerID == "" || seen[id] {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
result = append(result, bookingResource{
|
||||
id: id,
|
||||
peerID: peerID,
|
||||
instanceID: resolveInstanceID(res),
|
||||
})
|
||||
}
|
||||
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsCompute) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
id, peerID := res.GetID(), res.GetCreatorID()
|
||||
if peerID == "" || seen[id] {
|
||||
continue
|
||||
}
|
||||
seen[id] = true
|
||||
result = append(result, bookingResource{
|
||||
id: id,
|
||||
peerID: peerID,
|
||||
instanceID: resolveInstanceID(res),
|
||||
})
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// checkResourceAvailability returns the IDs of unavailable resources and
|
||||
// human-readable warning messages.
|
||||
func checkResourceAvailability(res []bookingResource, start time.Time, end *time.Time) (unavailable []string, warnings []string) {
|
||||
for _, r := range res {
|
||||
plannerMu.RLock()
|
||||
entry := PlannerCache[r.peerID]
|
||||
plannerMu.RUnlock()
|
||||
if entry == nil || entry.Planner == nil {
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"peer %s planner not in cache for resource %s – assuming available", r.peerID, r.id))
|
||||
continue
|
||||
}
|
||||
if !checkInstance(entry.Planner, r.id, r.instanceID, start, end) {
|
||||
unavailable = append(unavailable, r.id)
|
||||
warnings = append(warnings, fmt.Sprintf(
|
||||
"resource %s is not available in [%s – %s]",
|
||||
r.id, start.Format(time.RFC3339), formatOptTime(end)))
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// checkInstance checks availability for the specific instance resolved by the
|
||||
// scheduler. When instanceID is empty (no instance selected / none resolvable),
|
||||
// it falls back to checking all instances known in the planner and returns true
|
||||
// if any one has remaining capacity. Returns true when no capacity is recorded.
|
||||
func checkInstance(p *planner.Planner, resourceID string, instanceID string, start time.Time, end *time.Time) bool {
|
||||
if instanceID != "" {
|
||||
return p.Check(resourceID, instanceID, nil, start, end)
|
||||
}
|
||||
// Fallback: accept if any known instance has free capacity
|
||||
caps, ok := p.Capacities[resourceID]
|
||||
if !ok || len(caps) == 0 {
|
||||
return true // no recorded usage → assume free
|
||||
}
|
||||
for id := range caps {
|
||||
if p.Check(resourceID, id, nil, start, end) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// findNextSlot scans forward from 'from' in checkStepMin increments for up to
|
||||
// windowH hours and returns the first candidate start time at which all
|
||||
// resources are simultaneously free.
|
||||
func findNextSlot(resources []bookingResource, from time.Time, originalEnd *time.Time, windowH int) *time.Time {
|
||||
duration := time.Hour
|
||||
if originalEnd != nil {
|
||||
if d := originalEnd.Sub(from); d > 0 {
|
||||
duration = d
|
||||
}
|
||||
}
|
||||
step := time.Duration(checkStepMin) * time.Minute
|
||||
limit := from.Add(time.Duration(windowH) * time.Hour)
|
||||
for t := from.Add(step); t.Before(limit); t = t.Add(step) {
|
||||
e := t.Add(duration)
|
||||
if unavail, _ := checkResourceAvailability(resources, t, &e); len(unavail) == 0 {
|
||||
return &t
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatOptTime(t *time.Time) string {
|
||||
if t == nil {
|
||||
return "open"
|
||||
}
|
||||
return t.Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// GetWorkflowPeerIDs loads the workflow and returns the deduplicated list of
|
||||
// creator peer IDs for all its storage and compute resources.
|
||||
// These are the peers whose planners must be watched by a check stream.
|
||||
func GetWorkflowPeerIDs(wfID string, request *tools.APIRequest) ([]string, error) {
|
||||
obj, code, err := workflow.NewAccessor(request).LoadOne(wfID)
|
||||
if code != 200 || err != nil {
|
||||
msg := "could not load workflow " + wfID
|
||||
if err != nil {
|
||||
msg += ": " + err.Error()
|
||||
}
|
||||
return nil, errors.New(msg)
|
||||
}
|
||||
wf := obj.(*workflow.Workflow)
|
||||
if wf.Graph == nil {
|
||||
return nil, nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var peerIDs []string
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsStorage) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
if id := res.GetCreatorID(); id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
peerIDs = append(peerIDs, id)
|
||||
}
|
||||
}
|
||||
for _, item := range wf.GetGraphItems(wf.Graph.IsCompute) {
|
||||
i := item
|
||||
_, res := i.GetResource()
|
||||
if res == nil {
|
||||
continue
|
||||
}
|
||||
if id := res.GetCreatorID(); id != "" && !seen[id] {
|
||||
seen[id] = true
|
||||
peerIDs = append(peerIDs, id)
|
||||
}
|
||||
}
|
||||
realPeersID := []string{}
|
||||
access := oclib.NewRequestAdmin(oclib.LibDataEnum(tools.PEER), nil)
|
||||
for _, id := range peerIDs {
|
||||
if data := access.LoadOne(id); data.Data != nil {
|
||||
realPeersID = append(realPeersID, data.ToPeer().PeerID)
|
||||
}
|
||||
}
|
||||
return realPeersID, nil
|
||||
}
|
||||
|
||||
142
infrastructure/scheduling/objects.go
Normal file
142
infrastructure/scheduling/objects.go
Normal file
@@ -0,0 +1,142 @@
|
||||
package scheduling
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources/purchase_resource"
|
||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
)
|
||||
|
||||
type SchedulerObject interface {
|
||||
utils.DBObject
|
||||
SetIsDraft(bool)
|
||||
GetKey() string
|
||||
SetSchedulerPeerID(peerID string)
|
||||
SetExecutionsID(ei string)
|
||||
GetDestPeer() string
|
||||
GetPeerSession() string
|
||||
GetExecutionsId() string
|
||||
GetExecutionId() string
|
||||
}
|
||||
|
||||
type ScheduledPurchase struct {
|
||||
purchase_resource.PurchaseResource
|
||||
}
|
||||
|
||||
type ScheduledBooking struct {
|
||||
booking.Booking
|
||||
}
|
||||
|
||||
func FromSchedulerDBObject(dt tools.DataType, obj SchedulerObject) utils.DBObject {
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
o := &booking.Booking{}
|
||||
b, _ := json.Marshal(obj)
|
||||
json.Unmarshal(b, &o)
|
||||
return o
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
o := &purchase_resource.PurchaseResource{}
|
||||
b, _ := json.Marshal(obj)
|
||||
json.Unmarshal(b, &o)
|
||||
return o
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func FromSchedulerObject(dt tools.DataType, obj SchedulerObject) utils.ShallowDBObject {
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
o := &booking.Booking{}
|
||||
b, _ := json.Marshal(obj)
|
||||
json.Unmarshal(b, &o)
|
||||
return o
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
o := &purchase_resource.PurchaseResource{}
|
||||
b, _ := json.Marshal(obj)
|
||||
json.Unmarshal(b, &o)
|
||||
return o
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ToSchedulerObject(dt tools.DataType, obj utils.ShallowDBObject) SchedulerObject {
|
||||
switch dt {
|
||||
case tools.BOOKING:
|
||||
o := &ScheduledBooking{}
|
||||
b, _ := json.Marshal(obj)
|
||||
json.Unmarshal(b, &o)
|
||||
return o
|
||||
case tools.PURCHASE_RESOURCE:
|
||||
o := &ScheduledPurchase{}
|
||||
b, _ := json.Marshal(obj)
|
||||
json.Unmarshal(b, &o)
|
||||
return o
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) GetExecutionId() string {
|
||||
return b.ExecutionID
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) GetExecutionId() string {
|
||||
return b.ExecutionID
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) GetExecutionsId() string {
|
||||
return b.ExecutionsID
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) GetExecutionsId() string {
|
||||
return b.ExecutionsID
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) GetPeerSession() string {
|
||||
return b.SchedulerPeerID
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) GetPeerSession() string {
|
||||
return b.SchedulerPeerID
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) GetDestPeer() string {
|
||||
return b.DestPeerID
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) GetDestPeer() string {
|
||||
return b.DestPeerID
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) GetKey() string {
|
||||
return b.ResourceID + "/" + b.InstanceID + "/" + tools.BOOKING.String()
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) GetKey() string {
|
||||
return b.ResourceID + "/" + b.InstanceID + "/" + tools.PURCHASE_RESOURCE.String()
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) SetIsDraft(ok bool) {
|
||||
b.IsDraft = ok
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) SetIsDraft(ok bool) {
|
||||
b.IsDraft = ok
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) SetSchedulerPeerID(peerID string) {
|
||||
b.SchedulerPeerID = peerID
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) SetSchedulerPeerID(peerID string) {
|
||||
b.SchedulerPeerID = peerID
|
||||
}
|
||||
|
||||
func (b *ScheduledBooking) SetExecutionsID(ei string) {
|
||||
b.ExecutionsID = ei
|
||||
}
|
||||
|
||||
func (b *ScheduledPurchase) SetExecutionsID(ei string) {
|
||||
b.ExecutionsID = ei
|
||||
}
|
||||
345
infrastructure/session.go
Normal file
345
infrastructure/session.go
Normal file
@@ -0,0 +1,345 @@
|
||||
package infrastructure
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"oc-scheduler/infrastructure/scheduling"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"cloud.o-forge.io/core/oc-lib/dbs"
|
||||
"cloud.o-forge.io/core/oc-lib/models/booking"
|
||||
"cloud.o-forge.io/core/oc-lib/models/order"
|
||||
"cloud.o-forge.io/core/oc-lib/models/peer"
|
||||
"cloud.o-forge.io/core/oc-lib/models/resources/purchase_resource"
|
||||
"cloud.o-forge.io/core/oc-lib/models/utils"
|
||||
"cloud.o-forge.io/core/oc-lib/models/workflow_execution"
|
||||
"cloud.o-forge.io/core/oc-lib/tools"
|
||||
)
|
||||
|
||||
// removeResourcePayload is sent via NATS REMOVE_RESOURCE so the receiver can
|
||||
// verify the delete order comes from the original scheduler session.
|
||||
type removeResourcePayload struct {
|
||||
ID string `json:"id"`
|
||||
SchedulerPeerID string `json:"scheduler_peer_id"`
|
||||
ExecutionsID string `json:"executions_id"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// DB helpers — objects are found via executions_id
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func sessionIDFilter(field, id string) *dbs.Filters {
|
||||
return &dbs.Filters{
|
||||
And: map[string][]dbs.Filter{
|
||||
field: {{Operator: dbs.EQUAL.String(), Value: id}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func loadSession(executionsID string, dt tools.DataType) []scheduling.SchedulerObject {
|
||||
results := oclib.NewRequestAdmin(oclib.LibDataEnum(dt), nil).Search(
|
||||
sessionIDFilter("executions_id", executionsID), "", true)
|
||||
out := make([]scheduling.SchedulerObject, 0, len(results.Data))
|
||||
for _, obj := range results.Data {
|
||||
out = append(out, scheduling.ToSchedulerObject(dt, obj))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func loadSessionExecs(executionsID string) []*workflow_execution.WorkflowExecution {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
results, _, _ := workflow_execution.NewAccessor(adminReq).Search(
|
||||
sessionIDFilter("executions_id", executionsID), "", true)
|
||||
out := make([]*workflow_execution.WorkflowExecution, 0, len(results))
|
||||
for _, obj := range results {
|
||||
if exec, ok := obj.(*workflow_execution.WorkflowExecution); ok {
|
||||
out = append(out, exec)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func loadSessionOrder(executionsID string) *order.Order {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
results, _, _ := order.NewAccessor(adminReq).Search(
|
||||
sessionIDFilter("executions_id", executionsID), "", true)
|
||||
for _, obj := range results {
|
||||
if o, ok := obj.(*order.Order); ok {
|
||||
return o
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session upsert
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// UpsertSessionDrafts creates or updates draft bookings/purchases/executions for a
|
||||
// Check session. Existing objects are found via the DB (executions_id).
|
||||
// Called on first successful check and on user date changes.
|
||||
//
|
||||
// - bookings/purchases: upserted by (resourceID, instanceID); stale ones deleted
|
||||
// - executions: replaced on every call (dates may have changed)
|
||||
// - order: created once, updated on subsequent calls
|
||||
func (ws *WorkflowSchedule) UpsertSessionDrafts(wfID, executionsID string, selfID *peer.Peer, request *tools.APIRequest) {
|
||||
_, _, execs, purchases, bookings, err := ws.GetBuyAndBook(wfID, request)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
|
||||
// --- bookings ---
|
||||
existing := map[string]scheduling.SchedulerObject{}
|
||||
seen := map[string]bool{}
|
||||
for dt, datas := range map[tools.DataType][]scheduling.SchedulerObject{
|
||||
tools.BOOKING: bookings, tools.PURCHASE_RESOURCE: purchases,
|
||||
} {
|
||||
for _, bk := range loadSession(executionsID, dt) {
|
||||
existing[bk.GetKey()] = bk
|
||||
}
|
||||
upsertSessionDrafts(dt, datas, existing, seen, selfID, executionsID, request)
|
||||
for key, prev := range existing {
|
||||
if !seen[key] {
|
||||
deleteScheduling(dt, prev, selfID, request)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// --- executions: replace on every call (dates may have changed) ---
|
||||
for _, old := range loadSessionExecs(executionsID) {
|
||||
UnregisterExecLock(old.GetID())
|
||||
workflow_execution.NewAccessor(adminReq).DeleteOne(old.GetID())
|
||||
}
|
||||
for _, exec := range execs {
|
||||
exec.ExecutionsID = executionsID
|
||||
exec.IsDraft = true
|
||||
ex, _, err := utils.GenericStoreOne(exec, workflow_execution.NewAccessor(adminReq))
|
||||
if err == nil {
|
||||
RegisterExecLock(ex.GetID())
|
||||
go WatchExecDeadline(ex.GetID(), exec.ExecDate, selfID, request)
|
||||
}
|
||||
}
|
||||
|
||||
// --- order: create once, update on subsequent calls ---
|
||||
if existing := loadSessionOrder(executionsID); existing == nil {
|
||||
ws.GenerateOrder(purchases, bookings, executionsID, request)
|
||||
} else {
|
||||
for _, purch := range purchases {
|
||||
existing.Purchases = append(
|
||||
existing.Purchases, scheduling.FromSchedulerObject(tools.PURCHASE_RESOURCE, purch).(*purchase_resource.PurchaseResource))
|
||||
}
|
||||
for _, b := range bookings {
|
||||
existing.Bookings = append(
|
||||
existing.Bookings, scheduling.FromSchedulerObject(tools.BOOKING, b).(*booking.Booking))
|
||||
}
|
||||
utils.GenericRawUpdateOne(existing, existing.GetID(), order.NewAccessor(adminReq))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Session lifecycle
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
func upsertSessionDrafts(dt tools.DataType, datas []scheduling.SchedulerObject, existing map[string]scheduling.SchedulerObject,
|
||||
seen map[string]bool, selfID *peer.Peer,
|
||||
executionsID string, request *tools.APIRequest) {
|
||||
fmt.Println("UpsertSessionDrafts", len(datas), len(existing))
|
||||
for _, bk := range datas {
|
||||
bk.SetSchedulerPeerID(selfID.PeerID)
|
||||
bk.SetExecutionsID(executionsID)
|
||||
seen[bk.GetKey()] = true
|
||||
if prev, ok := existing[bk.GetKey()]; ok {
|
||||
bk.SetID(prev.GetID())
|
||||
bk.SetIsDraft(false)
|
||||
// Convert to concrete type (Booking/PurchaseResource) so that
|
||||
// GenericRawUpdateOne serializes the real struct, not the wrapper.
|
||||
propagateWriteResource(
|
||||
scheduling.FromSchedulerDBObject(dt, bk), bk.GetDestPeer(), dt, selfID, request)
|
||||
} else {
|
||||
errCh := make(chan error, 1)
|
||||
propagateResource(scheduling.FromSchedulerDBObject(dt, bk), bk.GetDestPeer(), dt, selfID, request, errCh)
|
||||
<-errCh
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// CleanupSession deletes all draft bookings/purchases/executions/order for a
|
||||
// session (called when the WebSocket closes without a confirm).
|
||||
func CleanupSession(self *peer.Peer, executionsID string, selfID *peer.Peer, request *tools.APIRequest) {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
for _, exec := range loadSessionExecs(executionsID) {
|
||||
UnscheduleExecution(exec.GetID(), selfID, request)
|
||||
workflow_execution.NewAccessor(adminReq).DeleteOne(exec.GetID())
|
||||
}
|
||||
if o := loadSessionOrder(executionsID); o != nil {
|
||||
order.NewAccessor(adminReq).DeleteOne(o.GetID())
|
||||
}
|
||||
}
|
||||
|
||||
// ConfirmSession flips all session drafts to IsDraft=false and propagates them.
|
||||
// The considers mechanism then transitions executions to IsDraft=false once
|
||||
// all remote peers acknowledge.
|
||||
func ConfirmSession(executionsID string, selfID *peer.Peer, request *tools.APIRequest) error {
|
||||
for _, dt := range []tools.DataType{tools.BOOKING, tools.PURCHASE_RESOURCE} {
|
||||
for _, bk := range loadSession(executionsID, dt) {
|
||||
bk.SetIsDraft(false)
|
||||
propagateWriteResource(
|
||||
scheduling.FromSchedulerDBObject(dt, bk), bk.GetDestPeer(), dt, selfID, request)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// confirmSessionOrder sets the order IsDraft=false once all considers are received.
|
||||
func confirmSessionOrder(executionsID string, adminReq *tools.APIRequest) {
|
||||
if o := loadSessionOrder(executionsID); o != nil {
|
||||
o.IsDraft = false
|
||||
utils.GenericRawUpdateOne(o, o.GetID(), order.NewAccessor(adminReq))
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Propagation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// propagateWriteResource routes a booking/purchase write to its destination:
|
||||
// - local peer → DB upsert; emits considers on confirm (IsDraft=false)
|
||||
// - remote peer → NATS CREATE_RESOURCE (receiver upserts)
|
||||
func propagateWriteResource(obj utils.DBObject, destPeerID string, dt tools.DataType, selfID *peer.Peer, request *tools.APIRequest) {
|
||||
if destPeerID == selfID.GetID() {
|
||||
if _, _, err := utils.GenericRawUpdateOne(obj, obj.GetID(), obj.GetAccessor(request)); err != nil {
|
||||
fmt.Printf("propagateWriteResource: local update failed for %s %s: %v\n", dt, obj.GetID(), err)
|
||||
return
|
||||
}
|
||||
if dt == tools.BOOKING {
|
||||
go refreshSelfPlanner(selfID.PeerID, request)
|
||||
}
|
||||
fmt.Println("IS DRAFTED", obj.IsDrafted())
|
||||
if !obj.IsDrafted() {
|
||||
if payload, err := json.Marshal(&executionConsidersPayload{
|
||||
ID: obj.GetID(),
|
||||
}); err == nil {
|
||||
go updateExecutionState(payload, dt)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
payload, err := json.Marshal(obj)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tools.NewNATSCaller().SetNATSPub(tools.CREATE_RESOURCE, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: dt,
|
||||
Method: int(tools.CREATE_RESOURCE),
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
||||
// deleteBooking deletes a booking from its destination peer (local DB or NATS).
|
||||
func deleteScheduling(dt tools.DataType, bk scheduling.SchedulerObject, selfID *peer.Peer, request *tools.APIRequest) {
|
||||
if bk.GetDestPeer() == selfID.GetID() {
|
||||
oclib.NewRequestAdmin(oclib.LibDataEnum(dt), nil).DeleteOne(bk.GetID())
|
||||
go refreshSelfPlanner(selfID.PeerID, request)
|
||||
return
|
||||
}
|
||||
emitNATSRemove(bk.GetID(), bk.GetPeerSession(), bk.GetExecutionsId(), dt)
|
||||
}
|
||||
|
||||
// emitNATSRemove sends a REMOVE_RESOURCE event to the remote peer carrying
|
||||
// auth fields so the receiver can verify the delete is legitimate.
|
||||
func emitNATSRemove(id, schedulerPeerID, executionsID string, dt tools.DataType) {
|
||||
payload, _ := json.Marshal(removeResourcePayload{
|
||||
ID: id,
|
||||
SchedulerPeerID: schedulerPeerID,
|
||||
ExecutionsID: executionsID,
|
||||
})
|
||||
tools.NewNATSCaller().SetNATSPub(tools.REMOVE_RESOURCE, tools.NATSResponse{
|
||||
FromApp: "oc-scheduler",
|
||||
Datatype: dt,
|
||||
Method: int(tools.REMOVE_RESOURCE),
|
||||
Payload: payload,
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Deadline watchers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// WatchExecDeadline purges all unconfirmed bookings/purchases for an execution
|
||||
// one minute before its scheduled start, to avoid stale drafts blocking resources.
|
||||
// If the deadline has already passed (e.g. after a process restart), it fires immediately.
|
||||
func WatchExecDeadline(executionID string, execDate time.Time, selfID *peer.Peer, request *tools.APIRequest) {
|
||||
fmt.Println("WatchExecDeadline")
|
||||
delay := time.Until(execDate.UTC().Add(-1 * time.Minute))
|
||||
if delay <= 0 {
|
||||
go purgeUnconfirmedExecution(executionID, selfID, request)
|
||||
return
|
||||
}
|
||||
time.AfterFunc(delay, func() { purgeUnconfirmedExecution(executionID, selfID, request) })
|
||||
}
|
||||
|
||||
func purgeUnconfirmedExecution(executionID string, selfID *peer.Peer, request *tools.APIRequest) {
|
||||
acc := workflow_execution.NewAccessor(&tools.APIRequest{Admin: true})
|
||||
UnscheduleExecution(executionID, selfID, request)
|
||||
_, _, err := acc.DeleteOne(executionID)
|
||||
fmt.Printf("purgeUnconfirmedExecution: cleaned up resources for execution %s\n", err)
|
||||
}
|
||||
|
||||
// RecoverDraftExecutions is called at startup to restore deadline watchers for
|
||||
// draft executions that survived a process restart. Executions already past
|
||||
// their deadline are purged immediately.
|
||||
func RecoverDraftExecutions() {
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
var selfID *peer.Peer
|
||||
for selfID == nil {
|
||||
selfID, _ = oclib.GetMySelf()
|
||||
if selfID == nil {
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
results, _, _ := workflow_execution.NewAccessor(adminReq).Search(nil, "*", true)
|
||||
for _, obj := range results {
|
||||
exec, ok := obj.(*workflow_execution.WorkflowExecution)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
RegisterExecLock(exec.GetID())
|
||||
go WatchExecDeadline(exec.GetID(), exec.ExecDate, selfID, adminReq)
|
||||
}
|
||||
fmt.Printf("RecoverDraftExecutions: recovered %d draft executions\n", len(results))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Unschedule
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// UnscheduleExecution deletes all bookings for an execution (via PeerBookByGraph)
|
||||
// then deletes the execution itself.
|
||||
func UnscheduleExecution(executionID string, selfID *peer.Peer, request *tools.APIRequest) error {
|
||||
fmt.Println("UnscheduleExecution")
|
||||
adminReq := &tools.APIRequest{Admin: true}
|
||||
res, _, err := workflow_execution.NewAccessor(adminReq).LoadOne(executionID)
|
||||
if err != nil || res == nil {
|
||||
return fmt.Errorf("execution %s not found: %w", executionID, err)
|
||||
}
|
||||
exec := res.(*workflow_execution.WorkflowExecution)
|
||||
for _, byResource := range exec.PeerBookByGraph {
|
||||
for _, bookingIDs := range byResource {
|
||||
for _, bkID := range bookingIDs {
|
||||
bkRes, _, loadErr := booking.NewAccessor(adminReq).LoadOne(bkID)
|
||||
fmt.Println("UnscheduleExecution", bkID, loadErr)
|
||||
if loadErr != nil || bkRes == nil {
|
||||
continue
|
||||
}
|
||||
deleteScheduling(tools.BOOKING, scheduling.ToSchedulerObject(tools.BOOKING, bkRes), selfID, request)
|
||||
}
|
||||
}
|
||||
}
|
||||
workflow_execution.NewAccessor(adminReq).DeleteOne(executionID)
|
||||
UnregisterExecLock(executionID)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user