core/oc-scheduler
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scheduling Node

Browse Source
This commit is contained in:
mr
2026-03-17 11:58:27 +01:00
parent b9df0b2731
commit 7fbc077cb1
20 changed files with 2281 additions and 1504 deletions
Download Patch File Download Diff File Expand all files Collapse all files

479
infrastructure/scheduler.go
View File

@@ -4,18 +4,17 @@ import (
"encoding/json"
"errors"
"fmt"
"oc-scheduler/infrastructure/scheduling"
"strings"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/models/bill"
"cloud.o-forge.io/core/oc-lib/models/booking"
"cloud.o-forge.io/core/oc-lib/models/booking/planner"
"cloud.o-forge.io/core/oc-lib/models/common/enum"
"cloud.o-forge.io/core/oc-lib/models/common/pricing"
"cloud.o-forge.io/core/oc-lib/models/order"
"cloud.o-forge.io/core/oc-lib/models/peer"
"cloud.o-forge.io/core/oc-lib/models/resources"
"cloud.o-forge.io/core/oc-lib/models/resources/purchase_resource"
"cloud.o-forge.io/core/oc-lib/models/utils"
"cloud.o-forge.io/core/oc-lib/models/workflow"
@@ -48,6 +47,9 @@ type WorkflowSchedule struct {
SelectedStrategies workflow.ConfigItem `json:"selected_strategies"`
SelectedBillingStrategy pricing.BillingStrategy `json:"selected_billing_strategy"`
// Confirm, when true, triggers Schedule() to confirm the drafts held by this session.
Confirm bool `json:"confirm,omitempty"`
}
// TODO PREEMPTION !
@@ -67,7 +69,7 @@ ne pourra se lancé que SI il n'existe pas d'exécution se lançant durant la p
func NewScheduler(mode int, start string, end string, durationInS float64, cron string) *WorkflowSchedule {
ws := &WorkflowSchedule{
UUID: uuid.New().String(),
Start: time.Now(),
Start: time.Now().Add(asapBuffer),
BookingMode: booking.BookingMode(mode),
DurationS: durationInS,
Cron: cron,
@@ -84,21 +86,18 @@ func NewScheduler(mode int, start string, end string, durationInS float64, cron
return ws
}
func (ws *WorkflowSchedule) GetBuyAndBook(wfID string, request *tools.APIRequest) (bool, *workflow.Workflow, []*workflow_execution.WorkflowExecution, []*purchase_resource.PurchaseResource, []*booking.Booking, error) {
if request.Caller == nil && request.Caller.URLS == nil && request.Caller.URLS[tools.BOOKING] == nil || request.Caller.URLS[tools.BOOKING][tools.GET] == "" {
return false, nil, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, errors.New("no caller defined")
}
func (ws *WorkflowSchedule) GetBuyAndBook(wfID string, request *tools.APIRequest) (bool, *workflow.Workflow, []*workflow_execution.WorkflowExecution, []scheduling.SchedulerObject, []scheduling.SchedulerObject, error) {
access := workflow.NewAccessor(request)
res, code, err := access.LoadOne(wfID)
if code != 200 {
return false, nil, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, errors.New("could not load the workflow with id: " + err.Error())
return false, nil, []*workflow_execution.WorkflowExecution{}, []scheduling.SchedulerObject{}, []scheduling.SchedulerObject{}, errors.New("could not load the workflow with id: " + err.Error())
}
wf := res.(*workflow.Workflow)
isPreemptible, longest, priceds, wf, err := wf.Planify(ws.Start, ws.End,
ws.SelectedInstances, ws.SelectedPartnerships, ws.SelectedBuyings, ws.SelectedStrategies,
int(ws.BookingMode), request)
if err != nil {
return false, wf, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, err
return false, wf, []*workflow_execution.WorkflowExecution{}, []scheduling.SchedulerObject{}, []scheduling.SchedulerObject{}, err
}
ws.DurationS = longest
ws.Message = "We estimate that the workflow will start at " + ws.Start.String() + " and last " + fmt.Sprintf("%v", ws.DurationS) + " seconds."
@@ -107,101 +106,94 @@ func (ws *WorkflowSchedule) GetBuyAndBook(wfID string, request *tools.APIRequest
}
execs, err := ws.GetExecutions(wf, isPreemptible)
if err != nil {
return false, wf, []*workflow_execution.WorkflowExecution{}, []*purchase_resource.PurchaseResource{}, []*booking.Booking{}, err
return false, wf, []*workflow_execution.WorkflowExecution{}, []scheduling.SchedulerObject{}, []scheduling.SchedulerObject{}, err
}
purchased := []*purchase_resource.PurchaseResource{}
bookings := []*booking.Booking{}
purchased := []scheduling.SchedulerObject{}
bookings := []scheduling.SchedulerObject{}
for _, exec := range execs {
purchased = append(purchased, exec.Buy(ws.SelectedBillingStrategy, ws.UUID, wfID, priceds)...)
bookings = append(bookings, exec.Book(ws.UUID, wfID, priceds)...)
for _, obj := range exec.Buy(ws.SelectedBillingStrategy, ws.UUID, wfID, priceds) {
purchased = append(purchased, scheduling.ToSchedulerObject(tools.PURCHASE_RESOURCE, obj))
}
for _, obj := range exec.Book(ws.UUID, wfID, priceds) {
bookings = append(bookings, scheduling.ToSchedulerObject(tools.BOOKING, obj))
}
}
return true, wf, execs, purchased, bookings, nil
}
func (ws *WorkflowSchedule) GenerateOrder(purchases []*purchase_resource.PurchaseResource, bookings []*booking.Booking, request *tools.APIRequest) error {
// GenerateOrder creates a draft order (+ draft bill) for the given purchases and bookings.
// Returns the created order ID and any error.
func (ws *WorkflowSchedule) GenerateOrder(purchases []scheduling.SchedulerObject, bookings []scheduling.SchedulerObject, executionsID string, request *tools.APIRequest) (string, error) {
newOrder := &order.Order{
AbstractObject: utils.AbstractObject{
Name: "order_" + request.PeerID + "_" + time.Now().UTC().Format("2006-01-02T15:04:05"),
IsDraft: true,
},
ExecutionsID: ws.UUID,
Purchases: purchases,
Bookings: bookings,
ExecutionsID: executionsID,
Purchases: []*purchase_resource.PurchaseResource{},
Bookings: []*booking.Booking{},
Status: enum.PENDING,
}
if res, _, err := order.NewAccessor(request).StoreOne(newOrder); err == nil {
if _, err := bill.DraftFirstBill(res.(*order.Order), request); err != nil {
return err
}
return nil
} else {
return err
for _, purch := range purchases {
newOrder.Purchases = append(
newOrder.Purchases, scheduling.FromSchedulerObject(tools.PURCHASE_RESOURCE, purch).(*purchase_resource.PurchaseResource))
}
for _, b := range bookings {
newOrder.Bookings = append(
newOrder.Bookings, scheduling.FromSchedulerObject(tools.BOOKING, b).(*booking.Booking))
}
res, _, err := order.NewAccessor(request).StoreOne(newOrder)
if err != nil {
return "", err
}
if _, err := bill.DraftFirstBill(res.(*order.Order), request); err != nil {
return res.GetID(), err
}
return res.GetID(), nil
}
func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*WorkflowSchedule, *workflow.Workflow, []*workflow_execution.WorkflowExecution, error) {
if request == nil {
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no request found")
}
c := request.Caller
if c == nil || c.URLS == nil || c.URLS[tools.BOOKING] == nil {
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no caller defined")
}
methods := c.URLS[tools.BOOKING]
if _, ok := methods[tools.GET]; !ok {
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no path found")
}
ok, wf, executions, purchases, bookings, err := ws.GetBuyAndBook(wfID, request)
ws.WorkflowExecution = executions
if !ok || err != nil {
return ws, nil, executions, errors.New("could not book the workflow : " + fmt.Sprintf("%v", err))
}
ws.Workflow = wf
// Resolve our own peer MongoDB-ID once; used to decide local vs NATS routing.
selfID, _ := oclib.GetMySelf()
errCh := make(chan error, len(purchases))
for _, purchase := range purchases {
purchase.IsDraft = true
go propagateResource(purchase, purchase.DestPeerID, tools.PURCHASE_RESOURCE, selfID, request, errCh)
}
for i := 0; i < len(purchases); i++ {
if err := <-errCh; err != nil {
return ws, wf, executions, errors.New("could not propagate purchase: " + fmt.Sprintf("%v", err))
// If the client provides a scheduling_id from a Check session, confirm the
// pre-created drafts (bookings/purchases). Executions already exist as drafts
// and will be confirmed later by the considers mechanism.
if ws.UUID != "" {
adminReq := &tools.APIRequest{Admin: true}
// Obsolescence check: abort if any session execution's start date has passed.
executions := loadSessionExecs(ws.UUID)
for _, exec := range executions {
if !exec.ExecDate.IsZero() && exec.ExecDate.Before(time.Now()) {
return ws, nil, nil, fmt.Errorf("execution %s is obsolete (start date in the past)", exec.GetID())
}
}
}
errCh = make(chan error, len(bookings))
for _, bk := range bookings {
bk.IsDraft = true
go propagateResource(bk, bk.DestPeerID, tools.BOOKING, selfID, request, errCh)
}
for i := 0; i < len(bookings); i++ {
if err := <-errCh; err != nil {
return ws, wf, executions, errors.New("could not propagate booking: " + fmt.Sprintf("%v", err))
if err := ConfirmSession(ws.UUID, selfID, request); err != nil {
return ws, nil, []*workflow_execution.WorkflowExecution{}, fmt.Errorf("confirm session failed: %w", err)
}
}
if err := ws.GenerateOrder(purchases, bookings, request); err != nil {
return ws, wf, executions, err
}
fmt.Println("Schedules")
for _, exec := range executions {
err := exec.PurgeDraft(request)
if err != nil {
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("purge draft" + fmt.Sprintf("%v", err))
for _, exec := range executions {
go WatchExecDeadline(exec.GetID(), exec.ExecDate, selfID, request)
}
exec.StoreDraftDefault()
utils.GenericStoreOne(exec, workflow_execution.NewAccessor(request))
go EmitConsidersExecution(exec, wf)
obj, _, _ := workflow.NewAccessor(request).LoadOne(wfID)
if obj == nil {
return ws, nil, executions, nil
}
wf := obj.(*workflow.Workflow)
ws.Workflow = wf
ws.WorkflowExecution = executions
wf.GetAccessor(adminReq).UpdateOne(wf.Serialize(wf), wf.GetID())
return ws, wf, executions, nil
}
fmt.Println("Schedules")
wf.GetAccessor(&tools.APIRequest{Admin: true}).UpdateOne(wf.Serialize(wf), wf.GetID())
return ws, wf, executions, nil
// Schedule must be called from a Check session (ws.UUID set above).
// Direct scheduling without a prior Check session is not supported.
return ws, nil, []*workflow_execution.WorkflowExecution{}, errors.New("no scheduling session: use the Check stream first")
}
// propagateResource routes a purchase or booking to its destination:
@@ -210,14 +202,12 @@ func (ws *WorkflowSchedule) Schedules(wfID string, request *tools.APIRequest) (*
// - Otherwise a NATS CREATE_RESOURCE message is emitted so the destination
// peer can process it asynchronously.
//
// The caller is responsible for setting obj.IsDraft = true before calling.
// The caller is responsible for setting obj.IsDraft before calling.
func propagateResource(obj utils.DBObject, destPeerID string, dt tools.DataType, selfMongoID *peer.Peer, request *tools.APIRequest, errCh chan error) {
if selfMongoID == nil {
return
} // booking or purchase
if destPeerID == selfMongoID.GetID() {
if _, _, err := obj.GetAccessor(request).StoreOne(obj); err != nil {
errCh <- fmt.Errorf("could not store %s locally: %w", dt.String(), err)
stored := oclib.NewRequestAdmin(oclib.LibDataEnum(dt), nil).StoreOne(obj.Serialize(obj))
if stored.Err != "" || stored.Data == nil {
errCh <- fmt.Errorf("could not store %s locally: %s", dt.String(), stored.Err)
return
}
// The planner tracks booking time-slots only; purchases do not affect it.
@@ -227,17 +217,32 @@ func propagateResource(obj utils.DBObject, destPeerID string, dt tools.DataType,
errCh <- nil
return
}
payload, err := json.Marshal(obj)
m := obj.Serialize(obj)
if m["dest_peer_id"] != nil {
if data := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil).LoadOne(fmt.Sprintf("%v", m["dest_peer_id"])); data.Data != nil {
m["peer_id"] = data.Data.(*peer.Peer).PeerID
}
} else {
fmt.Println("NO DEST ID")
return
}
payload, err := json.Marshal(m)
if err != nil {
errCh <- fmt.Errorf("could not serialize %s: %w", dt.String(), err)
return
}
tools.NewNATSCaller().SetNATSPub(tools.CREATE_RESOURCE, tools.NATSResponse{
FromApp: "oc-scheduler",
Datatype: dt,
Method: int(tools.CREATE_RESOURCE),
if b, err := json.Marshal(&tools.PropalgationMessage{
DataType: dt.EnumIndex(),
Action: tools.PB_CREATE,
Payload: payload,
})
}); err == nil {
tools.NewNATSCaller().SetNATSPub(tools.PROPALGATION_EVENT, tools.NATSResponse{
FromApp: "oc-scheduler",
Datatype: dt,
Method: int(tools.PROPALGATION_EVENT),
Payload: b,
})
}
errCh <- nil
}
@@ -335,303 +340,3 @@ type Schedule struct {
* TODO : LARGEST GRAIN PLANIFYING THE WORKFLOW WHEN OPTION IS SET
* SET PROTECTION BORDER TIME
*/
// ---------------------------------------------------------------------------
// Slot availability check
// ---------------------------------------------------------------------------
const (
checkWindowHours = 5 // how far ahead to scan for a free slot (hours)
checkStepMin = 15 // time increment per scan step (minutes)
)
// CheckResult holds the outcome of a slot availability check.
type CheckResult struct {
Available bool `json:"available"`
Start time.Time `json:"start"`
End *time.Time `json:"end,omitempty"`
// NextSlot is the nearest free slot found within checkWindowHours when
// the requested slot is unavailable, or the preferred (conflict-free) slot
// when running in preemption mode.
NextSlot *time.Time `json:"next_slot,omitempty"`
Warnings []string `json:"warnings,omitempty"`
// Preemptible is true when the check was run in preemption mode.
Preemptible bool `json:"preemptible,omitempty"`
}
// bookingResource is the minimum info needed to verify a resource against the
// planner cache.
type bookingResource struct {
id string
peerID string
instanceID string // resolved from WorkflowSchedule.SelectedInstances
}
// Check verifies that all booking-relevant resources (storage and compute) of
// the given workflow have capacity for the requested time slot.
//
// - asap=true → ignore ws.Start, begin searching from time.Now()
// - preemption → always return Available=true but populate Warnings with
// conflicts and NextSlot with the nearest conflict-free alternative
func (ws *WorkflowSchedule) Check(wfID string, asap bool, preemption bool, request *tools.APIRequest) (*CheckResult, error) {
// 1. Load workflow
obj, code, err := workflow.NewAccessor(request).LoadOne(wfID)
if code != 200 || err != nil {
msg := "could not load workflow " + wfID
if err != nil {
msg += ": " + err.Error()
}
return nil, errors.New(msg)
}
wf := obj.(*workflow.Workflow)
// 2. Resolve start
start := ws.Start
if asap || start.IsZero() {
start = time.Now()
}
// 3. Resolve end – use explicit end/duration or estimate via Planify
end := ws.End
if end == nil {
if ws.DurationS > 0 {
e := start.Add(time.Duration(ws.DurationS * float64(time.Second)))
end = &e
} else {
_, longest, _, _, planErr := wf.Planify(
start, nil,
ws.SelectedInstances, ws.SelectedPartnerships,
ws.SelectedBuyings, ws.SelectedStrategies,
int(ws.BookingMode), request,
)
if planErr == nil && longest > 0 {
e := start.Add(time.Duration(longest) * time.Second)
end = &e
}
}
}
// 4. Extract booking-relevant (storage + compute) resources from the graph,
// resolving the selected instance for each resource.
checkables := collectBookingResources(wf, ws.SelectedInstances)
fmt.Println(checkables)
// 5. Check every resource against its peer's planner
unavailable, warnings := checkResourceAvailability(checkables, start, end)
fmt.Println(unavailable, warnings)
result := &CheckResult{
Start: start,
End: end,
Warnings: warnings,
}
// 6. Preemption mode: mark as schedulable regardless of conflicts, but
// surface warnings and the nearest conflict-free alternative.
if preemption {
result.Available = true
result.Preemptible = true
if len(unavailable) > 0 {
result.NextSlot = findNextSlot(checkables, start, end, checkWindowHours)
}
return result, nil
}
// 7. All resources are free
if len(unavailable) == 0 {
result.Available = true
return result, nil
}
// 8. Slot unavailable – locate the nearest free slot within the window
result.Available = false
result.NextSlot = findNextSlot(checkables, start, end, checkWindowHours)
return result, nil
}
// collectBookingResources returns unique storage and compute resources from the
// workflow graph. For each resource the selected instance ID is resolved from
// selectedInstances (the scheduler's SelectedInstances ConfigItem) so the planner
// check targets the exact instance chosen by the user.
func collectBookingResources(wf *workflow.Workflow, selectedInstances workflow.ConfigItem) []bookingResource {
if wf.Graph == nil {
return nil
}
seen := map[string]bool{}
var result []bookingResource
resolveInstanceID := func(res interface {
GetID() string
GetCreatorID() string
}) string {
idx := selectedInstances.Get(res.GetID())
switch r := res.(type) {
case *resources.StorageResource:
if inst := r.GetSelectedInstance(idx); inst != nil {
return inst.GetID()
}
case *resources.ComputeResource:
if inst := r.GetSelectedInstance(idx); inst != nil {
return inst.GetID()
}
}
return ""
}
for _, item := range wf.GetGraphItems(wf.Graph.IsStorage) {
i := item
_, res := i.GetResource()
if res == nil {
continue
}
id, peerID := res.GetID(), res.GetCreatorID()
if peerID == "" || seen[id] {
continue
}
seen[id] = true
result = append(result, bookingResource{
id: id,
peerID: peerID,
instanceID: resolveInstanceID(res),
})
}
for _, item := range wf.GetGraphItems(wf.Graph.IsCompute) {
i := item
_, res := i.GetResource()
if res == nil {
continue
}
id, peerID := res.GetID(), res.GetCreatorID()
if peerID == "" || seen[id] {
continue
}
seen[id] = true
result = append(result, bookingResource{
id: id,
peerID: peerID,
instanceID: resolveInstanceID(res),
})
}
return result
}
// checkResourceAvailability returns the IDs of unavailable resources and
// human-readable warning messages.
func checkResourceAvailability(res []bookingResource, start time.Time, end *time.Time) (unavailable []string, warnings []string) {
for _, r := range res {
plannerMu.RLock()
entry := PlannerCache[r.peerID]
plannerMu.RUnlock()
if entry == nil || entry.Planner == nil {
warnings = append(warnings, fmt.Sprintf(
"peer %s planner not in cache for resource %s – assuming available", r.peerID, r.id))
continue
}
if !checkInstance(entry.Planner, r.id, r.instanceID, start, end) {
unavailable = append(unavailable, r.id)
warnings = append(warnings, fmt.Sprintf(
"resource %s is not available in [%s – %s]",
r.id, start.Format(time.RFC3339), formatOptTime(end)))
}
}
return
}
// checkInstance checks availability for the specific instance resolved by the
// scheduler. When instanceID is empty (no instance selected / none resolvable),
// it falls back to checking all instances known in the planner and returns true
// if any one has remaining capacity. Returns true when no capacity is recorded.
func checkInstance(p *planner.Planner, resourceID string, instanceID string, start time.Time, end *time.Time) bool {
if instanceID != "" {
return p.Check(resourceID, instanceID, nil, start, end)
}
// Fallback: accept if any known instance has free capacity
caps, ok := p.Capacities[resourceID]
if !ok || len(caps) == 0 {
return true // no recorded usage → assume free
}
for id := range caps {
if p.Check(resourceID, id, nil, start, end) {
return true
}
}
return false
}
// findNextSlot scans forward from 'from' in checkStepMin increments for up to
// windowH hours and returns the first candidate start time at which all
// resources are simultaneously free.
func findNextSlot(resources []bookingResource, from time.Time, originalEnd *time.Time, windowH int) *time.Time {
duration := time.Hour
if originalEnd != nil {
if d := originalEnd.Sub(from); d > 0 {
duration = d
}
}
step := time.Duration(checkStepMin) * time.Minute
limit := from.Add(time.Duration(windowH) * time.Hour)
for t := from.Add(step); t.Before(limit); t = t.Add(step) {
e := t.Add(duration)
if unavail, _ := checkResourceAvailability(resources, t, &e); len(unavail) == 0 {
return &t
}
}
return nil
}
func formatOptTime(t *time.Time) string {
if t == nil {
return "open"
}
return t.Format(time.RFC3339)
}
// GetWorkflowPeerIDs loads the workflow and returns the deduplicated list of
// creator peer IDs for all its storage and compute resources.
// These are the peers whose planners must be watched by a check stream.
func GetWorkflowPeerIDs(wfID string, request *tools.APIRequest) ([]string, error) {
obj, code, err := workflow.NewAccessor(request).LoadOne(wfID)
if code != 200 || err != nil {
msg := "could not load workflow " + wfID
if err != nil {
msg += ": " + err.Error()
}
return nil, errors.New(msg)
}
wf := obj.(*workflow.Workflow)
if wf.Graph == nil {
return nil, nil
}
seen := map[string]bool{}
var peerIDs []string
for _, item := range wf.GetGraphItems(wf.Graph.IsStorage) {
i := item
_, res := i.GetResource()
if res == nil {
continue
}
if id := res.GetCreatorID(); id != "" && !seen[id] {
seen[id] = true
peerIDs = append(peerIDs, id)
}
}
for _, item := range wf.GetGraphItems(wf.Graph.IsCompute) {
i := item
_, res := i.GetResource()
if res == nil {
continue
}
if id := res.GetCreatorID(); id != "" && !seen[id] {
seen[id] = true
peerIDs = append(peerIDs, id)
}
}
realPeersID := []string{}
access := oclib.NewRequestAdmin(oclib.LibDataEnum(tools.PEER), nil)
for _, id := range peerIDs {
if data := access.LoadOne(id); data.Data != nil {
realPeersID = append(realPeersID, data.ToPeer().PeerID)
}
}
return realPeersID, nil
}