improved graph/workflow retrieval
This commit is contained in:
358
workflow_builder/graph.go
Normal file
358
workflow_builder/graph.go
Normal file
@@ -0,0 +1,358 @@
|
||||
package workflow_builder
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"maps"
|
||||
|
||||
"oc-scheduler/conf"
|
||||
"oc-scheduler/logger"
|
||||
models "oc-scheduler/models"
|
||||
|
||||
catalog_models "cloud.o-forge.io/core/oc-catalog/models" // this will be replaced with oc-lib
|
||||
|
||||
"github.com/beego/beego/v2/core/logs"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
|
||||
|
||||
type Graph struct {
|
||||
workflow_name string // used to test if the graph has been instatiated, private so can only be set by a graph's method
|
||||
Datas []catalog_models.DataModel
|
||||
Computings []catalog_models.ComputingModel
|
||||
Datacenters []catalog_models.DatacenterModel
|
||||
Storages []catalog_models.StorageModel
|
||||
Links map[string]catalog_models.Link
|
||||
ws models.HttpQuery
|
||||
}
|
||||
|
||||
// Create a dictionnaries with each existing workflow from a workspace, associated to the JSON representation of its content
|
||||
func (g *Graph) GetGraphList(apiurl string) (map[string]string, error) {
|
||||
g.ws.Init(apiurl)
|
||||
body, err := g.ws.Get("v1/workspace/list")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
workspaces := make(map[string]string)
|
||||
result := gjson.Get(string(body), "Workflows")
|
||||
result.ForEach(func(key, value gjson.Result) bool {
|
||||
workspaces[key.Str] = value.String()
|
||||
return true // keep iterating
|
||||
})
|
||||
return workspaces, nil
|
||||
}
|
||||
|
||||
// Should the parameter be removed, since we have oc-catalog url in the conf ?
|
||||
func (g *Graph) GetGraph(apiurl string, workflow string) (string, error) {
|
||||
g.ws.Init(apiurl)
|
||||
body, err := g.ws.Get("v1/workflow/" + workflow)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
graph := string(body)
|
||||
|
||||
// result := gjson.Get(string(body), "Workflows")
|
||||
// result.ForEach(func(key, value gjson.Result) bool {
|
||||
// workspaces[key.Str] = value.String()
|
||||
// return true // keep iterating
|
||||
// })
|
||||
return graph, nil
|
||||
}
|
||||
|
||||
|
||||
// Create the objects from the mxgraphxml stored in the workflow given as a parameter
|
||||
func (g *Graph) LoadFrom(workflow_name string) error {
|
||||
// Extract the xmlgraph from the given workspace
|
||||
graph, err := g.GetGraph(conf.GetConfig().OcCatalogUrl,workflow_name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// os.WriteFile("graph.xml", []byte(decodedValue), 0660)
|
||||
|
||||
g.GetWorkflowComponents(graph)
|
||||
g.GetLinks(graph)
|
||||
|
||||
g.workflow_name = workflow_name
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
// Create the objects that correspond to each component
|
||||
// in a workflow, combining the user input and the base components attributes
|
||||
func (g *Graph) GetWorkflowComponents(workflow string){
|
||||
types := []string{"computing","datacenter","data","storage"} // create a constant for more maintainability OR even better get the list of all component's type for this WF
|
||||
for _, component_type := range types {
|
||||
// Retrieve the dict of component for a specific type in the workflow
|
||||
result := gjson.Get(workflow, component_type)
|
||||
|
||||
if (result.Type != gjson.Null) {
|
||||
result.ForEach(func(id, value gjson.Result) bool{
|
||||
comp_id := value.Get("referenceID").Str
|
||||
|
||||
if (comp_id != "") {
|
||||
switch component_type {
|
||||
case "computing":
|
||||
g.AddComputingModel(comp_id, value, id.Str)
|
||||
case "data":
|
||||
g.AddDataModel(comp_id, value, id.Str)
|
||||
case "datacenter":
|
||||
g.AddDatacenterModel(comp_id, value, id.Str)
|
||||
case "storage":
|
||||
g.AddStorageModel(comp_id, value, id.Str)
|
||||
default :
|
||||
logs.Critical("Component type doesn't match a know type : " + component_type)
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func (g *Graph) GetLinks(workflow string){
|
||||
g.Links = make(map[string]catalog_models.Link)
|
||||
result := gjson.Get(workflow, "link")
|
||||
|
||||
if (result.Type != gjson.Null) {
|
||||
result.ForEach(func(id, value gjson.Result) bool{
|
||||
var l catalog_models.Link
|
||||
|
||||
json.Unmarshal([]byte(value.Raw),&l)
|
||||
g.Links[id.Str] = l
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Graph) AddDataModel(id string, user_input gjson.Result, wf_id string) error {
|
||||
var d catalog_models.DataModel
|
||||
resp, err := g.ws.Get("v1/data/" + id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
json.Unmarshal(resp, &d)
|
||||
json.Unmarshal([]byte(user_input.Raw),&d.DataNEWModel)
|
||||
d.ID = wf_id
|
||||
g.Datas = append(g.Datas, d)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Graph) AddDatacenterModel(id string, user_input gjson.Result, wf_id string) error {
|
||||
var d catalog_models.DatacenterModel
|
||||
resp, err := g.ws.Get("v1/datacenter/" + id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
json.Unmarshal(resp, &d)
|
||||
json.Unmarshal([]byte(user_input.Raw),&d.DatacenterNEWModel)
|
||||
d.ID = wf_id
|
||||
g.Datacenters = append(g.Datacenters, d)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Graph) AddComputingModel(id string, user_input gjson.Result, wf_id string) error {
|
||||
var c catalog_models.ComputingModel
|
||||
resp, err := g.ws.Get("v1/computing/" + id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
json.Unmarshal(resp, &c)
|
||||
json.Unmarshal([]byte(user_input.Raw),&c.ComputingNEWModel)
|
||||
c.ID = wf_id
|
||||
g.Computings = append(g.Computings, c)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Graph) AddStorageModel(id string, user_input gjson.Result, wf_id string) error {
|
||||
var s catalog_models.StorageModel
|
||||
resp, err := g.ws.Get("v1/storage/" + id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
json.Unmarshal(resp, &s)
|
||||
json.Unmarshal([]byte(user_input.Raw),&s.StorageNEWModel)
|
||||
s.ID = wf_id
|
||||
g.Storages = append(g.Storages, s)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Graph) ExportToArgo() (string, error) {
|
||||
if len(g.workflow_name) == 0 {
|
||||
return "",fmt.Errorf("can't export a graph that has not been loaded yet")
|
||||
}
|
||||
end_links := make(map[string]catalog_models.Link)
|
||||
|
||||
for i, link := range g.Links {
|
||||
if (!link.DCLink && !g.isSource(link.Destination,i)){
|
||||
end_links[i] = link
|
||||
}
|
||||
}
|
||||
|
||||
// index_list := make([]int, len(g.Links))
|
||||
// list_branches := make([][]string,0)
|
||||
list_branches := g.getListBranches(end_links, nil,nil)
|
||||
|
||||
for _, branch := range list_branches{
|
||||
str := ""
|
||||
for _, link := range branch{
|
||||
str = str + " --> " + g.getComponentName(g.Links[link].Source) + " linked with " + g.getComponentName(g.Links[link].Destination)
|
||||
}
|
||||
|
||||
fmt.Println(str)
|
||||
}
|
||||
|
||||
fmt.Println("Identified branches : ", list_branches)
|
||||
argo_builder := ArgoBuilder{graph : *g, branches: list_branches}
|
||||
filename, err := argo_builder.CreateDAG()
|
||||
if err != nil {
|
||||
logger.Logger.Error().Msg("Could not create the argo file for " + g.workflow_name)
|
||||
return "", err
|
||||
}
|
||||
return filename, nil
|
||||
}
|
||||
|
||||
// Return a list containing the IDs of each link that make up a branch in the graph
|
||||
func (g *Graph) getListBranches(end_links map[string]catalog_models.Link, unvisited_links_list map[string]catalog_models.Link, current_branch []string) (list_branches [][]string) {
|
||||
|
||||
if current_branch == nil {
|
||||
current_branch = make([]string, 0)
|
||||
}
|
||||
|
||||
if unvisited_links_list == nil {
|
||||
unvisited_links_list = make(map[string]catalog_models.Link,len(g.Links))
|
||||
maps.Copy(unvisited_links_list,g.Links)
|
||||
fmt.Println(unvisited_links_list)
|
||||
}
|
||||
|
||||
for link_id, _ := range end_links {
|
||||
j := link_id
|
||||
new_branches := make([][]string,0)
|
||||
|
||||
previous_index := g.getPreviousLink(j, unvisited_links_list)
|
||||
if len(previous_index) == 0 {
|
||||
list_branches = append(list_branches, []string{link_id})
|
||||
}
|
||||
|
||||
for _, id_link := range previous_index {
|
||||
current_branch = append([]string{link_id},current_branch...)
|
||||
delete(unvisited_links_list, link_id)
|
||||
// create a new branch for each previous link, appending the current path to this node to the created branch
|
||||
new_end_link := make(map[string]catalog_models.Link,0)
|
||||
new_end_link[id_link] = g.Links[id_link]
|
||||
new_branches = g.getListBranches(new_end_link,unvisited_links_list,current_branch)
|
||||
|
||||
for _, new_branch := range new_branches{
|
||||
current_branch = append(new_branch,link_id)
|
||||
list_branches = append(list_branches, current_branch)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (g *Graph) ExportToHelm(id string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Return if it exists a link where Destination is the same as comp_id
|
||||
func (g *Graph) isDestination(comp_id string,link_id string) bool {
|
||||
|
||||
for i, link := range g.Links{
|
||||
if(i !=link_id && link.Destination == comp_id){
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
|
||||
}
|
||||
|
||||
// Return if it exists a link where Source is the same as comp_id
|
||||
func (g *Graph) isSource(comp_id string,link_id string) bool {
|
||||
|
||||
for i, link := range g.Links{
|
||||
if(i !=link_id && link.Source == comp_id && !link.DCLink){
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
|
||||
}
|
||||
|
||||
// Returns an index number if their is a link in g.Links
|
||||
// with the same Destination id that the Source id in g.Links[linkIndex]
|
||||
// or nil if not
|
||||
|
||||
func (g *Graph) getPreviousLink(link_id string,map_link map[string]catalog_models.Link) (previous_id []string) {
|
||||
for k, link := range map_link{
|
||||
if(k != link_id && link.Destination == g.Links[link_id].Source){
|
||||
previous_id = append(previous_id, k)
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (g *Graph) getComponentName(id string) string {
|
||||
|
||||
for _, comp := range g.Computings{
|
||||
if comp.ID == id {
|
||||
return comp.Name
|
||||
}
|
||||
}
|
||||
for _, storage := range g.Storages{
|
||||
if storage.ID == id {
|
||||
return storage.Name
|
||||
}
|
||||
}
|
||||
for _, data := range g.Datas{
|
||||
if data.ID == id {
|
||||
return data.Name
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// returns either computing, data or storage
|
||||
func (g *Graph) getComponentType(component_id string) string {
|
||||
for _, comp := range g.Computings {
|
||||
if comp.ID == component_id{
|
||||
return "computing"
|
||||
}
|
||||
}
|
||||
|
||||
for _, data := range g.Datas {
|
||||
if data.ID == component_id{
|
||||
return "data"
|
||||
}
|
||||
}
|
||||
|
||||
for _, storage := range g.Storages {
|
||||
if storage.ID == component_id{
|
||||
return "storage"
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// Returns a slice of id, in case the link is made of twice the same type of component
|
||||
|
||||
func (g *Graph) getComponentByType(compType string, link catalog_models.Link) (ids []string){
|
||||
if(g.getComponentType(link.Source) == compType){
|
||||
ids = append(ids, link.Source)
|
||||
}
|
||||
if(g.getComponentType(link.Destination) == compType){
|
||||
ids = append(ids, link.Destination)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user