diff --git a/README.md b/README.md index e429389..23a80f7 100644 --- a/README.md +++ b/README.md @@ -15,3 +15,50 @@ imagePullPolicy: Never Not doing so will end up in the pod having a `ErrorImagePull` +## Allow argo to create services + +In order for monitord to expose **open cloud services** on the node, we need to give him permission to create **k8s services**. + +For that we can update the RBAC configuration for a role already created by argo : + +### Manually edit the rbac authorization + +> kubectl edit roles.rbac.authorization.k8s.io -n argo argo-role + +In rules add a new entry : + +``` +- apiGroups: + - "" + resources: + - services + verbs: + - get + - create +``` + +### Patch the rbac authorization with a one liner + +> kubectl patch role argo-role -n argo --type='json' -p='[{"op": "add", "path": "/rules/-", "value": {"apiGroups": [""], "resources": ["services"], "verbs": ["get","create"]}}]' + +### Check wether the modification is effective + +> kubectl auth can-i create services --as=system:serviceaccount:argo:argo -n argo + +This command **must return "yes"** + + +## TODO + +- [ ] Logs the output of each pods : + - logsPods() function already exists + - need to implement the logic to create each pod's logger and start the monitoring routing +- [ ] Allow the front to known on which IP the service are reachable + - currently doing it by using `kubectl get nodes -o wide` + + +### Adding ingress handling to support reverse proxing + +- Test wether ingress-nginx is running or not + - Do something if not found : stop running and send error log OR start installation +- \ No newline at end of file diff --git a/conf/conf.go b/conf/conf.go index 8f0eda4..3de8b82 100644 --- a/conf/conf.go +++ b/conf/conf.go @@ -3,10 +3,14 @@ package conf import "sync" type Config struct { + MongoURL string + Database string LokiURL string + NatsURL string ExecutionID string Timeout int WorkflowID string + Logs string } var instance *Config diff --git a/demo_nginx/DTF.html b/demo_nginx/DTF.html new file mode 100644 index 0000000..30e4f97 --- /dev/null +++ b/demo_nginx/DTF.html @@ -0,0 +1,5 @@ + + +

This could be your DTF webpage

+ + \ No newline at end of file diff --git a/demo_nginx/cockpit.html b/demo_nginx/cockpit.html new file mode 100644 index 0000000..6c7b718 --- /dev/null +++ b/demo_nginx/cockpit.html @@ -0,0 +1,5 @@ + + +

Welcome to your DTF Cockpit

+ + \ No newline at end of file diff --git a/demo_nginx/workflow_nginx_services.json b/demo_nginx/workflow_nginx_services.json index acbee01..f07a50c 100644 --- a/demo_nginx/workflow_nginx_services.json +++ b/demo_nginx/workflow_nginx_services.json @@ -1,15 +1,13 @@ { "id": "9c7ffc7e-3e6e-4ea8-8eab-3a03258712ff", "name": "test-services", - "resourceset": { - "processings": [ - "7c71a15b-bdbc-46d7-9dab-67e369804136", - "0d565c87-50ae-4a73-843d-f8b2d4047772" - ], - "datacenters": [ - "7b989e97-c3e7-49d2-a3a7-f959da4870b5" - ] - }, + "processings": [ + "7c71a15b-bdbc-46d7-9dab-67e369804136", + "0d565c87-50ae-4a73-843d-f8b2d4047772" + ], + "datacenters": [ + "7b989e97-c3e7-49d2-a3a7-f959da4870b5" + ], "graph": { "zoom": 1, "items": { @@ -28,7 +26,7 @@ }, "args": { "type": "string", - "value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/cockpit.html -o /usr/share/ningx/cockpit.hmtl" + "value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/cockpit.html -o /usr/share/nginx/cockpit.hmtl" } } }, @@ -61,7 +59,7 @@ }, "args": { "type": "string", - "value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/DTF.html -o /usr/share/ningx/DTF.hmtl" + "value": "-SL https://cloud.o-forge.io/core/oc-monitord/raw/branch/services_demo/demo_nginx/DTF.html -o /usr/share/nginx/DTF.hmtl" } } }, @@ -100,6 +98,15 @@ }, "args": { "type": "string" + }, + "expose": { + "type": "dict", + "value": { + "80" :{ + "reverse" : "", + "PAT" : "308080" + } + } } } }, @@ -139,6 +146,15 @@ }, "args": { "type": "string" + }, + "expose": { + "type": "dict", + "value": { + "80" :{ + "reverse" : "", + "PAT" : "308081" + } + } } } }, @@ -222,7 +238,7 @@ "y": 0 }, "destination": { - "id": "0d565c87-50ae-4a73-843d-f8b2d4047772", + "id": "bf6916ff-b16f-44b3-818b-0bcd5bbaca00", "x": 0, "y": 0 } @@ -277,12 +293,12 @@ }, { "source": { - "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", + "id": "d83ac451-4690-44d9-af09-48e7588b2db9", "x": 0, "y": 0 }, "destination": { - "id": "d83ac451-4690-44d9-af09-48e7588b2db9", + "id": "6a7e8860-7c26-4b70-9b3a-1bd27adcdfe1", "x": 0, "y": 0 } diff --git a/go.mod b/go.mod index 5874c16..7a41244 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module oc-monitord go 1.22.0 require ( - cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c + cloud.o-forge.io/core/oc-lib v0.0.0-20240924075418-021b461b0a7d github.com/akamensky/argparse v1.4.0 github.com/goraz/onion v0.1.3 github.com/nats-io/nats-server/v2 v2.10.18 @@ -17,10 +17,10 @@ require ( github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.0 // indirect + github.com/go-playground/validator/v10 v10.22.1 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -38,10 +38,10 @@ require ( github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - go.mongodb.org/mongo-driver v1.16.1 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect + go.mongodb.org/mongo-driver v1.17.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect ) diff --git a/go.sum b/go.sum index 6d7d36e..c0fa3cd 100644 --- a/go.sum +++ b/go.sum @@ -44,6 +44,8 @@ cloud.o-forge.io/core/oc-lib v0.0.0-20240830071403-db78c70dc349 h1:bEIY1lCsA78/m cloud.o-forge.io/core/oc-lib v0.0.0-20240830071403-db78c70dc349/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0= cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c h1:4ZoM9ONJiaeLHSi0s8gsCe4lHuRHXkfK+eDSnTCspa0= cloud.o-forge.io/core/oc-lib v0.0.0-20240830131445-af18dba5563c/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0= +cloud.o-forge.io/core/oc-lib v0.0.0-20240924075418-021b461b0a7d h1:f8cT/NunF+eoZLU5B9gmiT4ky99zPmnQBbj4tj23KuA= +cloud.o-forge.io/core/oc-lib v0.0.0-20240924075418-021b461b0a7d/go.mod h1:FIJD0taWLJ5pjQLJ6sfE2KlTkvbmk5SMcyrxdjsaVz0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/akamensky/argparse v1.4.0 h1:YGzvsTqCvbEZhL8zZu2AiA5nq805NZh75JNj4ajn1xc= github.com/akamensky/argparse v1.4.0/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= @@ -69,6 +71,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.22.0 h1:k6HsTZ0sTnROkhS//R0O+55JgM8C4Bx7ia+JlgcnOao= github.com/go-playground/validator/v10 v10.22.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= +github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -87,6 +91,8 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= +github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -169,6 +175,8 @@ go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4B go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.mongodb.org/mongo-driver v1.17.0 h1:Hp4q2MCjvY19ViwimTs00wHi7G4yzxh4/2+nTx8r40k= +go.mongodb.org/mongo-driver v1.17.0/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191112222119-e1110fd1c708/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -177,6 +185,8 @@ golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -187,6 +197,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= @@ -210,6 +222,8 @@ golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -220,6 +234,8 @@ golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/main.go b/main.go index 0eb3ef0..fdc69db 100644 --- a/main.go +++ b/main.go @@ -21,7 +21,6 @@ import ( "cloud.o-forge.io/core/oc-lib/logs" "cloud.o-forge.io/core/oc-lib/models/utils" "cloud.o-forge.io/core/oc-lib/models/workflow_execution" - "cloud.o-forge.io/core/oc-lib/tools" "github.com/akamensky/argparse" "github.com/google/uuid" @@ -64,13 +63,19 @@ func main() { loadConfig(true, nil) } - logs.SetAppName("oc-monitord") - logger = logs.CreateLogger("oc-monitord", conf.GetConfig().LokiURL) + logger = logs.CreateLogger("oc-monitord") logger.Debug().Msg("Loki URL : " + conf.GetConfig().LokiURL) logger.Debug().Msg("Workflow executed : " + conf.GetConfig().ExecutionID) - oclib.Init("oc-monitord", "", "") + oclib.SetConfig( + conf.GetConfig().MongoURL, + conf.GetConfig().Database, + conf.GetConfig().NatsURL, + conf.GetConfig().LokiURL, + conf.GetConfig().Logs, + ) + oclib.Init("oc-monitord") wf_id := getWorkflowId(conf.GetConfig().ExecutionID) conf.GetConfig().WorkflowID = wf_id @@ -121,6 +126,8 @@ func getWorkflowId(exec_id string) string { return wf_exec.WorkflowID } +// So far we only log the output from + func executeWorkflow(argo_file_path string) { // var stdout, stderr, stdout_logs, stderr_logs io.ReadCloser var stdout, stderr io.ReadCloser @@ -187,7 +194,7 @@ func logWorkflow(pipe io.ReadCloser, wg *sync.WaitGroup) { // Debug, no logs sent func logPods(pipe io.ReadCloser, name string) { - pods_logger = wf_logger.With().Str("sortie name", name).Logger() + pods_logger = wf_logger.With().Str("pod_name", name).Logger() scanner := bufio.NewScanner(pipe) for scanner.Scan() { log := scanner.Text() @@ -202,7 +209,7 @@ func loadConfig(is_k8s bool, parser *argparse.Parser) { o = initOnion(o) // These variables can only be retrieved in the onion - // Variables that don't depend on the environmen (from conf file), can be loaded after + // Variables that don't depend on the environment (from conf file), can be loaded after // We can't use underscore in the env variable names because it's the delimitor with OCMONITOR too setConf(is_k8s, o, parser) @@ -223,7 +230,9 @@ func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) { conf.GetConfig().ExecutionID = o.GetString("workflow") mongo := o.GetStringDefault("mongourl", "mongodb://127.0.0.1:27017") db := o.GetStringDefault("database", "DC_myDC") - tools.SetConfig(mongo, db, "") + + conf.GetConfig().MongoURL = mongo + conf.GetConfig().Database = db } else { url := parser.String("u", "url", &argparse.Options{Required: true, Default: "http://127.0.0.1:3100", Help: "Url to the Loki database logs will be sent to"}) execution := parser.String("e", "execution", &argparse.Options{Required: true, Help: "Execution ID of the workflow to request from oc-catalog API"}) @@ -235,16 +244,20 @@ func setConf(is_k8s bool, o *onion.Onion, parser *argparse.Parser) { fmt.Println(parser.Usage(err)) os.Exit(1) } + + conf.GetConfig().Logs = "debug" + conf.GetConfig().LokiURL = *url + conf.GetConfig().MongoURL = *mongo + conf.GetConfig().Database = *db conf.GetConfig().Timeout = *timeout conf.GetConfig().ExecutionID = *execution - tools.SetConfig(*mongo, *db, "") } } func initOnion(o *onion.Onion) *onion.Onion { - logger = logs.CreateLogger("oc-monitord", "") + logger = logs.CreateLogger("oc-monitord") configFile := "" l3 := onion.NewEnvLayerPrefix("_", "OCMONITORD") diff --git a/models/services.go b/models/services.go new file mode 100644 index 0000000..ba0e10a --- /dev/null +++ b/models/services.go @@ -0,0 +1,40 @@ +package models + + +type ServiceResource struct { + Action string `yaml:"action,omitempty"` + SuccessCondition string `yaml:"successCondition,omitempty"` + FailureCondition string `yaml:"failureCondition,omitempty"` + SetOwnerReference bool `yaml:"setOwnerReference,omitempty"` + Manifest string `yaml:"manifest,omitempty"` +} + +type Service struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata Metadata `yaml:"metadata"` + Spec ServiceSpec `yaml:"spec"` +} + +type Metadata struct { + Name string `yaml:"name"` + +} + +// ServiceSpec is the specification of the Kubernetes Service +type ServiceSpec struct { + Selector map[string]string `yaml:"selector,omitempty"` + Ports []ServicePort `yaml:"ports"` + ClusterIP string `yaml:"clusterIP,omitempty"` + Type string `yaml:"type,omitempty"` +} + +// ServicePort defines a port for a Kubernetes Service +type ServicePort struct { + Name string `yaml:"name"` // Even if empty need to be in the yaml + + Protocol string `yaml:"protocol,omitempty"` + Port int64 `yaml:"port"` + TargetPort int64 `yaml:"targetPort,omitempty"` + NodePort int64 `yaml:"nodePort,omitempty"` +} \ No newline at end of file diff --git a/models/template.go b/models/template.go index 47feaf4..c7c9b0e 100644 --- a/models/template.go +++ b/models/template.go @@ -30,6 +30,10 @@ type Dag struct { Tasks []Task `yaml:"tasks,omitempty"` } +type TemplateMetadata struct { + Labels map[string]string `yaml:"labels,omitempty"` +} + type Template struct { Name string `yaml:"name"` Inputs struct { @@ -37,4 +41,6 @@ type Template struct { } `yaml:"inputs,omitempty"` Container Container `yaml:"container,omitempty"` Dag Dag `yaml:"dag,omitempty"` + Metadata TemplateMetadata `yaml:"metadata,omitempty"` + Resource ServiceResource `yaml:"resource,omitempty"` } diff --git a/models/translate_ports.go b/models/translate_ports.go new file mode 100644 index 0000000..080101b --- /dev/null +++ b/models/translate_ports.go @@ -0,0 +1,8 @@ +package models + +type PortTranslation map[string]PortConfig + +type PortConfig struct { + Reverse string `json:"reverse,omitempty"` + PAT string `json:"PAT,omitempty"` +} diff --git a/test-logs-workflow.json b/test-logs-workflow.json new file mode 100644 index 0000000..d4e9b71 --- /dev/null +++ b/test-logs-workflow.json @@ -0,0 +1,141 @@ +{ + "name": "test-log", + "processings": [ + "0d565c87-50ae-4a73-843d-f8b2d4047772", + "2ce0323f-a85d-4b8b-a783-5280f48d634a" + ], + "datacenters": [ + "7b989e97-c3e7-49d2-a3a7-f959da4870b5" + ], + "graph": { + "items": { + "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633": { + "id": "aa8d2265-9fe2-42c7-ba1f-46ea0da8e633", + "width": 0, + "height": 0, + "processing": { + "id": "0d565c87-50ae-4a73-843d-f8b2d4047772", + "resource_model": { + "resource_type": "processing", + "model": { + "command": { + "type": "string", + "value": "curlimages/curl:7.88.1" + }, + "args": { + "type": "string", + "value": "-SL https://toulousefc.com -o /mnt/vol/tfc.hmtl" + } + } + }, + "name": "CURL", + "short_description": "Transfer or retrieve information from or to a server ", + "description": "curl is a tool for transferring data from or to a server. It supports these protocols: DICT, FILE, FTP, FTPS, GOPHER, GOPHERS, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP, SMB, SMBS, SMTP, SMTPS, TELNET, TFTP, WS and WSS.", + "logo": "https://cloud.o-forge.io/core/deperecated-oc-catalog/src/branch/main/scripts/local_imgs/curl-logo.png", + "owner": "IRT", + "source_url": "http://www.google.com", + "license": "GPLv2", + "ram": {}, + "storage": 300, + "parallel": true, + "scaling_model": 2, + "disk_io": "30 MB/s" + } + }, + "b6f790ae-5db2-4458-8375-59450f12f65b": { + "id": "b6f790ae-5db2-4458-8375-59450f12f65b", + "width": 0, + "height": 0, + "processing": { + "id": "2ce0323f-a85d-4b8b-a783-5280f48d634a", + "resource_model": { + "resource_type": "processing", + "model": { + "command": { + "type": "string", + "value": "alpine:3.7 sh -c" + }, + "args": { + "type": "string", + "value": "grep ' store objects new_dag.Tasks = append(new_dag.Tasks, step) } - b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, Template{Name: "dag", Dag: new_dag}) + if b.Services != nil { + new_dag.Tasks = append(new_dag.Tasks, Task{Name:"workflow-service-pod", Template: "workflow-service-pod"}) + } + + b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, Template{Name: "dag", Dag: new_dag}) + } func (b *ArgoBuilder) createVolumes() { @@ -160,7 +183,7 @@ func (b *ArgoBuilder) createNginxVolumes() { func (b *ArgoBuilder) getDependency(current_computing_id string) (dependencies []string) { for _, link := range b.OriginWorkflow.Graph.Links { - if !b.IsProcessing(link.Source.ID) { + if b.OriginWorkflow.Graph.Items[link.Source.ID].Processing == nil { continue } source := b.OriginWorkflow.Graph.Items[link.Source.ID].Processing @@ -272,8 +295,9 @@ func (b *ArgoBuilder) getProcessings() (list_computings []graph.GraphItem) { return } -func (b *ArgoBuilder) IsProcessing(id string) bool { - return slices.Contains(b.OriginWorkflow.Processings, id) +// Pass a GraphItem's UUID and not the ID +func (b *ArgoBuilder) IsProcessing(component_uuid string) bool { + return slices.Contains(b.OriginWorkflow.Processings, component_uuid) } func getStringValue(comp resource_model.AbstractResource, key string) string { @@ -283,19 +307,26 @@ func getStringValue(comp resource_model.AbstractResource, key string) string { return "" } -func (b *ArgoBuilder) isService() bool{ - // for dev purpose do not commit to main - if os.Getenv("test_service") != ""{ - return true - } +func (b *ArgoBuilder) isService(id string) bool{ + + comp := b.OriginWorkflow.Graph.Items[id] - comp_list := b.getProcessings() - - if len(comp_list) != 1 { + if comp.Processing == nil { return false } - comp := comp_list[0] + _, is_exposed := comp.Processing.ResourceModel.Model["expose"] + return is_exposed +} + + +func (b *ArgoBuilder) addLabel(name string, id string) { + argo_name := getArgoName(name,id) + for _, template := range b.Workflow.Spec.Templates{ + if template.Name == argo_name{ + template.Metadata.Labels["app"] = "service-workflow" + return + } + } +} - return comp.Data.ResourceModel.Model["port"].Value != "" && comp.Data.ResourceModel.Model["hostname"].Value != "" -} \ No newline at end of file diff --git a/workflow_builder/argo_services.go b/workflow_builder/argo_services.go new file mode 100644 index 0000000..a990deb --- /dev/null +++ b/workflow_builder/argo_services.go @@ -0,0 +1,148 @@ +package workflow_builder + +import ( + "oc-monitord/models" + "strconv" + "strings" + + "cloud.o-forge.io/core/oc-lib/models/resource_model" + "cloud.o-forge.io/core/oc-lib/models/resources/workflow/graph" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "gopkg.in/yaml.v3" +) + +// TODO : refactor this method or the deserialization process in oc-lib to get rid of the mongo code +func getExposeContract(expose resource_model.Model) map[string]map[string]string { + contract := make(map[string]map[string]string,0) + + mapped_info := bson.M{} + // var contract PortTranslation + _ , byt, _ := bson.MarshalValue(expose.Value) + + bson.Unmarshal(byt,&mapped_info) + + for _,v := range mapped_info { + port := v.(primitive.M)["Key"].(string) + // exposed_port := map[string]interface{}{data["Key"] : ""} + port_translation := v.(primitive.M)["Value"] + contract[port] = map[string]string{} + for _,v2 := range port_translation.(primitive.A) { + if v2.(primitive.M)["Key"] == "reverse" { + contract[port]["reverse"] = v2.(primitive.M)["Value"].(string) + } + if v2.(primitive.M)["Key"] == "PAT" { + contract[port]["PAT"] = v2.(primitive.M)["Value"].(string) + } + } + } + return contract +} + + +func (b *ArgoBuilder) CreateService(processing graph.GraphItem) models.Service{ + + // model { + // Type : "dict", + // Value : { + // "80" : { + // "reverse" : "", + // "PAT" : "34000" + // }, + // "344" : { + // "reverse" : "", + // "PAT" : "34400" + // } + // } + // } + + + new_service := models.Service{APIVersion: "v1", + Kind: "Service", + Metadata: models.Metadata{ + Name: "workflow-service" , + }, + Spec: models.ServiceSpec{ + Selector: map[string]string{"app": "oc-service"}, + Ports: []models.ServicePort{ + }, + Type: "NodePort", + }, + } + + completeServicePorts(&new_service, processing) + yamlified, _ := yaml.Marshal(new_service) + x := string(yamlified) + _ = x + return new_service +} + +func completeServicePorts(service *models.Service, processing graph.GraphItem) { + + contract := getExposeContract(processing.Processing.ResourceModel.Model["expose"]) + + + for str_port,translation_dict := range contract{ + + port, err := strconv.ParseInt(str_port, 10, 64) + if err != nil { + logger.Error().Msg("Could not convert " + str_port + "to an int") + return + } + + + if _, ok := translation_dict["PAT"]; ok{ + port_translation, err := strconv.ParseInt(translation_dict["PAT"], 10, 64) + if err != nil { + logger.Error().Msg("Could not convert " + translation_dict["PAT"] + "to an int") + return + } + + + + new_port_translation := models.ServicePort{ + Name: strings.ToLower(processing.Processing.Name) + processing.ID, + Port: port_translation-30000, + TargetPort: port, + NodePort: port_translation, + Protocol: "TCP", + } + service.Spec.Ports = append(service.Spec.Ports, new_port_translation) + } + + } + + return +} + +func (b *ArgoBuilder) createService(service models.Service, processing_name string, processing_id string) { + if b.Services != nil{ + b.Services.Spec.Ports = append(b.Services.Spec.Ports, service.Spec.Ports...) + }else { + b.Services = &service + } + + b.addLabel(processing_name,processing_id) + +} + +func (b *ArgoBuilder) addServiceToArgo() error { + service_manifest, err := yaml.Marshal(b.Services) + if err != nil { + logger.Error().Msg("Could not marshal service manifest") + return err + } + + service_template := models.Template{Name: "workflow-service-pod", + Resource: models.ServiceResource{ + Action: "create", + SuccessCondition: "status.succeeded > 0", + FailureCondition: "status.failed > 3", + SetOwnerReference: true, + Manifest: string(service_manifest), + }, + } + b.Workflow.Spec.Templates = append(b.Workflow.Spec.Templates, service_template) + + return nil +} \ No newline at end of file