core/oc-discovery
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Daemons Search

Browse Source
This commit is contained in:
mr
2026-01-28 17:22:29 +01:00
parent 0ed2fc0f15
commit 7fd258dc9d
22 changed files with 1631 additions and 680 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
daemons/pubsub/handler.go Normal file
View File

@@ -0,0 +1,89 @@
package pubsub
import (
"context"
"encoding/json"
"errors"
"oc-discovery/daemons"
"oc-discovery/daemons/dht"
"oc-discovery/models"
"cloud.o-forge.io/core/oc-lib/models/peer"
"cloud.o-forge.io/core/oc-lib/models/resources"
"cloud.o-forge.io/core/oc-lib/tools"
)
func (ps *PubSubService) handleEvent(ctx context.Context, topicName string, evt models.Event) error {
action := ps.getTopicName(topicName)
if err := ps.handleEventFromPartner(evt, action); err != nil {
return err
}
if err := ps.handleEventSearch(ctx, evt, action); err != nil {
return err
}
return nil
}
func (ps *PubSubService) handleEventFromPartner(evt models.Event, action tools.PubSubAction) error {
if !(action == tools.PB_CREATE || action == tools.PB_UPDATE || action == tools.PB_DELETE) {
return nil
}
resource, err := resources.ToResource(int(evt.DataType), evt.Payload)
if err != nil {
return err
}
b, err := json.Marshal(resource)
if err != nil {
return err
}
switch action {
case tools.PB_CREATE:
case tools.PB_UPDATE:
tools.NewNATSCaller().SetNATSPub(tools.CREATE_RESOURCE, tools.NATSResponse{
FromApp: "oc-discovery",
Datatype: tools.DataType(evt.DataType),
Method: int(tools.CREATE_RESOURCE),
Payload: b,
})
case tools.PB_DELETE:
tools.NewNATSCaller().SetNATSPub(tools.REMOVE_RESOURCE, tools.NATSResponse{
FromApp: "oc-discovery",
Datatype: tools.DataType(evt.DataType),
Method: int(tools.REMOVE_RESOURCE),
Payload: b,
})
default:
return errors.New("no action authorized available : " + action.String())
}
return nil
}
func (ps *PubSubService) handleEventSearch( // only : on partner followings. 3 canals for every partner.
ctx context.Context,
evt models.Event,
action tools.PubSubAction,
) error {
if !(action == tools.PB_SEARCH_RESPONSE || action == tools.PB_SEARCH) {
return nil
}
if p, err := dht.GetDHTService().GetPeer(ctx, evt.From); err == nil {
if err := daemons.VerifyPeer([]*peer.Peer{p}, evt); err != nil {
return err
}
switch action {
case tools.PB_SEARCH_RESPONSE:
if err := ps.retrieveResponse(ctx, p, evt); err != nil {
return err
}
case tools.PB_SEARCH: // when someone ask for search.
if p, err := dht.GetDHTService().GetPeer(ctx, evt.From); err == nil {
if err := ps.sendResponse(ctx, p, evt); err != nil {
return err
}
}
default:
return nil
}
}
return nil
}

47
daemons/pubsub/nats.go Normal file
View File

@@ -0,0 +1,47 @@
package pubsub
import (
"context"
"encoding/json"
"fmt"
"cloud.o-forge.io/core/oc-lib/tools"
)
func ListenNATS() {
tools.NewNATSCaller().ListenNats(map[tools.NATSMethod]func(tools.NATSResponse){
tools.PROPALGATION_EVENT: func(resp tools.NATSResponse) {
var propalgation tools.PropalgationMessage
err := json.Unmarshal(resp.Payload, &propalgation)
var dt *tools.DataType
if propalgation.DataType > 0 {
dtt := tools.DataType(propalgation.DataType)
dt = &dtt
}
if err == nil {
switch propalgation.Action {
case tools.PB_CREATE:
case tools.PB_UPDATE:
case tools.PB_DELETE:
GetPubSubService().ToPartnerPublishEvent(
context.Background(),
propalgation.Action,
dt, propalgation.User,
propalgation.Payload,
)
case tools.PB_SEARCH:
m := map[string]interface{}{}
json.Unmarshal(propalgation.Payload, &m)
GetPubSubService().SearchPublishEvent(
context.Background(),
dt,
fmt.Sprintf("%v", m["type"]),
propalgation.User,
fmt.Sprintf("%v", m["search"]),
)
}
}
},
})
}

205
daemons/pubsub/publish.go Normal file
View File

@@ -0,0 +1,205 @@
package pubsub
import (
"context"
"encoding/json"
"errors"
"oc-discovery/daemons"
"oc-discovery/models"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/dbs"
"cloud.o-forge.io/core/oc-lib/models/peer"
"cloud.o-forge.io/core/oc-lib/tools"
)
func (ps *PubSubService) SearchPublishEvent(
ctx context.Context,
dt *tools.DataType,
typ string,
user string,
search string,
) error {
switch typ {
case "partner":
ps.searchPartnersPublishEvent(
ctx, dt, user, search,
)
case "all":
b, err := json.Marshal(map[string]string{
"search": search,
})
if err != nil {
return err
}
ps.searchPublishEvent(
ctx, dt, user, "", b,
)
case "known":
ps.searchKnownPublishEvent(
ctx, dt, user, search,
)
default:
return errors.New("no type of research found")
}
return nil
}
func (ps *PubSubService) searchPartnersPublishEvent(
ctx context.Context,
dt *tools.DataType,
user string,
search string,
) error {
access := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil)
f := &dbs.Filters{
And: map[string][]dbs.Filter{ // search by name if no filters are provided
"state": {{Operator: dbs.EQUAL.String(), Value: peer.ONLINE.EnumIndex()}},
"relation": {{Operator: dbs.EQUAL.String(), Value: peer.PARTNER.EnumIndex()}},
},
}
if search != "" {
f.Or = map[string][]dbs.Filter{ // filter by like name, short_description, description, owner, url if no filters are provided
"abstractintanciatedresource.abstractresource.abstractobject.name": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.type": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.short_description": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.description": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.owners.name": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.abstractobject.creator_id": {{Operator: dbs.EQUAL.String(), Value: search}},
}
}
b, err := json.Marshal(map[string]string{
"search": search,
})
if err != nil {
return err
}
peersKnown := access.Search(f, "", false)
for _, known := range peersKnown.Data {
if err := ps.searchPublishEvent(ctx, dt, user, known.GetID(), b); err != nil {
return err
}
}
return nil
}
func (ps *PubSubService) searchKnownPublishEvent(
ctx context.Context,
dt *tools.DataType,
user string,
search string,
) error {
access := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil)
f := &dbs.Filters{
And: map[string][]dbs.Filter{ // search by name if no filters are provided
"state": {{Operator: dbs.EQUAL.String(), Value: peer.ONLINE.EnumIndex()}},
"relation": {{Operator: dbs.NOT.String(), Value: &dbs.Filters{
And: map[string][]dbs.Filter{
"relation": {{Operator: dbs.EQUAL.String(), Value: peer.BLACKLIST.EnumIndex()}},
},
}}},
},
}
if search != "" {
f.Or = map[string][]dbs.Filter{ // filter by like name, short_description, description, owner, url if no filters are provided
"abstractintanciatedresource.abstractresource.abstractobject.name": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.type": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.short_description": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.description": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.owners.name": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.abstractobject.creator_id": {{Operator: dbs.EQUAL.String(), Value: search}},
}
}
b, err := json.Marshal(map[string]string{
"search": search,
})
if err != nil {
return err
}
peersKnown := access.Search(f, "", false)
for _, known := range peersKnown.Data {
if err := ps.searchPublishEvent(ctx, dt, user, known.GetID(), b); err != nil {
return err
}
}
return nil
}
func (ps *PubSubService) searchPublishEvent(
ctx context.Context,
dt *tools.DataType,
user string,
peerID string,
payload []byte,
) error {
id, err := oclib.GenerateNodeID()
if err != nil {
return err
}
if err := ps.subscribeEvents(ctx, dt, tools.PB_SEARCH_RESPONSE, id, 60); err != nil { // TODO Catpure Event !
return err
}
return ps.publishEvent(ctx, dt, tools.PB_SEARCH, user, peerID, payload, false)
}
func (ps *PubSubService) ToPartnerPublishEvent(
ctx context.Context, action tools.PubSubAction, dt *tools.DataType, user string, payload []byte) error {
id, err := oclib.GenerateNodeID()
if err != nil {
return err
}
return ps.publishEvent(ctx, dt, action, user, id, payload, false)
}
func (ps *PubSubService) publishEvent(
ctx context.Context,
dt *tools.DataType,
action tools.PubSubAction,
user string,
peerID string,
payload []byte,
chanNamedByDt bool,
) error {
name := action.String() + "#" + peerID
if chanNamedByDt && dt != nil { // if a datatype is precised then : app.action.datatype#peerID
name = action.String() + "." + (*dt).String() + "#" + peerID
}
from, err := oclib.GenerateNodeID()
if err != nil {
return err
}
evt := models.Event{
Type: name,
From: from,
User: user,
Timestamp: time.Now().Unix(),
Payload: payload,
}
if dt != nil {
evt.DataType = int64(dt.EnumIndex())
} else {
evt.DataType = -1
}
body, _ := json.Marshal(evt)
priv, err := daemons.LoadKeyFromFile(false)
if err != nil {
return err
}
sig, _ := priv.Sign(body)
evt.Signature = sig
msg, _ := json.Marshal(evt)
topic, err := ps.PS.Join(name)
if err != nil {
return err
}
return topic.Publish(ctx, msg)
}
// TODO REVIEW PUBLISHING + ADD SEARCH ON PUBLIC : YES
// TODO : Search should verify DataType

127
daemons/pubsub/service.go Normal file
View File

@@ -0,0 +1,127 @@
package pubsub
import (
"context"
"encoding/json"
"oc-discovery/models"
"strings"
"sync"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/dbs"
"cloud.o-forge.io/core/oc-lib/models/peer"
"cloud.o-forge.io/core/oc-lib/models/resources"
"cloud.o-forge.io/core/oc-lib/tools"
pubsub "github.com/libp2p/go-libp2p-pubsub"
)
type PubSubService struct {
PS *pubsub.PubSub
Subscription []string
mutex sync.RWMutex
}
var pubsubSingleton *PubSubService
func Init(ctx context.Context, ps *pubsub.PubSub) {
pubsubSingleton = &PubSubService{
PS: ps,
Subscription: []string{},
}
pubsubSingleton.initSubscribeEvents(ctx)
}
func GetPubSubService() *PubSubService {
return pubsubSingleton
}
func (ps *PubSubService) getTopicName(topicName string) tools.PubSubAction {
ns := strings.Split(topicName, ".")
if len(ns) > 0 {
return tools.GetActionString(ns[0])
}
return tools.NONE
}
func (abs *PubSubService) retrieveResponse(ctx context.Context, p *peer.Peer, event models.Event) error {
res, err := resources.ToResource(int(event.DataType), event.Payload)
if err != nil || res == nil {
return nil
}
b, err := json.Marshal(res.Serialize(res))
tools.NewNATSCaller().SetNATSPub(tools.CATALOG_SEARCH_EVENT, tools.NATSResponse{
FromApp: "oc-discovery",
Datatype: tools.DataType(event.DataType),
Method: int(tools.CATALOG_SEARCH_EVENT),
Payload: b,
})
return nil
}
func (abs *PubSubService) sendResponse(ctx context.Context, p *peer.Peer, event models.Event) error {
dts := []oclib.LibDataEnum{oclib.LibDataEnum(event.DataType)}
if event.DataType == -1 { // expect all resources
dts = []oclib.LibDataEnum{oclib.LibDataEnum(oclib.COMPUTE_RESOURCE), oclib.LibDataEnum(oclib.STORAGE_RESOURCE),
oclib.LibDataEnum(oclib.PROCESSING_RESOURCE), oclib.LibDataEnum(oclib.DATA_RESOURCE), oclib.LibDataEnum(oclib.WORKFLOW_RESOURCE)}
}
var m map[string]string
err := json.Unmarshal(event.Payload, &m)
if err != nil {
return err
}
for _, dt := range dts {
access := oclib.NewRequestAdmin(oclib.LibDataEnum(event.DataType), nil)
peerID := p.GetID()
searched := access.Search(abs.filterPeer(peerID, m["search"]), "", false)
for _, ss := range searched.Data {
if j, err := json.Marshal(ss); err == nil {
if event.DataType != -1 {
ndt := tools.DataType(dt.EnumIndex())
abs.publishEvent(ctx, &ndt, tools.PB_SEARCH_RESPONSE, event.User, peerID, j, true)
} else {
abs.publishEvent(ctx, nil, tools.PB_SEARCH_RESPONSE, event.User, peerID, j, true)
}
}
}
}
return nil
}
func (abs *PubSubService) filterPeer(peerID string, search string) *dbs.Filters {
id, err := oclib.GetMySelf()
if err != nil {
return nil
}
filter := map[string][]dbs.Filter{
"creator_id": {{Operator: dbs.EQUAL.String(), Value: id}}, // is my resource...
"": {{Operator: dbs.OR.String(), Value: &dbs.Filters{
Or: map[string][]dbs.Filter{
"abstractobject.access_mode": {{Operator: dbs.EQUAL.String(), Value: 1}}, // if public
"abstractinstanciatedresource.instances": {{Operator: dbs.ELEMMATCH.String(), Value: &dbs.Filters{ // or got a partners instances
And: map[string][]dbs.Filter{
"resourceinstance.partnerships": {{Operator: dbs.ELEMMATCH.String(), Value: &dbs.Filters{
And: map[string][]dbs.Filter{
"resourcepartnership.peer_groups." + peerID: {{Operator: dbs.EXISTS.String(), Value: true}},
},
}}},
},
}}},
},
}}},
}
if search != "" {
filter[" "] = []dbs.Filter{{Operator: dbs.OR.String(), Value: &dbs.Filters{
Or: map[string][]dbs.Filter{ // filter by like name, short_description, description, owner, url if no filters are provided
"abstractintanciatedresource.abstractresource.abstractobject.name": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.type": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.short_description": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.description": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.owners.name": {{Operator: dbs.LIKE.String(), Value: search}},
"abstractintanciatedresource.abstractresource.abstractobject.creator_id": {{Operator: dbs.EQUAL.String(), Value: search}},
},
}}}
}
return &dbs.Filters{
And: filter,
}
}

139
daemons/pubsub/subscribe.go Normal file
View File

@@ -0,0 +1,139 @@
package pubsub
import (
"context"
"encoding/json"
"errors"
"fmt"
"oc-discovery/daemons"
"oc-discovery/daemons/dht"
"oc-discovery/models"
"slices"
"time"
oclib "cloud.o-forge.io/core/oc-lib"
"cloud.o-forge.io/core/oc-lib/models/peer"
"cloud.o-forge.io/core/oc-lib/tools"
pubsub "github.com/libp2p/go-libp2p-pubsub"
)
func (ps *PubSubService) initSubscribeEvents(ctx context.Context) error {
ourPeerID, err := oclib.GenerateNodeID()
if err != nil {
return err
}
if err := ps.subscribeEvents(ctx, nil, tools.PB_SEARCH, "", -1); err != nil {
return err
}
if err := ps.subscribeEvents(ctx, nil, tools.PB_SEARCH, ourPeerID, -1); err != nil { // we subscribe at our proprer deductible search adresse.
return err
}
if err := ps.initPartnersSubscribeEvents(ctx); err != nil {
return nil
}
return nil
}
func (ps *PubSubService) initPartnersSubscribeEvents(ctx context.Context) error {
// search all your partners : we check in base for this because we keep actively peer state if partners
access := oclib.NewRequestAdmin(oclib.LibDataEnum(oclib.PEER), nil)
peers := access.Search(nil, fmt.Sprintf("%v", peer.PARTNER.EnumIndex()), false)
for _, p := range peers.Data {
if err := ps.PartnerSubscribeEvents(ctx, p.(*peer.Peer).PeerID); err != nil {
return err
}
}
return nil
}
func (ps *PubSubService) PartnerSubscribeEvents(ctx context.Context, myPartnerPeerID string) error {
if myPartnerPeerID == "" {
return errors.New("should discover a particular partner peer")
}
for _, action := range []tools.PubSubAction{tools.PB_CREATE, tools.PB_UPDATE, tools.PB_DELETE} {
if err := ps.subscribeEvents(ctx, nil, action, myPartnerPeerID, -1); err != nil {
return err
}
}
return nil
}
// generic function to subscribe to DHT flow of event
func (ps *PubSubService) subscribeEvents(
ctx context.Context, dt *tools.DataType, action tools.PubSubAction, peerID string, timeout int,
) error {
// define a name app.action#peerID
name := action.String() + "#" + peerID
if dt != nil { // if a datatype is precised then : app.action.datatype#peerID
name = action.String() + "." + (*dt).String() + "#" + peerID
}
topic, err := ps.PS.Join(name) // find out the topic
if err != nil {
return err
}
sub, err := topic.Subscribe() // then subscribe to it
if err != nil {
return err
}
ps.mutex.Lock() // add safely in cache your subscription.
ps.Subscription = append(ps.Subscription, name)
ps.mutex.Unlock()
// launch loop waiting for results.
go ps.waitResults(ctx, sub, name, timeout)
return nil
}
func (ps *PubSubService) waitResults(ctx context.Context, sub *pubsub.Subscription, topicName string, timeout int) {
logger := oclib.GetLogger()
defer ctx.Done()
for {
ps.mutex.Lock() // check safely if cache is actually notified subscribed to topic
if !slices.Contains(ps.Subscription, topicName) { // if not kill the loop.
break
}
ps.mutex.Unlock()
// if still subscribed -> wait for new message
var cancel context.CancelFunc
if timeout != -1 {
ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
defer cancel()
}
msg, err := sub.Next(ctx)
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
// timeout hit, no message before deadline kill subsciption.
ps.mutex.Lock()
subs := []string{}
for _, ss := range ps.Subscription {
if ss != topicName {
subs = append(subs, ss)
}
}
ps.Subscription = subs
ps.mutex.Unlock()
return
}
continue
}
var evt models.Event
if err := json.Unmarshal(msg.Data, &evt); err != nil { // map to event
continue
}
if p, err := dht.GetDHTService().GetPeer(ctx, evt.From); err == nil {
if err := ps.processEventPeerKnown(ctx, p, evt, topicName); err != nil {
logger.Err(err)
}
}
}
}
func (ps *PubSubService) processEventPeerKnown(
ctx context.Context, p *peer.Peer, event models.Event, topicName string) error {
if err := daemons.VerifyPeer([]*peer.Peer{p}, event); err != nil {
return err
}
return ps.handleEvent(ctx, topicName, event)
}