Simple Architecture
This commit is contained in:
331
daemons/node/common/common_cache.go
Normal file
331
daemons/node/common/common_cache.go
Normal file
@@ -0,0 +1,331 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/protocol"
|
||||
)
|
||||
|
||||
type Score struct {
|
||||
FirstContacted time.Time
|
||||
UptimeTracker *UptimeTracker
|
||||
LastFillRate float64
|
||||
Score float64
|
||||
// IsSeed marks indexers that came from the IndexerAddresses static config.
|
||||
// Seeds are sticky: they are never evicted by the score threshold alone.
|
||||
// A seed is only removed when: (a) heartbeat fails, or (b) it sends
|
||||
// SuggestMigrate and the node already has MinIndexer non-seed alternatives.
|
||||
IsSeed bool
|
||||
// challenge bookkeeping (2-3 peers per batch, raw data returned by indexer)
|
||||
hbCount int // heartbeats sent since last challenge batch
|
||||
nextChallenge int // send challenges when hbCount reaches this (rand 1-10)
|
||||
challengeTotal int // number of own-PeerID challenges sent (ground truth)
|
||||
challengeCorrect int // own PeerID found AND lastSeen within 2×interval
|
||||
// fill rate consistency: cross-check reported fillRate vs peerCount/maxNodes
|
||||
fillChecked int
|
||||
fillConsistent int
|
||||
// BornAt stability
|
||||
LastBornAt time.Time
|
||||
bornAtChanges int
|
||||
// DHT challenge
|
||||
dhtChecked int
|
||||
dhtSuccess int
|
||||
dhtBatchCounter int
|
||||
// Peer witnesses
|
||||
witnessChecked int
|
||||
witnessConsistent int
|
||||
// WitnessPool: up to 3 witnesses last reported by this indexer.
|
||||
// Used for indirect probing when the indexer becomes unreachable.
|
||||
// Oldest entry is replaced when the pool is full and a fresher witness arrives.
|
||||
WitnessPool []WitnessCacheEntry
|
||||
}
|
||||
|
||||
// WitnessCacheEntry holds one witness AddrInfo with its last-seen timestamp.
|
||||
const maxWitnessPool = 3
|
||||
|
||||
type WitnessCacheEntry struct {
|
||||
AI pp.AddrInfo
|
||||
SeenAt time.Time
|
||||
}
|
||||
|
||||
// UpdateWitnessPool inserts or refreshes a witness entry.
|
||||
// If the pool is full and the witness is new, the oldest entry is replaced.
|
||||
func (s *Score) UpdateWitnessPool(w pp.AddrInfo) {
|
||||
for i, e := range s.WitnessPool {
|
||||
if e.AI.ID == w.ID {
|
||||
s.WitnessPool[i].AI = w
|
||||
s.WitnessPool[i].SeenAt = time.Now()
|
||||
return
|
||||
}
|
||||
}
|
||||
entry := WitnessCacheEntry{AI: w, SeenAt: time.Now()}
|
||||
if len(s.WitnessPool) < maxWitnessPool {
|
||||
s.WitnessPool = append(s.WitnessPool, entry)
|
||||
return
|
||||
}
|
||||
// Replace oldest.
|
||||
oldest := 0
|
||||
for i, e := range s.WitnessPool {
|
||||
if e.SeenAt.Before(s.WitnessPool[oldest].SeenAt) {
|
||||
oldest = i
|
||||
}
|
||||
}
|
||||
s.WitnessPool[oldest] = entry
|
||||
}
|
||||
|
||||
// computeNodeSideScore computes the node's quality assessment of an indexer from raw metrics.
|
||||
// All ratios are in [0,1]; result is in [0,100].
|
||||
// - uptimeRatio : gap-aware fraction of lifetime the indexer was reachable
|
||||
// - challengeAccuracy: own-PeerID challenges answered correctly (found + recent lastSeen)
|
||||
// - latencyScore : 1 - RTT/maxRTT, clamped [0,1]
|
||||
// - fillScore : 1 - fillRate — prefer less-loaded indexers
|
||||
// - fillConsistency : fraction of ticks where peerCount/maxNodes ≈ fillRate (±10%)
|
||||
func (s *Score) ComputeNodeSideScore(latencyScore float64) float64 {
|
||||
uptime := s.UptimeTracker.UptimeRatio()
|
||||
challengeAccuracy := 1.0
|
||||
if s.challengeTotal > 0 {
|
||||
challengeAccuracy = float64(s.challengeCorrect) / float64(s.challengeTotal)
|
||||
}
|
||||
fillScore := 1.0 - s.LastFillRate
|
||||
fillConsistency := 1.0
|
||||
if s.fillChecked > 0 {
|
||||
fillConsistency = float64(s.fillConsistent) / float64(s.fillChecked)
|
||||
}
|
||||
witnessConsistency := 1.0
|
||||
if s.witnessChecked > 0 {
|
||||
witnessConsistency = float64(s.witnessConsistent) / float64(s.witnessChecked)
|
||||
}
|
||||
dhtSuccessRate := 1.0
|
||||
if s.dhtChecked > 0 {
|
||||
dhtSuccessRate = float64(s.dhtSuccess) / float64(s.dhtChecked)
|
||||
}
|
||||
base := ((0.20 * uptime) +
|
||||
(0.20 * challengeAccuracy) +
|
||||
(0.15 * latencyScore) +
|
||||
(0.10 * fillScore) +
|
||||
(0.10 * fillConsistency) +
|
||||
(0.15 * witnessConsistency) +
|
||||
(0.10 * dhtSuccessRate)) * 100
|
||||
// BornAt stability: each unexpected BornAt change penalises by 30%.
|
||||
bornAtPenalty := 1.0 - 0.30*float64(s.bornAtChanges)
|
||||
if bornAtPenalty < 0 {
|
||||
bornAtPenalty = 0
|
||||
}
|
||||
return base * bornAtPenalty
|
||||
}
|
||||
|
||||
type Directory struct {
|
||||
MuAddr sync.RWMutex
|
||||
MuScore sync.RWMutex
|
||||
MuStream sync.RWMutex
|
||||
Addrs map[string]*pp.AddrInfo
|
||||
Scores map[string]*Score
|
||||
Nudge chan struct{}
|
||||
Streams ProtocolStream
|
||||
}
|
||||
|
||||
func (d *Directory) ExistsScore(a string) bool {
|
||||
d.MuScore.RLock()
|
||||
defer d.MuScore.RUnlock()
|
||||
for addr, ai := range d.Scores {
|
||||
if ai != nil && (a == addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *Directory) GetScore(a string) *Score {
|
||||
d.MuScore.RLock()
|
||||
defer d.MuScore.RUnlock()
|
||||
for addr, s := range d.Scores {
|
||||
if s != nil && (a == addr) {
|
||||
sCopy := *s
|
||||
return &sCopy
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Directory) GetScores() map[string]*Score {
|
||||
d.MuScore.RLock()
|
||||
defer d.MuScore.RUnlock()
|
||||
score := map[string]*Score{}
|
||||
for addr, s := range d.Scores {
|
||||
score[addr] = s
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func (d *Directory) DeleteScore(a string) {
|
||||
d.MuScore.RLock()
|
||||
defer d.MuScore.RUnlock()
|
||||
score := map[string]*Score{}
|
||||
for addr, s := range d.Scores {
|
||||
if a != addr {
|
||||
score[addr] = s
|
||||
}
|
||||
}
|
||||
d.Scores = score
|
||||
}
|
||||
|
||||
func (d *Directory) SetScore(addr string, score *Score) *pp.AddrInfo {
|
||||
d.MuScore.Lock()
|
||||
defer d.MuScore.Unlock()
|
||||
d.Scores[addr] = score
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Directory) ExistsAddr(addrOrId string) bool {
|
||||
d.MuAddr.RLock()
|
||||
defer d.MuAddr.RUnlock()
|
||||
for addr, ai := range d.Addrs {
|
||||
if ai != nil && (addrOrId == ai.ID.String() || addrOrId == addr) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (d *Directory) GetAddr(addrOrId string) *pp.AddrInfo {
|
||||
d.MuAddr.RLock()
|
||||
defer d.MuAddr.RUnlock()
|
||||
for addr, ai := range d.Addrs {
|
||||
if ai != nil && (addrOrId == ai.ID.String() || addrOrId == addr) {
|
||||
aiCopy := *ai
|
||||
return &aiCopy
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Directory) DeleteAddr(a string) {
|
||||
d.MuAddr.RLock()
|
||||
defer d.MuAddr.RUnlock()
|
||||
addrs := map[string]*pp.AddrInfo{}
|
||||
for addr, s := range d.Addrs {
|
||||
if a != addr {
|
||||
addrs[addr] = s
|
||||
}
|
||||
}
|
||||
d.Addrs = addrs
|
||||
}
|
||||
|
||||
func (d *Directory) SetAddr(addr string, info *pp.AddrInfo) *pp.AddrInfo {
|
||||
d.MuAddr.Lock()
|
||||
defer d.MuAddr.Unlock()
|
||||
d.Addrs[addr] = info
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *Directory) GetAddrIDs() []pp.ID {
|
||||
d.MuAddr.RLock()
|
||||
defer d.MuAddr.RUnlock()
|
||||
indexers := make([]pp.ID, 0, len(d.Addrs))
|
||||
for _, ai := range d.Addrs {
|
||||
if ai != nil {
|
||||
indexers = append(indexers, ai.ID)
|
||||
}
|
||||
}
|
||||
return Shuffle(indexers)
|
||||
}
|
||||
|
||||
func (d *Directory) GetAddrsStr() []string {
|
||||
d.MuAddr.RLock()
|
||||
defer d.MuAddr.RUnlock()
|
||||
indexers := make([]string, 0, len(d.Addrs))
|
||||
for s, ai := range d.Addrs {
|
||||
if ai != nil {
|
||||
indexers = append(indexers, s)
|
||||
}
|
||||
}
|
||||
|
||||
return Shuffle(indexers)
|
||||
}
|
||||
|
||||
type Entry struct {
|
||||
Addr string
|
||||
Info *pp.AddrInfo
|
||||
}
|
||||
|
||||
func (d *Directory) GetAddrs() []Entry {
|
||||
d.MuAddr.RLock()
|
||||
defer d.MuAddr.RUnlock()
|
||||
indexers := make([]Entry, 0, len(d.Addrs))
|
||||
for addr, ai := range d.Addrs {
|
||||
if ai != nil {
|
||||
indexers = append(indexers, Entry{
|
||||
Addr: addr,
|
||||
Info: ai,
|
||||
})
|
||||
}
|
||||
}
|
||||
return Shuffle(indexers)
|
||||
}
|
||||
|
||||
// NudgeIndexerHeartbeat signals the indexer heartbeat goroutine to fire immediately.
|
||||
func (d *Directory) NudgeIt() {
|
||||
select {
|
||||
case d.Nudge <- struct{}{}:
|
||||
default: // nudge already pending, skip
|
||||
}
|
||||
}
|
||||
|
||||
type ProtocolStream map[protocol.ID]map[pp.ID]*Stream
|
||||
|
||||
func (ps ProtocolStream) Get(protocol protocol.ID) map[pp.ID]*Stream {
|
||||
if ps[protocol] == nil {
|
||||
ps[protocol] = map[pp.ID]*Stream{}
|
||||
}
|
||||
|
||||
return ps[protocol]
|
||||
}
|
||||
|
||||
func (ps ProtocolStream) GetPerID(protocol protocol.ID, peerID pp.ID) *Stream {
|
||||
if ps[protocol] == nil {
|
||||
ps[protocol] = map[pp.ID]*Stream{}
|
||||
}
|
||||
return ps[protocol][peerID]
|
||||
}
|
||||
|
||||
func (ps ProtocolStream) Add(protocol protocol.ID, peerID *pp.ID, s *Stream) error {
|
||||
if ps[protocol] == nil {
|
||||
ps[protocol] = map[pp.ID]*Stream{}
|
||||
}
|
||||
if peerID != nil {
|
||||
if s != nil {
|
||||
ps[protocol][*peerID] = s
|
||||
} else {
|
||||
return errors.New("unable to add stream : stream missing")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ps ProtocolStream) Delete(protocol protocol.ID, peerID *pp.ID) {
|
||||
if streams, ok := ps[protocol]; ok {
|
||||
if peerID != nil && streams[*peerID] != nil && streams[*peerID].Stream != nil {
|
||||
streams[*peerID].Stream.Close()
|
||||
delete(streams, *peerID)
|
||||
} else {
|
||||
for _, s := range ps {
|
||||
for _, v := range s {
|
||||
if v.Stream != nil {
|
||||
v.Stream.Close()
|
||||
}
|
||||
}
|
||||
}
|
||||
delete(ps, protocol)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var Indexers = &Directory{
|
||||
Addrs: map[string]*pp.AddrInfo{},
|
||||
Scores: map[string]*Score{},
|
||||
Nudge: make(chan struct{}, 1),
|
||||
Streams: ProtocolStream{},
|
||||
}
|
||||
295
daemons/node/common/common_heartbeat.go
Normal file
295
daemons/node/common/common_heartbeat.go
Normal file
@@ -0,0 +1,295 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
)
|
||||
|
||||
type Heartbeat struct {
|
||||
Name string `json:"name"`
|
||||
Stream *Stream `json:"stream"`
|
||||
DID string `json:"did"`
|
||||
PeerID string `json:"peer_id"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
IndexersBinded []string `json:"indexers_binded"`
|
||||
Score float64
|
||||
// Record carries a fresh signed PeerRecord (JSON) so the receiving indexer
|
||||
// can republish it to the DHT without an extra round-trip.
|
||||
// Only set by nodes (not indexers heartbeating other indexers).
|
||||
Record json.RawMessage `json:"record,omitempty"`
|
||||
// Need is how many more indexers this node wants (MaxIndexer - current pool size).
|
||||
// The receiving indexer uses this to know how many suggestions to return.
|
||||
// 0 means the pool is full — no suggestions needed unless SuggestMigrate.
|
||||
Need int `json:"need,omitempty"`
|
||||
// Challenges is a list of PeerIDs the node asks the indexer to spot-check.
|
||||
// Always includes the node's own PeerID (ground truth) + up to 2 additional
|
||||
// known peers. Nil means no challenge this tick.
|
||||
Challenges []string `json:"challenges,omitempty"`
|
||||
// ChallengeDID asks the indexer to retrieve this DID from the DHT (every 5th batch).
|
||||
ChallengeDID string `json:"challenge_did,omitempty"`
|
||||
// Referent marks this indexer as the node's designated search referent.
|
||||
// Only one indexer per node receives Referent=true at a time (the best-scored one).
|
||||
// The indexer stores the node in its referencedNodes for distributed search.
|
||||
Referent bool `json:"referent,omitempty"`
|
||||
}
|
||||
|
||||
// SearchPeerRequest is sent by a node to an indexer via ProtocolSearchPeer.
|
||||
// The indexer broadcasts it on the GossipSub search mesh and streams results back.
|
||||
type SearchPeerRequest struct {
|
||||
QueryID string `json:"query_id"`
|
||||
// At least one of PeerID, DID, Name must be set.
|
||||
PeerID string `json:"peer_id,omitempty"`
|
||||
DID string `json:"did,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
}
|
||||
|
||||
// SearchQuery is broadcast on TopicSearchPeer by the receiving indexer.
|
||||
// EmitterID is the indexer's own PeerID — responding indexers open a
|
||||
// ProtocolSearchPeerResponse stream back to it.
|
||||
type SearchQuery struct {
|
||||
QueryID string `json:"query_id"`
|
||||
PeerID string `json:"peer_id,omitempty"`
|
||||
DID string `json:"did,omitempty"`
|
||||
Name string `json:"name,omitempty"`
|
||||
EmitterID string `json:"emitter_id"`
|
||||
}
|
||||
|
||||
// SearchPeerResult is sent by a responding indexer to the emitting indexer
|
||||
// via ProtocolSearchPeerResponse, and forwarded by the emitting indexer to
|
||||
// the node on the open ProtocolSearchPeer stream.
|
||||
type SearchPeerResult struct {
|
||||
QueryID string `json:"query_id"`
|
||||
Records []SearchHit `json:"records"`
|
||||
}
|
||||
|
||||
// SearchHit is a single peer found during distributed search.
|
||||
type SearchHit struct {
|
||||
PeerID string `json:"peer_id"`
|
||||
DID string `json:"did"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
// ChallengeEntry is the indexer's raw answer for one challenged peer.
|
||||
type ChallengeEntry struct {
|
||||
PeerID string `json:"peer_id"`
|
||||
Found bool `json:"found"`
|
||||
LastSeen time.Time `json:"last_seen,omitempty"` // zero if not found
|
||||
}
|
||||
|
||||
// HeartbeatResponse carries raw metrics only — no pre-cooked score.
|
||||
type HeartbeatResponse struct {
|
||||
FillRate float64 `json:"fill_rate"`
|
||||
PeerCount int `json:"peer_count"`
|
||||
MaxNodes int `json:"max_nodes"` // capacity — lets node cross-check fillRate
|
||||
BornAt time.Time `json:"born_at"`
|
||||
Challenges []ChallengeEntry `json:"challenges,omitempty"`
|
||||
// DHTFound / DHTPayload: response to a ChallengeDID request.
|
||||
DHTFound bool `json:"dht_found,omitempty"`
|
||||
DHTPayload json.RawMessage `json:"dht_payload,omitempty"`
|
||||
// Witnesses: random sample of connected nodes so the querying node can cross-check.
|
||||
Witnesses []pp.AddrInfo `json:"witnesses,omitempty"`
|
||||
// Suggestions: better indexers this indexer knows about via its DHT cache.
|
||||
// The node should open heartbeat connections to these (they become StaticIndexers).
|
||||
Suggestions []pp.AddrInfo `json:"suggestions,omitempty"`
|
||||
// SuggestMigrate: set when this indexer is overloaded (fill rate > threshold)
|
||||
// and is actively trying to hand the node off to the Suggestions list.
|
||||
// Seeds: node de-stickies this indexer once it has MinIndexer non-seed alternatives.
|
||||
// Non-seeds: node removes this indexer immediately if it has enough alternatives.
|
||||
SuggestMigrate bool `json:"suggest_migrate,omitempty"`
|
||||
}
|
||||
|
||||
// ComputeIndexerScore computes a composite quality score [0, 100] for the connecting peer.
|
||||
// - uptimeRatio: fraction of tracked lifetime online (gap-aware) — peer reliability
|
||||
// - bpms: bandwidth normalized to MaxExpectedMbps — link capacity
|
||||
// - diversity: indexer's own /24 subnet diversity — network topology quality
|
||||
// - latencyScore: 1 - RTT/maxRoundTrip — link responsiveness
|
||||
// - fillRate: fraction of indexer slots used (0=empty, 1=full) — collective trust signal:
|
||||
// a fuller indexer has been chosen and retained by many peers, which is evidence of quality.
|
||||
func (hb *Heartbeat) ComputeIndexerScore(uptimeRatio float64, bpms float64, diversity float64, latencyScore float64, fillRate float64) {
|
||||
hb.Score = ((0.20 * uptimeRatio) +
|
||||
(0.20 * bpms) +
|
||||
(0.20 * diversity) +
|
||||
(0.15 * latencyScore) +
|
||||
(0.25 * fillRate)) * 100
|
||||
}
|
||||
|
||||
type HeartbeatInfo []struct {
|
||||
Info []byte `json:"info"`
|
||||
}
|
||||
|
||||
// WitnessRequest is sent by a node to a peer to ask its view of a given indexer.
|
||||
type WitnessRequest struct {
|
||||
IndexerPeerID string `json:"indexer_peer_id"`
|
||||
}
|
||||
|
||||
// WitnessReport is returned by a peer in response to a WitnessRequest.
|
||||
type WitnessReport struct {
|
||||
Seen bool `json:"seen"`
|
||||
BornAt time.Time `json:"born_at,omitempty"`
|
||||
FillRate float64 `json:"fill_rate,omitempty"`
|
||||
Score float64 `json:"score,omitempty"`
|
||||
}
|
||||
|
||||
// HandleBandwidthProbe echoes back everything written on the stream, then closes.
|
||||
// It is registered by all participants so the measuring side (the heartbeat receiver)
|
||||
// can open a dedicated probe stream and read the round-trip latency + throughput.
|
||||
func HandleBandwidthProbe(s network.Stream) {
|
||||
defer s.Close()
|
||||
s.SetDeadline(time.Now().Add(10 * time.Second))
|
||||
io.Copy(s, s) // echo every byte back to the sender
|
||||
}
|
||||
|
||||
// HandleWitnessQuery answers a witness query: the caller wants to know
|
||||
// what this node thinks of a given indexer (identified by its PeerID).
|
||||
func HandleWitnessQuery(h host.Host, s network.Stream) {
|
||||
defer s.Close()
|
||||
s.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
var req WitnessRequest
|
||||
if err := json.NewDecoder(s).Decode(&req); err != nil {
|
||||
return
|
||||
}
|
||||
report := WitnessReport{}
|
||||
for _, ai := range Indexers.GetAddrs() {
|
||||
if ai.Info == nil || ai.Info.ID.String() != req.IndexerPeerID {
|
||||
continue
|
||||
}
|
||||
if score := Indexers.GetScore(addrKey(*ai.Info)); score != nil {
|
||||
report.Seen = true
|
||||
report.BornAt = score.LastBornAt
|
||||
report.FillRate = score.LastFillRate
|
||||
report.Score = score.Score
|
||||
}
|
||||
break
|
||||
}
|
||||
json.NewEncoder(s).Encode(report)
|
||||
}
|
||||
|
||||
// IndirectProbeIndexer asks each witness in the cache whether it still sees
|
||||
// the given indexer (by PeerID). Returns true if at least one witness confirms
|
||||
// it is alive — meaning our direct link is asymmetrically broken, not the indexer.
|
||||
// All probes run in parallel; the function blocks at most 5 seconds.
|
||||
func IndirectProbeIndexer(h host.Host, indexerPeerID string, pool []WitnessCacheEntry) bool {
|
||||
if len(pool) == 0 {
|
||||
return false
|
||||
}
|
||||
results := make(chan bool, len(pool))
|
||||
for _, e := range pool {
|
||||
go func(ai pp.AddrInfo) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
s, err := h.NewStream(ctx, ai.ID, ProtocolWitnessQuery)
|
||||
if err != nil {
|
||||
results <- false
|
||||
return
|
||||
}
|
||||
defer s.Reset()
|
||||
s.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if err := json.NewEncoder(s).Encode(WitnessRequest{IndexerPeerID: indexerPeerID}); err != nil {
|
||||
results <- false
|
||||
return
|
||||
}
|
||||
var rep WitnessReport
|
||||
if err := json.NewDecoder(s).Decode(&rep); err != nil {
|
||||
results <- false
|
||||
return
|
||||
}
|
||||
results <- rep.Seen
|
||||
}(e.AI)
|
||||
}
|
||||
for range pool {
|
||||
if <-results {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SupportsHeartbeat probes pid with a short-lived stream to verify it has
|
||||
// a ProtocolHeartbeat handler (i.e. it is an indexer, not a plain node).
|
||||
// Only protocol negotiation is performed — no data is sent.
|
||||
// Returns false on any error, including "protocol not supported".
|
||||
func SupportsHeartbeat(h host.Host, pid pp.ID) bool {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
s, err := h.NewStream(ctx, pid, ProtocolHeartbeat)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.Reset()
|
||||
return true
|
||||
}
|
||||
|
||||
// queryWitnesses contacts each witness in parallel, collects their view of the
|
||||
// indexer, and updates score.witnessChecked / score.witnessConsistent.
|
||||
// Called in a goroutine — must not hold any lock.
|
||||
func queryWitnesses(h host.Host, indexerPeerID string, indexerBornAt time.Time, indexerFillRate float64, witnesses []pp.AddrInfo, score *Score) {
|
||||
logger := oclib.GetLogger()
|
||||
type result struct{ consistent bool }
|
||||
results := make(chan result, len(witnesses))
|
||||
|
||||
for _, ai := range witnesses {
|
||||
if ai.ID == h.ID() {
|
||||
// Never query ourselves — skip and count as inconclusive.
|
||||
results <- result{}
|
||||
continue
|
||||
}
|
||||
go func(ai pp.AddrInfo) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
s, err := h.NewStream(ctx, ai.ID, ProtocolWitnessQuery)
|
||||
if err != nil {
|
||||
results <- result{}
|
||||
return
|
||||
}
|
||||
defer s.Close()
|
||||
s.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if err := json.NewEncoder(s).Encode(WitnessRequest{IndexerPeerID: indexerPeerID}); err != nil {
|
||||
results <- result{}
|
||||
return
|
||||
}
|
||||
var rep WitnessReport
|
||||
if err := json.NewDecoder(s).Decode(&rep); err != nil || !rep.Seen {
|
||||
results <- result{}
|
||||
return
|
||||
}
|
||||
// BornAt must be identical (fixed timestamp).
|
||||
bornAtOK := !rep.BornAt.IsZero() && rep.BornAt.Equal(indexerBornAt)
|
||||
// FillRate coherent within ±25% (it fluctuates normally).
|
||||
diff := rep.FillRate - indexerFillRate
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
fillOK := diff < 0.25
|
||||
consistent := bornAtOK && fillOK
|
||||
logger.Debug().
|
||||
Str("witness", ai.ID.String()).
|
||||
Bool("bornAt_ok", bornAtOK).
|
||||
Bool("fill_ok", fillOK).
|
||||
Msg("witness report")
|
||||
results <- result{consistent: consistent}
|
||||
}(ai)
|
||||
}
|
||||
|
||||
checked, consistent := 0, 0
|
||||
for range witnesses {
|
||||
r := <-results
|
||||
checked++
|
||||
if r.consistent {
|
||||
consistent++
|
||||
}
|
||||
}
|
||||
|
||||
if checked == 0 {
|
||||
return
|
||||
}
|
||||
score.witnessChecked += checked
|
||||
score.witnessConsistent += consistent
|
||||
}
|
||||
588
daemons/node/common/common_indexer_hb.go
Normal file
588
daemons/node/common/common_indexer_hb.go
Normal file
@@ -0,0 +1,588 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"oc-discovery/conf"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/protocol"
|
||||
)
|
||||
|
||||
var TimeWatcher time.Time
|
||||
|
||||
// retryRunning guards against launching multiple retryUntilSeedResponds goroutines.
|
||||
var retryRunning atomic.Bool
|
||||
|
||||
func ConnectToIndexers(h host.Host, minIndexer int, maxIndexer int, recordFn ...func() json.RawMessage) error {
|
||||
TimeWatcher = time.Now().UTC()
|
||||
logger := oclib.GetLogger()
|
||||
|
||||
// Bootstrap from IndexerAddresses seed set.
|
||||
addresses := strings.Split(conf.GetConfig().IndexerAddresses, ",")
|
||||
if len(addresses) > maxIndexer {
|
||||
addresses = addresses[0:maxIndexer]
|
||||
}
|
||||
for _, indexerAddr := range addresses {
|
||||
indexerAddr = strings.TrimSpace(indexerAddr)
|
||||
if indexerAddr == "" {
|
||||
continue
|
||||
}
|
||||
ad, err := pp.AddrInfoFromString(indexerAddr)
|
||||
if err != nil {
|
||||
logger.Err(err)
|
||||
continue
|
||||
}
|
||||
key := ad.ID.String()
|
||||
Indexers.SetAddr(key, ad)
|
||||
// Pre-create score entry with IsSeed=true so the sticky flag is set before
|
||||
// the first heartbeat tick (lazy creation in doTick would lose the flag).
|
||||
if !Indexers.ExistsScore(key) {
|
||||
Indexers.SetScore(key, &Score{
|
||||
FirstContacted: time.Now().UTC(),
|
||||
UptimeTracker: &UptimeTracker{FirstSeen: time.Now().UTC()},
|
||||
nextChallenge: rand.Intn(10) + 1,
|
||||
IsSeed: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
seeds := Indexers.GetAddrs()
|
||||
indexerCount := len(seeds)
|
||||
|
||||
if indexerCount < minIndexer {
|
||||
return fmt.Errorf("you run a node without indexers... your gonna be isolated.")
|
||||
}
|
||||
|
||||
// Start long-lived heartbeat to seed indexers. The single goroutine follows
|
||||
// all subsequent StaticIndexers changes.
|
||||
SendHeartbeat(context.Background(), ProtocolHeartbeat, conf.GetConfig().Name,
|
||||
h, Indexers, 20*time.Second, maxIndexer, recordFn...)
|
||||
|
||||
// Watch for inbound connections: if a peer connects to us and our pool has
|
||||
// room, probe it first to confirm it supports ProtocolHeartbeat (i.e. it is
|
||||
// an indexer). Plain nodes don't register the handler — the negotiation fails
|
||||
// instantly so we never pollute the pool with non-indexer peers.
|
||||
h.Network().Notify(&network.NotifyBundle{
|
||||
ConnectedF: func(n network.Network, c network.Conn) {
|
||||
if c.Stat().Direction != network.DirInbound {
|
||||
return
|
||||
}
|
||||
if len(Indexers.GetAddrs()) >= maxIndexer {
|
||||
return
|
||||
}
|
||||
peerID := c.RemotePeer()
|
||||
if Indexers.ExistsAddr(peerID.String()) {
|
||||
return
|
||||
}
|
||||
// Probe in a goroutine — ConnectedF must not block.
|
||||
go func(pid pp.ID) {
|
||||
if !SupportsHeartbeat(h, pid) {
|
||||
return // plain node, skip
|
||||
}
|
||||
if len(Indexers.GetAddrs()) >= maxIndexer {
|
||||
return
|
||||
}
|
||||
if Indexers.ExistsAddr(pid.String()) {
|
||||
return
|
||||
}
|
||||
addrs := h.Peerstore().Addrs(pid)
|
||||
if len(addrs) == 0 {
|
||||
return
|
||||
}
|
||||
ai := FilterLoopbackAddrs(pp.AddrInfo{ID: pid, Addrs: addrs})
|
||||
if len(ai.Addrs) == 0 {
|
||||
return
|
||||
}
|
||||
adCopy := ai
|
||||
Indexers.SetAddr(pid.String(), &adCopy)
|
||||
Indexers.NudgeIt()
|
||||
log := oclib.GetLogger()
|
||||
log.Info().Str("peer", pid.String()).
|
||||
Msg("[pool] inbound indexer peer added as candidate")
|
||||
}(peerID)
|
||||
},
|
||||
})
|
||||
|
||||
// Proactive DHT upgrade: once seeds are connected and the DHT routing table
|
||||
// is warm, discover better indexers and add them to the pool alongside the seeds.
|
||||
// Seeds stay as guaranteed anchors; scoring will demote poor performers over time.
|
||||
go func(seeds []Entry) {
|
||||
// Let seed connections establish and the DHT routing table warm up.
|
||||
time.Sleep(5 * time.Second)
|
||||
// For pure nodes (no IndexerService), spin up a lightweight DHT client.
|
||||
if discoveryDHT == nil {
|
||||
if len(seeds) == 0 {
|
||||
return
|
||||
}
|
||||
initNodeDHT(h, seeds)
|
||||
}
|
||||
if discoveryDHT == nil {
|
||||
return
|
||||
}
|
||||
current := len(Indexers.GetAddrs())
|
||||
need := maxIndexer - current
|
||||
if need <= 0 {
|
||||
need = maxIndexer / 2 // diversify even when pool is already at capacity
|
||||
}
|
||||
logger.Info().Int("need", need).Msg("[dht] proactive indexer discovery from DHT")
|
||||
replenishIndexersFromDHT(h, need)
|
||||
}(seeds)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// reconnectToSeeds re-adds the configured seed indexers to StaticIndexers as
|
||||
// sticky fallback entries. Called when the pool drops to zero so the node
|
||||
// never becomes completely isolated.
|
||||
func reconnectToSeeds() {
|
||||
logger := oclib.GetLogger()
|
||||
logger.Warn().Msg("[pool] all indexers lost, reconnecting to configured seeds")
|
||||
addresses := strings.Split(conf.GetConfig().IndexerAddresses, ",")
|
||||
for _, addrStr := range addresses {
|
||||
addrStr = strings.TrimSpace(addrStr)
|
||||
if addrStr == "" {
|
||||
continue
|
||||
}
|
||||
ad, err := pp.AddrInfoFromString(addrStr)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
key := ad.ID.String()
|
||||
Indexers.SetAddr(key, ad)
|
||||
if score := Indexers.GetScore(key); score == nil {
|
||||
Indexers.SetScore(key, &Score{
|
||||
FirstContacted: time.Now().UTC(),
|
||||
UptimeTracker: &UptimeTracker{FirstSeen: time.Now().UTC()},
|
||||
nextChallenge: rand.Intn(10) + 1,
|
||||
IsSeed: true,
|
||||
})
|
||||
} else {
|
||||
// Restore sticky flag so the seed is not immediately re-ejected.
|
||||
score.IsSeed = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// retryUntilSeedResponds loops with exponential backoff until at least one
|
||||
// configured seed is reachable again. Once seeds are back in the pool it
|
||||
// nudges the heartbeat loop and lets the normal DHT upgrade path take over.
|
||||
// Should be called in a goroutine — it blocks until the situation resolves.
|
||||
// Panics immediately if no seeds are configured: there is nothing to wait for.
|
||||
func retryUntilSeedResponds() {
|
||||
if !retryRunning.CompareAndSwap(false, true) {
|
||||
return // another goroutine is already running the retry loop
|
||||
}
|
||||
defer retryRunning.Store(false)
|
||||
|
||||
logger := oclib.GetLogger()
|
||||
rawAddresses := strings.TrimSpace(conf.GetConfig().IndexerAddresses)
|
||||
if rawAddresses == "" {
|
||||
// No seeds configured: rely on the inbound-connection notifee to fill
|
||||
// the pool. Just wait patiently — the loop below will return as soon
|
||||
// as any peer connects and NudgeIt() is called.
|
||||
logger.Warn().Msg("[pool] pool empty and no seeds configured — waiting for inbound indexer")
|
||||
}
|
||||
backoff := 10 * time.Second
|
||||
const maxBackoff = 5 * time.Minute
|
||||
for {
|
||||
time.Sleep(backoff)
|
||||
if backoff < maxBackoff {
|
||||
backoff *= 2
|
||||
}
|
||||
// Check whether someone else already refilled the pool.
|
||||
if len(Indexers.GetAddrs()) > 0 {
|
||||
logger.Info().Msg("[pool] pool refilled externally, stopping seed retry")
|
||||
return
|
||||
}
|
||||
logger.Warn().Dur("backoff", backoff).Msg("[pool] still isolated, retrying seeds")
|
||||
reconnectToSeeds()
|
||||
if len(Indexers.GetAddrs()) > 0 {
|
||||
Indexers.NudgeIt()
|
||||
// Re-bootstrap DHT now that we have at least one connection candidate.
|
||||
if discoveryDHT != nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
discoveryDHT.Bootstrap(ctx) //nolint:errcheck
|
||||
cancel()
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ensureScore returns the Score for addr, creating it if absent.
|
||||
func ensureScore(d *Directory, addr string) *Score {
|
||||
if !d.ExistsScore(addr) {
|
||||
d.SetScore(addr, &Score{
|
||||
FirstContacted: time.Now().UTC(),
|
||||
UptimeTracker: &UptimeTracker{FirstSeen: time.Now().UTC()},
|
||||
nextChallenge: rand.Intn(10) + 1,
|
||||
})
|
||||
}
|
||||
return d.GetScore(addr)
|
||||
}
|
||||
|
||||
// evictPeer removes addr from directory atomically and returns a snapshot of
|
||||
// remaining AddrInfos (for consensus voter selection).
|
||||
func evictPeer(d *Directory, addr string, id pp.ID, proto protocol.ID) []pp.AddrInfo {
|
||||
d.Streams.Delete(proto, &id)
|
||||
d.DeleteAddr(addr)
|
||||
voters := make([]pp.AddrInfo, 0, len(d.Addrs))
|
||||
for _, ai := range d.GetAddrs() {
|
||||
if ai.Info != nil {
|
||||
voters = append(voters, *ai.Info)
|
||||
}
|
||||
}
|
||||
d.DeleteScore(addr)
|
||||
return voters
|
||||
}
|
||||
|
||||
// handleSuggestions adds unknown suggested indexers to the directory.
|
||||
func handleSuggestions(d *Directory, from string, suggestions []pp.AddrInfo) {
|
||||
added := 0
|
||||
for _, sug := range suggestions {
|
||||
key := addrKey(sug)
|
||||
if !d.ExistsAddr(key) {
|
||||
cpy := sug
|
||||
d.SetAddr(key, &cpy)
|
||||
added++
|
||||
}
|
||||
}
|
||||
if added > 0 {
|
||||
logger := oclib.GetLogger()
|
||||
logger.Info().Int("added", added).Str("from", from).
|
||||
Msg("added suggested indexers from heartbeat response")
|
||||
d.NudgeIt()
|
||||
}
|
||||
}
|
||||
|
||||
// SendHeartbeat starts a goroutine that sends periodic heartbeats to peers.
|
||||
// recordFn, when provided, is called on each tick and its output is embedded in
|
||||
// the heartbeat as a fresh signed PeerRecord so the receiving indexer can
|
||||
// republish it to the DHT without an extra round-trip.
|
||||
// Pass no recordFn (or nil) for indexer→indexer / native heartbeats.
|
||||
func SendHeartbeat(ctx context.Context, proto protocol.ID, name string, h host.Host, directory *Directory, interval time.Duration, maxPool int, recordFn ...func() json.RawMessage) {
|
||||
logger := oclib.GetLogger()
|
||||
isIndexerHB := directory == Indexers
|
||||
var recFn func() json.RawMessage
|
||||
if len(recordFn) > 0 {
|
||||
recFn = recordFn[0]
|
||||
}
|
||||
go func() {
|
||||
logger.Info().Str("proto", string(proto)).Int("peers", len(directory.Addrs)).Msg("heartbeat started")
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
|
||||
// peerEntry pairs addr key with AddrInfo so doTick can update score maps directly.
|
||||
type peerEntry struct {
|
||||
addr string
|
||||
ai *pp.AddrInfo
|
||||
}
|
||||
|
||||
doTick := func() {
|
||||
addrs := directory.GetAddrsStr()
|
||||
need := maxPool - len(addrs)
|
||||
if need < 0 {
|
||||
need = 0
|
||||
}
|
||||
baseHB := Heartbeat{
|
||||
Name: name,
|
||||
PeerID: h.ID().String(),
|
||||
Timestamp: time.Now().UTC().Unix(),
|
||||
IndexersBinded: addrs,
|
||||
Need: need,
|
||||
}
|
||||
if recFn != nil {
|
||||
baseHB.Record = recFn()
|
||||
}
|
||||
// Determine the referent indexer: highest-scored one receives Referent=true
|
||||
// so it stores us in its referencedNodes for distributed search.
|
||||
var referentAddr string
|
||||
if isIndexerHB {
|
||||
var bestScore float64 = -1
|
||||
for _, ai2 := range directory.GetAddrs() {
|
||||
if s := directory.GetScore(ai2.Addr); s != nil && s.Score > bestScore {
|
||||
bestScore = s.Score
|
||||
referentAddr = ai2.Addr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, ai := range directory.GetAddrs() {
|
||||
// Build per-peer heartbeat copy so challenge injection is peer-specific.
|
||||
hb := baseHB
|
||||
if isIndexerHB && referentAddr != "" && ai.Addr == referentAddr {
|
||||
hb.Referent = true
|
||||
}
|
||||
// Ensure an IndexerScore entry exists for this peer.
|
||||
var score *Score
|
||||
if isIndexerHB {
|
||||
score = ensureScore(directory, ai.Addr)
|
||||
|
||||
// Inject challenge batch if due (random 1-10 HBs between batches).
|
||||
score.hbCount++
|
||||
if score.hbCount >= score.nextChallenge {
|
||||
// Ground truth: node's own PeerID — indexer MUST have us.
|
||||
challenges := []string{h.ID().String()}
|
||||
// Add up to 2 more known peers (other indexers) for richer data.
|
||||
// Use the already-snapshotted entries to avoid re-locking.
|
||||
for _, ai2 := range directory.GetAddrs() {
|
||||
if ai2.Addr != ai.Addr && ai2.Info != nil {
|
||||
challenges = append(challenges, ai2.Info.ID.String())
|
||||
if len(challenges) >= 3 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
hb.Challenges = challenges
|
||||
score.hbCount = 0
|
||||
score.nextChallenge = rand.Intn(10) + 1
|
||||
score.challengeTotal++ // count own-PeerID challenge (ground truth)
|
||||
score.dhtBatchCounter++
|
||||
// DHT challenge every 5th batch: ask indexer to retrieve our own DID.
|
||||
if score.dhtBatchCounter%5 == 0 {
|
||||
var selfDID string
|
||||
if len(baseHB.Record) > 0 {
|
||||
var partial struct {
|
||||
DID string `json:"did"`
|
||||
}
|
||||
if json.Unmarshal(baseHB.Record, &partial) == nil {
|
||||
selfDID = partial.DID
|
||||
}
|
||||
}
|
||||
if selfDID != "" {
|
||||
hb.ChallengeDID = selfDID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resp, rtt, err := sendHeartbeat(ctx, h, proto, ai.Info, hb, directory.Streams, interval*time.Second)
|
||||
if err != nil { // Heartbeat fails
|
||||
fmt.Println("EERR", err)
|
||||
HeartbeatFailure(h, proto, directory, ai.Addr, ai.Info, isIndexerHB, maxPool, err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update IndexerScore — uptime recorded on any successful send,
|
||||
// even if the indexer does not support bidirectional heartbeat (Fix 1).
|
||||
if isIndexerHB && score != nil {
|
||||
score.UptimeTracker.RecordHeartbeat()
|
||||
|
||||
maxRTT := BaseRoundTrip * 10
|
||||
latencyScore := 1.0 - float64(rtt)/float64(maxRTT)
|
||||
if latencyScore < 0 {
|
||||
latencyScore = 0
|
||||
}
|
||||
if latencyScore > 1 {
|
||||
latencyScore = 1
|
||||
}
|
||||
|
||||
// Update fill / challenge fields only when the indexer responded.
|
||||
if resp != nil {
|
||||
// BornAt stability check.
|
||||
if score.LastBornAt.IsZero() {
|
||||
score.LastBornAt = resp.BornAt
|
||||
} else if !resp.BornAt.IsZero() && !resp.BornAt.Equal(score.LastBornAt) {
|
||||
score.bornAtChanges++
|
||||
score.LastBornAt = resp.BornAt
|
||||
logger.Warn().Str("peer", ai.Info.ID.String()).
|
||||
Int("changes", score.bornAtChanges).
|
||||
Msg("indexer BornAt changed — possible restart or impersonation")
|
||||
}
|
||||
score.LastFillRate = resp.FillRate
|
||||
|
||||
// Fill rate consistency: cross-check peerCount/maxNodes vs reported fillRate.
|
||||
if resp.MaxNodes > 0 {
|
||||
expected := float64(resp.PeerCount) / float64(resp.MaxNodes)
|
||||
diff := expected - resp.FillRate
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
score.fillChecked++
|
||||
if diff < 0.1 {
|
||||
score.fillConsistent++
|
||||
}
|
||||
}
|
||||
|
||||
// Validate challenge responses. Only own-PeerID counts as ground truth.
|
||||
if len(hb.Challenges) > 0 && len(resp.Challenges) > 0 {
|
||||
ownID := h.ID().String()
|
||||
for _, ce := range resp.Challenges {
|
||||
if ce.PeerID != ownID {
|
||||
continue // informational only
|
||||
}
|
||||
recentEnough := !ce.LastSeen.IsZero() &&
|
||||
time.Since(ce.LastSeen) < 2*RecommendedHeartbeatInterval
|
||||
if ce.Found && recentEnough {
|
||||
score.challengeCorrect++
|
||||
}
|
||||
logger.Info().Str("peer", ai.Info.ID.String()).
|
||||
Bool("found", ce.Found).
|
||||
Bool("recent", recentEnough).
|
||||
Msg("own-PeerID challenge result")
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// DHT challenge result.
|
||||
if hb.ChallengeDID != "" {
|
||||
score.dhtChecked++
|
||||
if resp.DHTFound {
|
||||
score.dhtSuccess++
|
||||
}
|
||||
}
|
||||
|
||||
// Refresh local witness cache for indirect probing on future failure.
|
||||
for _, w := range resp.Witnesses {
|
||||
score.UpdateWitnessPool(w)
|
||||
}
|
||||
|
||||
// Launch witness cross-check asynchronously (must not hold lock).
|
||||
if len(resp.Witnesses) > 0 {
|
||||
go queryWitnesses(h, ai.Info.ID.String(), resp.BornAt, resp.FillRate, resp.Witnesses, score)
|
||||
} else if resp.MaxNodes > 0 {
|
||||
// No witnesses offered. Valid if indexer only has us (PeerCount==1).
|
||||
// Cross-check: FillRate should equal 1/MaxNodes within ±10%.
|
||||
expected := 1.0 / float64(resp.MaxNodes)
|
||||
diff := resp.FillRate - expected
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
score.witnessChecked++
|
||||
if resp.PeerCount == 1 && diff < 0.1 {
|
||||
score.witnessConsistent++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
score.Score = score.ComputeNodeSideScore(latencyScore)
|
||||
age := score.UptimeTracker.Uptime()
|
||||
minScore := dynamicMinScore(age)
|
||||
// Fix 4: grace period — at least 2 full heartbeat cycles before ejecting.
|
||||
isSeed := score.IsSeed
|
||||
// Seeds are sticky: never evicted by score alone (SuggestMigrate handles it).
|
||||
// Never eject the last indexer by score alone — we would lose all connectivity.
|
||||
belowThreshold := score.Score < minScore &&
|
||||
score.UptimeTracker.TotalOnline >= 2*RecommendedHeartbeatInterval &&
|
||||
!isSeed &&
|
||||
len(directory.Addrs) > 1
|
||||
|
||||
if belowThreshold {
|
||||
logger.Info().Str("peer", ai.Info.ID.String()).
|
||||
Float64("score", score.Score).Float64("min", minScore).
|
||||
Msg("indexer score below threshold, removing from pool")
|
||||
voters := evictPeer(directory, ai.Addr, ai.Info.ID, proto)
|
||||
need := max(maxPool-len(voters), 1)
|
||||
if len(voters) > 0 {
|
||||
go TriggerConsensus(h, voters, need)
|
||||
} else {
|
||||
go replenishIndexersFromDHT(h, need)
|
||||
}
|
||||
}
|
||||
|
||||
// Accept suggestions from this indexer — add unknown ones to the directory.
|
||||
if resp != nil && len(resp.Suggestions) > 0 {
|
||||
handleSuggestions(directory, ai.Info.ID.String(), resp.Suggestions)
|
||||
}
|
||||
|
||||
// Handle SuggestMigrate: indexer is overloaded and wants us to move.
|
||||
if resp != nil && resp.SuggestMigrate && isIndexerHB {
|
||||
nonSeedCount := 0
|
||||
for _, sc := range directory.GetScores() {
|
||||
if !sc.IsSeed {
|
||||
nonSeedCount++
|
||||
}
|
||||
}
|
||||
if nonSeedCount >= conf.GetConfig().MinIndexer {
|
||||
if isSeed {
|
||||
// Seed has offloaded us: clear sticky flag, score eviction takes over.
|
||||
score.IsSeed = false
|
||||
logger.Info().Str("peer", ai.Info.ID.String()).
|
||||
Msg("seed discharged via SuggestMigrate, de-stickied")
|
||||
} else {
|
||||
evictPeer(directory, ai.Addr, ai.Info.ID, proto)
|
||||
logger.Info().Str("peer", ai.Info.ID.String()).Msg("accepted migration from overloaded indexer")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
doTick()
|
||||
case <-directory.Nudge:
|
||||
if isIndexerHB {
|
||||
logger.Info().Msg("nudge received, heartbeating new indexers immediately")
|
||||
doTick()
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func HeartbeatFailure(h host.Host, proto protocol.ID, directory *Directory,
|
||||
addr string, info *pp.AddrInfo, isIndexerHB bool, maxPool int, err error) {
|
||||
logger := oclib.GetLogger()
|
||||
logger.Err(err)
|
||||
// Seeds are never evicted on heartbeat failure.
|
||||
// Keeping them in the pool lets the regular 60-second ticker retry them
|
||||
// at a natural cadence — no reconnect storm, no libp2p dial-backoff accumulation.
|
||||
// A seed will self-heal once it comes back; DHT and inbound peers fill the gap.
|
||||
if isIndexerHB {
|
||||
if score := directory.GetScore(addr); score != nil {
|
||||
if score.IsSeed {
|
||||
logger.Warn().Str("peer", info.ID.String()).
|
||||
Msg("[pool] seed heartbeat failed — keeping in pool, ticker will retry " + err.Error())
|
||||
return
|
||||
}
|
||||
// Indirect probe: query cached witnesses before declaring the indexer dead.
|
||||
// If a witness confirms it is alive, the failure is a local asymmetric
|
||||
// link — not the indexer. Skip eviction; next tick will retry directly.
|
||||
if len(score.WitnessPool) > 0 {
|
||||
pool := append([]WitnessCacheEntry(nil), score.WitnessPool...)
|
||||
if IndirectProbeIndexer(h, info.ID.String(), pool) {
|
||||
logger.Warn().Str("peer", info.ID.String()).
|
||||
Msg("[indirect] witness confirms indexer alive — asymmetric link, skipping eviction " + err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info().Str("peer", info.ID.String()).Str("proto", string(proto)).
|
||||
Msg("heartbeat failed, removing peer from pool : " + err.Error())
|
||||
consensusVoters := evictPeer(directory, addr, info.ID, proto)
|
||||
if isIndexerHB {
|
||||
need := maxPool - len(consensusVoters)
|
||||
if need < 1 {
|
||||
need = 1
|
||||
}
|
||||
logger.Info().Int("remaining", len(consensusVoters)).Int("need", need).Msg("pool state after removal")
|
||||
poolSize := len(directory.GetAddrs())
|
||||
if poolSize == 0 {
|
||||
// Pool is truly empty (no seeds configured or no seeds in pool).
|
||||
// Start the backoff retry loop — it will re-add seeds and nudge
|
||||
// only once a seed actually responds.
|
||||
go retryUntilSeedResponds()
|
||||
} else if len(consensusVoters) > 0 {
|
||||
go TriggerConsensus(h, consensusVoters, need)
|
||||
} else {
|
||||
go replenishIndexersFromDHT(h, need)
|
||||
}
|
||||
}
|
||||
}
|
||||
182
daemons/node/common/common_scoring.go
Normal file
182
daemons/node/common/common_scoring.go
Normal file
@@ -0,0 +1,182 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
cr "crypto/rand"
|
||||
"io"
|
||||
"net"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
)
|
||||
|
||||
const MaxExpectedMbps = 100.0
|
||||
const MinPayloadChallenge = 512
|
||||
const MaxPayloadChallenge = 2048
|
||||
const BaseRoundTrip = 400 * time.Millisecond
|
||||
|
||||
type UptimeTracker struct {
|
||||
FirstSeen time.Time
|
||||
LastSeen time.Time
|
||||
TotalOnline time.Duration
|
||||
}
|
||||
|
||||
// RecordHeartbeat accumulates online time gap-aware: only counts the interval if
|
||||
// the gap since the last heartbeat is within 2× the recommended interval (i.e. no
|
||||
// extended outage). Call this each time a heartbeat is successfully processed.
|
||||
func (u *UptimeTracker) RecordHeartbeat() {
|
||||
now := time.Now().UTC()
|
||||
if !u.LastSeen.IsZero() {
|
||||
gap := now.Sub(u.LastSeen)
|
||||
if gap <= 2*RecommendedHeartbeatInterval {
|
||||
u.TotalOnline += gap
|
||||
}
|
||||
}
|
||||
u.LastSeen = now
|
||||
}
|
||||
|
||||
func (u *UptimeTracker) Uptime() time.Duration {
|
||||
return time.Since(u.FirstSeen)
|
||||
}
|
||||
|
||||
// UptimeRatio returns the fraction of tracked lifetime during which the peer was
|
||||
// continuously online (gap ≤ 2×RecommendedHeartbeatInterval). Returns 0 before
|
||||
// the first heartbeat interval has elapsed.
|
||||
func (u *UptimeTracker) UptimeRatio() float64 {
|
||||
total := time.Since(u.FirstSeen)
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
ratio := float64(u.TotalOnline) / float64(total)
|
||||
if ratio > 1 {
|
||||
ratio = 1
|
||||
}
|
||||
return ratio
|
||||
}
|
||||
|
||||
func (u *UptimeTracker) IsEligible(min time.Duration) bool {
|
||||
return u.Uptime() >= min
|
||||
}
|
||||
|
||||
// getBandwidthChallengeRate opens a dedicated ProtocolBandwidthProbe stream to
|
||||
// remotePeer, sends a random payload, reads the echo, and computes throughput
|
||||
// and a latency score. Returns (ok, bpms, latencyScore, error).
|
||||
// latencyScore is 1.0 when RTT is very fast and 0.0 when at or beyond maxRoundTrip.
|
||||
// Using a separate stream avoids mixing binary data on the JSON heartbeat stream
|
||||
// and ensures the echo handler is actually running on the remote side.
|
||||
func getBandwidthChallengeRate(h host.Host, remotePeer pp.ID, payloadSize int) (bool, float64, float64, error) {
|
||||
payload := make([]byte, payloadSize)
|
||||
if _, err := cr.Read(payload); err != nil {
|
||||
return false, 0, 0, err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
s, err := h.NewStream(ctx, remotePeer, ProtocolBandwidthProbe)
|
||||
if err != nil {
|
||||
return false, 0, 0, err
|
||||
}
|
||||
defer s.Reset()
|
||||
s.SetDeadline(time.Now().Add(10 * time.Second))
|
||||
start := time.Now()
|
||||
if _, err = s.Write(payload); err != nil {
|
||||
return false, 0, 0, err
|
||||
}
|
||||
s.CloseWrite()
|
||||
// Half-close the write side so the handler's io.Copy sees EOF and stops.
|
||||
// Read the echo.
|
||||
response := make([]byte, payloadSize)
|
||||
if _, err = io.ReadFull(s, response); err != nil {
|
||||
return false, 0, 0, err
|
||||
}
|
||||
|
||||
duration := time.Since(start)
|
||||
maxRoundTrip := BaseRoundTrip + (time.Duration(payloadSize) * (100 * time.Millisecond))
|
||||
mbps := float64(payloadSize*8) / duration.Seconds() / 1e6
|
||||
|
||||
// latencyScore: 1.0 = instant, 0.0 = at maxRoundTrip or beyond.
|
||||
latencyScore := 1.0 - float64(duration)/float64(maxRoundTrip)
|
||||
if latencyScore < 0 {
|
||||
latencyScore = 0
|
||||
}
|
||||
if latencyScore > 1 {
|
||||
latencyScore = 1
|
||||
}
|
||||
|
||||
if duration > maxRoundTrip || mbps < 5.0 {
|
||||
return false, float64(mbps / MaxExpectedMbps), latencyScore, nil
|
||||
}
|
||||
return true, float64(mbps / MaxExpectedMbps), latencyScore, nil
|
||||
}
|
||||
|
||||
func getDiversityRate(h host.Host, peers []string) float64 {
|
||||
peers, _ = checkPeers(h, peers)
|
||||
diverse := []string{}
|
||||
for _, p := range peers {
|
||||
ip, err := ExtractIP(p)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
div := ip.Mask(net.CIDRMask(24, 32)).String()
|
||||
if !slices.Contains(diverse, div) {
|
||||
diverse = append(diverse, div)
|
||||
}
|
||||
}
|
||||
if len(diverse) == 0 || len(peers) == 0 {
|
||||
return 1
|
||||
}
|
||||
return float64(len(diverse)) / float64(len(peers))
|
||||
}
|
||||
|
||||
// getOwnDiversityRate measures subnet /24 diversity of the indexer's own connected peers.
|
||||
// This evaluates the indexer's network position rather than the connecting node's topology.
|
||||
func getOwnDiversityRate(h host.Host) float64 {
|
||||
diverse := map[string]struct{}{}
|
||||
total := 0
|
||||
for _, pid := range h.Network().Peers() {
|
||||
for _, maddr := range h.Peerstore().Addrs(pid) {
|
||||
total++
|
||||
ip, err := ExtractIP(maddr.String())
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
diverse[ip.Mask(net.CIDRMask(24, 32)).String()] = struct{}{}
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
return 1
|
||||
}
|
||||
return float64(len(diverse)) / float64(total)
|
||||
}
|
||||
|
||||
func checkPeers(h host.Host, peers []string) ([]string, []string) {
|
||||
concretePeer := []string{}
|
||||
ips := []string{}
|
||||
for _, p := range peers {
|
||||
ad, err := pp.AddrInfoFromString(p)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if PeerIsAlive(h, *ad) {
|
||||
concretePeer = append(concretePeer, p)
|
||||
if ip, err := ExtractIP(p); err == nil {
|
||||
ips = append(ips, ip.Mask(net.CIDRMask(24, 32)).String())
|
||||
}
|
||||
}
|
||||
}
|
||||
return concretePeer, ips
|
||||
}
|
||||
|
||||
// dynamicMinScore returns the minimum acceptable score for a peer, starting
|
||||
// permissive (20%) for brand-new peers and hardening linearly to 80% over 24h.
|
||||
// This prevents ejecting newcomers in fresh networks while filtering parasites.
|
||||
func dynamicMinScore(age time.Duration) float64 {
|
||||
hours := age.Hours()
|
||||
score := 20.0 + 60.0*(hours/24.0)
|
||||
if score > 80.0 {
|
||||
score = 80.0
|
||||
}
|
||||
return score
|
||||
}
|
||||
302
daemons/node/common/common_service.go
Normal file
302
daemons/node/common/common_service.go
Normal file
@@ -0,0 +1,302 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
"github.com/libp2p/go-libp2p/core/protocol"
|
||||
)
|
||||
|
||||
type LongLivedStreamRecordedService[T interface{}] struct {
|
||||
*LongLivedPubSubService
|
||||
StreamRecords map[protocol.ID]map[pp.ID]*StreamRecord[T]
|
||||
StreamMU sync.RWMutex
|
||||
maxNodesConn int
|
||||
// AllowInbound, when set, is called once at stream open before any heartbeat
|
||||
// is decoded. remotePeer is the connecting peer; isNew is true when no
|
||||
// StreamRecord exists yet (first-ever connection). Return a non-nil error
|
||||
// to immediately reset the stream and refuse the peer.
|
||||
AllowInbound func(remotePeer pp.ID, isNew bool) error
|
||||
// ValidateHeartbeat, when set, is called inside the heartbeat loop after
|
||||
// each successful CheckHeartbeat decode. Return a non-nil error to reset
|
||||
// the stream and terminate the session.
|
||||
ValidateHeartbeat func(remotePeer pp.ID) error
|
||||
// AfterHeartbeat is called after each successful heartbeat with the full
|
||||
// decoded Heartbeat so the hook can use the fresh embedded PeerRecord.
|
||||
AfterHeartbeat func(hb *Heartbeat)
|
||||
// AfterDelete is called after gc() evicts an expired peer, outside the lock.
|
||||
// name and did may be empty if the HeartbeatStream had no metadata.
|
||||
AfterDelete func(pid pp.ID, name string, did string)
|
||||
// BuildHeartbeatResponse, when set, is called after each successfully decoded
|
||||
// heartbeat to build the response sent back to the node.
|
||||
// remotePeer is the peer that sent the heartbeat (used for offload routing).
|
||||
// need is how many more indexers the node wants (from hb.Need).
|
||||
// referent is true when the node designated this indexer as its search referent.
|
||||
BuildHeartbeatResponse func(remotePeer pp.ID, need int, challenges []string, challengeDID string, referent bool) *HeartbeatResponse
|
||||
}
|
||||
|
||||
func (ix *LongLivedStreamRecordedService[T]) MaxNodesConn() int {
|
||||
return ix.maxNodesConn
|
||||
}
|
||||
|
||||
func NewStreamRecordedService[T interface{}](h host.Host, maxNodesConn int) *LongLivedStreamRecordedService[T] {
|
||||
service := &LongLivedStreamRecordedService[T]{
|
||||
LongLivedPubSubService: NewLongLivedPubSubService(h),
|
||||
StreamRecords: map[protocol.ID]map[pp.ID]*StreamRecord[T]{},
|
||||
maxNodesConn: maxNodesConn,
|
||||
}
|
||||
go service.StartGC(30 * time.Second)
|
||||
// Garbage collection is needed on every Map of Long-Lived Stream... it may be a top level redesigned
|
||||
go service.Snapshot(1 * time.Hour)
|
||||
return service
|
||||
}
|
||||
|
||||
func (ix *LongLivedStreamRecordedService[T]) StartGC(interval time.Duration) {
|
||||
go func() {
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for range t.C {
|
||||
fmt.Println("ACTUALLY RELATED INDEXERS", Indexers.Addrs, len(Indexers.Addrs))
|
||||
ix.gc()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (ix *LongLivedStreamRecordedService[T]) gc() {
|
||||
ix.StreamMU.Lock()
|
||||
now := time.Now().UTC()
|
||||
if ix.StreamRecords[ProtocolHeartbeat] == nil {
|
||||
ix.StreamRecords[ProtocolHeartbeat] = map[pp.ID]*StreamRecord[T]{}
|
||||
ix.StreamMU.Unlock()
|
||||
return
|
||||
}
|
||||
streams := ix.StreamRecords[ProtocolHeartbeat]
|
||||
|
||||
type gcEntry struct {
|
||||
pid pp.ID
|
||||
name string
|
||||
did string
|
||||
}
|
||||
var evicted []gcEntry
|
||||
for pid, rec := range streams {
|
||||
if now.After(rec.HeartbeatStream.Expiry) || now.Sub(rec.HeartbeatStream.UptimeTracker.LastSeen) > 2*rec.HeartbeatStream.Expiry.Sub(now) {
|
||||
name, did := "", ""
|
||||
if rec.HeartbeatStream != nil {
|
||||
name = rec.HeartbeatStream.Name
|
||||
did = rec.HeartbeatStream.DID
|
||||
}
|
||||
evicted = append(evicted, gcEntry{pid, name, did})
|
||||
for _, sstreams := range ix.StreamRecords {
|
||||
if sstreams[pid] != nil {
|
||||
delete(sstreams, pid)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ix.StreamMU.Unlock()
|
||||
|
||||
if ix.AfterDelete != nil {
|
||||
for _, e := range evicted {
|
||||
ix.AfterDelete(e.pid, e.name, e.did)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (ix *LongLivedStreamRecordedService[T]) Snapshot(interval time.Duration) {
|
||||
go func() {
|
||||
logger := oclib.GetLogger()
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
for range t.C {
|
||||
infos := ix.snapshot()
|
||||
for _, inf := range infos {
|
||||
logger.Info().Msg(" -> " + inf.DID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// -------- Snapshot / Query --------
|
||||
func (ix *LongLivedStreamRecordedService[T]) snapshot() []*StreamRecord[T] {
|
||||
ix.StreamMU.Lock()
|
||||
defer ix.StreamMU.Unlock()
|
||||
|
||||
out := make([]*StreamRecord[T], 0, len(ix.StreamRecords))
|
||||
for _, streams := range ix.StreamRecords {
|
||||
for _, stream := range streams {
|
||||
out = append(out, stream)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (ix *LongLivedStreamRecordedService[T]) HandleHeartbeat(s network.Stream) {
|
||||
logger := oclib.GetLogger()
|
||||
defer s.Close()
|
||||
|
||||
// AllowInbound: burst guard + ban check before the first byte is read.
|
||||
if ix.AllowInbound != nil {
|
||||
remotePeer := s.Conn().RemotePeer()
|
||||
ix.StreamMU.RLock()
|
||||
_, exists := ix.StreamRecords[ProtocolHeartbeat][remotePeer]
|
||||
ix.StreamMU.RUnlock()
|
||||
if err := ix.AllowInbound(remotePeer, !exists); err != nil {
|
||||
logger.Warn().Err(err).Str("peer", remotePeer.String()).Msg("inbound connection refused")
|
||||
s.Reset()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
dec := json.NewDecoder(s)
|
||||
for {
|
||||
ix.StreamMU.Lock()
|
||||
if ix.StreamRecords[ProtocolHeartbeat] == nil {
|
||||
ix.StreamRecords[ProtocolHeartbeat] = map[pp.ID]*StreamRecord[T]{}
|
||||
}
|
||||
streams := ix.StreamRecords[ProtocolHeartbeat]
|
||||
streamsAnonym := map[pp.ID]HeartBeatStreamed{}
|
||||
for k, v := range streams {
|
||||
streamsAnonym[k] = v
|
||||
}
|
||||
ix.StreamMU.Unlock()
|
||||
pid, hb, err := CheckHeartbeat(ix.Host, s, dec, streamsAnonym, &ix.StreamMU, ix.maxNodesConn)
|
||||
if err != nil {
|
||||
// Stream-level errors (EOF, reset, closed) mean the connection is gone
|
||||
// — exit so the goroutine doesn't spin forever on a dead stream.
|
||||
// Metric/policy errors (score too low, too many connections) are transient
|
||||
// — those are also stream-terminal since the stream carries one session.
|
||||
if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) ||
|
||||
strings.Contains(err.Error(), "reset") ||
|
||||
strings.Contains(err.Error(), "closed") ||
|
||||
strings.Contains(err.Error(), "too many connections") {
|
||||
logger.Info().Err(err).Msg("heartbeat stream terminated, closing handler")
|
||||
return
|
||||
}
|
||||
logger.Warn().Err(err).Msg("heartbeat check failed, retrying on same stream")
|
||||
continue
|
||||
}
|
||||
// ValidateHeartbeat: per-tick behavioral check (rate limiting, bans).
|
||||
if ix.ValidateHeartbeat != nil {
|
||||
if err := ix.ValidateHeartbeat(*pid); err != nil {
|
||||
logger.Warn().Err(err).Str("peer", pid.String()).Msg("heartbeat rejected, closing stream")
|
||||
s.Reset()
|
||||
return
|
||||
}
|
||||
}
|
||||
ix.StreamMU.Lock()
|
||||
// if record already seen update last seen
|
||||
if rec, ok := streams[*pid]; ok {
|
||||
rec.DID = hb.DID
|
||||
// Preserve the existing UptimeTracker so TotalOnline accumulates correctly.
|
||||
// hb.Stream is a fresh Stream with no UptimeTracker; carry the old one over.
|
||||
oldTracker := rec.GetUptimeTracker()
|
||||
rec.HeartbeatStream = hb.Stream
|
||||
if oldTracker != nil {
|
||||
rec.HeartbeatStream.UptimeTracker = oldTracker
|
||||
} else {
|
||||
rec.HeartbeatStream.UptimeTracker = &UptimeTracker{FirstSeen: time.Now().UTC()}
|
||||
}
|
||||
rec.HeartbeatStream.UptimeTracker.RecordHeartbeat()
|
||||
rec.LastScore = hb.Score
|
||||
logger.Info().Msg("A new node is updated : " + pid.String())
|
||||
} else {
|
||||
tracker := &UptimeTracker{FirstSeen: time.Now().UTC()}
|
||||
tracker.RecordHeartbeat()
|
||||
hb.Stream.UptimeTracker = tracker
|
||||
streams[*pid] = &StreamRecord[T]{
|
||||
DID: hb.DID,
|
||||
HeartbeatStream: hb.Stream,
|
||||
LastScore: hb.Score,
|
||||
}
|
||||
logger.Info().Msg("A new node is subscribed : " + pid.String())
|
||||
}
|
||||
ix.StreamMU.Unlock()
|
||||
// Enrich hb.DID before calling the hook: nodes never set hb.DID directly;
|
||||
// extract it from the embedded signed PeerRecord if available, then fall
|
||||
// back to the DID stored by handleNodePublish in the stream record.
|
||||
if hb.DID == "" && len(hb.Record) > 0 {
|
||||
var partial struct {
|
||||
DID string `json:"did"`
|
||||
}
|
||||
if json.Unmarshal(hb.Record, &partial) == nil && partial.DID != "" {
|
||||
hb.DID = partial.DID
|
||||
}
|
||||
}
|
||||
if hb.DID == "" {
|
||||
ix.StreamMU.RLock()
|
||||
if rec, ok := streams[*pid]; ok {
|
||||
hb.DID = rec.DID
|
||||
}
|
||||
ix.StreamMU.RUnlock()
|
||||
}
|
||||
if ix.AfterHeartbeat != nil && hb.DID != "" {
|
||||
ix.AfterHeartbeat(hb)
|
||||
}
|
||||
// Send response back to the node (bidirectional heartbeat).
|
||||
if ix.BuildHeartbeatResponse != nil {
|
||||
if resp := ix.BuildHeartbeatResponse(s.Conn().RemotePeer(), hb.Need, hb.Challenges, hb.ChallengeDID, hb.Referent); resp != nil {
|
||||
s.SetWriteDeadline(time.Now().Add(3 * time.Second))
|
||||
json.NewEncoder(s).Encode(resp)
|
||||
s.SetWriteDeadline(time.Time{})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CheckHeartbeat(h host.Host, s network.Stream, dec *json.Decoder, streams map[pp.ID]HeartBeatStreamed, lock *sync.RWMutex, maxNodes int) (*pp.ID, *Heartbeat, error) {
|
||||
if len(h.Network().Peers()) >= maxNodes {
|
||||
return nil, nil, fmt.Errorf("too many connections, try another indexer")
|
||||
}
|
||||
var hb Heartbeat
|
||||
if err := dec.Decode(&hb); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
_, bpms, latencyScore, _ := getBandwidthChallengeRate(h, s.Conn().RemotePeer(), MinPayloadChallenge+int(rand.Float64()*(MaxPayloadChallenge-MinPayloadChallenge)))
|
||||
{
|
||||
pid, err := pp.Decode(hb.PeerID)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
uptimeRatio := float64(0)
|
||||
age := time.Duration(0)
|
||||
lock.Lock()
|
||||
if rec, ok := streams[pid]; ok && rec.GetUptimeTracker() != nil {
|
||||
uptimeRatio = rec.GetUptimeTracker().UptimeRatio()
|
||||
age = rec.GetUptimeTracker().Uptime()
|
||||
}
|
||||
lock.Unlock()
|
||||
// E: measure the indexer's own subnet diversity, not the node's view.
|
||||
diversity := getOwnDiversityRate(h)
|
||||
// fillRate: fraction of indexer capacity used — higher = more peers trust this indexer.
|
||||
fillRate := 0.0
|
||||
if maxNodes > 0 {
|
||||
fillRate = float64(len(h.Network().Peers())) / float64(maxNodes)
|
||||
if fillRate > 1 {
|
||||
fillRate = 1
|
||||
}
|
||||
}
|
||||
hb.ComputeIndexerScore(uptimeRatio, bpms, diversity, latencyScore, fillRate)
|
||||
// B: dynamic minScore — starts at 20% for brand-new peers, ramps to 80% at 24h.
|
||||
minScore := dynamicMinScore(age)
|
||||
if hb.Score < minScore {
|
||||
return nil, nil, errors.New("not enough trusting value")
|
||||
}
|
||||
hb.Stream = &Stream{
|
||||
Name: hb.Name,
|
||||
DID: hb.DID,
|
||||
Stream: s,
|
||||
Expiry: time.Now().UTC().Add(2 * time.Minute),
|
||||
} // here is the long-lived bidirectional heartbeat.
|
||||
return &pid, &hb, err
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
199
daemons/node/common/consensus.go
Normal file
199
daemons/node/common/consensus.go
Normal file
@@ -0,0 +1,199 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
"github.com/libp2p/go-libp2p/core/network"
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
)
|
||||
|
||||
// ProtocolIndexerCandidates is opened by a node toward its remaining indexers
|
||||
// to request candidate replacement indexers after an ejection event.
|
||||
const ProtocolIndexerCandidates = "/opencloud/indexer/candidates/1.0"
|
||||
|
||||
// IndexerCandidatesRequest is sent by a node to one of its indexers.
|
||||
// Count is how many candidates are needed.
|
||||
type IndexerCandidatesRequest struct {
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// IndexerCandidatesResponse carries a random sample of known indexers from
|
||||
// the responding indexer's DHT cache.
|
||||
type IndexerCandidatesResponse struct {
|
||||
Candidates []pp.AddrInfo `json:"candidates"`
|
||||
}
|
||||
|
||||
// TriggerConsensus asks each remaining indexer for a random pool of candidates,
|
||||
// scores them asynchronously via a one-shot probe heartbeat, and admits the
|
||||
// best ones to StaticIndexers. Falls back to DHT replenishment for any gap.
|
||||
//
|
||||
// Must be called in a goroutine — it blocks until all probes have returned
|
||||
// (or timed out), which can take up to ~10s.
|
||||
func TriggerConsensus(h host.Host, remaining []pp.AddrInfo, need int) {
|
||||
if need <= 0 || len(remaining) == 0 {
|
||||
return
|
||||
}
|
||||
logger := oclib.GetLogger()
|
||||
logger.Info().Int("voters", len(remaining)).Int("need", need).
|
||||
Msg("[consensus] starting indexer candidate consensus")
|
||||
|
||||
// Phase 1 — collect candidates from all remaining indexers in parallel.
|
||||
type collectResult struct{ candidates []pp.AddrInfo }
|
||||
collectCh := make(chan collectResult, len(remaining))
|
||||
for _, ai := range remaining {
|
||||
go func(ai pp.AddrInfo) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
s, err := h.NewStream(ctx, ai.ID, ProtocolIndexerCandidates)
|
||||
if err != nil {
|
||||
collectCh <- collectResult{}
|
||||
return
|
||||
}
|
||||
defer s.Close()
|
||||
s.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
if err := json.NewEncoder(s).Encode(IndexerCandidatesRequest{Count: need + 2}); err != nil {
|
||||
collectCh <- collectResult{}
|
||||
return
|
||||
}
|
||||
var resp IndexerCandidatesResponse
|
||||
if err := json.NewDecoder(s).Decode(&resp); err != nil {
|
||||
collectCh <- collectResult{}
|
||||
return
|
||||
}
|
||||
collectCh <- collectResult{candidates: resp.Candidates}
|
||||
}(ai)
|
||||
}
|
||||
|
||||
// Merge and deduplicate, excluding indexers already in the pool.
|
||||
seen := map[pp.ID]struct{}{}
|
||||
for _, ai := range Indexers.GetAddrIDs() {
|
||||
seen[ai] = struct{}{}
|
||||
|
||||
}
|
||||
var candidates []pp.AddrInfo
|
||||
for range remaining {
|
||||
r := <-collectCh
|
||||
for _, ai := range r.candidates {
|
||||
if _, dup := seen[ai.ID]; !dup {
|
||||
seen[ai.ID] = struct{}{}
|
||||
candidates = append(candidates, ai)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(candidates) == 0 {
|
||||
logger.Info().Msg("[consensus] no candidates from voters, falling back to DHT")
|
||||
replenishIndexersFromDHT(h, need)
|
||||
return
|
||||
}
|
||||
logger.Info().Int("candidates", len(candidates)).Msg("[consensus] scoring candidates")
|
||||
|
||||
// Phase 2 — score all candidates in parallel via a one-shot probe heartbeat.
|
||||
type scoreResult struct {
|
||||
ai pp.AddrInfo
|
||||
score float64
|
||||
}
|
||||
scoreCh := make(chan scoreResult, len(candidates))
|
||||
for _, ai := range candidates {
|
||||
go func(ai pp.AddrInfo) {
|
||||
resp, rtt, err := probeIndexer(h, ai)
|
||||
if err != nil {
|
||||
scoreCh <- scoreResult{ai: ai, score: 0}
|
||||
return
|
||||
}
|
||||
scoreCh <- scoreResult{ai: ai, score: quickScore(resp, rtt)}
|
||||
}(ai)
|
||||
}
|
||||
|
||||
results := make([]scoreResult, 0, len(candidates))
|
||||
for range candidates {
|
||||
results = append(results, <-scoreCh)
|
||||
}
|
||||
|
||||
// Sort descending by quick score, admit top `need` above the minimum bar.
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].score > results[j].score })
|
||||
minQ := dynamicMinScore(0) // fresh peer: threshold starts at 20
|
||||
|
||||
admitted := 0
|
||||
for _, res := range results {
|
||||
if admitted >= need {
|
||||
break
|
||||
}
|
||||
if res.score < minQ {
|
||||
break // sorted desc: everything after is worse
|
||||
}
|
||||
key := addrKey(res.ai)
|
||||
if Indexers.ExistsAddr(key) {
|
||||
continue // already in pool (race with heartbeat path)
|
||||
}
|
||||
cpy := res.ai
|
||||
Indexers.SetAddr(key, &cpy)
|
||||
admitted++
|
||||
}
|
||||
|
||||
if admitted > 0 {
|
||||
logger.Info().Int("admitted", admitted).Msg("[consensus] candidates admitted to pool")
|
||||
Indexers.NudgeIt()
|
||||
}
|
||||
|
||||
// Fill any remaining gap with DHT discovery.
|
||||
if gap := need - admitted; gap > 0 {
|
||||
logger.Info().Int("gap", gap).Msg("[consensus] gap after consensus, falling back to DHT")
|
||||
replenishIndexersFromDHT(h, gap)
|
||||
}
|
||||
}
|
||||
|
||||
// probeIndexer dials the candidate, sends one lightweight heartbeat, and
|
||||
// returns the HeartbeatResponse (nil if the indexer doesn't support it) and RTT.
|
||||
func probeIndexer(h host.Host, ai pp.AddrInfo) (*HeartbeatResponse, time.Duration, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
|
||||
defer cancel()
|
||||
if h.Network().Connectedness(ai.ID) != network.Connected {
|
||||
if err := h.Connect(ctx, ai); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
}
|
||||
s, err := h.NewStream(ctx, ai.ID, ProtocolHeartbeat)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
defer s.Close()
|
||||
|
||||
hb := Heartbeat{PeerID: h.ID().String(), Timestamp: time.Now().UTC().Unix()}
|
||||
s.SetWriteDeadline(time.Now().Add(3 * time.Second))
|
||||
if err := json.NewEncoder(s).Encode(hb); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
s.SetWriteDeadline(time.Time{})
|
||||
|
||||
sentAt := time.Now()
|
||||
s.SetReadDeadline(time.Now().Add(5 * time.Second))
|
||||
var resp HeartbeatResponse
|
||||
if err := json.NewDecoder(s).Decode(&resp); err != nil {
|
||||
// Indexer connected but no response: connection itself is the signal.
|
||||
return nil, time.Since(sentAt), nil
|
||||
}
|
||||
return &resp, time.Since(sentAt), nil
|
||||
}
|
||||
|
||||
// quickScore computes a lightweight score [0,100] from a probe result.
|
||||
// Uses only fill rate (inverse) and latency — the two signals available
|
||||
// without a full heartbeat history.
|
||||
func quickScore(resp *HeartbeatResponse, rtt time.Duration) float64 {
|
||||
maxRTT := BaseRoundTrip * 10
|
||||
latencyScore := 1.0 - float64(rtt)/float64(maxRTT)
|
||||
if latencyScore < 0 {
|
||||
latencyScore = 0
|
||||
}
|
||||
if resp == nil {
|
||||
// Connection worked but no response (old indexer): moderate score.
|
||||
return latencyScore * 50
|
||||
}
|
||||
fillScore := 1.0 - resp.FillRate // prefer less-loaded indexers
|
||||
return (0.5*latencyScore + 0.5*fillScore) * 100
|
||||
}
|
||||
219
daemons/node/common/dht_discovery.go
Normal file
219
daemons/node/common/dht_discovery.go
Normal file
@@ -0,0 +1,219 @@
|
||||
package common
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
oclib "cloud.o-forge.io/core/oc-lib"
|
||||
"github.com/ipfs/go-cid"
|
||||
dht "github.com/libp2p/go-libp2p-kad-dht"
|
||||
"github.com/libp2p/go-libp2p/core/host"
|
||||
pp "github.com/libp2p/go-libp2p/core/peer"
|
||||
ma "github.com/multiformats/go-multiaddr"
|
||||
mh "github.com/multiformats/go-multihash"
|
||||
)
|
||||
|
||||
// FilterLoopbackAddrs strips loopback (127.x, ::1) and unspecified addresses
|
||||
// from an AddrInfo so we never hand peers an address they cannot dial externally.
|
||||
func FilterLoopbackAddrs(ai pp.AddrInfo) pp.AddrInfo {
|
||||
filtered := make([]ma.Multiaddr, 0, len(ai.Addrs))
|
||||
for _, addr := range ai.Addrs {
|
||||
ip, err := ExtractIP(addr.String())
|
||||
if err != nil || ip.IsLoopback() || ip.IsUnspecified() {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, addr)
|
||||
}
|
||||
return pp.AddrInfo{ID: ai.ID, Addrs: filtered}
|
||||
}
|
||||
|
||||
// RecommendedHeartbeatInterval is the target period between heartbeat ticks.
|
||||
// Indexers use this as the DHT Provide refresh interval.
|
||||
const RecommendedHeartbeatInterval = 60 * time.Second
|
||||
|
||||
// discoveryDHT is the DHT instance used for indexer discovery.
|
||||
// Set by SetDiscoveryDHT once the indexer service initialises its DHT.
|
||||
var discoveryDHT *dht.IpfsDHT
|
||||
|
||||
// SetDiscoveryDHT stores the DHT instance used by replenishIndexersFromDHT.
|
||||
// Called by NewIndexerService once the DHT is ready.
|
||||
func SetDiscoveryDHT(d *dht.IpfsDHT) {
|
||||
discoveryDHT = d
|
||||
}
|
||||
|
||||
// initNodeDHT creates a lightweight DHT client for pure nodes (no IndexerService).
|
||||
// Uses the seed indexers as bootstrap peers. Called lazily by ConnectToIndexers
|
||||
// when discoveryDHT is still nil after the initial warm-up delay.
|
||||
func initNodeDHT(h host.Host, seeds []Entry) {
|
||||
logger := oclib.GetLogger()
|
||||
bootstrapPeers := []pp.AddrInfo{}
|
||||
for _, s := range seeds {
|
||||
bootstrapPeers = append(bootstrapPeers, *s.Info)
|
||||
}
|
||||
d, err := dht.New(context.Background(), h,
|
||||
dht.Mode(dht.ModeClient),
|
||||
dht.ProtocolPrefix("oc"),
|
||||
dht.BootstrapPeers(bootstrapPeers...),
|
||||
)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Msg("[dht] node DHT client init failed")
|
||||
return
|
||||
}
|
||||
SetDiscoveryDHT(d)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := d.Bootstrap(ctx); err != nil {
|
||||
logger.Warn().Err(err).Msg("[dht] node DHT client bootstrap failed")
|
||||
}
|
||||
logger.Info().Msg("[dht] node DHT client ready")
|
||||
}
|
||||
|
||||
// IndexerCID returns the well-known CID under which all indexers advertise.
|
||||
func IndexerCID() cid.Cid {
|
||||
h, _ := mh.Sum([]byte("/opencloud/indexers"), mh.SHA2_256, -1)
|
||||
return cid.NewCidV1(cid.Raw, h)
|
||||
}
|
||||
|
||||
// DiscoverIndexersFromDHT uses the DHT to find up to count indexers advertising
|
||||
// under the well-known key. Excludes self. Resolves addresses when the provider
|
||||
// record carries none.
|
||||
func DiscoverIndexersFromDHT(h host.Host, d *dht.IpfsDHT, count int) []pp.AddrInfo {
|
||||
logger := oclib.GetLogger()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
c := IndexerCID()
|
||||
ch := d.FindProvidersAsync(ctx, c, count*2)
|
||||
seen := map[pp.ID]struct{}{}
|
||||
var results []pp.AddrInfo
|
||||
for ai := range ch {
|
||||
if ai.ID == h.ID() {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[ai.ID]; dup {
|
||||
continue
|
||||
}
|
||||
seen[ai.ID] = struct{}{}
|
||||
if len(ai.Addrs) == 0 {
|
||||
resolved, err := d.FindPeer(ctx, ai.ID)
|
||||
if err != nil {
|
||||
logger.Warn().Str("peer", ai.ID.String()).Msg("[dht] no addrs and FindPeer failed, skipping")
|
||||
continue
|
||||
}
|
||||
ai = resolved
|
||||
}
|
||||
ai = FilterLoopbackAddrs(ai)
|
||||
if len(ai.Addrs) == 0 {
|
||||
continue
|
||||
}
|
||||
results = append(results, ai)
|
||||
if len(results) >= count {
|
||||
break
|
||||
}
|
||||
}
|
||||
logger.Info().Int("found", len(results)).Msg("[dht] indexer discovery complete")
|
||||
return results
|
||||
}
|
||||
|
||||
// SelectByFillRate picks up to want providers using fill-rate weighted random
|
||||
// selection w(F) = F*(1-F) — peaks at F=0.5, prefers less-loaded indexers.
|
||||
// Providers with unknown fill rate receive F=0.5 (neutral prior).
|
||||
// Enforces subnet /24 diversity: at most one indexer per /24.
|
||||
func SelectByFillRate(providers []pp.AddrInfo, fillRates map[pp.ID]float64, want int) []pp.AddrInfo {
|
||||
if len(providers) == 0 || want <= 0 {
|
||||
return nil
|
||||
}
|
||||
type weighted struct {
|
||||
ai pp.AddrInfo
|
||||
weight float64
|
||||
}
|
||||
ws := make([]weighted, 0, len(providers))
|
||||
for _, ai := range providers {
|
||||
f, ok := fillRates[ai.ID]
|
||||
if !ok {
|
||||
f = 0.5
|
||||
}
|
||||
ws = append(ws, weighted{ai: ai, weight: f * (1 - f)})
|
||||
}
|
||||
// Shuffle first for fairness among equal-weight peers.
|
||||
rand.Shuffle(len(ws), func(i, j int) { ws[i], ws[j] = ws[j], ws[i] })
|
||||
// Sort descending by weight (simple insertion sort — small N).
|
||||
for i := 1; i < len(ws); i++ {
|
||||
for j := i; j > 0 && ws[j].weight > ws[j-1].weight; j-- {
|
||||
ws[j], ws[j-1] = ws[j-1], ws[j]
|
||||
}
|
||||
}
|
||||
|
||||
subnets := map[string]struct{}{}
|
||||
var selected []pp.AddrInfo
|
||||
for _, w := range ws {
|
||||
if len(selected) >= want {
|
||||
break
|
||||
}
|
||||
subnet := subnetOf(w.ai)
|
||||
if subnet != "" {
|
||||
if _, dup := subnets[subnet]; dup {
|
||||
continue
|
||||
}
|
||||
subnets[subnet] = struct{}{}
|
||||
}
|
||||
selected = append(selected, w.ai)
|
||||
}
|
||||
return selected
|
||||
}
|
||||
|
||||
// subnetOf returns the /24 subnet string for the first non-loopback address of ai.
|
||||
func subnetOf(ai pp.AddrInfo) string {
|
||||
for _, ma := range ai.Addrs {
|
||||
ip, err := ExtractIP(ma.String())
|
||||
if err != nil || ip.IsLoopback() {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(ip.String(), ".")
|
||||
if len(parts) >= 3 {
|
||||
return parts[0] + "." + parts[1] + "." + parts[2]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// replenishIndexersFromDHT is called when an indexer heartbeat fails and more
|
||||
// indexers are needed. Queries the DHT and adds fresh entries to StaticIndexers.
|
||||
func replenishIndexersFromDHT(h host.Host, need int) {
|
||||
if need <= 0 || discoveryDHT == nil {
|
||||
return
|
||||
}
|
||||
logger := oclib.GetLogger()
|
||||
logger.Info().Int("need", need).Msg("[dht] replenishing indexer pool from DHT")
|
||||
|
||||
providers := DiscoverIndexersFromDHT(h, discoveryDHT, need*3)
|
||||
selected := SelectByFillRate(providers, nil, need)
|
||||
if len(selected) == 0 {
|
||||
logger.Warn().Msg("[dht] no indexers found in DHT for replenishment")
|
||||
return
|
||||
}
|
||||
|
||||
added := 0
|
||||
for _, ai := range selected {
|
||||
addr := addrKey(ai)
|
||||
if !Indexers.ExistsAddr(addr) {
|
||||
adCopy := ai
|
||||
Indexers.SetAddr(addr, &adCopy)
|
||||
added++
|
||||
}
|
||||
}
|
||||
|
||||
if added > 0 {
|
||||
logger.Info().Int("added", added).Msg("[dht] indexers added from DHT")
|
||||
Indexers.NudgeIt()
|
||||
}
|
||||
}
|
||||
|
||||
// addrKey returns the canonical map key for an AddrInfo.
|
||||
// The PeerID is used as key so the same peer is never stored twice regardless
|
||||
// of which of its addresses was seen first.
|
||||
func addrKey(ai pp.AddrInfo) string {
|
||||
return ai.ID.String()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,6 +3,7 @@ package common
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
@@ -37,3 +38,31 @@ func ExtractIP(addr string) (net.IP, error) {
|
||||
}
|
||||
return ip, nil
|
||||
}
|
||||
|
||||
func GetIndexer(addrOrId string) *pp.AddrInfo {
|
||||
return Indexers.GetAddr(addrOrId)
|
||||
}
|
||||
|
||||
func GetIndexersIDs() []pp.ID {
|
||||
return Indexers.GetAddrIDs()
|
||||
}
|
||||
|
||||
func GetIndexersStr() []string {
|
||||
return Indexers.GetAddrsStr()
|
||||
}
|
||||
|
||||
func GetIndexers() []*pp.AddrInfo {
|
||||
entries := Indexers.GetAddrs()
|
||||
result := make([]*pp.AddrInfo, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
result = append(result, e.Info)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func Shuffle[T any](slice []T) []T {
|
||||
rand.Shuffle(len(slice), func(i, j int) {
|
||||
slice[i], slice[j] = slice[j], slice[i]
|
||||
})
|
||||
return slice
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user