diff --git a/pkg/frost/roast/coordinator_state.go b/pkg/frost/roast/coordinator_state.go index 0dc22be852..2a1cfc8edd 100644 --- a/pkg/frost/roast/coordinator_state.go +++ b/pkg/frost/roast/coordinator_state.go @@ -326,6 +326,18 @@ func (c *inMemoryCoordinator) RecordEvidence( return fmt.Errorf("coordinator: %w", err) } + // Emit any equivocation evidence AFTER c.mu is released: a registered + // observer is host telemetry (possibly a blocking write) and must not + // run while the coordinator state mutex is held. The evidence value is + // fully materialized (bytes copied) under the lock; emission only reads + // that copy. Registered before the unlock defer so it runs after it. + var pendingEvidence *EquivocationEvidence + defer func() { + if pendingEvidence != nil { + emitEquivocationEvidence(*pendingEvidence) + } + }() + c.mu.Lock() defer c.mu.Unlock() record, ok := c.attempts[handle.id] @@ -358,6 +370,13 @@ func (c *inMemoryCoordinator) RecordEvidence( } if !bytes.Equal(existingBytes, newBytes) || !bytes.Equal(existing.OperatorSignature, snapshot.OperatorSignature) { + pendingEvidence = &EquivocationEvidence{ + Kind: EquivocationKindSnapshotConflict, + AttemptContextHash: append([]byte(nil), snapshot.AttemptContextHash...), + Sender: snapshot.SenderID(), + ExistingEnvelope: snapshotEnvelopeForEvidence(existing), + ConflictingEnvelope: snapshotEnvelopeForEvidence(snapshot), + } return ErrSnapshotConflict } // Identical re-submission: idempotent no-op. diff --git a/pkg/frost/roast/equivocation.go b/pkg/frost/roast/equivocation.go new file mode 100644 index 0000000000..0677dec8b1 --- /dev/null +++ b/pkg/frost/roast/equivocation.go @@ -0,0 +1,146 @@ +package roast + +import ( + "encoding/hex" + "fmt" + "sync" + + "github.com/ipfs/go-log/v2" + + "github.com/keep-network/keep-core/pkg/protocol/group" +) + +var equivocationLogger = log.Logger("keep-frost-roast-equivocation") + +// Equivocation evidence kinds. Each event carries the exact signed +// envelope bytes behind a detection, so telemetry retains enough data to +// diagnose whether targeted equivocation is occurring before the full +// proof-carrying-blame wire format (follow-up item 7) exists. Two +// operator-signed bodies from the same sender for the same attempt are +// self-incriminating: both signatures verify, the bodies differ. +const ( + // EquivocationKindSnapshotConflict: a sender re-submitted a signed + // snapshot for the same attempt that differs from its first + // submission (first-write-wins conflict at the coordinator). + EquivocationKindSnapshotConflict = "snapshot_conflict" + // EquivocationKindOwnSnapshotMutatedInBundle: a coordinator bundle + // carries this member's snapshot with a different signature than the + // member actually submitted. + EquivocationKindOwnSnapshotMutatedInBundle = "own_snapshot_mutated_in_bundle" + // EquivocationKindOwnSnapshotMissingFromBundle: a coordinator bundle + // omits this member's submitted snapshot entirely. + EquivocationKindOwnSnapshotMissingFromBundle = "own_snapshot_missing_from_bundle" +) + +// EquivocationEvidence carries the exact signed byte streams behind a +// detected conflict, censorship, or mutation event. Envelope fields hold +// SignedLocalEvidenceSnapshot wire bytes verbatim (body + operator +// signature) and may be nil when the corresponding side could not be +// encoded or does not exist (e.g. a snapshot missing from a bundle). +type EquivocationEvidence struct { + Kind string + AttemptContextHash []byte + Sender group.MemberIndex + // ExistingEnvelope is the first-accepted / self-submitted signed + // snapshot envelope. + ExistingEnvelope []byte + // ConflictingEnvelope is the re-submitted / bundled signed snapshot + // envelope that disagrees with ExistingEnvelope. + ConflictingEnvelope []byte +} + +// EquivocationEvidenceObserver consumes equivocation evidence events. +type EquivocationEvidenceObserver func(evidence EquivocationEvidence) + +var ( + equivocationEvidenceObserverMutex sync.RWMutex + equivocationEvidenceObserver EquivocationEvidenceObserver +) + +// RegisterEquivocationEvidenceObserver registers a process-wide observer +// used to retain equivocation evidence in the host's telemetry system. +// Only a single observer is supported. +func RegisterEquivocationEvidenceObserver( + observer EquivocationEvidenceObserver, +) error { + if observer == nil { + return fmt.Errorf("equivocation evidence observer is nil") + } + + equivocationEvidenceObserverMutex.Lock() + defer equivocationEvidenceObserverMutex.Unlock() + + if equivocationEvidenceObserver != nil { + return fmt.Errorf("equivocation evidence observer is already registered") + } + + equivocationEvidenceObserver = observer + + return nil +} + +// UnregisterEquivocationEvidenceObserver clears the observer registration. +func UnregisterEquivocationEvidenceObserver() { + equivocationEvidenceObserverMutex.Lock() + defer equivocationEvidenceObserverMutex.Unlock() + + equivocationEvidenceObserver = nil +} + +// emitEquivocationEvidence logs the full evidence (these events are rare +// and the bytes are the diagnosis) and forwards it to the registered +// observer, if any. Never fails: evidence retention must not perturb the +// protocol path that detected the event. +func emitEquivocationEvidence(evidence EquivocationEvidence) { + equivocationLogger.Warnf( + "equivocation evidence [%s]: sender [%d], attempt context hash [%s], "+ + "existing envelope [%s], conflicting envelope [%s]", + evidence.Kind, + evidence.Sender, + hex.EncodeToString(evidence.AttemptContextHash), + hex.EncodeToString(evidence.ExistingEnvelope), + hex.EncodeToString(evidence.ConflictingEnvelope), + ) + + equivocationEvidenceObserverMutex.RLock() + observer := equivocationEvidenceObserver + equivocationEvidenceObserverMutex.RUnlock() + + if observer != nil { + // Honor the never-fails contract: a panicking telemetry observer + // must not escape into the protocol path that detected the event. + func() { + defer func() { + if r := recover(); r != nil { + equivocationLogger.Errorf( + "equivocation evidence observer panicked: [%v]", r, + ) + } + }() + observer(evidence) + }() + } +} + +// snapshotEnvelopeForEvidence encodes a snapshot's signed envelope for +// evidence retention, tolerating encode failures (nil result) so the +// detection path never degrades. +func snapshotEnvelopeForEvidence(snapshot *LocalEvidenceSnapshot) []byte { + if snapshot == nil { + return nil + } + envelope, err := snapshot.Marshal() + if err != nil { + equivocationLogger.Warnf( + "could not encode snapshot envelope for evidence retention: [%v]", + err, + ) + return nil + } + // Defensive copy: Marshal returns the snapshot's internal wire-envelope + // cache (its contract forbids callers from mutating it), but evidence + // bytes are handed to an external observer that may retain and later + // normalize, zero, or otherwise mutate them - which would corrupt the + // cached signed bytes on the stored snapshot. + return append([]byte(nil), envelope...) +} diff --git a/pkg/frost/roast/equivocation_test.go b/pkg/frost/roast/equivocation_test.go new file mode 100644 index 0000000000..e4568ac47b --- /dev/null +++ b/pkg/frost/roast/equivocation_test.go @@ -0,0 +1,222 @@ +package roast + +import ( + "bytes" + "errors" + "testing" + + "github.com/keep-network/keep-core/pkg/frost/roast/attempt" + "github.com/keep-network/keep-core/pkg/protocol/group" +) + +func captureEquivocationEvidence(t *testing.T) *[]EquivocationEvidence { + t.Helper() + captured := &[]EquivocationEvidence{} + if err := RegisterEquivocationEvidenceObserver( + func(evidence EquivocationEvidence) { + *captured = append(*captured, evidence) + }, + ); err != nil { + t.Fatalf("register observer: %v", err) + } + t.Cleanup(UnregisterEquivocationEvidenceObserver) + return captured +} + +func TestSnapshotConflict_RetainsBothSignedEnvelopes(t *testing.T) { + captured := captureEquivocationEvidence(t) + + c := NewInMemoryCoordinator().(*inMemoryCoordinator) + ctx := newTestContext(t) + handle, err := c.BeginAttempt(ctx) + if err != nil { + t.Fatalf("begin attempt: %v", err) + } + + first := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(3, ctx.Hash(), attempt.Evidence{ + Overflows: map[group.MemberIndex]uint{1: 1}, + }), + ) + if err := c.RecordEvidence(handle, first); err != nil { + t.Fatalf("record first: %v", err) + } + + // The same sender equivocates: a different signed snapshot for the + // same attempt. + conflicting := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(3, ctx.Hash(), attempt.Evidence{ + Overflows: map[group.MemberIndex]uint{1: 2}, + }), + ) + if err := c.RecordEvidence(handle, conflicting); !errors.Is(err, ErrSnapshotConflict) { + t.Fatalf("expected ErrSnapshotConflict, got %v", err) + } + + if len(*captured) != 1 { + t.Fatalf("expected 1 evidence event, got %d", len(*captured)) + } + evidence := (*captured)[0] + if evidence.Kind != EquivocationKindSnapshotConflict { + t.Fatalf("kind = %q", evidence.Kind) + } + if evidence.Sender != 3 { + t.Fatalf("sender = %d", evidence.Sender) + } + wantExisting, _ := first.Marshal() + wantConflicting, _ := conflicting.Marshal() + if !bytes.Equal(evidence.ExistingEnvelope, wantExisting) { + t.Fatal("existing envelope bytes must match the first submission verbatim") + } + if !bytes.Equal(evidence.ConflictingEnvelope, wantConflicting) { + t.Fatal("conflicting envelope bytes must match the re-submission verbatim") + } + + // Idempotent identical re-submission must NOT emit evidence. + if err := c.RecordEvidence(handle, first); err != nil { + t.Fatalf("identical re-submission should be a no-op: %v", err) + } + if len(*captured) != 1 { + t.Fatalf("identical re-submission emitted evidence: %d events", len(*captured)) + } +} + +func TestOwnSnapshotMutatedInBundle_RetainsBothSignedEnvelopes(t *testing.T) { + captured := captureEquivocationEvidence(t) + + selfSubmission := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(7, pinnedContextHash, attempt.Evidence{}), + ) + + mutated := *selfSubmission + mutated.OperatorSignature = bytes.Repeat([]byte{0xff}, 64) + // Fresh caches: the mutated copy is a distinct signed object. + mutated.signedBody = nil + mutated.wireEnvelope = nil + + bundle := &TransitionMessage{ + AttemptContextHash: append([]byte{}, pinnedContextHash[:]...), + Bundle: []LocalEvidenceSnapshot{mutated}, + } + if err := verifyOwnObservationsPresent(bundle, 7, selfSubmission); !errors.Is(err, ErrCensorshipDetected) { + t.Fatalf("expected ErrCensorshipDetected, got %v", err) + } + + if len(*captured) != 1 { + t.Fatalf("expected 1 evidence event, got %d", len(*captured)) + } + evidence := (*captured)[0] + if evidence.Kind != EquivocationKindOwnSnapshotMutatedInBundle { + t.Fatalf("kind = %q", evidence.Kind) + } + wantSelf, _ := selfSubmission.Marshal() + if !bytes.Equal(evidence.ExistingEnvelope, wantSelf) { + t.Fatal("existing envelope must be the self submission verbatim") + } + if len(evidence.ConflictingEnvelope) == 0 { + t.Fatal("conflicting envelope must carry the bundled snapshot") + } +} + +func TestOwnSnapshotMissingFromBundle_RetainsSelfEnvelope(t *testing.T) { + captured := captureEquivocationEvidence(t) + + selfSubmission := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(7, pinnedContextHash, attempt.Evidence{}), + ) + bundle := &TransitionMessage{ + AttemptContextHash: append([]byte{}, pinnedContextHash[:]...), + Bundle: []LocalEvidenceSnapshot{}, + } + if err := verifyOwnObservationsPresent(bundle, 7, selfSubmission); !errors.Is(err, ErrCensorshipDetected) { + t.Fatalf("expected ErrCensorshipDetected, got %v", err) + } + + if len(*captured) != 1 { + t.Fatalf("expected 1 evidence event, got %d", len(*captured)) + } + evidence := (*captured)[0] + if evidence.Kind != EquivocationKindOwnSnapshotMissingFromBundle { + t.Fatalf("kind = %q", evidence.Kind) + } + wantSelf, _ := selfSubmission.Marshal() + if !bytes.Equal(evidence.ExistingEnvelope, wantSelf) { + t.Fatal("existing envelope must be the self submission verbatim") + } + if evidence.ConflictingEnvelope != nil { + t.Fatal("missing-snapshot evidence has no conflicting envelope") + } +} + +func TestEquivocationObserver_PanicDoesNotEscapeProtocolPath(t *testing.T) { + if err := RegisterEquivocationEvidenceObserver( + func(_ EquivocationEvidence) { panic("observer boom") }, + ); err != nil { + t.Fatalf("register observer: %v", err) + } + t.Cleanup(UnregisterEquivocationEvidenceObserver) + + c := NewInMemoryCoordinator().(*inMemoryCoordinator) + ctx := newTestContext(t) + handle, err := c.BeginAttempt(ctx) + if err != nil { + t.Fatalf("begin attempt: %v", err) + } + first := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(3, ctx.Hash(), attempt.Evidence{ + Overflows: map[group.MemberIndex]uint{1: 1}, + }), + ) + if err := c.RecordEvidence(handle, first); err != nil { + t.Fatalf("record first: %v", err) + } + conflicting := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(3, ctx.Hash(), attempt.Evidence{ + Overflows: map[group.MemberIndex]uint{1: 2}, + }), + ) + // A panicking observer must not crash the protocol path: the intended + // ErrSnapshotConflict must still surface. + if err := c.RecordEvidence(handle, conflicting); !errors.Is(err, ErrSnapshotConflict) { + t.Fatalf("expected ErrSnapshotConflict despite panicking observer, got %v", err) + } +} + +func TestEquivocationEvidence_ObserverCannotCorruptSnapshotCache(t *testing.T) { + snapshot := signSnapshotForTest( + t, + NewLocalEvidenceSnapshot(7, pinnedContextHash, attempt.Evidence{ + Overflows: map[group.MemberIndex]uint{1: 1}, + }), + ) + pristine, err := snapshot.Marshal() + if err != nil { + t.Fatalf("marshal: %v", err) + } + pristineCopy := append([]byte(nil), pristine...) + + // Evidence bytes must not alias the snapshot's internal cache: mutating + // the returned evidence envelope must leave the snapshot's signed bytes + // intact. + evidenceBytes := snapshotEnvelopeForEvidence(snapshot) + if len(evidenceBytes) == 0 { + t.Fatal("expected evidence envelope bytes") + } + for i := range evidenceBytes { + evidenceBytes[i] ^= 0xff + } + + after, err := snapshot.Marshal() + if err != nil { + t.Fatalf("re-marshal: %v", err) + } + if !bytes.Equal(after, pristineCopy) { + t.Fatal("mutating evidence bytes corrupted the snapshot's cached envelope") + } +} diff --git a/pkg/frost/roast/signature.go b/pkg/frost/roast/signature.go index fb107447e3..fe17bdc664 100644 --- a/pkg/frost/roast/signature.go +++ b/pkg/frost/roast/signature.go @@ -201,6 +201,13 @@ func verifyOwnObservationsPresent( msg.Bundle[i].OperatorSignature, selfSubmission.OperatorSignature, ) { + emitEquivocationEvidence(EquivocationEvidence{ + Kind: EquivocationKindOwnSnapshotMutatedInBundle, + AttemptContextHash: append([]byte(nil), msg.AttemptContextHash...), + Sender: selfMember, + ExistingEnvelope: snapshotEnvelopeForEvidence(selfSubmission), + ConflictingEnvelope: snapshotEnvelopeForEvidence(&msg.Bundle[i]), + }) return fmt.Errorf( "%w: own evidence snapshot signature mutated in bundle", ErrCensorshipDetected, @@ -208,5 +215,11 @@ func verifyOwnObservationsPresent( } return nil } + emitEquivocationEvidence(EquivocationEvidence{ + Kind: EquivocationKindOwnSnapshotMissingFromBundle, + AttemptContextHash: append([]byte(nil), msg.AttemptContextHash...), + Sender: selfMember, + ExistingEnvelope: snapshotEnvelopeForEvidence(selfSubmission), + }) return ErrCensorshipDetected }