From b8b62d114944ee1742023661d8fb8c9d49d00b3b Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 10:19:11 +0800 Subject: [PATCH 1/3] feat(cluster): add PCM-X writer conversion queue Add bounded FIFO writer arbitration, exact ownership handoff, distributed wait-for graph integration, GCS image transfer, cancellation, drain and retirement, observability, and liveness coverage. --- src/backend/access/heap/heapam.c | 17 +- src/backend/access/heap/visibilitymap.c | 2 +- src/backend/cluster/Makefile | 3 + src/backend/cluster/cluster_debug.c | 75 +- src/backend/cluster/cluster_gcs_block.c | 6142 ++++- src/backend/cluster/cluster_gcs_block_dedup.c | 898 +- src/backend/cluster/cluster_gcs_block_shard.c | 74 +- src/backend/cluster/cluster_ic.c | 3 + src/backend/cluster/cluster_lmd_graph.c | 451 +- src/backend/cluster/cluster_lmd_wait_state.c | 106 +- src/backend/cluster/cluster_lmon.c | 2 + src/backend/cluster/cluster_lms.c | 13 +- src/backend/cluster/cluster_lms_outbound.c | 57 +- src/backend/cluster/cluster_lock_acquire.c | 44 +- src/backend/cluster/cluster_pcm_direct_init.c | 110 + src/backend/cluster/cluster_pcm_lock.c | 331 +- src/backend/cluster/cluster_pcm_own.c | 305 +- src/backend/cluster/cluster_pcm_x_convert.c | 19695 ++++++++++++++++ .../cluster/cluster_pcm_x_image_fetch.c | 169 + src/backend/cluster/cluster_qvotec.c | 43 +- src/backend/cluster/cluster_sf_dep.c | 37 + src/backend/cluster/cluster_shmem.c | 5 + src/backend/cluster/cluster_tx_enqueue.c | 71 +- src/backend/storage/buffer/bufmgr.c | 4435 +++- src/backend/storage/freespace/freespace.c | 4 +- src/backend/storage/lmgr/lwlock.c | 28 +- src/include/cluster/cluster_gcs_block.h | 1122 +- src/include/cluster/cluster_gcs_block_dedup.h | 149 +- src/include/cluster/cluster_ic.h | 4 + src/include/cluster/cluster_ic_envelope.h | 60 +- src/include/cluster/cluster_lmd.h | 38 + src/include/cluster/cluster_lmd_wait_state.h | 82 +- src/include/cluster/cluster_pcm_direct_init.h | 58 + src/include/cluster/cluster_pcm_lock.h | 100 +- src/include/cluster/cluster_pcm_own.h | 120 +- src/include/cluster/cluster_pcm_x_bufmgr.h | 352 + src/include/cluster/cluster_pcm_x_convert.h | 1644 ++ .../cluster/cluster_pcm_x_image_fetch.h | 72 + src/include/cluster/cluster_qvotec.h | 4 + src/include/cluster/cluster_sf_dep.h | 19 + src/include/cluster/cluster_tx_enqueue.h | 9 +- src/include/storage/buf_internals.h | 34 + src/include/storage/bufmgr.h | 3 + src/include/storage/lwlock.h | 21 + src/test/cluster_tap/t/017_debug.pl | 5 + src/test/cluster_tap/t/020_shmem_registry.pl | 8 +- src/test/cluster_tap/t/021_block_format.pl | 6 +- src/test/cluster_tap/t/022_itl_slot.pl | 6 +- .../cluster_tap/t/023_buffer_descriptor.pl | 6 +- src/test/cluster_tap/t/024_pcm_lock.pl | 19 +- src/test/cluster_tap/t/106_lmd_smoke.pl | 12 +- .../cluster_tap/t/108_pcm_state_machine.pl | 22 +- .../cluster_tap/t/109_lmd_deadlock_smoke.pl | 6 +- src/test/cluster_tap/t/110_gcs_loopback.pl | 6 +- .../cluster_tap/t/111_gcs_block_ship_2node.pl | 12 +- .../t/112_gcs_block_retransmit_2node.pl | 12 +- .../cluster_tap/t/113_gcs_block_2way_2node.pl | 12 +- .../cluster_tap/t/114_gcs_block_3way_2node.pl | 12 +- .../cluster_tap/t/115_gcs_block_3way_3node.pl | 4 +- .../t/116_gcs_block_lost_write_2node.pl | 12 +- .../t/400_pcm_x_queue_4node_liveness.pl | 588 + src/test/cluster_unit/Makefile | 65 +- src/test/cluster_unit/test_cluster_debug.c | 201 +- .../cluster_unit/test_cluster_gcs_block.c | 2655 ++- .../test_cluster_gcs_block_3way.c | 2 +- .../test_cluster_gcs_block_dedup.c | 827 +- .../test_cluster_gcs_block_retransmit.c | 4 +- .../test_cluster_gcs_block_shard.c | 73 +- src/test/cluster_unit/test_cluster_ic.c | 44 +- .../cluster_unit/test_cluster_lmd_graph.c | 517 +- .../test_cluster_lmd_wait_state.c | 285 +- src/test/cluster_unit/test_cluster_lmon.c | 4 + .../cluster_unit/test_cluster_lms_outbound.c | 141 +- .../cluster_unit/test_cluster_lock_acquire.c | 62 +- .../test_cluster_pcm_direct_init.c | 435 + src/test/cluster_unit/test_cluster_pcm_lock.c | 364 +- src/test/cluster_unit/test_cluster_pcm_own.c | 2023 ++ .../cluster_unit/test_cluster_pcm_x_convert.c | 14270 +++++++++++ .../test_cluster_pcm_x_image_fetch.c | 256 + src/test/cluster_unit/test_cluster_qvotec.c | 4 + src/test/cluster_unit/test_cluster_sf_dep.c | 44 + src/test/cluster_unit/test_cluster_shmem.c | 6 + 82 files changed, 58784 insertions(+), 1257 deletions(-) create mode 100644 src/backend/cluster/cluster_pcm_direct_init.c create mode 100644 src/backend/cluster/cluster_pcm_x_convert.c create mode 100644 src/backend/cluster/cluster_pcm_x_image_fetch.c create mode 100644 src/include/cluster/cluster_pcm_direct_init.h create mode 100644 src/include/cluster/cluster_pcm_x_bufmgr.h create mode 100644 src/include/cluster/cluster_pcm_x_convert.h create mode 100644 src/include/cluster/cluster_pcm_x_image_fetch.h create mode 100644 src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl create mode 100644 src/test/cluster_unit/test_cluster_pcm_direct_init.c create mode 100644 src/test/cluster_unit/test_cluster_pcm_own.c create mode 100644 src/test/cluster_unit/test_cluster_pcm_x_convert.c create mode 100644 src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index 7d96f4d44a8..9a6ffef55ab 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -3238,6 +3238,11 @@ cluster_heap_writer_wait_failclosed(Relation relation, Buffer buffer, HeapTuple ClusterTxwResult txw; txw = cluster_tx_enqueue_wait(&ckey, cluster_ges_request_timeout_ms); + if (txw == CLUSTER_TXW_RETRY) + ereport(ERROR, (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("remote writer wait blocked by a PCM-X holder probe"), + errhint("Retry the transaction after the current Cache Fusion " + "conversion round completes."))); if (txw == CLUSTER_TXW_TIMEOUT) { cluster_vis_bump_vis_conflict_failclosed_count(); ereport(ERROR, @@ -6451,8 +6456,16 @@ heap_lock_tuple(Relation relation, HeapTuple tuple, { ClusterTxwResult txw; - txw = cluster_tx_enqueue_wait(&ckey, - cluster_ges_request_timeout_ms); + txw = cluster_tx_enqueue_wait( + &ckey, cluster_ges_request_timeout_ms); + if (txw == CLUSTER_TXW_RETRY) + ereport(ERROR, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("remote row-lock wait blocked by a " + "PCM-X holder probe"), + errhint("Retry the transaction after the " + "current Cache Fusion " + "conversion round completes."))); if (txw == CLUSTER_TXW_TIMEOUT) ereport(ERROR, (errcode(ERRCODE_CLUSTER_GES_TIMEOUT), diff --git a/src/backend/access/heap/visibilitymap.c b/src/backend/access/heap/visibilitymap.c index 93ab9d50796..f98a30191e6 100644 --- a/src/backend/access/heap/visibilitymap.c +++ b/src/backend/access/heap/visibilitymap.c @@ -661,7 +661,7 @@ vm_readbuf(Relation rel, BlockNumber blkno, bool extend) */ if (PageIsNew(BufferGetPage(buf))) { - LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + LockBufferForVisibilityMapPageInit(buf); if (PageIsNew(BufferGetPage(buf))) PageInit(BufferGetPage(buf), BLCKSZ, 0); LockBuffer(buf, BUFFER_LOCK_UNLOCK); diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index eec165c4b31..180b7d96690 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -44,6 +44,7 @@ OBJS = \ cluster_apply_master_election.o \ cluster_advisory.o \ cluster_pcm_own.o \ + cluster_pcm_direct_init.o \ cluster_cancel_token.o \ cluster_backup.o \ cluster_backup_manifest.o \ @@ -156,6 +157,8 @@ OBJS = \ cluster_node_remove_policy.o \ cluster_node_remove_views.o \ cluster_pcm_lock.o \ + cluster_pcm_x_convert.o \ + cluster_pcm_x_image_fetch.o \ cluster_pgstat.o \ cluster_pi_recovery_srf.o \ cluster_pi_shadow.o \ diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 3a1a52b0a86..791e58d5341 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -141,6 +141,7 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_itl_slot.h" /* CLUSTER_ITL_* constants (stage 1.5) */ #include "cluster/cluster_buffer_desc.h" /* BufferType / PcmState enums (stage 1.6) */ #include "cluster/cluster_pcm_lock.h" /* PCM state-machine API + grd helpers */ +#include "cluster/cluster_pcm_x_convert.h" /* PCM-X external FIFO observability */ #include "cluster/cluster_gcs.h" /* GCS request protocol surface (spec-2.32 D8) */ #include "cluster/cluster_gcs_block.h" /* GCS block-ship data plane (spec-2.33 D10) */ #include "cluster/cluster_gcs_block_dedup.h" /* per-worker dedup-shard counters (spec-7.3 D5/D9) */ @@ -1508,7 +1509,7 @@ dump_lms(ReturnSetInfo *rsinfo) /* * dump_lmd -- spec-2.19 Sprint A Step 4 D10. * - * Emits 47 rows under category='lmd' (spec-2.19 daemon state/counters + + * Emits 51 rows under category='lmd' (spec-2.19 daemon state/counters + * spec-2.22 graph/Tarjan + spec-2.23 probe + spec-2.24 cancel/cleanup + * spec-5.8 D6 two-round-confirm/reconfig-gate + spec-5.8 Hardening v1.0.1 * FC1 member_incomplete_count + spec-5.8 D8 shmem REPORT-collector @@ -1547,6 +1548,14 @@ dump_lmd(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_lmd_victim_cancel_sent_count_get())); emit_row(rsinfo, "lmd", "revalidate_fail_count", fmt_int64((int64)cluster_lmd_revalidate_fail_count_get())); + emit_row(rsinfo, "lmd", "pcm_convert_wfg_replace_count", + fmt_int64((int64)cluster_lmd_pcm_convert_wfg_replace_count_get())); + emit_row(rsinfo, "lmd", "pcm_convert_wfg_remove_count", + fmt_int64((int64)cluster_lmd_pcm_convert_wfg_remove_count_get())); + emit_row(rsinfo, "lmd", "pcm_convert_wfg_replace_fail_count", + fmt_int64((int64)cluster_lmd_pcm_convert_wfg_replace_fail_count_get())); + emit_row(rsinfo, "lmd", "pcm_convert_wfg_exact_remove_stale_count", + fmt_int64((int64)cluster_lmd_pcm_convert_wfg_exact_remove_stale_count_get())); emit_row(rsinfo, "lmd", "cross_node_victim_pending_count", fmt_int64((int64)cluster_lmd_cross_node_victim_pending_count_get())); emit_row(rsinfo, "lmd", "inject_call_count", @@ -1985,6 +1994,56 @@ dump_pcm(ReturnSetInfo *rsinfo) * (table full): non-zero means prov_query absence is inconclusive. */ emit_row(rsinfo, "pcm", "wm_prov_insert_fail_count", fmt_int64((int64)cluster_pcm_get_wm_prov_insert_fail_count())); + + /* PCM-X queue core: 30 stable keys in the existing pcm category. */ + { + PcmXStatsSnapshot stats = { 0 }; + PcmXRuntimeSnapshot runtime = cluster_pcm_x_runtime_snapshot(); + + (void)cluster_pcm_x_stats_snapshot(&stats); + emit_row(rsinfo, "pcm", "pcm_x_runtime_state", fmt_int32((int32)runtime.state)); + emit_row(rsinfo, "pcm", "pcm_x_runtime_generation", + fmt_int64((int64)runtime.gate_generation)); + emit_row(rsinfo, "pcm", "pcm_x_queue_enqueue_count", fmt_int64((int64)stats.enqueue_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_admit_count", fmt_int64((int64)stats.admit_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_confirm_count", fmt_int64((int64)stats.confirm_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_promotion_count", + fmt_int64((int64)stats.promotion_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_transfer_count", + fmt_int64((int64)stats.transfer_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_complete_count", + fmt_int64((int64)stats.complete_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_cancel_count", fmt_int64((int64)stats.cancel_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_revoke_count", fmt_int64((int64)stats.revoke_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_coalesced_count", + fmt_int64((int64)stats.coalesced_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_wait_count", fmt_int64((int64)stats.wait_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_full_count", fmt_int64((int64)stats.full_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_stale_count", fmt_int64((int64)stats.stale_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_miss_count", fmt_int64((int64)stats.miss_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_recovery_blocked_count", + fmt_int64((int64)stats.recovery_blocked_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_activating_reset_count", + fmt_int64((int64)stats.activating_reset_count)); + emit_row(rsinfo, "pcm", "pcm_x_queue_depth", fmt_int64((int64)stats.depth)); + emit_row(rsinfo, "pcm", "pcm_x_queue_depth_high_water", + fmt_int64((int64)stats.depth_high_water)); + emit_row(rsinfo, "pcm", "pcm_x_queue_active_tags", fmt_int64((int64)stats.active_tags)); + emit_row(rsinfo, "pcm", "pcm_x_queue_live_tickets", fmt_int64((int64)stats.live_tickets)); + emit_row(rsinfo, "pcm", "pcm_x_queue_live_slots", fmt_int64((int64)stats.live_slots)); + emit_row(rsinfo, "pcm", "pcm_x_local_retire_gate", + fmt_int64((int64)stats.local_retire_gate)); + emit_row(rsinfo, "pcm", "pcm_x_local_retire_marker_count", + fmt_int64((int64)stats.local_retire_marker_count)); + emit_row(rsinfo, "pcm", "pcm_x_local_retire_marker_ticket_id", + fmt_int64((int64)stats.local_retire_marker_ticket_id)); + emit_row(rsinfo, "pcm", "pcm_x_own_begin_count", fmt_int64((int64)stats.own_begin_count)); + emit_row(rsinfo, "pcm", "pcm_x_own_commit_count", fmt_int64((int64)stats.own_commit_count)); + emit_row(rsinfo, "pcm", "pcm_x_own_abort_count", fmt_int64((int64)stats.own_abort_count)); + emit_row(rsinfo, "pcm", "pcm_x_own_busy_count", fmt_int64((int64)stats.own_busy_count)); + emit_row(rsinfo, "pcm", "pcm_x_own_corrupt_count", + fmt_int64((int64)stats.own_corrupt_count)); + } } @@ -2180,6 +2239,14 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_block_dedup_hint_violation_count())); emit_row(rsinfo, "gcs", "dedup_legacy_pin_count", fmt_int64((int64)cluster_gcs_get_block_dedup_legacy_pin_count())); + emit_row(rsinfo, "gcs", "dedup_pcm_x_stage_count", + fmt_int64((int64)cluster_gcs_get_block_dedup_pcm_x_stage_count())); + emit_row(rsinfo, "gcs", "dedup_pcm_x_replay_count", + fmt_int64((int64)cluster_gcs_get_block_dedup_pcm_x_replay_count())); + emit_row(rsinfo, "gcs", "dedup_pcm_x_release_count", + fmt_int64((int64)cluster_gcs_get_block_dedup_pcm_x_release_count())); + emit_row(rsinfo, "gcs", "dedup_pcm_x_failclosed_count", + fmt_int64((int64)cluster_gcs_get_block_dedup_pcm_x_failclosed_count())); emit_row(rsinfo, "gcs", "done_enqueue_drop_count", fmt_int64((int64)cluster_gcs_get_block_done_enqueue_drop_count())); @@ -2218,6 +2285,12 @@ dump_gcs(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_gcs_get_invalidate_busy_sent_count())); emit_row(rsinfo, "gcs", "invalidate_busy_received_count", fmt_int64((int64)cluster_gcs_get_invalidate_busy_received_count())); + emit_row(rsinfo, "gcs", "invalidate_passive_s_release_count", + fmt_int64((int64)cluster_gcs_get_invalidate_passive_s_release_count())); + emit_row(rsinfo, "gcs", "pcm_x_self_handoff_count", + fmt_int64((int64)cluster_gcs_get_pcm_x_self_handoff_count())); + emit_row(rsinfo, "gcs", "pcm_x_self_handoff_drain_count", + fmt_int64((int64)cluster_gcs_get_pcm_x_self_handoff_drain_count())); emit_row(rsinfo, "gcs", "invalidate_park_expired_count", fmt_int64((int64)cluster_gcs_get_invalidate_park_expired_count())); emit_row(rsinfo, "gcs", "invalidate_park_overflow_count", diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index 1f31eae132e..f8bbef6c0e7 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -39,6 +39,7 @@ #ifdef USE_PGRAC_CLUSTER #include "access/multixact.h" /* MultiXactIdIsValid (spec-7.1 D3-b fetch) */ +#include "access/xact.h" #include "access/xlog.h" #include "access/xlogdefs.h" #include "cluster/cluster_clean_leave.h" /* spec-5.13 S6 — CL-I5 serve gate */ @@ -48,13 +49,15 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_cr_server.h" /* spec-6.12b CR-server park/fetch */ #include "cluster/cluster_gcs_block.h" -#include "cluster/cluster_lms_shard.h" /* PGRAC: spec-7.3 D4 — tag->worker shard */ +#include "cluster/cluster_lms_shard.h" /* PGRAC: spec-7.3 D4 — tag->worker shard */ +#include "cluster/cluster_lmd.h" #include "cluster/cluster_gcs_reqid.h" /* PGRAC: spec-6.14a D1 — id domains */ #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */ #include "cluster/cluster_grd.h" /* spec-4.6 D4 — block_path_failclosed counter */ #include "cluster/cluster_grd_outbound.h" #include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ #include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ +#include "cluster/cluster_reconfig.h" /* QVOTEC-observed live peer incarnation */ #include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */ #include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ @@ -66,21 +69,27 @@ #include "cluster/cluster_ic_rdma.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_ic_tier1.h" /* PGRAC: spec-7.3 D5 — my DATA channel = worker id */ +#include "cluster/cluster_lms.h" #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" #include "cluster/cluster_pcm_own.h" /* S3 forensics — ownership gen in 53R93 errdetail */ +#include "cluster/cluster_pcm_x_bufmgr.h" +#include "cluster/cluster_pcm_x_image_fetch.h" #include "cluster/cluster_shmem.h" #include "cluster/storage/cluster_shared_fs.h" #include "cluster/cluster_sf_dep.h" #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */ +#include "cluster/cluster_write_fence.h" #include "common/hashfn.h" #include "miscadmin.h" #include "port/atomics.h" #include "storage/backendid.h" #include "storage/bufmgr.h" #include "storage/buf_internals.h" +#include "storage/bufpage.h" #include "storage/latch.h" #include "storage/condition_variable.h" +#include "storage/ipc.h" #include "storage/lwlock.h" #include "storage/proc.h" #include "storage/shmem.h" @@ -208,6 +217,13 @@ typedef struct ClusterGcsBlockShared { * ACKs — holder side sent / master side consumed (slot-matching). */ pg_atomic_uint64 invalidate_busy_sent_count; pg_atomic_uint64 invalidate_busy_received_count; + /* Exact queue INVALIDATEs that broke the waiting-writer pin ring by + * normalizing a content-drained MAIN/INIT S mirror in place. */ + pg_atomic_uint64 invalidate_passive_s_release_count; + /* Sole-requester S source fused the existing revoke into its grant, and + * the matching DRAIN preserved the resulting current X descriptor. */ + pg_atomic_uint64 pcm_x_self_handoff_count; + pg_atomic_uint64 pcm_x_self_handoff_drain_count; pg_atomic_uint64 invalidate_park_overflow_count; pg_atomic_uint64 drop_pinned_deny_count; /* PGRAC: GCS serve-stall round-6 — the generation gate refused a drop @@ -415,6 +431,10 @@ static ClusterGcsBlockBackendBlock *gcs_block_my_block(void); static ClusterGcsBlockOutstandingSlot *gcs_block_reserve_slot(BufferTag tag, uint8 transition_id, int32 master_node, uint64 *out_request_id); +static ClusterGcsBlockOutstandingSlot *gcs_block_try_reserve_exact_slot(BufferTag tag, + uint8 transition_id, + int32 expected_source_node, + uint64 request_id); static void gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot); static void gcs_block_send_reply(int32 dest_node, const GcsBlockRequestPayload *req, GcsBlockReplyStatus status, XLogRecPtr page_lsn, @@ -430,7 +450,21 @@ static uint32 gcs_block_compute_checksum(const char *block_data); static uint32 gcs_block_compute_invalidate_checksum(const GcsBlockInvalidatePayload *inv); static uint32 gcs_block_compute_invalidate_ack_checksum(const GcsBlockInvalidateAckPayload *ack); static uint32 gcs_block_compute_redeclare_checksum(const GcsBlockRedeclarePayload *p); +static PcmXSessionAuthResult +gcs_block_pcm_x_authenticated_session_result(int32 node_id, uint64 expected_epoch, + uint64 *session_out, + ClusterGcsPcmXAuthSample *sample_out); +static bool gcs_block_pcm_x_authenticated_session(int32 node_id, uint64 expected_epoch, + uint64 *session_out); +static bool gcs_block_pcm_x_revalidate_peer_binding(int32 node_id, uint64 epoch, uint64 session); +static bool gcs_block_pcm_x_source_capable(int32 node_id); +static PcmXQueueResult gcs_block_pcm_x_fetch_own_result(ClusterPcmOwnResult result); +static PcmXQueueResult +gcs_block_pcm_x_fetch_reservation_mismatch(const ClusterPcmOwnSnapshot *live); static void gcs_block_install_block(BufferDesc *buf, const char *block_data, XLogRecPtr page_lsn); +static PcmXQueueResult gcs_block_pcm_x_install_reserved_image_exact( + BufferDesc *buf, const ClusterPcmOwnSnapshot *reservation_base, uint64 reservation_token, + const char *block_data, XLogRecPtr page_lsn, const PcmXRuntimeSnapshot *request_runtime); static void gcs_block_install_reply_block(BufferDesc *buf, const char *block_data, XLogRecPtr page_lsn, const ClusterGcsBlockOutstandingSlot *slot); @@ -519,6 +553,9 @@ cluster_gcs_block_shmem_init(void) pg_atomic_init_u64(&ClusterGcsBlock->invalidate_parked_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->invalidate_busy_sent_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->invalidate_busy_received_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->invalidate_passive_s_release_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->pcm_x_self_handoff_count, 0); + pg_atomic_init_u64(&ClusterGcsBlock->pcm_x_self_handoff_drain_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->invalidate_park_expired_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->invalidate_park_overflow_count, 0); pg_atomic_init_u64(&ClusterGcsBlock->drop_pinned_deny_count, 0); @@ -668,6 +705,34 @@ gcs_block_my_block(void) return &gcs_block_backend_blocks[idx]; } + +/* Mint a queue request id from the same per-backend domain as ordinary block + * requests without consuming a block-reply slot. The shared counter lock is + * the collision boundary between the two users. Queue identities are + * durable, so wrapping the 40-bit wire sequence is exhaustion, not reuse. */ +static bool +gcs_block_pcm_x_next_request_id(uint64 *request_id_out) +{ + ClusterGcsBlockBackendBlock *blk; + uint64 sequence; + + if (request_id_out != NULL) + *request_id_out = 0; + if (request_id_out == NULL || MyBackendId <= 0 || MyBackendId > MaxBackends) + return false; + blk = gcs_block_my_block(); + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + sequence = blk->next_request_id; + if (sequence == 0 || sequence > GCS_REQID_REQUESTER_SEQ_MASK) { + LWLockRelease(&blk->lock.lock); + return false; + } + blk->next_request_id++; + *request_id_out = gcs_reqid_requester(cluster_node_id, (int)MyBackendId - 1, sequence); + LWLockRelease(&blk->lock.lock); + return *request_id_out != 0; +} + static ClusterGcsBlockOutstandingSlot * gcs_block_reserve_slot(BufferTag tag, uint8 transition_id, int32 master_node, uint64 *out_request_id) @@ -724,6 +789,67 @@ gcs_block_reserve_slot(BufferTag tag, uint8 transition_id, int32 master_node, return slot; } + +/* Reserve the established reply table with a canonical PCM-X image handle. + * The handle is already generation-exact, so consuming a second generated + * request id would sever the record/reply correlation. Refuse a concurrent + * duplicate in this backend instead of letting one reply wake an arbitrary + * same-id slot. */ +static ClusterGcsBlockOutstandingSlot * +gcs_block_try_reserve_exact_slot(BufferTag tag, uint8 transition_id, int32 expected_source_node, + uint64 request_id) +{ + ClusterGcsBlockBackendBlock *blk; + ClusterGcsBlockOutstandingSlot *slot = NULL; + int i; + + if (!cluster_pcm_x_image_id_decode(request_id, NULL, NULL) || expected_source_node < 0 + || expected_source_node >= PCM_X_PROTOCOL_NODE_LIMIT) + return NULL; + blk = gcs_block_my_block(); + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + for (i = 0; i < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; i++) { + if (blk->slots[i].in_use && blk->slots[i].request_id == request_id) + goto reserve_done; + } + for (i = 0; i < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; i++) { + if (!blk->slots[i].in_use) { + slot = &blk->slots[i]; + slot->in_use = true; + slot->request_id = request_id; + slot->transition_id = transition_id; + slot->tag = tag; + slot->master_node = expected_source_node; + slot->reply_received = false; + memset(&slot->reply_header, 0, sizeof(slot->reply_header)); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->reply_undo_trailer_valid = false; + slot->reply_undo_tt_generation = 0; + slot->reply_undo_authority_scn = 0; + slot->request_epoch = 0; + slot->expected_master_node = expected_source_node; + slot->stale = false; + slot->direct_state = GCS_BLOCK_DIRECT_UNARMED; + slot->direct_expected_peer = -1; + slot->direct_arm_id = 0; + slot->direct_target_kind = GCS_BLOCK_DIRECT_TARGET_NONE; + slot->direct_target_buf = NULL; + slot->direct_target_addr = NULL; + slot->direct_target_lkey = 0; + slot->direct_target_prepared = false; + slot->direct_abort_reason = GCS_BLOCK_DIRECT_ABORT_NONE; + break; + } + } + +reserve_done: + LWLockRelease(&blk->lock.lock); + return slot; +} + static void gcs_block_release_slot(ClusterGcsBlockOutstandingSlot *slot) { @@ -1195,11 +1321,94 @@ gcs_block_install_block(BufferDesc *buf, const char *block_data, XLogRecPtr page content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) { + LWLockRelease(content_lock); + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("refusing to overwrite retained cluster PCM image"), + errdetail("buffer=%d", buf->buf_id))); + } + page = BufferGetPage(BufferDescriptorGetBuffer(buf)); + memcpy(page, block_data, GCS_BLOCK_DATA_SIZE); + gcs_block_note_install_copy(); + PageSetLSN(page, page_lsn); + LWLockRelease(content_lock); +} + + +/* + * Install one immutable PCM-X holder image without owning the surrounding + * reservation lifecycle. The queue driver began GRANT_PENDING and remains + * solely responsible for commit/abort. Recheck the complete ownership tuple + * on both sides of the content-lock window so descriptor reuse or a competing + * lifecycle can never turn a valid image into a write to the wrong page. + */ +static PcmXQueueResult +gcs_block_pcm_x_install_reserved_image_exact(BufferDesc *buf, + const ClusterPcmOwnSnapshot *reservation_base, + uint64 reservation_token, const char *block_data, + XLogRecPtr page_lsn, + const PcmXRuntimeSnapshot *request_runtime) +{ + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult own_result; + PcmXRuntimeSnapshot runtime; + LWLock *content_lock; + Page page; + + if (buf == NULL || reservation_base == NULL || block_data == NULL || request_runtime == NULL) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (!cluster_gcs_pcm_x_requester_runtime_exact(request_runtime, &runtime)) + return PCM_X_QUEUE_NOT_READY; + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + if (own_result != CLUSTER_PCM_OWN_OK) + return gcs_block_pcm_x_fetch_own_result(own_result); + if (!cluster_pcm_x_image_fetch_reservation_exact(&live, reservation_base, reservation_token)) + return gcs_block_pcm_x_fetch_reservation_mismatch(&live); + + content_lock = BufferDescriptorGetContentLock(buf); + LWLockAcquire(content_lock, LW_EXCLUSIVE); + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + if (own_result != CLUSTER_PCM_OWN_OK + || !cluster_pcm_x_image_fetch_reservation_exact(&live, reservation_base, + reservation_token)) { + LWLockRelease(content_lock); + return own_result != CLUSTER_PCM_OWN_OK ? gcs_block_pcm_x_fetch_own_result(own_result) + : gcs_block_pcm_x_fetch_reservation_mismatch(&live); + } + runtime = cluster_pcm_x_runtime_snapshot(); + if (!cluster_gcs_pcm_x_requester_runtime_exact(request_runtime, &runtime)) { + LWLockRelease(content_lock); + return PCM_X_QUEUE_NOT_READY; + } + page = BufferGetPage(BufferDescriptorGetBuffer(buf)); memcpy(page, block_data, GCS_BLOCK_DATA_SIZE); gcs_block_note_install_copy(); PageSetLSN(page, page_lsn); + own_result = cluster_bufmgr_pcm_own_publish_installed_x_image(buf, reservation_base, + reservation_token); + if (own_result != CLUSTER_PCM_OWN_OK) { + /* Bytes changed without a publishable exact reservation. No prior + * local image is available for rollback, so retain all evidence and + * stop queue admission. */ + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + return PCM_X_QUEUE_CORRUPT; + } + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + runtime = cluster_pcm_x_runtime_snapshot(); + if (own_result != CLUSTER_PCM_OWN_OK + || !cluster_pcm_x_image_fetch_reservation_exact(&live, reservation_base, reservation_token) + || !cluster_gcs_pcm_x_requester_runtime_exact(request_runtime, &runtime)) { + /* Bytes were copied while the reservation identity changed. Core has + * no sound rollback image for that boundary. */ + cluster_pcm_x_runtime_fail_closed(); + LWLockRelease(content_lock); + return PCM_X_QUEUE_CORRUPT; + } LWLockRelease(content_lock); + return PCM_X_QUEUE_OK; } static void @@ -2575,218 +2784,709 @@ cluster_gcs_send_block_request_and_wait(BufferDesc *buf, PcmLockTransition trans } -/* - * PGRAC: spec-5.2 D2 (sub-case B) — local-master read-image forward. - * - * When THIS node is the GCS master for a block that a REMOTE node holds in - * X, and a local reader needs an N→S image (e.g. to see an uncommitted ITL - * row-lock before a cross-node TX wait), the tag-only acquire path cannot - * serve the read (it has no data plane). Here the master forwards a - * read-image request straight to the holder and waits for the holder to - * direct-ship the current image (status READ_IMAGE_FROM_XHOLDER). The - * holder keeps its X; this node installs the bytes for THIS read only and - * never registers as an S holder (returns false so buf->pcm_state stays N). - * - * spec-6.12a ㉕: with cluster.read_scache on, the forward also carries the - * downgrade-request flag. If the holder accepts (ships - * S_GRANTED_XHOLDER_DOWNGRADE), this node — being the master — registers - * itself as an S holder via a LOCAL transition apply and returns true - * (durable S; the caller mirrors pcm_state). Registration failure - * degrades to the one-shot semantics below. - * - * Returns false for the one-shot read image (non-durable), true for the - * ㉕ durable downgraded S grant. Fails closed (ereport) if no image can - * be obtained — never a silent stale read (Rule 8.A). - */ -bool -cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) +/* The canonical image handle is bound to one epoch, holder, and master + * incarnation. Unlike an ordinary block request it must never follow an + * epoch-driven master rehash: loss of any authority component makes this + * exact attempt retryable only by the outer convert-queue driver. */ +static bool +gcs_block_pcm_x_fetch_authority_exact(const PcmXLocalProgress *progress) { - ClusterGcsBlockOutstandingSlot *slot; - uint64 request_id = 0; - BufferTag tag; - GcsBlockForwardPayload fwd; - bool got_reply = false; - bool installed = false; - bool durable_s = false; /* spec-6.12a ㉕ — holder downgraded, we registered */ + uint64 holder_session; + uint64 master_session; + uint64 epoch; + int32 holder_node; + int32 master_node; - if (buf == NULL) - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_gcs_local_master_read_image_and_wait: NULL BufferDesc"))); + if (progress == NULL) + return false; + epoch = progress->identity.cluster_epoch; + holder_node = (int32)progress->image.source_node; + master_node = progress->master_node; + if (cluster_epoch_get_current() != epoch || holder_node < 0 + || holder_node >= PCM_X_PROTOCOL_NODE_LIMIT || master_node < 0 + || master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || cluster_gcs_lookup_master(progress->identity.tag) != master_node + || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(holder_node) || !cluster_membership_is_member(master_node) + || cluster_gcs_block_phase_for_tag(progress->identity.tag) == GCS_BLOCK_RECOVERING + || (cluster_write_fence_enforcing() && !cluster_write_fence_allowed()) + || !gcs_block_pcm_x_source_capable(holder_node) + || !gcs_block_pcm_x_source_capable(master_node) + || !gcs_block_pcm_x_authenticated_session(holder_node, epoch, &holder_session) + || !gcs_block_pcm_x_revalidate_peer_binding(holder_node, epoch, holder_session) + || !gcs_block_pcm_x_authenticated_session(master_node, epoch, &master_session) + || master_session != progress->master_session_incarnation + || !gcs_block_pcm_x_revalidate_peer_binding(master_node, epoch, master_session)) + return false; + return true; +} - tag = buf->tag; - cluster_gcs_block_dedup_register_backend_exit_hook(); - /* expected_master == self: the holder's reply carries forwarding_master = - * self, which the HC108 authorized chain validates against this slot. */ - slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_S, cluster_node_id, &request_id); - PG_TRY(); - { - ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); - TimestampTz deadline; +/* Bind every requester-side mutation and sleep to the exact formation that + * admitted it. The runtime token protects the local gate generation/session; + * the peer binding protects the authenticated master connection inside that + * formation. */ +static bool +gcs_block_pcm_x_requester_authority_exact(const PcmXRuntimeSnapshot *request_runtime, + int32 master_node, uint64 cluster_epoch, + uint64 master_session) +{ + PcmXRuntimeSnapshot current = cluster_pcm_x_runtime_snapshot(); - LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); - slot->reply_received = false; - memset(&slot->reply_header, 0, sizeof(slot->reply_header)); - memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); - slot->reply_sf_dep_valid = false; - slot->reply_sf_flags = 0; - cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); - slot->request_epoch = cluster_epoch_get_current(); - slot->expected_master_node = cluster_node_id; - slot->stale = false; - LWLockRelease(&blk->lock.lock); + return cluster_gcs_pcm_x_requester_runtime_exact(request_runtime, ¤t) + && gcs_block_pcm_x_revalidate_peer_binding(master_node, cluster_epoch, master_session); +} - memset(&fwd, 0, sizeof(fwd)); - fwd.request_id = request_id; - fwd.epoch = cluster_epoch_get_current(); - fwd.tag = tag; - fwd.original_requester_node = cluster_node_id; /* reply returns to us */ - fwd.requester_backend_id = (int32)MyBackendId; - fwd.master_node = cluster_node_id; - fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; - GcsBlockForwardPayloadSetExpectedPiWatermarkScn( - &fwd, cluster_pcm_lock_pi_watermark_scn_query(tag)); - GcsBlockForwardPayloadSetReadImage(&fwd, true); - /* PGRAC: spec-6.12a ㉕ — ask the remote X holder to TRY the quiescent - * X->S downgrade so this read (and every later one) becomes a durable - * cached S. We ARE the master here, so on the holder's durable reply - * the registration is a local transition apply — no ACK wire. */ - if (cluster_read_scache) - GcsBlockForwardPayloadSetDowngradeRequest(&fwd, true); - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_sent_count, 1); - if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, - (uint32)holder_node, &fwd, sizeof(fwd))) - ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("cluster_gcs_block: failed to enqueue read-image FORWARD " - "to X holder %d", - holder_node))); +/* Image fetch also crosses the holder data-plane connection. Reuse the + * canonical fetch authority proof so both holder and master bindings remain + * exact while additionally pinning the original requester runtime token. */ +static bool +gcs_block_pcm_x_fetch_requester_authority_exact(const PcmXRuntimeSnapshot *request_runtime, + const PcmXLocalProgress *progress) +{ + PcmXRuntimeSnapshot current = cluster_pcm_x_runtime_snapshot(); - deadline = GetCurrentTimestamp() - + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; + return cluster_gcs_pcm_x_requester_runtime_exact(request_runtime, ¤t) + && gcs_block_pcm_x_fetch_authority_exact(progress); +} - ConditionVariablePrepareToSleep(&slot->reply_cv); - for (;;) { - TimestampTz now; - long timeout_ms; - bool have_reply; - LWLockAcquire(&blk->lock.lock, LW_SHARED); - have_reply = slot->in_use && slot->reply_received; - LWLockRelease(&blk->lock.lock); - if (have_reply) { - got_reply = true; - break; - } - now = GetCurrentTimestamp(); - if (now >= deadline) - break; - timeout_ms = (long)((deadline - now) / 1000); - if (timeout_ms <= 0) - timeout_ms = 1; - (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, - WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); - } - ConditionVariableCancelSleep(); +/* Map the opaque ownership substrate to the queue driver's result domain. + * A malformed live flag shape is evidence, not contention, and closes the + * PCM-X runtime before returning. */ +static PcmXQueueResult +gcs_block_pcm_x_fetch_own_result(ClusterPcmOwnResult result) +{ + switch (result) { + case CLUSTER_PCM_OWN_OK: + return PCM_X_QUEUE_OK; + case CLUSTER_PCM_OWN_STALE: + return PCM_X_QUEUE_STALE; + case CLUSTER_PCM_OWN_BUSY: + cluster_pcm_x_stats_note_own_busy(); + return PCM_X_QUEUE_BUSY; + case CLUSTER_PCM_OWN_EXHAUSTED: + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + case CLUSTER_PCM_OWN_NOT_READY: + return PCM_X_QUEUE_NOT_READY; + case CLUSTER_PCM_OWN_CORRUPT: + cluster_pcm_x_stats_note_own_corrupt(); + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + case CLUSTER_PCM_OWN_INVALID: + default: + return PCM_X_QUEUE_INVALID; + } +} - if (got_reply - && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER - || slot->reply_header.status - == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { - uint32 expected = slot->reply_header.checksum; - uint32 got = gcs_block_compute_checksum(slot->reply_block_data); - if (expected == got) { - gcs_block_install_reply_block(buf, slot->reply_block_data, - (XLogRecPtr)slot->reply_header.page_lsn, slot); - /* spec-5.14 D2 class 2: this node (local master) consumed the - * remote holder's volatile image. */ - gcs_block_stamp_touched(holder_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); - installed = true; - /* - * PGRAC: spec-6.12a ㉕ — the holder downgraded X->S and shipped - * a DURABLE S grant. We are the master: register ourselves as - * an S holder with a LOCAL transition apply (no ACK wire). The - * holder's own X->S notify travels on the LMON dispatch path; - * if it was applied first our N->S lands on state S (bitmap - * add); if it is still in flight the apply fails and we - * DEGRADE to the one-shot semantics — install stands, - * pcm_state stays N (Rule 8.A: never a durable copy the - * master entry does not track). - */ - if (slot->reply_header.status - == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE) { - if (cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, - cluster_node_id)) - durable_s = true; - else - cluster_lever_a_note_remote_ack_degraded(); - } - } else { - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); +static PcmXQueueResult +gcs_block_pcm_x_fetch_reservation_mismatch(const ClusterPcmOwnSnapshot *live) +{ + ClusterPcmOwnResult shape; + + if (live == NULL) + return PCM_X_QUEUE_INVALID; + shape = cluster_pcm_own_classify_live_flags(live->flags, live->reservation_token); + if (shape == CLUSTER_PCM_OWN_CORRUPT) + return gcs_block_pcm_x_fetch_own_result(shape); + /* A well-formed but non-exact live tuple belongs to another lifecycle. */ + return PCM_X_QUEUE_STALE; +} + + +static bool +gcs_block_pcm_x_self_page_exact(Page page, const PcmXImageToken *image) +{ + return page != NULL && image != NULL && PageGetLSN(page) == (XLogRecPtr)image->page_lsn + && (uint64)((PageHeader)page)->pd_block_scn == image->page_scn + && cluster_gcs_block_compute_checksum((const char *)page) == image->page_checksum; +} + + +static bool +gcs_block_pcm_x_self_progress_exact(const PcmXLocalProgress *progress, + const PcmXLocalHandle *leader, + const ClusterPcmOwnSnapshot *revoking_base, + uint16 expected_response) +{ + uint32 expected_member_state; + + if (progress == NULL || leader == NULL || revoking_base == NULL + || (expected_response != PGRAC_IC_MSG_PCM_X_PREPARE_GRANT + && expected_response != PGRAC_IC_MSG_PCM_X_COMMIT_X)) + return false; + expected_member_state = expected_response == PGRAC_IC_MSG_PCM_X_PREPARE_GRANT + ? PCM_XL_REMOTE_WAIT + : PCM_XL_CONTENT_ACTIVE; + return memcmp(&progress->identity, &leader->identity, sizeof(progress->identity)) == 0 + && BufferTagsEqual(&progress->identity.tag, &revoking_base->tag) + && progress->identity.base_own_generation == revoking_base->generation + && progress->role == PCM_X_LOCAL_ROLE_NODE_LEADER + && progress->member_state == expected_member_state && progress->pending_opcode == 0 + && progress->last_response_opcode == expected_response + && progress->image.source_node == (uint32)cluster_node_id + && progress->image.source_own_generation == revoking_base->generation + && progress->image.image_id != 0 && progress->master_session_incarnation != 0; +} + + +static ClusterPcmXGrantReservationKind +gcs_block_pcm_x_self_handoff_kind(uint8 pcm_state) +{ + switch ((PcmState)pcm_state) { + case PCM_STATE_N: + return CLUSTER_PCM_X_GRANT_RESERVATION_N_REVOKE_HANDOFF; + case PCM_STATE_S: + return CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF; + case PCM_STATE_X: + return CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF; + case PCM_STATE_READ_IMAGE: + default: + return CLUSTER_PCM_X_GRANT_RESERVATION_INVALID; + } +} + + +PcmXQueueResult +cluster_gcs_pcm_x_adopt_self_image(BufferDesc *buf, const PcmXLocalHandle *leader, + const ClusterPcmOwnSnapshot *revoking_base, + uint64 *out_reservation_token) +{ + GcsBlockRequestPayload request_proof; + PcmXLocalProgress progress_before; + PcmXLocalProgress progress_after; + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult own_result; + PcmXQueueResult queue_result; + LWLock *content_lock; + Page page; + volatile bool handed_off = false; + volatile bool handoff_transitioned = false; + volatile bool exact_revoking_before = false; + volatile bool content_locked = false; + uint64 reservation_token = 0; + ClusterPcmXGrantReservationKind expected_kind; + + if (out_reservation_token != NULL) + *out_reservation_token = 0; + if (buf == NULL || leader == NULL || revoking_base == NULL || out_reservation_token == NULL + || MyBackendId <= 0 || revoking_base->flags != PCM_OWN_FLAG_REVOKING + || revoking_base->reservation_token == 0) + return PCM_X_QUEUE_INVALID; + expected_kind = gcs_block_pcm_x_self_handoff_kind(revoking_base->pcm_state); + if (expected_kind == CLUSTER_PCM_X_GRANT_RESERVATION_INVALID) + return PCM_X_QUEUE_INVALID; + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_before); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result != PCM_X_QUEUE_OK) + return queue_result; + if (!gcs_block_pcm_x_self_progress_exact(&progress_before, leader, revoking_base, + PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) + || !cluster_pcm_x_image_fetch_build_request(&progress_before, cluster_node_id, + (int32)MyBackendId, &request_proof) + || !gcs_block_pcm_x_fetch_authority_exact(&progress_before)) + return PCM_X_QUEUE_STALE; + + content_lock = BufferDescriptorGetContentLock(buf); + PG_TRY(); + { + LWLockAcquire(content_lock, LW_EXCLUSIVE); + content_locked = true; + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result == PCM_X_QUEUE_OK) { + bool exact_revoking = BufferTagsEqual(&live.tag, &revoking_base->tag) + && live.generation == revoking_base->generation + && live.reservation_token == revoking_base->reservation_token + && live.flags == PCM_OWN_FLAG_REVOKING + && live.pcm_state == revoking_base->pcm_state; + bool exact_handoff = cluster_pcm_x_grant_reservation_kind( + &live, revoking_base, revoking_base->reservation_token) + == expected_kind; + + if (!exact_revoking && !exact_handoff) + queue_result = gcs_block_pcm_x_fetch_reservation_mismatch(&live); + exact_revoking_before = exact_revoking; + } + page = BufferGetPage(BufferDescriptorGetBuffer(buf)); + if (queue_result == PCM_X_QUEUE_OK + && !gcs_block_pcm_x_self_page_exact(page, &progress_before.image)) { + cluster_pcm_x_runtime_fail_closed(); + queue_result = PCM_X_QUEUE_CORRUPT; + } + if (queue_result == PCM_X_QUEUE_OK) { + own_result = cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(buf, revoking_base, + &reservation_token); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + handed_off = queue_result == PCM_X_QUEUE_OK; + handoff_transitioned = handed_off && exact_revoking_before; + } + if (queue_result == PCM_X_QUEUE_OK) { + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + if (gcs_block_pcm_x_fetch_own_result(own_result) != PCM_X_QUEUE_OK + || cluster_pcm_x_grant_reservation_kind(&live, revoking_base, reservation_token) + != expected_kind) { + cluster_pcm_x_runtime_fail_closed(); + queue_result = PCM_X_QUEUE_CORRUPT; } } + LWLockRelease(content_lock); + content_locked = false; } PG_CATCH(); { - gcs_block_release_slot(slot); + if (handed_off) + cluster_pcm_x_runtime_fail_closed(); + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); PG_RE_THROW(); } PG_END_TRY(); + if (queue_result != PCM_X_QUEUE_OK) + return queue_result; + + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_after); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result != PCM_X_QUEUE_OK + || memcmp(&progress_before, &progress_after, sizeof(progress_after)) != 0 + || !gcs_block_pcm_x_fetch_authority_exact(&progress_after)) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + gcs_block_stamp_touched(cluster_node_id, progress_after.master_node); + if (handoff_transitioned && ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->pcm_x_self_handoff_count, 1); + *out_reservation_token = reservation_token; + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_gcs_pcm_x_finish_self_image_x(BufferDesc *buf, const PcmXLocalHandle *leader, + const ClusterPcmOwnSnapshot *revoking_base, + uint64 reservation_token, uint64 *out_committed_generation) +{ + PcmXLocalProgress progress = { 0 }; + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult own_result; + volatile PcmXQueueResult queue_result; + LWLock *content_lock; + Page page; + volatile uint64 committed_generation = 0; + volatile bool content_locked = false; + volatile bool committed = false; + volatile bool handoff_live = false; + ClusterPcmXGrantReservationKind expected_kind; + + if (out_committed_generation != NULL) + *out_committed_generation = 0; + if (buf == NULL || leader == NULL || revoking_base == NULL || out_committed_generation == NULL + || reservation_token == 0) + return PCM_X_QUEUE_INVALID; + expected_kind = gcs_block_pcm_x_self_handoff_kind(revoking_base->pcm_state); + if (expected_kind == CLUSTER_PCM_X_GRANT_RESERVATION_INVALID) + return PCM_X_QUEUE_INVALID; + + /* PREPARE's role handoff is irreversible: first prove that exact tuple is + * live, so every later phase/session/interrupt failure is recovery-blocked + * rather than exposed as an ordinary retry. */ + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result != PCM_X_QUEUE_OK) + return queue_result; + if (cluster_pcm_x_grant_reservation_kind(&live, revoking_base, reservation_token) + != expected_kind) + return gcs_block_pcm_x_fetch_reservation_mismatch(&live); + handoff_live = true; + + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result != PCM_X_QUEUE_OK + || !gcs_block_pcm_x_self_progress_exact(&progress, leader, revoking_base, + PGRAC_IC_MSG_PCM_X_COMMIT_X) + || !gcs_block_pcm_x_fetch_authority_exact(&progress)) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } - gcs_block_release_slot(slot); - - if (durable_s) - return true; /* spec-6.12a ㉕ — durable S; caller mirrors pcm_state = S */ - if (installed) - return false; /* one-shot read image — non-durable, leave pcm_state N */ + content_lock = BufferDescriptorGetContentLock(buf); + PG_TRY(); + { + LWLockAcquire(content_lock, LW_EXCLUSIVE); + content_locked = true; + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result == PCM_X_QUEUE_OK + && cluster_pcm_x_grant_reservation_kind(&live, revoking_base, reservation_token) + != expected_kind) + queue_result = gcs_block_pcm_x_fetch_reservation_mismatch(&live); + page = BufferGetPage(BufferDescriptorGetBuffer(buf)); + if (queue_result == PCM_X_QUEUE_OK + && !gcs_block_pcm_x_self_page_exact(page, &progress.image)) + queue_result = PCM_X_QUEUE_CORRUPT; + if (queue_result == PCM_X_QUEUE_OK) { + uint64 next_generation = 0; + + own_result = cluster_bufmgr_pcm_own_finish_x_commit( + buf, revoking_base, reservation_token, &next_generation); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result == PCM_X_QUEUE_OK) { + committed_generation = next_generation; + committed = true; + } + } + LWLockRelease(content_lock); + content_locked = false; + } + PG_CATCH(); + { + /* LWLockRelease resumes interrupts. A pending cancel/die after the + * ownership bump must never bypass FINAL as an ordinary backend error. */ + if (handoff_live || committed) + cluster_pcm_x_runtime_fail_closed(); + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + PG_RE_THROW(); + } + PG_END_TRY(); + if (queue_result != PCM_X_QUEUE_OK) { + /* Handoff is irreversible even when the ownership bump itself failed. + * Preserve exact evidence for recovery instead of aborting. */ + cluster_pcm_x_runtime_fail_closed(); + return queue_result; + } + if (revoking_base->generation == UINT64_MAX + || committed_generation != revoking_base->generation + 1) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + *out_committed_generation = committed_generation; + return PCM_X_QUEUE_OK; +} - /* No read image obtained (timeout / holder evict / denial) — fail closed, - * never a silent stale read (Rule 8.A). */ - ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cluster_gcs_block: could not obtain read image from X holder %d " - "for tag spc=%u db=%u relNumber=%u block=%u", - holder_node, tag.spcOid, tag.dbOid, - (unsigned int)BufTagGetRelNumber(&tag), (unsigned int)tag.blockNum), - errhint("The X holder did not ship a current image in time; retry, or " - "inspect dump_gcs.cf_xheld_read_ship_count."))); - return true; /* unreachable */ + +/* + * Fetch the immutable holder READY record through the established GCS block + * data plane and install it under the exact GRANT_PENDING reservation. This + * function neither advances the queue phase nor commits/aborts ownership; + * the caller retains both lifecycles and must present the same leader/base/ + * token tuple to its next protocol step. + */ +PcmXQueueResult +cluster_gcs_pcm_x_fetch_image_and_install(BufferDesc *buf, const PcmXLocalHandle *leader, + const ClusterPcmOwnSnapshot *reservation_base, + uint64 reservation_token, + const PcmXRuntimeSnapshot *request_runtime) +{ + ClusterGcsBlockOutstandingSlot *slot; + PcmXLocalProgress progress_before; + ClusterPcmOwnSnapshot live_own; + GcsBlockRequestPayload request; + PcmXQueueResult queue_result; + ClusterPcmOwnResult own_result; + volatile PcmXQueueResult result = PCM_X_QUEUE_NOT_READY; + int max_retries; + int retry_attempt; + + if (buf == NULL || leader == NULL || reservation_base == NULL || request_runtime == NULL + || MyBackendId <= 0) + return PCM_X_QUEUE_INVALID; + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_before); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result != PCM_X_QUEUE_OK) + return queue_result; + if (!BufferTagsEqual(&buf->tag, &reservation_base->tag) + || !BufferTagsEqual(&leader->identity.tag, &reservation_base->tag) + || reservation_base->generation != leader->identity.base_own_generation) + return PCM_X_QUEUE_STALE; + if (!cluster_pcm_x_image_fetch_build_request(&progress_before, cluster_node_id, + (int32)MyBackendId, &request)) + return PCM_X_QUEUE_NOT_READY; + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, &progress_before)) + return PCM_X_QUEUE_NOT_READY; + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live_own); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result != PCM_X_QUEUE_OK) + return queue_result; + if (!cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, + reservation_token)) + return gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); + + cluster_gcs_block_dedup_register_backend_exit_hook(); + slot = gcs_block_try_reserve_exact_slot(request.tag, request.transition_id, + (int32)progress_before.image.source_node, + request.request_id); + if (slot == NULL) + return PCM_X_QUEUE_BUSY; + max_retries = cluster_gcs_block_retransmit_max_retries >= 0 + ? cluster_gcs_block_retransmit_max_retries + : 4; + + PG_TRY(); + { + for (retry_attempt = 0; retry_attempt <= max_retries; retry_attempt++) { + ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); + PcmXLocalProgress progress_now; + GcsBlockReplyHeader reply; + char reply_block[GCS_BLOCK_DATA_SIZE]; + TimestampTz deadline; + bool fence_lost = false; + bool got_reply = false; + bool slot_stale = false; + + if (retry_attempt > 0) { + long backoff_ms = gcs_block_backoff_ms_for_retry(retry_attempt); + + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, + &progress_before)) { + result = PCM_X_QUEUE_NOT_READY; + break; + } + pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_attempt_count, 1); + (void)WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, backoff_ms, + WAIT_EVENT_GCS_BLOCK_RETRANSMIT_WAIT); + ResetLatch(MyLatch); + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, + &progress_before)) { + result = PCM_X_QUEUE_NOT_READY; + break; + } + } + + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_now); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result != PCM_X_QUEUE_OK) { + result = queue_result; + break; + } + if (memcmp(&progress_before, &progress_now, sizeof(progress_now)) != 0) { + result = PCM_X_QUEUE_STALE; + break; + } + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, &progress_now)) { + result = cluster_epoch_get_current() == request.epoch ? PCM_X_QUEUE_NOT_READY + : PCM_X_QUEUE_STALE; + break; + } + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live_own); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result != PCM_X_QUEUE_OK) { + result = queue_result; + break; + } + if (!cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, + reservation_token)) { + result = gcs_block_pcm_x_fetch_reservation_mismatch(&live_own); + break; + } + + /* Rearm the exact image id atomically with its fixed authority. */ + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + slot->reply_received = false; + memset(&slot->reply_header, 0, sizeof(slot->reply_header)); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->reply_undo_trailer_valid = false; + slot->reply_undo_tt_generation = 0; + slot->reply_undo_authority_scn = 0; + slot->request_epoch = request.epoch; + slot->expected_master_node = (int32)progress_now.image.source_node; + slot->stale = false; + LWLockRelease(&blk->lock.lock); + + ConditionVariablePrepareToSleep(&slot->reply_cv); + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, &progress_now)) { + ConditionVariableCancelSleep(); + result = PCM_X_QUEUE_NOT_READY; + break; + } + if (retry_attempt == 0) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_request_count, 1); + else + pg_atomic_fetch_add_u64(&ClusterGcsBlock->retransmit_send_count, 1); + if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_REQUEST, + progress_now.image.source_node, &request, + sizeof(request))) { + ConditionVariableCancelSleep(); + if (retry_attempt < max_retries) + continue; + break; + } + + deadline = GetCurrentTimestamp() + + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; + for (;;) { + TimestampTz now; + long timeout_ms; + + LWLockAcquire(&blk->lock.lock, LW_SHARED); + got_reply = slot->in_use && slot->reply_received; + slot_stale = slot->in_use && slot->stale; + LWLockRelease(&blk->lock.lock); + if (got_reply || slot_stale) + break; + now = GetCurrentTimestamp(); + if (now >= deadline) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_timeout_count, 1); + break; + } + timeout_ms = (long)((deadline - now) / 1000); + if (timeout_ms <= 0) + timeout_ms = 1; + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, + &progress_now)) { + fence_lost = true; + break; + } + (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, + WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, + &progress_now)) { + fence_lost = true; + break; + } + } + ConditionVariableCancelSleep(); + if (fence_lost) { + result = PCM_X_QUEUE_NOT_READY; + break; + } + if (slot_stale) { + result = PCM_X_QUEUE_STALE; + break; + } + if (!got_reply) + continue; + if (!gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, &progress_now)) { + result = PCM_X_QUEUE_NOT_READY; + break; + } + + LWLockAcquire(&blk->lock.lock, LW_SHARED); + reply = slot->reply_header; + memcpy(reply_block, slot->reply_block_data, sizeof(reply_block)); + LWLockRelease(&blk->lock.lock); + if (reply.status == (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_master_not_holder_count, 1); + continue; + } + if (!cluster_pcm_x_image_fetch_reply_exact(&reply, reply_block, &progress_now, + cluster_node_id, (int32)MyBackendId)) { + cluster_pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_CORRUPT; + break; + } + + /* The reply is valid only while the queue and reservation that armed + * it remain byte-exact. */ + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_now); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result != PCM_X_QUEUE_OK + || memcmp(&progress_before, &progress_now, sizeof(progress_now)) != 0 + || !gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, + &progress_now)) { + result = queue_result == PCM_X_QUEUE_OK ? PCM_X_QUEUE_STALE : queue_result; + break; + } + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live_own); + queue_result = gcs_block_pcm_x_fetch_own_result(own_result); + if (queue_result != PCM_X_QUEUE_OK + || !cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, + reservation_token)) { + result = queue_result == PCM_X_QUEUE_OK + ? gcs_block_pcm_x_fetch_reservation_mismatch(&live_own) + : queue_result; + break; + } + + queue_result = gcs_block_pcm_x_install_reserved_image_exact( + buf, reservation_base, reservation_token, reply_block, (XLogRecPtr)reply.page_lsn, + request_runtime); + if (queue_result != PCM_X_QUEUE_OK) { + result = queue_result; + break; + } + + /* Once bytes landed, a disappearing queue/reservation has no local + * rollback proof. Preserve evidence and close the runtime. */ + queue_result = cluster_pcm_x_local_progress_exact(leader, &progress_now); + cluster_pcm_x_stats_note_queue_result(queue_result); + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live_own); + if (queue_result != PCM_X_QUEUE_OK + || memcmp(&progress_before, &progress_now, sizeof(progress_now)) != 0 + || !gcs_block_pcm_x_fetch_requester_authority_exact(request_runtime, &progress_now) + || gcs_block_pcm_x_fetch_own_result(own_result) != PCM_X_QUEUE_OK + || !cluster_pcm_x_image_fetch_reservation_exact(&live_own, reservation_base, + reservation_token)) { + cluster_pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_CORRUPT; + break; + } + gcs_block_stamp_touched((int32)progress_now.image.source_node, + progress_now.master_node); + result = PCM_X_QUEUE_OK; + break; + } + } + PG_CATCH(); + { + ConditionVariableCancelSleep(); + gcs_block_release_slot(slot); + PG_RE_THROW(); + } + PG_END_TRY(); + + gcs_block_release_slot(slot); + return (PcmXQueueResult)result; } /* - * PGRAC: spec-6.12b — requester-side CR fetch. + * PGRAC: spec-5.2 D2 (sub-case B) — local-master read-image forward. * - * Ask origin_node's CR-server for the CR page of `tag` at read_scn. The - * request rides the sub-case B wire shape (FORWARD payload direct to the - * serving node via the backend outbound ring; the HC108 chain on the - * direct-shipped reply validates forwarding_master == self). The SCN - * carrier holds the snapshot read_scn (the CR path never runs the - * lost-write watermark verdict — the result is historical by intent). + * When THIS node is the GCS master for a block that a REMOTE node holds in + * X, and a local reader needs an N→S image (e.g. to see an uncommitted ITL + * row-lock before a cross-node TX wait), the tag-only acquire path cannot + * serve the read (it has no data plane). Here the master forwards a + * read-image request straight to the holder and waits for the holder to + * direct-ship the current image (status READ_IMAGE_FROM_XHOLDER). The + * holder keeps its X; this node installs the bytes for THIS read only and + * never registers as an S holder (returns false so buf->pcm_state stays N). * - * true -> dst_page holds the shipped CR page; *out_partial says whether - * the local construction continues on it. - * false -> fail-closed: the caller keeps the unchanged 53R9G refusal - * (timeout, DENIED, checksum failure — Rule 8.A). The CR page - * is NEVER installed as current and never flushed; it exists - * only in the caller's CR destination. + * spec-6.12a ㉕: with cluster.read_scache on, the forward also carries the + * downgrade-request flag. If the holder accepts (ships + * S_GRANTED_XHOLDER_DOWNGRADE), this node — being the master — registers + * itself as an S holder via a LOCAL transition apply and returns true + * (durable S; the caller mirrors pcm_state). Registration failure + * degrades to the one-shot semantics below. + * + * Returns false for the one-shot read image (non-durable), true for the + * ㉕ durable downgraded S grant. Fails closed (ereport) if no image can + * be obtained — never a silent stale read (Rule 8.A). */ bool -cluster_gcs_block_cr_fetch_and_wait(BufferTag tag, SCN read_scn, int32 origin_node, char *dst_page, - bool *out_partial) +cluster_gcs_local_master_read_image_and_wait(BufferDesc *buf, int32 holder_node) { ClusterGcsBlockOutstandingSlot *slot; uint64 request_id = 0; + BufferTag tag; GcsBlockForwardPayload fwd; bool got_reply = false; - bool fetched = false; + bool installed = false; + bool durable_s = false; /* spec-6.12a ㉕ — holder downgraded, we registered */ - if (out_partial != NULL) - *out_partial = false; - if (dst_page == NULL || origin_node < 0 || origin_node == cluster_node_id) - return false; + if (buf == NULL) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_gcs_local_master_read_image_and_wait: NULL BufferDesc"))); + tag = buf->tag; cluster_gcs_block_dedup_register_backend_exit_hook(); + /* expected_master == self: the holder's reply carries forwarding_master = + * self, which the HC108 authorized chain validates against this slot. */ slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_S, cluster_node_id, &request_id); PG_TRY(); @@ -2810,19 +3510,27 @@ cluster_gcs_block_cr_fetch_and_wait(BufferTag tag, SCN read_scn, int32 origin_no fwd.request_id = request_id; fwd.epoch = cluster_epoch_get_current(); fwd.tag = tag; - fwd.original_requester_node = cluster_node_id; + fwd.original_requester_node = cluster_node_id; /* reply returns to us */ fwd.requester_backend_id = (int32)MyBackendId; fwd.master_node = cluster_node_id; fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; - GcsBlockForwardPayloadSetExpectedPiWatermarkScn(&fwd, read_scn); - GcsBlockForwardPayloadSetCrRequest(&fwd, true); + GcsBlockForwardPayloadSetExpectedPiWatermarkScn( + &fwd, cluster_pcm_lock_pi_watermark_scn_query(tag)); + GcsBlockForwardPayloadSetReadImage(&fwd, true); + /* PGRAC: spec-6.12a ㉕ — ask the remote X holder to TRY the quiescent + * X->S downgrade so this read (and every later one) becomes a durable + * cached S. We ARE the master here, so on the holder's durable reply + * the registration is a local transition apply — no ACK wire. */ + if (cluster_read_scache) + GcsBlockForwardPayloadSetDowngradeRequest(&fwd, true); + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_sent_count, 1); if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, - (uint32)origin_node, &fwd, sizeof(fwd))) + (uint32)holder_node, &fwd, sizeof(fwd))) ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), - errmsg("cluster_gcs_block: failed to enqueue CR request to " - "origin node %d", - (int)origin_node))); + errmsg("cluster_gcs_block: failed to enqueue read-image FORWARD " + "to X holder %d", + holder_node))); deadline = GetCurrentTimestamp() + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; @@ -2852,43 +3560,205 @@ cluster_gcs_block_cr_fetch_and_wait(BufferTag tag, SCN read_scn, int32 origin_no ConditionVariableCancelSleep(); if (got_reply - && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_CR_RESULT_FULL - || slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_CR_RESULT_PARTIAL)) { + && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || slot->reply_header.status + == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { uint32 expected = slot->reply_header.checksum; uint32 got = gcs_block_compute_checksum(slot->reply_block_data); if (expected == got) { - memcpy(dst_page, slot->reply_block_data, BLCKSZ); - /* spec-5.14 D2 class 2: this CR result is the origin's - * volatile construction — depend on it for fail-stop. */ - gcs_block_stamp_touched(origin_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); - if (out_partial != NULL) - *out_partial - = (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_CR_RESULT_PARTIAL); - fetched = true; - } else { - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); - } - } - } - PG_CATCH(); - { - gcs_block_release_slot(slot); - PG_RE_THROW(); - } - PG_END_TRY(); - - gcs_block_release_slot(slot); - - return fetched; /* false -> caller keeps the unchanged 53R9G refusal */ -} - - -/* - * PGRAC: spec-6.12i D-i1 — requester-side undo-TT fetch. - * - * Ask origin_node for its own TT-bearing undo header block (segment_id, - * block_no) plus the co-sampled live authority triple, riding the same + gcs_block_install_reply_block(buf, slot->reply_block_data, + (XLogRecPtr)slot->reply_header.page_lsn, slot); + /* spec-5.14 D2 class 2: this node (local master) consumed the + * remote holder's volatile image. */ + gcs_block_stamp_touched(holder_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + installed = true; + /* + * PGRAC: spec-6.12a ㉕ — the holder downgraded X->S and shipped + * a DURABLE S grant. We are the master: register ourselves as + * an S holder with a LOCAL transition apply (no ACK wire). The + * holder's own X->S notify travels on the LMON dispatch path; + * if it was applied first our N->S lands on state S (bitmap + * add); if it is still in flight the apply fails and we + * DEGRADE to the one-shot semantics — install stands, + * pcm_state stays N (Rule 8.A: never a durable copy the + * master entry does not track). + */ + if (slot->reply_header.status + == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE) { + if (cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_N_TO_S, + cluster_node_id)) + durable_s = true; + else + cluster_lever_a_note_remote_ack_degraded(); + } + } else { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); + } + } + } + PG_CATCH(); + { + gcs_block_release_slot(slot); + PG_RE_THROW(); + } + PG_END_TRY(); + + gcs_block_release_slot(slot); + + if (durable_s) + return true; /* spec-6.12a ㉕ — durable S; caller mirrors pcm_state = S */ + if (installed) + return false; /* one-shot read image — non-durable, leave pcm_state N */ + + /* No read image obtained (timeout / holder evict / denial) — fail closed, + * never a silent stale read (Rule 8.A). */ + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cluster_gcs_block: could not obtain read image from X holder %d " + "for tag spc=%u db=%u relNumber=%u block=%u", + holder_node, tag.spcOid, tag.dbOid, + (unsigned int)BufTagGetRelNumber(&tag), (unsigned int)tag.blockNum), + errhint("The X holder did not ship a current image in time; retry, or " + "inspect dump_gcs.cf_xheld_read_ship_count."))); + return true; /* unreachable */ +} + + +/* + * PGRAC: spec-6.12b — requester-side CR fetch. + * + * Ask origin_node's CR-server for the CR page of `tag` at read_scn. The + * request rides the sub-case B wire shape (FORWARD payload direct to the + * serving node via the backend outbound ring; the HC108 chain on the + * direct-shipped reply validates forwarding_master == self). The SCN + * carrier holds the snapshot read_scn (the CR path never runs the + * lost-write watermark verdict — the result is historical by intent). + * + * true -> dst_page holds the shipped CR page; *out_partial says whether + * the local construction continues on it. + * false -> fail-closed: the caller keeps the unchanged 53R9G refusal + * (timeout, DENIED, checksum failure — Rule 8.A). The CR page + * is NEVER installed as current and never flushed; it exists + * only in the caller's CR destination. + */ +bool +cluster_gcs_block_cr_fetch_and_wait(BufferTag tag, SCN read_scn, int32 origin_node, char *dst_page, + bool *out_partial) +{ + ClusterGcsBlockOutstandingSlot *slot; + uint64 request_id = 0; + GcsBlockForwardPayload fwd; + bool got_reply = false; + bool fetched = false; + + if (out_partial != NULL) + *out_partial = false; + if (dst_page == NULL || origin_node < 0 || origin_node == cluster_node_id) + return false; + + cluster_gcs_block_dedup_register_backend_exit_hook(); + slot = gcs_block_reserve_slot(tag, (uint8)PCM_TRANS_N_TO_S, cluster_node_id, &request_id); + + PG_TRY(); + { + ClusterGcsBlockBackendBlock *blk = gcs_block_my_block(); + TimestampTz deadline; + + LWLockAcquire(&blk->lock.lock, LW_EXCLUSIVE); + slot->reply_received = false; + memset(&slot->reply_header, 0, sizeof(slot->reply_header)); + memset(slot->reply_block_data, 0, sizeof(slot->reply_block_data)); + slot->reply_sf_dep_valid = false; + slot->reply_sf_flags = 0; + cluster_sf_dep_vec_reset(&slot->reply_sf_dep_vec); + slot->request_epoch = cluster_epoch_get_current(); + slot->expected_master_node = cluster_node_id; + slot->stale = false; + LWLockRelease(&blk->lock.lock); + + memset(&fwd, 0, sizeof(fwd)); + fwd.request_id = request_id; + fwd.epoch = cluster_epoch_get_current(); + fwd.tag = tag; + fwd.original_requester_node = cluster_node_id; + fwd.requester_backend_id = (int32)MyBackendId; + fwd.master_node = cluster_node_id; + fwd.transition_id = (uint8)PCM_TRANS_N_TO_S; + GcsBlockForwardPayloadSetExpectedPiWatermarkScn(&fwd, read_scn); + GcsBlockForwardPayloadSetCrRequest(&fwd, true); + + if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_FORWARD, + (uint32)origin_node, &fwd, sizeof(fwd))) + ereport(ERROR, (errcode(ERRCODE_CONNECTION_FAILURE), + errmsg("cluster_gcs_block: failed to enqueue CR request to " + "origin node %d", + (int)origin_node))); + + deadline = GetCurrentTimestamp() + + ((TimestampTz)cluster_gcs_reply_timeout_ms) * (TimestampTz)1000; + + ConditionVariablePrepareToSleep(&slot->reply_cv); + for (;;) { + TimestampTz now; + long timeout_ms; + bool have_reply; + + LWLockAcquire(&blk->lock.lock, LW_SHARED); + have_reply = slot->in_use && slot->reply_received; + LWLockRelease(&blk->lock.lock); + if (have_reply) { + got_reply = true; + break; + } + now = GetCurrentTimestamp(); + if (now >= deadline) + break; + timeout_ms = (long)((deadline - now) / 1000); + if (timeout_ms <= 0) + timeout_ms = 1; + (void)ConditionVariableTimedSleep(&slot->reply_cv, timeout_ms, + WAIT_EVENT_GCS_BLOCK_SHIP_WAIT); + } + ConditionVariableCancelSleep(); + + if (got_reply + && (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_CR_RESULT_FULL + || slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_CR_RESULT_PARTIAL)) { + uint32 expected = slot->reply_header.checksum; + uint32 got = gcs_block_compute_checksum(slot->reply_block_data); + + if (expected == got) { + memcpy(dst_page, slot->reply_block_data, BLCKSZ); + /* spec-5.14 D2 class 2: this CR result is the origin's + * volatile construction — depend on it for fail-stop. */ + gcs_block_stamp_touched(origin_node, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + if (out_partial != NULL) + *out_partial + = (slot->reply_header.status == (uint8)GCS_BLOCK_REPLY_CR_RESULT_PARTIAL); + fetched = true; + } else { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_checksum_fail_count, 1); + } + } + } + PG_CATCH(); + { + gcs_block_release_slot(slot); + PG_RE_THROW(); + } + PG_END_TRY(); + + gcs_block_release_slot(slot); + + return fetched; /* false -> caller keeps the unchanged 53R9G refusal */ +} + + +/* + * PGRAC: spec-6.12i D-i1 — requester-side undo-TT fetch. + * + * Ask origin_node for its own TT-bearing undo header block (segment_id, + * block_no) plus the co-sampled live authority triple, riding the same * sub-case B wire shape as the spec-6.12b CR fetch (FORWARD payload direct * to the serving node; HC108 chain on the direct-shipped reply validates * forwarding_master == self). The tag is the SYNTHETIC undo address; the @@ -4096,10 +4966,10 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) hdrv2->sf_flags = entry->sf_flags; for (i = 0; i < CLUSTER_SF_DEP_MAX_ORIGINS; i++) { - if (XLogRecPtrIsInvalid(entry->sf_dep_vec.required[i])) + if (XLogRecPtrIsInvalid(entry->payload_meta.sf_dep_vec.required[i])) continue; hdrv2->sf_dep[n].origin_node = i; - hdrv2->sf_dep[n].required_redo_lsn = (uint64)entry->sf_dep_vec.required[i]; + hdrv2->sf_dep[n].required_redo_lsn = (uint64)entry->payload_meta.sf_dep_vec.required[i]; n++; } hdrv2->sf_dep_count = (uint8)n; @@ -4132,6 +5002,106 @@ gcs_block_resend_cached_reply(int32 dest_node, const GcsBlockDedupEntry *entry) pfree(buf); } + +/* + * Serve the canonical PCM-X image-id request subdomain from the holder's + * dedicated dedup record. This branch deliberately precedes the generic + * request registration path: canonical image ids are reserved capabilities, + * not ordinary request ids, and generic dedup correctly rejects them. + * + * The request carries no new overlay byte. Its image_id names one immutable + * record; the live holder ledger supplies the full ref/image/master-session + * binding. Both the requester and the authoritative master are re-proved + * against the current peer frontier before the record is copied. A second + * holder snapshot closes the lookup-to-send window. No dedup pointer or pin + * escapes the lookup: cached is a by-value copy of the complete 8KB record. + */ +static bool +gcs_block_pcm_x_serve_image_fetch(const ClusterICEnvelope *env, const GcsBlockRequestPayload *req, + int worker_id) +{ + GcsBlockDedupEntry cached; + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageResult image_result; + PcmXLocalHolderProgress holder_after; + PcmXLocalHolderProgress holder_before; + PcmXQueueResult progress_result; + uint64 master_session; + uint64 requester_session; + uint64 current_epoch; + int32 encoded_master; + int32 tag_master; + + if (req == NULL || !cluster_pcm_x_image_id_decode(req->request_id, &encoded_master, NULL)) + return false; + /* From here onward the reserved namespace is always consumed here. */ + if (env == NULL || worker_id < 0 || worker_id >= cluster_lms_workers) + return true; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(req->tag); + if (encoded_master != tag_master || req->epoch != current_epoch + || !gcs_block_pcm_x_source_capable(req->sender_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(req->sender_node) + || !cluster_membership_is_member(tag_master) + || !gcs_block_pcm_x_authenticated_session(req->sender_node, current_epoch, + &requester_session) + || !gcs_block_pcm_x_revalidate_peer_binding(req->sender_node, current_epoch, + requester_session)) + return true; + + progress_result = cluster_pcm_x_local_holder_progress_exact(&req->tag, &holder_before); + if (progress_result != PCM_X_QUEUE_OK + || !cluster_pcm_x_image_fetch_request_exact(env, req, &holder_before, cluster_node_id, + tag_master, current_epoch) + || !gcs_block_pcm_x_authenticated_session(tag_master, current_epoch, &master_session) + || master_session != holder_before.master_session_incarnation + || !gcs_block_pcm_x_revalidate_peer_binding(tag_master, current_epoch, master_session)) + return true; + + memset(&key, 0, sizeof(key)); + key.origin_node_id = (uint32)req->sender_node; + key.requester_backend_id = req->requester_backend_id; + key.request_id = req->request_id; + key.cluster_epoch = req->epoch; + memset(&binding, 0, sizeof(binding)); + binding.identity.ref = holder_before.ref; + binding.identity.image = holder_before.image; + binding.master_session = holder_before.master_session_incarnation; + memset(&cached, 0, sizeof(cached)); + image_result + = cluster_gcs_block_dedup_pcm_x_lookup(worker_id, &key, &req->tag, &binding, &cached); + if (image_result == GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { + /* Materialization/publication is still progressing on this same shard. */ + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, + InvalidXLogRecPtr, NULL); + return true; + } + if (image_result != GCS_BLOCK_PCM_X_IMAGE_REPLAY) { + /* NOT_FOUND is an old/released handle and is safely retried by timeout. + * Any other result says live holder evidence and its dedicated record + * disagree, so retain both and close the runtime. */ + if (image_result != GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND) + cluster_pcm_x_runtime_fail_closed(); + return true; + } + + progress_result = cluster_pcm_x_local_holder_progress_exact(&req->tag, &holder_after); + if (progress_result != PCM_X_QUEUE_OK + || memcmp(&holder_before, &holder_after, sizeof(holder_before)) != 0 + || !cluster_pcm_x_image_fetch_request_exact(env, req, &holder_after, cluster_node_id, + tag_master, current_epoch)) + return true; + + gcs_block_resend_cached_reply(req->sender_node, &cached); + if (ClusterGcsBlock != NULL) { + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_reply_count, 1); + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_ship_bytes_total, GCS_BLOCK_DATA_SIZE); + } + return true; +} + /* * gcs_block_produce_reply — original (non-cached) master-side flow. * @@ -4422,6 +5392,12 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo } } + /* A canonical PCM-X image handle is served by the current holder, not by + * the generic resource-master decision tree. Intercept it before generic + * dedup, whose namespace guard intentionally rejects this id domain. */ + if (gcs_block_pcm_x_serve_image_fetch(env, req, dedup_worker_id)) + return; + /* PGRAC: spec-2.34 D5 — dedup lookup_or_register (HC90 + HC91 + HC92). */ memset(&key, 0, sizeof(key)); key.origin_node_id = (uint32)req->sender_node; @@ -4590,6 +5566,7 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo * subsequent N→S retry does not see a stale barrier. */ if (req->transition_id == PCM_TRANS_N_TO_X || req->transition_id == PCM_TRANS_S_TO_X_UPGRADE) { + PcmPendingXReserveResult reserve_result; PcmLockMode pre_state; uint64 current_lsn; int32 x_holder; @@ -4898,7 +5875,21 @@ cluster_gcs_handle_block_request_envelope(const ClusterICEnvelope *env, const vo } current_lsn = (uint64)GetXLogInsertRecPtr(); - cluster_pcm_lock_set_pending_x(req->tag, req->sender_node, current_lsn); + reserve_result = cluster_pcm_lock_set_pending_x(req->tag, req->sender_node, current_lsn); + if (reserve_result != PCM_PENDING_X_RESERVE_OK) { + /* Another exact round owns the starvation barrier. The dedup key + * for this request must be released so its bounded retry can be + * reconsidered after that owner completes. */ + if (reserve_result == PCM_PENDING_X_RESERVE_OCCUPIED) { + cluster_gcs_block_dedup_remove(dedup_worker_id, &key); + gcs_block_send_reply(req->sender_node, req, GCS_BLOCK_REPLY_DENIED_PENDING_X, + InvalidXLogRecPtr, NULL); + } else + gcs_block_send_reply(req->sender_node, req, + GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER, InvalidXLogRecPtr, + NULL); + return; + } pre_state = cluster_pcm_lock_query(req->tag); @@ -6249,6 +7240,12 @@ cluster_gcs_handle_block_reply_envelope(const ClusterICEnvelope *env, const void if (!gcs_block_decode_reply_payload(env, payload, &hdr, &block_data, &sf_dep_valid, &sf_flags, &sf_dep_vec, &undo_trailer)) return; + /* Bind the payload's claimed sender to the authenticated DATA envelope. + * The canonical PCM-X image-id path is direct holder -> requester, so + * accepting a forged header source here would bypass its holder proof. */ + if (env->source_node_id != (uint32)hdr->sender_node + || env->dest_node_id != (uint32)cluster_node_id) + return; /* HC80: direct index by requester_backend_id (1..MaxBackends → 0..MaxBackends-1). */ backend_idx = hdr->requester_backend_id - 1; @@ -6845,143 +7842,4388 @@ cluster_gcs_handle_block_forward_envelope(const ClusterICEnvelope *env, const vo ClusterBufmgrGcsDropResult dres = cluster_bufmgr_drop_block_for_gcs_no_wire( fwd->tag, page_lsn, &drop_lsn); - if (dres == CLUSTER_BUFMGR_GCS_DROP_PINNED - || dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { - if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) - pg_atomic_fetch_add_u64(&ClusterGcsBlock->xfer_stale_deny_count, 1); - else - pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, - 1); - /* undo the from-holder ship count taken with the - * grant status above — this reply is a deny now */ - pg_atomic_fetch_sub_u64(&ClusterGcsBlock->block_from_holder_ship_count, - 1); - hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; - } else { - /* PGRAC: spec-6.12h D-h3a — ordering pin: the PI - * conversion (inside the drop above) precedes the reply - * send at the bottom of this handler - * (cluster_ic_rdma_send_envelope_sge), so the requester - * observes an envelope stamped at-or-above the ship-SCN - * boundary (cluster_pi_shadow.h proof item 2). */ - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_x_transfer_ship_count, - 1); - if (GcsBlockForwardPayloadIsCleanEligible(fwd)) - pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_count, 1); - /* PGRAC: spec-6.12h D-h2 — if the D-h1 conversion kept our - * outgoing copy as a Past Image, report it to the master - * (unsolicited PI_KEPT ride; fire-and-forget — a lost note - * only leaves the PI untracked, fail-safe lingering). */ - if (cluster_bufmgr_block_is_pi(fwd->tag)) - gcs_block_pi_kept_note_send(fwd->tag, fwd->master_node); - } - } - } - } - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_ship_bytes_total, GCS_BLOCK_DATA_SIZE); - } - } else { - /* HC105 evict race */ - hdr->checksum = gcs_block_compute_checksum(buf + header_len); - hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; - pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_holder_evicted_count, 1); - } + if (dres == CLUSTER_BUFMGR_GCS_DROP_PINNED + || dres == CLUSTER_BUFMGR_GCS_DROP_STALE) { + if (dres == CLUSTER_BUFMGR_GCS_DROP_STALE) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->xfer_stale_deny_count, 1); + else + pg_atomic_fetch_add_u64(&ClusterGcsBlock->drop_pinned_deny_count, + 1); + /* undo the from-holder ship count taken with the + * grant status above — this reply is a deny now */ + pg_atomic_fetch_sub_u64(&ClusterGcsBlock->block_from_holder_ship_count, + 1); + hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; + } else { + /* PGRAC: spec-6.12h D-h3a — ordering pin: the PI + * conversion (inside the drop above) precedes the reply + * send at the bottom of this handler + * (cluster_ic_rdma_send_envelope_sge), so the requester + * observes an envelope stamped at-or-above the ship-SCN + * boundary (cluster_pi_shadow.h proof item 2). */ + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_x_transfer_ship_count, + 1); + if (GcsBlockForwardPayloadIsCleanEligible(fwd)) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->clean_page_xfer_count, 1); + /* PGRAC: spec-6.12h D-h2 — if the D-h1 conversion kept our + * outgoing copy as a Past Image, report it to the master + * (unsolicited PI_KEPT ride; fire-and-forget — a lost note + * only leaves the PI untracked, fail-safe lingering). */ + if (cluster_bufmgr_block_is_pi(fwd->tag)) + gcs_block_pi_kept_note_send(fwd->tag, fwd->master_node); + } + } + } + } + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_ship_bytes_total, GCS_BLOCK_DATA_SIZE); + } + } else { + /* HC105 evict race */ + hdr->checksum = gcs_block_compute_checksum(buf + header_len); + hdr->status = (uint8)GCS_BLOCK_REPLY_DENIED_MASTER_NOT_HOLDER; + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_forward_holder_evicted_count, 1); + } + + send_sf_dep = sf_peer_v2 && sf_dep_valid && block_payload != NULL + && (hdr->status == (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER + || hdr->status == (uint8)GCS_BLOCK_REPLY_X_GRANTED_FROM_HOLDER + || hdr->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER); + header_len + = send_sf_dep ? (uint32)sizeof(GcsBlockReplyHeaderV2) : (uint32)sizeof(GcsBlockReplyHeader); + total = header_len + GCS_BLOCK_DATA_SIZE; + if (send_sf_dep) { + GcsBlockReplyHeaderV2 *hdrv2 = (GcsBlockReplyHeaderV2 *)buf; + int i; + int n = 0; + + hdrv2->sf_flags = GCS_BLOCK_REPLY_SF_EARLY_TRANSFER | GCS_BLOCK_REPLY_SF_HAS_DEP_VEC; + for (i = 0; i < CLUSTER_SF_DEP_MAX_ORIGINS; i++) { + if (XLogRecPtrIsInvalid(sf_dep_vec.required[i])) + continue; + hdrv2->sf_dep[n].origin_node = i; + hdrv2->sf_dep[n].required_redo_lsn = (uint64)sf_dep_vec.required[i]; + n++; + } + hdrv2->sf_dep_count = (uint8)n; + } + + if (GcsBlockForwardPayloadIsDirectLandArmed(fwd)) { + (void)gcs_block_try_send_direct_reply(fwd->original_requester_node, true, hdr, + holder_ship_ok ? block_payload : NULL, + holder_ship_ok ? block_payload_lkey : 0, + block_payload_release_cb, block_payload_release_arg); + if (hdr->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER) + cluster_xp_end(&xp_fwd_ship); + block_payload_release_cb = NULL; + block_payload_release_arg = NULL; + pfree(buf); + return; + } + + if (holder_ship_ok && block_payload != NULL + && (hdr->status == (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER + || hdr->status == (uint8)GCS_BLOCK_REPLY_X_GRANTED_FROM_HOLDER + || hdr->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + /* PGRAC: spec-6.12a ㉕ — the downgraded durable S grant ships page + * bytes exactly like the statuses above; leaving it off this list + * would send the palloc0 zero pad under a real-page checksum + * (guaranteed verify failure at the requester). */ + || hdr->status == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { + ClusterICSge sge[2]; + ClusterICSendResult send_rc; + bool live_sge_payload = block_payload_release_cb == gcs_block_release_live_sge; + + memset(sge, 0, sizeof(sge)); + sge[0].addr = hdr; + sge[0].len = header_len; + sge[1].addr = (void *)block_payload; + sge[1].len = GCS_BLOCK_DATA_SIZE; + sge[1].lkey = block_payload_lkey; + sge[1].release_cb = block_payload_release_cb; + sge[1].release_arg = block_payload_release_arg; + send_rc = cluster_ic_rdma_send_envelope_sge( + PGRAC_IC_MSG_GCS_BLOCK_REPLY, fwd->original_requester_node, sge, lengthof(sge), total); + cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, send_rc); + if (send_rc == CLUSTER_IC_SEND_DONE && live_sge_payload) + gcs_block_note_live_sge_send(); + block_payload_release_cb = NULL; + block_payload_release_arg = NULL; + } else { + gcs_block_release_ship_image(block_payload_release_cb, block_payload_release_arg); + block_payload_release_cb = NULL; + block_payload_release_arg = NULL; + cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, + cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_REPLY, + fwd->original_requester_node, + buf, total)); + } + /* PGRAC: spec-5.59 D3 — close the holder-forward read-image ship scope + * (no-op unless the read-image branch above started it). */ + cluster_xp_end(&xp_fwd_ship); + + pfree(buf); +} + + +/* ============================================================ + * Dispatch table registration. + * + * PGRAC: spec-7.2 flip — the five block-family msg_types (REQUEST / + * REPLY / FORWARD / INVALIDATE / INVALIDATE_ACK) are registered on + * the DATA plane: the LMS-owned tier1 instance carries their frames, + * the LMS loop dispatches them, and the producer mask admits the LMS + * family for the drain-and-send leg. All five flip in this one edit + * (H-5: no half-migrated window; the registry probe above pivots the + * LMON tick sites and the LMS loop automatically). REDECLARE alone + * stays on the CONTROL plane (r4): recovery re-declare must survive a + * DATA-mesh teardown mid-episode, and the REDECLARE -> REDECLARE_DONE + * pair may not be split across planes. + * ============================================================ */ + +/* Resolve the boot session used by the PCM-X frontier. A remote session is + * fresh-alive QVOTEC authority, sampled twice inside one exact capability + * record generation (CONTROL-owned on the tier1 S3 path). DATA source is + * already bound by envelope verify. + * This deliberately does not use membership.last_admitted: that value is a + * historical anti-rejoin floor and can remain zero for initial members. */ +static PcmXSessionAuthResult +gcs_block_pcm_x_authenticated_session_result(int32 node_id, uint64 expected_epoch, + uint64 *session_out, + ClusterGcsPcmXAuthSample *sample_out) +{ + ClusterGcsPcmXAuthSample sample; + PcmXSessionAuthResult result; + uint64 session; + + memset(&sample, 0, sizeof(sample)); + if (session_out != NULL) + *session_out = 0; + if (sample_out != NULL) + memset(sample_out, 0, sizeof(*sample_out)); + if (node_id < 0 || node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_SESSION_AUTH_INVALID; + if (node_id == cluster_node_id) { + session = cluster_qvotec_get_self_incarnation(); + if (session == 0) + return PCM_X_SESSION_AUTH_SLOT_NOT_READY; + if (session_out != NULL) + *session_out = session; + return PCM_X_SESSION_AUTH_OK; + } + + sample.connection_before_valid = cluster_sf_peer_pcm_x_connection_generation( + node_id, &sample.connection_generation_before); + sample.slot_before_valid = cluster_reconfig_get_observed_slot(node_id, &sample.session_before, + &sample.slot_generation_before); + sample.observed_epoch_before = cluster_reconfig_get_observed_epoch(node_id); + sample.fresh_before = cluster_reconfig_get_observed_fresh_alive(node_id); + sample.slot_after_valid = cluster_reconfig_get_observed_slot(node_id, &sample.session_after, + &sample.slot_generation_after); + sample.observed_epoch_after = cluster_reconfig_get_observed_epoch(node_id); + sample.fresh_after = cluster_reconfig_get_observed_fresh_alive(node_id); + sample.connection_after_valid + = cluster_sf_peer_pcm_x_connection_generation(node_id, &sample.connection_generation_after); + result = cluster_gcs_pcm_x_auth_sample_classify(&sample, expected_epoch); + if (sample_out != NULL) + *sample_out = sample; + if (result != PCM_X_SESSION_AUTH_OK) + return result; + if (session_out != NULL) + *session_out = sample.session_before; + return PCM_X_SESSION_AUTH_OK; +} + + +static bool +gcs_block_pcm_x_authenticated_session(int32 node_id, uint64 expected_epoch, uint64 *session_out) +{ + return gcs_block_pcm_x_authenticated_session_result(node_id, expected_epoch, session_out, NULL) + == PCM_X_SESSION_AUTH_OK; +} + + +typedef struct GcsBlockPcmXRequesterCleanupContext { + bool active; + bool wait_published; + bool handle_live; + bool claim_live; + bool wfg_live; + bool cutoff_started; + bool reservation_started; + bool ownership_committed; + bool fail_closed_required; + uint64 wfg_generation; + bool *claim_handoff_out; + PcmXLocalHandle handle; + PcmXLocalWriterClaim claim; +} GcsBlockPcmXRequesterCleanupContext; + +static GcsBlockPcmXRequesterCleanupContext gcs_block_pcm_x_requester_cleanup_context; +static bool gcs_block_pcm_x_requester_exit_hook_registered = false; + +static bool gcs_block_pcm_x_revalidate_peer_binding(int32 node_id, uint64 epoch, uint64 session); + +static void +gcs_block_pcm_x_requester_wait(uint32 *wait_index) +{ + uint32 current; + + if (wait_index == NULL) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM-X requester wait has no backoff state"))); + current = *wait_index; + CHECK_FOR_INTERRUPTS(); + (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, + cluster_pcm_x_holder_retry_delay_ms(current), + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); + ResetLatch(MyLatch); + *wait_index = cluster_gcs_pcm_x_requester_wait_index_advance(current); + CHECK_FOR_INTERRUPTS(); +} + +/* Retry only inside the exact formation that admitted this requester. Both + * sides of the sleep are checked: a runtime fence or peer-session change that + * lands while the latch is asleep must be observed before the next protocol + * read or mutation. */ +static bool +gcs_block_pcm_x_requester_wait_exact(uint32 *wait_index, const PcmXRuntimeSnapshot *request_runtime, + int32 master_node, uint64 cluster_epoch, uint64 master_session) +{ + if (!gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, + master_session)) + return false; + gcs_block_pcm_x_requester_wait(wait_index); + return gcs_block_pcm_x_requester_authority_exact(request_runtime, master_node, cluster_epoch, + master_session); +} + +static void +gcs_block_pcm_x_requester_clear_wait(GcsBlockPcmXRequesterCleanupContext *cleanup) +{ + if (cleanup == NULL || !cleanup->wait_published) + return; + if (MyProc != NULL) + cluster_lmd_wait_state_clear(&MyProc->cluster_lmd_wait); + cleanup->wait_published = false; +} + +static PcmXQueueResult +gcs_block_pcm_x_requester_cleanup_impl(GcsBlockPcmXRequesterCleanupContext *cleanup, + bool owner_exit) +{ + GcsBlockPcmXCleanupAction action; + PcmXLocalHandle promoted; + PcmXQueueResult result; + PcmXRuntimeSnapshot runtime; + + if (cleanup == NULL || !cleanup->active) + return PCM_X_QUEUE_OK; + if (cleanup->claim_handoff_out != NULL && *cleanup->claim_handoff_out) { + /* bufmgr published the exact claim in its owner-exit ledger before + * requester authority was cleared. From that instant it is the sole + * cleanup owner, including an exit in this short handoff window. */ + pg_read_barrier(); + memset(cleanup, 0, sizeof(*cleanup)); + return PCM_X_QUEUE_OK; + } + if (cleanup->wfg_live) { + ClusterLmdGraphRemoveResult remove_result; + ClusterLmdVertex waiter; + + cluster_gcs_pcm_x_vertex_from_identity(&cleanup->handle.identity, &waiter); + remove_result = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter); + if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + cluster_pcm_x_runtime_fail_closed(); + gcs_block_pcm_x_requester_clear_wait(cleanup); + return PCM_X_QUEUE_CORRUPT; + } + if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) + cluster_lmd_pcm_convert_wfg_note_remove(); + result = cluster_pcm_x_local_follower_wfg_clear_exact(&cleanup->handle, + cleanup->wfg_generation); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + runtime = cluster_pcm_x_runtime_snapshot(); + if (owner_exit + && cluster_pcm_x_owner_exit_action(result, false, + runtime.state == PCM_X_RUNTIME_ACTIVE + && runtime.master_session_incarnation != 0) + == CLUSTER_PCM_X_OWNER_EXIT_RETRY) { + gcs_block_pcm_x_requester_clear_wait(cleanup); + return result; + } + cluster_pcm_x_runtime_fail_closed(); + gcs_block_pcm_x_requester_clear_wait(cleanup); + return result; + } + cleanup->wfg_live = false; + cleanup->wfg_generation = 0; + } + gcs_block_pcm_x_requester_clear_wait(cleanup); + action = cluster_gcs_pcm_x_requester_cleanup_action( + cleanup->handle_live, cleanup->claim_live, + cleanup->cutoff_started || cleanup->reservation_started, cleanup->ownership_committed); + /* A claimed writer is already the active member of a closed FIFO cohort. + * Every exit path must record WRITER_COMPLETE before either cancelling a + * still-reversible membership or preserving irreversible evidence behind the + * recovery gate. abort_exact would clear active_writer without COMPLETE and + * permanently strand every successor behind this backend. */ + if (cleanup->claim_live) { + result = cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(&cleanup->claim); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK) { + runtime = cluster_pcm_x_runtime_snapshot(); + if (owner_exit + && cluster_pcm_x_owner_exit_action(result, false, + runtime.state == PCM_X_RUNTIME_ACTIVE + && runtime.master_session_incarnation != 0) + == CLUSTER_PCM_X_OWNER_EXIT_RETRY) + return result; + cluster_pcm_x_runtime_fail_closed(); + return result; + } + cleanup->claim_live = false; + } + if (action == GCS_BLOCK_PCM_X_CLEANUP_PRESERVE_FAIL_CLOSED) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_NOT_READY; + } + if (action == GCS_BLOCK_PCM_X_CLEANUP_CANCEL_LOCAL) { + memset(&promoted, 0, sizeof(promoted)); + result = cluster_pcm_x_local_cancel_exact(&cleanup->handle, &promoted); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + result = cluster_pcm_x_local_detach_terminal_exact(&cleanup->handle); + cluster_pcm_x_stats_note_queue_result(result); + } + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_NOT_FOUND) { + runtime = cluster_pcm_x_runtime_snapshot(); + if (owner_exit + && cluster_pcm_x_owner_exit_action(result, false, + runtime.state == PCM_X_RUNTIME_ACTIVE + && runtime.master_session_incarnation != 0) + == CLUSTER_PCM_X_OWNER_EXIT_RETRY) + return result; + cluster_pcm_x_runtime_fail_closed(); + return result; + } + } + memset(cleanup, 0, sizeof(*cleanup)); + return PCM_X_QUEUE_OK; +} + +static PcmXQueueResult +gcs_block_pcm_x_requester_cleanup_attempt_noexcept(GcsBlockPcmXRequesterCleanupContext *cleanup, + bool owner_exit) +{ + volatile PcmXQueueResult result = PCM_X_QUEUE_CORRUPT; + + PG_TRY(); + { + result = gcs_block_pcm_x_requester_cleanup_impl(cleanup, owner_exit); + } + PG_CATCH(); + { + FlushErrorState(); + cluster_pcm_x_runtime_fail_closed(); + gcs_block_pcm_x_requester_clear_wait(cleanup); + result = PCM_X_QUEUE_CORRUPT; + } + PG_END_TRY(); + return (PcmXQueueResult)result; +} + +static void +gcs_block_pcm_x_requester_cleanup_noexcept(GcsBlockPcmXRequesterCleanupContext *cleanup) +{ + /* Ordinary ERROR cleanup stays non-waiting. If one exact attempt cannot + * close the claim/handle, cleanup_impl preserves shared evidence and closes + * the runtime; there is no safe reason to stall an error stack indefinitely. + * Backend death uses the dedicated retry loop below. */ + (void)gcs_block_pcm_x_requester_cleanup_attempt_noexcept(cleanup, false); +} + +static void +gcs_block_pcm_x_requester_owner_exit(int code pg_attribute_unused(), + Datum arg pg_attribute_unused()) +{ + GcsBlockPcmXRequesterCleanupContext *cleanup = &gcs_block_pcm_x_requester_cleanup_context; + + while (cleanup->active) { + PcmXQueueResult result; + PcmXRuntimeSnapshot runtime; + + result = gcs_block_pcm_x_requester_cleanup_attempt_noexcept(cleanup, true); + if (!cleanup->active || result == PCM_X_QUEUE_OK) + return; + runtime = cluster_pcm_x_runtime_snapshot(); + if (cluster_pcm_x_owner_exit_action(result, false, + runtime.state == PCM_X_RUNTIME_ACTIVE + && runtime.master_session_incarnation != 0) + != CLUSTER_PCM_X_OWNER_EXIT_RETRY) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + pg_usleep(1000L); + } +} + + +/* Bind an idle node leader to one stable ownership tuple before its first + * ENQUEUE. A predecessor round may have advanced ownership while this member + * waited as a follower. The queue key moves under its admission gate, then a + * second snapshot proves that no ownership lifecycle crossed that move. */ +static PcmXQueueResult +gcs_block_pcm_x_rekey_leader_base_exact(BufferDesc *buf, PcmXLocalHandle *leader, + ClusterPcmOwnSnapshot *base_out) +{ + ClusterPcmOwnSnapshot before; + ClusterPcmOwnSnapshot after; + PcmXLocalHandle rekeyed; + ClusterPcmOwnResult own_result; + ClusterPcmOwnResult live_result; + PcmXQueueResult result; + + if (base_out != NULL) + memset(base_out, 0, sizeof(*base_out)); + if (buf == NULL || leader == NULL || base_out == NULL + || leader->role != PCM_X_LOCAL_ROLE_NODE_LEADER) + return PCM_X_QUEUE_INVALID; + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &before); + result = gcs_block_pcm_x_fetch_own_result(own_result); + if (result != PCM_X_QUEUE_OK) + return result; + if (!BufferTagsEqual(&before.tag, &leader->identity.tag) + || (before.pcm_state != (uint8)PCM_STATE_N && before.pcm_state != (uint8)PCM_STATE_S + && before.pcm_state != (uint8)PCM_STATE_X)) + return PCM_X_QUEUE_STALE; + if (before.generation == UINT64_MAX) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + live_result = cluster_pcm_own_classify_live_flags(before.flags, before.reservation_token); + result = gcs_block_pcm_x_fetch_own_result(live_result); + if (result != PCM_X_QUEUE_OK) + return result; + + result = cluster_pcm_x_local_leader_rekey_generation_exact(leader, before.generation, &rekeyed); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + *leader = rekeyed; + + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &after); + result = gcs_block_pcm_x_fetch_own_result(own_result); + if (result != PCM_X_QUEUE_OK) + return result; + if (!BufferTagsEqual(&after.tag, &before.tag) || after.generation < before.generation) + return PCM_X_QUEUE_STALE; + live_result = cluster_pcm_own_classify_live_flags(after.flags, after.reservation_token); + result = gcs_block_pcm_x_fetch_own_result(live_result); + if (result != PCM_X_QUEUE_OK) + return result; + if (after.generation != before.generation || after.reservation_token != before.reservation_token + || after.flags != before.flags || after.pcm_state != before.pcm_state) + return PCM_X_QUEUE_BUSY; + *base_out = after; + return PCM_X_QUEUE_OK; +} + + +/* + * Drive one ordinary BM_VALID N/S/X -> X request through the node FIFO. + * + * The returned writer claim is deliberately still live. It is the local + * execution right, not merely an admission receipt: bufmgr keeps it across + * the content LWLock and records WRITER_COMPLETE only after UNLOCK. A node + * leader owns the one remote protocol; followers publish an exact local WFG + * edge and consume the already-granted node X in FIFO order. + */ +static PcmXQueueResult +gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim_out) +{ + PcmXLocalFollowerWfgSnapshot follower_snapshot; + PcmXLocalHandle handle; + PcmXLocalHandle fresh_handle; + PcmXLocalCutoff cutoff; + PcmXLocalProgress progress; + PcmXRuntimeSnapshot request_runtime; + ClusterLmdVertex blocker_vertex; + ClusterLmdVertex waiter_vertex; + ClusterPcmOwnSnapshot initial_own; + ClusterPcmOwnSnapshot reservation_base; + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult own_result; + GcsBlockPcmXRetryAction retry_action; + PcmXQueueResult arm_result = PCM_X_QUEUE_NOT_READY; + PcmXQueueResult result; + TransactionId xid; + uint64 cluster_epoch; + uint64 committed_generation = 0; + uint64 follower_graph_generation = 0; + uint64 master_session = 0; + uint64 request_id = 0; + uint64 reservation_token = 0; + uint64 wait_seq = 0; + uint32 wait_index = 0; + int32 master_node; + const char *fail_site = "entry"; + bool image_installed = false; + bool ownership_committed = false; + bool reservation_started = false; + bool self_source = false; + bool wait_published = false; + + if (claim_out != NULL) + memset(claim_out, 0, sizeof(*claim_out)); + if (buf == NULL || claim_out == NULL || ClusterPcmXConvertShmem == NULL || MyProc == NULL + || MyProc->pgprocno < 0 || MyBackendId <= 0 || cluster_node_id < 0 + || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_QUEUE_INVALID; + for (;;) { + GcsBlockPcmXFormationAction formation_action; + + request_runtime = cluster_pcm_x_runtime_snapshot(); + formation_action = cluster_gcs_pcm_x_requester_formation_action(&request_runtime); + if (formation_action == GCS_BLOCK_PCM_X_FORMATION_PROCEED) + break; + if (formation_action != GCS_BLOCK_PCM_X_FORMATION_WAIT) + return PCM_X_QUEUE_NOT_READY; + gcs_block_pcm_x_requester_wait(&wait_index); + } + cluster_epoch = cluster_epoch_get_current(); + master_node = cluster_gcs_lookup_master(buf->tag); + if (master_node < 0 || master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || !gcs_block_pcm_x_authenticated_session(master_node, cluster_epoch, &master_session) + || !gcs_block_pcm_x_revalidate_peer_binding(master_node, cluster_epoch, master_session)) + return PCM_X_QUEUE_NOT_READY; + /* The authenticated session above is a current-view proof. Before the + * first wait identity becomes visible, bind it back to the exact formation + * snapshot that admitted this call so a close/rejoin window cannot publish + * or enqueue a mixed-generation requester. */ + if (!gcs_block_pcm_x_requester_authority_exact(&request_runtime, master_node, cluster_epoch, + master_session)) + return PCM_X_QUEUE_NOT_READY; + if (!gcs_block_pcm_x_next_request_id(&request_id)) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + } + xid = GetTopTransactionIdIfAny(); + wait_seq = cluster_lmd_wait_state_publish( + &MyProc->cluster_lmd_wait, CLUSTER_LMD_WAIT_PCM_CONVERT, request_id, cluster_epoch, xid); + if (wait_seq == 0) { + cluster_pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto requester_done; + } + wait_published = true; + gcs_block_pcm_x_requester_cleanup_context.wait_published = true; + + /* Publish first, then close the freeze-before-wait race for every content + * lock already held by this backend. If a holder barrier is already frozen + * we must not block: that snapshot cannot retroactively acquire this nested + * edge. If it freezes later, the published PCM_CONVERT identity is visible + * to the normal blocker snapshot/WFG path. */ + for (;;) { + result = cluster_pcm_x_nested_wait_guard_before_block(); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) + break; + if (result == PCM_X_QUEUE_CORRUPT) + goto requester_fail_closed; + if (!cluster_gcs_pcm_x_nested_guard_retryable(result)) + goto requester_done; + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + + /* Ownership reservations and READ_IMAGE are protocol contention too. Keep + * their wait inside the published PCM_CONVERT interval; bufmgr must never + * sleep on these shapes before the nested-wait guard. */ + for (;;) { + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &initial_own); + result = gcs_block_pcm_x_fetch_own_result(own_result); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + goto requester_fail_closed; + goto requester_done; + } + if (!BufferTagsEqual(&initial_own.tag, &buf->tag) || initial_own.generation == UINT64_MAX) { + result = PCM_X_QUEUE_STALE; + goto requester_done; + } + live_result + = cluster_pcm_own_classify_live_flags(initial_own.flags, initial_own.reservation_token); + result = gcs_block_pcm_x_fetch_own_result(live_result); + if (result == PCM_X_QUEUE_CORRUPT) + goto requester_fail_closed; + if (initial_own.pcm_state == (uint8)PCM_STATE_READ_IMAGE || result == PCM_X_QUEUE_BUSY) { + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + continue; + } + if (result != PCM_X_QUEUE_OK) + goto requester_done; + if (initial_own.pcm_state != (uint8)PCM_STATE_N + && initial_own.pcm_state != (uint8)PCM_STATE_S + && initial_own.pcm_state != (uint8)PCM_STATE_X) { + result = PCM_X_QUEUE_STALE; + goto requester_done; + } + break; + } + memset(&handle, 0, sizeof(handle)); + handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + for (;;) { + PcmXWaitIdentity identity; + + memset(&identity, 0, sizeof(identity)); + identity.tag = buf->tag; + identity.node_id = cluster_node_id; + identity.procno = (uint32)MyProc->pgprocno; + identity.xid = xid; + identity.cluster_epoch = cluster_epoch; + identity.request_id = request_id; + identity.wait_seq = wait_seq; + identity.base_own_generation = initial_own.generation; + fail_site = "local-join"; + result = cluster_pcm_x_local_join_begin(&identity, master_node, master_session, &handle); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + break; + retry_action + = cluster_gcs_pcm_x_requester_retry_action(GCS_BLOCK_PCM_X_RETRY_SITE_JOIN, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + gcs_block_pcm_x_requester_cleanup_context.handle = handle; + gcs_block_pcm_x_requester_cleanup_context.handle_live = true; + + +requester_role_dispatch: + /* The leader claims before ENQUEUE, so every later local writer has one + * concrete FIFO/WFG blocker while the remote grant is in flight. */ + if (handle.role == PCM_X_LOCAL_ROLE_NODE_LEADER) { + for (;;) { + fail_site = "leader-rekey"; + result = gcs_block_pcm_x_rekey_leader_base_exact(buf, &handle, &initial_own); + gcs_block_pcm_x_requester_cleanup_context.handle = handle; + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) + break; + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + for (;;) { + fail_site = "leader-claim"; + result = cluster_pcm_x_local_writer_claim_exact(&handle, claim_out); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) { + gcs_block_pcm_x_requester_cleanup_context.claim = *claim_out; + gcs_block_pcm_x_requester_cleanup_context.claim_live = true; + break; + } + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + for (;;) { + /* Treat the call window as irreversible. The freeze helper may have + * closed the cohort before an injected ERROR reaches this frame. */ + gcs_block_pcm_x_requester_cleanup_context.cutoff_started = true; + fail_site = "leader-cutoff"; + result = cluster_pcm_x_local_begin_revoke_cutoff_exact(&handle, &cutoff); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + break; + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + gcs_block_pcm_x_requester_cleanup_context.cutoff_started = false; + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + } else if (handle.role == PCM_X_LOCAL_ROLE_FOLLOWER) { + for (;;) { + ClusterLmdGraphRemoveResult remove_result; + + ResetLatch(MyLatch); + if (follower_graph_generation != 0) { + cluster_gcs_pcm_x_vertex_from_identity(&handle.identity, &waiter_vertex); + remove_result + = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter_vertex); + if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + goto requester_fail_closed; + } + if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) + cluster_lmd_pcm_convert_wfg_note_remove(); + result = cluster_pcm_x_local_follower_wfg_clear_exact(&handle, + follower_graph_generation); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + goto requester_fail_closed; + follower_graph_generation = 0; + gcs_block_pcm_x_requester_cleanup_context.wfg_live = false; + gcs_block_pcm_x_requester_cleanup_context.wfg_generation = 0; + } + fail_site = "follower-claim"; + result = cluster_pcm_x_local_writer_claim_exact(&handle, claim_out); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) { + gcs_block_pcm_x_requester_cleanup_context.claim = *claim_out; + gcs_block_pcm_x_requester_cleanup_context.claim_live = true; + break; + } + if (result == PCM_X_QUEUE_STALE) { + fail_site = "follower-refresh-lookup"; + result = cluster_pcm_x_local_lookup_exact(&handle.identity, &fresh_handle); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) { + fail_site = "follower-refresh-compare"; + if (!cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)) + goto requester_fail_closed; + handle = fresh_handle; + gcs_block_pcm_x_requester_cleanup_context.handle = handle; + goto requester_role_dispatch; + } + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, result); + if (retry_action == GCS_BLOCK_PCM_X_RETRY_WAIT) { + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, + master_node, cluster_epoch, + master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + continue; + } + goto requester_fail_closed; + } + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + + fail_site = "follower-wfg-snapshot"; + result = cluster_pcm_x_local_follower_wfg_snapshot_exact(&handle, &follower_snapshot); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) { + cluster_gcs_pcm_x_vertex_from_identity(&follower_snapshot.waiter.identity, + &waiter_vertex); + cluster_gcs_pcm_x_vertex_from_identity(&follower_snapshot.blocker, &blocker_vertex); + follower_graph_generation = cluster_lmd_graph_replace_waiter_edges_exact( + &waiter_vertex, &blocker_vertex, 1, handle.identity.request_id); + if (follower_graph_generation == 0) { + cluster_lmd_pcm_convert_wfg_note_replace_fail(); + goto requester_fail_closed; + } + cluster_lmd_pcm_convert_wfg_note_replace(); + gcs_block_pcm_x_requester_cleanup_context.wfg_live = true; + gcs_block_pcm_x_requester_cleanup_context.wfg_generation + = follower_graph_generation; + result = cluster_pcm_x_local_follower_wfg_commit_exact(&follower_snapshot, + follower_graph_generation); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_WFG_COMMIT, result); + remove_result + = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter_vertex); + if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + goto requester_fail_closed; + } + if (remove_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) + cluster_lmd_pcm_convert_wfg_note_remove(); + follower_graph_generation = 0; + gcs_block_pcm_x_requester_cleanup_context.wfg_live = false; + gcs_block_pcm_x_requester_cleanup_context.wfg_generation = 0; + if (retry_action != GCS_BLOCK_PCM_X_RETRY_RESNAPSHOT_WFG) + goto requester_fail_closed; + } + } else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_FOLLOWER_SNAPSHOT, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + } + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + result = PCM_X_QUEUE_OK; + goto requester_done; + } else { + result = PCM_X_QUEUE_CORRUPT; + goto requester_fail_closed; + } + + /* Node leader: every iteration reconstructs the exact durable outbound + * leg. A full DATA ring is backpressure, never a client-visible timeout. */ + for (;;) { + bool staged = false; + + ResetLatch(MyLatch); + fail_site = "leader-progress"; + result = cluster_pcm_x_local_progress_exact(&handle, &progress); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK) { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + goto requester_wait; + } + if (progress.role != PCM_X_LOCAL_ROLE_NODE_LEADER || progress.master_node != master_node + || progress.master_session_incarnation != master_session) + goto requester_fail_closed; + + if (progress.member_state == PCM_XL_NODE_LEADER) { + if ((progress.pending_opcode == 0 && progress.last_response_opcode == 0) + || progress.pending_opcode == PGRAC_IC_MSG_PCM_X_ENQUEUE) { + PcmXEnqueuePayload enqueue; + PcmXLocalReliableToken token; + + arm_result = cluster_pcm_x_local_enqueue_arm_exact(&handle, &enqueue, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + result = arm_result; + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_ENQUEUE, master_node, + &enqueue, sizeof(enqueue)); + else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT + && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) + goto requester_fail_closed; + } + } else if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM + || (progress.pending_opcode == 0 + && progress.last_response_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_ACK)) { + PcmXPhasePayload confirm; + PcmXLocalReliableToken token; + + arm_result = cluster_pcm_x_local_admit_confirm_arm_exact(&handle, &confirm, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + result = arm_result; + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, + master_node, &confirm, sizeof(confirm)); + else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT + && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) + goto requester_fail_closed; + } + } + } else if (progress.member_state == PCM_XL_REMOTE_WAIT) { + if (progress.pending_opcode == 0 + && progress.last_response_opcode == PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) { + PcmXInstallReadyPayload install_ready; + PcmXLocalReliableToken token; + + if (!reservation_started) { + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &reservation_base); + result = gcs_block_pcm_x_fetch_own_result(own_result); + if (result != PCM_X_QUEUE_OK) + goto requester_fail_closed; + self_source = progress.image.source_node == (uint32)cluster_node_id; + if (self_source) + result = cluster_gcs_pcm_x_adopt_self_image(buf, &handle, &reservation_base, + &reservation_token); + else { + result = cluster_gcs_pcm_x_remote_reservation_preflight(&reservation_base, + &progress.identity); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK) + goto requester_fail_closed; + own_result = cluster_bufmgr_pcm_own_begin_x_reservation( + buf, &reservation_base, &reservation_token); + result = gcs_block_pcm_x_fetch_own_result(own_result); + } + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + goto requester_fail_closed; + reservation_started = true; + gcs_block_pcm_x_requester_cleanup_context.reservation_started = true; + } + if (!image_installed) { + if (self_source) + result = PCM_X_QUEUE_OK; + else + result = cluster_gcs_pcm_x_fetch_image_and_install( + buf, &handle, &reservation_base, reservation_token, &request_runtime); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK) + image_installed = true; + else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT) + goto requester_fail_closed; + } + } + if (image_installed) { + arm_result = cluster_pcm_x_local_install_ready_arm_exact( + &handle, &progress.ref, &progress.image, &install_ready, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + result = arm_result; + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_INSTALL_READY, + master_node, &install_ready, + sizeof(install_ready)); + else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT + && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) + goto requester_fail_closed; + } + } + } + } else if (progress.member_state == PCM_XL_CONTENT_ACTIVE) { + if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY) { + PcmXInstallReadyPayload install_ready; + PcmXLocalReliableToken token; + + arm_result = cluster_pcm_x_local_install_ready_arm_exact( + &handle, &progress.ref, &progress.image, &install_ready, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + result = arm_result; + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + staged = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_INSTALL_READY, + master_node, &install_ready, + sizeof(install_ready)); + else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, arm_result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_WAIT + && retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) + goto requester_fail_closed; + } + } else if (progress.pending_opcode == 0 + && progress.last_response_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) { + PcmXFinalAckPayload final_ack; + PcmXLocalReliableToken token; + + if (!ownership_committed) { + if (self_source) + result = cluster_gcs_pcm_x_finish_self_image_x( + buf, &handle, &reservation_base, reservation_token, + &committed_generation); + else { + own_result = cluster_bufmgr_pcm_own_finish_x_commit( + buf, &reservation_base, reservation_token, &committed_generation); + result = gcs_block_pcm_x_fetch_own_result(own_result); + } + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK) + goto requester_fail_closed; + ownership_committed = true; + gcs_block_pcm_x_requester_cleanup_context.ownership_committed = true; + } + arm_result = cluster_pcm_x_local_final_ack_arm_exact(&handle, committed_generation, + &final_ack, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + result = arm_result; + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + staged = cluster_gcs_pcm_x_stage_frame( + PGRAC_IC_MSG_PCM_X_FINAL_ACK, master_node, &final_ack, sizeof(final_ack)); + else + goto requester_fail_closed; + } else if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + PcmXFinalAckPayload final_ack; + PcmXLocalReliableToken token; + + arm_result = cluster_pcm_x_local_final_ack_arm_exact(&handle, committed_generation, + &final_ack, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + result = arm_result; + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + staged = cluster_gcs_pcm_x_stage_frame( + PGRAC_IC_MSG_PCM_X_FINAL_ACK, master_node, &final_ack, sizeof(final_ack)); + else { + retry_action = cluster_gcs_pcm_x_requester_retry_action( + GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, arm_result); + if (retry_action != GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS) + goto requester_fail_closed; + } + } + } else if (progress.member_state == PCM_XL_GRANTED) { + if (progress.pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM) { + PcmXPhasePayload final_confirm; + + memset(&final_confirm, 0, sizeof(final_confirm)); + final_confirm.ref = progress.ref; + final_confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + staged + = cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, master_node, + &final_confirm, sizeof(final_confirm)); + if (staged) { + result = PCM_X_QUEUE_OK; + break; + } + } else if (progress.pending_opcode == 0) { + result = PCM_X_QUEUE_OK; + break; + } else + goto requester_fail_closed; + } else + goto requester_fail_closed; + + requester_wait: + if (!gcs_block_pcm_x_requester_wait_exact(&wait_index, &request_runtime, master_node, + cluster_epoch, master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto requester_done; + } + } + goto requester_done; + +requester_fail_closed: + /* Cleanup may cancel a purely local membership. PREPARE/COMMIT evidence + * is never guessed away; the wrapper preserves it behind the recovery + * gate after clearing any exact external WFG edge. */ + gcs_block_pcm_x_requester_cleanup_context.fail_closed_required = true; + ereport(LOG, + (errmsg("PCM-X requester entered fail-closed at %s", fail_site), + errdetail("result=%d role=%u state=%u pending=%u last=%u request=%llu wait_seq=%llu", + (int)result, (unsigned int)handle.role, (unsigned int)progress.member_state, + (unsigned int)progress.pending_opcode, + (unsigned int)progress.last_response_opcode, (unsigned long long)request_id, + (unsigned long long)wait_seq))); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + result = PCM_X_QUEUE_CORRUPT; + +requester_done: + if (wait_published) + cluster_lmd_wait_state_clear(&MyProc->cluster_lmd_wait); + gcs_block_pcm_x_requester_cleanup_context.wait_published = false; + return result; +} + + +PcmXQueueResult +cluster_gcs_pcm_x_acquire_writer(BufferDesc *buf, PcmXLocalWriterClaim *claim_out, + bool *claim_handed_off) +{ + GcsBlockPcmXRequesterCleanupContext *cleanup = &gcs_block_pcm_x_requester_cleanup_context; + MemoryContext error_context = CurrentMemoryContext; + ErrorData *original_error; + PcmXQueueResult result = PCM_X_QUEUE_INVALID; + bool fail_closed_required; + + if (claim_handed_off != NULL) + *claim_handed_off = false; + if (claim_handed_off == NULL) + return PCM_X_QUEUE_INVALID; + + if (!gcs_block_pcm_x_requester_exit_hook_registered) { + before_shmem_exit(gcs_block_pcm_x_requester_owner_exit, (Datum)0); + gcs_block_pcm_x_requester_exit_hook_registered = true; + } + if (cleanup->active) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + memset(cleanup, 0, sizeof(*cleanup)); + cleanup->claim_handoff_out = claim_handed_off; + cleanup->active = true; + PG_TRY(); + { + result = gcs_block_pcm_x_acquire_writer_impl(buf, claim_out); + } + PG_CATCH(); + { + MemoryContextSwitchTo(error_context); + original_error = CopyErrorData(); + FlushErrorState(); + gcs_block_pcm_x_requester_cleanup_noexcept(cleanup); + ReThrowError(original_error); + } + PG_END_TRY(); + + fail_closed_required = cleanup->fail_closed_required; + if (result == PCM_X_QUEUE_OK) { + /* The caller pre-publishes claim_out in its owner-exit ledger. Transfer + * cleanup authority before clearing our requester context, so every + * asynchronous exit observes exactly one owner of the live claim. */ + pg_write_barrier(); + *claim_handed_off = true; + pg_write_barrier(); + memset(cleanup, 0, sizeof(*cleanup)); + } else + gcs_block_pcm_x_requester_cleanup_noexcept(cleanup); + if (fail_closed_required) + cluster_pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +gcs_block_pcm_x_revalidate_peer_binding(int32 node_id, uint64 epoch, uint64 session) +{ + PcmXQueueResult result; + + result = cluster_pcm_x_runtime_peer_binding_revalidate_exact(node_id, epoch, session); + cluster_pcm_x_stats_note_queue_result(result); + return result == PCM_X_QUEUE_OK; +} + + +/* A self-originating PCM-X frame has no HELLO record: its authority is the + * already-bound local formation entry. Remote frames must retain the exact + * current-connection capability proof. */ +static bool +gcs_block_pcm_x_source_capable(int32 node_id) +{ + return node_id == cluster_node_id || cluster_sf_peer_supports_pcm_x_convert(node_id); +} + + +/* + * Capture the complete MEMBER formation together with one epoch and one live + * incarnation per protocol peer. Membership has no public snapshot lock for + * this consumer, so bracket the authority reads with full before/after arrays, + * epoch and quorum. A transition inside one collection is an inconsistent + * sample; the formation tick ignores it and retries on its next pass. A stable + * epoch/session value that differs from the already-bound runtime is positive + * drift evidence and fails closed there. A MEMBER outside the core protocol's + * 32-node wire bitmap is likewise not representable. + */ +static bool +gcs_block_pcm_x_collect_formation(PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT], + uint64 *epoch_out, uint64 *self_session_out) +{ + ClusterMembershipState membership_after[CLUSTER_MAX_NODES]; + ClusterMembershipState membership_before[CLUSTER_MAX_NODES]; + uint64 epoch_after; + uint64 epoch_before; + uint64 peer_session; + int i; + + if (bindings == NULL || epoch_out == NULL || self_session_out == NULL || cluster_node_id < 0 + || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return false; + memset(bindings, 0, sizeof(PcmXPeerBinding) * PCM_X_PROTOCOL_NODE_LIMIT); + *epoch_out = 0; + *self_session_out = 0; + epoch_before = cluster_epoch_get_current(); + if (!cluster_qvotec_in_quorum()) + return false; + for (i = 0; i < CLUSTER_MAX_NODES; i++) + membership_before[i] = cluster_membership_get_state(i); + if (membership_before[cluster_node_id] != CLUSTER_MEMBER_MEMBER) + return false; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + PcmXSessionAuthResult auth_result; + + if (membership_before[i] != CLUSTER_MEMBER_MEMBER) + continue; + if (i >= PCM_X_PROTOCOL_NODE_LIMIT) + return false; + if (i != cluster_node_id && !cluster_sf_peer_supports_pcm_x_convert(i)) + return false; + auth_result + = gcs_block_pcm_x_authenticated_session_result(i, epoch_before, &peer_session, NULL); + if (auth_result != PCM_X_SESSION_AUTH_OK) + return false; + bindings[i].cluster_epoch = epoch_before; + bindings[i].peer_session_incarnation = peer_session; + } + + pg_read_barrier(); + for (i = 0; i < CLUSTER_MAX_NODES; i++) + membership_after[i] = cluster_membership_get_state(i); + epoch_after = cluster_epoch_get_current(); + if (epoch_before != epoch_after || !cluster_qvotec_in_quorum() + || memcmp(membership_before, membership_after, sizeof(membership_before)) != 0) + return false; + + *epoch_out = epoch_before; + *self_session_out = bindings[cluster_node_id].peer_session_incarnation; + return *self_session_out != 0; +} + + +#define PCM_X_MASTER_DRIVE_SCAN_BUDGET 1024 + +static void gcs_block_pcm_x_master_drive_retry_tick(void); +static void gcs_block_pcm_x_terminal_retry_tick(void); +static PcmXQueueResult +gcs_block_pcm_x_cancel_claimed_probe_exact(const PcmXMasterPendingXReleaseToken *token); + + +/* + * Steady-state core formation gate. Only pristine startup may activate; an + * ACTIVE runtime whose complete peer binding can no longer be re-proved is + * permanently closed until the deferred crash-recovery protocol intervenes. + */ +void +cluster_gcs_block_pcm_x_formation_tick(void) +{ + PcmXPeerBinding bindings_after[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXPeerBinding bindings_before[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXRuntimeSnapshot runtime; + PcmXRuntimeSnapshot runtime_after; + PcmXQueueResult result; + uint64 epoch_after; + uint64 epoch_before; + uint64 self_session_after; + uint64 self_session_before; + int i; + + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state == PCM_X_RUNTIME_ACTIVE) { + /* + * Do not compare a half-old/half-new membership view with the bound + * runtime. Both complete samples must first agree; any transient read, + * including an unrelated membership flip, is a no-op for this tick. + */ + if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, + &self_session_before)) + return; + if (!gcs_block_pcm_x_collect_formation(bindings_after, &epoch_after, &self_session_after)) + return; + if (!cluster_gcs_pcm_x_formation_samples_stable(true, bindings_before, true, bindings_after) + || epoch_before != epoch_after || self_session_before != self_session_after) + return; + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + if (bindings_before[i].peer_session_incarnation == 0) + continue; + result = cluster_pcm_x_runtime_peer_binding_revalidate_exact( + i, bindings_before[i].cluster_epoch, bindings_before[i].peer_session_incarnation); + if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_CORRUPT) + goto fail_closed; + if (result != PCM_X_QUEUE_OK) + return; + } + runtime_after = cluster_pcm_x_runtime_snapshot(); + if (runtime_after.state != PCM_X_RUNTIME_ACTIVE + || runtime_after.gate_generation != runtime.gate_generation + || runtime_after.master_session_incarnation != runtime.master_session_incarnation) + return; + gcs_block_pcm_x_master_drive_retry_tick(); + gcs_block_pcm_x_terminal_retry_tick(); + return; + } + + /* ACTIVATING and any post-activation BLOCKED state are non-pristine. */ + if (runtime.gate_generation != 0 || runtime.master_session_incarnation != 0) + return; + if (!gcs_block_pcm_x_collect_formation(bindings_before, &epoch_before, &self_session_before)) + return; + (void)cluster_pcm_x_runtime_activate_bound(self_session_before, bindings_before); + return; + +fail_closed: + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +/* + * PCM-X is an application protocol, including when resource master and + * requester are the same node. Always stage through the tag-sharded DATA + * ring: remote frames are sent by the owning LMS worker, while self frames + * are loopback-dispatched by that same worker. This preserves the per-tag + * single-consumer premise used by confirm compensation and blocker-set + * publication; cluster_ic_send_envelope(dest=self) would only return DONE + * without executing a handler. + */ +bool +cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, const void *payload, + uint16 payload_len) +{ + if (msg_type < PGRAC_IC_MSG_PCM_X_ENQUEUE || msg_type > PGRAC_IC_MSG_PCM_X_RETIRE_ACK + || dest_node_id < 0 || dest_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || payload == NULL + || payload_len == 0) + return false; + + return cluster_grd_outbound_enqueue_backend_msg(msg_type, (uint32)dest_node_id, payload, + payload_len); +} + + +/* RETIRE deliberately carries a cluster-wide ticket watermark rather than a + * BufferTag. The LMON terminal driver must therefore supply the ticket's tag + * out of band when handing the request to the DATA plane. The receiving LMS + * worker sends the tagless ACK directly on its own DATA connection. */ +static bool +gcs_block_pcm_x_stage_retire_up_to(int32 dest_node_id, const PcmXRetirePayload *payload, + const BufferTag *tag) +{ + int worker; + + if (dest_node_id < 0 || dest_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || payload == NULL + || tag == NULL || cluster_lms_workers <= 0) + return false; + worker = cluster_lms_shard_for_tag(tag, cluster_lms_workers); + if (worker < 0 || worker >= cluster_lms_workers) + return false; + return cluster_lms_outbound_enqueue(worker, PGRAC_IC_MSG_PCM_X_RETIRE_UP_TO, + (uint32)dest_node_id, payload, sizeof(*payload)); +} + + +static bool +gcs_block_pcm_x_send_retire_ack(int32 dest_node_id, const PcmXRetirePayload *payload) +{ + ClusterICSendResult send_result; + + if (dest_node_id < 0 || dest_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || payload == NULL) + return false; + send_result = cluster_ic_send_envelope(PGRAC_IC_MSG_PCM_X_RETIRE_ACK, dest_node_id, payload, + sizeof(*payload)); + return send_result == CLUSTER_IC_SEND_DONE || send_result == CLUSTER_IC_SEND_WOULD_BLOCK; +} + + +/* + * ADMIT_CONFIRM and BLOCKER_SET mutate an engine snapshot and the LMD graph + * as one logical per-tag action. Their compensation arms are safe only while + * every frame for the tag is serialized on worker[shard(tag)]. Keep this + * runtime guard beside the handlers as well as the staging-router tests: an + * assertion catches developer builds, and the explicit branch protects + * production builds from deleting a newer worker's graph generation. + */ +static bool +gcs_block_pcm_x_handler_tag_shard_exact(const BufferTag *tag) +{ + int receive_worker; + int tag_shard; + + if (tag == NULL || cluster_lms_workers <= 0) + return false; + receive_worker = cluster_ic_tier1_my_data_channel(); + tag_shard = cluster_lms_shard_for_tag(tag, cluster_lms_workers); + Assert(tag_shard == receive_worker); + if (tag_shard != receive_worker) { + ereport(LOG, + (errmsg_internal("PCM-X graph handler misrouted to LMS worker %d (tag shard %d); " + "dropping to preserve per-tag serialization", + receive_worker, tag_shard))); + return false; + } + return true; +} + + +static void +gcs_block_pcm_x_master_drive_fail_closed(PcmXQueueResult result) +{ + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || result == PCM_X_QUEUE_BAD_STATE || result == PCM_X_QUEUE_INVALID + || result == PCM_X_QUEUE_NO_CAPACITY) + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +static PcmXQueueResult +gcs_block_pcm_x_master_authority(const PcmXMasterDriveSnapshot *snapshot, + PcmAuthoritySnapshot *authority_out, uint32 *holders_out, + int32 *source_out) +{ + if (snapshot == NULL || authority_out == NULL || holders_out == NULL || source_out == NULL) + return PCM_X_QUEUE_INVALID; + if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) + return PCM_X_QUEUE_BAD_STATE; + if (!cluster_pcm_lock_authority_snapshot(snapshot->ref.identity.tag, authority_out)) + return PCM_X_QUEUE_NOT_READY; + return cluster_gcs_pcm_x_authority_holders_exact(authority_out, snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id, holders_out, + source_out); +} + + +/* Bind the node-only GRD starvation barrier to one immutable queue ticket. + * The GRD transition is idle-only, while the master-ticket flag supplies the + * durable replay proof that node identity alone cannot provide. */ +static PcmXQueueResult +gcs_block_pcm_x_ensure_pending_x_claim(const PcmXMasterDriveSnapshot *snapshot) +{ + PcmAuthoritySnapshot authority; + PcmXQueueResult result; + PcmPendingXReserveResult reserve_result; + bool claimed; + + if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_PROBE + || snapshot->ref.grant_generation != 0) + return PCM_X_QUEUE_INVALID; + result = cluster_pcm_x_master_pending_x_claim_state_exact(&snapshot->ref, &claimed); + if (result != PCM_X_QUEUE_OK) + return result; + if (claimed) { + if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) + return PCM_X_QUEUE_BAD_STATE; + return PCM_X_QUEUE_OK; + } + + reserve_result = cluster_pcm_lock_try_reserve_pending_x( + snapshot->ref.identity.tag, snapshot->ref.identity.node_id, snapshot->ref.handle.ticket_id); + if (reserve_result != PCM_PENDING_X_RESERVE_OK) { + if (reserve_result == PCM_PENDING_X_RESERVE_NO_CAPACITY) + return PCM_X_QUEUE_NO_CAPACITY; + if (reserve_result == PCM_PENDING_X_RESERVE_INVALID) + return PCM_X_QUEUE_INVALID; + if (!cluster_pcm_lock_authority_snapshot(snapshot->ref.identity.tag, &authority) + || authority.pending_x_requester_node == -1) + return PCM_X_QUEUE_NOT_READY; + return PCM_X_QUEUE_BUSY; + } + + result = cluster_pcm_x_master_pending_x_claim_exact(&snapshot->ref); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + /* Only RESERVE_OK reaches this point, so this driver owns the + * half-published cookie. Deterministic non-mutating claim failures + * must compensate it exactly; corruption keeps the evidence behind the + * already-closed runtime. */ + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || result == PCM_X_QUEUE_NO_CAPACITY) + return result; + if (!cluster_pcm_lock_clear_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id) + && cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + return result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_BAD_STATE + ? PCM_X_QUEUE_NOT_READY + : result; + } + /* A same-node legacy clear can race the cross-substrate publication window. + * Never drive a probe unless both halves of the proof are still present. */ + if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) { + /* CANCEL may win after the engine claim and clear the GRD cookie before + * this LMON continuation revalidates it. Re-read the ticket under its + * own domain lock: durable cancel/terminal progress is retryable, while + * an ACTIVE_PROBE+CLAIMED ticket with no cookie remains corruption. */ + claimed = false; + result = cluster_pcm_x_master_pending_x_claim_state_exact(&snapshot->ref, &claimed); + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_STALE + || result == PCM_X_QUEUE_NOT_FOUND || result == PCM_X_QUEUE_RETIRED) + return PCM_X_QUEUE_NOT_READY; + if (result != PCM_X_QUEUE_OK) + return result; + return claimed ? PCM_X_QUEUE_BAD_STATE : PCM_X_QUEUE_NOT_READY; + } + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +gcs_block_pcm_x_stage_queue_invalidations(const PcmXMasterDriveSnapshot *snapshot, + uint32 invalidate_bitmap) +{ + GcsBlockInvalidatePayload inv; + uint64 holder_session; + int node; + + if (snapshot == NULL || snapshot->ref.grant_generation == 0) + return PCM_X_QUEUE_INVALID; + memset(&inv, 0, sizeof(inv)); + inv.request_id = snapshot->ref.identity.request_id; + inv.epoch = snapshot->ref.identity.cluster_epoch; + inv.tag = snapshot->ref.identity.tag; + inv.master_node = cluster_node_id; + inv.invalidating_for_x_node = (uint8)snapshot->ref.identity.node_id; + inv.checksum = gcs_block_compute_invalidate_checksum(&inv); + + for (node = 0; node < PCM_X_PROTOCOL_NODE_LIMIT; node++) { + if ((invalidate_bitmap & (UINT32_C(1) << node)) == 0) + continue; + if (!cluster_membership_is_member(node) || !gcs_block_pcm_x_source_capable(node) + || !gcs_block_pcm_x_authenticated_session(node, inv.epoch, &holder_session) + || !gcs_block_pcm_x_revalidate_peer_binding(node, inv.epoch, holder_session)) + return PCM_X_QUEUE_NOT_READY; + if (!cluster_grd_outbound_enqueue_backend_msg(PGRAC_IC_MSG_GCS_BLOCK_INVALIDATE, + (uint32)node, &inv, sizeof(inv))) { + if (ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->invalidate_send_not_admitted_count, 1); + return PCM_X_QUEUE_BUSY; + } + if (ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->block_invalidate_broadcast_count, 1); + } + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +gcs_block_pcm_x_master_drive_transfer(const PcmXMasterDriveSnapshot *snapshot) +{ + PcmXRevokePayload revoke; + PcmXQueueResult result; + uint64 source_session = 0; + uint32 source_bit; + uint32 unacked; + int32 source; + + if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_TRANSFER) + return PCM_X_QUEUE_INVALID; + if (!cluster_gcs_pcm_x_transfer_pre_handoff_phase(snapshot->pending_opcode)) + return PCM_X_QUEUE_NOT_READY; + if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) + return PCM_X_QUEUE_BAD_STATE; + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE) + source = snapshot->expected_responder_node; + else { + uint32 source_bitmap = snapshot->acked_s_holders_bitmap; + + if (source_bitmap == 0 || (source_bitmap & (source_bitmap - 1)) != 0) + return PCM_X_QUEUE_NOT_READY; + result = cluster_gcs_pcm_x_next_unacked_holder(source_bitmap, 0, &source); + if (result != PCM_X_QUEUE_OK) + return result; + } + if (source < 0 || source >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_QUEUE_CORRUPT; + source_bit = UINT32_C(1) << source; + if ((snapshot->pending_s_holders_bitmap & source_bit) == 0 + || (snapshot->acked_s_holders_bitmap & source_bit) == 0) + return PCM_X_QUEUE_CORRUPT; + if (!gcs_block_pcm_x_authenticated_session(source, snapshot->ref.identity.cluster_epoch, + &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source, snapshot->ref.identity.cluster_epoch, + source_session)) + return PCM_X_QUEUE_NOT_READY; + result = cluster_pcm_x_master_revoke_arm_exact(&snapshot->ref, source, source_session, &revoke); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + unacked = snapshot->pending_s_holders_bitmap & ~snapshot->acked_s_holders_bitmap; + if (unacked != 0) + return gcs_block_pcm_x_stage_queue_invalidations(snapshot, unacked); + return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_REVOKE, source, &revoke, sizeof(revoke)) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; +} + + +static PcmXQueueResult +gcs_block_pcm_x_master_begin_transfer(PcmXMasterDriveSnapshot *snapshot, uint32 holders, + int32 source) +{ + PcmXMasterDriveSnapshot transfer; + PcmXMasterDriveSnapshot replaced; + PcmXTicketRef transfer_ref; + PcmXQueueResult begin_result; + PcmXQueueResult result; + uint32 source_bit; + + if (snapshot == NULL || source < 0 || source >= PCM_X_PROTOCOL_NODE_LIMIT + || (holders & (UINT32_C(1) << source)) == 0) + return PCM_X_QUEUE_INVALID; + if (!cluster_pcm_lock_queue_pending_x_exact(snapshot->ref.identity.tag, + snapshot->ref.identity.node_id, + snapshot->ref.handle.ticket_id)) + return PCM_X_QUEUE_BAD_STATE; + begin_result = cluster_pcm_x_master_begin_transfer_exact( + &snapshot->ref, snapshot->graph_generation, &transfer_ref); + cluster_pcm_x_stats_note_queue_result(begin_result); + if (begin_result != PCM_X_QUEUE_OK && begin_result != PCM_X_QUEUE_DUPLICATE) + return begin_result; + result = cluster_pcm_x_master_drive_snapshot_exact( + &snapshot->ref.identity.tag, snapshot->ref.identity.cluster_epoch, &transfer); + if (result != PCM_X_QUEUE_OK) + return result; + if (transfer.ticket_state != PCM_XT_ACTIVE_TRANSFER + || transfer.ref.handle.ticket_id != transfer_ref.handle.ticket_id + || transfer.ref.handle.queue_generation != transfer_ref.handle.queue_generation) + return PCM_X_QUEUE_STALE; + if (begin_result == PCM_X_QUEUE_DUPLICATE) + return gcs_block_pcm_x_master_drive_transfer(&transfer); + source_bit = UINT32_C(1) << source; + result = cluster_pcm_x_master_drive_bitmap_replace_exact(&transfer, holders, source_bit, + &replaced); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + *snapshot = replaced; + return gcs_block_pcm_x_master_drive_transfer(snapshot); +} + + +static PcmXQueueResult +gcs_block_pcm_x_master_drive_probe(PcmXMasterDriveSnapshot *snapshot) +{ + PcmAuthoritySnapshot authority; + PcmXMasterDriveSnapshot replaced; + PcmXQueueResult result; + uint32 acknowledged; + uint32 holders; + uint32 responder_bit; + int32 responder; + int32 source; + + if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_PROBE) + return PCM_X_QUEUE_INVALID; + if (snapshot->pending_s_holders_bitmap == 0) { + result = gcs_block_pcm_x_master_authority(snapshot, &authority, &holders, &source); + if (result != PCM_X_QUEUE_OK) + return result; + result = cluster_pcm_x_master_drive_bitmap_replace_exact(snapshot, holders, 0, &replaced); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + *snapshot = replaced; + } + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK) { + cluster_gcs_pcm_x_blocker_probe_kick(&snapshot->ref, snapshot->expected_responder_node); + return PCM_X_QUEUE_OK; + } + if (snapshot->pending_opcode != 0) + return PCM_X_QUEUE_BAD_STATE; + if (snapshot->last_response_opcode == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT + && snapshot->blocker_set_generation != 0) { + responder = (int32)snapshot->last_responder_node; + if (responder < 0 || responder >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_QUEUE_CORRUPT; + responder_bit = UINT32_C(1) << responder; + if ((snapshot->pending_s_holders_bitmap & responder_bit) != 0 + && (snapshot->acked_s_holders_bitmap & responder_bit) == 0) { + if (snapshot->blocker_count != 0) { + cluster_gcs_pcm_x_blocker_probe_kick(&snapshot->ref, responder); + return PCM_X_QUEUE_NOT_READY; + } + acknowledged = snapshot->acked_s_holders_bitmap | responder_bit; + result = cluster_pcm_x_master_drive_bitmap_replace_exact( + snapshot, snapshot->pending_s_holders_bitmap, acknowledged, &replaced); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + *snapshot = replaced; + } + } + result = cluster_gcs_pcm_x_next_unacked_holder(snapshot->pending_s_holders_bitmap, + snapshot->acked_s_holders_bitmap, &responder); + if (result == PCM_X_QUEUE_OK) { + cluster_gcs_pcm_x_blocker_probe_kick(&snapshot->ref, responder); + return PCM_X_QUEUE_OK; + } + if (result != PCM_X_QUEUE_NOT_FOUND) + return result; + result = gcs_block_pcm_x_master_authority(snapshot, &authority, &holders, &source); + if (result != PCM_X_QUEUE_OK) + return result; + if (holders != snapshot->pending_s_holders_bitmap) { + acknowledged = snapshot->acked_s_holders_bitmap & holders; + result = cluster_pcm_x_master_drive_bitmap_replace_exact(snapshot, holders, acknowledged, + &replaced); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + *snapshot = replaced; + if (acknowledged != holders) { + result = cluster_gcs_pcm_x_next_unacked_holder(holders, acknowledged, &responder); + if (result != PCM_X_QUEUE_OK) + return result; + cluster_gcs_pcm_x_blocker_probe_kick(&snapshot->ref, responder); + return PCM_X_QUEUE_OK; + } + } + return gcs_block_pcm_x_master_begin_transfer(snapshot, holders, source); +} + + +static PcmXQueueResult +gcs_block_pcm_x_master_drive_cancel_requested(const PcmXMasterDriveSnapshot *snapshot) +{ + PcmXMasterPendingXReleaseToken release; + PcmXPhasePayload ack; + PcmXQueueResult result; + + if (snapshot == NULL || snapshot->ticket_state != PCM_XT_ACTIVE_PROBE + || snapshot->flags + != (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + return PCM_X_QUEUE_INVALID; + /* Preserve the holder reliable FIFO: retry the one already-armed PROBE and + * let its normal BLOCKER_SET ACK path consume the leg. */ + if (snapshot->pending_opcode == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK) { + cluster_gcs_pcm_x_blocker_probe_kick(&snapshot->ref, snapshot->expected_responder_node); + return PCM_X_QUEUE_OK; + } + if (snapshot->pending_opcode != 0) + return PCM_X_QUEUE_CORRUPT; + + result = cluster_pcm_x_master_pending_x_cancel_prepare_exact(&snapshot->ref, &release); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + if (release.master_session_incarnation == 0) + return PCM_X_QUEUE_CORRUPT; + result = gcs_block_pcm_x_cancel_claimed_probe_exact(&release); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + + memset(&ack, 0, sizeof(ack)); + ack.ref = release.ref; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + return cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_CANCEL_ACK, + release.ref.identity.node_id, &ack, sizeof(ack)) + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BUSY; +} + + +static void +gcs_block_pcm_x_master_drive_tag(const BufferTag *tag, uint64 cluster_epoch) +{ + PcmXMasterDriveSnapshot snapshot; + PcmXTicketRef active; + PcmXQueueResult result; + + if (tag == NULL || cluster_epoch != cluster_epoch_get_current() || cluster_node_id < 0 + || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || cluster_gcs_lookup_master(*tag) != cluster_node_id + || cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING + || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id)) + return; + result = cluster_pcm_x_master_promote_head_exact(tag, cluster_epoch, &active); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_BUSY) { + gcs_block_pcm_x_master_drive_fail_closed(result); + return; + } + result = cluster_pcm_x_master_drive_snapshot_exact(tag, cluster_epoch, &snapshot); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK) { + gcs_block_pcm_x_master_drive_fail_closed(result); + return; + } + if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE + && snapshot.flags + == (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + result = gcs_block_pcm_x_master_drive_cancel_requested(&snapshot); + else if (snapshot.ticket_state == PCM_XT_ACTIVE_PROBE) { + result = gcs_block_pcm_x_ensure_pending_x_claim(&snapshot); + if (result == PCM_X_QUEUE_OK) + result = gcs_block_pcm_x_master_drive_probe(&snapshot); + } else if (snapshot.ticket_state == PCM_XT_ACTIVE_TRANSFER) + result = gcs_block_pcm_x_master_drive_transfer(&snapshot); + else + result = PCM_X_QUEUE_CORRUPT; + cluster_pcm_x_stats_note_queue_result(result); + gcs_block_pcm_x_master_drive_fail_closed(result); +} + + +/* + * First convert-queue DATA slice: authenticate and publish an ENQUEUE at the + * tag master, then return the exact admission handle. Transport completion + * is not application completion: a dropped/blocked ACK leaves the live master + * ticket available for byte-exact replay of the same prehandle. + */ +static void +cluster_gcs_handle_pcm_x_enqueue_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXEnqueuePayload *request; + PcmXMasterAdmission admission; + PcmXAdmitAckPayload ack; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session = 0; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXEnqueuePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + request = (const PcmXEnqueuePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(request->identity.tag); + if (!cluster_gcs_pcm_x_enqueue_ingress_valid(request, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id)) + return; + if (!gcs_block_pcm_x_source_capable(source_node)) + return; + if (!cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node)) + return; + if (cluster_gcs_block_phase_for_tag(request->identity.tag) == GCS_BLOCK_RECOVERING) + return; + if (!gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || request->prehandle.sender_session_incarnation != source_session) + return; + if (!gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + + cluster_pcm_x_stats_note_enqueue(); + result = cluster_pcm_x_master_admit_begin(request, &admission); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + + memset(&ack, 0, sizeof(ack)); + ack.ref = admission.ref; + ack.prehandle = admission.prehandle; + ack.result = (uint32)result; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + ack.flags = (uint16)admission.flags; + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_ADMIT_ACK, source_node, &ack, + sizeof(ack)); +} + + +/* + * Apply the exact ADMIT_ACK to the requester-node leader and wake its backend. + * WFG/blocker publication is the next layer: this handler deliberately does + * not arm or send ADMIT_CONFIRM. + */ +static void +cluster_gcs_handle_pcm_x_admit_ack_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXAdmitAckPayload *ack; + PcmXLocalHandle leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXAdmitAckPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + ack = (const PcmXAdmitAckPayload *)payload; + current_epoch = cluster_epoch_get_current(); + if (ack->ref.identity.node_id != cluster_node_id + || ack->ref.identity.cluster_epoch != current_epoch + || cluster_gcs_lookup_master(ack->ref.identity.tag) != source_node + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(ack->ref.identity.tag) == GCS_BLOCK_RECOVERING) + return; + if (!gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&ack->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_apply_admit_ack_exact(&leader, ack, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + PcmXPhasePayload confirm; + PcmXLocalReliableToken token; + PcmXQueueResult arm_result; + + arm_result = cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token); + cluster_pcm_x_stats_note_queue_result(arm_result); + if (arm_result == PCM_X_QUEUE_OK || arm_result == PCM_X_QUEUE_DUPLICATE) + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, source_node, + &confirm, sizeof(confirm)); + } + if ((result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) && ProcGlobal != NULL + && ack->ref.identity.procno < (uint32)ProcGlobal->allProcCount) + SetLatch(&ProcGlobal->allProcs[ack->ref.identity.procno].procLatch); +} + + +/* Publish the exact queued-leader blocker set into the existing WFG before + * making ADMITTING visible as QUEUED. The queue snapshot holds no lock on + * return; graph replace therefore obeys the C2 lock-order boundary. */ +static void +cluster_gcs_handle_pcm_x_admit_confirm_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *request; + PcmXMasterWfgSnapshot snapshot; + PcmXPhasePayload ack; + ClusterLmdVertex waiter; + PcmXQueueResult result; + uint64 current_epoch; + uint64 graph_generation; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + request = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(request->ref.identity.tag); + if (!cluster_gcs_pcm_x_admit_confirm_ingress_valid(request, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(request->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + if (!gcs_block_pcm_x_handler_tag_shard_exact(&request->ref.identity.tag)) + return; + + memset(&snapshot, 0, sizeof(snapshot)); + result = cluster_pcm_x_master_admit_wfg_snapshot_exact(&request->ref, &snapshot); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_DUPLICATE) + goto send_ack; + if (result != PCM_X_QUEUE_OK) + return; + + cluster_gcs_pcm_x_vertex_from_identity(&request->ref.identity, &waiter); + graph_generation = cluster_lmd_graph_replace_waiter_edges_exact( + &waiter, snapshot.blockers, (int)snapshot.blocker_count, waiter.request_id); + if (graph_generation == 0) { + cluster_lmd_pcm_convert_wfg_note_replace_fail(); + return; + } + cluster_lmd_pcm_convert_wfg_note_replace(); + result = cluster_pcm_x_master_admit_confirm_revalidate_exact(&snapshot.token, graph_generation); + cluster_pcm_x_stats_note_queue_result(result); + if (cluster_gcs_pcm_x_confirm_compensation_required(graph_generation, result)) { + if (cluster_lmd_graph_remove_edge_by_waiter_exact(&waiter)) + cluster_lmd_pcm_convert_wfg_note_remove(); + return; + } + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + +send_ack: + memset(&ack, 0, sizeof(ack)); + ack.ref = request->ref; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + if (cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, source_node, &ack, + sizeof(ack))) + gcs_block_pcm_x_master_drive_tag(&request->ref.identity.tag, + request->ref.identity.cluster_epoch); +} + + +static void +cluster_gcs_handle_pcm_x_admit_confirm_ack_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXPhasePayload *ack; + PcmXLocalHandle leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + ack = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(ack->ref.identity.tag); + if (!cluster_gcs_pcm_x_admit_confirm_ack_ingress_valid( + ack, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(ack->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&ack->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_admit_confirm_ack_exact(&leader, ack, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if ((result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) && ProcGlobal != NULL + && ack->ref.identity.procno < (uint32)ProcGlobal->allProcCount) + SetLatch(&ProcGlobal->allProcs[ack->ref.identity.procno].procLatch); +} + + +/* + * BLOCKER_SET BEGIN and EDGE intentionally have no application ACK. The + * holder retains one immutable outbound snapshot and retries the whole + * BEGIN/EDGE/COMMIT sequence until the final COMMIT ACK arrives. This keeps + * the 96-byte phase ACK exact: it identifies one complete set generation, + * rather than ambiguously acknowledging a truncated per-chunk cursor. + */ +static void +cluster_gcs_handle_pcm_x_blocker_set_begin_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXBlockerSetHeaderPayload *begin; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXBlockerSetHeaderPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + begin = (const PcmXBlockerSetHeaderPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(begin->ref.identity.tag); + if (!cluster_gcs_pcm_x_blocker_header_ingress_valid(begin, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(begin->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + if (!gcs_block_pcm_x_handler_tag_shard_exact(&begin->ref.identity.tag)) + return; + + result = cluster_pcm_x_master_blocker_stage_begin_exact(begin, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); +} + + +static PcmXQueueResult +cluster_gcs_pcm_x_remove_cancelled_waiter(const PcmXWaitIdentity *identity) +{ + ClusterLmdGraphRemoveResult graph_result; + ClusterLmdVertex waiter; + + cluster_gcs_pcm_x_vertex_from_identity(identity, &waiter); + graph_result = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter); + if (graph_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) { + cluster_lmd_pcm_convert_wfg_note_remove(); + return PCM_X_QUEUE_OK; + } + if (graph_result == CLUSTER_LMD_GRAPH_REMOVE_ABSENT) + return PCM_X_QUEUE_OK; + if (graph_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_STALE; + } + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; +} + + +/* Finish an already-prepared claimed cancellation without ever nesting the + * master queue lock with WFG or GRD locks. WFG goes first: STALE proves that + * the same 4-tuple names a newer wait_seq and therefore must retain the GRD + * barrier rather than expose that successor to an unsafe grant. */ +static PcmXQueueResult +gcs_block_pcm_x_cancel_claimed_probe_exact(const PcmXMasterPendingXReleaseToken *token) +{ + ClusterLmdGraphRemoveResult graph_result; + ClusterLmdVertex waiter; + PcmXQueueResult result; + + if (token == NULL || token->master_session_incarnation == 0) + return PCM_X_QUEUE_INVALID; + cluster_gcs_pcm_x_vertex_from_identity(&token->ref.identity, &waiter); + graph_result = cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter); + if (graph_result == CLUSTER_LMD_GRAPH_REMOVE_REMOVED) + cluster_lmd_pcm_convert_wfg_note_remove(); + else if (graph_result == CLUSTER_LMD_GRAPH_REMOVE_STALE) { + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_STALE; + } + if (!cluster_pcm_lock_clear_queue_pending_x_exact( + token->ref.identity.tag, token->ref.identity.node_id, token->ref.handle.ticket_id) + && cluster_pcm_lock_queue_pending_x_exact( + token->ref.identity.tag, token->ref.identity.node_id, token->ref.handle.ticket_id)) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + result = cluster_pcm_x_master_pending_x_cancel_finalize_exact(token); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + return result; + /* WFG and GRD have already crossed their irreversible exact barriers. + * Retaining RELEASE_PENDING without a recovery gate would expose a queue + * that can neither resume nor safely grant its successor. */ + cluster_pcm_x_runtime_fail_closed(); + return result; +} + + +/* Reconstruct cancellation cleanup entirely from shared ticket state. This + * is intentionally callable by LMON after the original LMS/backend exits: + * exact WFG removal is idempotent, the claimed path also clears only the + * ticket-bound GRD cookie, and finalize consumes RELEASE_PENDING last. */ +static PcmXQueueResult +gcs_block_pcm_x_cancel_terminal_cleanup_exact(const PcmXTicketRef *ref) +{ + PcmXMasterPendingXReleaseToken release; + PcmXQueueResult result; + + memset(&release, 0, sizeof(release)); + result = cluster_pcm_x_master_pending_x_cancel_prepare_exact(ref, &release); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + if (release.master_session_incarnation != 0) + return gcs_block_pcm_x_cancel_claimed_probe_exact(&release); + return cluster_gcs_pcm_x_remove_cancelled_waiter(&ref->identity); +} + + +static void +cluster_gcs_handle_pcm_x_blocker_set_edge_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXBlockerChunkPayload *edge; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXBlockerChunkPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + edge = (const PcmXBlockerChunkPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(edge->tag); + if (!cluster_gcs_pcm_x_blocker_edge_ingress_valid(edge, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(edge->tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + if (!gcs_block_pcm_x_handler_tag_shard_exact(&edge->tag)) + return; + + result = cluster_pcm_x_master_blocker_stage_edge_exact(edge, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); +} + + +/* Publish one complete holder blocker set into both the exact ticket snapshot + * and the existing LMD graph before returning its final application ACK. */ +static void +cluster_gcs_handle_pcm_x_blocker_set_commit_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXBlockerSetHeaderPayload *commit; + PcmXMasterBlockerEntry *entries = NULL; + PcmXMasterBlockerSnapshot snapshot; + ClusterLmdVertex *blockers = NULL; + ClusterLmdVertex waiter; + PcmXPhasePayload ack; + PcmXQueueResult result; + PcmXQueueResult stage_result; + uint64 current_epoch; + uint64 graph_generation; + uint64 source_session; + uint32 i; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXBlockerSetHeaderPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + commit = (const PcmXBlockerSetHeaderPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(commit->ref.identity.tag); + if (!cluster_gcs_pcm_x_blocker_header_ingress_valid(commit, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(commit->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + if (!gcs_block_pcm_x_handler_tag_shard_exact(&commit->ref.identity.tag)) + return; + + /* Ask the engine to prove the dynamic max_wait_edges bound and exact live + * stage before allocating from an untrusted wire count. */ + if (commit->nblockers != 0) { + result = cluster_pcm_x_master_blocker_stage_commit_exact( + commit, source_node, source_session, NULL, 0, &snapshot); + if (result != PCM_X_QUEUE_NO_CAPACITY) { + cluster_pcm_x_stats_note_queue_result(result); + goto blocker_commit_done; + } + entries = palloc0(mul_size(commit->nblockers, sizeof(*entries))); + blockers = palloc0(mul_size(commit->nblockers, sizeof(*blockers))); + } + result = cluster_pcm_x_master_blocker_stage_commit_exact(commit, source_node, source_session, + entries, commit->nblockers, &snapshot); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + goto blocker_commit_done; + stage_result = result; + if (result == PCM_X_QUEUE_DUPLICATE && snapshot.graph_generation != 0) + goto complete_blocker_probe; + result = cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&snapshot, entries, + commit->nblockers); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK) { + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + goto blocker_commit_done; + } + for (i = 0; i < commit->nblockers; i++) + blockers[i] = entries[i].blocker; + + cluster_gcs_pcm_x_vertex_from_identity(&commit->ref.identity, &waiter); + graph_generation = cluster_lmd_graph_replace_waiter_edges_exact( + &waiter, blockers, (int)commit->nblockers, waiter.request_id); + if (graph_generation == 0) { + cluster_lmd_pcm_convert_wfg_note_replace_fail(); + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + goto blocker_commit_done; + } + cluster_lmd_pcm_convert_wfg_note_replace(); + result = cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, entries, commit->nblockers, + graph_generation); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + /* The ticket set has already been published. Removing by waiter here + * could delete a concurrently newer graph generation; retain evidence + * and close the runtime for recovery instead. */ + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + goto blocker_commit_done; + } + +complete_blocker_probe: + /* Admit the generation-exact ACK into the durable per-peer outbound FIFO + * before consuming the master probe leg. A refused enqueue retains that + * leg, so the next bounded PROBE retry can replay the same immutable bytes. + * Type 49 is staged later on the same FIFO and therefore cannot overtake + * this ACK. */ + if (!cluster_gcs_pcm_x_blocker_ack_build(&commit->ref, commit->set_generation, &ack)) { + Assert(false); + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + goto blocker_commit_done; + } + if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, source_node, &ack, + sizeof(ack))) { + cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_BUSY); + goto blocker_commit_done; + } + result = cluster_pcm_x_master_blocker_probe_complete_exact(&commit->ref, commit->set_generation, + source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (cluster_pcm_x_blocker_commit_completion_requires_recovery(stage_result, result)) { + /* The first application may have put its ACK in flight without consuming + * the probe leg, so an unexpected completion remains fail-closed. A + * DUPLICATE stage is different: the same set was already published and + * the ACK above was merely replayed after a legal phase advance. The + * classifier keeps BAD_STATE/STALE/NOT_READY/BUSY benign in that replay + * case while structural CORRUPT still halts the runtime. */ + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + goto blocker_commit_done; + } + gcs_block_pcm_x_master_drive_tag(&commit->ref.identity.tag, commit->ref.identity.cluster_epoch); + +blocker_commit_done: + if (blockers != NULL) + pfree(blockers); + if (entries != NULL) + pfree(entries); +} + + +static void +cluster_gcs_pcm_x_wake_cancel_rotation(const PcmXWaitIdentity *cancelled, + const PcmXLocalHandle *new_leader) +{ + if (ProcGlobal == NULL) + return; + if (cancelled != NULL && cancelled->procno < (uint32)ProcGlobal->allProcCount) + SetLatch(&ProcGlobal->allProcs[cancelled->procno].procLatch); + if (new_leader != NULL && new_leader->identity.request_id != 0 + && new_leader->identity.procno < (uint32)ProcGlobal->allProcCount) + SetLatch(&ProcGlobal->allProcs[new_leader->identity.procno].procLatch); +} + + +static void +cluster_gcs_handle_pcm_x_prehandle_cancel_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXPrehandleCancelPayload *request; + PcmXMasterAdmission cancelled; + PcmXAdmitAckPayload ack; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPrehandleCancelPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + request = (const PcmXPrehandleCancelPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(request->identity.tag); + if (!cluster_gcs_pcm_x_prehandle_cancel_ingress_valid( + request, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(request->identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || request->prehandle.sender_session_incarnation != source_session + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + + memset(&cancelled, 0, sizeof(cancelled)); + result = cluster_pcm_x_master_prehandle_cancel_exact(request, &cancelled); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + result = cluster_gcs_pcm_x_remove_cancelled_waiter(&cancelled.ref.identity); + if (result != PCM_X_QUEUE_OK) + return; + memset(&ack, 0, sizeof(ack)); + ack.ref = cancelled.ref; + ack.prehandle = cancelled.prehandle; + /* Canonical application outcome is byte-identical for first apply/replay. */ + ack.result = PCM_X_QUEUE_OK; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL; + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK, source_node, &ack, + sizeof(ack)); +} + + +static void +cluster_gcs_handle_pcm_x_prehandle_cancel_ack_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXAdmitAckPayload *ack; + PcmXLocalHandle leader; + PcmXLocalHandle new_leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXAdmitAckPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + ack = (const PcmXAdmitAckPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(ack->ref.identity.tag); + if (!cluster_gcs_pcm_x_prehandle_cancel_ack_ingress_valid( + ack, env->payload_length, source_node, current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(ack->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&ack->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_prehandle_cancel_ack_exact(&leader, ack, source_node, + source_session, &new_leader); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + cluster_gcs_pcm_x_wake_cancel_rotation(&ack->ref.identity, &new_leader); +} + + +static void +cluster_gcs_handle_pcm_x_cancel_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *request; + PcmXPhasePayload ack; + PcmXMasterPendingXReleaseToken release; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + request = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(request->ref.identity.tag); + if (!cluster_gcs_pcm_x_cancel_ingress_valid(request, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(request->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + /* One master-lock classification closes the ordinary CANCEL-vs-transfer + * race. A claimed in-flight PROBE records CANCEL_REQUESTED and returns + * NOT_READY; its normal application ACK path will resume this release. */ + result = cluster_pcm_x_master_pending_x_cancel_prepare_exact(&request->ref, &release); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + if (release.master_session_incarnation != 0) { + result = gcs_block_pcm_x_cancel_claimed_probe_exact(&release); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + } else { + result = cluster_gcs_pcm_x_remove_cancelled_waiter(&request->ref.identity); + if (result != PCM_X_QUEUE_OK) + return; + } + memset(&ack, 0, sizeof(ack)); + ack.ref = request->ref; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_CANCEL_ACK, source_node, &ack, + sizeof(ack)); +} + + +static int +gcs_block_pcm_x_blocker_compare(const void *left_ptr, const void *right_ptr) +{ + const ClusterLmdVertex *left = (const ClusterLmdVertex *)left_ptr; + const ClusterLmdVertex *right = (const ClusterLmdVertex *)right_ptr; + + if (left->node_id != right->node_id) + return left->node_id < right->node_id ? -1 : 1; + if (left->procno != right->procno) + return left->procno < right->procno ? -1 : 1; + if (left->cluster_epoch != right->cluster_epoch) + return left->cluster_epoch < right->cluster_epoch ? -1 : 1; + if (left->request_id != right->request_id) + return left->request_id < right->request_id ? -1 : 1; + return 0; +} + + +/* + * Resolve a type-48 PROBE into one exact holder/blocker snapshot and replay + * its complete 45-47 sequence. Passive holders remain barrier occupancy but + * contribute no WFG edge. A torn PGPROC seqlock is explicitly BUSY and is + * never normalized to an empty set. The snapshot persists until the nonzero + * generation-exact type-48 ACK consumes it, so duplicate probes replay the + * same application bytes after a partial/refused send. + */ +static PcmXQueueResult +gcs_block_pcm_x_probe_collect_and_stage(const PcmXPhasePayload *probe, int32 master_node, + uint64 master_session) +{ + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXBlockerSetHeaderPayload header_payload; + PcmXLocalHolderHandle *holders = NULL; + PcmXBlockerChunkPayload *edges = NULL; + ClusterLmdVertex *blockers = NULL; + PcmXQueueResult result; + Size blocker_count = 0; + Size i; + + /* Replay is a lookup, never a re-sample. Once a 45-47 snapshot exists its + * token and bytes are immutable until the generation-exact ACK consumes + * it; this path therefore remains available even when the blocker pool is + * otherwise full or holder wait-state changes after the first arm. */ + result = cluster_pcm_x_local_blocker_snapshot_lookup_exact(&probe->ref, master_node, + master_session, &blocker_snapshot); + if (result == PCM_X_QUEUE_OK) + goto probe_copy; + if (result != PCM_X_QUEUE_NOT_FOUND) + goto probe_done; + + /* Freeze the local FIFO round and holder registry before consulting any + * PGPROC wait state. The generation-zero reservation is durable across + * this count-first allocation retry, so a late writer cannot enter the + * sampled round and a duplicate PROBE cannot reopen the barrier. */ + result = cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe->ref, master_node, master_session, NULL, 0, &holder_snapshot); + if (result == PCM_X_QUEUE_NO_CAPACITY) { + holders = palloc0(mul_size(holder_snapshot.holder_count, sizeof(*holders))); + result = cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe->ref, master_node, master_session, holders, holder_snapshot.holder_count, + &holder_snapshot); + } + if (result == PCM_X_QUEUE_DUPLICATE) { + result = cluster_pcm_x_local_blocker_snapshot_lookup_exact( + &probe->ref, master_node, master_session, &blocker_snapshot); + if (result == PCM_X_QUEUE_OK) + goto probe_copy; + } + if (result != PCM_X_QUEUE_OK) + goto probe_done; + if (holder_snapshot.holder_count != 0) { + if (ProcGlobal == NULL) { + result = PCM_X_QUEUE_NOT_READY; + goto probe_done; + } + blockers = palloc0(mul_size(holder_snapshot.holder_count, sizeof(*blockers))); + for (i = 0; i < holder_snapshot.holder_count; i++) { + const PcmXWaitIdentity *holder = &holders[i].key.identity; + ClusterLmdWaitStateSnapshot wait_snapshot; + ClusterLmdWaitStateReadResult read_result; + ClusterLmdVertex *blocker; + + if (holder->node_id != cluster_node_id + || holder->cluster_epoch != probe->ref.identity.cluster_epoch + || holder->procno >= (uint32)ProcGlobal->allProcCount) { + result = PCM_X_QUEUE_STALE; + goto probe_done; + } + read_result = cluster_lmd_wait_state_read_exact( + &ProcGlobal->allProcs[holder->procno].cluster_lmd_wait, &wait_snapshot); + if (read_result == CLUSTER_LMD_WAIT_STATE_READ_BUSY) { + result = PCM_X_QUEUE_BUSY; + goto probe_done; + } + if (read_result == CLUSTER_LMD_WAIT_STATE_READ_INACTIVE) + continue; + if (read_result != CLUSTER_LMD_WAIT_STATE_READ_ACTIVE + || (wait_snapshot.kind != CLUSTER_LMD_WAIT_GES + && wait_snapshot.kind != CLUSTER_LMD_WAIT_TX + && wait_snapshot.kind != CLUSTER_LMD_WAIT_PCM_CONVERT) + || wait_snapshot.wait_seq == 0 + || wait_snapshot.cluster_epoch != probe->ref.identity.cluster_epoch + || (wait_snapshot.kind == CLUSTER_LMD_WAIT_TX + ? (wait_snapshot.request_id != 0 + || !TransactionIdIsValid(wait_snapshot.xid)) + : wait_snapshot.request_id == 0)) { + result = PCM_X_QUEUE_STALE; + goto probe_done; + } + blocker = &blockers[blocker_count++]; + blocker->node_id = cluster_node_id; + blocker->procno = holder->procno; + blocker->cluster_epoch = wait_snapshot.cluster_epoch; + blocker->request_id = wait_snapshot.request_id; + blocker->xid = wait_snapshot.xid; + blocker->local_start_ts_ms = 0; + blocker->wait_seq = wait_snapshot.wait_seq; + } + if (blocker_count > 1) + qsort(blockers, blocker_count, sizeof(*blockers), gcs_block_pcm_x_blocker_compare); + } + + result = cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe->ref, master_node, master_session, &holder_snapshot, holders, + holder_snapshot.holder_count, blockers, blocker_count, &blocker_snapshot); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + goto probe_done; + +probe_copy: + if (blocker_snapshot.blocker_count != 0) + edges = palloc0(mul_size(blocker_snapshot.blocker_count, sizeof(*edges))); + result = cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &header_payload, + edges, blocker_snapshot.blocker_count); + if (result != PCM_X_QUEUE_OK) + goto probe_done; + /* The ACK producer may never collapse into the reserved PROBE value. */ + Assert(header_payload.set_generation != 0 && header_payload.set_generation != UINT64_MAX); + if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_BEGIN, master_node, + &header_payload, sizeof(header_payload))) { + result = PCM_X_QUEUE_BUSY; + goto probe_done; + } + for (i = 0; i < blocker_snapshot.blocker_count; i++) { + if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_EDGE, master_node, + &edges[i], sizeof(edges[i]))) { + result = PCM_X_QUEUE_BUSY; + goto probe_done; + } + } + if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, master_node, + &header_payload, sizeof(header_payload))) { + result = PCM_X_QUEUE_BUSY; + goto probe_done; + } + result = PCM_X_QUEUE_OK; + +probe_done: + if (edges != NULL) + pfree(edges); + if (blockers != NULL) + pfree(blockers); + if (holders != NULL) + pfree(holders); + return result; +} + + +/* Arm and stage one exact zero-generation PROBE. The tag-sharded master + * driver may call this once per bounded retry tick; DUPLICATE replays the + * identical durable type-48 leg. */ +void +cluster_gcs_pcm_x_blocker_probe_kick(const PcmXTicketRef *ref, int32 holder_node) +{ + PcmXMasterProbeToken token; + PcmXPhasePayload probe; + PcmXQueueResult result; + uint64 current_epoch; + uint64 holder_session; + + current_epoch = cluster_epoch_get_current(); + if (!cluster_gcs_pcm_x_ticket_ref_wire_valid(ref, current_epoch) || holder_node < 0 + || holder_node >= PCM_X_PROTOCOL_NODE_LIMIT || cluster_node_id < 0 + || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || cluster_gcs_lookup_master(ref->identity.tag) != cluster_node_id + || cluster_gcs_block_phase_for_tag(ref->identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_source_capable(holder_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(holder_node) + || !gcs_block_pcm_x_authenticated_session(holder_node, current_epoch, &holder_session) + || !gcs_block_pcm_x_revalidate_peer_binding(holder_node, current_epoch, holder_session)) + return; + result = cluster_pcm_x_master_blocker_probe_arm_exact(ref, holder_node, holder_session, &probe, + &token); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + Assert(cluster_gcs_pcm_x_blocker_ack_generation(&probe) == 0); + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, holder_node, &probe, + sizeof(probe)); +} + + +static void +cluster_gcs_handle_pcm_x_blocker_set_ack_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *control; + PcmXQueueResult result; + uint64 set_generation; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + control = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(control->ref.identity.tag); + if (!cluster_gcs_pcm_x_blocker_control_ingress_valid(control, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(control->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + set_generation = cluster_gcs_pcm_x_blocker_ack_generation(control); + if (set_generation == 0) { + if (!cluster_gcs_pcm_x_blocker_probe_ingress_valid(control, env->payload_length, + source_node, current_epoch, tag_master, + cluster_node_id)) + return; + result = gcs_block_pcm_x_probe_collect_and_stage(control, source_node, source_session); + } else { + if (!cluster_gcs_pcm_x_blocker_ack_ingress_valid(control, env->payload_length, source_node, + current_epoch, tag_master, + cluster_node_id)) + return; + /* Every 45-47 commit ACK is generation-exact and therefore nonzero. */ + Assert(set_generation != UINT64_MAX); + result = cluster_pcm_x_local_blocker_ack_exact(&control->ref, set_generation, source_node, + source_session); + } + cluster_pcm_x_stats_note_queue_result(result); +} + + +static bool +gcs_block_pcm_x_transfer_ingress_authorized(const BufferTag *tag, int32 source_node, + uint64 current_epoch, uint64 *source_session_out) +{ + uint64 source_session; + + if (source_session_out != NULL) + *source_session_out = 0; + if (tag == NULL || source_node < 0 || source_node >= PCM_X_PROTOCOL_NODE_LIMIT + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(*tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return false; + if (source_session_out != NULL) + *source_session_out = source_session; + return true; +} + + +static void +gcs_block_pcm_x_wake_requester(const PcmXWaitIdentity *identity) +{ + if (identity != NULL && identity->node_id == cluster_node_id && ProcGlobal != NULL + && identity->procno < (uint32)ProcGlobal->allProcCount) + SetLatch(&ProcGlobal->allProcs[identity->procno].procLatch); +} + + +/* RETIRE may promote a different local writer than the backend that drove the + * terminal exchange. Validate the complete seqlock-protected wait identity + * before using its procno; a missed hint is recovered by the requester's + * bounded progress recheck, whereas waking a recycled wait generation is not + * accepted as exact protocol behavior. */ +static bool +gcs_block_pcm_x_wake_requester_exact(const PcmXWaitIdentity *identity, uint32 all_proc_count) +{ + ClusterLmdWaitStateSnapshot snapshot; + ClusterLmdWaitStateReadResult read_result; + + if (identity == NULL || ProcGlobal == NULL || ProcGlobal->allProcCount <= 0 + || (uint32)ProcGlobal->allProcCount != all_proc_count + || identity->node_id != cluster_node_id || identity->procno >= all_proc_count + || identity->request_id == 0 || identity->wait_seq == 0) + return false; + read_result = cluster_lmd_wait_state_read_exact( + &ProcGlobal->allProcs[identity->procno].cluster_lmd_wait, &snapshot); + if (!cluster_gcs_pcm_x_wait_identity_matches(identity, cluster_node_id, all_proc_count, + read_result, &snapshot)) + return false; + SetLatch(&ProcGlobal->allProcs[identity->procno].procLatch); + return true; +} + + +/* Completing a local writer is the condition transition for its direct FIFO + * successor. The engine copies that successor under the same local-tag lock + * that records WRITER_COMPLETE; this adapter validates the full published wait + * generation before setting the latch. Polling remains only a lost-wake + * fallback, and a missed hint never changes the committed release result. */ +PcmXQueueResult +cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(const PcmXLocalWriterClaim *claim) +{ + PcmXWaitIdentity successor; + PcmXQueueResult result; + uint32 all_proc_count; + + memset(&successor, 0, sizeof(successor)); + result = cluster_pcm_x_local_writer_claim_release_collect_exact(claim, &successor); + if (result != PCM_X_QUEUE_OK || successor.request_id == 0 || ProcGlobal == NULL + || ProcGlobal->allProcCount <= 0) + return result; + all_proc_count = (uint32)ProcGlobal->allProcCount; + (void)gcs_block_pcm_x_wake_requester_exact(&successor, all_proc_count); + return result; +} + +/* Cleanup runs while an ERROR stack is already active or after backend owner + * death. Contain any ERROR from the exact engine/wake adapter: the caller + * retains its ledger entry unless OK is returned, and RECOVERY_BLOCKED makes + * the preserved claim explicit recovery evidence instead of an orphan. */ +PcmXQueueResult +cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(const PcmXLocalWriterClaim *claim) +{ + volatile PcmXQueueResult result = PCM_X_QUEUE_CORRUPT; + + PG_TRY(); + { + result = cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(claim); + } + PG_CATCH(); + { + FlushErrorState(); + cluster_pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_CORRUPT; + } + PG_END_TRY(); + return (PcmXQueueResult)result; +} + + +/* Allocate before the engine call so no ERROR-capable allocation follows a + * committed watermark. The engine publishes batch.count only after every + * terminal detach, the retired frontier, and the global admission gate have + * committed. Exact wake attempts are best-effort latency hints and never + * suppress the RETIRE ACK on BUSY, INACTIVE, or identity mismatch. */ +static PcmXQueueResult +gcs_block_pcm_x_local_retire_apply_and_wake(const PcmXRetirePayload *request, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXWaitIdentity *wake_items; + PcmXLocalWakeBatch wake_batch; + PcmXQueueResult result; + Size i; + uint32 all_proc_count; + + if (request == NULL) + return PCM_X_QUEUE_INVALID; + if (ProcGlobal == NULL || ProcGlobal->allProcCount <= 0) + return PCM_X_QUEUE_NOT_READY; + all_proc_count = (uint32)ProcGlobal->allProcCount; + wake_items = palloc0(mul_size((Size)all_proc_count, sizeof(*wake_items))); + wake_batch.items = wake_items; + wake_batch.capacity = (Size)all_proc_count; + wake_batch.count = 0; + result = cluster_pcm_x_local_retire_up_to_collect_exact( + request, authenticated_master_node, authenticated_master_session, &wake_batch); + if (result == PCM_X_QUEUE_OK) { + for (i = 0; i < wake_batch.count && i < wake_batch.capacity; i++) + (void)gcs_block_pcm_x_wake_requester_exact(&wake_batch.items[i], all_proc_count); + } + pfree(wake_items); + return result; +} + + +/* A type-49 ref names the remote requester, never a local holder backend. + * Locate holder processes only through the exact per-tag holder registry; a + * cached page with no registered holder remains durably staged for the outer + * data-plane worker and must not wake an unrelated same-numbered procno. */ +static void +gcs_block_pcm_x_wake_registered_holders(const BufferTag *tag) +{ + PcmXLocalHolderSnapshot snapshot; + PcmXLocalHolderHandle *holders = NULL; + PcmXQueueResult result; + Size i; + + if (tag == NULL || ProcGlobal == NULL) + return; + result = cluster_pcm_x_local_holder_snapshot(tag, NULL, 0, &snapshot); + if (result != PCM_X_QUEUE_NO_CAPACITY || snapshot.holder_count == 0) + return; + holders = palloc0(mul_size(snapshot.holder_count, sizeof(*holders))); + result = cluster_pcm_x_local_holder_snapshot(tag, holders, snapshot.holder_count, &snapshot); + if (result == PCM_X_QUEUE_OK + && cluster_pcm_x_local_holder_snapshot_revalidate(tag, &snapshot) == PCM_X_QUEUE_OK) { + for (i = 0; i < snapshot.holder_count; i++) { + const PcmXWaitIdentity *identity = &holders[i].key.identity; + + if (identity->node_id == cluster_node_id + && identity->procno < (uint32)ProcGlobal->allProcCount) + SetLatch(&ProcGlobal->allProcs[identity->procno].procLatch); + } + } + pfree(holders); +} + + +static bool +gcs_block_pcm_x_ticket_ref_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) +{ + return left != NULL && right != NULL && memcmp(left, right, sizeof(*left)) == 0; +} + + +static bool +gcs_block_pcm_x_image_key(const PcmXTicketRef *ref, uint64 image_id, GcsBlockDedupKey *key_out) +{ + int32 requester_backend_id; + int32 requester_node; + + if (key_out != NULL) + memset(key_out, 0, sizeof(*key_out)); + if (ref == NULL || key_out == NULL || ref->identity.node_id < 0 + || ref->identity.node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || !cluster_pcm_x_image_id_decode(image_id, NULL, NULL) + || !cluster_gcs_requester_id_decode(ref->identity.request_id, &requester_node, + &requester_backend_id, NULL) + || requester_node != ref->identity.node_id || requester_backend_id <= 0) + return false; + key_out->origin_node_id = (uint32)requester_node; + key_out->requester_backend_id = requester_backend_id; + key_out->request_id = image_id; + key_out->cluster_epoch = ref->identity.cluster_epoch; + return true; +} + + +static void +gcs_block_pcm_x_reserved_binding(const PcmXRevokePayload *revoke, uint64 source_generation, + uint64 master_session, GcsBlockPcmXImageBinding *binding_out) +{ + memset(binding_out, 0, sizeof(*binding_out)); + binding_out->identity.ref = revoke->ref; + binding_out->identity.image.image_id = revoke->image_id; + binding_out->identity.image.source_own_generation = source_generation; + binding_out->identity.image.source_node = (uint32)cluster_node_id; + binding_out->master_session = master_session; +} + + +static void +gcs_block_pcm_x_note_own_result(ClusterPcmOwnResult result) +{ + if (result == CLUSTER_PCM_OWN_BUSY) + cluster_pcm_x_stats_note_own_busy(); + else if (result == CLUSTER_PCM_OWN_CORRUPT) + cluster_pcm_x_stats_note_own_corrupt(); +} + + +static bool +gcs_block_pcm_x_release_image_exact(int worker_id, const GcsBlockPcmXImageWork *work, + const GcsBlockPcmXImageBinding *binding) +{ + return cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &work->key, &work->tag, binding) + == GCS_BLOCK_PCM_X_IMAGE_RELEASED; +} + + +/* Before ownership commit, a failed byte leg must unstage its exact capacity + * reservation before clearing REVOKING. After commit this helper is never + * legal: the immutable READY bytes are then the only recoverable copy owned by + * this protocol attempt. */ +static void +gcs_block_pcm_x_abort_image_before_finish(int worker_id, const GcsBlockPcmXImageWork *work, + const GcsBlockPcmXImageBinding *binding, BufferDesc *buf, + const ClusterPcmOwnSnapshot *revoking, + bool revoke_started) +{ + ClusterPcmOwnResult abort_result = CLUSTER_PCM_OWN_OK; + bool released; + + released = gcs_block_pcm_x_release_image_exact(worker_id, work, binding); + if (revoke_started) { + if (revoking->pcm_state == (uint8)PCM_STATE_N) + abort_result = cluster_bufmgr_pcm_own_abort_n_revoke(buf, revoking); + else if (revoking->pcm_state == (uint8)PCM_STATE_S) + abort_result = cluster_bufmgr_pcm_own_abort_s_revoke(buf, revoking); + else if (revoking->pcm_state == (uint8)PCM_STATE_X) + abort_result = cluster_bufmgr_pcm_own_abort_x_revoke(buf, revoking); + else + abort_result = CLUSTER_PCM_OWN_CORRUPT; + } + gcs_block_pcm_x_note_own_result(abort_result); + if (!released || abort_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_x_runtime_fail_closed(); +} + + +static void +gcs_block_pcm_x_stage_ready_work(int worker_id, const GcsBlockPcmXImageWork *work) +{ + PcmXGrantPayload image_ready; + PcmXGrantPayload replay; + PcmXQueueResult result; + GcsBlockPcmXImageResult mark_result; + GcsBlockPcmXImageResult rollback_result; + int32 master_node; + + if (!cluster_pcm_x_image_id_decode(work->binding.identity.image.image_id, &master_node, NULL)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + memset(&image_ready, 0, sizeof(image_ready)); + image_ready.ref = work->binding.identity.ref; + image_ready.image = work->binding.identity.image; + result = cluster_pcm_x_local_holder_image_ready_arm_exact( + &image_ready, master_node, work->binding.master_session, &replay); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_NOT_READY) + return; + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + mark_result = cluster_gcs_block_dedup_pcm_x_mark_staged_exact(worker_id, &work->key, &work->tag, + &work->binding); + if (mark_result == GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + return; + if (mark_result != GCS_BLOCK_PCM_X_IMAGE_STAGED) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + if (cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_IMAGE_READY, master_node, &replay, + sizeof(replay))) + return; + rollback_result = cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(worker_id, &work->key, + &work->tag, &work->binding); + if (rollback_result != GCS_BLOCK_PCM_X_IMAGE_REARMED + && rollback_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + cluster_pcm_x_runtime_fail_closed(); +} + + +static void +gcs_block_pcm_x_materialize_reserved_work(int worker_id, const GcsBlockPcmXImageWork *work) +{ + char block_data[GCS_BLOCK_DATA_SIZE]; + ClusterPcmOwnSnapshot current; + ClusterPcmOwnSnapshot retained; + ClusterPcmOwnSnapshot revoking; + GcsBlockPcmXImageBinding ready_binding; + GcsBlockPcmXImageResult image_result; + GcsBlockReplyHeader reply_header; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXQueueResult holder_result; + ClusterPcmOwnResult own_result; + ClusterPcmXRevokeFinishMode finish_mode; + PageHeaderData page_header; + BufferDesc *buf; + XLogRecPtr page_lsn; + uint64 page_scn = 0; + bool source_is_n; + bool source_is_s; + bool source_is_x; + bool descriptor_retained; + bool self_source_handoff; + int buffer_id; + + holder_result = cluster_pcm_x_local_holder_snapshot(&work->tag, NULL, 0, &holder_snapshot); + if (holder_result == PCM_X_QUEUE_NO_CAPACITY && holder_snapshot.holder_count > 0) + return; + if (holder_result == PCM_X_QUEUE_NOT_READY || holder_result == PCM_X_QUEUE_BUSY) + return; + if (holder_result != PCM_X_QUEUE_OK || holder_snapshot.holder_count != 0) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + + own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&work->tag, &buffer_id, ¤t); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_NOT_READY) + return; + source_is_n = current.pcm_state == (uint8)PCM_STATE_N; + source_is_s = current.pcm_state == (uint8)PCM_STATE_S; + source_is_x = current.pcm_state == (uint8)PCM_STATE_X; + self_source_handoff = (source_is_n || source_is_s || source_is_x) + && work->binding.identity.ref.identity.node_id == cluster_node_id; + if (own_result != CLUSTER_PCM_OWN_OK || (!source_is_n && !source_is_s && !source_is_x) + || (source_is_n && !self_source_handoff) || current.flags != 0 + || current.generation != work->binding.identity.image.source_own_generation) { + (void)gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding); + cluster_pcm_x_runtime_fail_closed(); + return; + } + buf = GetBufferDescriptor(buffer_id); + if (source_is_n) + own_result = cluster_bufmgr_pcm_own_prepare_n_source_image( + buf, ¤t, &revoking, block_data, &page_lsn, &page_scn); + else if (source_is_s) + own_result = cluster_bufmgr_pcm_own_begin_s_revoke(buf, ¤t, &revoking); + else + own_result = cluster_bufmgr_pcm_own_begin_x_revoke(buf, ¤t, &revoking); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result == CLUSTER_PCM_OWN_BUSY || own_result == CLUSTER_PCM_OWN_NOT_READY) + return; + if (own_result != CLUSTER_PCM_OWN_OK) { + (void)gcs_block_pcm_x_release_image_exact(worker_id, work, &work->binding); + cluster_pcm_x_runtime_fail_closed(); + return; + } + + if (!source_is_n && !cluster_bufmgr_copy_block_for_gcs(work->tag, &page_lsn, block_data)) { + gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, + true); + return; + } + memcpy(&page_header, block_data, sizeof(page_header)); + if ((uint64)page_lsn != (uint64)PageXLogRecPtrGet(page_header.pd_lsn) + || (source_is_n && page_scn != (uint64)page_header.pd_block_scn)) { + gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, + true); + return; + } + page_scn = (uint64)page_header.pd_block_scn; + + ready_binding = work->binding; + ready_binding.identity.image.page_scn = page_scn; + ready_binding.identity.image.page_lsn = (uint64)page_lsn; + ready_binding.identity.image.page_checksum = cluster_gcs_block_compute_checksum(block_data); + memset(&reply_header, 0, sizeof(reply_header)); + reply_header.request_id = work->key.request_id; + reply_header.page_lsn = ready_binding.identity.image.page_lsn; + reply_header.epoch = work->key.cluster_epoch; + reply_header.checksum = ready_binding.identity.image.page_checksum; + reply_header.sender_node = cluster_node_id; + reply_header.requester_backend_id = work->key.requester_backend_id; + reply_header.transition_id = (uint8)PCM_TRANS_N_TO_S; + reply_header.status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; + GcsBlockReplyHeaderSetForwardingMasterNode(&reply_header, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + image_result = cluster_gcs_block_dedup_pcm_x_materialize( + worker_id, &work->key, &work->tag, &ready_binding, &reply_header, block_data); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_STORED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { + gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &work->binding, buf, &revoking, + true); + return; + } + + if (self_source_handoff) { + /* A requester acting as its own N/S/X source keeps the one exact + * source frozen under the same REVOKING token until PREPARE atomically + * reclassifies it as GRANT_PENDING. Turning it into a retained remote + * source here would create a FINAL-before-DRAIN self-cycle. */ + own_result = cluster_bufmgr_pcm_own_snapshot(buf, ¤t); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result != CLUSTER_PCM_OWN_OK || !BufferTagsEqual(¤t.tag, &revoking.tag) + || current.generation != revoking.generation + || current.reservation_token != revoking.reservation_token + || current.flags != PCM_OWN_FLAG_REVOKING || current.pcm_state != revoking.pcm_state) { + /* Immutable record materialization already succeeded. Preserve both + * pieces of evidence and stop admission; a rollback could race a + * duplicate PREPARE using the same exact lifecycle. */ + cluster_pcm_x_runtime_fail_closed(); + return; + } + } else { + own_result + = cluster_bufmgr_pcm_own_finish_revoke_retain(buf, &revoking, page_lsn, &retained); + gcs_block_pcm_x_note_own_result(own_result); + if (own_result != CLUSTER_PCM_OWN_OK) { + gcs_block_pcm_x_abort_image_before_finish(worker_id, work, &ready_binding, buf, + &revoking, true); + return; + } + finish_mode = cluster_pcm_x_revoke_finish_mode(&revoking.tag, 0); + descriptor_retained = finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN; + if (revoking.generation == UINT64_MAX || retained.generation != revoking.generation + 1 + || retained.reservation_token != revoking.reservation_token + || (descriptor_retained ? retained.flags != PCM_OWN_FLAG_REVOKING : retained.flags != 0) + || retained.pcm_state != (uint8)PCM_STATE_N + || (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN + && finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) + || !BufferTagsEqual(&retained.tag, &revoking.tag)) { + /* Finish already crossed the S/X->N byte boundary. A malformed retained + * projection cannot be rolled back without inventing bytes or ownership. */ + cluster_pcm_x_runtime_fail_closed(); + return; + } + } + image_result = cluster_gcs_block_dedup_pcm_x_publish_ready_exact(worker_id, &work->key, + &work->tag, &ready_binding); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_STORED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) { + /* The source revoke is already irreversible (remote S/X or self-frozen N/S/X): + * retain the immutable intermediate evidence and stop the protocol. Only + * recovery may resolve this boundary. */ + cluster_pcm_x_runtime_fail_closed(); + return; + } + + /* Remote-source ownership is now N at generation+1; requester-as-source + * instead remains exact N/S/X+REVOKING at its base generation for the + * fused PREPARE handoff. No later failure may unstage the A-record; READY + * remains retryable until DRAIN. */ + { + GcsBlockPcmXImageWork ready_work = *work; + + ready_work.binding = ready_binding; + ready_work.entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + gcs_block_pcm_x_stage_ready_work(worker_id, &ready_work); + } +} + + +static void +gcs_block_pcm_x_owner_exit(int code, Datum arg pg_attribute_unused()) +{ + /* A nonzero aux-process exit can interrupt any instruction between the + * holder barrier, byte staging, X->N, and outbound admission. Core has no + * crash resolver, so never infer which side committed. */ + if (code != 0) + cluster_pcm_x_runtime_fail_closed(); +} + + +void +cluster_gcs_block_pcm_x_owner_start(int worker_id) +{ + if (worker_id < 0 || worker_id >= cluster_lms_workers) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + before_shmem_exit(gcs_block_pcm_x_owner_exit, Int32GetDatum(worker_id)); + if (cluster_gcs_block_dedup_pcm_x_restart_audit(worker_id)) + cluster_pcm_x_runtime_fail_closed(); +} + + +/* One bounded unit of holder-byte work, owned by the same DATA worker that + * dispatches every frame for this BufferTag. */ +void +cluster_gcs_block_pcm_x_image_pump_tick(int worker_id) +{ + GcsBlockPcmXImageWork work; + GcsBlockPcmXImageResult result; + + if (worker_id < 0 || worker_id >= cluster_lms_workers + || (cluster_write_fence_enforcing() && !cluster_write_fence_allowed())) + return; + result = cluster_gcs_block_dedup_pcm_x_next_work(worker_id, &work); + if (result == GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND || result == GCS_BLOCK_PCM_X_IMAGE_FULL) + return; + if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_REPLAY) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + if (cluster_lms_shard_for_tag(&work.tag, cluster_lms_workers) != worker_id) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + if (result == GCS_BLOCK_PCM_X_IMAGE_RESERVED) + gcs_block_pcm_x_materialize_reserved_work(worker_id, &work); + else + gcs_block_pcm_x_stage_ready_work(worker_id, &work); +} + + +/* 49 only installs the holder-side revoke ledger and wakes exact registered + * holders. It never copies a page, drops ownership, or synthesizes type 50 + * inside the IC handler. */ +static void +cluster_gcs_handle_pcm_x_revoke_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXRevokePayload *revoke; + ClusterPcmOwnSnapshot own_snapshot; + GcsBlockDedupKey image_key; + GcsBlockPcmXImageBinding reserved_binding; + GcsBlockPcmXImageResult image_result; + PcmXLocalHolderProgress holder_progress; + PcmXQueueResult progress_result; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + uint64 source_generation = 0; + int buffer_id; + int worker_id; + int32 source_node; + int32 tag_master; + bool have_source_generation = false; + bool new_reservation = false; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXRevokePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + revoke = (const PcmXRevokePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(revoke->ref.identity.tag); + if (!cluster_gcs_pcm_x_revoke_ingress_valid(revoke, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized(&revoke->ref.identity.tag, source_node, + current_epoch, &source_session)) + return; + if (!gcs_block_pcm_x_handler_tag_shard_exact(&revoke->ref.identity.tag)) + return; + worker_id = cluster_ic_tier1_my_data_channel(); + if (!gcs_block_pcm_x_image_key(&revoke->ref, revoke->image_id, &image_key)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + + /* Once type 50 is armed, the resident source buffer has already been retired. + * Recover its source generation from the exact holder ledger instead of + * looking through the now-empty buffer mapping. A fully drained duplicate + * is application-complete and must not reserve a fresh 8KB slot. */ + progress_result + = cluster_pcm_x_local_holder_progress_exact(&revoke->ref.identity.tag, &holder_progress); + if (progress_result == PCM_X_QUEUE_OK) { + if (!gcs_block_pcm_x_ticket_ref_equal(&holder_progress.ref, &revoke->ref) + || holder_progress.image.image_id != revoke->image_id) { + cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_STALE); + return; + } + if ((holder_progress.flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0) { + if ((holder_progress.flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + result = cluster_pcm_x_local_holder_revoke_apply_exact(revoke, source_node, + source_session); + cluster_pcm_x_stats_note_queue_result(result); + return; + } + if (holder_progress.pending_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY + || holder_progress.phase == PGRAC_IC_MSG_PCM_X_IMAGE_READY) { + if (!cluster_gcs_pcm_x_holder_image_ready_exact(&holder_progress, &revoke->ref, + revoke->image_id, cluster_node_id)) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + source_generation = holder_progress.image.source_own_generation; + have_source_generation = true; + } + } else if (progress_result != PCM_X_QUEUE_NOT_FOUND) { + cluster_pcm_x_stats_note_queue_result(progress_result); + return; + } + + if (!have_source_generation) { + ClusterPcmOwnResult own_result; + bool source_is_n; + bool source_is_s; + bool source_is_x; + + own_result = cluster_bufmgr_pcm_own_snapshot_by_tag(&revoke->ref.identity.tag, &buffer_id, + &own_snapshot); + gcs_block_pcm_x_note_own_result(own_result); + source_is_n = own_snapshot.pcm_state == (uint8)PCM_STATE_N; + source_is_s = own_snapshot.pcm_state == (uint8)PCM_STATE_S; + source_is_x = own_snapshot.pcm_state == (uint8)PCM_STATE_X; + if (own_result != CLUSTER_PCM_OWN_OK || (!source_is_n && !source_is_s && !source_is_x) + || own_snapshot.flags != 0) + return; + source_generation = own_snapshot.generation; + } + gcs_block_pcm_x_reserved_binding(revoke, source_generation, source_session, &reserved_binding); + image_result = cluster_gcs_block_dedup_pcm_x_reserve( + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_RESERVED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + return; + new_reservation = image_result == GCS_BLOCK_PCM_X_IMAGE_RESERVED; + if (!new_reservation) { + image_result = cluster_gcs_block_dedup_pcm_x_rearm_exact( + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding); + if (image_result != GCS_BLOCK_PCM_X_IMAGE_REARMED + && image_result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE + && image_result != GCS_BLOCK_PCM_X_IMAGE_NOT_READY) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + } + result = cluster_pcm_x_local_holder_revoke_apply_exact(revoke, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + gcs_block_pcm_x_wake_registered_holders(&revoke->ref.identity.tag); + else if (new_reservation + && cluster_gcs_block_dedup_pcm_x_release_exact( + worker_id, &image_key, &revoke->ref.identity.tag, &reserved_binding) + != GCS_BLOCK_PCM_X_IMAGE_RELEASED) + cluster_pcm_x_runtime_fail_closed(); +} + + +static void +cluster_gcs_handle_pcm_x_image_ready_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXGrantPayload *image_ready; + PcmXGrantPayload prepare; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXGrantPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + image_ready = (const PcmXGrantPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(image_ready->ref.identity.tag); + if (!cluster_gcs_pcm_x_image_ready_ingress_valid(image_ready, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized(&image_ready->ref.identity.tag, source_node, + current_epoch, &source_session)) + return; + result = cluster_pcm_x_master_image_ready_exact(image_ready, source_node, source_session, + &prepare); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + prepare.ref.identity.node_id, &prepare, + sizeof(prepare)); +} + + +static void +cluster_gcs_handle_pcm_x_prepare_grant_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXGrantPayload *prepare; + PcmXLocalHandle leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXGrantPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + prepare = (const PcmXGrantPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(prepare->ref.identity.tag); + if (!cluster_gcs_pcm_x_prepare_grant_ingress_valid(prepare, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized(&prepare->ref.identity.tag, source_node, + current_epoch, &source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&prepare->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_prepare_grant_exact(&leader, prepare, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + gcs_block_pcm_x_wake_requester(&prepare->ref.identity); +} + + +static void +cluster_gcs_handle_pcm_x_install_ready_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXInstallReadyPayload *install_ready; + PcmXPhasePayload commit; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXInstallReadyPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + install_ready = (const PcmXInstallReadyPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(install_ready->ref.identity.tag); + if (!cluster_gcs_pcm_x_install_ready_ingress_valid(install_ready, env->payload_length, + source_node, current_epoch, tag_master, + cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized( + &install_ready->ref.identity.tag, source_node, current_epoch, &source_session)) + return; + result = cluster_pcm_x_master_install_ready_exact(install_ready, source_node, source_session, + &commit); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_COMMIT_X, + commit.ref.identity.node_id, &commit, sizeof(commit)); +} + + +static void +cluster_gcs_handle_pcm_x_commit_x_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *commit; + PcmXLocalHandle leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + commit = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(commit->ref.identity.tag); + if (!cluster_gcs_pcm_x_commit_x_ingress_valid(commit, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized(&commit->ref.identity.tag, source_node, + current_epoch, &source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&commit->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_commit_x_exact(&leader, commit, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + gcs_block_pcm_x_wake_requester(&commit->ref.identity); +} + + +static void +cluster_gcs_handle_pcm_x_final_ack_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXFinalAckPayload *final_ack; + PcmAuthoritySnapshot authority; + PcmXGrdHandoffToken handoff; + PcmXGrdHandoffResult handoff_result; + PcmXMasterFinalAckToken token; + PcmXPhasePayload final_commit; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXFinalAckPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + final_ack = (const PcmXFinalAckPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(final_ack->ref.identity.tag); + if (!cluster_gcs_pcm_x_final_ack_ingress_valid(final_ack, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized(&final_ack->ref.identity.tag, source_node, + current_epoch, &source_session)) + return; + result = cluster_pcm_x_master_final_ack_prepare_exact(final_ack, source_node, source_session, + &token); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + gcs_block_pcm_x_master_drive_fail_closed(result); + return; + } + if (!cluster_pcm_lock_authority_snapshot(final_ack->ref.identity.tag, &authority) + || !cluster_gcs_pcm_x_grd_handoff_token_build(&token, &authority, &handoff)) { + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + return; + } + handoff_result = cluster_pcm_lock_queue_handoff_x_exact(&handoff); + if (handoff_result != PCM_X_GRD_HANDOFF_OK && handoff_result != PCM_X_GRD_HANDOFF_DUPLICATE) { + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + return; + } + result = cluster_pcm_x_master_final_ack_finalize_exact(&token, &final_commit); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + /* GRD already committed; retaining the engine/image evidence and + * closing the runtime is the only 8.A-safe outcome. */ + (void)cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + return; + } + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, + final_commit.ref.identity.node_id, &final_commit, + sizeof(final_commit)); +} + + +static void +cluster_gcs_handle_pcm_x_final_commit_ack_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXPhasePayload *final_commit; + PcmXLocalReliableToken token; + PcmXPhasePayload final_confirm; + PcmXLocalHandle leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + final_commit = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(final_commit->ref.identity.tag); + if (!cluster_gcs_pcm_x_final_commit_ack_ingress_valid(final_commit, env->payload_length, + source_node, current_epoch, tag_master, + cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized( + &final_commit->ref.identity.tag, source_node, current_epoch, &source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&final_commit->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_final_commit_ack_exact(&leader, final_commit, source_node, + source_session, &final_confirm, &token); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, source_node, + &final_confirm, sizeof(final_confirm)); + gcs_block_pcm_x_wake_requester(&final_commit->ref.identity); + } +} + + +static void +cluster_gcs_handle_pcm_x_final_confirm_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *final_confirm; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + final_confirm = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(final_confirm->ref.identity.tag); + if (!cluster_gcs_pcm_x_final_confirm_ingress_valid(final_confirm, env->payload_length, + source_node, current_epoch, tag_master, + cluster_node_id) + || !gcs_block_pcm_x_transfer_ingress_authorized( + &final_confirm->ref.identity.tag, source_node, current_epoch, &source_session)) + return; + result = cluster_pcm_x_master_final_confirm_exact(final_confirm, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + cluster_gcs_pcm_x_terminal_kick(&final_confirm->ref); +} + + +static void +cluster_gcs_handle_pcm_x_cancel_ack_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *ack; + PcmXLocalHandle leader; + PcmXLocalHandle new_leader; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + ack = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(ack->ref.identity.tag); + if (!cluster_gcs_pcm_x_cancel_ack_ingress_valid(ack, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(ack->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = cluster_pcm_x_local_lookup_exact(&ack->ref.identity, &leader); + if (result != PCM_X_QUEUE_OK) + return; + result = cluster_pcm_x_local_cancel_ack_exact(&leader, ack, source_node, source_session, + &new_leader); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + cluster_gcs_pcm_x_wake_cancel_rotation(&ack->ref.identity, &new_leader); +} + + +static PcmXQueueResult +gcs_block_pcm_x_local_drain_apply_exact(const PcmXDrainPollPayload *poll, + int32 authenticated_master_node, + uint64 authenticated_master_session, int worker_id) +{ + GcsBlockPcmXImageBinding binding; + GcsBlockDedupKey key; + PcmXLocalHolderProgress progress; + PcmXQueueResult progress_result; + PcmXQueueResult result; + ClusterPcmXRevokeFinishMode finish_mode; + uint64 holder_image_id = 0; + bool holder_image = false; + bool holder_ref = false; + bool self_source_handoff = false; + + progress_result = cluster_pcm_x_local_holder_progress_exact(&poll->ref.identity.tag, &progress); + if (progress_result == PCM_X_QUEUE_OK + && gcs_block_pcm_x_ticket_ref_equal(&progress.ref, &poll->ref)) { + holder_ref = true; + holder_image_id = progress.image.image_id; + } else if (progress_result != PCM_X_QUEUE_OK && progress_result != PCM_X_QUEUE_NOT_FOUND) + return progress_result; + + result = cluster_pcm_x_local_drain_poll_exact(poll, authenticated_master_node, + authenticated_master_session); + if (result != PCM_X_QUEUE_OK) + return result; + + /* DRAIN mutates the holder leg. Capture the image after that transition so + * IMAGE_READY cannot race between an old snapshot and the drain commit. */ + progress_result = cluster_pcm_x_local_holder_progress_exact(&poll->ref.identity.tag, &progress); + if (progress_result == PCM_X_QUEUE_OK + && gcs_block_pcm_x_ticket_ref_equal(&progress.ref, &poll->ref)) { + holder_ref = true; + if (holder_image_id == 0) + holder_image_id = progress.image.image_id; + if (!cluster_gcs_pcm_x_holder_image_drained_exact(&progress, &poll->ref, holder_image_id, + cluster_node_id) + || !cluster_pcm_x_image_id_decode(progress.image.image_id, NULL, NULL) + || !gcs_block_pcm_x_image_key(&progress.ref, progress.image.image_id, &key)) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + memset(&binding, 0, sizeof(binding)); + binding.identity.ref = progress.ref; + binding.identity.image = progress.image; + binding.master_session = authenticated_master_session; + holder_image = true; + } else if (progress_result != PCM_X_QUEUE_OK && progress_result != PCM_X_QUEUE_NOT_FOUND) { + /* Local DRAIN is already durable; losing its exact image evidence cannot + * be retried as ordinary queue work. */ + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (holder_ref && !holder_image) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (!holder_image) + return PCM_X_QUEUE_OK; + self_source_handoff = binding.identity.image.source_node == (uint32)cluster_node_id + && poll->ref.identity.node_id == cluster_node_id; + if (self_source_handoff + && cluster_bufmgr_pcm_own_self_handoff_x_exact(&poll->ref.identity.tag, + binding.identity.image.source_own_generation) + != CLUSTER_PCM_OWN_OK) { + /* FINAL makes the in-place X descriptor authoritative before DRAIN. + * Preserve the immutable record if that exact proof is missing. */ + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (cluster_gcs_block_dedup_pcm_x_release_exact(worker_id, &key, &poll->ref.identity.tag, + &binding) + != GCS_BLOCK_PCM_X_IMAGE_RELEASED) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (self_source_handoff) { + if (ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64(&ClusterGcsBlock->pcm_x_self_handoff_drain_count, 1); + return PCM_X_QUEUE_OK; + } + finish_mode = cluster_pcm_x_revoke_finish_mode(&poll->ref.identity.tag, 0); + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_DROP) + return PCM_X_QUEUE_OK; + if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (cluster_bufmgr_pcm_own_release_retained_image(&poll->ref.identity.tag, + binding.identity.image.source_own_generation) + != CLUSTER_PCM_OWN_OK) { + cluster_pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + return PCM_X_QUEUE_OK; +} + + +static void +cluster_gcs_handle_pcm_x_drain_poll_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXDrainPollPayload *poll; + PcmXPhasePayload ack; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int worker_id; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXDrainPollPayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + poll = (const PcmXDrainPollPayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(poll->ref.identity.tag); + if (!cluster_gcs_pcm_x_drain_poll_ingress_valid(poll, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(poll->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + if (!gcs_block_pcm_x_handler_tag_shard_exact(&poll->ref.identity.tag)) + return; + worker_id = cluster_ic_tier1_my_data_channel(); + /* The 49-56 integration drains exact holder type 50 and requester type 56 + * here first. This DRAIN_POLL proves master application and publishes each + * local terminal tombstone; cancellation is already ready. */ + result = gcs_block_pcm_x_local_drain_apply_exact(poll, source_node, source_session, worker_id); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + memset(&ack, 0, sizeof(ack)); + ack.ref = poll->ref; + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_ACK, source_node, &ack, + sizeof(ack)); +} + + +static void +cluster_gcs_handle_pcm_x_drain_ack_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXPhasePayload *ack; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + int32 tag_master; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXPhasePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + ack = (const PcmXPhasePayload *)payload; + current_epoch = cluster_epoch_get_current(); + tag_master = cluster_gcs_lookup_master(ack->ref.identity.tag); + if (!cluster_gcs_pcm_x_drain_ack_ingress_valid(ack, env->payload_length, source_node, + current_epoch, tag_master, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || cluster_gcs_block_phase_for_tag(ack->ref.identity.tag) == GCS_BLOCK_RECOVERING + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = cluster_pcm_x_master_terminal_leg_ack_exact(&ack->ref, PCM_X_TERMINAL_LEG_DRAIN, + source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + cluster_gcs_pcm_x_terminal_kick(&ack->ref); +} + + +static void +cluster_gcs_handle_pcm_x_retire_up_to_envelope(const ClusterICEnvelope *env, const void *payload) +{ + const PcmXRetirePayload *request; + PcmXRetirePayload ack; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXRetirePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + request = (const PcmXRetirePayload *)payload; + current_epoch = cluster_epoch_get_current(); + if (!gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !cluster_gcs_pcm_x_retire_request_ingress_valid(request, env->payload_length, + source_node, source_session, + current_epoch, cluster_node_id) + || !gcs_block_pcm_x_source_capable(source_node) || !cluster_qvotec_in_quorum() + || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = gcs_block_pcm_x_local_retire_apply_and_wake(request, source_node, source_session); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + ack = *request; + (void)gcs_block_pcm_x_send_retire_ack(source_node, &ack); +} + + +static void +cluster_gcs_handle_pcm_x_retire_up_to_ack_envelope(const ClusterICEnvelope *env, + const void *payload) +{ + const PcmXRetirePayload *ack; + PcmXRuntimeSnapshot runtime; + PcmXTicketRef ref; + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + int32 source_node; + + if (env == NULL || payload == NULL || env->payload_length != sizeof(PcmXRetirePayload) + || env->source_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + source_node = (int32)env->source_node_id; + ack = (const PcmXRetirePayload *)payload; + current_epoch = cluster_epoch_get_current(); + runtime = cluster_pcm_x_runtime_snapshot(); + if (!cluster_gcs_pcm_x_retire_ack_ingress_valid( + ack, env->payload_length, source_node, current_epoch, + runtime.master_session_incarnation, cluster_node_id) + || runtime.state != PCM_X_RUNTIME_ACTIVE || !gcs_block_pcm_x_source_capable(source_node) + || !cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id) + || !cluster_membership_is_member(source_node) + || !gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return; + result = cluster_pcm_x_master_retire_ack_resolve_exact(ack, source_node, source_session, &ref); + cluster_pcm_x_stats_note_queue_result(result); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + cluster_gcs_pcm_x_terminal_kick(&ref); +} + + +/* Drive one ticket without time-based reclamation. An already-armed leg + * returns an exact duplicate token and is resent; a refused send leaves that + * durable leg intact for the next LMON formation tick. */ +void +cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref) +{ + static uint8 last_auth_failure[PCM_X_PROTOCOL_NODE_LIMIT]; + ClusterGcsPcmXAuthSample auth_sample; + PcmXRuntimeSnapshot runtime; + PcmXMasterAdmission cancelled; + PcmXTerminalLegToken token; + PcmXDrainPollPayload poll; + PcmXPhasePayload cancel_ack; + PcmXAdmitAckPayload prehandle_ack; + PcmXRetirePayload retire; + PcmXTicketRef resolved; + PcmXQueueResult result; + PcmXSessionAuthResult auth_result; + uint64 current_epoch; + uint64 responder_session; + int kind; + int node; + int local_steps; + bool prehandle_cancel; + + current_epoch = cluster_epoch_get_current(); + if (!cluster_gcs_pcm_x_terminal_ref_wire_valid(ref, current_epoch)) + return; + if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) + return; + if (cluster_gcs_lookup_master(ref->identity.tag) != cluster_node_id) + return; + if (cluster_gcs_block_phase_for_tag(ref->identity.tag) == GCS_BLOCK_RECOVERING) + return; + if (!cluster_qvotec_in_quorum() || !cluster_membership_is_member(cluster_node_id)) + return; + if (ref->grant_generation == 0) { + result = gcs_block_pcm_x_cancel_terminal_cleanup_exact(ref); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + gcs_block_pcm_x_master_drive_fail_closed(result); + return; + } + result = cluster_pcm_x_master_cancel_ack_snapshot_exact(ref, &cancelled, &prehandle_cancel); + cluster_pcm_x_stats_note_queue_result(result); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) { + gcs_block_pcm_x_master_drive_fail_closed(result); + return; + } + /* The requester must observe the phase-exact cancel ACK before DRAIN. + * Both frames use the same tag-shard DATA FIFO; refusing either ACK arm + * leaves DRAIN unarmed so the next LMON tick retries this order exactly. */ + if (prehandle_cancel) { + memset(&prehandle_ack, 0, sizeof(prehandle_ack)); + prehandle_ack.ref = cancelled.ref; + prehandle_ack.prehandle = cancelled.prehandle; + prehandle_ack.result = PCM_X_QUEUE_OK; + prehandle_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL; + if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK, + ref->identity.node_id, &prehandle_ack, + sizeof(prehandle_ack))) + return; + } else { + memset(&cancel_ack, 0, sizeof(cancel_ack)); + cancel_ack.ref = cancelled.ref; + cancel_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + if (!cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_CANCEL_ACK, ref->identity.node_id, + &cancel_ack, sizeof(cancel_ack))) + return; + } + } + + for (local_steps = 0; local_steps < 2 * PCM_X_PROTOCOL_NODE_LIMIT; local_steps++) { + /* All ACKs may already be present. Detach first; global ticket order is + * enforced inside the engine and the periodic oldest-ticket scan retries. */ + result = cluster_pcm_x_master_detach_terminal_exact(ref); + if (result == PCM_X_QUEUE_OK) { + gcs_block_pcm_x_master_drive_tag(&ref->identity.tag, ref->identity.cluster_epoch); + return; + } + if (result == PCM_X_QUEUE_STALE) { + gcs_block_pcm_x_master_drive_tag(&ref->identity.tag, ref->identity.cluster_epoch); + return; + } + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + return; + + for (kind = PCM_X_TERMINAL_LEG_DRAIN; kind <= PCM_X_TERMINAL_LEG_RETIRE; kind++) { + for (node = 0; node < PCM_X_PROTOCOL_NODE_LIMIT; node++) { + if (!cluster_membership_is_member(node) || !gcs_block_pcm_x_source_capable(node)) + continue; + auth_result = gcs_block_pcm_x_authenticated_session_result( + node, current_epoch, &responder_session, &auth_sample); + if (auth_result != PCM_X_SESSION_AUTH_OK) { + if (last_auth_failure[node] != (uint8)auth_result) { + last_auth_failure[node] = (uint8)auth_result; + ereport( + LOG, + (errmsg("PCM-X terminal authority for node %d is not ready (reason %u)", + node, (unsigned int)auth_result), + errdetail( + "expected epoch %llu, slot=(%llu,%llu,%llu)->(%llu,%llu,%llu), " + "fresh=%d->%d, connection=%s/%u->%s/%u", + (unsigned long long)current_epoch, + (unsigned long long)auth_sample.session_before, + (unsigned long long)auth_sample.slot_generation_before, + (unsigned long long)auth_sample.observed_epoch_before, + (unsigned long long)auth_sample.session_after, + (unsigned long long)auth_sample.slot_generation_after, + (unsigned long long)auth_sample.observed_epoch_after, + auth_sample.fresh_before, auth_sample.fresh_after, + auth_sample.connection_before_valid ? "valid" : "invalid", + auth_sample.connection_generation_before, + auth_sample.connection_after_valid ? "valid" : "invalid", + auth_sample.connection_generation_after))); + } + continue; + } + last_auth_failure[node] = (uint8)PCM_X_SESSION_AUTH_OK; + if (!gcs_block_pcm_x_revalidate_peer_binding(node, current_epoch, + responder_session)) + continue; + result = cluster_pcm_x_master_terminal_leg_arm_exact( + ref, (PcmXTerminalLegKind)kind, node, responder_session, &token); + if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_NOT_READY) + continue; + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + if (kind == PCM_X_TERMINAL_LEG_DRAIN) { + memset(&poll, 0, sizeof(poll)); + poll.ref = *ref; + poll.drain_generation = token.drain_generation; + /* Even self DRAIN is application traffic. Dispatch it through + * the tag shard so holder-byte release and terminal publication + * remain one single-consumer action. */ + (void)cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_POLL, node, &poll, + sizeof(poll)); + return; + } else { + runtime = cluster_pcm_x_runtime_snapshot(); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = ref->identity.cluster_epoch; + retire.master_session_incarnation = runtime.master_session_incarnation; + retire.retire_through_ticket_id = ref->handle.ticket_id; + retire.sender_node = node; + if (node != cluster_node_id) { + (void)gcs_block_pcm_x_stage_retire_up_to(node, &retire, &ref->identity.tag); + return; + } + result = gcs_block_pcm_x_local_retire_apply_and_wake(&retire, cluster_node_id, + responder_session); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + result = cluster_pcm_x_master_retire_ack_resolve_exact( + &retire, cluster_node_id, responder_session, &resolved); + } + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return; + /* A local leg completed synchronously; restart from detach/next node. */ + break; + } + if (node < PCM_X_PROTOCOL_NODE_LIMIT) + break; + } + if (kind > PCM_X_TERMINAL_LEG_RETIRE) + return; + } +} + + +static void +gcs_block_pcm_x_master_drive_retry_tick(void) +{ + static Size cursor = 0; + BufferTag tag; + uint64 cluster_epoch; + + if (cluster_pcm_x_master_drive_work_next(&cursor, PCM_X_MASTER_DRIVE_SCAN_BUDGET, &tag, + &cluster_epoch) + == PCM_X_QUEUE_OK) + gcs_block_pcm_x_master_drive_tag(&tag, cluster_epoch); +} + + +static void +gcs_block_pcm_x_terminal_retry_tick(void) +{ + PcmXTicketRef ref; + PcmXQueueResult result; + + result = cluster_pcm_x_master_terminal_work_next(&ref); + if (result == PCM_X_QUEUE_OK) + cluster_gcs_pcm_x_terminal_kick(&ref); +} + + +static const ClusterICMsgTypeInfo pcm_x_enqueue_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_ENQUEUE, + .name = "pcm_x_enqueue", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_enqueue_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_admit_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_ADMIT_ACK, + .name = "pcm_x_admit_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_admit_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_admit_confirm_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, + .name = "pcm_x_admit_confirm", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_admit_confirm_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_admit_confirm_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, + .name = "pcm_x_admit_confirm_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_admit_confirm_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_blocker_set_begin_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_BEGIN, + .name = "pcm_x_blocker_set_begin", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_blocker_set_begin_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_blocker_set_edge_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_EDGE, + .name = "pcm_x_blocker_set_edge", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_blocker_set_edge_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_blocker_set_commit_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + .name = "pcm_x_blocker_set_commit", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_blocker_set_commit_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_blocker_set_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, + .name = "pcm_x_blocker_set_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_blocker_set_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_revoke_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_REVOKE, + .name = "pcm_x_revoke", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_revoke_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_image_ready_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_IMAGE_READY, + .name = "pcm_x_image_ready", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_image_ready_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_prepare_grant_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + .name = "pcm_x_prepare_grant", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_prepare_grant_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_install_ready_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_INSTALL_READY, + .name = "pcm_x_install_ready", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_install_ready_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_commit_x_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_COMMIT_X, + .name = "pcm_x_commit_x", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_commit_x_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; - send_sf_dep = sf_peer_v2 && sf_dep_valid && block_payload != NULL - && (hdr->status == (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER - || hdr->status == (uint8)GCS_BLOCK_REPLY_X_GRANTED_FROM_HOLDER - || hdr->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER); - header_len - = send_sf_dep ? (uint32)sizeof(GcsBlockReplyHeaderV2) : (uint32)sizeof(GcsBlockReplyHeader); - total = header_len + GCS_BLOCK_DATA_SIZE; - if (send_sf_dep) { - GcsBlockReplyHeaderV2 *hdrv2 = (GcsBlockReplyHeaderV2 *)buf; - int i; - int n = 0; +static const ClusterICMsgTypeInfo pcm_x_final_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_FINAL_ACK, + .name = "pcm_x_final_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_final_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; - hdrv2->sf_flags = GCS_BLOCK_REPLY_SF_EARLY_TRANSFER | GCS_BLOCK_REPLY_SF_HAS_DEP_VEC; - for (i = 0; i < CLUSTER_SF_DEP_MAX_ORIGINS; i++) { - if (XLogRecPtrIsInvalid(sf_dep_vec.required[i])) - continue; - hdrv2->sf_dep[n].origin_node = i; - hdrv2->sf_dep[n].required_redo_lsn = (uint64)sf_dep_vec.required[i]; - n++; - } - hdrv2->sf_dep_count = (uint8)n; - } +static const ClusterICMsgTypeInfo pcm_x_final_commit_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, + .name = "pcm_x_final_commit_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_final_commit_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; - if (GcsBlockForwardPayloadIsDirectLandArmed(fwd)) { - (void)gcs_block_try_send_direct_reply(fwd->original_requester_node, true, hdr, - holder_ship_ok ? block_payload : NULL, - holder_ship_ok ? block_payload_lkey : 0, - block_payload_release_cb, block_payload_release_arg); - if (hdr->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER) - cluster_xp_end(&xp_fwd_ship); - block_payload_release_cb = NULL; - block_payload_release_arg = NULL; - pfree(buf); - return; - } +static const ClusterICMsgTypeInfo pcm_x_final_confirm_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, + .name = "pcm_x_final_confirm", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_final_confirm_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; - if (holder_ship_ok && block_payload != NULL - && (hdr->status == (uint8)GCS_BLOCK_REPLY_GRANTED_FROM_HOLDER - || hdr->status == (uint8)GCS_BLOCK_REPLY_X_GRANTED_FROM_HOLDER - || hdr->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER - /* PGRAC: spec-6.12a ㉕ — the downgraded durable S grant ships page - * bytes exactly like the statuses above; leaving it off this list - * would send the palloc0 zero pad under a real-page checksum - * (guaranteed verify failure at the requester). */ - || hdr->status == (uint8)GCS_BLOCK_REPLY_S_GRANTED_XHOLDER_DOWNGRADE)) { - ClusterICSge sge[2]; - ClusterICSendResult send_rc; - bool live_sge_payload = block_payload_release_cb == gcs_block_release_live_sge; +static const ClusterICMsgTypeInfo pcm_x_prehandle_cancel_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL, + .name = "pcm_x_prehandle_cancel", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_prehandle_cancel_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; - memset(sge, 0, sizeof(sge)); - sge[0].addr = hdr; - sge[0].len = header_len; - sge[1].addr = (void *)block_payload; - sge[1].len = GCS_BLOCK_DATA_SIZE; - sge[1].lkey = block_payload_lkey; - sge[1].release_cb = block_payload_release_cb; - sge[1].release_arg = block_payload_release_arg; - send_rc = cluster_ic_rdma_send_envelope_sge( - PGRAC_IC_MSG_GCS_BLOCK_REPLY, fwd->original_requester_node, sge, lengthof(sge), total); - cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, send_rc); - if (send_rc == CLUSTER_IC_SEND_DONE && live_sge_payload) - gcs_block_note_live_sge_send(); - block_payload_release_cb = NULL; - block_payload_release_arg = NULL; - } else { - gcs_block_release_ship_image(block_payload_release_cb, block_payload_release_arg); - block_payload_release_cb = NULL; - block_payload_release_arg = NULL; - cluster_gcs_block_note_send_outcome(GCS_BLOCK_SEND_FAMILY_REPLY, - cluster_ic_send_envelope(PGRAC_IC_MSG_GCS_BLOCK_REPLY, - fwd->original_requester_node, - buf, total)); - } - /* PGRAC: spec-5.59 D3 — close the holder-forward read-image ship scope - * (no-op unless the read-image branch above started it). */ - cluster_xp_end(&xp_fwd_ship); +static const ClusterICMsgTypeInfo pcm_x_prehandle_cancel_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK, + .name = "pcm_x_prehandle_cancel_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_prehandle_cancel_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; - pfree(buf); -} +static const ClusterICMsgTypeInfo pcm_x_cancel_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_CANCEL, + .name = "pcm_x_cancel", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_cancel_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; +static const ClusterICMsgTypeInfo pcm_x_cancel_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_CANCEL_ACK, + .name = "pcm_x_cancel_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_cancel_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; -/* ============================================================ - * Dispatch table registration. - * - * PGRAC: spec-7.2 flip — the five block-family msg_types (REQUEST / - * REPLY / FORWARD / INVALIDATE / INVALIDATE_ACK) are registered on - * the DATA plane: the LMS-owned tier1 instance carries their frames, - * the LMS loop dispatches them, and the producer mask admits the LMS - * family for the drain-and-send leg. All five flip in this one edit - * (H-5: no half-migrated window; the registry probe above pivots the - * LMON tick sites and the LMS loop automatically). REDECLARE alone - * stays on the CONTROL plane (r4): recovery re-declare must survive a - * DATA-mesh teardown mid-episode, and the REDECLARE -> REDECLARE_DONE - * pair may not be split across planes. - * ============================================================ */ +static const ClusterICMsgTypeInfo pcm_x_drain_poll_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_DRAIN_POLL, + .name = "pcm_x_drain_poll", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_drain_poll_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_drain_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_DRAIN_ACK, + .name = "pcm_x_drain_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_drain_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_retire_up_to_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_RETIRE_UP_TO, + .name = "pcm_x_retire_up_to", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_retire_up_to_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; + +static const ClusterICMsgTypeInfo pcm_x_retire_up_to_ack_info = { + .msg_type = PGRAC_IC_MSG_PCM_X_RETIRE_ACK, + .name = "pcm_x_retire_ack", + .allowed_producer_mask + = CLUSTER_IC_PRODUCER_BUFFER_CLIENTS | CLUSTER_IC_PRODUCER_LMON | CLUSTER_IC_PRODUCER_LMS_DATA, + .broadcast_ok = false, + .handler = cluster_gcs_handle_pcm_x_retire_up_to_ack_envelope, + .plane = CLUSTER_IC_PLANE_DATA, +}; static const ClusterICMsgTypeInfo gcs_block_request_info = { .msg_type = PGRAC_IC_MSG_GCS_BLOCK_REQUEST, @@ -7653,6 +12895,7 @@ bool cluster_gcs_block_local_x_upgrade_ext(BufferTag tag, bool *out_busy) { GcsBlockRequestPayload synth; + PcmPendingXReserveResult reserve_result; uint32 holders_bm; uint32 self_bit; int n; @@ -7665,7 +12908,13 @@ cluster_gcs_block_local_x_upgrade_ext(BufferTag tag, bool *out_busy) return false; self_bit = (uint32)1u << cluster_node_id; - cluster_pcm_lock_set_pending_x(tag, cluster_node_id, (uint64)GetXLogInsertRecPtr()); + reserve_result + = cluster_pcm_lock_set_pending_x(tag, cluster_node_id, (uint64)GetXLogInsertRecPtr()); + if (reserve_result != PCM_PENDING_X_RESERVE_OK) { + if (out_busy != NULL && reserve_result == PCM_PENDING_X_RESERVE_OCCUPIED) + *out_busy = true; + return false; + } /* pending_x is armed from here on: readers are being PENDING_X-denied. * A throw anywhere below (cancel in the claim wait, an armed :error @@ -7814,14 +13063,18 @@ static bool gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) { GcsBlockInvalidateAckPayload ack; + ClusterPcmOwnResult own_result; + PcmXQueueResult queue_result; PcmLockMode pre_state; XLogRecPtr page_lsn = InvalidXLogRecPtr; SCN page_scn = InvalidScn; /* spec-2.41 D3 — ACK SCN carrier */ - uint8 ack_status = 0; /* OK */ - bool kept_pi = false; /* spec-6.12h D-h2 — drop converted to a PI */ + uint64 page_scn_raw = 0; + uint64 master_session = 0; + uint8 ack_status = 0; /* OK */ + bool kept_pi = false; /* spec-6.12h D-h2 — drop converted to a PI */ uint64 current_epoch = cluster_epoch_get_current(); - if (inv->epoch < current_epoch) { + if (inv->epoch != current_epoch) { ack_status = 1; /* epoch_stale */ goto send_ack; } @@ -7913,6 +13166,44 @@ gcs_block_invalidate_execute(const GcsBlockInvalidatePayload *inv) ack_status = 2; /* race: not resident */ break; case CLUSTER_BUFMGR_GCS_DROP_PINNED: + /* A generation-less type-17 may bypass the physical drop only while + * the local convert engine proves it names the exact frozen non-source + * participant whose blocker set this master already ACKed. This keeps + * old/legacy INVALIDATEs on their original pin-intolerant path. */ + if (pre_state == PCM_LOCK_MODE_S && cluster_gcs_lookup_master(inv->tag) == inv->master_node + && gcs_block_pcm_x_authenticated_session(inv->master_node, current_epoch, + &master_session) + && gcs_block_pcm_x_revalidate_peer_binding(inv->master_node, current_epoch, + master_session)) { + queue_result = cluster_pcm_x_local_queue_invalidate_authorize_exact( + &inv->tag, inv->epoch, inv->request_id, (int32)inv->invalidating_for_x_node, + inv->master_node, master_session); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result == PCM_X_QUEUE_OK) { + own_result = cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(&inv->tag, &page_lsn, + &page_scn_raw); + if (own_result == CLUSTER_PCM_OWN_OK) { + page_scn = (SCN)page_scn_raw; + (void)cluster_pcm_lock_apply_gcs_transition( + inv->tag, PCM_TRANS_S_TO_N_INVALIDATE, cluster_node_id); + if (ClusterGcsBlock != NULL) + pg_atomic_fetch_add_u64( + &ClusterGcsBlock->invalidate_passive_s_release_count, 1); + break; + } + if (own_result == CLUSTER_PCM_OWN_CORRUPT || own_result == CLUSTER_PCM_OWN_EXHAUSTED + || own_result == CLUSTER_PCM_OWN_INVALID) { + cluster_pcm_x_runtime_fail_closed(); + return false; + } + } else if (queue_result == PCM_X_QUEUE_CORRUPT + || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || queue_result == PCM_X_QUEUE_INVALID) { + gcs_block_pcm_x_master_drive_fail_closed(queue_result); + return false; + } + } + /* FALLTHROUGH */ case CLUSTER_BUFMGR_GCS_DROP_STALE: /* GCS serve-stall round-5 (A2): nothing changed, no ACK — the * caller parks the directive and the LMS loop retries it. STALE is @@ -8092,6 +13383,8 @@ static void cluster_gcs_handle_block_invalidate_envelope(const ClusterICEnvelope *env, const void *payload) { const GcsBlockInvalidatePayload *inv = (const GcsBlockInvalidatePayload *)payload; + uint64 current_epoch; + uint64 source_session; /* D16 inject — stall ack for timeout testing. */ CLUSTER_INJECTION_POINT("cluster-gcs-block-invalidate-stall-ack"); @@ -8114,6 +13407,13 @@ cluster_gcs_handle_block_invalidate_envelope(const ClusterICEnvelope *env, const cluster_gcs_block_dedup_note_misroute(); return; } + current_epoch = cluster_epoch_get_current(); + if (inv->epoch != current_epoch || cluster_gcs_lookup_master(inv->tag) != inv->master_node + || !gcs_block_pcm_x_authenticated_session(inv->master_node, current_epoch, + &source_session)) { + cluster_gcs_block_dedup_note_misroute(); + return; + } /* * PGRAC: spec-7.3 D5 (review P2-1) — per-worker shard routing guard, @@ -8175,15 +13475,48 @@ cluster_gcs_handle_block_invalidate_envelope(const ClusterICEnvelope *env, const * (stale rejected silently). On valid ack, sets the sender's bit * in invalidate_broadcast_acked_bm and broadcasts the CV. * ============================================================ */ +static PcmXQueueResult +gcs_block_pcm_x_queue_invalidate_ack_match(const GcsBlockInvalidateAckPayload *ack, + int32 source_node, PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXQueueResult result; + uint64 current_epoch; + uint64 source_session; + + if (ack == NULL || snapshot_out == NULL) + return PCM_X_QUEUE_INVALID; + memset(snapshot_out, 0, sizeof(*snapshot_out)); + current_epoch = cluster_epoch_get_current(); + if (cluster_gcs_lookup_master(ack->tag) != cluster_node_id) + return PCM_X_QUEUE_NOT_FOUND; + result = cluster_pcm_x_master_drive_snapshot_exact(&ack->tag, current_epoch, snapshot_out); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_CORRUPT ? result : PCM_X_QUEUE_NOT_FOUND; + result = cluster_gcs_pcm_x_invalidate_ack_match_exact(snapshot_out, ack, current_epoch, + source_node); + if (result == PCM_X_QUEUE_NOT_FOUND) + return result; + if (!gcs_block_pcm_x_authenticated_session(source_node, current_epoch, &source_session) + || !gcs_block_pcm_x_revalidate_peer_binding(source_node, current_epoch, source_session)) + return PCM_X_QUEUE_STALE; + return result; +} + + static void cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, const void *payload) { const GcsBlockInvalidateAckPayload *ack = (const GcsBlockInvalidateAckPayload *)payload; + PcmXMasterDriveSnapshot queue_snapshot; + PcmXMasterDriveSnapshot queue_updated; + PcmXQueueResult queue_result = PCM_X_QUEUE_NOT_FOUND; uint64 current_req_id; uint64 expected_epoch; uint32 expected_bm; SCN ack_page_scn = InvalidScn; /* spec-2.41 D3 — ACK now carries pd_block_scn */ BufferTag ack_tag = { 0 }; + bool queue_authority_applied = false; + bool queue_positive = false; bool valid = false; if (ClusterGcsBlock == NULL) @@ -8261,6 +13594,20 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c return; } + queue_result = gcs_block_pcm_x_queue_invalidate_ack_match(ack, (int32)env->source_node_id, + &queue_snapshot); + if (queue_result != PCM_X_QUEUE_NOT_FOUND) { + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result == PCM_X_QUEUE_OK) + queue_positive = true; + else if (queue_result == PCM_X_QUEUE_BAD_STATE || queue_result == PCM_X_QUEUE_CORRUPT + || queue_result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || queue_result == PCM_X_QUEUE_INVALID) + gcs_block_pcm_x_master_drive_fail_closed(queue_result); + if (!queue_positive) + return; + } + /* * PGRAC ownership-generation wave (ruling ②) — RETRYABLE_BUSY(5), * solicited negative ACK. Diverted BEFORE the slotless S-bit clear (a @@ -8313,8 +13660,12 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c */ if ((ack->ack_status == 0 || ack->ack_status == 2) && ack->epoch == cluster_epoch_get_current() && ack->sender_node >= 0 && ack->sender_node < 32) { - (void)cluster_pcm_lock_apply_gcs_transition(ack->tag, PCM_TRANS_S_TO_N_INVALIDATE, - ack->sender_node); + queue_authority_applied = cluster_pcm_lock_apply_gcs_transition( + ack->tag, PCM_TRANS_S_TO_N_INVALIDATE, ack->sender_node); + if (queue_positive && !queue_authority_applied) { + gcs_block_pcm_x_master_drive_fail_closed(PCM_X_QUEUE_BAD_STATE); + return; + } /* PGRAC: spec-6.12h D-h2 — the holder reported its dropped copy was * kept as a Past Image (D-h1); record it on the PI holder bitmap so * the discard protocol can target it later. Runs here (before the @@ -8341,6 +13692,19 @@ cluster_gcs_handle_block_invalidate_ack_envelope(const ClusterICEnvelope *env, c } } } + if (queue_positive) { + uint32 sender_bit = UINT32_C(1) << (uint32)ack->sender_node; + + queue_result = cluster_pcm_x_master_drive_bitmap_replace_exact( + &queue_snapshot, queue_snapshot.pending_s_holders_bitmap, + queue_snapshot.acked_s_holders_bitmap | sender_bit, &queue_updated); + cluster_pcm_x_stats_note_queue_result(queue_result); + if (queue_result == PCM_X_QUEUE_OK || queue_result == PCM_X_QUEUE_DUPLICATE) + gcs_block_pcm_x_master_drive_tag(&ack->tag, ack->epoch); + else + gcs_block_pcm_x_master_drive_fail_closed(queue_result); + return; + } LWLockAcquire(&ClusterGcsBlock->invalidate_broadcast_lock.lock, LW_EXCLUSIVE); current_req_id = pg_atomic_read_u64(&ClusterGcsBlock->invalidate_broadcast_request_id); @@ -8530,6 +13894,30 @@ static const ClusterICMsgTypeInfo gcs_block_redeclare_info = { void cluster_gcs_register_block_msg_types(void) { + cluster_ic_register_msg_type(&pcm_x_enqueue_info); + cluster_ic_register_msg_type(&pcm_x_admit_ack_info); + cluster_ic_register_msg_type(&pcm_x_admit_confirm_info); + cluster_ic_register_msg_type(&pcm_x_admit_confirm_ack_info); + cluster_ic_register_msg_type(&pcm_x_blocker_set_begin_info); + cluster_ic_register_msg_type(&pcm_x_blocker_set_edge_info); + cluster_ic_register_msg_type(&pcm_x_blocker_set_commit_info); + cluster_ic_register_msg_type(&pcm_x_blocker_set_ack_info); + cluster_ic_register_msg_type(&pcm_x_revoke_info); + cluster_ic_register_msg_type(&pcm_x_image_ready_info); + cluster_ic_register_msg_type(&pcm_x_prepare_grant_info); + cluster_ic_register_msg_type(&pcm_x_install_ready_info); + cluster_ic_register_msg_type(&pcm_x_commit_x_info); + cluster_ic_register_msg_type(&pcm_x_final_ack_info); + cluster_ic_register_msg_type(&pcm_x_final_commit_ack_info); + cluster_ic_register_msg_type(&pcm_x_final_confirm_info); + cluster_ic_register_msg_type(&pcm_x_prehandle_cancel_info); + cluster_ic_register_msg_type(&pcm_x_prehandle_cancel_ack_info); + cluster_ic_register_msg_type(&pcm_x_cancel_info); + cluster_ic_register_msg_type(&pcm_x_cancel_ack_info); + cluster_ic_register_msg_type(&pcm_x_drain_poll_info); + cluster_ic_register_msg_type(&pcm_x_drain_ack_info); + cluster_ic_register_msg_type(&pcm_x_retire_up_to_info); + cluster_ic_register_msg_type(&pcm_x_retire_up_to_ack_info); cluster_ic_register_msg_type(&gcs_block_request_info); cluster_ic_register_msg_type(&gcs_block_reply_info); cluster_ic_register_msg_type(&gcs_block_done_info); @@ -8757,6 +14145,27 @@ cluster_gcs_get_invalidate_busy_received_count(void) : 0; } +uint64 +cluster_gcs_get_invalidate_passive_s_release_count(void) +{ + return ClusterGcsBlock + ? pg_atomic_read_u64(&ClusterGcsBlock->invalidate_passive_s_release_count) + : 0; +} + +uint64 +cluster_gcs_get_pcm_x_self_handoff_count(void) +{ + return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->pcm_x_self_handoff_count) : 0; +} + +uint64 +cluster_gcs_get_pcm_x_self_handoff_drain_count(void) +{ + return ClusterGcsBlock ? pg_atomic_read_u64(&ClusterGcsBlock->pcm_x_self_handoff_drain_count) + : 0; +} + /* PGRAC: GCS serve-stall round-5 A2 — 4 bounded-drop accessors. */ uint64 cluster_gcs_get_invalidate_parked_count(void) @@ -9182,6 +14591,30 @@ cluster_gcs_get_block_dedup_legacy_pin_count(void) return cluster_gcs_block_dedup_get_legacy_pin_count(); } +uint64 +cluster_gcs_get_block_dedup_pcm_x_stage_count(void) +{ + return cluster_gcs_block_dedup_get_pcm_x_stage_count(); +} + +uint64 +cluster_gcs_get_block_dedup_pcm_x_replay_count(void) +{ + return cluster_gcs_block_dedup_get_pcm_x_replay_count(); +} + +uint64 +cluster_gcs_get_block_dedup_pcm_x_release_count(void) +{ + return cluster_gcs_block_dedup_get_pcm_x_release_count(); +} + +uint64 +cluster_gcs_get_block_dedup_pcm_x_failclosed_count(void) +{ + return cluster_gcs_block_dedup_get_pcm_x_failclosed_count(); +} + /* ============================================================ * PGRAC: spec-2.34 D4 — eager wake on epoch advance. @@ -9217,7 +14650,8 @@ cluster_gcs_block_on_epoch_advance(uint64 new_epoch) for (j = 0; j < MAX_OUTSTANDING_BLOCK_REQUESTS_PER_BACKEND; j++) { ClusterGcsBlockOutstandingSlot *slot = &blk->slots[j]; - if (slot->in_use && slot->request_epoch != 0 && slot->request_epoch < new_epoch + if (cluster_gcs_block_epoch_advance_stales_slot(slot->in_use, slot->request_epoch, + new_epoch) && !slot->stale) { slot->stale = true; ConditionVariableBroadcast(&slot->reply_cv); diff --git a/src/backend/cluster/cluster_gcs_block_dedup.c b/src/backend/cluster/cluster_gcs_block_dedup.c index b165c6e14c3..eb687b0b8b3 100644 --- a/src/backend/cluster/cluster_gcs_block_dedup.c +++ b/src/backend/cluster/cluster_gcs_block_dedup.c @@ -55,6 +55,8 @@ #include "cluster/cluster_shmem.h" #include "miscadmin.h" #include "port/atomics.h" +#include "port/pg_crc32c.h" +#include "storage/bufpage.h" #include "storage/ipc.h" #include "storage/lwlock.h" #include "storage/shmem.h" @@ -83,9 +85,13 @@ typedef struct ClusterGcsBlockDedupShard { pg_atomic_uint64 done_marked_count; /* identity-verified DONE stamped */ pg_atomic_uint64 done_mismatch_count; /* DONE dropped: miss / identity / in-flight */ /* GCS-race round-2 review F5 (calibration 2): registration routing. */ - pg_atomic_uint64 hint_violation_count; /* capable peer, hint 0 / over-max: denied */ - pg_atomic_uint64 legacy_pin_count; /* no-capability peer: protocol-max pin */ - pg_atomic_uint32 entry_count; /* live in-flight + completed entries */ + pg_atomic_uint64 hint_violation_count; /* capable peer, hint 0 / over-max: denied */ + pg_atomic_uint64 legacy_pin_count; /* no-capability peer: protocol-max pin */ + pg_atomic_uint64 pcm_x_stage_count; /* RESERVED -> immutable image */ + pg_atomic_uint64 pcm_x_replay_count; /* exact image replay */ + pg_atomic_uint64 pcm_x_release_count; /* exact terminal release */ + pg_atomic_uint64 pcm_x_failclosed_count; /* malformed, stale, or full image leg */ + pg_atomic_uint32 entry_count; /* live in-flight + completed entries */ } ClusterGcsBlockDedupShard; typedef struct ClusterGcsBlockDedupCtl { @@ -102,6 +108,18 @@ static ClusterGcsBlockDedupShard *cluster_gcs_block_dedup_shards = NULL; static HTAB *cluster_gcs_block_dedup_htabs[CLUSTER_LMS_MAX_WORKERS]; static int cluster_gcs_block_dedup_n_shards = 0; static bool dedup_backend_exit_hook_registered = false; +/* Process-local LMS cursors. Each DATA worker owns exactly one shard in one + * process, so this adds no shared-memory region or cross-worker authority. */ +static GcsBlockDedupKey dedup_pcm_x_work_cursor[CLUSTER_LMS_MAX_WORKERS]; +static bool dedup_pcm_x_work_cursor_valid[CLUSTER_LMS_MAX_WORKERS]; +/* When both classes stay runnable, alternate the single-work LMS tick budget. + * The initial false value preserves RESERVED-first admission while bounding a + * READY replay behind at most one reservation tick. */ +static bool dedup_pcm_x_prefer_ready_next[CLUSTER_LMS_MAX_WORKERS]; +/* Process-local wake hint. An empty scan clears it; exact reserve/rearm sets + * it. This keeps ordinary GCS traffic from rescanning every 8KB entry on + * every LMS tick when no PCM-X image work exists. */ +static bool dedup_pcm_x_work_pending[CLUSTER_LMS_MAX_WORKERS]; /* * Upper bound on entries examined per cap-full eager-reclaim probe. We do @@ -243,6 +261,12 @@ cluster_gcs_block_dedup_shmem_init(void) cluster_gcs_block_dedup_shards = (ClusterGcsBlockDedupShard *)(base + MAXALIGN(sizeof(ClusterGcsBlockDedupCtl))); cluster_gcs_block_dedup_n_shards = n; + memset(dedup_pcm_x_work_cursor, 0, sizeof(dedup_pcm_x_work_cursor)); + memset(dedup_pcm_x_work_cursor_valid, 0, sizeof(dedup_pcm_x_work_cursor_valid)); + memset(dedup_pcm_x_prefer_ready_next, 0, sizeof(dedup_pcm_x_prefer_ready_next)); + memset(dedup_pcm_x_work_pending, 0, sizeof(dedup_pcm_x_work_pending)); + for (i = 0; i < n; i++) + dedup_pcm_x_work_pending[i] = true; if (!found) { pg_atomic_init_u64(&cluster_gcs_block_dedup_ctl->misroute_failclosed_count, 0); @@ -266,6 +290,10 @@ cluster_gcs_block_dedup_shmem_init(void) pg_atomic_init_u64(&shard->done_mismatch_count, 0); pg_atomic_init_u64(&shard->hint_violation_count, 0); pg_atomic_init_u64(&shard->legacy_pin_count, 0); + pg_atomic_init_u64(&shard->pcm_x_stage_count, 0); + pg_atomic_init_u64(&shard->pcm_x_replay_count, 0); + pg_atomic_init_u64(&shard->pcm_x_release_count, 0); + pg_atomic_init_u64(&shard->pcm_x_failclosed_count, 0); pg_atomic_init_u32(&shard->entry_count, 0); } } @@ -358,6 +386,142 @@ cluster_gcs_block_dedup_register_backend_exit_hook(void) } +/* ============================================================ + * PCM-X image-entry validation. + * ============================================================ */ + +static void +dedup_pcm_x_note_failclosed(ClusterGcsBlockDedupShard *shard) +{ + if (shard != NULL) + pg_atomic_fetch_add_u64(&shard->pcm_x_failclosed_count, 1); +} + +static uint32 +dedup_pcm_x_block_checksum(const char *block_data) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, block_data, GCS_BLOCK_DATA_SIZE); + FIN_CRC32C(crc); + return (uint32)crc; +} + +static bool +dedup_pcm_x_key_valid(const GcsBlockDedupKey *key) +{ + return key != NULL && key->origin_node_id < PCM_X_PROTOCOL_NODE_LIMIT + && key->requester_backend_id > 0 + && cluster_pcm_x_image_id_decode(key->request_id, NULL, NULL); +} + +static bool +dedup_pcm_x_binding_valid(const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, bool reserved) +{ + const PcmXTicketRef *ref; + const PcmXImageToken *image; + int32 requester_node; + int32 requester_backend_id; + + if (!dedup_pcm_x_key_valid(key) || tag == NULL || binding == NULL + || binding->master_session == 0) + return false; + + ref = &binding->identity.ref; + image = &binding->identity.image; + if (memcmp(&ref->identity.tag, tag, sizeof(*tag)) != 0 + || ref->identity.node_id != (int32)key->origin_node_id + || ref->identity.cluster_epoch != key->cluster_epoch || ref->identity.wait_seq == 0 + || ref->handle.ticket_id == 0 || ref->handle.queue_generation == 0 + || ref->grant_generation == 0 || image->image_id != key->request_id + || image->source_node >= PCM_X_PROTOCOL_NODE_LIMIT || cluster_node_id < 0 + || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || image->source_node != (uint32)cluster_node_id) + return false; + + if (!cluster_gcs_requester_id_decode(ref->identity.request_id, &requester_node, + &requester_backend_id, NULL) + || requester_node != ref->identity.node_id + || requester_backend_id != key->requester_backend_id) + return false; + + if (reserved && (image->page_scn != 0 || image->page_lsn != 0 || image->page_checksum != 0)) + return false; + return true; +} + +static bool +dedup_pcm_x_reservation_equal(const GcsBlockDedupEntry *entry, + const GcsBlockPcmXImageBinding *binding) +{ + const GcsBlockPcmXImageIdentity *stored = &entry->payload_meta.pcm_x_identity; + const GcsBlockPcmXImageIdentity *incoming = &binding->identity; + + return entry->pcm_x_master_session == binding->master_session + && memcmp(&stored->ref, &incoming->ref, sizeof(stored->ref)) == 0 + && stored->image.image_id == incoming->image.image_id + && stored->image.source_own_generation == incoming->image.source_own_generation + && stored->image.source_node == incoming->image.source_node; +} + +static bool +dedup_pcm_x_ready_payload_valid(const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, + const GcsBlockReplyHeader *reply_header, const char *block_data) +{ + const PcmXImageToken *image; + PageHeaderData page_header; + static const uint8 zero_reserved[sizeof(reply_header->reserved_0)] = { 0 }; + + if (!dedup_pcm_x_binding_valid(key, tag, binding, false) || reply_header == NULL + || block_data == NULL) + return false; + + image = &binding->identity.image; + if (reply_header->request_id != key->request_id || reply_header->page_lsn != image->page_lsn + || reply_header->epoch != key->cluster_epoch + || reply_header->checksum != image->page_checksum + || reply_header->sender_node != (int32)image->source_node + || reply_header->requester_backend_id != key->requester_backend_id + || reply_header->transition_id != (uint8)PCM_TRANS_N_TO_S + || reply_header->status != (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || GcsBlockReplyHeaderGetForwardingMasterNode(reply_header) + != GCS_BLOCK_REPLY_NO_FORWARDING_MASTER + || memcmp(reply_header->reserved_0, zero_reserved, sizeof(zero_reserved)) != 0) + return false; + + memcpy(&page_header, block_data, sizeof(page_header)); + return image->page_lsn == (uint64)PageXLogRecPtrGet(page_header.pd_lsn) + && image->page_scn == (uint64)page_header.pd_block_scn + && image->page_checksum == dedup_pcm_x_block_checksum(block_data); +} + +static bool +dedup_pcm_x_entry_payload_valid(const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockDedupEntry *entry) +{ + GcsBlockPcmXImageBinding binding; + + binding.identity = entry->payload_meta.pcm_x_identity; + binding.master_session = entry->pcm_x_master_session; + return entry->transition_id == (uint8)PCM_TRANS_N_TO_S + && entry->status == (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + && dedup_pcm_x_ready_payload_valid(key, tag, &binding, &entry->reply_header, + entry->block_data); +} + + +static bool +dedup_pcm_x_entry_ready_valid(const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockDedupEntry *entry) +{ + return entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && dedup_pcm_x_entry_payload_valid(key, tag, entry); +} + + /* ============================================================ * Public API. * ============================================================ */ @@ -382,12 +546,22 @@ cluster_gcs_block_dedup_lookup_or_register(int worker_id, const GcsBlockDedupKey shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); if (shard == NULL) return GCS_BLOCK_DEDUP_FULL; /* not initialized / mis-route; fail closed */ + if (cluster_pcm_x_image_id_decode(key->request_id, NULL, NULL)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_DEDUP_VALIDATION_FAIL; + } LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); if (found) { + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_DEDUP_VALIDATION_FAIL; + } + /* HC91 — entry value collision check */ if (memcmp(&entry->tag, &tag, sizeof(BufferTag)) != 0 || entry->transition_id != transition_id) { @@ -480,6 +654,7 @@ cluster_gcs_block_dedup_lookup_or_register(int worker_id, const GcsBlockDedupKey entry->tag = tag; entry->transition_id = transition_id; entry->status = 0; + entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_GENERIC; entry->completed_at_ts = 0; entry->registered_at_ts = GetCurrentTimestamp(); @@ -514,6 +689,660 @@ cluster_gcs_block_dedup_lookup_or_register(int worker_id, const GcsBlockDedupKey return result; } +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *reserved_binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, reserved_binding, true)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (found) { + if ((entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) + && entry->transition_id == (uint8)PCM_TRANS_N_TO_S + && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 + && dedup_pcm_x_reservation_equal(entry, reserved_binding)) { + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_ENTER_NULL, &found); + if (entry == NULL + && dedup_reclaim_reclaimable_locked(shard, htab, GetCurrentTimestamp(), 1) > 0) + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_ENTER_NULL, &found); + if (entry == NULL || found) { + pg_atomic_fetch_add_u64(&shard->full_count, 1); + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_FULL; + } + + memset(((char *)entry) + sizeof(GcsBlockDedupKey), 0, + sizeof(GcsBlockDedupEntry) - sizeof(GcsBlockDedupKey)); + entry->tag = *tag; + entry->transition_id = (uint8)PCM_TRANS_N_TO_S; + entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED; + entry->pcm_x_master_session = reserved_binding->master_session; + entry->payload_meta.pcm_x_identity = reserved_binding->identity; + entry->registered_at_ts = GetCurrentTimestamp(); + pg_atomic_fetch_add_u32(&shard->entry_count, 1); + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_RESERVED; +} + +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_materialize(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding, + const GcsBlockReplyHeader *reply_header, + const char *block_data) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_ready_payload_valid(key, tag, ready_binding, reply_header, block_data)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + || entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { + GcsBlockPcmXImageBinding stored_binding; + + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (entry->transition_id == (uint8)PCM_TRANS_N_TO_S + && memcmp(&entry->tag, tag, sizeof(*tag)) == 0 + && GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) + && dedup_pcm_x_entry_payload_valid(key, tag, entry) + && memcmp(&entry->reply_header, reply_header, sizeof(*reply_header)) == 0 + && memcmp(entry->block_data, block_data, GCS_BLOCK_DATA_SIZE) == 0) { + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !dedup_pcm_x_reservation_equal(entry, ready_binding)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + + { + GcsBlockPcmXImageIdentity reserved_identity = entry->payload_meta.pcm_x_identity; + GcsBlockPcmXImageBinding stored_binding; + + entry->reply_header = *reply_header; + entry->status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; + entry->payload_meta.pcm_x_identity = ready_binding->identity; + memcpy(entry->block_data, block_data, GCS_BLOCK_DATA_SIZE); + entry->completed_at_ts = GetCurrentTimestamp(); + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (!dedup_pcm_x_ready_payload_valid(key, tag, &stored_binding, &entry->reply_header, + entry->block_data)) { + memset(&entry->reply_header, 0, sizeof(entry->reply_header)); + memset(entry->block_data, 0, GCS_BLOCK_DATA_SIZE); + entry->payload_meta.pcm_x_identity = reserved_identity; + entry->status = 0; + entry->completed_at_ts = 0; + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + } + entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED; + pg_atomic_fetch_add_u64(&shard->pcm_x_stage_count, 1); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STORED; +} + + +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, ready_binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) + || !dedup_pcm_x_entry_payload_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) { + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + + entry->entry_kind = GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE; + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STORED; +} + +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_lookup(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *expected_binding, + GcsBlockDedupEntry *cached_reply_out) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + bool binding_is_reserved; + + if (cached_reply_out != NULL) + memset(cached_reply_out, 0, sizeof(*cached_reply_out)); + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + binding_is_reserved = expected_binding != NULL && expected_binding->identity.image.page_scn == 0 + && expected_binding->identity.image.page_lsn == 0 + && expected_binding->identity.image.page_checksum == 0; + if (!dedup_pcm_x_binding_valid(key, tag, expected_binding, binding_is_reserved)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (!GcsBlockPcmXImageBindingEqual(&stored_binding, expected_binding)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) { + if (!dedup_pcm_x_entry_payload_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; + } + if (!dedup_pcm_x_entry_ready_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + + if (cached_reply_out != NULL) + *cached_reply_out = *entry; + pg_atomic_fetch_add_u64(&shard->pcm_x_replay_count, 1); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_REPLAY; +} + +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + bool binding_is_reserved; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + binding_is_reserved = binding != NULL && binding->identity.image.page_scn == 0 + && binding->identity.image.page_lsn == 0 + && binding->identity.image.page_checksum == 0; + if (!dedup_pcm_x_binding_valid(key, tag, binding, binding_is_reserved)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED) + || !GcsBlockPcmXImageBindingEqual(&stored_binding, binding)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + + (void)hash_search(htab, key, HASH_REMOVE, &found); + Assert(found); + pg_atomic_fetch_sub_u32(&shard->entry_count, 1); + pg_atomic_fetch_add_u64(&shard->pcm_x_release_count, 1); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_RELEASED; +} + + +static void +dedup_pcm_x_copy_work(const GcsBlockDedupEntry *entry, GcsBlockPcmXImageWork *work) +{ + memset(work, 0, sizeof(*work)); + work->key = entry->key; + work->binding.identity = entry->payload_meta.pcm_x_identity; + work->binding.master_session = entry->pcm_x_master_session; + work->tag = entry->tag; + work->entry_kind = entry->entry_kind; +} + + +static int +dedup_pcm_x_key_compare(const GcsBlockDedupKey *left, const GcsBlockDedupKey *right) +{ + if (left->origin_node_id != right->origin_node_id) + return left->origin_node_id < right->origin_node_id ? -1 : 1; + if (left->requester_backend_id != right->requester_backend_id) + return left->requester_backend_id < right->requester_backend_id ? -1 : 1; + if (left->request_id != right->request_id) + return left->request_id < right->request_id ? -1 : 1; + if (left->cluster_epoch != right->cluster_epoch) + return left->cluster_epoch < right->cluster_epoch ? -1 : 1; + return 0; +} + + +static void +dedup_pcm_x_consider_work(const GcsBlockDedupEntry *entry, const GcsBlockDedupKey *cursor, + bool cursor_valid, GcsBlockPcmXImageWork *after, bool *have_after, + GcsBlockPcmXImageWork *wrap, bool *have_wrap) +{ + bool is_after = cursor_valid && dedup_pcm_x_key_compare(&entry->key, cursor) > 0; + + if (is_after && (!*have_after || dedup_pcm_x_key_compare(&entry->key, &after->key) < 0)) { + dedup_pcm_x_copy_work(entry, after); + *have_after = true; + } + if (!*have_wrap || dedup_pcm_x_key_compare(&entry->key, &wrap->key) < 0) { + dedup_pcm_x_copy_work(entry, wrap); + *have_wrap = true; + } +} + + +/* Return at most one immutable work token per LMS tick. RESERVED wins the + * first mixed-class tick, then mixed-class selections alternate so neither + * fresh byte materialization nor READY replay can starve. Within each class + * a process-local exact-key cursor prevents one pinned holder from + * monopolizing every tick. */ +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *work_out) +{ + ClusterGcsBlockDedupShard *shard; + GcsBlockPcmXImageWork ready_after; + GcsBlockPcmXImageWork ready_wrap; + GcsBlockPcmXImageWork reserved_after; + GcsBlockPcmXImageWork reserved_wrap; + HTAB *htab = NULL; + HASH_SEQ_STATUS scan; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageResult result = GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + bool have_ready_after = false; + bool have_ready_wrap = false; + bool have_reserved_after = false; + bool have_reserved_wrap = false; + + if (work_out != NULL) + memset(work_out, 0, sizeof(*work_out)); + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (work_out == NULL) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + if (!dedup_pcm_x_work_pending[worker_id]) + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + hash_seq_init(&scan, htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) + continue; + /* A READY entry admitted to the outbound ring sleeps until an exact + * type-49 retransmit clears this marker. */ + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE && entry->done_at_ts != 0) + continue; + binding.identity = entry->payload_meta.pcm_x_identity; + binding.master_session = entry->pcm_x_master_session; + if (entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + && !dedup_pcm_x_binding_valid(&entry->key, &entry->tag, &binding, true)) + || (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + && !dedup_pcm_x_entry_ready_valid(&entry->key, &entry->tag, entry))) { + dedup_pcm_x_note_failclosed(shard); + result = GCS_BLOCK_PCM_X_IMAGE_INVALID; + break; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED) { + dedup_pcm_x_consider_work(entry, &dedup_pcm_x_work_cursor[worker_id], + dedup_pcm_x_work_cursor_valid[worker_id], &reserved_after, + &have_reserved_after, &reserved_wrap, &have_reserved_wrap); + } else + dedup_pcm_x_consider_work(entry, &dedup_pcm_x_work_cursor[worker_id], + dedup_pcm_x_work_cursor_valid[worker_id], &ready_after, + &have_ready_after, &ready_wrap, &have_ready_wrap); + } + /* hash_seq_search() terminates a naturally exhausted scan itself. Only + * the validation-failure break above leaves an open scan to close here. */ + if (result != GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND) + hash_seq_term(&scan); + if (result == GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND) { + bool have_ready = have_ready_after || have_ready_wrap; + bool have_reserved = have_reserved_after || have_reserved_wrap; + bool choose_ready = have_ready && have_reserved && dedup_pcm_x_prefer_ready_next[worker_id]; + + if (have_reserved && !choose_ready) { + *work_out = have_reserved_after ? reserved_after : reserved_wrap; + result = GCS_BLOCK_PCM_X_IMAGE_RESERVED; + } else if (have_ready) { + *work_out = have_ready_after ? ready_after : ready_wrap; + result = GCS_BLOCK_PCM_X_IMAGE_REPLAY; + } + if (result != GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND) { + dedup_pcm_x_work_cursor[worker_id] = work_out->key; + dedup_pcm_x_work_cursor_valid[worker_id] = true; + if (have_ready && have_reserved) + dedup_pcm_x_prefer_ready_next[worker_id] = !choose_ready; + } + } + if (result == GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND || result == GCS_BLOCK_PCM_X_IMAGE_INVALID) + dedup_pcm_x_work_pending[worker_id] = false; + LWLockRelease(&shard->lock.lock); + return result; +} + + +/* Mark only outbound-ring admission, not application completion. The exact + * DRAIN_POLL consumer remains the sole owner of byte release. */ +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_mark_staged_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, ready_binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) + || !dedup_pcm_x_entry_ready_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->done_at_ts != 0) { + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + entry->done_at_ts = GetCurrentTimestamp(); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STAGED; +} + + +/* Roll back only a failed outbound-ring admission. The complete READY + * binding is required because the reservation-shaped type-49 rearm API has a + * different authority: it is remote retransmit evidence, not a local enqueue + * transaction. */ +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + GcsBlockPcmXImageBinding stored_binding; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, ready_binding, false)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + stored_binding.identity = entry->payload_meta.pcm_x_identity; + stored_binding.master_session = entry->pcm_x_master_session; + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !GcsBlockPcmXImageBindingEqual(&stored_binding, ready_binding) + || !dedup_pcm_x_entry_ready_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->done_at_ts == 0) { + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + entry->done_at_ts = 0; + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_REARMED; +} + + +/* A byte-exact type-49 retransmit means the master has not applied type 50. + * It may re-open only the matching READY outbound marker; a still-RESERVED + * entry already appears in the normal work scan and needs no state change. */ +GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_rearm_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *reserved_binding) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + GcsBlockDedupEntry *entry; + bool found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return GCS_BLOCK_PCM_X_IMAGE_FULL; + if (!dedup_pcm_x_binding_valid(key, tag, reserved_binding, true)) { + dedup_pcm_x_note_failclosed(shard); + return GCS_BLOCK_PCM_X_IMAGE_INVALID; + } + LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (!found) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND; + } + if ((entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED + && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE) + || entry->transition_id != (uint8)PCM_TRANS_N_TO_S + || memcmp(&entry->tag, tag, sizeof(*tag)) != 0 + || !dedup_pcm_x_reservation_equal(entry, reserved_binding)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED) { + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_NOT_READY; + } + if (!dedup_pcm_x_entry_ready_valid(key, tag, entry)) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_STALE; + } + if (entry->done_at_ts == 0) { + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_DUPLICATE; + } + entry->done_at_ts = 0; + dedup_pcm_x_work_pending[worker_id] = true; + LWLockRelease(&shard->lock.lock); + return GCS_BLOCK_PCM_X_IMAGE_REARMED; +} + + +/* A newly forked LMS has no proof about which instruction the previous owner + * completed. Any dedicated entry is therefore retained recovery evidence; + * the caller transitions the PCM-X runtime to RECOVERY_BLOCKED. */ +bool +cluster_gcs_block_dedup_pcm_x_restart_audit(int worker_id) +{ + ClusterGcsBlockDedupShard *shard; + HTAB *htab = NULL; + HASH_SEQ_STATUS scan; + GcsBlockDedupEntry *entry; + bool evidence_found = false; + + shard = cluster_gcs_block_dedup_resolve_shard(worker_id, &htab); + if (shard == NULL) + return true; + + LWLockAcquire(&shard->lock.lock, LW_SHARED); + hash_seq_init(&scan, htab); + while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&scan)) != NULL) { + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) { + evidence_found = true; + hash_seq_term(&scan); + break; + } + } + if (evidence_found) + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return evidence_found; +} + /* * cluster_gcs_block_dedup_mark_done — GCS-race round-2 RC-F: consume a * requester completion proof. Full identity verification + COMPLETED @@ -539,14 +1368,18 @@ cluster_gcs_block_dedup_mark_done(int worker_id, const GcsBlockDedupKey *key, co LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); - if (found && memcmp(&entry->tag, tag, sizeof(BufferTag)) == 0 - && entry->transition_id == transition_id && entry->completed_at_ts != 0) { + if (found && entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && memcmp(&entry->tag, tag, sizeof(BufferTag)) == 0 && entry->transition_id == transition_id + && entry->completed_at_ts != 0) { if (entry->done_at_ts == 0) entry->done_at_ts = GetCurrentTimestamp(); stamped = true; /* duplicate DONE re-stamps nothing: idempotent */ pg_atomic_fetch_add_u64(&shard->done_marked_count, 1); - } else + } else { + if (found && entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + dedup_pcm_x_note_failclosed(shard); pg_atomic_fetch_add_u64(&shard->done_mismatch_count, 1); + } LWLockRelease(&shard->lock.lock); return stamped; } @@ -624,6 +1457,11 @@ cluster_gcs_block_dedup_install_reply_ex(int worker_id, const GcsBlockDedupKey * LWLockRelease(&shard->lock.lock); return; } + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) { + dedup_pcm_x_note_failclosed(shard); + LWLockRelease(&shard->lock.lock); + return; + } entry->status = (uint8)status; entry->reply_header = *header; @@ -631,14 +1469,14 @@ cluster_gcs_block_dedup_install_reply_ex(int worker_id, const GcsBlockDedupKey * entry->sf_flags = has_sf_dep ? (GCS_BLOCK_REPLY_SF_HAS_DEP_VEC | GCS_BLOCK_REPLY_SF_EARLY_TRANSFER) : 0; entry->sf_dep_count = 0; - cluster_sf_dep_vec_reset(&entry->sf_dep_vec); + cluster_sf_dep_vec_reset(&entry->payload_meta.sf_dep_vec); if (has_sf_dep && sf_dep_vec != NULL) { int i; for (i = 0; i < CLUSTER_SF_DEP_MAX_ORIGINS; i++) { if (XLogRecPtrIsInvalid(sf_dep_vec->required[i])) continue; - entry->sf_dep_vec.required[i] = sf_dep_vec->required[i]; + entry->payload_meta.sf_dep_vec.required[i] = sf_dep_vec->required[i]; entry->sf_dep_count++; } } @@ -668,6 +1506,7 @@ cluster_gcs_block_dedup_remove(int worker_id, const GcsBlockDedupKey *key) { ClusterGcsBlockDedupShard *shard; HTAB *htab = NULL; + GcsBlockDedupEntry *entry; bool found; Assert(key != NULL); @@ -677,9 +1516,13 @@ cluster_gcs_block_dedup_remove(int worker_id, const GcsBlockDedupKey *key) return; LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); - (void)hash_search(htab, key, HASH_REMOVE, &found); - if (found) + entry = (GcsBlockDedupEntry *)hash_search(htab, key, HASH_FIND, &found); + if (found && entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC) { + (void)hash_search(htab, key, HASH_REMOVE, &found); + Assert(found); pg_atomic_fetch_sub_u32(&shard->entry_count, 1); + } else if (found) + dedup_pcm_x_note_failclosed(shard); LWLockRelease(&shard->lock.lock); } @@ -837,6 +1680,9 @@ cluster_gcs_block_dedup_sweep_expired(TimestampTz now) int64 age_us; int64 deadline_us; + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + continue; + /* * GCS-race round-2 RC-F: per-entry pinned deadlines. A * DONE-proven entry only lingers its pinned quarantine (reorder @@ -897,7 +1743,8 @@ cluster_gcs_block_dedup_cleanup_on_backend_exit(uint32 origin_node_id, int32 bac LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); hash_seq_init(&seq, htab); while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&seq)) != NULL) { - if (entry->key.origin_node_id == origin_node_id + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && entry->key.origin_node_id == origin_node_id && entry->key.requester_backend_id == backend_id) { (void)hash_search(htab, &entry->key, HASH_REMOVE, NULL); removed++; @@ -930,7 +1777,8 @@ cluster_gcs_block_dedup_cleanup_on_node_dead(uint32 node_id) LWLockAcquire(&shard->lock.lock, LW_EXCLUSIVE); hash_seq_init(&seq, htab); while ((entry = (GcsBlockDedupEntry *)hash_seq_search(&seq)) != NULL) { - if (entry->key.origin_node_id == node_id) { + if (entry->entry_kind == GCS_BLOCK_DEDUP_ENTRY_GENERIC + && entry->key.origin_node_id == node_id) { (void)hash_search(htab, &entry->key, HASH_REMOVE, NULL); removed++; } @@ -1034,6 +1882,32 @@ cluster_gcs_block_dedup_get_legacy_pin_count(void) return cluster_gcs_block_dedup_sum_u64(offsetof(ClusterGcsBlockDedupShard, legacy_pin_count)); } +uint64 +cluster_gcs_block_dedup_get_pcm_x_stage_count(void) +{ + return cluster_gcs_block_dedup_sum_u64(offsetof(ClusterGcsBlockDedupShard, pcm_x_stage_count)); +} + +uint64 +cluster_gcs_block_dedup_get_pcm_x_replay_count(void) +{ + return cluster_gcs_block_dedup_sum_u64(offsetof(ClusterGcsBlockDedupShard, pcm_x_replay_count)); +} + +uint64 +cluster_gcs_block_dedup_get_pcm_x_release_count(void) +{ + return cluster_gcs_block_dedup_sum_u64( + offsetof(ClusterGcsBlockDedupShard, pcm_x_release_count)); +} + +uint64 +cluster_gcs_block_dedup_get_pcm_x_failclosed_count(void) +{ + return cluster_gcs_block_dedup_sum_u64( + offsetof(ClusterGcsBlockDedupShard, pcm_x_failclosed_count)); +} + /* * cluster_gcs_block_dedup_get_max_entries -- effective PER-SHARD dedup * capacity. diff --git a/src/backend/cluster/cluster_gcs_block_shard.c b/src/backend/cluster/cluster_gcs_block_shard.c index 7d6ef8bb4e9..c4e69918d0f 100644 --- a/src/backend/cluster/cluster_gcs_block_shard.c +++ b/src/backend/cluster/cluster_gcs_block_shard.c @@ -31,6 +31,7 @@ #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_lms_shard.h" +#include "cluster/cluster_pcm_x_convert.h" #include "storage/buf_internals.h" #ifdef USE_PGRAC_CLUSTER @@ -39,11 +40,10 @@ * cluster_gcs_block_payload_shard — spec-7.3 D4 (8.A). * * Pick the DATA worker for a staged block-family frame by hashing its - * BufferTag. Only the four tag-carrying staging-path types reach the - * outbound ring (REQUEST / FORWARD / INVALIDATE / DONE — each with the tag at a - * fixed offset); REPLY (no tag, request_id-correlated) and INVALIDATE-ACK - * are sent DIRECTLY from the receiving worker's dispatch handler, so they - * already ride the correct worker channel and never reach this function. + * BufferTag. The legacy GCS staging types carry the tag at a fixed offset; + * every staged PCM-X conversion payload starts with the tag through its + * identity/ref carrier. Tagless replies and PCM-X RETIRE frames are sent + * directly by the receiving worker and never reach this function. * * Returns the worker id in [0, n_workers), or -1 if the (msg_type, payload) * pair carries no routable tag. -1 is an 8.A fail-closed signal: an @@ -63,12 +63,22 @@ StaticAssertDecl(offsetof(GcsBlockInvalidatePayload, tag) == 16, "spec-7.3 D4: GcsBlockInvalidatePayload.tag offset moved"); StaticAssertDecl(offsetof(GcsBlockDonePayload, tag) == 16, "GCS-race round-2 review F4: GcsBlockDonePayload.tag offset moved"); +StaticAssertDecl(offsetof(PcmXWaitIdentity, tag) == 0, "PCM-X wait tag must lead payloads"); +StaticAssertDecl(offsetof(PcmXTicketRef, identity) == 0, + "PCM-X ticket identity must lead payloads"); +StaticAssertDecl(offsetof(PcmXEnqueuePayload, identity) == 0, + "PCM-X enqueue tag carrier must lead payload"); +StaticAssertDecl(offsetof(PcmXAdmitAckPayload, ref) == 0, "PCM-X ref carrier must lead payload"); +StaticAssertDecl(offsetof(PcmXBlockerChunkPayload, tag) == 0, + "PCM-X blocker tag must lead payload"); int cluster_gcs_block_payload_shard(uint8 msg_type, const void *payload, uint16 payload_len, int n_workers) { const BufferTag *tag; + BufferTag pcm_x_tag; + uint16 pcm_x_expected_len = 0; if (payload == NULL) return -1; @@ -99,11 +109,61 @@ cluster_gcs_block_payload_shard(uint8 msg_type, const void *payload, uint16 payl return -1; tag = &((const GcsBlockDonePayload *)payload)->tag; break; + case PGRAC_IC_MSG_PCM_X_ENQUEUE: + pcm_x_expected_len = sizeof(PcmXEnqueuePayload); + break; + case PGRAC_IC_MSG_PCM_X_ADMIT_ACK: + case PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK: + pcm_x_expected_len = sizeof(PcmXAdmitAckPayload); + break; + case PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM: + case PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK: + case PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK: + case PGRAC_IC_MSG_PCM_X_COMMIT_X: + case PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK: + case PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM: + case PGRAC_IC_MSG_PCM_X_CANCEL: + case PGRAC_IC_MSG_PCM_X_CANCEL_ACK: + case PGRAC_IC_MSG_PCM_X_DRAIN_ACK: + pcm_x_expected_len = sizeof(PcmXPhasePayload); + break; + case PGRAC_IC_MSG_PCM_X_BLOCKER_SET_BEGIN: + case PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT: + pcm_x_expected_len = sizeof(PcmXBlockerSetHeaderPayload); + break; + case PGRAC_IC_MSG_PCM_X_BLOCKER_SET_EDGE: + pcm_x_expected_len = sizeof(PcmXBlockerChunkPayload); + break; + case PGRAC_IC_MSG_PCM_X_REVOKE: + pcm_x_expected_len = sizeof(PcmXRevokePayload); + break; + case PGRAC_IC_MSG_PCM_X_IMAGE_READY: + case PGRAC_IC_MSG_PCM_X_PREPARE_GRANT: + pcm_x_expected_len = sizeof(PcmXGrantPayload); + break; + case PGRAC_IC_MSG_PCM_X_INSTALL_READY: + pcm_x_expected_len = sizeof(PcmXInstallReadyPayload); + break; + case PGRAC_IC_MSG_PCM_X_FINAL_ACK: + pcm_x_expected_len = sizeof(PcmXFinalAckPayload); + break; + case PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL: + pcm_x_expected_len = sizeof(PcmXPrehandleCancelPayload); + break; + case PGRAC_IC_MSG_PCM_X_DRAIN_POLL: + pcm_x_expected_len = sizeof(PcmXDrainPollPayload); + break; default: - /* REPLY / INVALIDATE-ACK are direct-sent, not staged; any other - * DATA type would need an explicit shard key (spec-7.3 §3.6). */ + /* Tagless replies are direct-sent, not staged; any other DATA type + * needs an explicit shard key before it may enter an outbound ring. */ return -1; } + if (pcm_x_expected_len != 0) { + if (payload_len != pcm_x_expected_len) + return -1; + memcpy(&pcm_x_tag, payload, sizeof(pcm_x_tag)); + tag = &pcm_x_tag; + } return cluster_lms_shard_for_tag(tag, n_workers); } diff --git a/src/backend/cluster/cluster_ic.c b/src/backend/cluster/cluster_ic.c index 1a86edc4f7b..738681173e6 100644 --- a/src/backend/cluster/cluster_ic.c +++ b/src/backend/cluster/cluster_ic.c @@ -660,6 +660,9 @@ cluster_ic_build_hello(uint8 out_buf[PGRAC_IC_HELLO_BYTES], uint16 hello_version * sentinel protocol capability, same unconditional discipline (see * cluster_ic.h). */ capabilities |= PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1; + /* PCM-X conversion is a protocol capability. Every opcode has a distinct + * registered DATA-plane type, so mixed-version senders gate on this bit. */ + capabilities |= PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1; if (capabilities != 0) ic_le_write_uint32(out_buf + PGRAC_IC_HELLO_CAPABILITIES_OFFSET, capabilities); diff --git a/src/backend/cluster/cluster_lmd_graph.c b/src/backend/cluster/cluster_lmd_graph.c index 0da5f9f8b49..b2fb18176a5 100644 --- a/src/backend/cluster/cluster_lmd_graph.c +++ b/src/backend/cluster/cluster_lmd_graph.c @@ -53,6 +53,7 @@ #include "storage/lwlock.h" #include "storage/shmem.h" #include "utils/hsearch.h" +#include "utils/palloc.h" /* ============================================================ @@ -148,8 +149,26 @@ typedef struct ClusterLmdGraphShared { * never honored, so it cannot wrongly clear / escalate the live victim. */ pg_atomic_uint64 cancel_ack_mismatch_count; int max_edges; /* snapshot of cluster.lmd_max_wait_edges at init */ + /* Appended connector accounting; existing field offsets stay unchanged. */ + pg_atomic_uint64 pcm_convert_wfg_replace_count; + pg_atomic_uint64 pcm_convert_wfg_remove_count; + pg_atomic_uint64 pcm_convert_wfg_replace_fail_count; + pg_atomic_uint64 pcm_convert_wfg_exact_remove_stale_count; } ClusterLmdGraphShared; +/* Pin the old tail and the appended connector counters byte-for-byte. */ +StaticAssertDecl(offsetof(ClusterLmdGraphShared, max_edges) == 296, + "LMD graph max_edges offset ABI"); +StaticAssertDecl(offsetof(ClusterLmdGraphShared, pcm_convert_wfg_replace_count) == 304, + "LMD graph PCM-X replace counter offset ABI"); +StaticAssertDecl(offsetof(ClusterLmdGraphShared, pcm_convert_wfg_remove_count) == 312, + "LMD graph PCM-X remove counter offset ABI"); +StaticAssertDecl(offsetof(ClusterLmdGraphShared, pcm_convert_wfg_replace_fail_count) == 320, + "LMD graph PCM-X replace-fail counter offset ABI"); +StaticAssertDecl(offsetof(ClusterLmdGraphShared, pcm_convert_wfg_exact_remove_stale_count) == 328, + "LMD graph PCM-X exact-remove-stale counter offset ABI"); +StaticAssertDecl(sizeof(ClusterLmdGraphShared) == 336, "LMD graph shared-memory ABI"); + static ClusterLmdGraphShared *cluster_lmd_graph_state = NULL; static HTAB *cluster_lmd_graph_htab = NULL; @@ -162,6 +181,15 @@ static void fill_vertex_key(const ClusterLmdVertex *v, LmdVertexKey *out); static void make_key(const ClusterLmdVertex *waiter, const ClusterLmdVertex *blocker, LmdEdgeKey *out); static bool key_waiter_matches(const LmdEdgeKey *key, const ClusterLmdVertex *waiter); +static bool vertex_identity_equal(const ClusterLmdVertex *a, const ClusterLmdVertex *b); +static int vertex_identity_qsort_cmp(const void *a, const void *b); +static bool vertex_metadata_equal(const ClusterLmdVertex *a, const ClusterLmdVertex *b); +static void restore_replaced_waiter_set(const ClusterLmdVertex *waiter, + const ClusterLmdVertex *new_blockers, int inserted_count, + const ClusterLmdWaitEdge *old_edges, int removed_count, + uint64 edge_count_before, uint64 generation_before); + +#define LMD_REMOVE_BATCH 32 /* ============================================================ @@ -210,6 +238,10 @@ cluster_lmd_graph_shmem_init(void) pg_atomic_init_u64(&cluster_lmd_graph_state->victim_cancel_sent_count, 0); pg_atomic_init_u64(&cluster_lmd_graph_state->revalidate_fail_count, 0); pg_atomic_init_u64(&cluster_lmd_graph_state->cross_node_victim_pending_count, 0); + pg_atomic_init_u64(&cluster_lmd_graph_state->pcm_convert_wfg_replace_count, 0); + pg_atomic_init_u64(&cluster_lmd_graph_state->pcm_convert_wfg_remove_count, 0); + pg_atomic_init_u64(&cluster_lmd_graph_state->pcm_convert_wfg_replace_fail_count, 0); + pg_atomic_init_u64(&cluster_lmd_graph_state->pcm_convert_wfg_exact_remove_stale_count, 0); pg_atomic_init_u64(&cluster_lmd_graph_state->probe_broadcast_count, 0); pg_atomic_init_u64(&cluster_lmd_graph_state->probe_partial_count, 0); /* spec-2.24 D12 init. */ @@ -281,6 +313,80 @@ key_waiter_matches(const LmdEdgeKey *key, const ClusterLmdVertex *waiter) && key->waiter.request_id == waiter->request_id; } +static bool +vertex_identity_equal(const ClusterLmdVertex *a, const ClusterLmdVertex *b) +{ + return a->node_id == b->node_id && a->procno == b->procno + && a->cluster_epoch == b->cluster_epoch && a->request_id == b->request_id; +} + +static int +vertex_identity_qsort_cmp(const void *a, const void *b) +{ + const ClusterLmdVertex *va = (const ClusterLmdVertex *)a; + const ClusterLmdVertex *vb = (const ClusterLmdVertex *)b; + + if (va->node_id != vb->node_id) + return va->node_id < vb->node_id ? -1 : 1; + if (va->procno != vb->procno) + return va->procno < vb->procno ? -1 : 1; + if (va->cluster_epoch != vb->cluster_epoch) + return va->cluster_epoch < vb->cluster_epoch ? -1 : 1; + if (va->request_id != vb->request_id) + return va->request_id < vb->request_id ? -1 : 1; + return 0; +} + +static bool +vertex_metadata_equal(const ClusterLmdVertex *a, const ClusterLmdVertex *b) +{ + return a->xid == b->xid && a->local_start_ts_ms == b->local_start_ts_ms + && a->wait_seq == b->wait_seq; +} + +/* + * Roll back a replacement that hit the dynahash's non-throwing insertion + * guard after mutation began. The caller holds the graph lock EXCLUSIVE. + * Removing the inserted prefix recreates exactly removed_count free slots, + * so restoring that many saved old edges cannot normally fail. A failure in + * that second invariant means the shared graph itself is corrupt; continuing + * would let the deadlock detector act on a partial graph, so PANIC is the only + * fail-closed outcome. + */ +static void +restore_replaced_waiter_set(const ClusterLmdVertex *waiter, const ClusterLmdVertex *new_blockers, + int inserted_count, const ClusterLmdWaitEdge *old_edges, + int removed_count, uint64 edge_count_before, uint64 generation_before) +{ + int i; + + for (i = 0; i < inserted_count; i++) { + LmdEdgeKey key; + bool found; + + make_key(waiter, &new_blockers[i], &key); + (void)hash_search(cluster_lmd_graph_htab, &key, HASH_REMOVE, &found); + if (!found) + ereport(PANIC, (errmsg("could not roll back partial LMD blocker-set replacement"))); + } + + for (i = 0; i < removed_count; i++) { + LmdEdgeKey key; + LmdEdgeEntry *entry; + bool found; + + make_key(&old_edges[i].waiter, &old_edges[i].blocker, &key); + entry = (LmdEdgeEntry *)hash_search(cluster_lmd_graph_htab, &key, HASH_ENTER_NULL, &found); + if (entry == NULL || found) + ereport(PANIC, (errmsg("could not restore LMD blocker set after replacement failure"))); + entry->edge = old_edges[i]; + } + + /* No snapshot can have observed the intermediate values under EXCLUSIVE. */ + pg_atomic_write_u64(&cluster_lmd_graph_state->edge_count, edge_count_before); + pg_atomic_write_u64(&cluster_lmd_graph_state->generation, generation_before); +} + bool cluster_lmd_graph_add_edge(const ClusterLmdWaitEdge *edge) { @@ -335,6 +441,205 @@ cluster_lmd_graph_add_edge(const ClusterLmdWaitEdge *edge) return true; } +/* + * spec-2.36a C2 -- publish a waiter's complete blocker set atomically. + * + * The old cancel-then-submit loop exposes an empty or partial graph to a + * concurrent Tarjan snapshot and cannot roll back a prefix when the graph is + * full. This operation holds the graph LWLock across capacity reservation, + * removal, and insertion. Capacity is reserved against the FINAL cardinality + * before the first mutation, so a normal saturation failure leaves the prior + * set and graph generation untouched. + * + * The dynahash is fixed-size and the exclusive lock prevents competing + * inserts between the final-cardinality check and HASH_ENTER_NULL. A + * surprising insertion NULL is nevertheless handled: remove the inserted + * prefix, restore the saved old set and its count/generation, then return + * false. If that restoration invariant itself fails, PANIC prevents any + * detector from acting on a partial shared graph. The exact entry point + * returns the graph generation committed by this batch while still holding + * the graph lock; zero means no batch was published. The legacy boolean API + * remains a compatibility wrapper. + */ +uint64 +cluster_lmd_graph_replace_waiter_edges_exact(const ClusterLmdVertex *waiter, + const ClusterLmdVertex *blockers, int nblockers, + uint64 request_id) +{ + HASH_SEQ_STATUS scan; + LmdEdgeEntry *e; + ClusterLmdVertex *canonical = NULL; + ClusterLmdWaitEdge *old_edges; + uint64 cur_count; + uint64 committed_generation; + uint64 generation_before; + uint64 survivor_count; + int old_capacity; + int old_count; + int unique_count = 0; + int i; + + if (waiter == NULL || nblockers < 0 || (nblockers > 0 && blockers == NULL) + || cluster_lmd_graph_state == NULL || cluster_lmd_graph_htab == NULL + || request_id != waiter->request_id || nblockers > cluster_lmd_graph_state->max_edges) + return 0; + + /* + * Canonicalize outside the global graph lock. Equal identities may be + * folded only when victim-selection/ABA metadata is also exact; silently + * choosing one of two wait_seq values could cancel a reused backend wait. + */ + if (nblockers > 0) { + canonical = palloc_array(ClusterLmdVertex, nblockers); + memcpy(canonical, blockers, sizeof(ClusterLmdVertex) * nblockers); + qsort(canonical, nblockers, sizeof(ClusterLmdVertex), vertex_identity_qsort_cmp); + + for (i = 0; i < nblockers; i++) { + if (vertex_identity_equal(waiter, &canonical[i])) { + pfree(canonical); + return 0; + } + if (unique_count > 0 + && vertex_identity_equal(&canonical[unique_count - 1], &canonical[i])) { + if (!vertex_metadata_equal(&canonical[unique_count - 1], &canonical[i])) { + pfree(canonical); + return 0; + } + continue; + } + canonical[unique_count++] = canonical[i]; + } + } + + /* + * Save the old edge payloads both for O(old_count) deletion and for exact + * rollback if HASH_ENTER_NULL reports an invariant failure. Size the + * first pass from the replacement, then grow outside the LWLock if a + * waiter currently has a larger set. + */ + old_capacity = Max(LMD_REMOVE_BATCH, unique_count); + old_capacity = Min(old_capacity, cluster_lmd_graph_state->max_edges); + old_edges = palloc_array(ClusterLmdWaitEdge, old_capacity); + + for (;;) { + bool overflow = false; + + old_count = 0; + LWLockAcquire(&cluster_lmd_graph_state->lwlock, LW_EXCLUSIVE); + hash_seq_init(&scan, cluster_lmd_graph_htab); + while ((e = (LmdEdgeEntry *)hash_seq_search(&scan)) != NULL) { + if (!key_waiter_matches(&e->key, waiter)) + continue; + if (old_count == old_capacity) { + overflow = true; + hash_seq_term(&scan); + break; + } + old_edges[old_count++] = e->edge; + } + if (!overflow) + break; + + LWLockRelease(&cluster_lmd_graph_state->lwlock); + if (old_capacity == cluster_lmd_graph_state->max_edges) { + pfree(old_edges); + if (canonical != NULL) + pfree(canonical); + return 0; + } + old_capacity = Min(old_capacity * 2, cluster_lmd_graph_state->max_edges); + old_edges = repalloc_array(old_edges, ClusterLmdWaitEdge, old_capacity); + } + + cur_count = pg_atomic_read_u64(&cluster_lmd_graph_state->edge_count); + Assert((uint64)old_count <= cur_count); + if ((uint64)old_count > cur_count) { + LWLockRelease(&cluster_lmd_graph_state->lwlock); + pfree(old_edges); + if (canonical != NULL) + pfree(canonical); + return 0; + } + survivor_count = cur_count - (uint64)old_count; + if (survivor_count > (uint64)cluster_lmd_graph_state->max_edges + || (uint64)unique_count > (uint64)cluster_lmd_graph_state->max_edges - survivor_count) { + pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->wait_edge_full_count, 1); + LWLockRelease(&cluster_lmd_graph_state->lwlock); + pfree(old_edges); + if (canonical != NULL) + pfree(canonical); + return 0; + } + generation_before = pg_atomic_read_u64(&cluster_lmd_graph_state->generation); + + /* Delete the saved exact set in O(old_count), still under EXCLUSIVE. */ + for (i = 0; i < old_count; i++) { + LmdEdgeKey key; + bool found; + + make_key(&old_edges[i].waiter, &old_edges[i].blocker, &key); + (void)hash_search(cluster_lmd_graph_htab, &key, HASH_REMOVE, &found); + if (!found) { + restore_replaced_waiter_set(waiter, canonical, 0, old_edges, i, cur_count, + generation_before); + LWLockRelease(&cluster_lmd_graph_state->lwlock); + pfree(old_edges); + if (canonical != NULL) + pfree(canonical); + return 0; + } + pg_atomic_fetch_sub_u64(&cluster_lmd_graph_state->edge_count, 1); + pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->generation, 1); + } + + /* Insert the canonical set. A surprising NULL is rolled back exactly. */ + for (i = 0; i < unique_count; i++) { + ClusterLmdWaitEdge edge; + LmdEdgeKey key; + LmdEdgeEntry *entry; + bool found; + + memset(&edge, 0, sizeof(edge)); + edge.waiter = *waiter; + edge.blocker = canonical[i]; + edge.request_id = request_id; + make_key(waiter, &canonical[i], &key); + entry = (LmdEdgeEntry *)hash_search(cluster_lmd_graph_htab, &key, HASH_ENTER_NULL, &found); + if (entry == NULL) { + pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->wait_edge_full_count, 1); + restore_replaced_waiter_set(waiter, canonical, i, old_edges, old_count, cur_count, + generation_before); + LWLockRelease(&cluster_lmd_graph_state->lwlock); + pfree(old_edges); + if (canonical != NULL) + pfree(canonical); + return 0; + } + if (found) + ereport(PANIC, (errmsg("duplicate LMD edge appeared during atomic replacement"))); + entry->edge = edge; + entry->edge.graph_generation + = pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->generation, 1); + pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->edge_count, 1); + } + + committed_generation = pg_atomic_read_u64(&cluster_lmd_graph_state->generation); + LWLockRelease(&cluster_lmd_graph_state->lwlock); + pfree(old_edges); + if (canonical != NULL) + pfree(canonical); + return committed_generation; +} + +bool +cluster_lmd_graph_replace_waiter_edges(const ClusterLmdVertex *waiter, + const ClusterLmdVertex *blockers, int nblockers, + uint64 request_id) +{ + return cluster_lmd_graph_replace_waiter_edges_exact(waiter, blockers, nblockers, request_id) + != 0; +} + bool cluster_lmd_graph_has_waiter(const ClusterLmdVertex *waiter) { @@ -380,8 +685,6 @@ cluster_lmd_graph_has_waiter(const ClusterLmdVertex *waiter) * the generic graph. Held EXCLUSIVE across the whole operation so a * waiter's edges are removed atomically. */ -#define LMD_REMOVE_BATCH 32 - bool cluster_lmd_graph_remove_edge_by_waiter(const ClusterLmdVertex *waiter) { @@ -430,6 +733,88 @@ cluster_lmd_graph_remove_edge_by_waiter(const ClusterLmdVertex *waiter) return removed_any; } +/* + * PCM-X cancellation additionally binds removal to the backend wait_seq. The + * graph key intentionally remains the canonical waiter 4-tuple, so first + * prove that every matching edge still belongs to this exact wait instance; + * a stale cancel must leave a newer instance byte-stable. The exclusive lock + * makes ABSENT a stable observation relative to graph writers, rather than a + * false alias for STALE. + */ +ClusterLmdGraphRemoveResult +cluster_lmd_graph_remove_edge_by_waiter_exact_result(const ClusterLmdVertex *waiter) +{ + HASH_SEQ_STATUS proof_scan; + LmdEdgeEntry *entry; + bool matched_any = false; + bool removed_any = false; + + Assert(waiter != NULL); + if (waiter == NULL || waiter->wait_seq == 0) + return CLUSTER_LMD_GRAPH_REMOVE_STALE; + if (cluster_lmd_graph_state == NULL || cluster_lmd_graph_htab == NULL) + return CLUSTER_LMD_GRAPH_REMOVE_STALE; + + LWLockAcquire(&cluster_lmd_graph_state->lwlock, LW_EXCLUSIVE); + hash_seq_init(&proof_scan, cluster_lmd_graph_htab); + while ((entry = (LmdEdgeEntry *)hash_seq_search(&proof_scan)) != NULL) { + if (!key_waiter_matches(&entry->key, waiter)) + continue; + matched_any = true; + if (entry->edge.waiter.wait_seq != waiter->wait_seq) { + hash_seq_term(&proof_scan); + LWLockRelease(&cluster_lmd_graph_state->lwlock); + return CLUSTER_LMD_GRAPH_REMOVE_STALE; + } + } + if (!matched_any) { + LWLockRelease(&cluster_lmd_graph_state->lwlock); + return CLUSTER_LMD_GRAPH_REMOVE_ABSENT; + } + + for (;;) { + HASH_SEQ_STATUS scan; + LmdEdgeEntry *candidate; + LmdEdgeKey batch[LMD_REMOVE_BATCH]; + int nbatch = 0; + bool more = false; + int i; + + hash_seq_init(&scan, cluster_lmd_graph_htab); + while ((candidate = (LmdEdgeEntry *)hash_seq_search(&scan)) != NULL) { + if (!key_waiter_matches(&candidate->key, waiter)) + continue; + if (nbatch == LMD_REMOVE_BATCH) { + more = true; + hash_seq_term(&scan); + break; + } + batch[nbatch++] = candidate->key; + } + for (i = 0; i < nbatch; i++) { + bool found; + + (void)hash_search(cluster_lmd_graph_htab, &batch[i], HASH_REMOVE, &found); + if (found) { + pg_atomic_fetch_sub_u64(&cluster_lmd_graph_state->edge_count, 1); + pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->generation, 1); + removed_any = true; + } + } + if (!more) + break; + } + LWLockRelease(&cluster_lmd_graph_state->lwlock); + return removed_any ? CLUSTER_LMD_GRAPH_REMOVE_REMOVED : CLUSTER_LMD_GRAPH_REMOVE_STALE; +} + +bool +cluster_lmd_graph_remove_edge_by_waiter_exact(const ClusterLmdVertex *waiter) +{ + return cluster_lmd_graph_remove_edge_by_waiter_exact_result(waiter) + == CLUSTER_LMD_GRAPH_REMOVE_REMOVED; +} + int cluster_lmd_graph_snapshot_copy(ClusterLmdWaitEdge *out_buf, int max_edges, uint64 *out_gen_at_snapshot) @@ -488,6 +873,68 @@ cluster_lmd_wait_edge_full_count_get(void) return pg_atomic_read_u64(&cluster_lmd_graph_state->wait_edge_full_count); } +uint64 +cluster_lmd_pcm_convert_wfg_replace_count_get(void) +{ + if (cluster_lmd_graph_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_lmd_graph_state->pcm_convert_wfg_replace_count); +} + +uint64 +cluster_lmd_pcm_convert_wfg_remove_count_get(void) +{ + if (cluster_lmd_graph_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_lmd_graph_state->pcm_convert_wfg_remove_count); +} + +uint64 +cluster_lmd_pcm_convert_wfg_replace_fail_count_get(void) +{ + if (cluster_lmd_graph_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_lmd_graph_state->pcm_convert_wfg_replace_fail_count); +} + +uint64 +cluster_lmd_pcm_convert_wfg_exact_remove_stale_count_get(void) +{ + if (cluster_lmd_graph_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_lmd_graph_state->pcm_convert_wfg_exact_remove_stale_count); +} + +void +cluster_lmd_pcm_convert_wfg_note_replace(void) +{ + if (cluster_lmd_graph_state != NULL) + (void)pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->pcm_convert_wfg_replace_count, 1); +} + +void +cluster_lmd_pcm_convert_wfg_note_remove(void) +{ + if (cluster_lmd_graph_state != NULL) + (void)pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->pcm_convert_wfg_remove_count, 1); +} + +void +cluster_lmd_pcm_convert_wfg_note_replace_fail(void) +{ + if (cluster_lmd_graph_state != NULL) + (void)pg_atomic_fetch_add_u64(&cluster_lmd_graph_state->pcm_convert_wfg_replace_fail_count, + 1); +} + +void +cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(void) +{ + if (cluster_lmd_graph_state != NULL) + (void)pg_atomic_fetch_add_u64( + &cluster_lmd_graph_state->pcm_convert_wfg_exact_remove_stale_count, 1); +} + uint64 cluster_lmd_inject_call_count_get(void) { diff --git a/src/backend/cluster/cluster_lmd_wait_state.c b/src/backend/cluster/cluster_lmd_wait_state.c index 45b2f79e3f6..a7308d7afc2 100644 --- a/src/backend/cluster/cluster_lmd_wait_state.c +++ b/src/backend/cluster/cluster_lmd_wait_state.c @@ -29,11 +29,34 @@ #include "access/transam.h" #include "cluster/cluster_lmd_wait_state.h" +#define CLUSTER_LMD_WAIT_STATE_READ_RETRIES 16 + +static uint32 +cluster_lmd_wait_state_write_begin(ClusterLmdProcWaitState *ws) +{ + uint32 current; + uint32 odd; + + current = pg_atomic_read_u32(&ws->change_seq); + odd = current | 1U; + pg_atomic_write_u32(&ws->change_seq, odd); + pg_write_barrier(); + return odd; +} + +static void +cluster_lmd_wait_state_write_end(ClusterLmdProcWaitState *ws, uint32 odd) +{ + pg_write_barrier(); + pg_atomic_write_u32(&ws->change_seq, odd + 1U); +} + void cluster_lmd_wait_state_init(ClusterLmdProcWaitState *ws) { - pg_atomic_init_u32(&ws->active, 0); + pg_atomic_init_u32(&ws->change_seq, 0); pg_atomic_init_u64(&ws->wait_seq, 0); + ws->active = 0; ws->kind = CLUSTER_LMD_WAIT_NONE; ws->request_id = 0; ws->cluster_epoch = 0; @@ -43,13 +66,18 @@ cluster_lmd_wait_state_init(ClusterLmdProcWaitState *ws) void cluster_lmd_wait_state_reset(ClusterLmdProcWaitState *ws) { + uint32 odd; + /* * Backstop for a backend that reused this proc slot whose predecessor - * died mid-wait without clearing. Clear active only — wait_seq is left - * monotonically climbing so a stale victim tuple recorded for the - * predecessor can never re-match (procno reuse ABA guard). + * died mid-wait without clearing. Beginning from an already-odd sequence + * is deliberate: it closes a predecessor that died midway through a + * publish. wait_seq is left monotonically climbing so a stale victim + * tuple recorded for the predecessor can never re-match. */ - pg_atomic_write_u32(&ws->active, 0); + odd = cluster_lmd_wait_state_write_begin(ws); + ws->active = 0; + cluster_lmd_wait_state_write_end(ws, odd); } uint64 @@ -57,19 +85,16 @@ cluster_lmd_wait_state_publish(ClusterLmdProcWaitState *ws, uint8 kind, uint64 r uint64 cluster_epoch, TransactionId xid) { uint64 seq; + uint32 odd; - /* Write the data fields and advance wait_seq first, then publish via the - * active flag with a write barrier between. A reader that observes - * active == 1 (with a read barrier) sees this record, not a half-written - * one. */ + odd = cluster_lmd_wait_state_write_begin(ws); ws->kind = kind; ws->request_id = request_id; ws->cluster_epoch = cluster_epoch; ws->xid = xid; seq = pg_atomic_fetch_add_u64(&ws->wait_seq, 1) + 1; - - pg_write_barrier(); - pg_atomic_write_u32(&ws->active, 1); + ws->active = 1; + cluster_lmd_wait_state_write_end(ws, odd); return seq; } @@ -77,26 +102,55 @@ cluster_lmd_wait_state_publish(ClusterLmdProcWaitState *ws, uint8 kind, uint64 r void cluster_lmd_wait_state_clear(ClusterLmdProcWaitState *ws) { + uint32 odd; + /* The single funnel every wait-exit path (grant / timeout / ERROR / * cancel) calls. Leaves the data fields and wait_seq in place; active * is the authoritative "still waiting" signal. */ - pg_atomic_write_u32(&ws->active, 0); + odd = cluster_lmd_wait_state_write_begin(ws); + ws->active = 0; + cluster_lmd_wait_state_write_end(ws, odd); } -bool -cluster_lmd_wait_state_read(ClusterLmdProcWaitState *ws, ClusterLmdWaitStateSnapshot *out) +ClusterLmdWaitStateReadResult +cluster_lmd_wait_state_read_exact(ClusterLmdProcWaitState *ws, ClusterLmdWaitStateSnapshot *out) { + ClusterLmdWaitStateSnapshot snapshot; + uint32 before; + uint32 after; + int retry; + memset(out, 0, sizeof(*out)); - if (pg_atomic_read_u32(&ws->active) == 0) - return false; - - pg_read_barrier(); - out->active = true; - out->kind = ws->kind; - out->request_id = ws->request_id; - out->cluster_epoch = ws->cluster_epoch; - out->xid = ws->xid; - out->wait_seq = pg_atomic_read_u64(&ws->wait_seq); - return true; + for (retry = 0; retry < CLUSTER_LMD_WAIT_STATE_READ_RETRIES; retry++) { + before = pg_atomic_read_u32(&ws->change_seq); + if ((before & 1U) != 0) + continue; + + pg_read_barrier(); + snapshot.active = ws->active != 0; + snapshot.kind = ws->kind; + snapshot.request_id = ws->request_id; + snapshot.cluster_epoch = ws->cluster_epoch; + snapshot.xid = ws->xid; + snapshot.wait_seq = pg_atomic_read_u64(&ws->wait_seq); + pg_read_barrier(); + + after = pg_atomic_read_u32(&ws->change_seq); + if (before != after || (after & 1U) != 0) + continue; + if (!snapshot.active) + return CLUSTER_LMD_WAIT_STATE_READ_INACTIVE; + + *out = snapshot; + return CLUSTER_LMD_WAIT_STATE_READ_ACTIVE; + } + + return CLUSTER_LMD_WAIT_STATE_READ_BUSY; +} + +bool +cluster_lmd_wait_state_read(ClusterLmdProcWaitState *ws, ClusterLmdWaitStateSnapshot *out) +{ + return cluster_lmd_wait_state_read_exact(ws, out) == CLUSTER_LMD_WAIT_STATE_READ_ACTIVE; } diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 4d286b65751..f0d0e396993 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -1287,6 +1287,7 @@ LmonMain(void) * (event/epoch source) and AFTER dead_sweep (I47, P2). */ cluster_grd_recovery_lmon_tick(); + cluster_gcs_block_pcm_x_formation_tick(); /* * spec-2.9 D2 review fix: BOC_BROADCAST is triggered by @@ -1937,6 +1938,7 @@ LmonMain(void) cluster_reconfig_lmon_tick(); /* spec-4.6 D1: GRD recovery sequence (see main-loop site). */ cluster_grd_recovery_lmon_tick(); + cluster_gcs_block_pcm_x_formation_tick(); /* * PGRAC: spec-5.3 — drain the GRD work queue in the stub / diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index 930b00f7979..bebf5a5566a 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -747,6 +747,7 @@ LmsMain(void) * the new generation. */ cluster_lms_bump_restart_generation_at_main_entry(); (void)cluster_ges_dedup_drop_stale_entries(); + cluster_gcs_block_pcm_x_owner_start(0); /* PGRAC: spec-7.2 D2 — bring up the LMS-owned DATA-plane listener + * mesh (false = plane off for this node; the loop below then keeps @@ -817,14 +818,18 @@ LmsMain(void) * and a fenced node's reads must keep flowing exactly as * they do on the ungated CONTROL ring pre-flip; gating * them wedges post-crash rejoin behind its own catalog - * reads. Thaw wakes the latch and the held image legs - * resume (pure-read probe, no-throw, per the - * write_fence_allowed contract). */ + * reads. PCM-X types 50-56 are the exception: the ring + * drain retains those grant legs behind its final runtime + + * fence gate while unrelated read traffic keeps flowing. + * Thaw wakes the latch and the held image legs resume + * (pure-read probe, no-throw, per the write_fence_allowed + * contract). */ if (cluster_gcs_block_family_on_data_plane() && !(cluster_write_fence_enforcing() && !cluster_write_fence_allowed())) { cluster_lms_cr_ship_ready(); cluster_gcs_block_pi_discard_drain(); } + cluster_gcs_block_pcm_x_image_pump_tick(0); (void)cluster_lms_outbound_drain_send(0); /* spec-7.3 D4: worker 0's ring */ /* GCS serve-stall round-5 A2 — retry PINNED invalidate * directives parked by the dispatch handler (bounded, one @@ -922,6 +927,7 @@ LmsWorkerMain(int worker_id) ereport(LOG, (errmsg("cluster_lms: DATA-plane worker %d started (pid %d)", worker_id, (int)MyProcPid))); + cluster_gcs_block_pcm_x_owner_start(worker_id); /* * PGRAC: spec-7.3 D3 — bring up this worker's DATA-plane channel + mesh @@ -963,6 +969,7 @@ LmsWorkerMain(int worker_id) * the mesh. REPLY / INVALIDATE-ACK for blocks we received are * sent directly from the dispatch handler in THIS process, so * they already ride this worker's channel. */ + cluster_gcs_block_pcm_x_image_pump_tick(worker_id); (void)cluster_lms_outbound_drain_send(worker_id); /* GCS serve-stall round-5 A2 — retry PINNED invalidate * directives parked by this worker's dispatch handler. */ diff --git a/src/backend/cluster/cluster_lms_outbound.c b/src/backend/cluster/cluster_lms_outbound.c index c7359ef36d1..db9697fe548 100644 --- a/src/backend/cluster/cluster_lms_outbound.c +++ b/src/backend/cluster/cluster_lms_outbound.c @@ -43,6 +43,7 @@ #include "cluster/cluster_ic_router.h" #include "cluster/cluster_lms.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_write_fence.h" #include "miscadmin.h" #include "storage/lwlock.h" #include "storage/shmem.h" @@ -116,6 +117,24 @@ static const ClusterShmemRegion cluster_lms_outbound_region = { .reserved_flags = 0, }; + +/* Type 50 reports the holder's irreversible X->N handoff; types 51-56 drive + * the resulting X grant to completion. None may cross the final DATA + * transport boundary while the protocol runtime or node write authority is + * closed. Admission/cancel/drain traffic remains independently routable. */ +static bool +lms_outbound_pcm_x_grant_held(uint8 msg_type) +{ + PcmXRuntimeSnapshot runtime; + + if (msg_type < PGRAC_IC_MSG_PCM_X_IMAGE_READY || msg_type > PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM) + return false; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE) + return true; + return cluster_write_fence_enforcing() && !cluster_write_fence_allowed(); +} + void cluster_lms_outbound_shmem_register(void) { @@ -251,13 +270,47 @@ cluster_lms_outbound_drain_send(int worker_id) continue; } + /* Revalidate irreversible PCM-X grant authority immediately before + * transport admission. Retaining also blocks later same-peer frames, + * preserving the DATA FIFO while unrelated peers keep flowing. */ + if (lms_outbound_pcm_x_grant_held(slot.msg_type)) { + if (slot.dest_node_id < CLUSTER_MAX_NODES) + peer_blocked[slot.dest_node_id] = true; + Assert(n_retained < (int)lengthof(retained)); + retained[n_retained++] = slot; + continue; + } + if (slot.msg_type == PGRAC_IC_MSG_GCS_BLOCK_REQUEST && slot.payload_len == sizeof(GcsBlockRequestPayload)) cluster_gcs_block_lmon_prepare_outbound_request((GcsBlockRequestPayload *)slot.payload, (int32)slot.dest_node_id); - rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, - slot.payload_len > 0 ? slot.payload : NULL, slot.payload_len); + /* + * The generic IC send path deliberately treats dest=self as a no-op + * DONE. That is correct for transport diagnostics, but not for a + * staged DATA actor: PCM-X frequently maps a tag's resource master to + * the requesting node, and its ENQUEUE/ACK must still execute on the + * tag-owning LMS worker. Build the same envelope and dispatch it here, + * after dequeue and without the ring lock. A handler response is + * staged back onto this tag's ring, preserving the one-worker FIFO and + * avoiding recursive handler execution. + */ + if ((int32)slot.dest_node_id == cluster_node_id) { + ClusterICEnvelope env; + + if (cluster_ic_envelope_build( + &env, slot.msg_type, (uint32)cluster_node_id, slot.dest_node_id, + slot.payload_len > 0 ? slot.payload : NULL, slot.payload_len) + && cluster_ic_dispatch_envelope(&env, slot.payload_len > 0 ? slot.payload : NULL, + cluster_node_id)) + rc = CLUSTER_IC_SEND_DONE; + else + rc = CLUSTER_IC_SEND_HARD_ERROR; + } else + rc = cluster_ic_send_envelope(slot.msg_type, (int32)slot.dest_node_id, + slot.payload_len > 0 ? slot.payload : NULL, + slot.payload_len); switch (rc) { case CLUSTER_IC_SEND_DONE: case CLUSTER_IC_SEND_WOULD_BLOCK: diff --git a/src/backend/cluster/cluster_lock_acquire.c b/src/backend/cluster/cluster_lock_acquire.c index 450054f16c7..2db5d08881f 100644 --- a/src/backend/cluster/cluster_lock_acquire.c +++ b/src/backend/cluster/cluster_lock_acquire.c @@ -61,6 +61,7 @@ #include "cluster/cluster_lms.h" #include "cluster/cluster_lock_acquire.h" #include "cluster/cluster_native_lock_probe.h" +#include "cluster/cluster_pcm_x_convert.h" #include "cluster/cluster_cancel_token.h" /* spec-5.9 D3 cluster_cancel_token_consume */ #include "cluster/cluster_signal.h" /* cluster_ges_cancel_pending sig_atomic_t */ #include "storage/latch.h" /* spec-4.6 D4 — freeze-gate WaitLatch */ @@ -109,6 +110,27 @@ ensure_counter_initialized(void) } } + +/* A backend holding a frozen buffer content lock must fail before entering a + * second distributed wait. BUSY/NOT_READY are retryable closed decisions; + * structural corruption remains an internal fail-closed error. */ +static ClusterLockAcquireResult +cluster_lock_pcm_x_nested_guard_result(PcmXQueueResult result) +{ + switch (result) { + case PCM_X_QUEUE_OK: + return CLUSTER_LOCK_ACQUIRE_OK_GRANTED; + case PCM_X_QUEUE_BARRIER_CLOSED: + case PCM_X_QUEUE_BUSY: + case PCM_X_QUEUE_NOT_READY: + case PCM_X_QUEUE_STALE: + case PCM_X_QUEUE_GATE_RETRY: + return CLUSTER_LOCK_ACQUIRE_FAIL_SHARD_REMASTERING; + default: + return CLUSTER_LOCK_ACQUIRE_FAIL_INTERNAL; + } +} + /* * spec-2.21 D4 — encode ClusterGrdHolderId into LOCALLOCK 24-byte raw image. * Mirror: see lock.h LOCALLOCK.cluster_holder_raw definition. @@ -327,6 +349,7 @@ cluster_lock_acquire_s4_remote_request_wait(const ClusterLockAcquireRequest *req * waiter edge (spec-5.8 D1b/D1e). */ ClusterLmdProcWaitState *ws = (MyProc != NULL) ? &MyProc->cluster_lmd_wait : NULL; + volatile PcmXQueueResult guard_result = PCM_X_QUEUE_OK; if (ws != NULL) (void)cluster_lmd_wait_state_publish(ws, CLUSTER_LMD_WAIT_GES, req->request_id, @@ -334,9 +357,11 @@ cluster_lock_acquire_s4_remote_request_wait(const ClusterLockAcquireRequest *req GetTopTransactionIdIfAny()); PG_TRY(); { - reject = cluster_ges_send_request_and_wait(&req->resid, (uint32)req->lockmode, - &req->holder, req->request_id, - req->timeout_ms, req->wait_event); + guard_result = cluster_pcm_x_nested_wait_guard_before_block(); + if (guard_result == PCM_X_QUEUE_OK) + reject = cluster_ges_send_request_and_wait(&req->resid, (uint32)req->lockmode, + &req->holder, req->request_id, + req->timeout_ms, req->wait_event); } PG_CATCH(); { @@ -347,6 +372,8 @@ cluster_lock_acquire_s4_remote_request_wait(const ClusterLockAcquireRequest *req PG_END_TRY(); if (ws != NULL) cluster_lmd_wait_state_clear(ws); + if (guard_result != PCM_X_QUEUE_OK) + return cluster_lock_pcm_x_nested_guard_result((PcmXQueueResult)guard_result); } if (reject == GES_REJECT_REASON_NONE) { if (dontwait && is_advisory) @@ -450,6 +477,7 @@ static ClusterLockAcquireResult cluster_lock_acquire_s5_convert(const ClusterLockAcquireRequest *req) { uint32 reject; + volatile PcmXQueueResult guard_result = PCM_X_QUEUE_OK; /* spec-5.8 D1d — a cross-node CONVERT (S->X upgrade) blocks and can * deadlock; publish/clear the wait-state around it like the S4 request * wait so the resolver can revalidate the victim. */ @@ -461,9 +489,11 @@ cluster_lock_acquire_s5_convert(const ClusterLockAcquireRequest *req) GetTopTransactionIdIfAny()); PG_TRY(); { - reject = cluster_ges_send_convert_and_wait(&req->resid, (uint32)req->lockmode, - (uint32)req->current_mode, &req->holder, - req->request_id, /* timeout = GUC default */ 0); + guard_result = cluster_pcm_x_nested_wait_guard_before_block(); + if (guard_result == PCM_X_QUEUE_OK) + reject = cluster_ges_send_convert_and_wait( + &req->resid, (uint32)req->lockmode, (uint32)req->current_mode, &req->holder, + req->request_id, /* timeout = GUC default */ 0); } PG_CATCH(); { @@ -474,6 +504,8 @@ cluster_lock_acquire_s5_convert(const ClusterLockAcquireRequest *req) PG_END_TRY(); if (ws != NULL) cluster_lmd_wait_state_clear(ws); + if (guard_result != PCM_X_QUEUE_OK) + return cluster_lock_pcm_x_nested_guard_result((PcmXQueueResult)guard_result); if (reject == GES_REJECT_REASON_NONE) { pg_atomic_fetch_add_u64(&stub_s5_promote_count, 1); diff --git a/src/backend/cluster/cluster_pcm_direct_init.c b/src/backend/cluster/cluster_pcm_direct_init.c new file mode 100644 index 00000000000..81f698eaecf --- /dev/null +++ b/src/backend/cluster/cluster_pcm_direct_init.c @@ -0,0 +1,110 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_direct_init.c + * Process-local exact proof for PCM direct initialization. + * + * Author: SqlRush + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_pcm_direct_init.h" + +static bool +cluster_pcm_direct_init_kind_valid(ClusterPcmDirectInitKind kind) +{ + return kind >= CLUSTER_PCM_DIRECT_INIT_READ_MISS && kind <= CLUSTER_PCM_DIRECT_INIT_FSM; +} + +static ClusterPcmOwnResult +cluster_pcm_direct_init_snapshot_validate(ClusterPcmDirectInitKind kind, + const ClusterPcmDirectInitSnapshot *snapshot) +{ + ClusterPcmOwnResult live_result; + bool valid_page_kind; + + if (!cluster_pcm_direct_init_kind_valid(kind) || snapshot == NULL) + return CLUSTER_PCM_OWN_INVALID; + + live_result = cluster_pcm_own_classify_live_flags(snapshot->flags, snapshot->reservation_token); + if (live_result != CLUSTER_PCM_OWN_OK) + return live_result; + + if (snapshot->generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + if (snapshot->buf_id < 0 || snapshot->private_refcount <= 0 + || BUF_STATE_GET_REFCOUNT(snapshot->buf_state) == 0) + return CLUSTER_PCM_OWN_STALE; + if ((snapshot->buf_state & BM_TAG_VALID) == 0 + || (snapshot->buf_state & (BM_DIRTY | BM_JUST_DIRTIED)) != 0 + || snapshot->buffer_type != (uint8)BUF_TYPE_CURRENT + || snapshot->pcm_state != (uint8)PCM_STATE_N || !snapshot->page_is_new) + return CLUSTER_PCM_OWN_STALE; + + valid_page_kind = kind == CLUSTER_PCM_DIRECT_INIT_VM || kind == CLUSTER_PCM_DIRECT_INIT_FSM; + if (valid_page_kind) { + if ((snapshot->buf_state & BM_VALID) == 0 || (snapshot->buf_state & BM_IO_IN_PROGRESS) != 0) + return CLUSTER_PCM_OWN_STALE; + } else if ((snapshot->buf_state & BM_VALID) != 0 + || (snapshot->buf_state & BM_IO_IN_PROGRESS) == 0) + return CLUSTER_PCM_OWN_STALE; + + if ((kind == CLUSTER_PCM_DIRECT_INIT_VM && snapshot->tag.forkNum != VISIBILITYMAP_FORKNUM) + || (kind == CLUSTER_PCM_DIRECT_INIT_FSM && snapshot->tag.forkNum != FSM_FORKNUM)) + return CLUSTER_PCM_OWN_STALE; + + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_direct_init_proof_arm(ClusterPcmDirectInitKind kind, + const ClusterPcmDirectInitSnapshot *snapshot, + ClusterPcmDirectInitProof *out_proof) +{ + ClusterPcmOwnResult result; + + if (out_proof == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_proof, 0, sizeof(*out_proof)); + + result = cluster_pcm_direct_init_snapshot_validate(kind, snapshot); + if (result != CLUSTER_PCM_OWN_OK) + return result; + + out_proof->identity = *snapshot; + out_proof->kind = kind; + out_proof->armed = true; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_direct_init_proof_consume(ClusterPcmDirectInitKind kind, + const ClusterPcmDirectInitSnapshot *snapshot, + ClusterPcmDirectInitProof *proof) +{ + ClusterPcmOwnResult result; + ClusterPcmDirectInitSnapshot *expected; + + if (proof == NULL || !proof->armed) + return CLUSTER_PCM_OWN_STALE; + + /* Failure consumes too: callers cannot refresh a rejected operation by + * retrying an old proof after the buffer identity has changed. */ + proof->armed = false; + if (kind != proof->kind) + return CLUSTER_PCM_OWN_STALE; + + result = cluster_pcm_direct_init_snapshot_validate(kind, snapshot); + if (result != CLUSTER_PCM_OWN_OK) + return result; + + expected = &proof->identity; + if (snapshot->buf_id != expected->buf_id || !BufferTagsEqual(&snapshot->tag, &expected->tag) + || snapshot->generation != expected->generation + || snapshot->reservation_token != expected->reservation_token + || snapshot->private_refcount != expected->private_refcount) + return CLUSTER_PCM_OWN_STALE; + + return CLUSTER_PCM_OWN_OK; +} diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index bfc5f0d4e86..dd29642b4ac 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -194,15 +194,16 @@ struct GrdEntry { * flight (master-side broadcast pending). * Read by N→S handler to short-circuit * with DENIED_PENDING_X reply (HC117). - * pending_x_since_lsn: observability only — XLogCtl LSN at the - * moment pending_x was set; do NOT use - * for timeout / retry math (LSN does not - * advance on idle DB; see Q7 rationale). + * pending_x_since_lsn: legacy requests store a low-63-bit XLogCtl + * LSN for observability; PCM-X queue heads + * store high-bit|ticket_id as an exact cookie. + * Decode the namespace before displaying it; + * never use either form for timeout math. * Cleared paths: (a) X grant install ack; (b) reconfig epoch advance; * (c) HC124 LMON node-dead sweep when requester crashes. */ int32 pending_x_requester_node; /* 4B [ 88, 92) -1 = none */ int32 _pad_pending_x; /* 4B [ 92, 96) 8B align */ - uint64 pending_x_since_lsn; /* 8B [ 96, 104) HC117 observability */ + uint64 pending_x_since_lsn; /* 8B [ 96, 104) LSN or queue cookie */ /* PGRAC: spec-2.37 D2 HC125+HC126 NEW — single max-historical PI watermark. * * pi_watermark_lsn: max(holder_last_page_lsn) observed across all @@ -481,9 +482,18 @@ pcm_master_holder_set_node(struct GrdEntry *entry, int32 node_id) if (entry->master_holder.node_id == (uint32)node_id) return; /* no-op; do not bump lifecycle counter */ entry->master_holder.node_id = (uint32)node_id; - /* procno / cluster_epoch / request_id intentionally left at current - * values (zero on fresh entry from pcm_get_or_create_entry); spec-2.35 - * scope does not consume them. HC110. */ + /* procno / cluster_epoch / request_id intentionally remain zero on a + * fresh entry; node_id starts at the no-holder sentinel and is set here. + * HC110. */ + cluster_gcs_block_bump_master_holder_lifecycle(); +} + +static inline void +pcm_master_holder_set_exact(struct GrdEntry *entry, const ClusterGrdHolderId *holder) +{ + if (memcmp(&entry->master_holder, holder, sizeof(*holder)) == 0) + return; + entry->master_holder = *holder; cluster_gcs_block_bump_master_holder_lifecycle(); } @@ -503,6 +513,30 @@ pcm_master_holder_is_valid(const struct GrdEntry *entry) return entry->master_holder.node_id != INVALID_PCM_MASTER_HOLDER_NODE; } +static void +pcm_authority_snapshot_locked(struct GrdEntry *entry, PcmAuthoritySnapshot *out) +{ + memset(out, 0, sizeof(*out)); + out->master_holder = entry->master_holder; + out->transition_count = pg_atomic_read_u64(&entry->transition_count_local); + out->pending_x_since_lsn = entry->pending_x_since_lsn; + out->state = (PcmState)pg_atomic_read_u32(&entry->master_state); + out->x_holder_node = entry->x_holder_node; + out->s_holders_bitmap = pg_atomic_read_u32(&entry->s_holders_bitmap); + out->pending_x_requester_node = entry->pending_x_requester_node; +} + +static bool +pcm_authority_snapshot_equal(const PcmAuthoritySnapshot *left, const PcmAuthoritySnapshot *right) +{ + return memcmp(&left->master_holder, &right->master_holder, sizeof(left->master_holder)) == 0 + && left->transition_count == right->transition_count + && left->pending_x_since_lsn == right->pending_x_since_lsn && left->state == right->state + && left->x_holder_node == right->x_holder_node + && left->s_holders_bitmap == right->s_holders_bitmap + && left->pending_x_requester_node == right->pending_x_requester_node; +} + static inline int32 pcm_lowest_set_bit_node(uint32 bitmap) { @@ -542,6 +576,161 @@ cluster_pcm_master_holder_node_by_tag(BufferTag tag) return node_id; } +/* + * Return a single-entry-lock view of the PCM authority. A missing entry is + * represented by false; out is still initialized to the canonical empty + * sentinels so diagnostic callers never observe uninitialized bytes. + */ +bool +cluster_pcm_lock_authority_snapshot(BufferTag tag, PcmAuthoritySnapshot *out) +{ + struct GrdEntry *entry; + bool found; + + if (out == NULL) + return false; + + memset(out, 0, sizeof(*out)); + out->state = PCM_STATE_N; + out->x_holder_node = -1; + out->pending_x_requester_node = -1; + out->master_holder.node_id = INVALID_PCM_MASTER_HOLDER_NODE; + + if (ClusterPcm == NULL || cluster_pcm_htab == NULL) + return false; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (found && entry != NULL) { + LWLockAcquire(&entry->entry_lock.lock, LW_SHARED); + pcm_authority_snapshot_locked(entry, out); + LWLockRelease(&entry->entry_lock.lock); + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + + return found && entry != NULL; +} + +/* + * Atomically install the queue winner as the sole X holder, but only if the + * authority still exactly matches the snapshot captured when the transfer + * began. No engine lock is held here; callers prepare an immutable token, + * perform this GRD barrier, then finalize their queue state. + */ +PcmXGrdHandoffResult +cluster_pcm_lock_queue_handoff_x_exact(const PcmXGrdHandoffToken *token) +{ + struct GrdEntry *entry; + PcmAuthoritySnapshot current; + ClusterGrdHolderId requester; + PcmXGrdHandoffResult result = PCM_X_GRD_HANDOFF_BAD_STATE; + bool found; + bool broadcast = false; + uint64 pending_x_value; + uint32 source_bit; + + if (token == NULL || token->requester_node < 0 || token->requester_node >= 32 + || token->source_node < 0 || token->source_node >= 32 || token->request_id == 0 + || token->grant_generation == 0 + || !cluster_pcm_x_image_id_decode(token->image_id, NULL, NULL)) + return PCM_X_GRD_HANDOFF_INVALID; + if (!PcmPendingXQueueValue(token->ticket_id, &pending_x_value)) + return PCM_X_GRD_HANDOFF_INVALID; + if (ClusterPcm == NULL || cluster_pcm_htab == NULL) + return PCM_X_GRD_HANDOFF_NOT_FOUND; + + requester.node_id = (uint32)token->requester_node; + requester.procno = token->requester_procno; + requester.cluster_epoch = token->cluster_epoch; + requester.request_id = token->request_id; + source_bit = (uint32)1u << (uint32)token->source_node; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &token->tag, HASH_FIND, &found); + if (!found || entry == NULL) { + LWLockRelease(&ClusterPcm->htab_lock.lock); + return PCM_X_GRD_HANDOFF_NOT_FOUND; + } + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + pcm_authority_snapshot_locked(entry, ¤t); + + /* A retry after the commit barrier is harmless only for the same full + * requester identity and an image no newer than the installed Lamport-SCN + * authority. page_lsn is per-node A-record evidence, not a cross-node + * version comparator. */ + if (current.state == PCM_STATE_X && current.x_holder_node == token->requester_node + && current.s_holders_bitmap == 0 && current.pending_x_requester_node == -1 + && memcmp(¤t.master_holder, &requester, sizeof(requester)) == 0 + && (!SCN_VALID(token->page_scn) + || scn_local(entry->pi_watermark_scn) >= scn_local(token->page_scn))) { + result = PCM_X_GRD_HANDOFF_DUPLICATE; + goto done; + } + + if (!pcm_authority_snapshot_equal(¤t, &token->authority)) { + result = PCM_X_GRD_HANDOFF_STALE; + goto done; + } + if (current.pending_x_requester_node != token->requester_node + || current.pending_x_since_lsn != pending_x_value) + goto done; + /* The authority snapshot deliberately excludes watermarks, so close the + * SCN window under the same entry lock as the pending-X cookie and GRD + * transition. Cross-node version authority is Lamport page_scn only; + * page_lsn belongs to its source WAL stream and remains A-record evidence. */ + if (SCN_VALID(entry->pi_watermark_scn) + && (!SCN_VALID(token->page_scn) + || scn_local(token->page_scn) < scn_local(entry->pi_watermark_scn))) + goto done; + + if (current.state == PCM_STATE_X) { + if (current.x_holder_node != token->source_node || current.s_holders_bitmap != 0 + || current.master_holder.node_id != (uint32)token->source_node) + goto done; + } else if (current.state == PCM_STATE_S) { + if (current.x_holder_node != -1 || current.s_holders_bitmap != source_bit + || current.master_holder.node_id != (uint32)token->source_node) + goto done; + } else if (current.state == PCM_STATE_N) { + /* A cold global-N round has no physical cache holder. The queue names + * the requester as synthetic source so the existing image/A-record path + * remains mandatory and the final barrier stays ticket-exact. */ + if (token->source_node != token->requester_node || current.x_holder_node != -1 + || current.s_holders_bitmap != 0 || pcm_master_holder_is_valid(entry)) + goto done; + } else { + goto done; + } + + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); + entry->x_holder_node = token->requester_node; + pg_atomic_write_u32(&entry->s_holders_bitmap, 0); + entry->s_holder_refcount_local = 0; + pcm_master_holder_set_exact(entry, &requester); + entry->pending_x_requester_node = -1; + entry->pending_x_since_lsn = 0; + if (SCN_VALID(token->page_scn) + && scn_local(token->page_scn) > scn_local(entry->pi_watermark_scn)) { + SCN old_scn = entry->pi_watermark_scn; + + entry->pi_watermark_scn = token->page_scn; + pcm_wm_prov_record(token->tag, old_scn, token->page_scn, CLUSTER_PCM_WM_SRC_GRANT_X, + token->source_node, token->request_id, token->cluster_epoch); + } + entry->last_transition_at = GetCurrentTimestamp(); + pg_atomic_fetch_add_u64(&entry->transition_count_local, 1); + broadcast = true; + result = PCM_X_GRD_HANDOFF_OK; + +done: + LWLockRelease(&entry->entry_lock.lock); + LWLockRelease(&ClusterPcm->htab_lock.lock); + if (broadcast) + ConditionVariableBroadcast(&entry->wait_cv); + return result; +} + /* ============================================================ * PGRAC: spec-2.36 D5 HC117 / HC124 — S barrier helpers. @@ -560,24 +749,112 @@ cluster_pcm_master_holder_node_by_tag(BufferTag tag) * htab_lock SHARED; per-entry mutation under entry_lock * EXCLUSIVE; idempotent recheck inside the lock. * ============================================================ */ -void +static PcmPendingXReserveResult +pcm_lock_try_reserve_pending_x_value(BufferTag tag, int32 requester_node, uint64 value) +{ + struct GrdEntry *entry; + bool found; + PcmPendingXReserveResult result = PCM_PENDING_X_RESERVE_OCCUPIED; + + if (ClusterPcm == NULL || cluster_pcm_htab == NULL || requester_node < 0 + || requester_node >= 32) + return PCM_PENDING_X_RESERVE_INVALID; + /* Admission itself is the first authoritative touch for a cold tag. Use + * the canonical lazy allocator so both legacy and queue writers can arm + * the barrier before any transition exists. */ + if (pcm_get_or_create_entry(tag) == NULL) + return PCM_PENDING_X_RESERVE_NO_CAPACITY; + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (found && entry != NULL) { + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + if (entry->pending_x_requester_node == -1) { + entry->pending_x_requester_node = requester_node; + entry->pending_x_since_lsn = value; + result = PCM_PENDING_X_RESERVE_OK; + } + LWLockRelease(&entry->entry_lock.lock); + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + return result; +} + +PcmPendingXReserveResult cluster_pcm_lock_set_pending_x(BufferTag tag, int32 requester_node, uint64 current_lsn) +{ + if ((current_lsn & PCM_PENDING_X_QUEUE_KIND) != 0) + return PCM_PENDING_X_RESERVE_INVALID; + return pcm_lock_try_reserve_pending_x_value(tag, requester_node, current_lsn); +} + +/* + * Reserve the one GrdEntry pending-X barrier for an active PCM-X queue head. + * + * Legacy request paths still use cluster_pcm_lock_set_pending_x() for their + * request-scoped round. Queue arbitration cannot: a delayed drive of another + * tag-head must never overwrite the requester that owns the live transfer. + * A node-only match is not replay proof: a legacy request from the same node + * may own that mark. This primitive therefore succeeds only on the idle + * transition; the queue engine separately persists the exact ticket claim. + */ +PcmPendingXReserveResult +cluster_pcm_lock_try_reserve_pending_x(BufferTag tag, int32 requester_node, uint64 ticket_id) +{ + uint64 value; + + if (!PcmPendingXQueueValue(ticket_id, &value)) + return PCM_PENDING_X_RESERVE_INVALID; + return pcm_lock_try_reserve_pending_x_value(tag, requester_node, value); +} + +bool +cluster_pcm_lock_queue_pending_x_exact(BufferTag tag, int32 requester_node, uint64 ticket_id) { struct GrdEntry *entry; + uint64 expected; bool found; + bool exact = false; - if (cluster_pcm_htab == NULL) - return; + if (ClusterPcm == NULL || cluster_pcm_htab == NULL || requester_node < 0 || requester_node >= 32 + || !PcmPendingXQueueValue(ticket_id, &expected)) + return false; + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); + if (found && entry != NULL) { + LWLockAcquire(&entry->entry_lock.lock, LW_SHARED); + exact = entry->pending_x_requester_node == requester_node + && entry->pending_x_since_lsn == expected; + LWLockRelease(&entry->entry_lock.lock); + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + return exact; +} +bool +cluster_pcm_lock_clear_queue_pending_x_exact(BufferTag tag, int32 requester_node, uint64 ticket_id) +{ + struct GrdEntry *entry; + uint64 expected; + bool found; + bool cleared = false; + + if (ClusterPcm == NULL || cluster_pcm_htab == NULL || requester_node < 0 || requester_node >= 32 + || !PcmPendingXQueueValue(ticket_id, &expected)) + return false; LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); if (found && entry != NULL) { LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); - entry->pending_x_requester_node = requester_node; - entry->pending_x_since_lsn = current_lsn; + if (entry->pending_x_requester_node == requester_node + && entry->pending_x_since_lsn == expected) { + entry->pending_x_requester_node = -1; + entry->pending_x_since_lsn = 0; + cleared = true; + } LWLockRelease(&entry->entry_lock.lock); } LWLockRelease(&ClusterPcm->htab_lock.lock); + return cleared; } void @@ -593,8 +870,10 @@ cluster_pcm_lock_clear_pending_x(BufferTag tag) entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); if (found && entry != NULL) { LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); - entry->pending_x_requester_node = -1; - entry->pending_x_since_lsn = 0; + if ((entry->pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) == 0) { + entry->pending_x_requester_node = -1; + entry->pending_x_since_lsn = 0; + } LWLockRelease(&entry->entry_lock.lock); } LWLockRelease(&ClusterPcm->htab_lock.lock); @@ -625,7 +904,8 @@ cluster_pcm_lock_clear_pending_x_if(BufferTag tag, int32 expected_requester) entry = (struct GrdEntry *)hash_search(cluster_pcm_htab, &tag, HASH_FIND, &found); if (found && entry != NULL) { LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); - if (entry->pending_x_requester_node == expected_requester) { + if (entry->pending_x_requester_node == expected_requester + && (entry->pending_x_since_lsn & PCM_PENDING_X_QUEUE_KIND) == 0) { entry->pending_x_requester_node = -1; entry->pending_x_since_lsn = 0; cleared = true; @@ -2826,16 +3106,11 @@ cluster_pcm_lock_release(BufferTag tag) * eviction) so X locks release with X→N rather than the S→N transition. */ void -cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode) +cluster_pcm_lock_release_saved_tag_for_eviction(BufferTag tag, PcmLockMode mode) { - BufferTag tag; int master_node; PcmLockTransition trans; - if (buf == NULL) - ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), - errmsg("cluster_pcm_lock_release_buffer_for_eviction: NULL BufferDesc"))); - if (cluster_pcm_htab == NULL) PCM_STUB_DISABLED_PATH; @@ -2856,7 +3131,6 @@ cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode) if (mode != PCM_LOCK_MODE_S && mode != PCM_LOCK_MODE_X) return; - tag = buf->tag; master_node = cluster_gcs_lookup_master(tag); if (master_node != cluster_node_id) { @@ -2901,6 +3175,16 @@ cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode) cluster_pcm_lock_release(tag); } +void +cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode) +{ + if (buf == NULL) + ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("cluster_pcm_lock_release_buffer_for_eviction: NULL BufferDesc"))); + + cluster_pcm_lock_release_saved_tag_for_eviction(buf->tag, mode); +} + /* * PGRAC: spec-2.35 D5 (HC111 + HC112) — content-lock unlock variant. * @@ -3312,6 +3596,7 @@ pcm_get_or_create_entry(BufferTag tag) entry->tag = saved_tag; pg_atomic_init_u32(&entry->master_state, (uint32)PCM_STATE_N); entry->x_holder_node = -1; + entry->master_holder.node_id = INVALID_PCM_MASTER_HOLDER_NODE; pg_atomic_init_u32(&entry->s_holders_bitmap, 0); pg_atomic_init_u32(&entry->pi_holders_bitmap, 0); pg_atomic_init_u64(&entry->transition_count_local, 0); diff --git a/src/backend/cluster/cluster_pcm_own.c b/src/backend/cluster/cluster_pcm_own.c index f0b5355dccf..348a7802999 100644 --- a/src/backend/cluster/cluster_pcm_own.c +++ b/src/backend/cluster/cluster_pcm_own.c @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * cluster_pcm_own.c - * pgrac per-buffer node-local ownership generation + flags (shmem array). + * pgrac per-buffer node-local ownership tuple (generation + token + flags). * * See cluster_pcm_own.h for the design. This file owns the shmem array * allocation (NBuffers entries, indexed by buf_id) and its registration @@ -54,17 +54,318 @@ cluster_pcm_own_shmem_init(void) for (i = 0; i < NBuffers; i++) { pg_atomic_init_u64(&ClusterPcmOwnArray[i].generation, 0); + pg_atomic_init_u64(&ClusterPcmOwnArray[i].reservation_token, 0); pg_atomic_init_u32(&ClusterPcmOwnArray[i].flags, 0); ClusterPcmOwnArray[i]._pad = 0; } } } +static bool +cluster_pcm_own_entry_for_buf(int buf_id, ClusterPcmOwnEntry **out_entry) +{ + if (out_entry == NULL) + return false; + *out_entry = NULL; + if (ClusterPcmOwnArray == NULL || buf_id < 0 || buf_id >= NBuffers) + return false; + *out_entry = &ClusterPcmOwnArray[buf_id]; + return true; +} + +static bool +cluster_pcm_own_reservation_flag_valid(uint32 reservation_flag) +{ + return reservation_flag == PCM_OWN_FLAG_GRANT_PENDING + || reservation_flag == PCM_OWN_FLAG_REVOKING; +} + +ClusterPcmOwnResult +cluster_pcm_own_reservation_begin_exact(int buf_id, uint64 expected_generation, + uint32 reservation_flag, uint64 *out_token) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 generation; + uint64 old_token; + uint32 flags; + + if (out_token == NULL) + return CLUSTER_PCM_OWN_INVALID; + *out_token = 0; + if (!cluster_pcm_own_reservation_flag_valid(reservation_flag)) + return CLUSTER_PCM_OWN_INVALID; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + + generation = pg_atomic_read_u64(&entry->generation); + old_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + if (generation != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_result = cluster_pcm_own_classify_live_flags(flags, old_token); + if (live_result != CLUSTER_PCM_OWN_OK) + return live_result; + if (generation == UINT64_MAX || old_token == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + + old_token++; + /* Both stores are serialized by the BufferDesc header lock required by + * the public contract. The flag is the active marker; token stays + * monotonic and is never cleared. */ + pg_atomic_write_u64(&entry->reservation_token, old_token); + pg_atomic_write_u32(&entry->flags, reservation_flag); + *out_token = old_token; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_reservation_abort_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, uint32 reservation_flag) +{ + ClusterPcmOwnEntry *entry; + + if (!cluster_pcm_own_reservation_flag_valid(reservation_flag) || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->generation) != expected_generation + || pg_atomic_read_u64(&entry->reservation_token) != reservation_token + || pg_atomic_read_u32(&entry->flags) != reservation_flag) + return CLUSTER_PCM_OWN_STALE; + + pg_atomic_write_u32(&entry->flags, 0); + return CLUSTER_PCM_OWN_OK; +} + +static ClusterPcmOwnResult +cluster_pcm_own_reservation_handoff_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, uint32 source_flag, + uint32 target_flag) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 live_token; + uint32 flags; + + if (reservation_token == 0 || !cluster_pcm_own_reservation_flag_valid(source_flag) + || !cluster_pcm_own_reservation_flag_valid(target_flag) || source_flag == target_flag) + return CLUSTER_PCM_OWN_INVALID; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->generation) != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + /* An exact duplicate observes the already-published role and succeeds + * without touching the tuple. */ + if (flags == target_flag) + return CLUSTER_PCM_OWN_OK; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != source_flag) + return CLUSTER_PCM_OWN_BUSY; + + pg_atomic_write_u32(&entry->flags, target_flag); + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_revoke_to_grant_handoff_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token) +{ + return cluster_pcm_own_reservation_handoff_exact(buf_id, expected_generation, reservation_token, + PCM_OWN_FLAG_REVOKING, + PCM_OWN_FLAG_GRANT_PENDING); +} + +ClusterPcmOwnResult +cluster_pcm_own_grant_commit_exact(int buf_id, uint64 expected_generation, uint64 reservation_token, + uint64 *out_committed_generation) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 generation; + uint64 live_token; + uint32 flags; + + if (out_committed_generation == NULL || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + *out_committed_generation = 0; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + + generation = pg_atomic_read_u64(&entry->generation); + if (generation != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != PCM_OWN_FLAG_GRANT_PENDING) + return CLUSTER_PCM_OWN_BUSY; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + if (generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + + generation++; + pg_atomic_write_u64(&entry->generation, generation); + pg_atomic_write_u32(&entry->flags, 0); + *out_committed_generation = generation; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_revoke_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, uint64 *out_committed_generation) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 generation; + uint64 live_token; + uint32 flags; + + if (out_committed_generation == NULL || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + *out_committed_generation = 0; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + + generation = pg_atomic_read_u64(&entry->generation); + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + if (generation != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != PCM_OWN_FLAG_REVOKING) + return CLUSTER_PCM_OWN_BUSY; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + if (generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + + generation++; + pg_atomic_write_u64(&entry->generation, generation); + pg_atomic_write_u32(&entry->flags, 0); + *out_committed_generation = generation; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_revoke_retain_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 generation; + uint64 live_token; + uint32 flags; + + if (out_committed_generation == NULL || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + *out_committed_generation = 0; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + + generation = pg_atomic_read_u64(&entry->generation); + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + if (generation != expected_generation) + return CLUSTER_PCM_OWN_STALE; + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != PCM_OWN_FLAG_REVOKING) + return CLUSTER_PCM_OWN_BUSY; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + if (generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + + generation++; + pg_atomic_write_u64(&entry->generation, generation); + /* Unlike an ordinary revoke, this token is the descriptor-retention pin. + * Keep REVOKING published until exact image-record drain. */ + *out_committed_generation = generation; + return CLUSTER_PCM_OWN_OK; +} + +ClusterPcmOwnResult +cluster_pcm_own_revoke_retain_release_exact(int buf_id, uint64 committed_generation, + uint64 reservation_token) +{ + ClusterPcmOwnEntry *entry; + ClusterPcmOwnResult live_result; + uint64 live_token; + uint32 flags; + + if (reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return CLUSTER_PCM_OWN_NOT_READY; + if (pg_atomic_read_u64(&entry->generation) != committed_generation) + return CLUSTER_PCM_OWN_STALE; + live_token = pg_atomic_read_u64(&entry->reservation_token); + flags = pg_atomic_read_u32(&entry->flags); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return live_result; + if (flags == 0) + return CLUSTER_PCM_OWN_STALE; + if (flags != PCM_OWN_FLAG_REVOKING) + return CLUSTER_PCM_OWN_BUSY; + if (live_token != reservation_token) + return CLUSTER_PCM_OWN_STALE; + + pg_atomic_write_u32(&entry->flags, 0); + return CLUSTER_PCM_OWN_OK; +} + +bool +cluster_pcm_own_gen_bump_checked(int buf_id, uint64 *out_generation) +{ + ClusterPcmOwnEntry *entry; + uint64 generation; + uint32 flags; + + if (out_generation != NULL) + *out_generation = 0; + if (!cluster_pcm_own_entry_for_buf(buf_id, &entry)) + return false; + generation = pg_atomic_read_u64(&entry->generation); + flags = pg_atomic_read_u32(&entry->flags); + if (out_generation != NULL) + *out_generation = generation; + if (generation == UINT64_MAX || flags != 0) + return false; + generation++; + pg_atomic_write_u64(&entry->generation, generation); + if (out_generation != NULL) + *out_generation = generation; + return true; +} + static const ClusterShmemRegion cluster_pcm_own_region = { .name = "pgrac cluster pcm ownership", .size_fn = cluster_pcm_own_shmem_size, .init_fn = cluster_pcm_own_shmem_init, - .lwlock_count = 0, /* atomics only; the buffer header spinlock serializes the triple */ + .lwlock_count = 0, /* atomics only; the buffer header spinlock serializes the tuple */ .owner_subsys = "cluster_pcm_own", .reserved_flags = 0, }; diff --git a/src/backend/cluster/cluster_pcm_x_convert.c b/src/backend/cluster/cluster_pcm_x_convert.c new file mode 100644 index 00000000000..d627ab85805 --- /dev/null +++ b/src/backend/cluster/cluster_pcm_x_convert.c @@ -0,0 +1,19695 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_x_convert.c + * PCM-X checked five-pool external shared-memory substrate. + * + * The region is self-describing and uses offsets and slot indexes only. + * Creation initializes six allocators over five logical pools because the + * local membership pool has fixed wait and holder partitions. Attach + * validates the complete immutable layout before publishing the local + * region pointer. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_pcm_x_convert.c + * + * NOTES + * This is a pgrac-original file. Exported symbols use the + * cluster_pcm_x prefix. Queue state transitions and message dispatch are + * intentionally outside this substrate. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "miscadmin.h" +#include "storage/buf_internals.h" +#include "storage/proc.h" +#include "storage/shmem.h" + +#include "port/pg_crc32c.h" + +#include "cluster/cluster_guc.h" +#include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_pcm_x_convert.h" +#include "cluster/cluster_shmem.h" + + +/* Process-local pointer published only after create/attach validation. */ +PcmXShmemHeader *ClusterPcmXConvertShmem = NULL; + + +StaticAssertDecl(offsetof(PcmXSlotHeader, next_free) == 0, "PCM-X common free link must be first"); +StaticAssertDecl(offsetof(PcmXMasterTagSlot, slot) == 0, + "PCM-X master-tag slot header must be first"); +StaticAssertDecl(offsetof(PcmXMasterTicketSlot, slot) == 0, + "PCM-X master-ticket slot header must be first"); +StaticAssertDecl(offsetof(PcmXBlockerSlot, slot) == 0, "PCM-X blocker slot header must be first"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, slot) == 0, + "PCM-X local-tag slot header must be first"); +StaticAssertDecl(offsetof(PcmXLocalMembershipSlot, slot) == 0, + "PCM-X local-membership slot header must be first"); +StaticAssertDecl((PCM_X_LOCK_PARTITIONS & (PCM_X_LOCK_PARTITIONS - 1)) == 0, + "PCM-X lock partition count must be a power of two"); +StaticAssertDecl(PCM_X_RUNTIME_SHUTTING_DOWN < PCM_X_RUNTIME_GATE_STATE_MASK, + "PCM-X runtime states must fit in the packed gate"); + + +/* Private fail-closed phase used to serialize session publication. */ +#define PCM_X_RUNTIME_GATE_ACTIVATING 3U +#define PCM_X_RUNTIME_GATE_EXHAUSTED 4U +#define PCM_X_SLOT_GENERATION_READ_RETRIES 16 +#define PCM_X_STATE_BIT(state) (UINT32_C(1) << (state)) +#define PCM_X_MASTER_TICKET_DOMAIN_STATES \ + (PCM_X_STATE_BIT(PCM_XT_ADMITTING) | PCM_X_STATE_BIT(PCM_XT_QUEUED) \ + | PCM_X_STATE_BIT(PCM_XT_ACTIVE_PROBE) | PCM_X_STATE_BIT(PCM_XT_ACTIVE_TRANSFER) \ + | PCM_X_STATE_BIT(PCM_XT_COMPLETE) | PCM_X_STATE_BIT(PCM_XT_CANCELLED) \ + | PCM_X_STATE_BIT(PCM_XT_RETIRE_CREDIT) | PCM_X_STATE_BIT(PCM_XT_RECOVERY_BLOCKED)) +#define PCM_X_LOCAL_MEMBER_DOMAIN_STATES \ + (PCM_X_STATE_BIT(PCM_XL_JOINED_NONWAITABLE) | PCM_X_STATE_BIT(PCM_XL_WAITABLE_FOLLOWER) \ + | PCM_X_STATE_BIT(PCM_XL_NODE_LEADER) | PCM_X_STATE_BIT(PCM_XL_REMOTE_WAIT) \ + | PCM_X_STATE_BIT(PCM_XL_CONTENT_ACTIVE) | PCM_X_STATE_BIT(PCM_XL_GRANTED) \ + | PCM_X_STATE_BIT(PCM_XL_CANCELLED) | PCM_X_STATE_BIT(PCM_XL_RECOVERY_BLOCKED)) +#define PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES \ + (PCM_X_STATE_BIT(PCM_XL_HOLDER_ACQUIRING) | PCM_X_STATE_BIT(PCM_XL_HOLDER_ACTIVE) \ + | PCM_X_STATE_BIT(PCM_XL_HOLDER_RELEASING)) +StaticAssertDecl(PCM_X_RUNTIME_GATE_ACTIVATING > PCM_X_RUNTIME_SHUTTING_DOWN, + "PCM-X activating phase must not overlap public runtime states"); +StaticAssertDecl(PCM_X_RUNTIME_GATE_EXHAUSTED > PCM_X_RUNTIME_SHUTTING_DOWN, + "PCM-X exhausted phase must not overlap public runtime states"); +StaticAssertDecl(PCM_X_RUNTIME_GATE_ACTIVATING <= PCM_X_RUNTIME_GATE_STATE_MASK, + "PCM-X activating phase must fit in the packed gate"); +StaticAssertDecl(PCM_X_RUNTIME_GATE_EXHAUSTED <= PCM_X_RUNTIME_GATE_STATE_MASK, + "PCM-X exhausted phase must fit in the packed gate"); + + +static Size pcm_x_maxalign(Size value); +static Size pcm_x_append_array(Size *cursor, Size capacity, Size element_size); +static void pcm_x_set_pool(PcmXShmemLayout *layout, PcmXPoolKind kind, Size capacity, + Size slot_size, bool has_directory, Size *cursor); +static void pcm_x_compute_runtime_layout(PcmXShmemLayout *layout); +static void pcm_x_init_free_list(char *base, Size slots_offset, Size slot_size, + Size first_slot_index, Size capacity, + PcmXAllocatorState *allocator); +static void pcm_x_init_allocators(PcmXShmemHeader *header); +static void pcm_x_init_stats(PcmXStats *stats); +static void pcm_x_stats_increment(pg_atomic_uint64 *counter); +static bool pcm_x_stats_depth_increment(PcmXShmemHeader *header); +static bool pcm_x_stats_depth_decrement(PcmXShmemHeader *header); +static bool pcm_x_master_terminal_leg_is_clear(const PcmXReliableLegState *leg); +static bool pcm_x_master_blocker_stage_metadata_valid(const PcmXMasterTicketSlot *ticket); +static void pcm_x_master_blocker_snapshot_clear(PcmXMasterBlockerSnapshot *snapshot); +static bool pcm_x_ticket_ref_is_zero(const PcmXTicketRef *ref); +static bool pcm_x_local_terminal_ref_valid(const PcmXTicketRef *ref); +static bool pcm_x_local_admission_ref_valid(const PcmXTicketRef *ref, + const PcmXWaitIdentity *identity); +static bool pcm_x_local_terminal_unlinked_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalMembershipSlot *terminal, + PcmXSlotRef terminal_ref); + + +static inline uint32 +pcm_x_runtime_gate_pack(uint32 generation, uint32 state) +{ + Assert(generation <= PCM_X_RUNTIME_GATE_GENERATION_MAX); + Assert(state <= PCM_X_RUNTIME_GATE_STATE_MASK); + return (generation << PCM_X_RUNTIME_GATE_STATE_BITS) | state; +} + +static inline uint32 +pcm_x_runtime_gate_generation(uint32 gate) +{ + return gate >> PCM_X_RUNTIME_GATE_STATE_BITS; +} + +static inline uint32 +pcm_x_runtime_gate_state(uint32 gate) +{ + return gate & PCM_X_RUNTIME_GATE_STATE_MASK; +} + + +/* Align only after a checked addition proves MAXALIGN cannot wrap. */ +static Size +pcm_x_maxalign(Size value) +{ + (void)add_size(value, MAXIMUM_ALIGNOF - 1); + return MAXALIGN(value); +} + +/* Append one aligned fixed-record array and return its region offset. */ +static Size +pcm_x_append_array(Size *cursor, Size capacity, Size element_size) +{ + Size offset; + Size bytes; + + offset = pcm_x_maxalign(*cursor); + bytes = mul_size(capacity, element_size); + *cursor = add_size(offset, bytes); + return offset; +} + +/* Populate one logical pool and append its optional bounded directory. */ +static void +pcm_x_set_pool(PcmXShmemLayout *layout, PcmXPoolKind kind, Size capacity, Size slot_size, + bool has_directory, Size *cursor) +{ + PcmXPoolLayout *pool = &layout->pools[kind]; + + pool->slots_offset = pcm_x_append_array(cursor, capacity, slot_size); + pool->slot_size = slot_size; + pool->capacity = capacity; + if (has_directory) { + pool->directory_capacity = mul_size(capacity, 2); + pool->directory_offset + = pcm_x_append_array(cursor, pool->directory_capacity, sizeof(PcmXDirectoryEntry)); + } +} + + +/* + * cluster_pcm_x_layout_compute -- Compute the complete immutable layout. + * + * All capacity arithmetic and every byte/offset step uses PostgreSQL's + * checked shared-memory helpers. An overflow raises ERROR before the + * resulting offset can be used. + * + * Inputs: + * max_backends: configured backend process count. + * auxiliary_procs: fixed auxiliary process count. + * n_buffers: shared buffer count. + * max_wait_edges: configured WFG edge bound. + * layout: writable output object. + * + * Returns: + * Nothing. On success, layout contains the complete region description. + * + * Side Effects: + * Raises ERROR for a null output or checked-arithmetic overflow. + * + * Author: SqlRush + */ +void +cluster_pcm_x_layout_compute(Size max_backends, Size auxiliary_procs, Size n_buffers, + Size max_wait_edges, PcmXShmemLayout *layout) +{ + Size cursor; + Size blocker_capacity; + Size local_tag_capacity; + Size membership_capacity; + Size master_ticket_directory_capacity; + PcmXPoolLayout *membership; + + if (layout == NULL) + ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), + errmsg("PCM-X shared memory layout output is null"), + errhint("Pass a writable PcmXShmemLayout object."))); + + memset(layout, 0, sizeof(*layout)); + layout->magic = PCM_X_SHMEM_MAGIC; + layout->version = PCM_X_SHMEM_LAYOUT_VERSION; + layout->pool_count = PCM_X_POOL_COUNT; + layout->allocator_count = PCM_X_ALLOC_COUNT; + layout->lock_partition_count = PCM_X_LOCK_PARTITIONS; + layout->lwlock_count = PCM_X_LWLOCK_COUNT; + layout->max_backends = max_backends; + layout->auxiliary_procs = auxiliary_procs; + layout->n_buffers = n_buffers; + layout->max_wait_edges = max_wait_edges; + layout->process_capacity = add_size(max_backends, auxiliary_procs); + layout->locks_per_process = Min(n_buffers, (Size)LWLOCK_MAX_HELD_BY_PROC); + layout->active_holder_capacity = mul_size(layout->process_capacity, layout->locks_per_process); + layout->holder_tag_capacity = Min(n_buffers, layout->active_holder_capacity); + layout->node_ticket_capacity + = mul_size(layout->process_capacity, (Size)PCM_X_PROTOCOL_NODE_LIMIT); + master_ticket_directory_capacity = mul_size(layout->node_ticket_capacity, 2); + + blocker_capacity = mul_size(max_wait_edges, 2); + local_tag_capacity = add_size(layout->node_ticket_capacity, layout->holder_tag_capacity); + membership_capacity = add_size(layout->process_capacity, layout->active_holder_capacity); + + layout->header_size = pcm_x_maxalign(sizeof(PcmXShmemHeader)); + cursor = layout->header_size; + pcm_x_set_pool(layout, PCM_X_POOL_MASTER_TAG, layout->node_ticket_capacity, + sizeof(PcmXMasterTagSlot), true, &cursor); + pcm_x_set_pool(layout, PCM_X_POOL_MASTER_TICKET, layout->node_ticket_capacity, + sizeof(PcmXMasterTicketSlot), false, &cursor); + layout->master_ticket_directories.prehandle_capacity = master_ticket_directory_capacity; + layout->master_ticket_directories.prehandle_offset = pcm_x_append_array( + &cursor, layout->master_ticket_directories.prehandle_capacity, sizeof(PcmXDirectoryEntry)); + layout->master_ticket_directories.handle_capacity = master_ticket_directory_capacity; + layout->master_ticket_directories.handle_offset = pcm_x_append_array( + &cursor, layout->master_ticket_directories.handle_capacity, sizeof(PcmXDirectoryEntry)); + layout->master_ticket_directories.retire_capacity = master_ticket_directory_capacity; + layout->master_ticket_directories.retire_offset = pcm_x_append_array( + &cursor, layout->master_ticket_directories.retire_capacity, sizeof(PcmXDirectoryEntry)); + pcm_x_set_pool(layout, PCM_X_POOL_BLOCKER, blocker_capacity, sizeof(PcmXBlockerSlot), false, + &cursor); + pcm_x_set_pool(layout, PCM_X_POOL_LOCAL_TAG, local_tag_capacity, sizeof(PcmXLocalTagSlot), true, + &cursor); + pcm_x_set_pool(layout, PCM_X_POOL_LOCAL_MEMBERSHIP, membership_capacity, + sizeof(PcmXLocalMembershipSlot), false, &cursor); + + membership = &layout->pools[PCM_X_POOL_LOCAL_MEMBERSHIP]; + layout->local_wait.first_slot_index = 0; + layout->local_wait.slots_offset = membership->slots_offset; + layout->local_wait.capacity = layout->process_capacity; + layout->local_wait.directory_capacity = mul_size(layout->local_wait.capacity, 2); + layout->local_wait.directory_offset = pcm_x_append_array( + &cursor, layout->local_wait.directory_capacity, sizeof(PcmXDirectoryEntry)); + + layout->local_holder.first_slot_index = layout->process_capacity; + layout->local_holder.slots_offset = add_size( + membership->slots_offset, mul_size(layout->local_wait.capacity, membership->slot_size)); + layout->local_holder.capacity = layout->active_holder_capacity; + layout->local_holder.directory_capacity = mul_size(layout->local_holder.capacity, 2); + layout->local_holder.directory_offset = pcm_x_append_array( + &cursor, layout->local_holder.directory_capacity, sizeof(PcmXDirectoryEntry)); + layout->total_size = pcm_x_maxalign(cursor); +} + + +/* + * cluster_pcm_x_validate_attach -- Validate an existing region description. + * + * Inputs: + * header: existing shared-memory header. + * expected: layout computed from this process's runtime configuration. + * + * Returns: + * PCM_X_ATTACH_OK for an exact match, or a specific mismatch result. + * + * Side Effects: + * None. + * + * Author: SqlRush + */ +PcmXAttachResult +cluster_pcm_x_validate_attach(const PcmXShmemHeader *header, const PcmXShmemLayout *expected) +{ + if (header == NULL || expected == NULL) + return PCM_X_ATTACH_NULL; + if (header->layout.magic != expected->magic) + return PCM_X_ATTACH_MAGIC_MISMATCH; + if (header->layout.version != expected->version) + return PCM_X_ATTACH_VERSION_MISMATCH; + if (memcmp(&header->layout, expected, sizeof(*expected)) != 0) + return PCM_X_ATTACH_LAYOUT_MISMATCH; + return PCM_X_ATTACH_OK; +} + + +/* + * cluster_pcm_x_generation_next -- Advance a bounded generation counter. + * + * Inputs: + * current: current generation; zero is valid. + * next: writable output for the successor. + * + * Returns: + * True after storing the successor. False for a null output or UINT64_MAX; + * in that case, caller storage is unchanged. + * + * Side Effects: + * None beyond writing *next on success. + * + * Author: SqlRush + */ +bool +cluster_pcm_x_generation_next(uint64 current, uint64 *next) +{ + if (next == NULL || current == UINT64_MAX) + return false; + *next = current + 1; + return true; +} + + +static bool +pcm_x_slot_generation_read(const PcmXSlotHeader *slot, uint64 *generation_out) +{ + uint32 after; + uint32 before; + uint64 generation; + int retry; + + if (slot == NULL || generation_out == NULL) + return false; + *generation_out = 0; + for (retry = 0; retry < PCM_X_SLOT_GENERATION_READ_RETRIES; retry++) { + before = pg_atomic_read_u32((volatile pg_atomic_uint32 *)&slot->generation_change_seq); + if ((before & 1U) != 0) + continue; + pg_read_barrier(); + generation = ((uint64)slot->slot_generation_hi << 32) | (uint64)slot->slot_generation_lo; + pg_read_barrier(); + after = pg_atomic_read_u32((volatile pg_atomic_uint32 *)&slot->generation_change_seq); + if (before == after && (after & 1U) == 0) { + *generation_out = generation; + return true; + } + } + return false; +} + + +/* Caller holds the allocator lock for this slot. */ +static void +pcm_x_slot_generation_write(PcmXSlotHeader *slot, uint64 generation) +{ + uint32 odd; + + Assert(ClusterPcmXConvertShmem != NULL); + Assert(LWLockHeldByMeInMode(&ClusterPcmXConvertShmem->allocator_lock.lock, LW_EXCLUSIVE)); + odd = pg_atomic_read_u32(&slot->generation_change_seq) | 1U; + pg_atomic_write_u32(&slot->generation_change_seq, odd); + pg_write_barrier(); + slot->slot_generation_lo = (uint32)generation; + slot->slot_generation_hi = (uint32)(generation >> 32); + pg_write_barrier(); + pg_atomic_write_u32(&slot->generation_change_seq, odd + 1U); +} + + +static uint32 +pcm_x_slot_state_read(const PcmXSlotHeader *slot) +{ + return pg_atomic_read_u32((volatile pg_atomic_uint32 *)&slot->state_flags) + & PCM_X_SLOT_STATE_MASK; +} + + +static void +pcm_x_slot_state_write(PcmXSlotHeader *slot, uint32 state) +{ + uint32 current; + uint32 desired; + + if (slot == NULL || (state & ~PCM_X_SLOT_STATE_MASK) != 0) + return; + current = pg_atomic_read_u32(&slot->state_flags); + do { + desired = (current & PCM_X_SLOT_FLAGS_MASK) | state; + } while (!pg_atomic_compare_exchange_u32(&slot->state_flags, ¤t, desired)); +} + + +static void +pcm_x_slot_state_flags_write(PcmXSlotHeader *slot, uint32 state, uint32 flags) +{ + uint32 flag_value_mask = PCM_X_SLOT_FLAGS_MASK >> PCM_X_SLOT_FLAGS_SHIFT; + + if (slot == NULL || (state & ~PCM_X_SLOT_STATE_MASK) != 0 || (flags & ~flag_value_mask) != 0) + return; + pg_atomic_write_u32(&slot->state_flags, state | (flags << PCM_X_SLOT_FLAGS_SHIFT)); +} + + +static uint32 +pcm_x_slot_flags_read(const PcmXSlotHeader *slot) +{ + if (slot == NULL) + return 0; + return pg_atomic_read_u32((volatile pg_atomic_uint32 *)&slot->state_flags) + >> PCM_X_SLOT_FLAGS_SHIFT; +} + + +static bool +pcm_x_slot_flags_compare_exchange(PcmXSlotHeader *slot, uint32 *expected_flags, + uint32 desired_flags) +{ + uint32 current; + uint32 current_flags; + uint32 desired; + uint32 flag_value_mask = PCM_X_SLOT_FLAGS_MASK >> PCM_X_SLOT_FLAGS_SHIFT; + + if (slot == NULL || expected_flags == NULL || (desired_flags & ~flag_value_mask) != 0) + return false; + current = pg_atomic_read_u32(&slot->state_flags); + for (;;) { + current_flags = current >> PCM_X_SLOT_FLAGS_SHIFT; + if (current_flags != *expected_flags) { + *expected_flags = current_flags; + return false; + } + desired = (current & PCM_X_SLOT_STATE_MASK) | (desired_flags << PCM_X_SLOT_FLAGS_SHIFT); + if (pg_atomic_compare_exchange_u32(&slot->state_flags, ¤t, desired)) + return true; + } +} + + +static void +pcm_x_slot_flags_write(PcmXSlotHeader *slot, uint32 flags) +{ + uint32 current; + uint32 desired; + uint32 flag_value_mask = PCM_X_SLOT_FLAGS_MASK >> PCM_X_SLOT_FLAGS_SHIFT; + + if (slot == NULL || (flags & ~flag_value_mask) != 0) + return; + current = pg_atomic_read_u32(&slot->state_flags); + do { + desired = (current & PCM_X_SLOT_STATE_MASK) | (flags << PCM_X_SLOT_FLAGS_SHIFT); + } while (!pg_atomic_compare_exchange_u32(&slot->state_flags, ¤t, desired)); +} + + +static uint32 +pcm_x_slot_flags_fetch_or(PcmXSlotHeader *slot, uint32 flags) +{ + uint32 current; + uint32 desired; + uint32 old_flags; + uint32 flag_value_mask = PCM_X_SLOT_FLAGS_MASK >> PCM_X_SLOT_FLAGS_SHIFT; + + if (slot == NULL || (flags & ~flag_value_mask) != 0) + return 0; + current = pg_atomic_read_u32(&slot->state_flags); + for (;;) { + old_flags = current >> PCM_X_SLOT_FLAGS_SHIFT; + desired + = (current & PCM_X_SLOT_STATE_MASK) | ((old_flags | flags) << PCM_X_SLOT_FLAGS_SHIFT); + if (pg_atomic_compare_exchange_u32(&slot->state_flags, ¤t, desired)) + return old_flags; + } +} + + +static uint32 +pcm_x_slot_flags_fetch_and(PcmXSlotHeader *slot, uint32 flags) +{ + uint32 current; + uint32 desired; + uint32 old_flags; + uint32 flag_value_mask = PCM_X_SLOT_FLAGS_MASK >> PCM_X_SLOT_FLAGS_SHIFT; + + if (slot == NULL) + return 0; + flags &= flag_value_mask; + current = pg_atomic_read_u32(&slot->state_flags); + for (;;) { + old_flags = current >> PCM_X_SLOT_FLAGS_SHIFT; + desired + = (current & PCM_X_SLOT_STATE_MASK) | ((old_flags & flags) << PCM_X_SLOT_FLAGS_SHIFT); + if (pg_atomic_compare_exchange_u32(&slot->state_flags, ¤t, desired)) + return old_flags; + } +} + + +/* + * Make one FREE slot non-visible under the sole allocator lock. + * + * The checked generation bump happens before a caller can publish a directory + * entry. Exhaustion is itself persisted in the slot and removes the slot from + * reuse; a stale generation can therefore never wrap to zero. + */ +static bool +pcm_x_slot_generation_reserve_locked(PcmXSlotHeader *slot) +{ + uint64 current; + uint64 next; + + if (slot == NULL || pcm_x_slot_state_read(slot) != PCM_X_SLOT_FREE + || pcm_x_slot_flags_read(slot) != 0) + return false; + if (!pcm_x_slot_generation_read(slot, ¤t)) + return false; + if (!cluster_pcm_x_generation_next(current, &next)) { + slot->next_free = PCM_X_INVALID_SLOT_INDEX; + pg_write_barrier(); + pcm_x_slot_state_write(slot, PCM_X_SLOT_GENERATION_EXHAUSTED); + return false; + } + + slot->next_free = PCM_X_INVALID_SLOT_INDEX; + pcm_x_slot_generation_write(slot, next); + pg_write_barrier(); + pcm_x_slot_state_write(slot, PCM_X_SLOT_RESERVED_NONVISIBLE); + return true; +} + + +typedef struct PcmXAllocatorView { + Size slots_offset; + Size slot_size; + Size first_slot_index; + Size capacity; + PcmXAllocatorState *state; +} PcmXAllocatorView; + +typedef struct PcmXDirectoryView { + PcmXDirectoryEntry *entries; + Size capacity; + PcmXAllocatorKind allocator_kind; +} PcmXDirectoryView; + +typedef enum PcmXDirectoryMatch { + PCM_X_DIRECTORY_MATCH_EXACT = 0, + PCM_X_DIRECTORY_MATCH_DIFFERENT, + PCM_X_DIRECTORY_MATCH_STALE, + PCM_X_DIRECTORY_MATCH_CORRUPT +} PcmXDirectoryMatch; + + +static void pcm_x_runtime_fail_closed(void); + + +/* Required runtime precondition: never nest allocator under a tag domain. */ +static bool +pcm_x_allocator_entry_unlocked(const PcmXShmemHeader *header) +{ + return !LWLockHeldByMe((LWLock *)&header->allocator_lock.lock) + && !LWLockAnyHeldByMe((LWLock *)&header->master_locks[0].lock, PCM_X_LOCK_PARTITIONS, + sizeof(LWLockPadded)) + && !LWLockAnyHeldByMe((LWLock *)&header->local_locks[0].lock, PCM_X_LOCK_PARTITIONS, + sizeof(LWLockPadded)); +} + + +static bool +pcm_x_allocator_view(PcmXAllocatorKind kind, PcmXAllocatorView *view) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + const PcmXPoolLayout *pool; + + if (header == NULL || view == NULL || kind < 0 || kind >= PCM_X_ALLOC_COUNT) + return false; + memset(view, 0, sizeof(*view)); + view->state = &header->allocator[kind]; + + switch (kind) { + case PCM_X_ALLOC_MASTER_TAG: + pool = &header->layout.pools[PCM_X_POOL_MASTER_TAG]; + break; + case PCM_X_ALLOC_MASTER_TICKET: + pool = &header->layout.pools[PCM_X_POOL_MASTER_TICKET]; + break; + case PCM_X_ALLOC_BLOCKER: + pool = &header->layout.pools[PCM_X_POOL_BLOCKER]; + break; + case PCM_X_ALLOC_LOCAL_TAG: + pool = &header->layout.pools[PCM_X_POOL_LOCAL_TAG]; + break; + case PCM_X_ALLOC_LOCAL_WAIT: + view->slots_offset = header->layout.local_wait.slots_offset; + view->slot_size = sizeof(PcmXLocalMembershipSlot); + view->first_slot_index = header->layout.local_wait.first_slot_index; + view->capacity = header->layout.local_wait.capacity; + return true; + case PCM_X_ALLOC_LOCAL_HOLDER: + view->slots_offset = header->layout.local_holder.slots_offset; + view->slot_size = sizeof(PcmXLocalMembershipSlot); + view->first_slot_index = header->layout.local_holder.first_slot_index; + view->capacity = header->layout.local_holder.capacity; + return true; + case PCM_X_ALLOC_COUNT: + return false; + } + + view->slots_offset = pool->slots_offset; + view->slot_size = pool->slot_size; + view->first_slot_index = 0; + view->capacity = pool->capacity; + return true; +} + + +static PcmXSlotHeader * +pcm_x_allocator_slot(const PcmXAllocatorView *view, Size slot_index) +{ + Size local_index; + + if (view == NULL || slot_index < view->first_slot_index) + return NULL; + local_index = slot_index - view->first_slot_index; + if (local_index >= view->capacity) + return NULL; + return (PcmXSlotHeader *)((char *)ClusterPcmXConvertShmem + view->slots_offset + + local_index * view->slot_size); +} + + +/* Validate the O(1) allocator conservation facts before mutating its free list. */ +static bool +pcm_x_allocator_state_valid_locked(const PcmXAllocatorView *view) +{ + PcmXSlotHeader *head; + Size available; + + if (view == NULL || view->state == NULL || view->slot_size < sizeof(PcmXSlotHeader) + || view->state->generation_exhausted > view->capacity) + return false; + available = view->capacity - view->state->generation_exhausted; + if (view->state->used > available || view->state->high_water > view->capacity + || view->state->high_water < view->state->used) + return false; + if (view->state->free_head == PCM_X_INVALID_SLOT_INDEX) + return view->state->used == available; + if (view->state->used >= available) + return false; + head = pcm_x_allocator_slot(view, view->state->free_head); + return head != NULL && pcm_x_slot_state_read(head) == PCM_X_SLOT_FREE; +} + + +/* Reserve from one fixed allocator while the caller owns allocator_lock. */ +static PcmXAllocatorResult +pcm_x_allocator_reserve_locked(PcmXAllocatorKind kind, PcmXSlotRef *ref_out, + PcmXSlotHeader **slot_out) +{ + PcmXAllocatorView view; + PcmXAllocatorResult result = PCM_X_ALLOC_NO_CAPACITY; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + Size scanned = 0; + + if (header == NULL || ref_out == NULL || slot_out == NULL + || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE) + || !pcm_x_allocator_view(kind, &view)) + return PCM_X_ALLOC_INVALID; + ref_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + ref_out->slot_generation = 0; + *slot_out = NULL; + if (!pcm_x_allocator_state_valid_locked(&view)) + return PCM_X_ALLOC_CORRUPT; + + while (view.state->free_head != PCM_X_INVALID_SLOT_INDEX && scanned < view.capacity) { + PcmXSlotHeader *slot; + PcmXSlotHeader *successor = NULL; + Size slot_index = view.state->free_head; + Size saved_next; + Size free_slots; + + slot = pcm_x_allocator_slot(&view, slot_index); + if (slot == NULL || pcm_x_slot_state_read(slot) != PCM_X_SLOT_FREE) + return PCM_X_ALLOC_CORRUPT; + saved_next = slot->next_free; + if (saved_next != PCM_X_INVALID_SLOT_INDEX) + successor = pcm_x_allocator_slot(&view, saved_next); + if (saved_next == slot_index + || (saved_next != PCM_X_INVALID_SLOT_INDEX + && (successor == NULL || pcm_x_slot_state_read(successor) != PCM_X_SLOT_FREE))) + return PCM_X_ALLOC_CORRUPT; + free_slots = view.capacity - view.state->generation_exhausted - view.state->used; + if ((saved_next == PCM_X_INVALID_SLOT_INDEX) != (free_slots == 1)) + return PCM_X_ALLOC_CORRUPT; + + view.state->free_head = saved_next; + scanned++; + if (!pcm_x_slot_generation_reserve_locked(slot)) { + if (pcm_x_slot_state_read(slot) != PCM_X_SLOT_GENERATION_EXHAUSTED + || view.state->generation_exhausted >= view.capacity) + return PCM_X_ALLOC_CORRUPT; + view.state->generation_exhausted++; + if (!pcm_x_allocator_state_valid_locked(&view)) + return PCM_X_ALLOC_CORRUPT; + continue; + } + + view.state->used++; + view.state->high_water = Max(view.state->high_water, view.state->used); + if (!pcm_x_allocator_state_valid_locked(&view)) + return PCM_X_ALLOC_CORRUPT; + ref_out->slot_index = slot_index; + if (!pcm_x_slot_generation_read(slot, &ref_out->slot_generation)) + return PCM_X_ALLOC_CORRUPT; + *slot_out = slot; + return PCM_X_ALLOC_OK; + } + if (scanned == view.capacity && view.state->free_head != PCM_X_INVALID_SLOT_INDEX) + result = PCM_X_ALLOC_CORRUPT; + if (result == PCM_X_ALLOC_NO_CAPACITY && !pcm_x_allocator_state_valid_locked(&view)) + result = PCM_X_ALLOC_CORRUPT; + return result; +} + + +/* Release one RESERVED/DETACHING slot while the caller owns allocator_lock. */ +static PcmXAllocatorResult +pcm_x_allocator_release_locked(PcmXAllocatorKind kind, PcmXSlotRef expected, uint32 expected_state) +{ + PcmXAllocatorView view; + PcmXAllocatorResult result = PCM_X_ALLOC_OK; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXSlotHeader *slot; + Size old_head; + uint64 generation; + + if (header == NULL + || (expected_state != PCM_X_SLOT_RESERVED_NONVISIBLE + && expected_state != PCM_X_SLOT_DETACHING) + || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE) + || !pcm_x_allocator_view(kind, &view)) + return PCM_X_ALLOC_INVALID; + if (!pcm_x_allocator_state_valid_locked(&view)) + return PCM_X_ALLOC_CORRUPT; + slot = pcm_x_allocator_slot(&view, expected.slot_index); + if (slot == NULL) + return PCM_X_ALLOC_INVALID; + if (!pcm_x_slot_generation_read(slot, &generation)) + return PCM_X_ALLOC_CORRUPT; + if (generation != expected.slot_generation) + return PCM_X_ALLOC_STALE_REF; + if (pcm_x_slot_state_read(slot) != expected_state) + return PCM_X_ALLOC_BAD_STATE; + if (view.state->used == 0) + return PCM_X_ALLOC_CORRUPT; + + old_head = view.state->free_head; + if (old_head != PCM_X_INVALID_SLOT_INDEX + && (pcm_x_allocator_slot(&view, old_head) == NULL + || pcm_x_slot_state_read(pcm_x_allocator_slot(&view, old_head)) != PCM_X_SLOT_FREE)) + return PCM_X_ALLOC_CORRUPT; + memset((char *)slot + sizeof(PcmXSlotHeader), 0, view.slot_size - sizeof(PcmXSlotHeader)); + slot->next_free = old_head; + pcm_x_slot_generation_write(slot, generation); + pg_write_barrier(); + pcm_x_slot_state_flags_write(slot, PCM_X_SLOT_FREE, 0); + view.state->free_head = expected.slot_index; + view.state->used--; + if (!pcm_x_allocator_state_valid_locked(&view)) + result = PCM_X_ALLOC_CORRUPT; + return result; +} + + +static PcmXSlotHeader * +pcm_x_slot_ref_resolve_locked(PcmXAllocatorKind kind, PcmXSlotRef expected) +{ + PcmXAllocatorView view; + PcmXSlotHeader *slot; + uint64 generation; + + if (!pcm_x_allocator_view(kind, &view)) + return NULL; + slot = pcm_x_allocator_slot(&view, expected.slot_index); + if (slot == NULL) + return NULL; + if (!pcm_x_slot_generation_read(slot, &generation)) { + pcm_x_runtime_fail_closed(); + return NULL; + } + if (generation != expected.slot_generation || pcm_x_slot_state_read(slot) == PCM_X_SLOT_FREE + || pcm_x_slot_state_read(slot) == PCM_X_SLOT_GENERATION_EXHAUSTED) + return NULL; + return slot; +} + + +static void +pcm_x_init_stats(PcmXStats *stats) +{ + pg_atomic_init_u64(&stats->enqueue_count, 0); + pg_atomic_init_u64(&stats->admit_count, 0); + pg_atomic_init_u64(&stats->confirm_count, 0); + pg_atomic_init_u64(&stats->promotion_count, 0); + pg_atomic_init_u64(&stats->transfer_count, 0); + pg_atomic_init_u64(&stats->complete_count, 0); + pg_atomic_init_u64(&stats->cancel_count, 0); + pg_atomic_init_u64(&stats->revoke_count, 0); + pg_atomic_init_u64(&stats->coalesced_count, 0); + pg_atomic_init_u64(&stats->wait_count, 0); + pg_atomic_init_u64(&stats->full_count, 0); + pg_atomic_init_u64(&stats->stale_count, 0); + pg_atomic_init_u64(&stats->miss_count, 0); + pg_atomic_init_u64(&stats->recovery_blocked_count, 0); + pg_atomic_init_u64(&stats->activating_reset_count, 0); + pg_atomic_init_u64(&stats->depth, 0); + pg_atomic_init_u64(&stats->depth_high_water, 0); + pg_atomic_init_u64(&stats->own_begin_count, 0); + pg_atomic_init_u64(&stats->own_commit_count, 0); + pg_atomic_init_u64(&stats->own_abort_count, 0); + pg_atomic_init_u64(&stats->own_busy_count, 0); + pg_atomic_init_u64(&stats->own_corrupt_count, 0); +} + + +static void +pcm_x_stats_increment(pg_atomic_uint64 *counter) +{ + if (counter != NULL) + (void)pg_atomic_fetch_add_u64(counter, 1); +} + + +static bool +pcm_x_stats_depth_increment(PcmXShmemHeader *header) +{ + uint64 depth; + uint64 observed; + + if (header == NULL) + return false; + observed = pg_atomic_read_u64(&header->stats.depth); + for (;;) { + uint64 expected; + + if (observed == UINT64_MAX) + return false; + expected = observed; + if (pg_atomic_compare_exchange_u64(&header->stats.depth, &expected, observed + 1)) + break; + observed = expected; + } + depth = observed + 1; + observed = pg_atomic_read_u64(&header->stats.depth_high_water); + while (depth > observed) { + uint64 expected = observed; + + if (pg_atomic_compare_exchange_u64(&header->stats.depth_high_water, &expected, depth)) + break; + observed = expected; + } + return true; +} + + +static bool +pcm_x_stats_depth_decrement(PcmXShmemHeader *header) +{ + uint64 observed; + + if (header == NULL) + return false; + observed = pg_atomic_read_u64(&header->stats.depth); + while (observed != 0) { + uint64 expected = observed; + + if (pg_atomic_compare_exchange_u64(&header->stats.depth, &expected, observed - 1)) + return true; + observed = expected; + } + return false; +} + + +bool +cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + bool allocator_lock_held; + int i; + + if (snapshot_out == NULL) + return false; + memset(snapshot_out, 0, sizeof(*snapshot_out)); + if (header == NULL) + return false; + + allocator_lock_held = LWLockHeldByMe(&header->allocator_lock.lock); + if (!allocator_lock_held) + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + snapshot_out->enqueue_count = pg_atomic_read_u64(&header->stats.enqueue_count); + snapshot_out->admit_count = pg_atomic_read_u64(&header->stats.admit_count); + snapshot_out->confirm_count = pg_atomic_read_u64(&header->stats.confirm_count); + snapshot_out->promotion_count = pg_atomic_read_u64(&header->stats.promotion_count); + snapshot_out->transfer_count = pg_atomic_read_u64(&header->stats.transfer_count); + snapshot_out->complete_count = pg_atomic_read_u64(&header->stats.complete_count); + snapshot_out->cancel_count = pg_atomic_read_u64(&header->stats.cancel_count); + snapshot_out->revoke_count = pg_atomic_read_u64(&header->stats.revoke_count); + snapshot_out->coalesced_count = pg_atomic_read_u64(&header->stats.coalesced_count); + snapshot_out->wait_count = pg_atomic_read_u64(&header->stats.wait_count); + snapshot_out->full_count = pg_atomic_read_u64(&header->stats.full_count); + snapshot_out->stale_count = pg_atomic_read_u64(&header->stats.stale_count); + snapshot_out->miss_count = pg_atomic_read_u64(&header->stats.miss_count); + snapshot_out->recovery_blocked_count + = pg_atomic_read_u64(&header->stats.recovery_blocked_count); + snapshot_out->activating_reset_count + = pg_atomic_read_u64(&header->stats.activating_reset_count); + snapshot_out->depth = pg_atomic_read_u64(&header->stats.depth); + snapshot_out->depth_high_water = pg_atomic_read_u64(&header->stats.depth_high_water); + snapshot_out->own_begin_count = pg_atomic_read_u64(&header->stats.own_begin_count); + snapshot_out->own_commit_count = pg_atomic_read_u64(&header->stats.own_commit_count); + snapshot_out->own_abort_count = pg_atomic_read_u64(&header->stats.own_abort_count); + snapshot_out->own_busy_count = pg_atomic_read_u64(&header->stats.own_busy_count); + snapshot_out->own_corrupt_count = pg_atomic_read_u64(&header->stats.own_corrupt_count); + snapshot_out->active_tags = (uint64)header->allocator[PCM_X_ALLOC_MASTER_TAG].used; + snapshot_out->live_tickets = (uint64)header->allocator[PCM_X_ALLOC_MASTER_TICKET].used; + snapshot_out->local_retire_gate = (uint64)pg_atomic_read_u32(&header->local_retire_gate); + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + uint64 marker = header->peer_frontiers[i].local_retire_in_progress_ticket_id; + + if (marker == 0) + continue; + if (snapshot_out->local_retire_marker_count == 0) + snapshot_out->local_retire_marker_ticket_id = marker; + snapshot_out->local_retire_marker_count++; + } + for (i = 0; i < PCM_X_ALLOC_COUNT; i++) + snapshot_out->live_slots += (uint64)header->allocator[i].used; + if (!allocator_lock_held) + LWLockRelease(&header->allocator_lock.lock); + return true; +} + + +void +cluster_pcm_x_stats_note_enqueue(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.enqueue_count); +} + + +void +cluster_pcm_x_stats_note_wait(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.wait_count); +} + + +void +cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header == NULL) + return; + switch (result) { + case PCM_X_QUEUE_NO_CAPACITY: + pcm_x_stats_increment(&header->stats.full_count); + break; + case PCM_X_QUEUE_STALE: + pcm_x_stats_increment(&header->stats.stale_count); + break; + case PCM_X_QUEUE_NOT_FOUND: + pcm_x_stats_increment(&header->stats.miss_count); + break; + default: + break; + } +} + + +void +cluster_pcm_x_stats_note_own_begin(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.own_begin_count); +} + + +void +cluster_pcm_x_stats_note_own_commit(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.own_commit_count); +} + + +void +cluster_pcm_x_stats_note_own_abort(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.own_abort_count); +} + + +void +cluster_pcm_x_stats_note_own_busy(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.own_busy_count); +} + + +void +cluster_pcm_x_stats_note_own_corrupt(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header != NULL) + pcm_x_stats_increment(&header->stats.own_corrupt_count); +} + + +static void +pcm_x_runtime_fail_closed(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + bool transitioned; + + transitioned + = cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED); + if (!transitioned) + transitioned = cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_SHUTTING_DOWN, + PCM_X_RUNTIME_RECOVERY_BLOCKED); + if (transitioned && header != NULL) + pcm_x_stats_increment(&header->stats.recovery_blocked_count); +} + + +void +cluster_pcm_x_runtime_fail_closed(void) +{ + pcm_x_runtime_fail_closed(); +} + + +/* Reserve one slot under the sole allocator authority. */ +PcmXAllocatorResult +cluster_pcm_x_allocator_reserve(PcmXAllocatorKind kind, PcmXSlotRef *ref_out, + PcmXSlotHeader **slot_out) +{ + PcmXAllocatorView view; + PcmXAllocatorResult result = PCM_X_ALLOC_NO_CAPACITY; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + Size scanned = 0; + + if (ref_out == NULL || slot_out == NULL || header == NULL || !pcm_x_allocator_view(kind, &view)) + return PCM_X_ALLOC_INVALID; + ref_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + ref_out->slot_generation = 0; + *slot_out = NULL; + if (!pcm_x_allocator_entry_unlocked(header)) { + pcm_x_runtime_fail_closed(); + return PCM_X_ALLOC_INVALID; + } + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_allocator_state_valid_locked(&view)) { + result = PCM_X_ALLOC_CORRUPT; + goto done; + } + while (view.state->free_head != PCM_X_INVALID_SLOT_INDEX && scanned < view.capacity) { + PcmXSlotHeader *slot; + PcmXSlotHeader *successor = NULL; + Size slot_index = view.state->free_head; + Size saved_next; + Size free_slots; + + slot = pcm_x_allocator_slot(&view, slot_index); + if (slot == NULL || pcm_x_slot_state_read(slot) != PCM_X_SLOT_FREE) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + saved_next = slot->next_free; + if (saved_next != PCM_X_INVALID_SLOT_INDEX) + successor = pcm_x_allocator_slot(&view, saved_next); + if (saved_next == slot_index + || (saved_next != PCM_X_INVALID_SLOT_INDEX + && (successor == NULL || pcm_x_slot_state_read(successor) != PCM_X_SLOT_FREE))) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + free_slots = view.capacity - view.state->generation_exhausted - view.state->used; + if ((saved_next == PCM_X_INVALID_SLOT_INDEX) != (free_slots == 1)) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + + /* Save and unlink the successor before reserve clears next_free. */ + view.state->free_head = saved_next; + scanned++; + if (!pcm_x_slot_generation_reserve_locked(slot)) { + if (pcm_x_slot_state_read(slot) != PCM_X_SLOT_GENERATION_EXHAUSTED + || view.state->generation_exhausted >= view.capacity) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + view.state->generation_exhausted++; + if (!pcm_x_allocator_state_valid_locked(&view)) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + continue; + } + + view.state->used++; + view.state->high_water = Max(view.state->high_water, view.state->used); + if (!pcm_x_allocator_state_valid_locked(&view)) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + ref_out->slot_index = slot_index; + if (!pcm_x_slot_generation_read(slot, &ref_out->slot_generation)) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + *slot_out = slot; + result = PCM_X_ALLOC_OK; + break; + } + if (result == PCM_X_ALLOC_NO_CAPACITY && scanned == view.capacity + && view.state->free_head != PCM_X_INVALID_SLOT_INDEX) + result = PCM_X_ALLOC_CORRUPT; + if (result == PCM_X_ALLOC_NO_CAPACITY && !pcm_x_allocator_state_valid_locked(&view)) + result = PCM_X_ALLOC_CORRUPT; + +done: + LWLockRelease(&header->allocator_lock.lock); + + if (result == PCM_X_ALLOC_CORRUPT) { + ref_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + ref_out->slot_generation = 0; + *slot_out = NULL; + pcm_x_runtime_fail_closed(); + } + return result; +} + + +/* Release an unpublished rollback or a quiesced DETACHING slot by exact ref. */ +PcmXAllocatorResult +cluster_pcm_x_allocator_release_exact(PcmXAllocatorKind kind, PcmXSlotRef expected, + uint32 expected_state) +{ + PcmXAllocatorView view; + PcmXAllocatorResult result = PCM_X_ALLOC_OK; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXSlotHeader *slot; + Size old_head; + uint64 generation; + + if (header == NULL + || (expected_state != PCM_X_SLOT_RESERVED_NONVISIBLE + && expected_state != PCM_X_SLOT_DETACHING) + || !pcm_x_allocator_view(kind, &view)) + return PCM_X_ALLOC_INVALID; + if (!pcm_x_allocator_entry_unlocked(header)) { + pcm_x_runtime_fail_closed(); + return PCM_X_ALLOC_INVALID; + } + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_allocator_state_valid_locked(&view)) { + result = PCM_X_ALLOC_CORRUPT; + goto done; + } + slot = pcm_x_allocator_slot(&view, expected.slot_index); + if (slot == NULL) + result = PCM_X_ALLOC_INVALID; + else if (!pcm_x_slot_generation_read(slot, &generation)) + result = PCM_X_ALLOC_CORRUPT; + else if (generation != expected.slot_generation) + result = PCM_X_ALLOC_STALE_REF; + else if (pcm_x_slot_state_read(slot) != expected_state) + result = PCM_X_ALLOC_BAD_STATE; + else if (view.state->used == 0) + result = PCM_X_ALLOC_CORRUPT; + else { + old_head = view.state->free_head; + if (old_head != PCM_X_INVALID_SLOT_INDEX + && (pcm_x_allocator_slot(&view, old_head) == NULL + || pcm_x_slot_state_read(pcm_x_allocator_slot(&view, old_head)) != PCM_X_SLOT_FREE)) + result = PCM_X_ALLOC_CORRUPT; + else { + memset((char *)slot + sizeof(PcmXSlotHeader), 0, + view.slot_size - sizeof(PcmXSlotHeader)); + slot->next_free = old_head; + pcm_x_slot_generation_write(slot, generation); + pg_write_barrier(); + pcm_x_slot_state_flags_write(slot, PCM_X_SLOT_FREE, 0); + view.state->free_head = expected.slot_index; + view.state->used--; + if (!pcm_x_allocator_state_valid_locked(&view)) + result = PCM_X_ALLOC_CORRUPT; + } + } + +done: + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_ALLOC_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* + * Revalidate a directory ref only after the caller owns its tag-domain lock. + * + * The allocator deliberately does not nest that lock. A delayed lookup can + * therefore overlap FREE->RESERVED reuse. The monotonic generation counter + * brackets the complete lifecycle-state and BufferTag snapshot; a torn + * multi-field read or slot reuse cannot escape as one incarnation. Once + * accepted, DETACHING must take the same domain lock before the allocator is + * allowed to reclaim the slot. + */ +static const BufferTag * +pcm_x_slot_tag(PcmXAllocatorKind kind, const PcmXSlotHeader *slot) +{ + switch (kind) { + case PCM_X_ALLOC_MASTER_TAG: + return &((const PcmXMasterTagSlot *)slot)->tag; + case PCM_X_ALLOC_MASTER_TICKET: + return &((const PcmXMasterTicketSlot *)slot)->ref.identity.tag; + case PCM_X_ALLOC_LOCAL_TAG: + return &((const PcmXLocalTagSlot *)slot)->tag; + case PCM_X_ALLOC_LOCAL_WAIT: + case PCM_X_ALLOC_LOCAL_HOLDER: + return &((const PcmXLocalMembershipSlot *)slot)->identity.tag; + case PCM_X_ALLOC_BLOCKER: + case PCM_X_ALLOC_COUNT: + return NULL; + } + return NULL; +} + + +PcmXSlotHeader * +cluster_pcm_x_slot_ref_revalidate(PcmXAllocatorKind kind, PcmXSlotRef expected, + uint32 expected_state, const BufferTag *expected_tag, + uint32 tag_hash) +{ + PcmXAllocatorView view; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXSlotHeader *slot; + const BufferTag *slot_tag; + BufferTag tag_snapshot; + uint64 generation1; + uint64 generation2; + uint32 state1; + uint32 state2; + uint32 authoritative_hash; + uint32 partition; + + /* Blockers have no independent tag; revalidate them through their owner ticket. */ + if (header == NULL || expected_tag == NULL || kind == PCM_X_ALLOC_BLOCKER + || !pcm_x_allocator_view(kind, &view) || expected_state == PCM_X_SLOT_FREE + || expected_state == PCM_X_SLOT_RESERVED_NONVISIBLE + || expected_state == PCM_X_SLOT_GENERATION_EXHAUSTED) + return NULL; + authoritative_hash = cluster_pcm_x_tag_hash(expected_tag); + if (tag_hash != authoritative_hash) { + pcm_x_runtime_fail_closed(); + return NULL; + } + partition = cluster_pcm_x_lock_partition(authoritative_hash); + if (LWLockHeldByMe(&header->allocator_lock.lock) + || ((kind == PCM_X_ALLOC_MASTER_TAG || kind == PCM_X_ALLOC_MASTER_TICKET) + && !LWLockHeldByMe(&header->master_locks[partition].lock)) + || (kind >= PCM_X_ALLOC_LOCAL_TAG + && !LWLockHeldByMe(&header->local_locks[partition].lock))) { + pcm_x_runtime_fail_closed(); + return NULL; + } + + slot = pcm_x_allocator_slot(&view, expected.slot_index); + if (slot == NULL) + return NULL; + /* Allocator reuse does not nest the domain lock. An unstable seqlock is a + * legal overlap here; the caller retries from its exact directory ref. */ + if (!pcm_x_slot_generation_read(slot, &generation1)) + return NULL; + if (generation1 != expected.slot_generation) + return NULL; + pg_read_barrier(); + state1 = pcm_x_slot_state_read(slot); + if (state1 != expected_state) + return NULL; + pg_read_barrier(); + slot_tag = pcm_x_slot_tag(kind, slot); + if (slot_tag == NULL) + return NULL; + tag_snapshot = *slot_tag; + pg_read_barrier(); + state2 = pcm_x_slot_state_read(slot); + pg_read_barrier(); + if (!pcm_x_slot_generation_read(slot, &generation2)) + return NULL; + if (state1 != state2 || generation1 != generation2) + return NULL; + if (!BufferTagsEqual(&tag_snapshot, expected_tag)) + return NULL; + return slot; +} + + +static bool +pcm_x_directory_view(PcmXDirectoryKind kind, PcmXDirectoryView *view) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + Size offset; + + if (header == NULL || view == NULL || kind < 0 || kind >= PCM_X_DIR_COUNT) + return false; + memset(view, 0, sizeof(*view)); + + switch (kind) { + case PCM_X_DIR_MASTER_TAG: + offset = header->layout.pools[PCM_X_POOL_MASTER_TAG].directory_offset; + view->capacity = header->layout.pools[PCM_X_POOL_MASTER_TAG].directory_capacity; + view->allocator_kind = PCM_X_ALLOC_MASTER_TAG; + break; + case PCM_X_DIR_MASTER_TICKET_PREHANDLE: + offset = header->layout.master_ticket_directories.prehandle_offset; + view->capacity = header->layout.master_ticket_directories.prehandle_capacity; + view->allocator_kind = PCM_X_ALLOC_MASTER_TICKET; + break; + case PCM_X_DIR_MASTER_TICKET_HANDLE: + offset = header->layout.master_ticket_directories.handle_offset; + view->capacity = header->layout.master_ticket_directories.handle_capacity; + view->allocator_kind = PCM_X_ALLOC_MASTER_TICKET; + break; + case PCM_X_DIR_MASTER_TICKET_RETIRE: + offset = header->layout.master_ticket_directories.retire_offset; + view->capacity = header->layout.master_ticket_directories.retire_capacity; + view->allocator_kind = PCM_X_ALLOC_MASTER_TICKET; + break; + case PCM_X_DIR_LOCAL_TAG: + offset = header->layout.pools[PCM_X_POOL_LOCAL_TAG].directory_offset; + view->capacity = header->layout.pools[PCM_X_POOL_LOCAL_TAG].directory_capacity; + view->allocator_kind = PCM_X_ALLOC_LOCAL_TAG; + break; + case PCM_X_DIR_LOCAL_WAIT: + offset = header->layout.local_wait.directory_offset; + view->capacity = header->layout.local_wait.directory_capacity; + view->allocator_kind = PCM_X_ALLOC_LOCAL_WAIT; + break; + case PCM_X_DIR_LOCAL_HOLDER: + offset = header->layout.local_holder.directory_offset; + view->capacity = header->layout.local_holder.directory_capacity; + view->allocator_kind = PCM_X_ALLOC_LOCAL_HOLDER; + break; + case PCM_X_DIR_COUNT: + return false; + } + if (view->capacity == 0) + return false; + view->entries = (PcmXDirectoryEntry *)((char *)header + offset); + return true; +} + + +static uint64 +pcm_x_hash_scalar(uint64 hash, uint64 value) +{ + int i; + + for (i = 0; i < 8; i++) { + hash ^= (uint8)(value >> (i * 8)); + hash *= UINT64CONST(1099511628211); + } + return hash; +} + + +static uint64 +pcm_x_hash_tag(uint64 hash, const BufferTag *tag) +{ + hash = pcm_x_hash_scalar(hash, (uint64)tag->spcOid); + hash = pcm_x_hash_scalar(hash, (uint64)tag->dbOid); + hash = pcm_x_hash_scalar(hash, (uint64)tag->relNumber); + hash = pcm_x_hash_scalar(hash, (uint64)(uint32)tag->forkNum); + return pcm_x_hash_scalar(hash, (uint64)tag->blockNum); +} + + +/* The sole hash authority for every PCM-X master/local lock partition. */ +uint32 +cluster_pcm_x_tag_hash(const BufferTag *tag) +{ + uint64 hash = UINT64CONST(1469598103934665603); + + if (tag == NULL) + return 0; + hash = pcm_x_hash_tag(hash, tag); + return (uint32)(hash ^ (hash >> 32)); +} + + +static uint64 +pcm_x_hash_wait_identity(uint64 hash, const PcmXWaitIdentity *identity) +{ + hash = pcm_x_hash_tag(hash, &identity->tag); + hash = pcm_x_hash_scalar(hash, (uint64)(uint32)identity->node_id); + hash = pcm_x_hash_scalar(hash, (uint64)identity->procno); + hash = pcm_x_hash_scalar(hash, (uint64)identity->xid); + hash = pcm_x_hash_scalar(hash, identity->cluster_epoch); + hash = pcm_x_hash_scalar(hash, identity->request_id); + hash = pcm_x_hash_scalar(hash, identity->wait_seq); + return pcm_x_hash_scalar(hash, identity->base_own_generation); +} + + +/* Hash fields canonically; structure padding is never part of directory identity. */ +bool +cluster_pcm_x_directory_key_hash(PcmXDirectoryKind kind, const void *key, uint64 *hash_out) +{ + uint64 hash = UINT64CONST(1469598103934665603); + + if (key == NULL || hash_out == NULL || kind < 0 || kind >= PCM_X_DIR_COUNT) + return false; + hash = pcm_x_hash_scalar(hash, (uint64)kind); + + switch (kind) { + case PCM_X_DIR_MASTER_TAG: + case PCM_X_DIR_LOCAL_TAG: + hash = pcm_x_hash_tag(hash, (const BufferTag *)key); + break; + case PCM_X_DIR_MASTER_TICKET_PREHANDLE: { + const PcmXPrehandleKey *prehandle = (const PcmXPrehandleKey *)key; + + hash = pcm_x_hash_scalar(hash, prehandle->sender_session_incarnation); + hash = pcm_x_hash_scalar(hash, prehandle->prehandle_sequence); + break; + } + case PCM_X_DIR_MASTER_TICKET_HANDLE: { + const PcmXTicketHandle *handle = (const PcmXTicketHandle *)key; + + hash = pcm_x_hash_scalar(hash, handle->ticket_id); + hash = pcm_x_hash_scalar(hash, handle->queue_generation); + break; + } + case PCM_X_DIR_MASTER_TICKET_RETIRE: { + const PcmXRetirePayload *retire = (const PcmXRetirePayload *)key; + + hash = pcm_x_hash_scalar(hash, retire->cluster_epoch); + hash = pcm_x_hash_scalar(hash, retire->master_session_incarnation); + hash = pcm_x_hash_scalar(hash, retire->retire_through_ticket_id); + break; + } + case PCM_X_DIR_LOCAL_WAIT: { + const PcmXWaitIdentity *identity = (const PcmXWaitIdentity *)key; + + hash = pcm_x_hash_wait_identity(hash, identity); + break; + } + case PCM_X_DIR_LOCAL_HOLDER: { + const PcmXLocalHolderKey *holder = (const PcmXLocalHolderKey *)key; + + hash = pcm_x_hash_wait_identity(hash, &holder->identity); + hash = pcm_x_hash_scalar(hash, (uint64)(uint32)holder->buffer_id); + break; + } + case PCM_X_DIR_COUNT: + return false; + } + + *hash_out = hash; + return true; +} + + +static bool +pcm_x_wait_identity_equal(const PcmXWaitIdentity *left, const PcmXWaitIdentity *right) +{ + return BufferTagsEqual(&left->tag, &right->tag) && left->node_id == right->node_id + && left->procno == right->procno && left->xid == right->xid + && left->cluster_epoch == right->cluster_epoch && left->request_id == right->request_id + && left->wait_seq == right->wait_seq + && left->base_own_generation == right->base_own_generation; +} + + +static void +pcm_x_retire_key_from_ticket(const PcmXMasterTicketSlot *ticket, PcmXRetirePayload *key_out) +{ + memset(key_out, 0, sizeof(*key_out)); + key_out->cluster_epoch = ticket->ref.identity.cluster_epoch; + key_out->master_session_incarnation = ticket->master_session_incarnation; + key_out->retire_through_ticket_id = ticket->ref.handle.ticket_id; +} + + +static bool +pcm_x_retire_key_equal(const PcmXRetirePayload *left, const PcmXRetirePayload *right) +{ + return left->cluster_epoch == right->cluster_epoch + && left->master_session_incarnation == right->master_session_incarnation + && left->retire_through_ticket_id == right->retire_through_ticket_id; +} + + +static bool +pcm_x_directory_key_matches_slot(PcmXDirectoryKind kind, const void *key, + const PcmXSlotHeader *slot) +{ + switch (kind) { + case PCM_X_DIR_MASTER_TAG: + return BufferTagsEqual((const BufferTag *)key, &((const PcmXMasterTagSlot *)slot)->tag); + case PCM_X_DIR_MASTER_TICKET_PREHANDLE: { + const PcmXPrehandleKey *left = (const PcmXPrehandleKey *)key; + const PcmXPrehandleKey *right = &((const PcmXMasterTicketSlot *)slot)->prehandle; + + return left->sender_session_incarnation == right->sender_session_incarnation + && left->prehandle_sequence == right->prehandle_sequence; + } + case PCM_X_DIR_MASTER_TICKET_HANDLE: { + const PcmXTicketHandle *left = (const PcmXTicketHandle *)key; + const PcmXTicketHandle *right = &((const PcmXMasterTicketSlot *)slot)->ref.handle; + + return left->ticket_id == right->ticket_id + && left->queue_generation == right->queue_generation; + } + case PCM_X_DIR_MASTER_TICKET_RETIRE: { + const PcmXRetirePayload *retire = (const PcmXRetirePayload *)key; + const PcmXMasterTicketSlot *ticket = (const PcmXMasterTicketSlot *)slot; + PcmXRetirePayload resident; + + pcm_x_retire_key_from_ticket(ticket, &resident); + return pcm_x_retire_key_equal(retire, &resident); + } + case PCM_X_DIR_LOCAL_TAG: + return BufferTagsEqual((const BufferTag *)key, &((const PcmXLocalTagSlot *)slot)->tag); + case PCM_X_DIR_LOCAL_WAIT: + return pcm_x_wait_identity_equal((const PcmXWaitIdentity *)key, + &((const PcmXLocalMembershipSlot *)slot)->identity); + case PCM_X_DIR_LOCAL_HOLDER: { + const PcmXLocalHolderKey *holder = (const PcmXLocalHolderKey *)key; + const PcmXLocalMembershipSlot *membership = (const PcmXLocalMembershipSlot *)slot; + + return holder->buffer_id == membership->buffer_id + && pcm_x_wait_identity_equal(&holder->identity, &membership->identity); + } + case PCM_X_DIR_COUNT: + return false; + } + return false; +} + + +static bool +pcm_x_directory_slot_key_hash(PcmXDirectoryKind kind, const PcmXSlotHeader *slot, uint64 *hash_out) +{ + switch (kind) { + case PCM_X_DIR_MASTER_TAG: + return cluster_pcm_x_directory_key_hash(kind, &((const PcmXMasterTagSlot *)slot)->tag, + hash_out); + case PCM_X_DIR_MASTER_TICKET_PREHANDLE: + return cluster_pcm_x_directory_key_hash( + kind, &((const PcmXMasterTicketSlot *)slot)->prehandle, hash_out); + case PCM_X_DIR_MASTER_TICKET_HANDLE: + return cluster_pcm_x_directory_key_hash( + kind, &((const PcmXMasterTicketSlot *)slot)->ref.handle, hash_out); + case PCM_X_DIR_MASTER_TICKET_RETIRE: { + PcmXRetirePayload retire; + + pcm_x_retire_key_from_ticket((const PcmXMasterTicketSlot *)slot, &retire); + return cluster_pcm_x_directory_key_hash(kind, &retire, hash_out); + } + case PCM_X_DIR_LOCAL_TAG: + return cluster_pcm_x_directory_key_hash(kind, &((const PcmXLocalTagSlot *)slot)->tag, + hash_out); + case PCM_X_DIR_LOCAL_WAIT: + return cluster_pcm_x_directory_key_hash( + kind, &((const PcmXLocalMembershipSlot *)slot)->identity, hash_out); + case PCM_X_DIR_LOCAL_HOLDER: { + const PcmXLocalMembershipSlot *membership = (const PcmXLocalMembershipSlot *)slot; + PcmXLocalHolderKey holder; + + memset(&holder, 0, sizeof(holder)); + holder.identity = membership->identity; + holder.buffer_id = membership->buffer_id; + return cluster_pcm_x_directory_key_hash(kind, &holder, hash_out); + } + case PCM_X_DIR_COUNT: + return false; + } + return false; +} + + +static PcmXDirectoryMatch +pcm_x_directory_entry_match_locked(PcmXDirectoryKind kind, const PcmXDirectoryView *directory, + const PcmXDirectoryEntry *entry, const void *key) +{ + PcmXAllocatorView allocator; + PcmXSlotHeader *slot; + uint64 slot_generation; + uint64 resident_key_hash; + uint32 slot_state; + + if (!pcm_x_allocator_view(directory->allocator_kind, &allocator)) + return PCM_X_DIRECTORY_MATCH_CORRUPT; + slot = pcm_x_allocator_slot(&allocator, entry->slot_index); + if (slot == NULL) + return PCM_X_DIRECTORY_MATCH_CORRUPT; + slot_state = pcm_x_slot_state_read(slot); + if (!pcm_x_slot_generation_read(slot, &slot_generation)) + return PCM_X_DIRECTORY_MATCH_CORRUPT; + if (slot_generation != entry->slot_generation || slot_state == PCM_X_SLOT_FREE + || slot_state == PCM_X_SLOT_GENERATION_EXHAUSTED) + return PCM_X_DIRECTORY_MATCH_STALE; + /* The entry hash is also the immutable-key checksum for its resident slot. */ + if (!pcm_x_directory_slot_key_hash(kind, slot, &resident_key_hash) + || resident_key_hash != entry->key_hash) + return PCM_X_DIRECTORY_MATCH_CORRUPT; + return pcm_x_directory_key_matches_slot(kind, key, slot) ? PCM_X_DIRECTORY_MATCH_EXACT + : PCM_X_DIRECTORY_MATCH_DIFFERENT; +} + + +static void +pcm_x_directory_entry_publish(PcmXDirectoryEntry *entry, uint64 key_hash, PcmXSlotRef ref) +{ + entry->key_hash = key_hash; + entry->slot_index = ref.slot_index; + entry->slot_generation = ref.slot_generation; + entry->reserved = 0; + pg_write_barrier(); + entry->state = PCM_X_DIRECTORY_OCCUPIED; +} + + +static void +pcm_x_directory_entry_clear(PcmXDirectoryEntry *entry) +{ + entry->key_hash = 0; + entry->slot_index = 0; + entry->slot_generation = 0; + entry->reserved = 0; + pg_write_barrier(); + entry->state = PCM_X_DIRECTORY_EMPTY; +} + + +static Size +pcm_x_directory_distance(Size home, Size position, Size capacity) +{ + return position >= home ? position - home : capacity - home + position; +} + + +static PcmXDirectoryResult +pcm_x_directory_backshift_delete_locked(PcmXDirectoryView *directory, Size hole) +{ + Size scan = (hole + 1) % directory->capacity; + Size steps; + + for (steps = 0; steps < directory->capacity; steps++) { + PcmXDirectoryEntry *entry = &directory->entries[scan]; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) { + pcm_x_directory_entry_clear(&directory->entries[hole]); + return PCM_X_DIRECTORY_OK; + } + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return PCM_X_DIRECTORY_CORRUPT; + if (pcm_x_directory_distance(entry->key_hash % directory->capacity, hole, + directory->capacity) + < pcm_x_directory_distance(entry->key_hash % directory->capacity, scan, + directory->capacity)) { + PcmXSlotRef ref = { entry->slot_index, entry->slot_generation }; + + pcm_x_directory_entry_publish(&directory->entries[hole], entry->key_hash, ref); + hole = scan; + } + scan = (scan + 1) % directory->capacity; + } + return PCM_X_DIRECTORY_CORRUPT; +} + + +static PcmXDirectoryResult +pcm_x_directory_find_locked(PcmXDirectoryKind kind, const void *key, PcmXSlotRef *found_out) +{ + PcmXDirectoryView directory; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || found_out == NULL || !LWLockHeldByMe(&header->allocator_lock.lock) + || !cluster_pcm_x_directory_key_hash(kind, key, &key_hash) + || !pcm_x_directory_view(kind, &directory)) + return PCM_X_DIRECTORY_INVALID; + found_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + found_out->slot_generation = 0; + bucket = key_hash % directory.capacity; + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) + return PCM_X_DIRECTORY_NOT_FOUND; + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return PCM_X_DIRECTORY_CORRUPT; + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(kind, &directory, entry, key); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + found_out->slot_index = entry->slot_index; + found_out->slot_generation = entry->slot_generation; + return PCM_X_DIRECTORY_OK; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) + return PCM_X_DIRECTORY_STALE_REF; + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) + return PCM_X_DIRECTORY_CORRUPT; + } + return PCM_X_DIRECTORY_NOT_FOUND; +} + + +static PcmXDirectoryResult +pcm_x_directory_insert_locked(PcmXDirectoryKind kind, const void *key, PcmXSlotRef candidate, + PcmXSlotRef *existing_out) +{ + PcmXDirectoryView directory; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXSlotHeader *candidate_slot; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || existing_out == NULL + || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE) + || !cluster_pcm_x_directory_key_hash(kind, key, &key_hash) + || !pcm_x_directory_view(kind, &directory)) + return PCM_X_DIRECTORY_INVALID; + existing_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + existing_out->slot_generation = 0; + candidate_slot = pcm_x_slot_ref_resolve_locked(directory.allocator_kind, candidate); + if (candidate_slot == NULL) + return PCM_X_DIRECTORY_STALE_REF; + if (!pcm_x_directory_key_matches_slot(kind, key, candidate_slot)) + return PCM_X_DIRECTORY_INVALID; + + bucket = key_hash % directory.capacity; + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) { + pcm_x_directory_entry_publish(entry, key_hash, candidate); + return PCM_X_DIRECTORY_OK; + } + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return PCM_X_DIRECTORY_CORRUPT; + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(kind, &directory, entry, key); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + existing_out->slot_index = entry->slot_index; + existing_out->slot_generation = entry->slot_generation; + return PCM_X_DIRECTORY_EXISTS; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) + return PCM_X_DIRECTORY_STALE_REF; + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) + return PCM_X_DIRECTORY_CORRUPT; + } + return PCM_X_DIRECTORY_FULL; +} + + +/* + * Publish the target key of one gated local-leader rekey while the resident + * membership still carries its old identity. The admission gate is the + * cross-lock transaction marker; only the later local-lock phase may replace + * resident_identity. This helper therefore validates the exact old resident + * instead of applying the ordinary directory checksum prerequisite to the + * target key. + */ +static PcmXDirectoryResult +pcm_x_directory_insert_rekey_target_locked(const PcmXWaitIdentity *target_identity, + const PcmXWaitIdentity *resident_identity, + PcmXSlotRef candidate, PcmXSlotRef *existing_out) +{ + PcmXDirectoryView directory; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalMembershipSlot *candidate_slot; + PcmXWaitIdentity expected_target; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || target_identity == NULL || resident_identity == NULL + || existing_out == NULL || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE) + || !cluster_pcm_x_directory_key_hash(PCM_X_DIR_LOCAL_WAIT, target_identity, &key_hash) + || !pcm_x_directory_view(PCM_X_DIR_LOCAL_WAIT, &directory)) + return PCM_X_DIRECTORY_INVALID; + existing_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + existing_out->slot_generation = 0; + expected_target = *resident_identity; + expected_target.base_own_generation = target_identity->base_own_generation; + if (!pcm_x_wait_identity_equal(&expected_target, target_identity)) + return PCM_X_DIRECTORY_INVALID; + candidate_slot = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked( + PCM_X_ALLOC_LOCAL_WAIT, candidate); + if (candidate_slot == NULL) + return PCM_X_DIRECTORY_STALE_REF; + if (candidate_slot->partition != PCM_X_LOCAL_PARTITION_WAIT + || !pcm_x_wait_identity_equal(&candidate_slot->identity, resident_identity)) + return PCM_X_DIRECTORY_INVALID; + + bucket = key_hash % directory.capacity; + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) { + pcm_x_directory_entry_publish(entry, key_hash, candidate); + return PCM_X_DIRECTORY_OK; + } + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return PCM_X_DIRECTORY_CORRUPT; + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(PCM_X_DIR_LOCAL_WAIT, &directory, entry, + target_identity); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + existing_out->slot_index = entry->slot_index; + existing_out->slot_generation = entry->slot_generation; + return PCM_X_DIRECTORY_EXISTS; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) + return PCM_X_DIRECTORY_STALE_REF; + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) + return PCM_X_DIRECTORY_CORRUPT; + } + return PCM_X_DIRECTORY_FULL; +} + + +static PcmXDirectoryResult +pcm_x_directory_delete_exact_locked(PcmXDirectoryKind kind, const void *key, PcmXSlotRef expected) +{ + PcmXDirectoryView directory; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE) + || !cluster_pcm_x_directory_key_hash(kind, key, &key_hash) + || !pcm_x_directory_view(kind, &directory)) + return PCM_X_DIRECTORY_INVALID; + bucket = key_hash % directory.capacity; + for (step = 0; step < directory.capacity; step++) { + Size position = (bucket + step) % directory.capacity; + PcmXDirectoryEntry *entry = &directory.entries[position]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) + return PCM_X_DIRECTORY_NOT_FOUND; + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return PCM_X_DIRECTORY_CORRUPT; + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(kind, &directory, entry, key); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + if (entry->slot_index != expected.slot_index + || entry->slot_generation != expected.slot_generation) + return PCM_X_DIRECTORY_STALE_REF; + return pcm_x_directory_backshift_delete_locked(&directory, position); + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) + return PCM_X_DIRECTORY_STALE_REF; + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) + return PCM_X_DIRECTORY_CORRUPT; + } + return PCM_X_DIRECTORY_NOT_FOUND; +} + + +PcmXDirectoryResult +cluster_pcm_x_directory_find(PcmXDirectoryKind kind, const void *key, PcmXSlotRef *found_out) +{ + PcmXDirectoryView directory; + PcmXDirectoryResult result = PCM_X_DIRECTORY_NOT_FOUND; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + uint64 key_hash; + Size bucket; + Size step; + + if (found_out == NULL || header == NULL + || !cluster_pcm_x_directory_key_hash(kind, key, &key_hash) + || !pcm_x_directory_view(kind, &directory)) + return PCM_X_DIRECTORY_INVALID; + found_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + found_out->slot_generation = 0; + if (!pcm_x_allocator_entry_unlocked(header)) { + pcm_x_runtime_fail_closed(); + return PCM_X_DIRECTORY_INVALID; + } + bucket = key_hash % directory.capacity; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) + break; + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) { + result = PCM_X_DIRECTORY_CORRUPT; + break; + } + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(kind, &directory, entry, key); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + found_out->slot_index = entry->slot_index; + found_out->slot_generation = entry->slot_generation; + result = PCM_X_DIRECTORY_OK; + break; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) { + result = PCM_X_DIRECTORY_STALE_REF; + break; + } + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) { + result = PCM_X_DIRECTORY_CORRUPT; + break; + } + } + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_DIRECTORY_STALE_REF || result == PCM_X_DIRECTORY_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXDirectoryResult +cluster_pcm_x_directory_insert(PcmXDirectoryKind kind, const void *key, PcmXSlotRef candidate, + PcmXSlotRef *existing_out) +{ + PcmXDirectoryView directory; + PcmXDirectoryResult result = PCM_X_DIRECTORY_FULL; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXSlotHeader *candidate_slot; + uint64 key_hash; + Size bucket; + Size step; + + if (existing_out == NULL || header == NULL + || !cluster_pcm_x_directory_key_hash(kind, key, &key_hash) + || !pcm_x_directory_view(kind, &directory)) + return PCM_X_DIRECTORY_INVALID; + existing_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + existing_out->slot_generation = 0; + if (!pcm_x_allocator_entry_unlocked(header)) { + pcm_x_runtime_fail_closed(); + return PCM_X_DIRECTORY_INVALID; + } + bucket = key_hash % directory.capacity; + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + candidate_slot = pcm_x_slot_ref_resolve_locked(directory.allocator_kind, candidate); + if (candidate_slot == NULL) { + result = PCM_X_DIRECTORY_STALE_REF; + goto done; + } + if (!pcm_x_directory_key_matches_slot(kind, key, candidate_slot)) { + result = PCM_X_DIRECTORY_INVALID; + goto done; + } + + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) { + pcm_x_directory_entry_publish(entry, key_hash, candidate); + result = PCM_X_DIRECTORY_OK; + break; + } + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) { + result = PCM_X_DIRECTORY_CORRUPT; + break; + } + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(kind, &directory, entry, key); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + existing_out->slot_index = entry->slot_index; + existing_out->slot_generation = entry->slot_generation; + result = PCM_X_DIRECTORY_EXISTS; + break; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) { + result = PCM_X_DIRECTORY_STALE_REF; + break; + } + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) { + result = PCM_X_DIRECTORY_CORRUPT; + break; + } + } + +done: + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_DIRECTORY_STALE_REF || result == PCM_X_DIRECTORY_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXDirectoryResult +cluster_pcm_x_directory_delete_exact(PcmXDirectoryKind kind, const void *key, PcmXSlotRef expected) +{ + PcmXDirectoryView directory; + PcmXDirectoryResult result = PCM_X_DIRECTORY_NOT_FOUND; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || !cluster_pcm_x_directory_key_hash(kind, key, &key_hash) + || !pcm_x_directory_view(kind, &directory)) + return PCM_X_DIRECTORY_INVALID; + if (!pcm_x_allocator_entry_unlocked(header)) { + pcm_x_runtime_fail_closed(); + return PCM_X_DIRECTORY_INVALID; + } + bucket = key_hash % directory.capacity; + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + for (step = 0; step < directory.capacity; step++) { + Size position = (bucket + step) % directory.capacity; + PcmXDirectoryEntry *entry = &directory.entries[position]; + PcmXDirectoryMatch match; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) + break; + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) { + result = PCM_X_DIRECTORY_CORRUPT; + break; + } + if (entry->key_hash != key_hash) + continue; + match = pcm_x_directory_entry_match_locked(kind, &directory, entry, key); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + if (entry->slot_index != expected.slot_index + || entry->slot_generation != expected.slot_generation) + result = PCM_X_DIRECTORY_STALE_REF; + else + result = pcm_x_directory_backshift_delete_locked(&directory, position); + break; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) { + result = PCM_X_DIRECTORY_STALE_REF; + break; + } + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) { + result = PCM_X_DIRECTORY_CORRUPT; + break; + } + } + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_DIRECTORY_STALE_REF || result == PCM_X_DIRECTORY_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Return the master/local partition selected by a stable BufferTag hash. */ +uint32 +cluster_pcm_x_lock_partition(uint32 tag_hash) +{ + return tag_hash & (PCM_X_LOCK_PARTITIONS - 1); +} + + +/* Frozen application-message successor table for live and terminal ticket slots. */ +static bool +pcm_x_master_event_successor_state(PcmXMasterTicketState current, PcmXMasterEvent event) +{ + switch (event) { + case PCM_X_EVENT_ADMIT_CONFIRM: + return current >= PCM_XT_QUEUED && current <= PCM_XT_RETIRE_CREDIT; + case PCM_X_EVENT_COMMIT_COMPLETE: + return current == PCM_XT_COMPLETE || current == PCM_XT_RETIRE_CREDIT; + case PCM_X_EVENT_CANCEL_EXACT: + return current == PCM_XT_CANCELLED || current == PCM_XT_RETIRE_CREDIT; + default: + return false; + } +} + + +static uint64 +pcm_x_master_event_tombstone(PcmXMasterEvent event) +{ + switch (event) { + case PCM_X_EVENT_ADMIT_CONFIRM: + return PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM; + case PCM_X_EVENT_COMMIT_COMPLETE: + return PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + case PCM_X_EVENT_CANCEL_EXACT: + return PCM_X_RESPONSE_TOMBSTONE_CANCEL; + default: + return 0; + } +} + + +static bool +pcm_x_master_event_replay_exact(PcmXMasterTicketState current, PcmXMasterEvent event, + uint64 response_tombstones) +{ + uint64 required = pcm_x_master_event_tombstone(event); + + return required != 0 && pcm_x_master_event_successor_state(current, event) + && (response_tombstones & required) != 0; +} + + +/* + * cluster_pcm_x_master_step -- Pure authority for master ticket transitions. + * + * Callers must separately prove exact slot/ref identity. The EXACT guard is + * mandatory for every transition after local reserve. Missing positive + * gates leave the state unchanged; post-authorization cancel is converted to + * RECOVERY_BLOCKED instead of guessing rollback. + */ +PcmXStepResult +cluster_pcm_x_master_step(PcmXMasterTicketState current, PcmXMasterEvent event, uint32 guards, + PcmXMasterTicketState *next) +{ + uint64 response_tombstones = 0; + + if (next == NULL) + return PCM_X_STEP_BLOCKED; + *next = current; + + if ((guards & PCM_X_G_EXACT) == 0) + return PCM_X_STEP_STALE; + if ((guards & PCM_X_G_CONFIRM_TOMBSTONE) != 0) + response_tombstones |= PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM; + if ((guards & PCM_X_G_COMPLETE_TOMBSTONE) != 0) + response_tombstones |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + if ((guards & PCM_X_G_CANCEL_TOMBSTONE) != 0) + response_tombstones |= PCM_X_RESPONSE_TOMBSTONE_CANCEL; + + if (current == PCM_XT_RECOVERY_BLOCKED && event == PCM_X_EVENT_FAULT) + return PCM_X_STEP_DUPLICATE; + if (event == PCM_X_EVENT_FAULT && current != PCM_XT_FREE + && current != PCM_XT_GENERATION_EXHAUSTED) { + *next = PCM_XT_RECOVERY_BLOCKED; + return PCM_X_STEP_APPLIED; + } + + if (event == PCM_X_EVENT_ADMIT_CONFIRM && current != PCM_XT_ADMITTING) { + if (pcm_x_master_event_replay_exact(current, event, response_tombstones)) + return PCM_X_STEP_DUPLICATE; + return PCM_X_STEP_BLOCKED; + } + if ((event == PCM_X_EVENT_COMMIT_COMPLETE || event == PCM_X_EVENT_CANCEL_EXACT) + && pcm_x_master_event_replay_exact(current, event, response_tombstones)) + return PCM_X_STEP_DUPLICATE; + if (event == PCM_X_EVENT_DRAIN_EXACT && current == PCM_XT_RETIRE_CREDIT) + return PCM_X_STEP_DUPLICATE; + + switch (event) { + case PCM_X_EVENT_PUBLISH_ADMIT: + if (current == PCM_XT_RESERVED_NONVISIBLE) { + *next = PCM_XT_ADMITTING; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_ADMIT_CONFIRM: + if (current == PCM_XT_ADMITTING && (guards & PCM_X_G_WFG_READY) != 0) { + *next = PCM_XT_QUEUED; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_PROMOTE_HEAD: + if (current == PCM_XT_QUEUED && (guards & PCM_X_G_HEAD) != 0) { + *next = PCM_XT_ACTIVE_PROBE; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_BLOCKERS_COMMITTED: + if (current == PCM_XT_ACTIVE_PROBE && (guards & PCM_X_G_WFG_READY) != 0 + && (guards & PCM_X_G_REVERSIBLE) != 0) { + *next = PCM_XT_ACTIVE_TRANSFER; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_COMMIT_COMPLETE: + if (current == PCM_XT_ACTIVE_TRANSFER) { + *next = PCM_XT_COMPLETE; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_CANCEL_EXACT: + if ((current == PCM_XT_RESERVED_NONVISIBLE || current == PCM_XT_ADMITTING + || current == PCM_XT_QUEUED || current == PCM_XT_ACTIVE_PROBE) + && (guards & PCM_X_G_REVERSIBLE) != 0) { + *next = PCM_XT_CANCELLED; + return PCM_X_STEP_APPLIED; + } + if (current == PCM_XT_ACTIVE_TRANSFER) { + *next = PCM_XT_RECOVERY_BLOCKED; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_DRAIN_EXACT: + if ((current == PCM_XT_COMPLETE || current == PCM_XT_CANCELLED) + && (guards & PCM_X_G_DRAINED) != 0) { + *next = PCM_XT_RETIRE_CREDIT; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_RETIRE_ACK_EXACT: + if (current == PCM_XT_RETIRE_CREDIT && (guards & PCM_X_G_RETIRED) != 0) { + *next = PCM_XT_DETACHING; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_DETACH_EXACT: + if (current == PCM_XT_DETACHING) { + *next = PCM_XT_FREE; + return PCM_X_STEP_APPLIED; + } + break; + case PCM_X_EVENT_FAULT: + break; + } + + return PCM_X_STEP_BLOCKED; +} + + +/* + * Sample the cross-partition gate together with its master-session authority. + * + * pg_atomic_read_u32() has no barrier semantics in this tree. An ACTIVE + * observation therefore needs an explicit read barrier before consuming the + * session published by the activating CAS, followed by a second barrier and + * state read. Any transition during the sample fails closed instead of + * returning a session detached from the ACTIVE incarnation that published it. + */ +PcmXRuntimeSnapshot +cluster_pcm_x_runtime_snapshot(void) +{ + PcmXRuntimeSnapshot snapshot = { 0 }; + uint32 gate1; + uint32 gate2; + uint32 state1; + uint64 session; + + if (ClusterPcmXConvertShmem == NULL) + return snapshot; + + gate1 = pg_atomic_read_u32(&ClusterPcmXConvertShmem->runtime_gate); + state1 = pcm_x_runtime_gate_state(gate1); + snapshot.gate_generation = pcm_x_runtime_gate_generation(gate1); + if (state1 != PCM_X_RUNTIME_ACTIVE) { + if (state1 <= PCM_X_RUNTIME_SHUTTING_DOWN) + snapshot.state = (PcmXRuntimeState)state1; + return snapshot; + } + + pg_read_barrier(); + session = ClusterPcmXConvertShmem->master_session_incarnation; + pg_read_barrier(); + gate2 = pg_atomic_read_u32(&ClusterPcmXConvertShmem->runtime_gate); + if (gate1 != gate2 || session == 0) + return snapshot; + + snapshot.state = PCM_X_RUNTIME_ACTIVE; + snapshot.master_session_incarnation = session; + return snapshot; +} + + +/* + * Revalidate one fail-stop acting token at the final owner lock. + * + * The second exact snapshot is the operation's authority linearization point: + * a close observed first returns NOT_READY with no visible mutation; a close + * that follows is ordered after this already-authorized operation. Composite + * admit/join operations take this point under allocator_lock before publishing + * any directory entry, then must finish their domain phase or leave evidence. + */ +static bool +pcm_x_runtime_token_exact(const PcmXRuntimeSnapshot *start, uint64 expected_master_session) +{ + PcmXRuntimeSnapshot current; + + if (start == NULL || start->state != PCM_X_RUNTIME_ACTIVE + || start->master_session_incarnation == 0 || start->gate_generation == 0) + return false; + current = cluster_pcm_x_runtime_snapshot(); + return current.state == PCM_X_RUNTIME_ACTIVE + && current.gate_generation == start->gate_generation + && current.master_session_incarnation == start->master_session_incarnation + && (expected_master_session == 0 + || expected_master_session == start->master_session_incarnation); +} + + +/* + * Claim a blocked gate, install a strictly newer session, then publish ACTIVE. + * + * The private ACTIVATING phase prevents concurrent publishers from racing the + * fenced session store. Its generation is already the next public generation, + * so the final ACTIVATING-to-ACTIVE CAS does not consume another value. A + * process failure after the claim leaves the gate in a private phase that all + * readers normalize to RECOVERY_BLOCKED. + */ +bool +cluster_pcm_x_runtime_activate_bound(uint64 master_session_incarnation, + const PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]) +{ + uint32 active_gate; + uint32 activation_retry_generation; + uint32 claimed_gate; + uint64 current_session; + uint32 expected_gate; + uint32 generation; + int i; + + if (ClusterPcmXConvertShmem == NULL || master_session_incarnation == 0) + return false; + if (bindings != NULL) { + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + /* CLUSTER_EPOCH_INITIAL (zero) is a live, wire-visible epoch. + * A binding is absent only when its peer session is absent. */ + if (bindings[i].peer_session_incarnation == 0 && bindings[i].cluster_epoch != 0) + return false; + } + } + + expected_gate = pg_atomic_read_u32(&ClusterPcmXConvertShmem->runtime_gate); + if (pcm_x_runtime_gate_state(expected_gate) != PCM_X_RUNTIME_RECOVERY_BLOCKED) + return false; + generation = pcm_x_runtime_gate_generation(expected_gate); + if (generation >= PCM_X_RUNTIME_GATE_GENERATION_MAX - 1) + return false; + current_session = ClusterPcmXConvertShmem->master_session_incarnation; + activation_retry_generation + = pg_atomic_read_u32(&ClusterPcmXConvertShmem->activation_retry_generation); + /* + * A normal ACTIVE-to-BLOCKED transition is permanently fail-stop in the + * steady-state core. Re-entry is authorized only for pristine startup or + * by the exact marker published after recovering a stranded ACTIVATING CAS. + */ + if (!((generation == 0 && current_session == 0 && activation_retry_generation == 0) + || (generation != 0 && activation_retry_generation == generation))) + return false; + if (master_session_incarnation <= current_session) + return false; + + claimed_gate = pcm_x_runtime_gate_pack(generation + 1, PCM_X_RUNTIME_GATE_ACTIVATING); + if (!pg_atomic_compare_exchange_u32(&ClusterPcmXConvertShmem->runtime_gate, &expected_gate, + claimed_gate)) + return false; + + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + PcmXOutboundTargetFrontier *outbound = &ClusterPcmXConvertShmem->outbound_targets[i]; + uint64 peer_epoch = bindings == NULL ? 0 : bindings[i].cluster_epoch; + uint64 peer_session = bindings == NULL ? 0 : bindings[i].peer_session_incarnation; + + ClusterPcmXConvertShmem->peer_frontiers[i].cluster_epoch = peer_epoch; + ClusterPcmXConvertShmem->peer_frontiers[i].sender_session_incarnation = peer_session; + ClusterPcmXConvertShmem->peer_frontiers[i].next_expected_prehandle_sequence = 1; + ClusterPcmXConvertShmem->peer_frontiers[i].retired_prehandle_sequence = 0; + pg_atomic_write_u32(&outbound->mint_gate, 0); + outbound->flags = PCM_X_OUTBOUND_TARGET_INITIALIZED + | (peer_session == 0 ? 0 : PCM_X_OUTBOUND_TARGET_BOUND); + outbound->cluster_epoch = peer_epoch; + outbound->target_session_incarnation = peer_session; + outbound->next_prehandle_sequence = 1; + } + ClusterPcmXConvertShmem->master_session_incarnation = master_session_incarnation; + pg_atomic_write_u32(&ClusterPcmXConvertShmem->activation_retry_generation, 0); + pg_write_barrier(); + active_gate = pcm_x_runtime_gate_pack(generation + 1, PCM_X_RUNTIME_ACTIVE); + if (!pg_atomic_compare_exchange_u32(&ClusterPcmXConvertShmem->runtime_gate, &claimed_gate, + active_gate)) + return false; + return true; +} + + +bool +cluster_pcm_x_runtime_activate(uint64 master_session_incarnation) +{ + return cluster_pcm_x_runtime_activate_bound(master_session_incarnation, NULL); +} + + +/* + * Revalidate one already-published peer binding without extending authority. + * + * The caller enters without allocator or tag-domain locks. allocator_lock + * makes the two frontiers one immutable identity sample, while the runtime + * token binds that sample to the ACTIVE gate observed before lock acquisition. + * An empty frontier is a mismatch: only activation/recovery may publish it. + */ +PcmXQueueResult +cluster_pcm_x_runtime_peer_binding_revalidate_exact(int32 peer_node, uint64 cluster_epoch, + uint64 peer_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXPeerFrontier *frontier; + PcmXOutboundTargetFrontier *outbound; + PcmXRuntimeSnapshot runtime; + PcmXQueueResult result; + bool fail_closed = false; + + if (header == NULL || peer_node < 0 || peer_node >= PCM_X_PROTOCOL_NODE_LIMIT + || peer_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (!pcm_x_allocator_entry_unlocked(header)) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_INVALID; + } + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + if (!pcm_x_runtime_token_exact(&runtime, runtime.master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto done; + } + frontier = &header->peer_frontiers[peer_node]; + outbound = &header->outbound_targets[peer_node]; + if (frontier->cluster_epoch == cluster_epoch + && frontier->sender_session_incarnation == peer_session + && frontier->next_expected_prehandle_sequence != 0 + && frontier->retired_prehandle_sequence < frontier->next_expected_prehandle_sequence + && outbound->flags == (PCM_X_OUTBOUND_TARGET_INITIALIZED | PCM_X_OUTBOUND_TARGET_BOUND) + && outbound->cluster_epoch == cluster_epoch + && outbound->target_session_incarnation == peer_session + && outbound->next_prehandle_sequence != 0) + result = PCM_X_QUEUE_OK; + else { + result = PCM_X_QUEUE_STALE; + fail_closed = true; + } + +done: + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* + * Recovery-only escape from a publisher that died after claiming ACTIVATING. + * + * The caller must already have external proof that the activating publisher is + * gone. The exact packed generation prevents an old recovery action from + * closing a later activation. The fenced session value remains monotonic; a + * subsequent activation must publish a strictly newer session. + */ +bool +cluster_pcm_x_runtime_reset_activating(uint32 expected_gate_generation) +{ + uint32 blocked_gate; + uint32 expected_gate; + + if (ClusterPcmXConvertShmem == NULL || expected_gate_generation == 0 + || expected_gate_generation > PCM_X_RUNTIME_GATE_GENERATION_MAX) + return false; + expected_gate + = pcm_x_runtime_gate_pack(expected_gate_generation, PCM_X_RUNTIME_GATE_ACTIVATING); + blocked_gate + = pcm_x_runtime_gate_pack(expected_gate_generation, PCM_X_RUNTIME_RECOVERY_BLOCKED); + if (!pg_atomic_compare_exchange_u32(&ClusterPcmXConvertShmem->runtime_gate, &expected_gate, + blocked_gate)) + return false; + /* A crash before this store only leaves a safely non-restartable gate. */ + pg_write_barrier(); + pg_atomic_write_u32(&ClusterPcmXConvertShmem->activation_retry_generation, + expected_gate_generation); + pcm_x_stats_increment(&ClusterPcmXConvertShmem->stats.activating_reset_count); + return true; +} + + +/* Publish one legal non-activation transition and advance the ABA generation. */ +bool +cluster_pcm_x_runtime_transition(PcmXRuntimeState expected, PcmXRuntimeState desired) +{ + uint32 desired_gate; + uint32 expected_gate; + uint32 exhausted_gate; + uint32 generation; + + if (ClusterPcmXConvertShmem == NULL || expected == desired + || expected > PCM_X_RUNTIME_SHUTTING_DOWN || desired > PCM_X_RUNTIME_SHUTTING_DOWN + || desired == PCM_X_RUNTIME_ACTIVE) + return false; + if (!((expected == PCM_X_RUNTIME_ACTIVE && desired == PCM_X_RUNTIME_RECOVERY_BLOCKED) + || (expected == PCM_X_RUNTIME_ACTIVE && desired == PCM_X_RUNTIME_SHUTTING_DOWN) + || (expected == PCM_X_RUNTIME_SHUTTING_DOWN + && desired == PCM_X_RUNTIME_RECOVERY_BLOCKED))) + return false; + + expected_gate = pg_atomic_read_u32(&ClusterPcmXConvertShmem->runtime_gate); + if (pcm_x_runtime_gate_state(expected_gate) != (uint32)expected) + return false; + generation = pcm_x_runtime_gate_generation(expected_gate); + if (generation == PCM_X_RUNTIME_GATE_GENERATION_MAX + || (desired == PCM_X_RUNTIME_SHUTTING_DOWN + && generation >= PCM_X_RUNTIME_GATE_GENERATION_MAX - 1)) { + /* Never leave an ACTIVE gate open merely because its version is spent. */ + exhausted_gate = pcm_x_runtime_gate_pack( + generation == PCM_X_RUNTIME_GATE_GENERATION_MAX ? generation : generation + 1, + PCM_X_RUNTIME_GATE_EXHAUSTED); + (void)pg_atomic_compare_exchange_u32(&ClusterPcmXConvertShmem->runtime_gate, &expected_gate, + exhausted_gate); + return false; + } + desired_gate = pcm_x_runtime_gate_pack(generation + 1, (uint32)desired); + + return pg_atomic_compare_exchange_u32(&ClusterPcmXConvertShmem->runtime_gate, &expected_gate, + desired_gate); +} + + +/* Only the short PREPARE-to-COMMIT transfer window may reserve ownership. */ +bool +cluster_pcm_x_ownership_reservation_allowed(PcmXMasterTicketState state) +{ + return state == PCM_XT_ACTIVE_TRANSFER; +} + + +/* Terminal tickets detach from the hot queue only after every node drains. */ +bool +cluster_pcm_x_ticket_drain_ready(const PcmXMasterTicketSlot *ticket) +{ + uint32 state; + + if (ticket == NULL || ticket->involved_nodes_bitmap == 0) + return false; + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_COMPLETE && state != PCM_XT_CANCELLED) + return false; + return ticket->drained_nodes_bitmap == ticket->involved_nodes_bitmap; +} + + +/* RETIRE_CREDIT is reusable only after every involved peer application-ACKs. */ +bool +cluster_pcm_x_ticket_retire_ready(const PcmXMasterTicketSlot *ticket) +{ + if (ticket == NULL || ticket->involved_nodes_bitmap == 0 + || pcm_x_slot_state_read(&ticket->slot) != PCM_XT_RETIRE_CREDIT) + return false; + return ticket->drained_nodes_bitmap == ticket->involved_nodes_bitmap + && ticket->retire_acked_nodes_bitmap == ticket->involved_nodes_bitmap; +} + + +static bool +pcm_x_prehandle_equal(const PcmXPrehandleKey *left, const PcmXPrehandleKey *right) +{ + return left->sender_session_incarnation == right->sender_session_incarnation + && left->prehandle_sequence == right->prehandle_sequence; +} + + +static bool +pcm_x_ticket_handle_equal(const PcmXTicketHandle *left, const PcmXTicketHandle *right) +{ + return left->ticket_id == right->ticket_id && left->queue_generation == right->queue_generation; +} + + +static bool +pcm_x_ticket_locator_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) +{ + return pcm_x_wait_identity_equal(&left->identity, &right->identity) + && pcm_x_ticket_handle_equal(&left->handle, &right->handle); +} + + +static bool +pcm_x_ticket_ref_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) +{ + return pcm_x_ticket_locator_equal(left, right) + && left->grant_generation == right->grant_generation; +} + + +static bool +pcm_x_image_token_equal(const PcmXImageToken *left, const PcmXImageToken *right) +{ + return left != NULL && right != NULL && left->image_id == right->image_id + && left->source_own_generation == right->source_own_generation + && left->page_scn == right->page_scn && left->page_lsn == right->page_lsn + && left->source_node == right->source_node + && left->page_checksum == right->page_checksum; +} + + +static bool +pcm_x_image_token_valid(const PcmXImageToken *image) +{ + return image != NULL && image->source_node < PCM_X_PROTOCOL_NODE_LIMIT + && cluster_pcm_x_image_id_decode(image->image_id, NULL, NULL); +} + + +static bool +pcm_x_image_id_master_exact(uint64 image_id, int32 expected_master_node) +{ + int32 encoded_master_node; + + return expected_master_node >= 0 && expected_master_node < PCM_X_PROTOCOL_NODE_LIMIT + && cluster_pcm_x_image_id_decode(image_id, &encoded_master_node, NULL) + && encoded_master_node == expected_master_node; +} + + +/* Admission-phase wire references are immutable and always carry grant 0. */ +static bool +pcm_x_ticket_admission_ref_equal(const PcmXTicketRef *current, const PcmXTicketRef *admission) +{ + return admission->grant_generation == 0 && pcm_x_ticket_locator_equal(current, admission); +} + + +static bool +pcm_x_wait_identity_valid(const PcmXWaitIdentity *identity) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + return header != NULL && identity != NULL && identity->node_id >= 0 + && identity->node_id < PCM_X_PROTOCOL_NODE_LIMIT + && identity->procno < header->layout.process_capacity && identity->request_id != 0 + && identity->wait_seq != 0; +} + + +static void +pcm_x_master_admission_clear(PcmXMasterAdmission *admission) +{ + memset(admission, 0, sizeof(*admission)); + admission->tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + admission->ticket_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static void +pcm_x_local_handle_clear(PcmXLocalHandle *handle) +{ + if (handle == NULL) + return; + memset(handle, 0, sizeof(*handle)); + handle->tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + handle->membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static void +pcm_x_local_follower_wfg_snapshot_clear(PcmXLocalFollowerWfgSnapshot *snapshot) +{ + if (snapshot == NULL) + return; + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->waiter.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->waiter.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->blocker_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->leader_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->active_writer_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static void +pcm_x_local_writer_claim_clear(PcmXLocalWriterClaim *claim) +{ + if (claim == NULL) + return; + memset(claim, 0, sizeof(*claim)); + claim->writer.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + claim->writer.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + claim->active_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static void +pcm_x_master_admission_from_ticket(PcmXMasterAdmission *admission, PcmXSlotRef ticket_ref, + const PcmXMasterTicketSlot *ticket) +{ + pcm_x_master_admission_clear(admission); + /* grant_generation is mutable under the master-domain lock. Admission + * replay copies only the immutable locator while holding allocator_lock. */ + admission->ref.identity = ticket->ref.identity; + admission->ref.handle = ticket->ref.handle; + admission->ref.grant_generation = 0; + admission->prehandle = ticket->prehandle; + admission->tag_slot.slot_index = ticket->tag_slot_index; + admission->tag_slot.slot_generation = ticket->tag_slot_generation; + admission->ticket_slot = ticket_ref; + admission->master_session_incarnation = ticket->master_session_incarnation; + admission->admission_sequence = ticket->admission_sequence; +} + + +static void +pcm_x_local_handle_from_member(PcmXLocalHandle *handle, PcmXSlotRef membership_ref, + const PcmXLocalMembershipSlot *member, PcmXLocalRole role) +{ + pcm_x_local_handle_clear(handle); + handle->identity = member->identity; + handle->tag_slot.slot_index = member->tag_slot_index; + handle->tag_slot.slot_generation = member->tag_slot_generation; + handle->membership_slot = membership_ref; + handle->local_sequence = member->local_sequence; + handle->local_round = member->admitted_round; + handle->role = (uint16)role; +} + + +static PcmXQueueResult +pcm_x_queue_result_from_allocator(PcmXAllocatorResult result) +{ + switch (result) { + case PCM_X_ALLOC_OK: + return PCM_X_QUEUE_OK; + case PCM_X_ALLOC_NO_CAPACITY: + return PCM_X_QUEUE_NO_CAPACITY; + case PCM_X_ALLOC_STALE_REF: + return PCM_X_QUEUE_STALE; + case PCM_X_ALLOC_BAD_STATE: + return PCM_X_QUEUE_BAD_STATE; + case PCM_X_ALLOC_CORRUPT: + return PCM_X_QUEUE_CORRUPT; + case PCM_X_ALLOC_INVALID: + return PCM_X_QUEUE_INVALID; + } + return PCM_X_QUEUE_CORRUPT; +} + + +static PcmXQueueResult +pcm_x_queue_result_from_directory(PcmXDirectoryResult result) +{ + switch (result) { + case PCM_X_DIRECTORY_OK: + return PCM_X_QUEUE_OK; + case PCM_X_DIRECTORY_NOT_FOUND: + return PCM_X_QUEUE_NOT_FOUND; + case PCM_X_DIRECTORY_EXISTS: + return PCM_X_QUEUE_DUPLICATE; + case PCM_X_DIRECTORY_FULL: + return PCM_X_QUEUE_NO_CAPACITY; + case PCM_X_DIRECTORY_STALE_REF: + /* + * Internal lookups hold the sole allocator lock. An occupied entry + * cannot legally outlive or change its resident slot there; this is + * persistent directory corruption, not an old wire request. + */ + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + case PCM_X_DIRECTORY_CORRUPT: + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + case PCM_X_DIRECTORY_INVALID: + return PCM_X_QUEUE_INVALID; + } + return PCM_X_QUEUE_CORRUPT; +} + + +static PcmXSlotHeader * +pcm_x_domain_slot(PcmXAllocatorKind kind, PcmXSlotRef ref, const BufferTag *expected_tag, + uint32 allowed_state_mask) +{ + PcmXAllocatorView view; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXSlotHeader *slot; + const BufferTag *slot_tag; + BufferTag tag_snapshot; + uint64 generation1; + uint64 generation2; + uint32 partition; + uint32 state1; + uint32 state2; + + if (header == NULL || expected_tag == NULL || allowed_state_mask == 0 + || !pcm_x_allocator_view(kind, &view)) + return NULL; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(expected_tag)); + if (LWLockHeldByMe(&header->allocator_lock.lock) + || ((kind == PCM_X_ALLOC_MASTER_TAG || kind == PCM_X_ALLOC_MASTER_TICKET) + && !LWLockHeldByMe(&header->master_locks[partition].lock)) + || (kind >= PCM_X_ALLOC_LOCAL_TAG + && !LWLockHeldByMe(&header->local_locks[partition].lock))) { + pcm_x_runtime_fail_closed(); + return NULL; + } + slot = pcm_x_allocator_slot(&view, ref.slot_index); + if (slot == NULL) + return NULL; + /* FREE/reuse can overlap this domain snapshot under the allocator lock. */ + if (!pcm_x_slot_generation_read(slot, &generation1)) + return NULL; + if (generation1 != ref.slot_generation) + return NULL; + pg_read_barrier(); + state1 = pcm_x_slot_state_read(slot); + if (state1 >= 32 || (allowed_state_mask & (UINT32_C(1) << state1)) == 0) + return NULL; + pg_read_barrier(); + slot_tag = pcm_x_slot_tag(kind, slot); + if (slot_tag == NULL) + return NULL; + tag_snapshot = *slot_tag; + pg_read_barrier(); + state2 = pcm_x_slot_state_read(slot); + pg_read_barrier(); + if (!pcm_x_slot_generation_read(slot, &generation2)) + return NULL; + if (state1 != state2 || generation1 != generation2 + || !BufferTagsEqual(&tag_snapshot, expected_tag)) + return NULL; + return slot; +} + + +static bool +pcm_x_master_stage_rollback_locked(bool new_tag, bool tag_directory_published, + bool prehandle_published, bool handle_published, + const BufferTag *tag, PcmXSlotRef tag_ref, + PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref) +{ + bool ok = true; + + if (handle_published + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, &ticket->ref.handle, + ticket_ref) + != PCM_X_DIRECTORY_OK) + ok = false; + if (prehandle_published + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &ticket->prehandle, ticket_ref) + != PCM_X_DIRECTORY_OK) + ok = false; + if (ticket != NULL + && pcm_x_allocator_release_locked(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + PCM_X_SLOT_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + ok = false; + if (tag_directory_published + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TAG, tag, tag_ref) + != PCM_X_DIRECTORY_OK) + ok = false; + if (new_tag + && pcm_x_allocator_release_locked(PCM_X_ALLOC_MASTER_TAG, tag_ref, + PCM_X_SLOT_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + ok = false; + return ok; +} + + +PcmXQueueResult +cluster_pcm_x_peer_bind_ack_publish(int32 peer_node, uint64 cluster_epoch, uint64 peer_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXPeerFrontier *frontier; + PcmXOutboundTargetFrontier *outbound; + PcmXRuntimeSnapshot runtime; + PcmXQueueResult result; + uint32 expected_gate = 0; + bool fail_closed = false; + bool gate_claimed = false; + + if (header == NULL || peer_node < 0 || peer_node >= PCM_X_PROTOCOL_NODE_LIMIT + || peer_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_runtime_token_exact(&runtime, runtime.master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto done; + } + frontier = &header->peer_frontiers[peer_node]; + outbound = &header->outbound_targets[peer_node]; + if (!pg_atomic_compare_exchange_u32(&outbound->mint_gate, &expected_gate, 1)) { + result = PCM_X_QUEUE_BUSY; + goto done; + } + gate_claimed = true; + if ((outbound->flags & ~PCM_X_OUTBOUND_TARGET_KNOWN_FLAGS) != 0 + || (outbound->flags & PCM_X_OUTBOUND_TARGET_INITIALIZED) == 0 + || outbound->next_prehandle_sequence == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto done; + } + if (frontier->cluster_epoch == 0 && frontier->sender_session_incarnation == 0 + && frontier->next_expected_prehandle_sequence == 1 + && frontier->retired_prehandle_sequence == 0 + && outbound->flags == PCM_X_OUTBOUND_TARGET_INITIALIZED && outbound->cluster_epoch == 0 + && outbound->target_session_incarnation == 0 && outbound->next_prehandle_sequence == 1) { + frontier->cluster_epoch = cluster_epoch; + frontier->sender_session_incarnation = peer_session; + outbound->cluster_epoch = cluster_epoch; + outbound->target_session_incarnation = peer_session; + outbound->flags |= PCM_X_OUTBOUND_TARGET_BOUND; + result = PCM_X_QUEUE_OK; + } else if (frontier->cluster_epoch == cluster_epoch + && frontier->sender_session_incarnation == peer_session + && frontier->next_expected_prehandle_sequence != 0 + && frontier->retired_prehandle_sequence < frontier->next_expected_prehandle_sequence + && outbound->flags + == (PCM_X_OUTBOUND_TARGET_INITIALIZED | PCM_X_OUTBOUND_TARGET_BOUND) + && outbound->cluster_epoch == cluster_epoch + && outbound->target_session_incarnation == peer_session + && outbound->next_prehandle_sequence != 0) { + result = PCM_X_QUEUE_DUPLICATE; + } else { + result = PCM_X_QUEUE_STALE; + fail_closed = true; + } + +done: + if (gate_claimed) { + pg_write_barrier(); + pg_atomic_write_u32(&outbound->mint_gate, 0); + } + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +pcm_x_master_gate_release_exact(PcmXMasterTagSlot *tag_slot, uint32 claimed_value) +{ + uint32 expected = claimed_value; + + pg_write_barrier(); + return pg_atomic_compare_exchange_u32(&tag_slot->admission_gate, &expected, 0); +} + + +static bool +pcm_x_master_admission_gate_release(PcmXMasterTagSlot *tag_slot) +{ + return pcm_x_master_gate_release_exact(tag_slot, 1); +} + + +/* + * Cross one allocator/domain lock-order window without stranding its gate. + * + * LWLockAcquire can ERROR before ownership (for example at the per-backend + * held-lock limit). Publication before this call cannot be rolled back + * safely across lock domains, so first make the partial phase explicit in the + * runtime recovery gate, then release only our exact short gate and rethrow. + */ +static void +pcm_x_master_gate_acquire_guarded(LWLock *lock, LWLockMode mode, PcmXMasterTagSlot *tag_slot, + uint32 claimed_value) +{ + PG_TRY(); + { + LWLockAcquire(lock, mode); + } + PG_CATCH(); + { + pcm_x_runtime_fail_closed(); + if (LWLockHeldByMe(lock)) + LWLockRelease(lock); + if (claimed_value != 0 && tag_slot != NULL) + (void)pcm_x_master_gate_release_exact(tag_slot, claimed_value); + PG_RE_THROW(); + } + PG_END_TRY(); +} + + +static PcmXQueueResult +pcm_x_resident_tag_state_result(const PcmXSlotHeader *slot) +{ + uint32 state; + + if (slot == NULL) + return PCM_X_QUEUE_CORRUPT; + state = pcm_x_slot_state_read(slot); + if (state == PCM_X_TAG_LIVE) + return PCM_X_QUEUE_OK; + if (state == PCM_X_TAG_RESERVED_NONVISIBLE || state == PCM_X_TAG_DETACHING) + return PCM_X_QUEUE_BUSY; + if (state == PCM_X_TAG_RECOVERY_BLOCKED) + return PCM_X_QUEUE_NOT_READY; + return PCM_X_QUEUE_CORRUPT; +} + + +static PcmXQueueResult +pcm_x_master_busy_for_node(PcmXSlotRef tag_ref, const BufferTag *tag, int32 node_id, + PcmXMasterAdmission *admission_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXMasterTagSlot *tag_slot; + Size current; + uint32 partition; + uint32 admission_gate; + uint32 queued_node_bitmap; + Size visited = 0; + bool scan_corrupt = false; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || !BufferTagsEqual(&tag_slot->tag, tag)) { + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_BUSY; + } + current = tag_slot->head_index; + while (current != PCM_X_INVALID_SLOT_INDEX && visited < PCM_X_PROTOCOL_NODE_LIMIT) { + PcmXAllocatorView view; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view)) { + scan_corrupt = true; + break; + } + ticket = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&view, current); + if (ticket == NULL) { + scan_corrupt = true; + break; + } + ticket_ref.slot_index = current; + if (!pcm_x_slot_generation_read(&ticket->slot, &ticket_ref.slot_generation)) { + scan_corrupt = true; + break; + } + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) { + scan_corrupt = true; + break; + } + if (ticket->ref.identity.node_id == node_id) { + pcm_x_master_admission_from_ticket(admission_out, ticket_ref, ticket); + admission_out->flags = PCM_X_ADMIT_F_NODE_COALESCED; + if (ticket->prev_index == PCM_X_INVALID_SLOT_INDEX) + admission_out->flags |= PCM_X_ADMIT_F_QUEUE_HEAD; + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_BUSY; + } + current = ticket->next_index; + visited++; + } + if (current != PCM_X_INVALID_SLOT_INDEX) + scan_corrupt = true; + queued_node_bitmap = pg_atomic_read_u32(&tag_slot->queued_node_bitmap); + admission_gate = pg_atomic_read_u32(&tag_slot->admission_gate); + LWLockRelease(&header->master_locks[partition].lock); + /* + * A same-node bit observed under allocator_lock can legitimately disappear + * before this scan when the old ticket cancels. Conversely, a replacement + * publisher can own admission_gate and republish the bit before its domain + * link becomes visible. Only a stable bit with no publisher and no linked + * ticket is contradictory authority. + */ + if (!scan_corrupt + && ((queued_node_bitmap & (UINT32_C(1) << node_id)) == 0 || admission_gate != 0)) + return PCM_X_QUEUE_BUSY; + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; +} + + +/* Atomically publish one master tag/ticket/directory admission, then link it. */ +static PcmXQueueResult +pcm_x_master_admit_begin_impl(const PcmXEnqueuePayload *request, PcmXMasterAdmission *admission_out, + bool cancel_first) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXMasterTagSlot *tag_slot = NULL; + PcmXMasterTicketSlot *ticket = NULL; + PcmXSlotHeader *raw_slot; + PcmXSlotRef tag_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef ticket_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef found; + PcmXSlotRef existing; + PcmXDirectoryResult directory_result; + PcmXAllocatorResult allocator_result; + PcmXQueueResult result = PCM_X_QUEUE_OK; + uint64 ticket_id; + uint64 admission_sequence; + uint32 node_bit; + uint32 partition; + uint32 expected_gate; + bool new_tag = false; + bool tag_directory_published = false; + bool prehandle_published = false; + bool gate_claimed = false; + bool fail_closed = false; + + if (admission_out != NULL) + pcm_x_master_admission_clear(admission_out); + if (header == NULL || request == NULL || admission_out == NULL + || !pcm_x_wait_identity_valid(&request->identity) + || request->prehandle.sender_session_incarnation == 0 + || request->prehandle.prehandle_sequence == 0) + return PCM_X_QUEUE_INVALID; + if (request->prehandle.prehandle_sequence == UINT64_MAX) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + /* + * This queue primitive consumes an already authenticated sender context. + * The production wire adapter must complete BIND before invoking it. + */ + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + node_bit = UINT32_C(1) << request->identity.node_id; + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_runtime_token_exact(&runtime, runtime.master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_done; + } + frontier = &header->peer_frontiers[request->identity.node_id]; + if (frontier->cluster_epoch == 0 && frontier->sender_session_incarnation == 0 + && frontier->next_expected_prehandle_sequence == 1 + && frontier->retired_prehandle_sequence == 0) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_done; + } + if (frontier->cluster_epoch != request->identity.cluster_epoch + || frontier->sender_session_incarnation != request->prehandle.sender_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto allocator_done; + } + if (frontier->next_expected_prehandle_sequence == 0 + || frontier->retired_prehandle_sequence >= frontier->next_expected_prehandle_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &request->prehandle, &found); + if (directory_result == PCM_X_DIRECTORY_OK) { + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + found); + if (ticket == NULL || !pcm_x_prehandle_equal(&ticket->prehandle, &request->prehandle) + || !pcm_x_wait_identity_equal(&ticket->ref.identity, &request->identity)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + pcm_x_master_admission_from_ticket(admission_out, found, ticket); + admission_out->flags = PCM_X_ADMIT_F_EXACT_REPLAY; + result = pcm_x_slot_state_read(&ticket->slot) == PCM_X_SLOT_RESERVED_NONVISIBLE + ? PCM_X_QUEUE_BUSY + : PCM_X_QUEUE_DUPLICATE; + goto allocator_done; + } + if (directory_result == PCM_X_DIRECTORY_STALE_REF + || directory_result == PCM_X_DIRECTORY_CORRUPT) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + if (directory_result != PCM_X_DIRECTORY_NOT_FOUND) { + result = pcm_x_queue_result_from_directory(directory_result); + goto allocator_done; + } + + if (request->prehandle.prehandle_sequence <= frontier->retired_prehandle_sequence) { + result = PCM_X_QUEUE_RETIRED; + goto allocator_done; + } + if (request->prehandle.prehandle_sequence < frontier->next_expected_prehandle_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + if (request->prehandle.prehandle_sequence > frontier->next_expected_prehandle_sequence) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_done; + } + if (frontier->next_expected_prehandle_sequence == UINT64_MAX || header->next_ticket_id == 0 + || header->next_ticket_id > PCM_X_MASTER_TICKET_ID_MAX) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto allocator_done; + } + + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TAG, &request->identity.tag, &tag_ref); + if (directory_result == PCM_X_DIRECTORY_OK) { + tag_slot + = (PcmXMasterTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TAG, tag_ref); + result = pcm_x_resident_tag_state_result(tag_slot == NULL ? NULL : &tag_slot->slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + if (!BufferTagsEqual(&tag_slot->tag, &request->identity.tag) + || tag_slot->cluster_epoch != request->identity.cluster_epoch) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + expected_gate = 0; + if (!pg_atomic_compare_exchange_u32(&tag_slot->admission_gate, &expected_gate, 1)) { + result = PCM_X_QUEUE_BUSY; + goto allocator_done; + } + gate_claimed = true; + pg_read_barrier(); + if ((pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & node_bit) != 0) { + if (!pcm_x_master_admission_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + gate_claimed = false; + result = PCM_X_QUEUE_BUSY; + goto allocator_busy; + } + if (tag_slot->next_admission_sequence == 0 + || tag_slot->next_admission_sequence == UINT64_MAX) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto allocator_done; + } + admission_sequence = tag_slot->next_admission_sequence; + } else if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_MASTER_TAG, &tag_ref, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + new_tag = true; + tag_slot = (PcmXMasterTagSlot *)raw_slot; + tag_slot->tag = request->identity.tag; + tag_slot->cluster_epoch = request->identity.cluster_epoch; + tag_slot->next_admission_sequence = 1; + tag_slot->queue_state_sequence = 0; + tag_slot->head_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->tail_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->outstanding_ticket_count = 0; + pg_atomic_init_u32(&tag_slot->queued_node_bitmap, 0); + pg_atomic_init_u32(&tag_slot->admission_gate, 1); + gate_claimed = true; + admission_sequence = 1; + } else { + result = pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto allocator_done; + } + + ticket_id = header->next_ticket_id; + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_MASTER_TICKET, &ticket_ref, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (new_tag + && !pcm_x_master_stage_rollback_locked( + true, false, false, false, &request->identity.tag, tag_ref, NULL, ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + ticket = (PcmXMasterTicketSlot *)raw_slot; + ticket->ref.identity = request->identity; + ticket->ref.handle.ticket_id = ticket_id; + ticket->ref.handle.queue_generation = ticket_ref.slot_generation; + ticket->ref.grant_generation = 0; + ticket->prehandle = request->prehandle; + ticket->master_session_incarnation = runtime.master_session_incarnation; + ticket->admission_sequence = admission_sequence; + ticket->tag_slot_index = tag_ref.slot_index; + ticket->tag_slot_generation = tag_ref.slot_generation; + ticket->next_index = PCM_X_INVALID_SLOT_INDEX; + ticket->prev_index = PCM_X_INVALID_SLOT_INDEX; + ticket->blocker_head_index = PCM_X_INVALID_SLOT_INDEX; + ticket->blocker_stage_head_index = PCM_X_INVALID_SLOT_INDEX; + ticket->blocker_stage_source_session = 0; + ticket->blocker_set_source_session = 0; + ticket->blocker_stage_source_node = -1; + ticket->blocker_set_source_node = -1; + ticket->involved_nodes_bitmap = node_bit; + + if (new_tag) { + directory_result = pcm_x_directory_insert_locked( + PCM_X_DIR_MASTER_TAG, &request->identity.tag, tag_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + if (!pcm_x_master_stage_rollback_locked(true, false, false, false, + &request->identity.tag, tag_ref, ticket, + ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + tag_directory_published = true; + } + directory_result = pcm_x_directory_insert_locked(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &ticket->prehandle, ticket_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + if (!pcm_x_master_stage_rollback_locked(new_tag, tag_directory_published, false, false, + &request->identity.tag, tag_ref, ticket, + ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + prehandle_published = true; + directory_result = pcm_x_directory_insert_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, + &ticket->ref.handle, ticket_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + if (!pcm_x_master_stage_rollback_locked(new_tag, tag_directory_published, + prehandle_published, false, &request->identity.tag, + tag_ref, ticket, ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + if (!cancel_first) + (void)pg_atomic_fetch_or_u32(&tag_slot->queued_node_bitmap, node_bit); + tag_slot->next_admission_sequence = admission_sequence + 1; + header->next_ticket_id = ticket_id + 1; + frontier->next_expected_prehandle_sequence = request->prehandle.prehandle_sequence + 1; + +allocator_done: + if (result != PCM_X_QUEUE_OK && gate_claimed && !new_tag) { + if (fail_closed) { + /* + * A post-claim allocator invariant failed. Normalize internal + * stale references to corruption and retain the gate as recovery + * evidence; all retryable outcomes take the release arm below. + */ + result = PCM_X_QUEUE_CORRUPT; + } else { + if (!pcm_x_master_admission_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + gate_claimed = false; + } + } + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&request->identity.tag)); + pcm_x_master_gate_acquire_guarded(&header->master_locks[partition].lock, LW_EXCLUSIVE, tag_slot, + 1); + /* + * Allocator publication is now durable. Every domain-phase invariant + * failure below deliberately leaves admission_gate claimed and moves the + * runtime to RECOVERY_BLOCKED; only the successful link releases it. + */ + { + PcmXAllocatorView tag_view; + PcmXMasterTagSlot *raw_tag; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TAG, &tag_view)) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + raw_tag = (PcmXMasterTagSlot *)pcm_x_allocator_slot(&tag_view, tag_ref.slot_index); + if (raw_tag == NULL || pg_atomic_read_u32(&raw_tag->admission_gate) != 1) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + pg_read_barrier(); + } + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &request->identity.tag, + PCM_X_STATE_BIT(new_tag ? PCM_X_TAG_RESERVED_NONVISIBLE : PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &request->identity.tag, + PCM_X_STATE_BIT(PCM_XT_RESERVED_NONVISIBLE)); + if (tag_slot == NULL || ticket == NULL + || pcm_x_slot_state_read(&ticket->slot) != PCM_XT_RESERVED_NONVISIBLE + || (new_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_RESERVED_NONVISIBLE) + || (!new_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE) + || !BufferTagsEqual(&tag_slot->tag, &request->identity.tag) + || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation + || pg_atomic_read_u32(&tag_slot->admission_gate) != 1) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + pg_read_barrier(); + if ((!cancel_first && (pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & node_bit) == 0) + || pg_atomic_read_u32(&tag_slot->admission_gate) != 1) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (cancel_first) { + /* PREHANDLE_CANCEL may overtake ENQUEUE. Publish the formal terminal + * while admission_gate still excludes every competing admission: no + * observer can ever see this ticket as ADMITTING or hot-linked. */ + if (tag_slot->outstanding_ticket_count == SIZE_MAX) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + } + tag_slot->outstanding_ticket_count++; + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_CANCEL; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_CANCELLED); + if (new_tag) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + } + pg_write_barrier(); + if (!pcm_x_master_admission_gate_release(tag_slot)) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + pcm_x_master_admission_from_ticket(admission_out, ticket_ref, ticket); + pcm_x_stats_increment(&header->stats.admit_count); + pcm_x_stats_increment(&header->stats.cancel_count); + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_OK; + } + if (tag_slot->queue_state_sequence == UINT64_MAX + || tag_slot->outstanding_ticket_count == SIZE_MAX) { + /* Publication cannot be rolled back across lock domains; fence recovery. */ + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + } + if (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + tag_slot->head_index = ticket_ref.slot_index; + } else { + PcmXAllocatorView ticket_view; + PcmXMasterTicketSlot *tail; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &ticket_view)) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + tail = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&ticket_view, tag_slot->tail_index); + if (tail == NULL || tail->next_index != PCM_X_INVALID_SLOT_INDEX + || tail->tag_slot_index != tag_ref.slot_index + || tail->tag_slot_generation != tag_ref.slot_generation) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + ticket->prev_index = tag_slot->tail_index; + tail->next_index = ticket_ref.slot_index; + } + tag_slot->tail_index = ticket_ref.slot_index; + tag_slot->queue_state_sequence++; + tag_slot->outstanding_ticket_count++; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_ADMITTING); + if (new_tag) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + } + pg_write_barrier(); + if (!pcm_x_master_admission_gate_release(tag_slot)) { + LWLockRelease(&header->master_locks[partition].lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + pcm_x_master_admission_from_ticket(admission_out, ticket_ref, ticket); + if (ticket->prev_index == PCM_X_INVALID_SLOT_INDEX) + admission_out->flags |= PCM_X_ADMIT_F_QUEUE_HEAD; + pcm_x_stats_increment(&header->stats.admit_count); + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_OK; + +allocator_busy: + LWLockRelease(&header->allocator_lock.lock); + return pcm_x_master_busy_for_node(tag_ref, &request->identity.tag, request->identity.node_id, + admission_out); +} + + +PcmXQueueResult +cluster_pcm_x_master_admit_begin(const PcmXEnqueuePayload *request, + PcmXMasterAdmission *admission_out) +{ + return pcm_x_master_admit_begin_impl(request, admission_out, false); +} + + +static PcmXQueueResult +pcm_x_master_ticket_lookup_locked(const PcmXTicketRef *ref, PcmXSlotRef *ticket_ref_out, + PcmXMasterTicketSlot **ticket_out) +{ + PcmXDirectoryResult directory_result; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, &ref->handle, &ticket_ref); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + ticket_ref); + if (ticket == NULL) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + /* grant_generation is published under the master-domain lock. Locator + * lookup under allocator_lock uses immutable identity+handle only; every + * caller rechecks the full ref after acquiring the domain lock. */ + if (!pcm_x_ticket_locator_equal(&ticket->ref, ref)) + return PCM_X_QUEUE_STALE; + *ticket_ref_out = ticket_ref; + *ticket_out = ticket; + return PCM_X_QUEUE_OK; +} + + +/* Locate only admission-phase messages; later phases keep full grant exactness. */ +static PcmXQueueResult +pcm_x_master_admission_ticket_lookup_locked(const PcmXTicketRef *ref, PcmXSlotRef *ticket_ref_out, + PcmXMasterTicketSlot **ticket_out) +{ + PcmXDirectoryResult directory_result; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, &ref->handle, &ticket_ref); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + ticket_ref); + if (ticket == NULL) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (!pcm_x_ticket_admission_ref_equal(&ticket->ref, ref)) + return PCM_X_QUEUE_STALE; + *ticket_ref_out = ticket_ref; + *ticket_out = ticket; + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +pcm_x_master_tag_lookup_locked(const BufferTag *tag, PcmXSlotRef *tag_ref_out, + PcmXMasterTagSlot **tag_out) +{ + PcmXDirectoryResult directory_result; + PcmXMasterTagSlot *tag_slot; + PcmXSlotRef tag_ref; + + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TAG, tag, &tag_ref); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_NOT_FOUND; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + tag_slot = (PcmXMasterTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TAG, tag_ref); + if (tag_slot == NULL) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (!BufferTagsEqual(&tag_slot->tag, tag)) + return PCM_X_QUEUE_STALE; + *tag_ref_out = tag_ref; + *tag_out = tag_slot; + return PCM_X_QUEUE_OK; +} + + +static void +pcm_x_master_wfg_snapshot_clear(PcmXMasterWfgSnapshot *snapshot) +{ + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->token.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->token.ticket_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->token.predecessor_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + snapshot->token.active_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static bool +pcm_x_slot_ref_equal(PcmXSlotRef left, PcmXSlotRef right) +{ + return left.slot_index == right.slot_index && left.slot_generation == right.slot_generation; +} + + +static bool +pcm_x_lmd_vertex_identity_equal(const ClusterLmdVertex *left, const ClusterLmdVertex *right) +{ + return left->node_id == right->node_id && left->procno == right->procno + && left->cluster_epoch == right->cluster_epoch && left->request_id == right->request_id; +} + + +static void +pcm_x_master_ticket_vertex(const PcmXMasterTicketSlot *ticket, ClusterLmdVertex *vertex) +{ + memset(vertex, 0, sizeof(*vertex)); + vertex->node_id = ticket->ref.identity.node_id; + vertex->procno = ticket->ref.identity.procno; + vertex->cluster_epoch = ticket->ref.identity.cluster_epoch; + vertex->request_id = ticket->ref.identity.request_id; + vertex->xid = ticket->ref.identity.xid; + vertex->wait_seq = ticket->ref.identity.wait_seq; +} + + +static bool +pcm_x_master_wfg_token_equal(const PcmXMasterWfgToken *left, const PcmXMasterWfgToken *right) +{ + return pcm_x_ticket_ref_equal(&left->ticket, &right->ticket) + && pcm_x_slot_ref_equal(left->tag_slot, right->tag_slot) + && pcm_x_slot_ref_equal(left->ticket_slot, right->ticket_slot) + && pcm_x_slot_ref_equal(left->predecessor_slot, right->predecessor_slot) + && pcm_x_slot_ref_equal(left->active_slot, right->active_slot) + && left->queue_state_sequence == right->queue_state_sequence; +} + + +/* + * Capture only master-queue blockers while the caller holds the tag partition + * lock. This function does not call LMD: the graph replace happens after the + * public snapshot API has released every PCM queue lock. + */ +static PcmXQueueResult +pcm_x_master_wfg_capture_locked(PcmXRuntimeSnapshot runtime, const PcmXTicketRef *ref, + PcmXSlotRef tag_ref, PcmXSlotRef ticket_ref, + PcmXMasterWfgSnapshot *snapshot) +{ + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *active = NULL; + PcmXMasterTicketSlot *predecessor = NULL; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView ticket_view; + PcmXSlotRef active_ref; + PcmXSlotRef predecessor_ref; + uint32 active_state; + uint32 state; + + pcm_x_master_wfg_snapshot_clear(snapshot); + active_ref.slot_index = PCM_X_INVALID_SLOT_INDEX; + active_ref.slot_generation = 0; + predecessor_ref.slot_index = PCM_X_INVALID_SLOT_INDEX; + predecessor_ref.slot_generation = 0; + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref) + || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || tag_slot->cluster_epoch != ref->identity.cluster_epoch + || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) + return PCM_X_QUEUE_STALE; + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + return PCM_X_QUEUE_NOT_READY; + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_ADMITTING) { + if (ticket->graph_generation != 0 + && pcm_x_master_event_replay_exact((PcmXMasterTicketState)state, + PCM_X_EVENT_ADMIT_CONFIRM, + ticket->reliable.response_tombstone_mask)) + return PCM_X_QUEUE_DUPLICATE; + return PCM_X_QUEUE_BAD_STATE; + } + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &ticket_view)) + return PCM_X_QUEUE_CORRUPT; + + if (ticket->prev_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->head_index != ticket_ref.slot_index) + return PCM_X_QUEUE_CORRUPT; + } else { + if (ticket->prev_index == ticket_ref.slot_index + || tag_slot->head_index == ticket_ref.slot_index) + return PCM_X_QUEUE_CORRUPT; + predecessor + = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&ticket_view, ticket->prev_index); + if (predecessor == NULL + || !pcm_x_slot_generation_read(&predecessor->slot, &predecessor_ref.slot_generation)) + return PCM_X_QUEUE_CORRUPT; + predecessor_ref.slot_index = ticket->prev_index; + predecessor = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, + predecessor_ref, &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (predecessor == NULL || predecessor->next_index != ticket_ref.slot_index + || predecessor->tag_slot_index != tag_ref.slot_index + || predecessor->tag_slot_generation != tag_ref.slot_generation + || (pcm_x_slot_state_read(&predecessor->slot) != PCM_XT_ADMITTING + && pcm_x_slot_state_read(&predecessor->slot) != PCM_XT_QUEUED + && pcm_x_slot_state_read(&predecessor->slot) != PCM_XT_ACTIVE_PROBE + && pcm_x_slot_state_read(&predecessor->slot) != PCM_XT_ACTIVE_TRANSFER)) + return PCM_X_QUEUE_CORRUPT; + } + + if (tag_slot->active_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->active_slot_generation != 0) + return PCM_X_QUEUE_CORRUPT; + } else { + if (tag_slot->active_index == ticket_ref.slot_index + || tag_slot->active_index != tag_slot->head_index) + return PCM_X_QUEUE_CORRUPT; + active_ref.slot_index = tag_slot->active_index; + active_ref.slot_generation = tag_slot->active_slot_generation; + active = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, active_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (active == NULL || active->tag_slot_index != tag_ref.slot_index + || active->tag_slot_generation != tag_ref.slot_generation) + return PCM_X_QUEUE_CORRUPT; + active_state = pcm_x_slot_state_read(&active->slot); + if (active_state != PCM_XT_ACTIVE_PROBE && active_state != PCM_XT_ACTIVE_TRANSFER) + return PCM_X_QUEUE_CORRUPT; + } + + snapshot->token.ticket = ticket->ref; + snapshot->token.tag_slot = tag_ref; + snapshot->token.ticket_slot = ticket_ref; + snapshot->token.predecessor_slot = predecessor_ref; + snapshot->token.active_slot = active_ref; + snapshot->token.queue_state_sequence = tag_slot->queue_state_sequence; + if (active != NULL) + pcm_x_master_ticket_vertex(active, &snapshot->blockers[snapshot->blocker_count++]); + if (predecessor != NULL) { + ClusterLmdVertex predecessor_vertex; + + pcm_x_master_ticket_vertex(predecessor, &predecessor_vertex); + if (snapshot->blocker_count == 0 + || !pcm_x_lmd_vertex_identity_equal(&snapshot->blockers[0], &predecessor_vertex)) + snapshot->blockers[snapshot->blocker_count++] = predecessor_vertex; + } + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_master_admit_wfg_snapshot_exact(const PcmXTicketRef *ref, + PcmXMasterWfgSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef requested_ref; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (ref != NULL) + requested_ref = *ref; + if (snapshot_out != NULL) + pcm_x_master_wfg_snapshot_clear(snapshot_out); + if (header == NULL || ref == NULL || snapshot_out == NULL || requested_ref.grant_generation != 0 + || !pcm_x_wait_identity_valid(&requested_ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_admission_ticket_lookup_locked(&requested_ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&requested_ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + result = pcm_x_master_wfg_capture_locked(runtime, &requested_ref, tag_ref, ticket_ref, + snapshot_out); + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_master_admit_confirm_transition_locked(PcmXShmemHeader *header, PcmXMasterTicketSlot *ticket, + uint64 graph_generation) +{ + if (!pcm_x_stats_depth_increment(header)) + return PCM_X_QUEUE_CORRUPT; + ticket->graph_generation = graph_generation; + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_QUEUED); + pcm_x_stats_increment(&header->stats.confirm_count); + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_master_admit_confirm_revalidate_exact(const PcmXMasterWfgToken *token, + uint64 graph_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXMasterWfgSnapshot current; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + uint32 state; + + if (header == NULL || token == NULL || graph_generation == 0 + || token->ticket.grant_generation != 0 + || !pcm_x_wait_identity_valid(&token->ticket.identity) + || token->tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || token->ticket_slot.slot_index == PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_admission_ticket_lookup_locked(&token->ticket, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + if (!pcm_x_slot_ref_equal(ticket_ref, token->ticket_slot) + || !pcm_x_slot_ref_equal(tag_ref, token->tag_slot)) + result = PCM_X_QUEUE_STALE; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&token->ticket.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &token->ticket.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, &token->ticket)) { + result = PCM_X_QUEUE_STALE; + goto done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto done; + } + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_ADMITTING) { + if (pcm_x_master_event_replay_exact((PcmXMasterTicketState)state, PCM_X_EVENT_ADMIT_CONFIRM, + ticket->reliable.response_tombstone_mask) + && ticket->graph_generation == graph_generation) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_BAD_STATE; + goto done; + } + result + = pcm_x_master_wfg_capture_locked(runtime, &token->ticket, tag_ref, ticket_ref, ¤t); + if (result == PCM_X_QUEUE_OK && !pcm_x_master_wfg_token_equal(¤t.token, token)) + result = PCM_X_QUEUE_STALE; + if (result == PCM_X_QUEUE_OK) + result = pcm_x_master_admit_confirm_transition_locked(header, ticket, graph_generation); + +done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_admit_confirm_exact(const PcmXTicketRef *ref, uint64 graph_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (header == NULL || ref == NULL || graph_generation == 0 || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_admission_ticket_lookup_locked(ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) == PCM_XT_ADMITTING) + result = pcm_x_master_admit_confirm_transition_locked(header, ticket, graph_generation); + else if (pcm_x_master_event_replay_exact( + (PcmXMasterTicketState)pcm_x_slot_state_read(&ticket->slot), + PCM_X_EVENT_ADMIT_CONFIRM, ticket->reliable.response_tombstone_mask) + && ticket->graph_generation == graph_generation) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_BAD_STATE; + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_promote_head_exact(const BufferTag *tag, uint64 cluster_epoch, + PcmXTicketRef *active_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView ticket_view; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + uint32 state; + Size ticket_index; + + if (active_out != NULL) + memset(active_out, 0, sizeof(*active_out)); + if (header == NULL || tag == NULL || active_out == NULL) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_tag_lookup_locked(tag, &tag_ref, &tag_slot); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || !BufferTagsEqual(&tag_slot->tag, tag) || tag_slot->cluster_epoch != cluster_epoch) { + result = PCM_X_QUEUE_STALE; + goto done; + } + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &ticket_view)) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + if (tag_slot->active_index != PCM_X_INVALID_SLOT_INDEX) { + ticket = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&ticket_view, tag_slot->active_index); + if (ticket == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + ticket_ref.slot_index = tag_slot->active_index; + ticket_ref.slot_generation = tag_slot->active_slot_generation; + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + tag, PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + state = pcm_x_slot_state_read(&ticket->slot); + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED) { + result = PCM_X_QUEUE_NOT_READY; + goto done; + } + if (state != PCM_XT_ACTIVE_PROBE && state != PCM_XT_ACTIVE_TRANSFER) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + *active_out = ticket->ref; + result = PCM_X_QUEUE_BUSY; + goto done; + } + ticket_index = tag_slot->head_index; + if (ticket_index == PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_NOT_FOUND; + goto done; + } + ticket = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&ticket_view, ticket_index); + if (ticket == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + ticket_ref.slot_index = ticket_index; + if (!pcm_x_slot_generation_read(&ticket->slot, &ticket_ref.slot_generation)) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_QUEUED) { + result = PCM_X_QUEUE_NOT_READY; + goto done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto done; + } + if (!pcm_x_stats_depth_decrement(header)) { + result = PCM_X_QUEUE_CORRUPT; + goto done; + } + tag_slot->active_index = ticket_index; + tag_slot->active_slot_generation = ticket_ref.slot_generation; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_ACTIVE_PROBE); + *active_out = ticket->ref; + pcm_x_stats_increment(&header->stats.promotion_count); + result = PCM_X_QUEUE_OK; + +done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +pcm_x_master_pending_x_flags_valid(uint32 flags) +{ + const uint32 terminal = PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED; + + if ((flags & ~PCM_X_MASTER_TICKET_F_PENDING_X_KNOWN) != 0) + return false; + if ((flags & terminal) != 0 && (flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) == 0) + return false; + return (flags & terminal) != terminal; +} + + +static PcmXQueueResult +pcm_x_master_pending_x_claim_exact_impl(const PcmXTicketRef *ref, bool claim, bool *claimed_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 flags; + uint32 partition; + uint32 state; + + if (claimed_out != NULL) + *claimed_out = false; + if (header == NULL || ref == NULL || (claim && claimed_out != NULL) + || (!claim && claimed_out == NULL) || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_admission_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else { + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_ACTIVE_PROBE) { + if (!claim + && (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED + || state == PCM_XT_RETIRE_CREDIT)) { + if (ticket->next_index != PCM_X_INVALID_SLOT_INDEX + || ticket->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_index == ticket_ref.slot_index + || tag_slot->head_index == ticket_ref.slot_index + || tag_slot->tail_index == ticket_ref.slot_index) + result = PCM_X_QUEUE_CORRUPT; + else + result = PCM_X_QUEUE_NOT_READY; + } else + result = PCM_X_QUEUE_BAD_STATE; + } else if (tag_slot->active_index != ticket_ref.slot_index + || tag_slot->active_slot_generation != ticket_ref.slot_generation) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_pending_x_flags_valid( + (flags = pcm_x_slot_flags_read(&ticket->slot)))) + result = PCM_X_QUEUE_CORRUPT; + else if (!claim) { + if ((flags + & (PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + != 0) + result = PCM_X_QUEUE_NOT_READY; + else { + *claimed_out = (flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) != 0; + result = PCM_X_QUEUE_OK; + } + } else if ((flags + & (PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + != 0) + result = PCM_X_QUEUE_NOT_READY; + else if ((flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) != 0) + result = PCM_X_QUEUE_DUPLICATE; + else { + pcm_x_slot_flags_write(&ticket->slot, PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED); + result = PCM_X_QUEUE_OK; + } + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_pending_x_claim_state_exact(const PcmXTicketRef *ref, bool *claimed_out) +{ + return pcm_x_master_pending_x_claim_exact_impl(ref, false, claimed_out); +} + + +PcmXQueueResult +cluster_pcm_x_master_pending_x_claim_exact(const PcmXTicketRef *ref) +{ + return pcm_x_master_pending_x_claim_exact_impl(ref, true, NULL); +} + + +static PcmXQueueResult +pcm_x_master_pending_x_claim_required(const PcmXMasterTicketSlot *ticket) +{ + uint32 flags; + + if (ticket == NULL) + return PCM_X_QUEUE_CORRUPT; + flags = pcm_x_slot_flags_read(&ticket->slot); + if (!pcm_x_master_pending_x_flags_valid(flags)) + return PCM_X_QUEUE_CORRUPT; + if ((flags + & (PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED)) + != 0) + return PCM_X_QUEUE_NOT_READY; + return flags == PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_BAD_STATE; +} + + +static PcmXQueueResult +pcm_x_master_drive_candidate_exact(PcmXShmemHeader *header, PcmXSlotRef tag_ref, + const BufferTag *tag, BufferTag *tag_out, + uint64 *cluster_epoch_out) +{ + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + uint32 partition; + uint32 ticket_state; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || tag_slot->active_index == PCM_X_INVALID_SLOT_INDEX) { + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_NOT_FOUND; + } + ticket_ref.slot_index = tag_slot->active_index; + ticket_ref.slot_generation = tag_slot->active_slot_generation; + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL) { + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_CORRUPT; + } + ticket_state = pcm_x_slot_state_read(&ticket->slot); + if (ticket_state != PCM_XT_ACTIVE_PROBE && ticket_state != PCM_XT_ACTIVE_TRANSFER) { + LWLockRelease(&header->master_locks[partition].lock); + return ticket_state == PCM_XT_COMPLETE || ticket_state == PCM_XT_CANCELLED + ? PCM_X_QUEUE_NOT_FOUND + : PCM_X_QUEUE_CORRUPT; + } + *tag_out = tag_slot->tag; + *cluster_epoch_out = tag_slot->cluster_epoch; + LWLockRelease(&header->master_locks[partition].lock); + return PCM_X_QUEUE_OK; +} + + +/* Inspect at most scan_budget master-tag slots from a caller-owned cursor. + * Formation retry therefore makes bounded progress without adding a shared + * cursor to the fixed shmem header. */ +PcmXQueueResult +cluster_pcm_x_master_drive_work_next(Size *cursor_io, Size scan_budget, BufferTag *tag_out, + uint64 *cluster_epoch_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXAllocatorView view; + PcmXMasterTagSlot *raw; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + BufferTag tag; + Size budget; + Size i; + Size index; + Size start; + uint64 generation_after; + + if (tag_out != NULL) + memset(tag_out, 0, sizeof(*tag_out)); + if (cluster_epoch_out != NULL) + *cluster_epoch_out = 0; + if (header == NULL || cursor_io == NULL || scan_budget == 0 || tag_out == NULL + || cluster_epoch_out == NULL) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TAG, &view) || view.capacity == 0) + return PCM_X_QUEUE_INVALID; + + start = *cursor_io % view.capacity; + budget = Min(scan_budget, view.capacity); + for (i = 0; i < budget; i++) { + index = (start + i) % view.capacity; + *cursor_io = (index + 1) % view.capacity; + raw = (PcmXMasterTagSlot *)pcm_x_allocator_slot(&view, index); + if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE + || !pcm_x_slot_generation_read(&raw->slot, &tag_ref.slot_generation)) + continue; + tag_ref.slot_index = index; + tag = raw->tag; + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&raw->slot, &generation_after) + || generation_after != tag_ref.slot_generation + || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE) + continue; + + /* Formation retries only work that has crossed the ADMIT_CONFIRM_ACK + * outbound admission boundary. A queued-only head is first promoted by + * the event path after that ACK is durably staged; otherwise a full + * outbound ring could let type 49 overtake an unstaged type 44. */ + result + = pcm_x_master_drive_candidate_exact(header, tag_ref, &tag, tag_out, cluster_epoch_out); + if (result == PCM_X_QUEUE_CORRUPT) { + pcm_x_runtime_fail_closed(); + return result; + } + if (result == PCM_X_QUEUE_OK) { + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + memset(tag_out, 0, sizeof(*tag_out)); + *cluster_epoch_out = 0; + return PCM_X_QUEUE_NOT_READY; + } + return PCM_X_QUEUE_OK; + } + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) + return PCM_X_QUEUE_NOT_READY; + return PCM_X_QUEUE_NOT_FOUND; +} + + +static void +pcm_x_master_drive_snapshot_clear(PcmXMasterDriveSnapshot *snapshot) +{ + if (snapshot != NULL) + memset(snapshot, 0, sizeof(*snapshot)); +} + + +static bool +pcm_x_master_drive_leg_valid(const PcmXReliableLegState *leg) +{ + if (leg == NULL || leg->flags != 0 || leg->reserved != 0) + return false; + if (leg->pending_opcode == 0) + return leg->retry_deadline_ms == 0 && leg->expected_responder_session == 0 + && leg->retry_count == 0 && leg->expected_responder_node == 0 && leg->phase == 0; + return leg->phase == leg->pending_opcode && leg->expected_responder_session != 0 + && leg->expected_responder_node >= 0 + && leg->expected_responder_node < PCM_X_PROTOCOL_NODE_LIMIT; +} + + +static PcmXQueueResult +pcm_x_master_drive_capture_locked(PcmXRuntimeSnapshot runtime, PcmXMasterTagSlot *tag_slot, + PcmXSlotRef tag_ref, PcmXMasterTicketSlot *ticket, + PcmXSlotRef ticket_ref, PcmXMasterDriveSnapshot *snapshot) +{ + uint32 state; + uint32 ticket_flags; + + pcm_x_master_drive_snapshot_clear(snapshot); + if (tag_slot == NULL || ticket == NULL || snapshot == NULL) + return PCM_X_QUEUE_CORRUPT; + state = pcm_x_slot_state_read(&ticket->slot); + if (pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || !BufferTagsEqual(&tag_slot->tag, &ticket->ref.identity.tag) + || tag_slot->cluster_epoch != ticket->ref.identity.cluster_epoch + || tag_slot->active_index != ticket_ref.slot_index + || tag_slot->active_slot_generation != ticket_ref.slot_generation + || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) + return PCM_X_QUEUE_CORRUPT; + /* COMPLETE/CANCELLED legitimately remain the tag's active locator until + * exact drain/retire detaches them. They are not master-drive work and + * must not turn an unrelated retry/ACK lookup into RECOVERY_BLOCKED. */ + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED) + return PCM_X_QUEUE_NOT_READY; + if (state != PCM_XT_ACTIVE_PROBE && state != PCM_XT_ACTIVE_TRANSFER) + return PCM_X_QUEUE_CORRUPT; + if ((state == PCM_XT_ACTIVE_PROBE && ticket->ref.grant_generation != 0) + || (state == PCM_XT_ACTIVE_TRANSFER && ticket->ref.grant_generation == 0)) + return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + return PCM_X_QUEUE_NOT_READY; + ticket_flags = pcm_x_slot_flags_read(&ticket->slot); + if (!pcm_x_master_pending_x_flags_valid(ticket_flags)) + return PCM_X_QUEUE_CORRUPT; + if ((state == PCM_XT_ACTIVE_TRANSFER && ticket_flags != PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) + || (state == PCM_XT_ACTIVE_PROBE + && (ticket_flags & PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING) != 0)) + return PCM_X_QUEUE_CORRUPT; + if ((ticket->acked_s_holders_bitmap & ~ticket->pending_s_holders_bitmap) != 0 + || !pcm_x_master_drive_leg_valid(&ticket->reliable) + || !pcm_x_master_blocker_stage_metadata_valid(ticket)) + return PCM_X_QUEUE_CORRUPT; + + snapshot->ref = ticket->ref; + snapshot->image = ticket->image; + snapshot->master_session_incarnation = ticket->master_session_incarnation; + snapshot->blocker_set_generation = ticket->blocker_set_generation; + snapshot->blocker_stage_set_generation = ticket->blocker_stage_set_generation; + snapshot->graph_generation = ticket->graph_generation; + snapshot->state_sequence = ticket->reliable.state_sequence; + snapshot->retry_deadline_ms = ticket->reliable.retry_deadline_ms; + snapshot->response_tombstone_mask = ticket->reliable.response_tombstone_mask; + snapshot->expected_responder_session = ticket->reliable.expected_responder_session; + snapshot->pending_s_holders_bitmap = ticket->pending_s_holders_bitmap; + snapshot->acked_s_holders_bitmap = ticket->acked_s_holders_bitmap; + snapshot->involved_nodes_bitmap = ticket->involved_nodes_bitmap; + snapshot->blocker_count = ticket->blocker_count; + snapshot->retry_count = ticket->reliable.retry_count; + snapshot->last_responder_node = ticket->reliable.last_responder_node; + snapshot->expected_responder_node = ticket->reliable.expected_responder_node; + snapshot->ticket_state = (uint16)state; + snapshot->pending_opcode = ticket->reliable.pending_opcode; + snapshot->last_response_opcode = ticket->reliable.last_response_opcode; + snapshot->phase = ticket->reliable.phase; + /* PcmXReliableLegState flags/reserved are validated zero above. Reuse the + * process-local tail for master-ticket control flags so bitmap revalidate + * and the tag-sharded driver both observe CANCEL_REQUESTED exactly. */ + snapshot->flags = (uint16)ticket_flags; + snapshot->reserved = ticket->reliable.reserved; + return PCM_X_QUEUE_OK; +} + + +static bool +pcm_x_master_drive_snapshot_base_equal(const PcmXMasterDriveSnapshot *left, + const PcmXMasterDriveSnapshot *right) +{ + return left != NULL && right != NULL && pcm_x_ticket_ref_equal(&left->ref, &right->ref) + && pcm_x_image_token_equal(&left->image, &right->image) + && left->master_session_incarnation == right->master_session_incarnation + && left->blocker_set_generation == right->blocker_set_generation + && left->blocker_stage_set_generation == right->blocker_stage_set_generation + && left->graph_generation == right->graph_generation + && left->state_sequence == right->state_sequence + && left->retry_deadline_ms == right->retry_deadline_ms + && left->response_tombstone_mask == right->response_tombstone_mask + && left->expected_responder_session == right->expected_responder_session + && left->blocker_count == right->blocker_count && left->retry_count == right->retry_count + && left->last_responder_node == right->last_responder_node + && left->expected_responder_node == right->expected_responder_node + && left->ticket_state == right->ticket_state + && left->pending_opcode == right->pending_opcode + && left->last_response_opcode == right->last_response_opcode + && left->phase == right->phase && left->flags == right->flags + && left->reserved == right->reserved; +} + + +static bool +pcm_x_master_drive_snapshot_equal(const PcmXMasterDriveSnapshot *left, + const PcmXMasterDriveSnapshot *right) +{ + return pcm_x_master_drive_snapshot_base_equal(left, right) + && left->pending_s_holders_bitmap == right->pending_s_holders_bitmap + && left->acked_s_holders_bitmap == right->acked_s_holders_bitmap + && left->involved_nodes_bitmap == right->involved_nodes_bitmap; +} + + +PcmXQueueResult +cluster_pcm_x_master_drive_snapshot_exact(const BufferTag *tag, uint64 cluster_epoch, + PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (header == NULL || tag == NULL || snapshot_out == NULL) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_tag_lookup_locked(tag, &tag_ref, &tag_slot); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || !BufferTagsEqual(&tag_slot->tag, tag) + || tag_slot->cluster_epoch != cluster_epoch) { + result = PCM_X_QUEUE_STALE; + goto snapshot_done; + } + if (tag_slot->active_index == PCM_X_INVALID_SLOT_INDEX) { + result + = tag_slot->active_slot_generation == 0 ? PCM_X_QUEUE_NOT_READY : PCM_X_QUEUE_CORRUPT; + goto snapshot_done; + } + ticket_ref.slot_index = tag_slot->active_index; + ticket_ref.slot_generation = tag_slot->active_slot_generation; + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot_out); + +snapshot_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_drive_bitmap_replace_exact(const PcmXMasterDriveSnapshot *expected, + uint32 pending_s_holders_bitmap, + uint32 acked_s_holders_bitmap, + PcmXMasterDriveSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXMasterDriveSnapshot current; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + pcm_x_master_drive_snapshot_clear(snapshot_out); + if (header == NULL || expected == NULL || snapshot_out == NULL + || (acked_s_holders_bitmap & ~pending_s_holders_bitmap) != 0 + || (expected->acked_s_holders_bitmap & ~expected->pending_s_holders_bitmap) != 0 + || (expected->ticket_state != PCM_XT_ACTIVE_PROBE + && expected->ticket_state != PCM_XT_ACTIVE_TRANSFER) + || expected->reserved != 0 || !pcm_x_wait_identity_valid(&expected->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&expected->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&expected->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, + &expected->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &expected->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + ¤t); + if (result != PCM_X_QUEUE_OK) + goto replace_done; + *snapshot_out = current; + if (pcm_x_master_drive_snapshot_base_equal(¤t, expected) + && current.pending_s_holders_bitmap == pending_s_holders_bitmap + && current.acked_s_holders_bitmap == acked_s_holders_bitmap + && current.involved_nodes_bitmap == expected->involved_nodes_bitmap) { + result = PCM_X_QUEUE_DUPLICATE; + goto replace_done; + } + if (!pcm_x_master_drive_snapshot_equal(¤t, expected)) { + result = PCM_X_QUEUE_STALE; + goto replace_done; + } + + ticket->pending_s_holders_bitmap = pending_s_holders_bitmap; + ticket->acked_s_holders_bitmap = acked_s_holders_bitmap; + pg_write_barrier(); + result = pcm_x_master_drive_capture_locked(runtime, tag_slot, tag_ref, ticket, ticket_ref, + snapshot_out); + +replace_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +typedef struct PcmXBlockerRead { + PcmXSlotRef slot_ref; + ClusterLmdVertex blocker; + Size next_index; +} PcmXBlockerRead; + + +static bool +pcm_x_lmd_vertex_metadata_equal(const ClusterLmdVertex *left, const ClusterLmdVertex *right) +{ + return left->xid == right->xid && left->local_start_ts_ms == right->local_start_ts_ms + && left->wait_seq == right->wait_seq; +} + + +static bool +pcm_x_lmd_vertex_equal(const ClusterLmdVertex *left, const ClusterLmdVertex *right) +{ + return pcm_x_lmd_vertex_identity_equal(left, right) + && pcm_x_lmd_vertex_metadata_equal(left, right); +} + + +static int +pcm_x_lmd_vertex_identity_compare(const ClusterLmdVertex *left, const ClusterLmdVertex *right) +{ + if (left->node_id != right->node_id) + return left->node_id < right->node_id ? -1 : 1; + if (left->procno != right->procno) + return left->procno < right->procno ? -1 : 1; + if (left->cluster_epoch != right->cluster_epoch) + return left->cluster_epoch < right->cluster_epoch ? -1 : 1; + if (left->request_id != right->request_id) + return left->request_id < right->request_id ? -1 : 1; + return 0; +} + + +static bool +pcm_x_lmd_vertex_valid(const ClusterLmdVertex *vertex) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + return header != NULL && vertex != NULL && vertex->node_id >= 0 + && vertex->node_id < PCM_X_PROTOCOL_NODE_LIMIT + && (vertex->request_id != 0 + || (vertex->procno != CLUSTER_LMD_TX_HOLDER_PROCNO && vertex->wait_seq != 0 + && TransactionIdIsValid(vertex->xid))) + && (vertex->procno == CLUSTER_LMD_TX_HOLDER_PROCNO + || vertex->procno < header->layout.process_capacity); +} + + +/* ClusterLmdVertex has an ABI padding word before local_start_ts_ms. Never + * include that indeterminate byte range in the canonical blocker-set CRC. */ +static void +pcm_x_blocker_crc32c_update(pg_crc32c *crc, const ClusterLmdVertex *blocker) +{ + COMP_CRC32C(*crc, &blocker->node_id, sizeof(blocker->node_id)); + COMP_CRC32C(*crc, &blocker->procno, sizeof(blocker->procno)); + COMP_CRC32C(*crc, &blocker->cluster_epoch, sizeof(blocker->cluster_epoch)); + COMP_CRC32C(*crc, &blocker->request_id, sizeof(blocker->request_id)); + COMP_CRC32C(*crc, &blocker->xid, sizeof(blocker->xid)); + COMP_CRC32C(*crc, &blocker->local_start_ts_ms, sizeof(blocker->local_start_ts_ms)); + COMP_CRC32C(*crc, &blocker->wait_seq, sizeof(blocker->wait_seq)); +} + + +uint32 +cluster_pcm_x_blocker_set_crc32c(const ClusterLmdVertex *blockers, Size nblockers) +{ + pg_crc32c crc; + Size i; + + if (nblockers != 0 && blockers == NULL) + return 0; + INIT_CRC32C(crc); + for (i = 0; i < nblockers; i++) + pcm_x_blocker_crc32c_update(&crc, &blockers[i]); + FIN_CRC32C(crc); + return (uint32)crc; +} + + +/* Validate and count canonical identities without allocating process memory. */ +static PcmXQueueResult +pcm_x_blocker_input_validate(const PcmXTicketRef *ref, const ClusterLmdVertex *blockers, + Size nblockers, Size *unique_count_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + Size i; + Size j; + Size unique_count = 0; + + if (unique_count_out != NULL) + *unique_count_out = 0; + if (header == NULL || ref == NULL || unique_count_out == NULL + || (nblockers > 0 && blockers == NULL) || nblockers > UINT32_MAX) + return PCM_X_QUEUE_INVALID; + /* The pool is double-buffered (2 * max_wait_edges); one published set + * may consume at most one half so a full replacement can reserve first. */ + if (nblockers > header->layout.max_wait_edges) + return PCM_X_QUEUE_NO_CAPACITY; + + for (i = 0; i < nblockers; i++) { + bool first_identity = true; + + if (!pcm_x_lmd_vertex_valid(&blockers[i])) + return PCM_X_QUEUE_INVALID; + if (blockers[i].node_id == ref->identity.node_id + && blockers[i].procno == ref->identity.procno + && blockers[i].cluster_epoch == ref->identity.cluster_epoch + && blockers[i].request_id == ref->identity.request_id) + return PCM_X_QUEUE_CORRUPT; + for (j = 0; j < i; j++) { + if (!pcm_x_lmd_vertex_identity_equal(&blockers[j], &blockers[i])) + continue; + if (!pcm_x_lmd_vertex_metadata_equal(&blockers[j], &blockers[i])) + return PCM_X_QUEUE_CORRUPT; + first_identity = false; + break; + } + if (first_identity) + unique_count++; + } + *unique_count_out = unique_count; + return PCM_X_QUEUE_OK; +} + + +static const ClusterLmdVertex * +pcm_x_blocker_next_canonical(const ClusterLmdVertex *blockers, Size nblockers, + const ClusterLmdVertex *previous) +{ + const ClusterLmdVertex *candidate = NULL; + Size i; + Size j; + + for (i = 0; i < nblockers; i++) { + bool first_identity = true; + + for (j = 0; j < i; j++) { + if (pcm_x_lmd_vertex_identity_equal(&blockers[j], &blockers[i])) { + first_identity = false; + break; + } + } + if (!first_identity + || (previous != NULL && pcm_x_lmd_vertex_identity_compare(previous, &blockers[i]) >= 0)) + continue; + if (candidate == NULL || pcm_x_lmd_vertex_identity_compare(&blockers[i], candidate) < 0) + candidate = &blockers[i]; + } + return candidate; +} + + +/* Read one blocker only when its complete owner tuple is stable and exact. */ +static bool +pcm_x_blocker_read_exact(const PcmXAllocatorView *view, Size slot_index, + const PcmXTicketHandle *owner_handle, Size owner_slot_index, + uint64 owner_slot_generation, uint64 set_generation, + uint32 expected_chunk_no, uint32 expected_state, PcmXBlockerRead *read_out) +{ + PcmXBlockerSlot *slot; + PcmXTicketHandle handle; + ClusterLmdVertex blocker; + uint64 generation1; + uint64 generation2; + uint64 copied_owner_generation; + uint64 copied_set_generation; + Size copied_next_index; + Size copied_owner_index; + uint32 copied_chunk_no; + uint32 copied_direction; + uint32 state1; + uint32 state2; + + if (view == NULL || owner_handle == NULL || read_out == NULL) + return false; + slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(view, slot_index); + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &generation1) || generation1 == 0) + return false; + pg_read_barrier(); + state1 = pcm_x_slot_state_read(&slot->slot); + if (state1 != expected_state) + return false; + pg_read_barrier(); + handle = slot->handle; + blocker = slot->blocker; + copied_owner_generation = slot->owner_slot_generation; + copied_set_generation = slot->set_generation; + copied_next_index = slot->next_index; + copied_owner_index = slot->owner_slot_index; + copied_chunk_no = slot->chunk_no; + copied_direction = slot->direction; + pg_read_barrier(); + state2 = pcm_x_slot_state_read(&slot->slot); + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&slot->slot, &generation2) || generation1 != generation2 + || state1 != state2) + return false; + if (!pcm_x_ticket_handle_equal(&handle, owner_handle) + || owner_handle->queue_generation != owner_slot_generation + || copied_owner_generation != owner_slot_generation + || copied_owner_index != owner_slot_index || copied_set_generation != set_generation + || copied_chunk_no != expected_chunk_no + || copied_direction != PCM_X_BLOCKER_DIRECTION_MASTER + || (copied_next_index != PCM_X_INVALID_SLOT_INDEX && copied_next_index >= view->capacity) + || !pcm_x_lmd_vertex_valid(&blocker)) + return false; + read_out->slot_ref.slot_index = slot_index; + read_out->slot_ref.slot_generation = generation1; + read_out->blocker = blocker; + read_out->next_index = copied_next_index; + return true; +} + + +/* Validate a complete canonical chain before any caller mutates or copies it. */ +static bool +pcm_x_blocker_chain_validate(const PcmXAllocatorView *view, Size head_index, uint32 count, + const PcmXTicketHandle *owner_handle, Size owner_slot_index, + uint64 owner_slot_generation, uint64 set_generation, + uint32 expected_state, const PcmXMasterBlockerEntry *expected_entries, + Size expected_count, PcmXMasterBlockerEntry *entries_out) +{ + ClusterLmdVertex previous; + Size current_index = head_index; + uint32 i; + bool have_previous = false; + + if (view == NULL || owner_handle == NULL || count > view->capacity + || (expected_entries != NULL && expected_count != count)) + return false; + if (count == 0) + return head_index == PCM_X_INVALID_SLOT_INDEX; + if (head_index == PCM_X_INVALID_SLOT_INDEX) + return false; + + for (i = 0; i < count; i++) { + PcmXBlockerRead read; + + if (!pcm_x_blocker_read_exact(view, current_index, owner_handle, owner_slot_index, + owner_slot_generation, set_generation, i, expected_state, + &read)) + return false; + if (have_previous && pcm_x_lmd_vertex_identity_compare(&previous, &read.blocker) >= 0) + return false; + if (expected_entries != NULL) { + const PcmXMasterBlockerEntry *expected = &expected_entries[i]; + + if (expected->reserved != 0 || expected->chunk_no != i + || expected->slot_ref.slot_index != read.slot_ref.slot_index + || expected->slot_ref.slot_generation != read.slot_ref.slot_generation + || !pcm_x_lmd_vertex_equal(&expected->blocker, &read.blocker)) + return false; + } + if (entries_out != NULL) { + entries_out[i].slot_ref = read.slot_ref; + entries_out[i].blocker = read.blocker; + entries_out[i].chunk_no = i; + entries_out[i].reserved = 0; + } + previous = read.blocker; + have_previous = true; + current_index = read.next_index; + } + return current_index == PCM_X_INVALID_SLOT_INDEX; +} + + +/* Caller already validated this chain while holding its owner domain lock. */ +static void +pcm_x_blocker_chain_state_write(const PcmXAllocatorView *view, Size head_index, uint32 count, + uint32 state) +{ + Size current_index = head_index; + uint32 i; + + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(view, current_index); + + current_index = slot->next_index; + pcm_x_slot_state_write(&slot->slot, state); + } +} + + +/* Capacity is proved before reserve so ordinary saturation mutates no slot. */ +static PcmXAllocatorResult +pcm_x_blocker_capacity_preflight_locked(Size needed) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + Size available; + Size current_index; + Size reservable = 0; + Size scanned = 0; + + if (header == NULL || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view)) + return PCM_X_ALLOC_INVALID; + if (!pcm_x_allocator_state_valid_locked(&view)) + return PCM_X_ALLOC_CORRUPT; + available = view.capacity - view.state->generation_exhausted - view.state->used; + current_index = view.state->free_head; + while (current_index != PCM_X_INVALID_SLOT_INDEX) { + PcmXSlotHeader *slot; + uint64 generation; + + if (scanned >= view.capacity) + return PCM_X_ALLOC_CORRUPT; + slot = pcm_x_allocator_slot(&view, current_index); + if (slot == NULL || pcm_x_slot_state_read(slot) != PCM_X_SLOT_FREE + || pcm_x_slot_flags_read(slot) != 0 || !pcm_x_slot_generation_read(slot, &generation) + || slot->next_free == current_index) + return PCM_X_ALLOC_CORRUPT; + if (slot->next_free != PCM_X_INVALID_SLOT_INDEX + && pcm_x_allocator_slot(&view, slot->next_free) == NULL) + return PCM_X_ALLOC_CORRUPT; + if (generation != UINT64_MAX) + reservable++; + current_index = slot->next_free; + scanned++; + } + if (scanned != available) + return PCM_X_ALLOC_CORRUPT; + return reservable >= needed ? PCM_X_ALLOC_OK : PCM_X_ALLOC_NO_CAPACITY; +} + + +/* Caller owns allocator_lock EXCLUSIVE; validation precedes the first free. */ +static bool +pcm_x_blocker_chain_release_locked_exact(Size head_index, uint32 count, + const PcmXTicketHandle *owner_handle, + Size owner_slot_index, uint64 owner_slot_generation, + uint64 set_generation, uint32 expected_state) +{ + PcmXAllocatorView view; + Size current_index = head_index; + uint32 i; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view) + || !pcm_x_blocker_chain_validate(&view, head_index, count, owner_handle, owner_slot_index, + owner_slot_generation, set_generation, expected_state, + NULL, 0, NULL)) + return false; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current_index); + PcmXSlotRef ref; + Size next_index = slot->next_index; + + ref.slot_index = current_index; + if (!pcm_x_slot_generation_read(&slot->slot, &ref.slot_generation) + || pcm_x_allocator_release_locked(PCM_X_ALLOC_BLOCKER, ref, expected_state) + != PCM_X_ALLOC_OK) + return false; + current_index = next_index; + } + return current_index == PCM_X_INVALID_SLOT_INDEX; +} + + +static bool +pcm_x_blocker_chain_release_exact(Size head_index, uint32 count, + const PcmXTicketHandle *owner_handle, Size owner_slot_index, + uint64 owner_slot_generation, uint64 set_generation, + uint32 expected_state) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + bool released; + + if (header == NULL || !pcm_x_allocator_entry_unlocked(header)) + return false; + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + released = pcm_x_blocker_chain_release_locked_exact(head_index, count, owner_handle, + owner_slot_index, owner_slot_generation, + set_generation, expected_state); + LWLockRelease(&header->allocator_lock.lock); + return released; +} + + +static PcmXAllocatorResult +pcm_x_blocker_chain_reserve_locked(const PcmXTicketRef *ref, PcmXSlotRef owner_ref, + uint64 set_generation, const ClusterLmdVertex *blockers, + Size nblockers, Size unique_count, Size *head_index_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorResult result; + PcmXBlockerSlot *previous_slot = NULL; + const ClusterLmdVertex *previous = NULL; + Size head_index = PCM_X_INVALID_SLOT_INDEX; + Size reserved_count = 0; + + if (head_index_out != NULL) + *head_index_out = PCM_X_INVALID_SLOT_INDEX; + if (header == NULL || ref == NULL || head_index_out == NULL + || !LWLockHeldByMeInMode(&header->allocator_lock.lock, LW_EXCLUSIVE)) + return PCM_X_ALLOC_INVALID; + result = pcm_x_blocker_capacity_preflight_locked(unique_count); + if (result != PCM_X_ALLOC_OK) + return result; + + while (reserved_count < unique_count) { + const ClusterLmdVertex *candidate + = pcm_x_blocker_next_canonical(blockers, nblockers, previous); + PcmXSlotHeader *raw_slot; + PcmXBlockerSlot *slot; + PcmXSlotRef slot_ref; + + if (candidate == NULL) { + result = PCM_X_ALLOC_CORRUPT; + break; + } + result = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_BLOCKER, &slot_ref, &raw_slot); + if (result != PCM_X_ALLOC_OK) { + /* Full was ruled out before the first mutation. */ + if (result == PCM_X_ALLOC_NO_CAPACITY) + result = PCM_X_ALLOC_CORRUPT; + break; + } + slot = (PcmXBlockerSlot *)raw_slot; + slot->handle = ref->handle; + slot->blocker = *candidate; + slot->owner_slot_generation = owner_ref.slot_generation; + slot->set_generation = set_generation; + slot->next_index = PCM_X_INVALID_SLOT_INDEX; + slot->owner_slot_index = owner_ref.slot_index; + slot->chunk_no = (uint32)reserved_count; + slot->direction = PCM_X_BLOCKER_DIRECTION_MASTER; + if (previous_slot == NULL) + head_index = slot_ref.slot_index; + else + previous_slot->next_index = slot_ref.slot_index; + previous_slot = slot; + previous = candidate; + reserved_count++; + } + if (result != PCM_X_ALLOC_OK) { + if (!pcm_x_blocker_chain_release_locked_exact( + head_index, (uint32)reserved_count, &ref->handle, owner_ref.slot_index, + owner_ref.slot_generation, set_generation, PCM_XB_RESERVED_NONVISIBLE)) + result = PCM_X_ALLOC_CORRUPT; + return result; + } + *head_index_out = head_index; + return PCM_X_ALLOC_OK; +} + + +/* Inbound BLOCKER_SET staging is separate from the currently published set. + * The old graph and its exact blocker chain remain authoritative until COMMIT + * atomically swaps a complete canonical replacement into the ticket. */ +static bool +pcm_x_master_blocker_stage_is_clear(const PcmXMasterTicketSlot *ticket) +{ + return ticket != NULL && ticket->blocker_stage_head_index == PCM_X_INVALID_SLOT_INDEX + && ticket->blocker_stage_set_generation == 0 && ticket->blocker_stage_count == 0 + && ticket->blocker_stage_crc32c == 0 && ticket->blocker_stage_next_chunk == 0 + && ticket->blocker_stage_source_session == 0 && ticket->blocker_stage_source_node == -1; +} + + +static void +pcm_x_master_blocker_stage_clear(PcmXMasterTicketSlot *ticket) +{ + if (ticket == NULL) + return; + ticket->blocker_stage_head_index = PCM_X_INVALID_SLOT_INDEX; + ticket->blocker_stage_set_generation = 0; + ticket->blocker_stage_count = 0; + ticket->blocker_stage_crc32c = 0; + ticket->blocker_stage_next_chunk = 0; + ticket->blocker_stage_source_session = 0; + ticket->blocker_stage_source_node = -1; +} + + +static bool +pcm_x_master_blocker_stage_metadata_valid(const PcmXMasterTicketSlot *ticket) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + if (header == NULL || ticket == NULL) + return false; + if (ticket->blocker_stage_set_generation == 0) + return pcm_x_master_blocker_stage_is_clear(ticket); + return ticket->blocker_stage_set_generation != UINT64_MAX + && ticket->blocker_stage_count <= header->layout.max_wait_edges + && ticket->blocker_stage_next_chunk <= ticket->blocker_stage_count + && ticket->blocker_stage_source_session != 0 && ticket->blocker_stage_source_node >= 0 + && ticket->blocker_stage_source_node < PCM_X_PROTOCOL_NODE_LIMIT + && ((ticket->blocker_stage_count == 0 + && ticket->blocker_stage_head_index == PCM_X_INVALID_SLOT_INDEX) + || (ticket->blocker_stage_count != 0 + && ticket->blocker_stage_head_index != PCM_X_INVALID_SLOT_INDEX)); +} + + +static bool +pcm_x_master_blocker_published_source_valid(const PcmXMasterTicketSlot *ticket) +{ + if (ticket == NULL) + return false; + if (ticket->blocker_set_generation == 0) + return ticket->blocker_set_source_node == -1 && ticket->blocker_set_source_session == 0; + return (ticket->blocker_set_source_node == -1 && ticket->blocker_set_source_session == 0) + || (ticket->blocker_set_source_node >= 0 + && ticket->blocker_set_source_node < PCM_X_PROTOCOL_NODE_LIMIT + && ticket->blocker_set_source_session != 0); +} + + +static bool +pcm_x_master_blocker_published_source_exact(const PcmXMasterTicketSlot *ticket, int32 source_node, + uint64 source_session) +{ + return ticket != NULL && ticket->blocker_set_generation != 0 + && ticket->blocker_set_source_node == source_node + && ticket->blocker_set_source_session == source_session; +} + + +static bool +pcm_x_master_blocker_ack_replay_exact(const PcmXMasterTicketSlot *ticket, int32 source_node, + uint64 source_session) +{ + const PcmXReliableLegState *leg; + + if (ticket == NULL || !pcm_x_master_blocker_stage_is_clear(ticket) + || ticket->graph_generation == 0 + || !pcm_x_master_blocker_published_source_exact(ticket, source_node, source_session)) + return false; + leg = &ticket->reliable; + return leg->retry_deadline_ms == 0 && leg->expected_responder_session == 0 + && leg->retry_count == 0 && leg->expected_responder_node == 0 && leg->pending_opcode == 0 + && leg->last_response_opcode == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT && leg->phase == 0 + && leg->flags == 0 && leg->reserved == 0; +} + + +static bool +pcm_x_master_probe_source_exact(const PcmXMasterTicketSlot *ticket, int32 source_node, + uint64 source_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + const PcmXPeerFrontier *frontier; + + if (header == NULL || ticket == NULL || source_node < 0 + || source_node >= PCM_X_PROTOCOL_NODE_LIMIT || source_session == 0) + return false; + frontier = &header->peer_frontiers[source_node]; + return ticket->reliable.pending_opcode == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + && ticket->reliable.phase == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + && ticket->reliable.expected_responder_node == source_node + && ticket->reliable.expected_responder_session == source_session + && frontier->cluster_epoch == ticket->ref.identity.cluster_epoch + && frontier->sender_session_incarnation == source_session; +} + + +static bool +pcm_x_blocker_header_valid(const PcmXBlockerSetHeaderPayload *header_payload) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + return header != NULL && header_payload != NULL && header_payload->set_generation != 0 + && header_payload->set_generation != UINT64_MAX + && header_payload->nblockers <= header->layout.max_wait_edges + && header_payload->ref.grant_generation == 0 + && pcm_x_wait_identity_valid(&header_payload->ref.identity); +} + + +static bool +pcm_x_blocker_edge_locator_equal(const PcmXMasterTicketSlot *ticket, + const PcmXBlockerChunkPayload *edge) +{ + return ticket != NULL && edge != NULL && BufferTagsEqual(&ticket->ref.identity.tag, &edge->tag) + && ticket->ref.identity.node_id == edge->requester_node + && ticket->ref.identity.procno == edge->requester_procno + && ticket->ref.identity.cluster_epoch == edge->cluster_epoch + && ticket->ref.identity.request_id == edge->request_id + && pcm_x_ticket_handle_equal(&ticket->ref.handle, &edge->handle) + && edge->grant_generation == 0; +} + + +static bool +pcm_x_blocker_stage_slot_exact(const PcmXAllocatorView *view, Size slot_index, + const PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref, + uint64 set_generation, uint32 chunk_no, uint32 expected_state, + PcmXBlockerSlot **slot_out, PcmXSlotRef *slot_ref_out) +{ + PcmXBlockerSlot *slot; + uint64 slot_generation; + + if (view == NULL || ticket == NULL || slot_out == NULL || slot_ref_out == NULL) + return false; + slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(view, slot_index); + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &slot_generation) + || slot_generation == 0 || pcm_x_slot_state_read(&slot->slot) != expected_state + || !pcm_x_ticket_handle_equal(&slot->handle, &ticket->ref.handle) + || slot->owner_slot_generation != ticket_ref.slot_generation + || slot->owner_slot_index != ticket_ref.slot_index || slot->set_generation != set_generation + || slot->chunk_no != chunk_no || slot->direction != PCM_X_BLOCKER_DIRECTION_MASTER + || (slot->next_index != PCM_X_INVALID_SLOT_INDEX && slot->next_index >= view->capacity)) + return false; + slot_ref_out->slot_index = slot_index; + slot_ref_out->slot_generation = slot_generation; + *slot_out = slot; + return true; +} + + +/* Resolve one exact staging chunk by its bounded owner chain, never by a pool scan. */ +static bool +pcm_x_blocker_stage_nth_exact(const PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref, + uint32 target_chunk, PcmXBlockerSlot **slot_out, + PcmXSlotRef *slot_ref_out) +{ + PcmXAllocatorView view; + PcmXBlockerSlot *slot = NULL; + PcmXSlotRef slot_ref; + Size current; + uint32 i; + + if (ticket == NULL || target_chunk >= ticket->blocker_stage_count + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view)) + return false; + current = ticket->blocker_stage_head_index; + for (i = 0; i <= target_chunk; i++) { + uint32 expected_state + = i < ticket->blocker_stage_next_chunk ? PCM_XB_LIVE : PCM_XB_RESERVED_NONVISIBLE; + + if (!pcm_x_blocker_stage_slot_exact(&view, current, ticket, ticket_ref, + ticket->blocker_stage_set_generation, i, expected_state, + &slot, &slot_ref)) + return false; + if (i != target_chunk && slot->next_index == PCM_X_INVALID_SLOT_INDEX) + return false; + current = slot->next_index; + } + *slot_out = slot; + *slot_ref_out = slot_ref; + return true; +} + + +static bool +pcm_x_blocker_stage_chain_validate(const PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref, + PcmXMasterBlockerEntry *entries_out, pg_crc32c *crc_out) +{ + PcmXAllocatorView view; + ClusterLmdVertex previous; + pg_crc32c crc; + Size current; + uint32 i; + bool have_previous = false; + + if (ticket == NULL || ticket->blocker_stage_next_chunk != ticket->blocker_stage_count + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view)) + return false; + INIT_CRC32C(crc); + current = ticket->blocker_stage_head_index; + for (i = 0; i < ticket->blocker_stage_count; i++) { + PcmXBlockerSlot *slot; + PcmXSlotRef slot_ref; + + if (!pcm_x_blocker_stage_slot_exact(&view, current, ticket, ticket_ref, + ticket->blocker_stage_set_generation, i, PCM_XB_LIVE, + &slot, &slot_ref) + || !pcm_x_lmd_vertex_valid(&slot->blocker) + || (slot->blocker.node_id == ticket->ref.identity.node_id + && slot->blocker.procno == ticket->ref.identity.procno + && slot->blocker.cluster_epoch == ticket->ref.identity.cluster_epoch + && slot->blocker.request_id == ticket->ref.identity.request_id) + || (have_previous && pcm_x_lmd_vertex_identity_compare(&previous, &slot->blocker) >= 0)) + return false; + pcm_x_blocker_crc32c_update(&crc, &slot->blocker); + if (entries_out != NULL) { + entries_out[i].slot_ref = slot_ref; + entries_out[i].blocker = slot->blocker; + entries_out[i].chunk_no = i; + entries_out[i].reserved = 0; + } + previous = slot->blocker; + have_previous = true; + current = slot->next_index; + } + if (current != PCM_X_INVALID_SLOT_INDEX) + return false; + FIN_CRC32C(crc); + if (crc_out != NULL) + *crc_out = crc; + return true; +} + + +/* Caller owns allocator_lock EXCLUSIVE; every slot is still nonvisible. */ +static bool +pcm_x_blocker_stage_chain_release_reserved_locked(Size head_index, uint32 count, + const PcmXTicketHandle *handle, + PcmXSlotRef ticket_ref, uint64 set_generation) +{ + PcmXAllocatorView view; + Size current = head_index; + uint32 i; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view)) + return false; + if (count == 0) + return head_index == PCM_X_INVALID_SLOT_INDEX; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot; + PcmXSlotRef slot_ref; + Size next; + + if (handle == NULL || current == PCM_X_INVALID_SLOT_INDEX) + return false; + slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &slot_ref.slot_generation) + || pcm_x_slot_state_read(&slot->slot) != PCM_XB_RESERVED_NONVISIBLE + || !pcm_x_ticket_handle_equal(&slot->handle, handle) + || slot->owner_slot_index != ticket_ref.slot_index + || slot->owner_slot_generation != ticket_ref.slot_generation + || slot->set_generation != set_generation || slot->chunk_no != i + || slot->direction != PCM_X_BLOCKER_DIRECTION_MASTER) + return false; + next = slot->next_index; + slot_ref.slot_index = current; + if (pcm_x_allocator_release_locked(PCM_X_ALLOC_BLOCKER, slot_ref, + PCM_XB_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + return false; + current = next; + } + return current == PCM_X_INVALID_SLOT_INDEX; +} + + +/* Caller owns allocator_lock EXCLUSIVE. An aborted stage can contain a LIVE + * prefix and an unfilled RESERVED suffix; after the owner lock marks the + * whole chain DETACHING, validate every owner/link before releasing any slot. + * Unfilled suffix vertices are intentionally not interpreted. */ +static bool +pcm_x_blocker_stage_chain_release_detaching_locked(Size head_index, uint32 count, + const PcmXTicketHandle *handle, + PcmXSlotRef ticket_ref, uint64 set_generation) +{ + PcmXAllocatorView view; + Size current = head_index; + uint32 i; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view) || handle == NULL + || count > view.capacity) + return false; + if (count == 0) + return head_index == PCM_X_INVALID_SLOT_INDEX; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot; + uint64 slot_generation; + + if (current == PCM_X_INVALID_SLOT_INDEX) + return false; + slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &slot_generation) + || pcm_x_slot_state_read(&slot->slot) != PCM_XB_DETACHING + || !pcm_x_ticket_handle_equal(&slot->handle, handle) + || slot->owner_slot_index != ticket_ref.slot_index + || slot->owner_slot_generation != ticket_ref.slot_generation + || slot->set_generation != set_generation || slot->chunk_no != i + || slot->direction != PCM_X_BLOCKER_DIRECTION_MASTER + || (slot->next_index != PCM_X_INVALID_SLOT_INDEX && slot->next_index >= view.capacity)) + return false; + current = slot->next_index; + } + if (current != PCM_X_INVALID_SLOT_INDEX) + return false; + current = head_index; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + PcmXSlotRef slot_ref; + Size next = slot->next_index; + + slot_ref.slot_index = current; + if (!pcm_x_slot_generation_read(&slot->slot, &slot_ref.slot_generation) + || pcm_x_allocator_release_locked(PCM_X_ALLOC_BLOCKER, slot_ref, PCM_XB_DETACHING) + != PCM_X_ALLOC_OK) + return false; + current = next; + } + return current == PCM_X_INVALID_SLOT_INDEX; +} + + +static PcmXAllocatorResult +pcm_x_blocker_stage_chain_reserve_locked(const PcmXTicketRef *ref, PcmXSlotRef ticket_ref, + uint64 set_generation, uint32 count, Size *head_index_out) +{ + PcmXAllocatorResult result; + PcmXBlockerSlot *previous = NULL; + Size head_index = PCM_X_INVALID_SLOT_INDEX; + uint32 reserved = 0; + + if (ref == NULL || head_index_out == NULL) + return PCM_X_ALLOC_INVALID; + *head_index_out = PCM_X_INVALID_SLOT_INDEX; + result = pcm_x_blocker_capacity_preflight_locked(count); + if (result != PCM_X_ALLOC_OK) + return result; + while (reserved < count) { + PcmXSlotHeader *raw_slot; + PcmXBlockerSlot *slot; + PcmXSlotRef slot_ref; + + result = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_BLOCKER, &slot_ref, &raw_slot); + if (result != PCM_X_ALLOC_OK) { + if (result == PCM_X_ALLOC_NO_CAPACITY) + result = PCM_X_ALLOC_CORRUPT; + break; + } + slot = (PcmXBlockerSlot *)raw_slot; + slot->handle = ref->handle; + slot->owner_slot_generation = ticket_ref.slot_generation; + slot->set_generation = set_generation; + slot->next_index = PCM_X_INVALID_SLOT_INDEX; + slot->owner_slot_index = ticket_ref.slot_index; + slot->chunk_no = reserved; + slot->direction = PCM_X_BLOCKER_DIRECTION_MASTER; + if (previous == NULL) + head_index = slot_ref.slot_index; + else + previous->next_index = slot_ref.slot_index; + previous = slot; + reserved++; + } + if (result != PCM_X_ALLOC_OK) { + if (!pcm_x_blocker_stage_chain_release_reserved_locked(head_index, reserved, &ref->handle, + ticket_ref, set_generation)) + result = PCM_X_ALLOC_CORRUPT; + return result; + } + *head_index_out = head_index; + return PCM_X_ALLOC_OK; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_probe_arm_exact(const PcmXTicketRef *ref, int32 responder_node, + uint64 responder_session, PcmXPhasePayload *probe_out, + PcmXMasterProbeToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXPeerFrontier *frontier; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 next_sequence; + uint32 flags; + uint32 partition; + + if (probe_out != NULL) + memset(probe_out, 0, sizeof(*probe_out)); + if (token_out != NULL) + memset(token_out, 0, sizeof(*token_out)); + if (header == NULL || ref == NULL || probe_out == NULL || token_out == NULL + || ref->grant_generation != 0 || responder_node < 0 + || responder_node >= PCM_X_PROTOCOL_NODE_LIMIT || responder_session == 0 + || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_PROBE) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_pending_x_flags_valid((flags = pcm_x_slot_flags_read(&ticket->slot)))) + result = PCM_X_QUEUE_CORRUPT; + else if ((flags & PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING) != 0) + result = PCM_X_QUEUE_CORRUPT; + else { + frontier = &header->peer_frontiers[responder_node]; + if (frontier->cluster_epoch != ref->identity.cluster_epoch + || frontier->sender_session_incarnation != responder_session) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_master_terminal_leg_is_clear(&ticket->reliable)) { + if (pcm_x_master_probe_source_exact(ticket, responder_node, responder_session)) { + token_out->ref = ticket->ref; + token_out->state_sequence = ticket->reliable.state_sequence; + token_out->expected_responder_session = responder_session; + token_out->expected_responder_node = responder_node; + result = PCM_X_QUEUE_DUPLICATE; + } else + result = PCM_X_QUEUE_BUSY; + } else if ((flags & PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED) != 0) { + /* An exact replay of the already durable leg is allowed above. Once + * that leg clears, cancellation owns forward progress and no later + * holder may be added to the terminal participant set. */ + result = PCM_X_QUEUE_NOT_READY; + } else if (!cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &next_sequence)) + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + else { + ticket->reliable.state_sequence = next_sequence; + ticket->reliable.retry_deadline_ms = 0; + ticket->reliable.expected_responder_session = responder_session; + ticket->reliable.retry_count = 0; + ticket->reliable.expected_responder_node = responder_node; + ticket->reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK; + ticket->reliable.phase = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK; + ticket->reliable.flags = 0; + ticket->involved_nodes_bitmap |= UINT32_C(1) << responder_node; + pg_write_barrier(); + token_out->ref = ticket->ref; + token_out->state_sequence = next_sequence; + token_out->expected_responder_session = responder_session; + token_out->expected_responder_node = responder_node; + result = PCM_X_QUEUE_OK; + } + } + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + probe_out->ref = ticket->ref; + /* The complete [88,96) generation remains zero by construction. */ + Assert(probe_out->reason == 0 && probe_out->phase == 0 && probe_out->flags == 0); + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Caller owns the exact ticket's master-domain lock. */ +static PcmXQueueResult +pcm_x_master_blocker_stage_begin_precheck_locked(const PcmXMasterTicketSlot *ticket, + PcmXSlotRef ticket_ref, + const PcmXBlockerSetHeaderPayload *begin, + int32 source_node, uint64 source_session) +{ + PcmXAllocatorView blocker_view; + uint64 current_generation; + uint32 state; + + if (ticket == NULL || begin == NULL || !pcm_x_master_blocker_stage_metadata_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view)) + return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_ticket_admission_ref_equal(&ticket->ref, &begin->ref)) + return PCM_X_QUEUE_STALE; + current_generation = ticket->blocker_set_generation; + if ((current_generation == 0 + && (ticket->blocker_head_index != PCM_X_INVALID_SLOT_INDEX || ticket->blocker_count != 0 + || ticket->blocker_set_crc32c != 0 + || !pcm_x_master_blocker_published_source_valid(ticket))) + || (current_generation != 0 + && (!pcm_x_blocker_chain_validate(&blocker_view, ticket->blocker_head_index, + ticket->blocker_count, &ticket->ref.handle, + ticket_ref.slot_index, ticket_ref.slot_generation, + current_generation, PCM_XB_LIVE, NULL, 0, NULL) + || !pcm_x_master_blocker_published_source_valid(ticket)))) + return PCM_X_QUEUE_CORRUPT; + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_ACTIVE_PROBE) { + if (state != PCM_XT_ACTIVE_TRANSFER && state != PCM_XT_COMPLETE && state != PCM_XT_CANCELLED + && state != PCM_XT_RETIRE_CREDIT) + return PCM_X_QUEUE_BAD_STATE; + if (!pcm_x_master_blocker_stage_is_clear(ticket)) + return PCM_X_QUEUE_CORRUPT; + return begin->set_generation == current_generation + && begin->nblockers == ticket->blocker_count + && begin->set_crc32c == ticket->blocker_set_crc32c + && pcm_x_master_blocker_published_source_exact(ticket, source_node, + source_session) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + } + if (!pcm_x_master_probe_source_exact(ticket, source_node, source_session) + && !pcm_x_master_blocker_ack_replay_exact(ticket, source_node, source_session)) + return PCM_X_QUEUE_STALE; + if (ticket->blocker_stage_set_generation != 0) { + if (begin->set_generation == ticket->blocker_stage_set_generation + && begin->nblockers == ticket->blocker_stage_count + && begin->set_crc32c == ticket->blocker_stage_crc32c + && source_node == ticket->blocker_stage_source_node + && source_session == ticket->blocker_stage_source_session) + return PCM_X_QUEUE_DUPLICATE; + return begin->set_generation <= ticket->blocker_stage_set_generation ? PCM_X_QUEUE_STALE + : PCM_X_QUEUE_BUSY; + } + if (begin->set_generation == current_generation) { + return begin->nblockers == ticket->blocker_count + && begin->set_crc32c == ticket->blocker_set_crc32c + && pcm_x_master_blocker_published_source_exact(ticket, source_node, + source_session) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + } + if (begin->set_generation < current_generation) + return PCM_X_QUEUE_STALE; + if (current_generation == UINT64_MAX || begin->set_generation == UINT64_MAX) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + if (current_generation != 0 && ticket->graph_generation == 0) + return PCM_X_QUEUE_NOT_READY; + return begin->set_generation == current_generation + 1 ? PCM_X_QUEUE_OK : PCM_X_QUEUE_NOT_READY; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_stage_begin_exact(const PcmXBlockerSetHeaderPayload *begin, + int32 authenticated_source_node, + uint64 authenticated_source_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + PcmXAllocatorResult allocator_result; + Size staged_head = PCM_X_INVALID_SLOT_INDEX; + uint32 partition; + bool release_staged = false; + + if (!pcm_x_blocker_header_valid(begin) || authenticated_source_node < 0 + || authenticated_source_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_source_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&begin->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&begin->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &begin->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = ticket == NULL ? PCM_X_QUEUE_STALE : PCM_X_QUEUE_NOT_READY; + else + result = pcm_x_master_blocker_stage_begin_precheck_locked( + ticket, ticket_ref, begin, authenticated_source_node, authenticated_source_session); + LWLockRelease(&header->master_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; + } + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + allocator_result = pcm_x_blocker_stage_chain_reserve_locked( + &begin->ref, ticket_ref, begin->set_generation, begin->nblockers, &staged_head); + LWLockRelease(&header->allocator_lock.lock); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &begin->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = ticket == NULL ? PCM_X_QUEUE_STALE : PCM_X_QUEUE_NOT_READY; + else + result = pcm_x_master_blocker_stage_begin_precheck_locked( + ticket, ticket_ref, begin, authenticated_source_node, authenticated_source_session); + if (result == PCM_X_QUEUE_OK) { + ticket->blocker_stage_head_index = staged_head; + ticket->blocker_stage_set_generation = begin->set_generation; + ticket->blocker_stage_count = begin->nblockers; + ticket->blocker_stage_crc32c = begin->set_crc32c; + ticket->blocker_stage_next_chunk = 0; + ticket->blocker_stage_source_session = authenticated_source_session; + ticket->blocker_stage_source_node = authenticated_source_node; + pg_write_barrier(); + } else + release_staged = true; + LWLockRelease(&header->master_locks[partition].lock); + + if (release_staged) { + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_blocker_stage_chain_release_reserved_locked(staged_head, begin->nblockers, + &begin->ref.handle, ticket_ref, + begin->set_generation)) + result = PCM_X_QUEUE_CORRUPT; + LWLockRelease(&header->allocator_lock.lock); + } + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_master_blocker_published_edge_replay(const PcmXMasterTicketSlot *ticket, + PcmXSlotRef ticket_ref, + const PcmXBlockerChunkPayload *edge, int32 source_node, + uint64 source_session) +{ + PcmXAllocatorView view; + PcmXBlockerRead read; + Size current; + uint32 i; + + if (ticket == NULL || edge == NULL || edge->set_generation != ticket->blocker_set_generation + || !pcm_x_master_blocker_published_source_exact(ticket, source_node, source_session) + || edge->chunk_no >= ticket->blocker_count) + return PCM_X_QUEUE_STALE; + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view) + || !pcm_x_blocker_chain_validate(&view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, + ticket_ref.slot_generation, ticket->blocker_set_generation, + PCM_XB_LIVE, NULL, 0, NULL)) + return PCM_X_QUEUE_CORRUPT; + current = ticket->blocker_head_index; + for (i = 0; i <= edge->chunk_no; i++) { + if (!pcm_x_blocker_read_exact(&view, current, &ticket->ref.handle, ticket_ref.slot_index, + ticket_ref.slot_generation, ticket->blocker_set_generation, i, + PCM_XB_LIVE, &read)) + return PCM_X_QUEUE_CORRUPT; + current = read.next_index; + } + return pcm_x_lmd_vertex_equal(&read.blocker, &edge->blocker) ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_stage_edge_exact(const PcmXBlockerChunkPayload *edge, + int32 authenticated_source_node, + uint64 authenticated_source_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXBlockerSlot *slot; + PcmXBlockerSlot *previous; + PcmXSlotRef found; + PcmXSlotRef slot_ref; + PcmXSlotRef previous_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 partition; + uint32 state; + + if (header == NULL || edge == NULL || authenticated_source_node < 0 + || authenticated_source_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_source_session == 0 || edge->set_generation == 0 + || edge->set_generation == UINT64_MAX || edge->grant_generation != 0 + || edge->requester_node < 0 || edge->requester_node >= PCM_X_PROTOCOL_NODE_LIMIT + || edge->requester_procno >= header->layout.process_capacity || edge->request_id == 0 + || edge->handle.ticket_id == 0 || edge->handle.queue_generation == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, &edge->handle, &found); + if (directory_result == PCM_X_DIRECTORY_OK) + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + found); + else + ticket = NULL; + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + if (ticket == NULL) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&edge->tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, found, &edge->tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_blocker_edge_locator_equal(ticket, edge)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_master_blocker_stage_metadata_valid(ticket)) + result = PCM_X_QUEUE_CORRUPT; + else { + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_ACTIVE_PROBE) { + if (state != PCM_XT_ACTIVE_TRANSFER && state != PCM_XT_COMPLETE + && state != PCM_XT_CANCELLED && state != PCM_XT_RETIRE_CREDIT) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_blocker_stage_is_clear(ticket)) + result = PCM_X_QUEUE_CORRUPT; + else + result = pcm_x_master_blocker_published_edge_replay( + ticket, found, edge, authenticated_source_node, authenticated_source_session); + } else if (!pcm_x_master_probe_source_exact(ticket, authenticated_source_node, + authenticated_source_session) + && !pcm_x_master_blocker_ack_replay_exact(ticket, authenticated_source_node, + authenticated_source_session)) + result = PCM_X_QUEUE_STALE; + else if (ticket->blocker_stage_set_generation == 0) + result = pcm_x_master_blocker_published_edge_replay( + ticket, found, edge, authenticated_source_node, authenticated_source_session); + else if (edge->set_generation != ticket->blocker_stage_set_generation + || authenticated_source_node != ticket->blocker_stage_source_node + || authenticated_source_session != ticket->blocker_stage_source_session) + result = PCM_X_QUEUE_STALE; + else if (edge->chunk_no >= ticket->blocker_stage_count) + result = PCM_X_QUEUE_STALE; + else if (edge->chunk_no > ticket->blocker_stage_next_chunk) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_lmd_vertex_valid(&edge->blocker) + || edge->blocker.node_id != authenticated_source_node + || edge->blocker.cluster_epoch != ticket->ref.identity.cluster_epoch + || (edge->blocker.node_id == ticket->ref.identity.node_id + && edge->blocker.procno == ticket->ref.identity.procno + && edge->blocker.request_id == ticket->ref.identity.request_id)) + result = PCM_X_QUEUE_INVALID; + else if (!pcm_x_blocker_stage_nth_exact(ticket, found, edge->chunk_no, &slot, &slot_ref)) + result = PCM_X_QUEUE_CORRUPT; + else if (edge->chunk_no < ticket->blocker_stage_next_chunk) + result = pcm_x_lmd_vertex_equal(&slot->blocker, &edge->blocker) ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + else { + result = PCM_X_QUEUE_OK; + if (edge->chunk_no > 0) { + if (!pcm_x_blocker_stage_nth_exact(ticket, found, edge->chunk_no - 1, &previous, + &previous_ref)) + result = PCM_X_QUEUE_CORRUPT; + else if (pcm_x_lmd_vertex_identity_compare(&previous->blocker, &edge->blocker) >= 0) + result = PCM_X_QUEUE_INVALID; + } + if (result == PCM_X_QUEUE_OK) { + slot->blocker = edge->blocker; + pg_write_barrier(); + pcm_x_slot_state_write(&slot->slot, PCM_XB_LIVE); + ticket->blocker_stage_next_chunk++; + } + } + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_stage_commit_exact(const PcmXBlockerSetHeaderPayload *commit, + int32 authenticated_source_node, + uint64 authenticated_source_session, + PcmXMasterBlockerEntry *entries_out, + Size entry_capacity, + PcmXMasterBlockerSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView blocker_view; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + PcmXTicketHandle owner_handle; + pg_crc32c actual_crc; + Size old_head = PCM_X_INVALID_SLOT_INDEX; + uint64 old_generation = 0; + uint32 old_count = 0; + uint32 partition; + uint32 state; + bool release_old = false; + + pcm_x_master_blocker_snapshot_clear(snapshot_out); + if (!pcm_x_blocker_header_valid(commit) || snapshot_out == NULL || authenticated_source_node < 0 + || authenticated_source_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_source_session == 0 || (entry_capacity > 0 && entries_out == NULL)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&commit->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&commit->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &commit->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, &commit->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_master_blocker_stage_metadata_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view)) + result = PCM_X_QUEUE_CORRUPT; + else { + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_ACTIVE_PROBE) { + if (state != PCM_XT_ACTIVE_TRANSFER && state != PCM_XT_COMPLETE + && state != PCM_XT_CANCELLED && state != PCM_XT_RETIRE_CREDIT) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_blocker_stage_is_clear(ticket) + || !pcm_x_master_blocker_published_source_valid(ticket)) + result = PCM_X_QUEUE_CORRUPT; + else if (commit->set_generation != ticket->blocker_set_generation + || commit->nblockers != ticket->blocker_count + || commit->set_crc32c != ticket->blocker_set_crc32c + || !pcm_x_master_blocker_published_source_exact( + ticket, authenticated_source_node, authenticated_source_session)) + result = PCM_X_QUEUE_STALE; + else if (ticket->graph_generation == 0) + result = PCM_X_QUEUE_CORRUPT; + else if (ticket->blocker_count > entry_capacity) + result = PCM_X_QUEUE_NO_CAPACITY; + else if (!pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, entries_out)) + result = PCM_X_QUEUE_CORRUPT; + else { + snapshot_out->ref = ticket->ref; + snapshot_out->set_generation = ticket->blocker_set_generation; + snapshot_out->graph_generation = ticket->graph_generation; + snapshot_out->blocker_count = ticket->blocker_count; + snapshot_out->set_crc32c = ticket->blocker_set_crc32c; + result = PCM_X_QUEUE_DUPLICATE; + } + } else if (!pcm_x_master_probe_source_exact(ticket, authenticated_source_node, + authenticated_source_session) + && !pcm_x_master_blocker_ack_replay_exact(ticket, authenticated_source_node, + authenticated_source_session)) + result = PCM_X_QUEUE_STALE; + else if (ticket->blocker_stage_set_generation == 0) { + if (commit->set_generation != ticket->blocker_set_generation + || commit->nblockers != ticket->blocker_count + || commit->set_crc32c != ticket->blocker_set_crc32c + || !pcm_x_master_blocker_published_source_exact(ticket, authenticated_source_node, + authenticated_source_session)) + result = PCM_X_QUEUE_STALE; + else if (ticket->blocker_count > entry_capacity) + result = PCM_X_QUEUE_NO_CAPACITY; + else if (!pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, entries_out)) + result = PCM_X_QUEUE_CORRUPT; + else { + snapshot_out->ref = ticket->ref; + snapshot_out->set_generation = ticket->blocker_set_generation; + snapshot_out->graph_generation = ticket->graph_generation; + snapshot_out->blocker_count = ticket->blocker_count; + snapshot_out->set_crc32c = ticket->blocker_set_crc32c; + result = PCM_X_QUEUE_DUPLICATE; + } + } else if (commit->set_generation != ticket->blocker_stage_set_generation + || commit->nblockers != ticket->blocker_stage_count + || commit->set_crc32c != ticket->blocker_stage_crc32c + || authenticated_source_node != ticket->blocker_stage_source_node + || authenticated_source_session != ticket->blocker_stage_source_session) + result = PCM_X_QUEUE_STALE; + else if (ticket->blocker_stage_next_chunk != ticket->blocker_stage_count) + result = PCM_X_QUEUE_NOT_READY; + else if (ticket->blocker_stage_count > entry_capacity) + result = PCM_X_QUEUE_NO_CAPACITY; + else if (!pcm_x_blocker_stage_chain_validate(ticket, ticket_ref, entries_out, &actual_crc)) + result = PCM_X_QUEUE_CORRUPT; + else if ((uint32)actual_crc != ticket->blocker_stage_crc32c) + result = PCM_X_QUEUE_STALE; + else if ((ticket->blocker_set_generation == 0 + && (ticket->blocker_head_index != PCM_X_INVALID_SLOT_INDEX + || ticket->blocker_count != 0 || ticket->blocker_set_crc32c != 0 + || !pcm_x_master_blocker_published_source_valid(ticket))) + || (ticket->blocker_set_generation != 0 + && (!pcm_x_master_blocker_published_source_valid(ticket) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)))) + result = PCM_X_QUEUE_CORRUPT; + else { + owner_handle = ticket->ref.handle; + old_head = ticket->blocker_head_index; + old_count = ticket->blocker_count; + old_generation = ticket->blocker_set_generation; + pcm_x_blocker_chain_state_write(&blocker_view, old_head, old_count, PCM_XB_DETACHING); + ticket->blocker_head_index = ticket->blocker_stage_head_index; + ticket->blocker_set_generation = ticket->blocker_stage_set_generation; + ticket->blocker_count = ticket->blocker_stage_count; + ticket->blocker_set_crc32c = ticket->blocker_stage_crc32c; + ticket->blocker_set_source_session = ticket->blocker_stage_source_session; + ticket->blocker_set_source_node = ticket->blocker_stage_source_node; + ticket->graph_generation = 0; + pcm_x_master_blocker_stage_clear(ticket); + pg_write_barrier(); + snapshot_out->ref = ticket->ref; + snapshot_out->set_generation = ticket->blocker_set_generation; + snapshot_out->graph_generation = 0; + snapshot_out->blocker_count = ticket->blocker_count; + snapshot_out->set_crc32c = ticket->blocker_set_crc32c; + release_old = true; + result = PCM_X_QUEUE_OK; + } + } + LWLockRelease(&header->master_locks[partition].lock); + + if (release_old + && !pcm_x_blocker_chain_release_exact(old_head, old_count, &owner_handle, + ticket_ref.slot_index, ticket_ref.slot_generation, + old_generation, PCM_XB_DETACHING)) + result = PCM_X_QUEUE_CORRUPT; + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_stage_abort_exact(const PcmXTicketRef *ref, uint64 set_generation, + int32 authenticated_source_node, + uint64 authenticated_source_session, + uint64 expected_state_sequence) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXBlockerSlot *slot = NULL; + PcmXSlotRef slot_ref; + PcmXSlotRef ticket_ref; + PcmXTicketHandle owner_handle; + PcmXQueueResult result; + Size staged_head = PCM_X_INVALID_SLOT_INDEX; + uint64 staged_generation = 0; + uint64 next_sequence; + uint32 staged_count = 0; + uint32 partition; + uint32 i; + bool release_stage = false; + + if (header == NULL || ref == NULL || set_generation == 0 || set_generation == UINT64_MAX + || expected_state_sequence == 0 || ref->grant_generation != 0 + || authenticated_source_node < 0 || authenticated_source_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_source_session == 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_PROBE) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_blocker_stage_metadata_valid(ticket)) + result = PCM_X_QUEUE_CORRUPT; + else if (ticket->reliable.state_sequence != expected_state_sequence + || !pcm_x_master_probe_source_exact(ticket, authenticated_source_node, + authenticated_source_session) + || ticket->blocker_stage_set_generation != set_generation + || ticket->blocker_stage_source_node != authenticated_source_node + || ticket->blocker_stage_source_session != authenticated_source_session) + result = PCM_X_QUEUE_STALE; + else if (!cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &next_sequence)) + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + else { + for (i = 0; i < ticket->blocker_stage_count; i++) { + if (!pcm_x_blocker_stage_nth_exact(ticket, ticket_ref, i, &slot, &slot_ref)) { + result = PCM_X_QUEUE_CORRUPT; + break; + } + } + if (result == PCM_X_QUEUE_OK && ticket->blocker_stage_count != 0 + && (slot == NULL || slot->next_index != PCM_X_INVALID_SLOT_INDEX)) + result = PCM_X_QUEUE_CORRUPT; + if (result == PCM_X_QUEUE_OK) { + staged_head = ticket->blocker_stage_head_index; + staged_count = ticket->blocker_stage_count; + staged_generation = ticket->blocker_stage_set_generation; + owner_handle = ticket->ref.handle; + if (staged_count != 0) { + PcmXAllocatorView blocker_view; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view)) + result = PCM_X_QUEUE_CORRUPT; + else + pcm_x_blocker_chain_state_write(&blocker_view, staged_head, staged_count, + PCM_XB_DETACHING); + } + if (result == PCM_X_QUEUE_OK) { + pcm_x_master_blocker_stage_clear(ticket); + ticket->reliable.state_sequence = next_sequence; + pg_write_barrier(); + release_stage = true; + } + } + } + LWLockRelease(&header->master_locks[partition].lock); + + if (release_stage) { + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_blocker_stage_chain_release_detaching_locked( + staged_head, staged_count, &owner_handle, ticket_ref, staged_generation)) + result = PCM_X_QUEUE_CORRUPT; + LWLockRelease(&header->allocator_lock.lock); + } + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Caller owns the exact tag's master-domain lock. */ +static PcmXQueueResult +pcm_x_master_blocker_replace_precheck_locked(const PcmXMasterTicketSlot *ticket, + PcmXSlotRef ticket_ref, uint64 requested_generation, + uint32 requested_crc32c) +{ + PcmXAllocatorView blocker_view; + uint64 current_generation; + uint32 state; + + if (ticket == NULL || !pcm_x_master_blocker_stage_metadata_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view)) + return PCM_X_QUEUE_CORRUPT; + if (ticket->blocker_stage_set_generation != 0) + return PCM_X_QUEUE_BUSY; + current_generation = ticket->blocker_set_generation; + if ((current_generation == 0 + && (ticket->blocker_head_index != PCM_X_INVALID_SLOT_INDEX || ticket->blocker_count != 0 + || ticket->blocker_set_crc32c != 0 + || !pcm_x_master_blocker_published_source_valid(ticket))) + || (current_generation != 0 + && (!pcm_x_blocker_chain_validate(&blocker_view, ticket->blocker_head_index, + ticket->blocker_count, &ticket->ref.handle, + ticket_ref.slot_index, ticket_ref.slot_generation, + current_generation, PCM_XB_LIVE, NULL, 0, NULL) + || !pcm_x_master_blocker_published_source_valid(ticket)))) + return PCM_X_QUEUE_CORRUPT; + if (requested_generation == UINT64_MAX || current_generation == UINT64_MAX) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + if (requested_generation < current_generation) + return PCM_X_QUEUE_STALE; + if (requested_generation == current_generation) + return requested_crc32c == ticket->blocker_set_crc32c ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_CORRUPT; + if (requested_generation != current_generation + 1) + return PCM_X_QUEUE_NOT_READY; + state = pcm_x_slot_state_read(&ticket->slot); + return state == PCM_XT_ACTIVE_PROBE ? PCM_X_QUEUE_OK : PCM_X_QUEUE_BAD_STATE; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_set_replace_exact(const PcmXTicketRef *ref, uint64 set_generation, + const ClusterLmdVertex *blockers, Size nblockers, + uint32 set_crc32c) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView blocker_view; + PcmXSlotRef ticket_ref; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + PcmXAllocatorResult allocator_result; + PcmXTicketHandle owner_handle; + Size unique_count; + Size new_head_index = PCM_X_INVALID_SLOT_INDEX; + Size old_head_index = PCM_X_INVALID_SLOT_INDEX; + uint64 old_set_generation = 0; + uint32 old_count = 0; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || ref == NULL || set_generation == 0 || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + result = pcm_x_blocker_input_validate(ref, blockers, nblockers, &unique_count); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + if (set_generation == UINT64_MAX) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + } + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref) + || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else + result = pcm_x_master_blocker_replace_precheck_locked(ticket, ticket_ref, set_generation, + set_crc32c); + LWLockRelease(&header->master_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; + } + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + allocator_result = pcm_x_blocker_chain_reserve_locked(ref, ticket_ref, set_generation, blockers, + nblockers, unique_count, &new_head_index); + LWLockRelease(&header->allocator_lock.lock); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref) + || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else + result = pcm_x_master_blocker_replace_precheck_locked(ticket, ticket_ref, set_generation, + set_crc32c); + if (result == PCM_X_QUEUE_OK) { + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate(&blocker_view, new_head_index, (uint32)unique_count, + &ref->handle, ticket_ref.slot_index, + ticket_ref.slot_generation, set_generation, + PCM_XB_RESERVED_NONVISIBLE, NULL, 0, NULL)) + result = PCM_X_QUEUE_CORRUPT; + else { + owner_handle = ticket->ref.handle; + old_head_index = ticket->blocker_head_index; + old_count = ticket->blocker_count; + old_set_generation = ticket->blocker_set_generation; + pcm_x_blocker_chain_state_write(&blocker_view, old_head_index, old_count, + PCM_XB_DETACHING); + pcm_x_blocker_chain_state_write(&blocker_view, new_head_index, (uint32)unique_count, + PCM_XB_LIVE); + /* A newly published blocker set has no matching WFG commit yet. */ + ticket->graph_generation = 0; + pg_write_barrier(); + ticket->blocker_head_index = new_head_index; + ticket->blocker_count = (uint32)unique_count; + ticket->blocker_set_crc32c = set_crc32c; + ticket->blocker_set_generation = set_generation; + ticket->blocker_set_source_session = 0; + ticket->blocker_set_source_node = -1; + result = PCM_X_QUEUE_OK; + } + } + LWLockRelease(&header->master_locks[partition].lock); + + if (result != PCM_X_QUEUE_OK) { + if (!pcm_x_blocker_chain_release_exact(new_head_index, (uint32)unique_count, &ref->handle, + ticket_ref.slot_index, ticket_ref.slot_generation, + set_generation, PCM_XB_RESERVED_NONVISIBLE)) + result = PCM_X_QUEUE_CORRUPT; + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED; + } else if (!pcm_x_blocker_chain_release_exact(old_head_index, old_count, &owner_handle, + ticket_ref.slot_index, ticket_ref.slot_generation, + old_set_generation, PCM_XB_DETACHING)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static void +pcm_x_master_blocker_snapshot_clear(PcmXMasterBlockerSnapshot *snapshot) +{ + if (snapshot == NULL) + return; + memset(snapshot, 0, sizeof(*snapshot)); +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_snapshot_exact(const PcmXTicketRef *ref, + PcmXMasterBlockerEntry *entries_out, + Size entry_capacity, + PcmXMasterBlockerSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView blocker_view; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + pcm_x_master_blocker_snapshot_clear(snapshot_out); + if (header == NULL || ref == NULL || snapshot_out == NULL + || (entry_capacity > 0 && entries_out == NULL) + || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (ticket->blocker_set_generation == 0) + result = PCM_X_QUEUE_NOT_READY; + else if (ticket->blocker_set_generation == UINT64_MAX + || !pcm_x_master_blocker_published_source_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)) + result = PCM_X_QUEUE_CORRUPT; + else if (ticket->blocker_count > entry_capacity) + result = PCM_X_QUEUE_NO_CAPACITY; + else if (!pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, entries_out)) + result = PCM_X_QUEUE_CORRUPT; + else { + snapshot_out->ref = ticket->ref; + snapshot_out->set_generation = ticket->blocker_set_generation; + snapshot_out->graph_generation = ticket->graph_generation; + snapshot_out->blocker_count = ticket->blocker_count; + snapshot_out->set_crc32c = ticket->blocker_set_crc32c; + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_snapshot_revalidate_exact(const PcmXMasterBlockerSnapshot *snapshot, + const PcmXMasterBlockerEntry *entries, + Size entry_count) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView blocker_view; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (header == NULL || snapshot == NULL || snapshot->set_generation == 0 + || snapshot->set_generation == UINT64_MAX || entry_count != snapshot->blocker_count + || (entry_count > 0 && entries == NULL) + || !pcm_x_wait_identity_valid(&snapshot->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&snapshot->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&snapshot->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &snapshot->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &snapshot->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (ticket->blocker_set_generation != snapshot->set_generation + || ticket->graph_generation != snapshot->graph_generation) + result = PCM_X_QUEUE_STALE; + else if (ticket->blocker_count != snapshot->blocker_count + || ticket->blocker_set_crc32c != snapshot->set_crc32c + || !pcm_x_master_blocker_published_source_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)) + result = PCM_X_QUEUE_CORRUPT; + else if (!pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, entries, entry_count, NULL)) + result = PCM_X_QUEUE_STALE; + else + result = PCM_X_QUEUE_OK; + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_graph_commit_exact(const PcmXMasterBlockerSnapshot *snapshot, + const PcmXMasterBlockerEntry *entries, + Size entry_count, uint64 graph_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXAllocatorView blocker_view; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (header == NULL || snapshot == NULL || snapshot->set_generation == 0 + || snapshot->set_generation == UINT64_MAX || graph_generation == 0 + || snapshot->ref.grant_generation != 0 || entry_count != snapshot->blocker_count + || (entry_count > 0 && entries == NULL) + || !pcm_x_wait_identity_valid(&snapshot->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&snapshot->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + /* + * The caller performs the LMD batch replace before entering here. This + * lock never nests the graph lock: it only revalidates the immutable copy + * token and publishes the graph generation against that exact blocker set. + */ + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&snapshot->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &snapshot->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &snapshot->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (ticket->blocker_set_generation != snapshot->set_generation) + result = PCM_X_QUEUE_STALE; + else if (ticket->blocker_count != snapshot->blocker_count + || ticket->blocker_set_crc32c != snapshot->set_crc32c + || !pcm_x_master_blocker_published_source_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)) + result = PCM_X_QUEUE_CORRUPT; + else if (!pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, entries, entry_count, NULL)) + result = PCM_X_QUEUE_STALE; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_PROBE) + result = PCM_X_QUEUE_BAD_STATE; + else if (ticket->graph_generation == graph_generation) + result = PCM_X_QUEUE_DUPLICATE; + else if (ticket->graph_generation != 0 && graph_generation < ticket->graph_generation) + result = PCM_X_QUEUE_STALE; + else { + ticket->graph_generation = graph_generation; + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +pcm_x_master_unlink_ticket_locked(PcmXMasterTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXMasterTicketSlot *ticket, PcmXSlotRef ticket_ref) +{ + PcmXAllocatorView ticket_view; + PcmXMasterTicketSlot *next = NULL; + PcmXMasterTicketSlot *previous = NULL; + bool active; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &ticket_view)) + return false; + if (ticket->prev_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->head_index != ticket_ref.slot_index) + return false; + } else { + if (ticket->prev_index == ticket_ref.slot_index + || tag_slot->head_index == ticket_ref.slot_index) + return false; + previous = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&ticket_view, ticket->prev_index); + if (previous == NULL || previous->next_index != ticket_ref.slot_index + || previous->tag_slot_index != tag_ref.slot_index + || previous->tag_slot_generation != tag_ref.slot_generation) + return false; + } + if (ticket->next_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->tail_index != ticket_ref.slot_index) + return false; + } else { + if (ticket->next_index == ticket_ref.slot_index || ticket->next_index == ticket->prev_index + || tag_slot->tail_index == ticket_ref.slot_index) + return false; + next = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&ticket_view, ticket->next_index); + if (next == NULL || next->prev_index != ticket_ref.slot_index + || next->tag_slot_index != tag_ref.slot_index + || next->tag_slot_generation != tag_ref.slot_generation) + return false; + } + active = tag_slot->active_index == ticket_ref.slot_index; + if (active && tag_slot->active_slot_generation != ticket_ref.slot_generation) + return false; + + /* Every structural check above is read-only. Preserve the original FIFO + * as recovery evidence if any backlink or active locator is corrupt. */ + if (previous == NULL) + tag_slot->head_index = ticket->next_index; + else + previous->next_index = ticket->next_index; + if (next == NULL) + tag_slot->tail_index = ticket->prev_index; + else + next->prev_index = ticket->prev_index; + if (active) { + tag_slot->active_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_slot_generation = 0; + } + ticket->next_index = PCM_X_INVALID_SLOT_INDEX; + ticket->prev_index = PCM_X_INVALID_SLOT_INDEX; + return true; +} + + +static bool +pcm_x_transfer_leg_is_clear(const PcmXReliableLegState *leg) +{ + return leg != NULL && leg->retry_deadline_ms == 0 && leg->expected_responder_session == 0 + && leg->retry_count == 0 && leg->expected_responder_node == 0 && leg->pending_opcode == 0 + && leg->phase == 0 && leg->flags == 0 && leg->reserved == 0; +} + + +static bool +pcm_x_transfer_leg_is_pristine(const PcmXReliableLegState *leg) +{ + return pcm_x_transfer_leg_is_clear(leg) && leg->state_sequence == 0 + && leg->response_tombstone_mask == 0 && leg->last_responder_node == 0 + && leg->last_response_opcode == 0; +} + + +static bool +pcm_x_transfer_leg_exact(const PcmXReliableLegState *leg, uint16 opcode, int32 responder_node, + uint64 responder_session) +{ + return leg != NULL && opcode != 0 && responder_node >= 0 + && responder_node < PCM_X_PROTOCOL_NODE_LIMIT && responder_session != 0 + && leg->pending_opcode == opcode && leg->phase == opcode && leg->flags == 0 + && leg->reserved == 0 && leg->expected_responder_node == responder_node + && leg->expected_responder_session == responder_session; +} + + +static bool +pcm_x_transfer_peer_exact(const PcmXMasterTicketSlot *ticket, int32 responder_node, + uint64 responder_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + const PcmXPeerFrontier *frontier; + + if (header == NULL || ticket == NULL || responder_node < 0 + || responder_node >= PCM_X_PROTOCOL_NODE_LIMIT || responder_session == 0) + return false; + frontier = &header->peer_frontiers[responder_node]; + return frontier->cluster_epoch == ticket->ref.identity.cluster_epoch + && frontier->sender_session_incarnation == responder_session; +} + + +static void +pcm_x_transfer_leg_clear(PcmXReliableLegState *leg, uint16 response_opcode, + int32 authenticated_node) +{ + leg->retry_deadline_ms = 0; + leg->expected_responder_session = 0; + leg->retry_count = 0; + leg->last_responder_node = (uint32)authenticated_node; + leg->expected_responder_node = 0; + leg->pending_opcode = 0; + leg->last_response_opcode = response_opcode; + leg->phase = 0; + leg->flags = 0; + leg->reserved = 0; +} + + +static PcmXQueueResult +pcm_x_transfer_leg_arm_locked(PcmXMasterTicketSlot *ticket, uint16 opcode, int32 responder_node, + uint64 responder_session) +{ + uint64 next_sequence; + + if (ticket == NULL || opcode == 0 + || !pcm_x_transfer_peer_exact(ticket, responder_node, responder_session)) + return PCM_X_QUEUE_STALE; + if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) + return pcm_x_transfer_leg_exact(&ticket->reliable, opcode, responder_node, + responder_session) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_BUSY; + if (!cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &next_sequence)) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + ticket->reliable.state_sequence = next_sequence; + ticket->reliable.retry_deadline_ms = 0; + ticket->reliable.expected_responder_session = responder_session; + ticket->reliable.retry_count = 0; + ticket->reliable.expected_responder_node = responder_node; + ticket->reliable.pending_opcode = opcode; + ticket->reliable.phase = opcode; + ticket->reliable.flags = 0; + ticket->reliable.reserved = 0; + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +pcm_x_transfer_leg_advance_locked(PcmXMasterTicketSlot *ticket, uint16 expected_opcode, + int32 authenticated_node, uint64 authenticated_session, + uint16 response_opcode, uint16 next_opcode, int32 next_node, + uint64 next_session) +{ + uint64 next_sequence = 0; + + if (ticket == NULL + || !pcm_x_transfer_leg_exact(&ticket->reliable, expected_opcode, authenticated_node, + authenticated_session)) + return PCM_X_QUEUE_STALE; + if (!pcm_x_transfer_peer_exact(ticket, authenticated_node, authenticated_session)) + return PCM_X_QUEUE_STALE; + if (next_opcode != 0) { + if (!pcm_x_transfer_peer_exact(ticket, next_node, next_session)) + return PCM_X_QUEUE_STALE; + if (!cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &next_sequence)) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + } + pcm_x_transfer_leg_clear(&ticket->reliable, response_opcode, authenticated_node); + if (next_opcode != 0) { + ticket->reliable.state_sequence = next_sequence; + ticket->reliable.expected_responder_session = next_session; + ticket->reliable.expected_responder_node = next_node; + ticket->reliable.pending_opcode = next_opcode; + ticket->reliable.phase = next_opcode; + } + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +pcm_x_image_id_mint(uint64 *image_id_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXQueueResult result; + uint64 raw_sequence; + + if (image_id_out != NULL) + *image_id_out = 0; + if (header == NULL || image_id_out == NULL) + return PCM_X_QUEUE_INVALID; + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + raw_sequence = header->next_image_id; + if (raw_sequence == 0 || raw_sequence > PCM_X_IMAGE_ID_SEQ_MASK) + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + else if (!cluster_pcm_x_image_id_encode(cluster_node_id, raw_sequence, image_id_out)) + result = PCM_X_QUEUE_CORRUPT; + else { + header->next_image_id = raw_sequence + 1; + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->allocator_lock.lock); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_blocker_probe_complete_exact(const PcmXTicketRef *ref, uint64 set_generation, + int32 authenticated_source_node, + uint64 authenticated_source_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (header == NULL || ref == NULL || set_generation == 0 || set_generation == UINT64_MAX + || ref->grant_generation != 0 || authenticated_source_node < 0 + || authenticated_source_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_source_session == 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_PROBE) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_blocker_stage_metadata_valid(ticket) + || !pcm_x_master_blocker_stage_is_clear(ticket)) + result = PCM_X_QUEUE_CORRUPT; + else if (ticket->blocker_set_generation != set_generation || ticket->graph_generation == 0 + || !pcm_x_master_blocker_published_source_exact(ticket, authenticated_source_node, + authenticated_source_session)) + result = PCM_X_QUEUE_STALE; + else if (pcm_x_transfer_leg_is_clear(&ticket->reliable)) + result = ticket->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_master_probe_source_exact(ticket, authenticated_source_node, + authenticated_source_session)) + result = PCM_X_QUEUE_STALE; + else { + pcm_x_transfer_leg_clear(&ticket->reliable, PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_source_node); + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_begin_transfer_exact(const PcmXTicketRef *ref, uint64 graph_generation, + PcmXTicketRef *transfer_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 grant_generation = 0; + uint32 partition; + uint32 state; + + if (transfer_out != NULL) + memset(transfer_out, 0, sizeof(*transfer_out)); + if (header == NULL || ref == NULL || transfer_out == NULL || graph_generation == 0 + || ref->grant_generation != 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref) + || tag_slot->active_index != ticket_ref.slot_index + || tag_slot->active_slot_generation != ticket_ref.slot_generation) { + result = PCM_X_QUEUE_STALE; + goto begin_transfer_done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto begin_transfer_done; + } + state = pcm_x_slot_state_read(&ticket->slot); + result = pcm_x_master_pending_x_claim_required(ticket); + if (result != PCM_X_QUEUE_OK) + goto begin_transfer_done; + if (state == PCM_XT_ACTIVE_TRANSFER) { + if (ticket->graph_generation == graph_generation && ticket->ref.grant_generation != 0) { + *transfer_out = ticket->ref; + result = PCM_X_QUEUE_DUPLICATE; + } else + result = PCM_X_QUEUE_STALE; + goto begin_transfer_done; + } + if (state != PCM_XT_ACTIVE_PROBE) { + result = PCM_X_QUEUE_BAD_STATE; + goto begin_transfer_done; + } + if (!pcm_x_master_blocker_stage_metadata_valid(ticket)) { + result = PCM_X_QUEUE_CORRUPT; + goto begin_transfer_done; + } + if (ticket->blocker_stage_set_generation != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto begin_transfer_done; + } + if (ticket->blocker_set_generation == 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto begin_transfer_done; + } + { + PcmXAllocatorView blocker_view; + + if (ticket->blocker_set_generation == UINT64_MAX + || !pcm_x_master_blocker_published_source_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)) { + result = PCM_X_QUEUE_CORRUPT; + goto begin_transfer_done; + } + } + if (ticket->blocker_count != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto begin_transfer_done; + } + if (ticket->graph_generation != graph_generation || ticket->ref.grant_generation != 0) { + result = PCM_X_QUEUE_STALE; + goto begin_transfer_done; + } + if (!pcm_x_transfer_leg_is_clear(&ticket->reliable) + || ticket->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT) { + result = PCM_X_QUEUE_BAD_STATE; + goto begin_transfer_done; + } + /* grant_generation belongs to the durable master ticket, not to an outer + * caller. Mint it from the ticket's checked reliable sequence while the + * exact tag partition is locked, then persist the same value in both the + * full wire reference and the sequence that fences every later leg. */ + if (!cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &grant_generation)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto begin_transfer_done; + } + ticket->reliable.state_sequence = grant_generation; + ticket->ref.grant_generation = grant_generation; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_ACTIVE_TRANSFER); + *transfer_out = ticket->ref; + pcm_x_stats_increment(&header->stats.transfer_count); + result = PCM_X_QUEUE_OK; + +begin_transfer_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_revoke_arm_exact(const PcmXTicketRef *ref, int32 responder_node, + uint64 responder_session, PcmXRevokePayload *revoke_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 image_id = 0; + uint32 partition; + bool mint_image = false; + + if (revoke_out != NULL) + memset(revoke_out, 0, sizeof(*revoke_out)); + if (header == NULL || ref == NULL || revoke_out == NULL || ref->grant_generation == 0 + || responder_node < 0 || responder_node >= PCM_X_PROTOCOL_NODE_LIMIT + || responder_session == 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if (ticket->blocker_count != 0 || ticket->graph_generation == 0 + || !pcm_x_master_blocker_stage_is_clear(ticket)) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) { + if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, + responder_session) + && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_BUSY; + } else if (ticket->image.image_id != 0) { + result = ticket->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY + ? PCM_X_QUEUE_NOT_READY + : PCM_X_QUEUE_CORRUPT; + } else if (!pcm_x_transfer_peer_exact(ticket, responder_node, responder_session)) + result = PCM_X_QUEUE_STALE; + else { + result = PCM_X_QUEUE_OK; + mint_image = true; + } + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + if (!mint_image) { + revoke_out->ref = ticket->ref; + revoke_out->image_id = ticket->image.image_id; + } + } + LWLockRelease(&header->master_locks[partition].lock); + if (!mint_image) + goto revoke_done; + + result = pcm_x_image_id_mint(&image_id); + if (result != PCM_X_QUEUE_OK) + goto revoke_done; + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if (ticket->blocker_count != 0 || ticket->graph_generation == 0 + || !pcm_x_master_blocker_stage_is_clear(ticket)) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_transfer_leg_is_clear(&ticket->reliable)) { + if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, + responder_session) + && ticket->image.image_id != 0 && ticket->image.source_node == (uint32)responder_node) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_BUSY; + } else if (ticket->image.image_id != 0) + result = PCM_X_QUEUE_CORRUPT; + else { + result = pcm_x_transfer_leg_arm_locked(ticket, PGRAC_IC_MSG_PCM_X_REVOKE, responder_node, + responder_session); + if (result == PCM_X_QUEUE_OK) { + ticket->image.image_id = image_id; + ticket->image.source_node = (uint32)responder_node; + ticket->involved_nodes_bitmap |= UINT32_C(1) << responder_node; + pg_write_barrier(); + } + } + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + revoke_out->ref = ticket->ref; + revoke_out->image_id = ticket->image.image_id; + } + LWLockRelease(&header->master_locks[partition].lock); + +revoke_done: + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_image_ready_exact(const PcmXGrantPayload *image_ready, + int32 authenticated_node, uint64 authenticated_session, + PcmXGrantPayload *prepare_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 requester_session; + int32 requester_node; + uint32 partition; + + if (prepare_out != NULL) + memset(prepare_out, 0, sizeof(*prepare_out)); + if (header == NULL || image_ready == NULL || prepare_out == NULL + || image_ready->ref.grant_generation == 0 || !pcm_x_image_token_valid(&image_ready->image) + || !pcm_x_image_id_master_exact(image_ready->image.image_id, cluster_node_id) + || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_session == 0 + || image_ready->image.source_node != (uint32)authenticated_node + || !pcm_x_wait_identity_valid(&image_ready->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&image_ready->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition + = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&image_ready->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &image_ready->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + requester_node = image_ready->ref.identity.node_id; + /* Admission fixed the requester session in the prehandle. A frontier + * change cannot retarget an already admitted ticket to a new backend. */ + requester_session = ticket == NULL ? 0 : ticket->prehandle.sender_session_incarnation; + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &image_ready->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if (ticket->image.image_id != image_ready->image.image_id + || ticket->image.source_node != image_ready->image.source_node) + result = PCM_X_QUEUE_STALE; + else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + requester_node, requester_session)) + result = pcm_x_image_token_equal(&ticket->image, &image_ready->image) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + else if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_REVOKE, + authenticated_node, authenticated_session)) + result = PCM_X_QUEUE_STALE; + else { + result = pcm_x_transfer_leg_advance_locked( + ticket, PGRAC_IC_MSG_PCM_X_REVOKE, authenticated_node, authenticated_session, + PGRAC_IC_MSG_PCM_X_IMAGE_READY, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, requester_node, + requester_session); + if (result == PCM_X_QUEUE_OK) { + ticket->image = image_ready->image; + ticket->involved_nodes_bitmap |= UINT32_C(1) << requester_node; + pg_write_barrier(); + } + } + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + prepare_out->ref = ticket->ref; + prepare_out->image = ticket->image; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ready, + int32 authenticated_node, uint64 authenticated_session, + PcmXPhasePayload *commit_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + + if (commit_out != NULL) + memset(commit_out, 0, sizeof(*commit_out)); + if (header == NULL || install_ready == NULL || commit_out == NULL + || install_ready->ref.grant_generation == 0 || install_ready->image_id == 0 + || install_ready->result != PCM_X_QUEUE_OK + || install_ready->phase != PGRAC_IC_MSG_PCM_X_INSTALL_READY || install_ready->flags != 0 + || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_session == 0 || install_ready->ref.identity.node_id != authenticated_node + || !pcm_x_wait_identity_valid(&install_ready->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&install_ready->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + partition + = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&install_ready->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &install_ready->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &install_ready->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if (ticket->image.image_id != install_ready->image_id) + result = PCM_X_QUEUE_STALE; + else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, + authenticated_node, authenticated_session)) + result = PCM_X_QUEUE_DUPLICATE; + else + result = pcm_x_transfer_leg_advance_locked( + ticket, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, authenticated_node, authenticated_session, + PGRAC_IC_MSG_PCM_X_INSTALL_READY, PGRAC_IC_MSG_PCM_X_COMMIT_X, authenticated_node, + authenticated_session); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + commit_out->ref = ticket->ref; + commit_out->phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_final_ack_prepare_exact(const PcmXFinalAckPayload *final_ack, + int32 authenticated_node, uint64 authenticated_session, + PcmXMasterFinalAckToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult claim_result; + PcmXQueueResult result; + uint64 expected_generation; + uint32 partition; + + if (token_out != NULL) + memset(token_out, 0, sizeof(*token_out)); + if (header == NULL || final_ack == NULL || token_out == NULL + || final_ack->ref.grant_generation == 0 || final_ack->image_id == 0 + || final_ack->committed_own_generation == 0 || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || final_ack->ref.identity.node_id != authenticated_node + || !pcm_x_wait_identity_valid(&final_ack->ref.identity)) + return PCM_X_QUEUE_INVALID; + if (!cluster_pcm_x_generation_next(final_ack->ref.identity.base_own_generation, + &expected_generation)) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + if (final_ack->committed_own_generation != expected_generation) + return PCM_X_QUEUE_STALE; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&final_ack->ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&final_ack->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &final_ack->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &final_ack->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if ((claim_result = pcm_x_master_pending_x_claim_required(ticket)) != PCM_X_QUEUE_OK) + result = claim_result; + else if (ticket->image.image_id != final_ack->image_id + || !pcm_x_image_token_valid(&ticket->image)) + result = PCM_X_QUEUE_STALE; + else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, + authenticated_node, authenticated_session)) + result = pcm_x_transfer_peer_exact(ticket, authenticated_node, authenticated_session) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + else if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, + authenticated_node, authenticated_session) + || !pcm_x_transfer_peer_exact(ticket, authenticated_node, authenticated_session)) + result = PCM_X_QUEUE_STALE; + else + result = PCM_X_QUEUE_OK; + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + token_out->final_ack = *final_ack; + token_out->image = ticket->image; + token_out->master_session_incarnation = ticket->master_session_incarnation; + token_out->state_sequence = ticket->reliable.state_sequence; + token_out->authenticated_session = authenticated_session; + token_out->authenticated_node = authenticated_node; + token_out->reserved = 0; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_final_ack_finalize_exact(const PcmXMasterFinalAckToken *token, + PcmXPhasePayload *final_commit_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult claim_result; + PcmXQueueResult result; + uint64 expected_generation; + uint64 next_state_sequence = 0; + uint32 partition; + + if (final_commit_out != NULL) + memset(final_commit_out, 0, sizeof(*final_commit_out)); + if (header == NULL || token == NULL || final_commit_out == NULL || token->reserved != 0 + || token->master_session_incarnation == 0 || token->state_sequence == 0 + || token->authenticated_node < 0 || token->authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || token->authenticated_session == 0 || token->final_ack.ref.grant_generation == 0 + || token->final_ack.image_id == 0 || token->final_ack.committed_own_generation == 0 + || token->final_ack.ref.identity.node_id != token->authenticated_node + || !pcm_x_wait_identity_valid(&token->final_ack.ref.identity) + || !pcm_x_image_token_valid(&token->image) + || token->image.image_id != token->final_ack.image_id) + return PCM_X_QUEUE_INVALID; + if (!cluster_pcm_x_generation_next(token->final_ack.ref.identity.base_own_generation, + &expected_generation)) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + if (token->final_ack.committed_own_generation != expected_generation) + return PCM_X_QUEUE_STALE; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0 + || !pcm_x_runtime_token_exact(&runtime, token->master_session_incarnation)) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&token->final_ack.ref, &ticket_ref, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + partition + = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&token->final_ack.ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &token->final_ack.ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &token->final_ack.ref)) + result = PCM_X_QUEUE_STALE; + else if (ticket->master_session_incarnation != token->master_session_incarnation + || !pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if ((claim_result = pcm_x_master_pending_x_claim_required(ticket)) != PCM_X_QUEUE_OK) + result = claim_result; + else if (!pcm_x_image_token_equal(&ticket->image, &token->image) + || ticket->image.image_id != token->final_ack.image_id + || !pcm_x_transfer_peer_exact(ticket, token->authenticated_node, + token->authenticated_session)) + result = PCM_X_QUEUE_STALE; + else if (pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, + token->authenticated_node, token->authenticated_session)) { + if (ticket->reliable.state_sequence == token->state_sequence) + result = PCM_X_QUEUE_DUPLICATE; + else if (cluster_pcm_x_generation_next(token->state_sequence, &next_state_sequence) + && ticket->reliable.state_sequence == next_state_sequence) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_STALE; + } else if (ticket->reliable.state_sequence != token->state_sequence + || !pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, + token->authenticated_node, + token->authenticated_session)) + result = PCM_X_QUEUE_STALE; + else + result = pcm_x_transfer_leg_advance_locked( + ticket, PGRAC_IC_MSG_PCM_X_COMMIT_X, token->authenticated_node, + token->authenticated_session, PGRAC_IC_MSG_PCM_X_FINAL_ACK, + PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, token->authenticated_node, + token->authenticated_session); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + final_commit_out->ref = ticket->ref; + final_commit_out->phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_final_ack_exact(const PcmXFinalAckPayload *final_ack, int32 authenticated_node, + uint64 authenticated_session, + PcmXPhasePayload *final_commit_out) +{ + PcmXMasterFinalAckToken token; + PcmXQueueResult result; + + if (final_commit_out != NULL) + memset(final_commit_out, 0, sizeof(*final_commit_out)); + if (final_commit_out == NULL) + return PCM_X_QUEUE_INVALID; + result = cluster_pcm_x_master_final_ack_prepare_exact(final_ack, authenticated_node, + authenticated_session, &token); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + return cluster_pcm_x_master_final_ack_finalize_exact(&token, final_commit_out); +} + + +PcmXQueueResult +cluster_pcm_x_master_final_confirm_exact(const PcmXPhasePayload *final_confirm, + int32 authenticated_node, uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 flags; + uint32 node_bit; + uint32 partition; + + if (header == NULL || final_confirm == NULL || final_confirm->ref.grant_generation == 0 + || final_confirm->reason != 0 || final_confirm->phase != PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM + || final_confirm->flags != 0 || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || final_confirm->ref.identity.node_id != authenticated_node + || !pcm_x_wait_identity_valid(&final_confirm->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + node_bit = UINT32_C(1) << final_confirm->ref.identity.node_id; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&final_confirm->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + partition + = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&final_confirm->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TAG, tag_ref, + &final_confirm->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &final_confirm->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL + || !pcm_x_ticket_ref_equal(&ticket->ref, &final_confirm->ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (((flags = pcm_x_slot_flags_read(&ticket->slot)) + & ~PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) + != 0) + result = PCM_X_QUEUE_CORRUPT; + else if (pcm_x_slot_state_read(&ticket->slot) == PCM_XT_COMPLETE + && pcm_x_transfer_leg_is_clear(&ticket->reliable) + && ticket->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM) + result = PCM_X_QUEUE_DUPLICATE; + else if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_ACTIVE_TRANSFER) + result = PCM_X_QUEUE_BAD_STATE; + else if ((flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) == 0) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_transfer_leg_exact(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, + authenticated_node, authenticated_session) + || !pcm_x_transfer_peer_exact(ticket, authenticated_node, authenticated_session)) + result = PCM_X_QUEUE_STALE; + else if (tag_slot->active_index != ticket_ref.slot_index + || tag_slot->active_slot_generation != ticket_ref.slot_generation + || tag_slot->head_index != ticket_ref.slot_index + || ticket->prev_index != PCM_X_INVALID_SLOT_INDEX + || (pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & node_bit) == 0) + result = PCM_X_QUEUE_CORRUPT; + else if (tag_slot->queue_state_sequence == UINT64_MAX) + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + else if (!pcm_x_master_unlink_ticket_locked(tag_slot, tag_ref, ticket, ticket_ref)) + result = PCM_X_QUEUE_CORRUPT; + else { + pcm_x_transfer_leg_clear(&ticket->reliable, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, + authenticated_node); + tag_slot->queue_state_sequence++; + (void)pg_atomic_fetch_and_u32(&tag_slot->queued_node_bitmap, ~node_bit); + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + if ((flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) != 0) + pcm_x_slot_flags_write(&ticket->slot, 0); + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_COMPLETE); + pcm_x_stats_increment(&header->stats.complete_count); + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_complete_exact(const PcmXTicketRef *ref) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 node_bit; + uint32 partition; + uint32 state; + uint32 flags; + + if (header == NULL || ref == NULL || ref->grant_generation == 0 + || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + node_bit = UINT32_C(1) << ref->identity.node_id; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) { + result = PCM_X_QUEUE_STALE; + goto complete_done; + } + if (runtime.state != PCM_X_RUNTIME_ACTIVE + || !pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto complete_done; + } + state = pcm_x_slot_state_read(&ticket->slot); + flags = pcm_x_slot_flags_read(&ticket->slot); + if ((flags & ~PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto complete_done; + } + /* This legacy completion primitive has no GRD handoff token. Once the + * ticket owns pending-X it must complete through FINAL confirmation. */ + if ((flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto complete_done; + } + if (pcm_x_master_event_replay_exact((PcmXMasterTicketState)state, PCM_X_EVENT_COMMIT_COMPLETE, + ticket->reliable.response_tombstone_mask)) { + result = PCM_X_QUEUE_DUPLICATE; + goto complete_done; + } + if (pcm_x_master_event_successor_state((PcmXMasterTicketState)state, + PCM_X_EVENT_COMMIT_COMPLETE)) { + result = PCM_X_QUEUE_BAD_STATE; + goto complete_done; + } + if (state != PCM_XT_ACTIVE_TRANSFER) { + result = PCM_X_QUEUE_BAD_STATE; + goto complete_done; + } + /* + * RECOVERY_BLOCKED retains ACTIVE_TRANSFER evidence. grant_generation is + * assigned before FINAL_ACK and is therefore not positive completion proof. + */ + if (tag_slot->active_index != ticket_ref.slot_index + || tag_slot->active_slot_generation != ticket_ref.slot_generation + || tag_slot->head_index != ticket_ref.slot_index + || ticket->prev_index != PCM_X_INVALID_SLOT_INDEX + || (pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & node_bit) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto complete_done; + } + if (tag_slot->queue_state_sequence == UINT64_MAX) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto complete_done; + } + if (!pcm_x_master_unlink_ticket_locked(tag_slot, tag_ref, ticket, ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto complete_done; + } + tag_slot->queue_state_sequence++; + (void)pg_atomic_fetch_and_u32(&tag_slot->queued_node_bitmap, ~node_bit); + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_COMPLETE); + pcm_x_stats_increment(&header->stats.complete_count); + result = PCM_X_QUEUE_OK; + +complete_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_master_cancel_exact_impl(const PcmXTicketRef *ref, bool reversible_only, + bool pending_x_release, PcmXMasterPendingXReleaseToken *release_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + uint32 node_bit; + uint32 partition; + uint32 state; + uint32 flags; + uint64 release_sequence = 0; + bool pending_x_claimed = false; + bool recovery_blocked = false; + + if (release_out != NULL) + memset(release_out, 0, sizeof(*release_out)); + if (header == NULL || ref == NULL || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity) || (pending_x_release && release_out == NULL) + || (!pending_x_release && release_out != NULL)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + node_bit = UINT32_C(1) << ref->identity.node_id; + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + result = pcm_x_master_admission_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result != PCM_X_QUEUE_OK) { + LWLockRelease(&header->allocator_lock.lock); + return result; + } + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + LWLockRelease(&header->allocator_lock.lock); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_admission_ref_equal(&ticket->ref, ref) + || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE) { + result = PCM_X_QUEUE_STALE; + goto cancel_done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto cancel_done; + } + state = pcm_x_slot_state_read(&ticket->slot); + flags = pcm_x_slot_flags_read(&ticket->slot); + if (!pcm_x_master_pending_x_flags_valid(flags)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + pending_x_claimed = (flags & PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) != 0; + if (pending_x_release && state == PCM_XT_CANCELLED + && flags + == (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING) + && ticket->ref.grant_generation == 0 + && pcm_x_master_event_replay_exact((PcmXMasterTicketState)state, PCM_X_EVENT_CANCEL_EXACT, + ticket->reliable.response_tombstone_mask) + && pcm_x_transfer_leg_is_clear(&ticket->reliable)) { + PcmXAllocatorView blocker_view; + + /* External WFG/GRD release is irreversible. A replay token is safe only + * when the tombstone is already structurally detached and no partial + * blocker stage can still publish under it. */ + if (ticket->reliable.state_sequence == 0 || ticket->graph_generation == 0 + || !pcm_x_master_blocker_stage_metadata_valid(ticket) + || !pcm_x_master_blocker_stage_is_clear(ticket) + || !pcm_x_master_blocker_published_source_valid(ticket) + || (ticket->blocker_set_generation == 0 + && (ticket->blocker_head_index != PCM_X_INVALID_SLOT_INDEX + || ticket->blocker_count != 0 || ticket->blocker_set_crc32c != 0)) + || (ticket->blocker_set_generation != 0 + && (ticket->blocker_set_generation == UINT64_MAX + || ticket->reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL))) + || ticket->next_index != PCM_X_INVALID_SLOT_INDEX + || ticket->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_index == ticket_ref.slot_index + || tag_slot->head_index == ticket_ref.slot_index + || tag_slot->tail_index == ticket_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + release_out->ref = ticket->ref; + release_out->master_session_incarnation = ticket->master_session_incarnation; + release_out->state_sequence = ticket->reliable.state_sequence; + result = PCM_X_QUEUE_DUPLICATE; + goto cancel_done; + } + if (pending_x_release && flags != 0 + && (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED + || state == PCM_XT_RETIRE_CREDIT)) { + /* A terminal ticket with pending external authority must have matched + * the complete replay proof above. Never downgrade a torn token/leg to + * an ordinary duplicate, which would skip GRD release forever. */ + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + if (state == PCM_XT_ACTIVE_TRANSFER) { + if (ticket->ref.grant_generation == 0 || flags != PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + /* A cancellation that loses the single master-lock classification race + * to ACTIVE_TRANSFER has crossed the irreversible grant boundary. The + * FINAL protocol owns completion; a late CANCEL is retryable, not global + * corruption. */ + if (pending_x_release) { + result = PCM_X_QUEUE_BUSY; + goto cancel_done; + } + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_RECOVERY_BLOCKED); + recovery_blocked = true; + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + /* Cancellation cannot silently discard the ticket-exact proof for an + * externally retained GRD barrier. Such an ACTIVE_PROBE is driven to + * FINAL or released through the explicit two-phase gate choreography. */ + if (pending_x_release && pending_x_claimed) { + if (state != PCM_XT_ACTIVE_PROBE + || (flags != PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + && flags + != (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED))) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + } else if (!pending_x_release && pending_x_claimed) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED || state == PCM_XT_RETIRE_CREDIT) { + if (ticket->ref.grant_generation == 0 + && pcm_x_master_event_replay_exact((PcmXMasterTicketState)state, + PCM_X_EVENT_CANCEL_EXACT, + ticket->reliable.response_tombstone_mask)) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if (ticket->ref.grant_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + /* Once promotion has exposed ACTIVE_PROBE, wire messages 57-60 defer to the + * blocker/transfer exit protocol. The legacy internal primitive retains + * its explicit ACTIVE_PROBE cleanup authority for existing recovery tests. */ + if (reversible_only && !pending_x_release && state == PCM_XT_ACTIVE_PROBE) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if (state != PCM_XT_ADMITTING && state != PCM_XT_QUEUED && state != PCM_XT_ACTIVE_PROBE) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if (!pcm_x_master_blocker_stage_metadata_valid(ticket)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + if (ticket->blocker_stage_set_generation != 0) { + if (pending_x_release && pending_x_claimed) { + pcm_x_slot_flags_write(&ticket->slot, + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED); + result = PCM_X_QUEUE_NOT_READY; + } else + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if ((state == PCM_XT_ACTIVE_PROBE + && (tag_slot->active_index != ticket_ref.slot_index + || tag_slot->active_slot_generation != ticket_ref.slot_generation)) + || (state != PCM_XT_ACTIVE_PROBE && tag_slot->active_index == ticket_ref.slot_index)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + if (state == PCM_XT_ACTIVE_PROBE) { + PcmXAllocatorView blocker_view; + + if (!pending_x_release + && (ticket->blocker_set_generation == 0 || ticket->graph_generation == 0)) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if (ticket->blocker_set_generation == 0) { + if (ticket->blocker_head_index != PCM_X_INVALID_SLOT_INDEX || ticket->blocker_count != 0 + || ticket->blocker_set_crc32c != 0 + || !pcm_x_master_blocker_published_source_valid(ticket)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + } else if (ticket->blocker_set_generation == UINT64_MAX + || !pcm_x_master_blocker_published_source_valid(ticket) + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + if (!pending_x_release && ticket->blocker_count != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_done; + } + if (pending_x_release && pending_x_claimed + && (ticket->graph_generation == 0 || !pcm_x_transfer_leg_is_clear(&ticket->reliable) + || (ticket->blocker_set_generation != 0 + && ticket->reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT))) { + pcm_x_slot_flags_write(&ticket->slot, + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED); + result = PCM_X_QUEUE_NOT_READY; + goto cancel_done; + } + } + if (tag_slot->queue_state_sequence == UINT64_MAX) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto cancel_done; + } + if ((pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & node_bit) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + if (pending_x_release && pending_x_claimed + && !cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &release_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto cancel_done; + } + if (!pcm_x_master_unlink_ticket_locked(tag_slot, tag_ref, ticket, ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + if (state == PCM_XT_QUEUED && !pcm_x_stats_depth_decrement(header)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_done; + } + tag_slot->queue_state_sequence++; + (void)pg_atomic_fetch_and_u32(&tag_slot->queued_node_bitmap, ~node_bit); + if (pending_x_release && pending_x_claimed) + ticket->reliable.state_sequence = release_sequence; + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_CANCEL; + if (pending_x_release && pending_x_claimed) + pcm_x_slot_flags_write(&ticket->slot, + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING); + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_CANCELLED); + pcm_x_stats_increment(&header->stats.cancel_count); + if (pending_x_release && pending_x_claimed) { + release_out->ref = ticket->ref; + release_out->master_session_incarnation = ticket->master_session_incarnation; + release_out->state_sequence = ticket->reliable.state_sequence; + } + result = PCM_X_QUEUE_OK; + +cancel_done: + LWLockRelease(&header->master_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || recovery_blocked) + pcm_x_runtime_fail_closed(); + return result; + } + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_master_cancel_exact(const PcmXTicketRef *ref) +{ + return pcm_x_master_cancel_exact_impl(ref, false, false, NULL); +} + + +PcmXQueueResult +cluster_pcm_x_master_cancel_reversible_exact(const PcmXTicketRef *ref) +{ + return pcm_x_master_cancel_exact_impl(ref, true, false, NULL); +} + + +PcmXQueueResult +cluster_pcm_x_master_pending_x_cancel_prepare_exact(const PcmXTicketRef *ref, + PcmXMasterPendingXReleaseToken *token_out) +{ + return pcm_x_master_cancel_exact_impl(ref, true, true, token_out); +} + + +/* A release worker can be preempted after another worker finalizes the same + * token and terminal retirement advances. Revalidate the two states that no + * longer belong to the live master domain without nesting allocator/master + * locks: a contiguous retired frontier, or the exact immutable DETACHING + * tombstone while allocator reclamation is excluded by the shared lock. */ +static PcmXQueueResult +pcm_x_master_pending_x_release_post_terminal_exact(const PcmXMasterPendingXReleaseToken *token) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 flags; + uint32 state; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + if (header->fully_retired_ticket_id != UINT64_MAX + && token->ref.handle.ticket_id <= header->fully_retired_ticket_id) { + result = PCM_X_QUEUE_DUPLICATE; + goto replay_done; + } + result = pcm_x_master_ticket_lookup_locked(&token->ref, &ticket_ref, &ticket); + if (result != PCM_X_QUEUE_OK) + goto replay_done; + state = pcm_x_slot_state_read(&ticket->slot); + if (state != PCM_XT_DETACHING) { + result = PCM_X_QUEUE_NOT_READY; + goto replay_done; + } + flags = pcm_x_slot_flags_read(&ticket->slot); + if (!pcm_x_ticket_ref_equal(&ticket->ref, &token->ref) + || ticket->master_session_incarnation != token->master_session_incarnation) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_master_pending_x_flags_valid(flags) || flags != 0 + || ticket->next_index != PCM_X_INVALID_SLOT_INDEX + || ticket->prev_index != PCM_X_INVALID_SLOT_INDEX + || (ticket->reliable.response_tombstone_mask & PCM_X_RESPONSE_TOMBSTONE_CANCEL) == 0 + || (ticket->reliable.response_tombstone_mask & PCM_X_RESPONSE_TOMBSTONE_COMPLETE) != 0) + result = PCM_X_QUEUE_CORRUPT; + else + result = PCM_X_QUEUE_DUPLICATE; + +replay_done: + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_pending_x_cancel_finalize_exact(const PcmXMasterPendingXReleaseToken *token) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + PcmXQueueResult replay_result; + uint32 flags; + uint32 partition; + uint32 state; + + if (header == NULL || token == NULL || token->master_session_incarnation == 0 + || token->state_sequence == 0 || token->ref.grant_generation != 0 + || !pcm_x_wait_identity_valid(&token->ref.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (runtime.master_session_incarnation != token->master_session_incarnation) + return PCM_X_QUEUE_STALE; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(&token->ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + replay_result = pcm_x_master_pending_x_release_post_terminal_exact(token); + if (replay_result == PCM_X_QUEUE_DUPLICATE || replay_result == PCM_X_QUEUE_CORRUPT) + return replay_result; + return result; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&token->ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &token->ref.identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &token->ref.identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, &token->ref)) + result = PCM_X_QUEUE_STALE; + else if (ticket->master_session_incarnation != token->master_session_incarnation) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_master_pending_x_flags_valid((flags = pcm_x_slot_flags_read(&ticket->slot)))) + result = PCM_X_QUEUE_CORRUPT; + else if ((state = pcm_x_slot_state_read(&ticket->slot)) != PCM_XT_CANCELLED + && state != PCM_XT_RETIRE_CREDIT) + result = PCM_X_QUEUE_BAD_STATE; + else if (!pcm_x_master_event_replay_exact((PcmXMasterTicketState)state, + PCM_X_EVENT_CANCEL_EXACT, + ticket->reliable.response_tombstone_mask)) + result = PCM_X_QUEUE_STALE; + else if (ticket->next_index != PCM_X_INVALID_SLOT_INDEX + || ticket->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_index == ticket_ref.slot_index + || tag_slot->head_index == ticket_ref.slot_index + || tag_slot->tail_index == ticket_ref.slot_index) + result = PCM_X_QUEUE_CORRUPT; + else if (flags == 0) + result = PCM_X_QUEUE_DUPLICATE; + else if (state != PCM_XT_CANCELLED) + result = PCM_X_QUEUE_CORRUPT; + else if (flags + != (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING)) + result = PCM_X_QUEUE_CORRUPT; + else if (ticket->reliable.state_sequence != token->state_sequence + || !pcm_x_transfer_leg_is_clear(&ticket->reliable) + || (ticket->blocker_set_generation != 0 + && ticket->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT)) + result = PCM_X_QUEUE_STALE; + else { + pcm_x_slot_flags_write(&ticket->slot, 0); + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_BAD_STATE) { + replay_result = pcm_x_master_pending_x_release_post_terminal_exact(token); + if (replay_result == PCM_X_QUEUE_DUPLICATE || replay_result == PCM_X_QUEUE_CORRUPT) + result = replay_result; + } + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Resolve a cancellation sent before ADMIT_ACK by the immutable prehandle. + * The directory slot generation is copied into cancelled_out before dropping + * allocator_lock; the full-ref cancel below therefore cannot touch a reused + * ticket even if terminal retirement races this adapter. */ +PcmXQueueResult +cluster_pcm_x_master_prehandle_cancel_exact(const PcmXPrehandleCancelPayload *request, + PcmXMasterAdmission *cancelled_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload staged_request; + PcmXSlotRef ticket_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + bool fail_closed = false; + bool cancel_first = false; + + if (cancelled_out != NULL) + pcm_x_master_admission_clear(cancelled_out); + if (header == NULL || request == NULL || cancelled_out == NULL + || !pcm_x_wait_identity_valid(&request->identity) + || request->prehandle.sender_session_incarnation == 0 + || request->prehandle.prehandle_sequence == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + if (!pcm_x_runtime_token_exact(&runtime, runtime.master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto prehandle_lookup_done; + } + frontier = &header->peer_frontiers[request->identity.node_id]; + if (frontier->cluster_epoch != request->identity.cluster_epoch + || frontier->sender_session_incarnation != request->prehandle.sender_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto prehandle_lookup_done; + } + if (frontier->next_expected_prehandle_sequence == 0 + || frontier->retired_prehandle_sequence >= frontier->next_expected_prehandle_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto prehandle_lookup_done; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &request->prehandle, &ticket_ref); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + if (request->prehandle.prehandle_sequence <= frontier->retired_prehandle_sequence) + result = PCM_X_QUEUE_RETIRED; + else if (request->prehandle.prehandle_sequence + < frontier->next_expected_prehandle_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else if (request->prehandle.prehandle_sequence + == frontier->next_expected_prehandle_sequence) { + /* CANCEL may overtake its ENQUEUE transport. Consume the frontier + * below through the normal composite admission path, then publish a + * formal CANCELLED tombstone before acknowledging application. */ + result = PCM_X_QUEUE_OK; + cancel_first = true; + } else + result = PCM_X_QUEUE_NOT_READY; + goto prehandle_lookup_done; + } + if (directory_result != PCM_X_DIRECTORY_OK) { + result = pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto prehandle_lookup_done; + } + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + ticket_ref); + if (ticket == NULL || !pcm_x_prehandle_equal(&ticket->prehandle, &request->prehandle)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto prehandle_lookup_done; + } + if (!pcm_x_wait_identity_equal(&ticket->ref.identity, &request->identity)) { + /* One prehandle is an immutable application idempotency key. Resolving + * it to another identity is internal namespace corruption, not an old + * generation that may be ignored. */ + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto prehandle_lookup_done; + } + pcm_x_master_admission_from_ticket(cancelled_out, ticket_ref, ticket); + result = PCM_X_QUEUE_OK; + +prehandle_lookup_done: + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) { + pcm_x_runtime_fail_closed(); + return result; + } + if (result != PCM_X_QUEUE_OK) + return result; + if (cancel_first) { + memset(&staged_request, 0, sizeof(staged_request)); + staged_request.identity = request->identity; + staged_request.prehandle = request->prehandle; + result = pcm_x_master_admit_begin_impl(&staged_request, cancelled_out, true); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + if (result == PCM_X_QUEUE_OK) + return PCM_X_QUEUE_OK; + } + + result = cluster_pcm_x_master_cancel_reversible_exact(&cancelled_out->ref); + if (result == PCM_X_QUEUE_DUPLICATE) + cancelled_out->flags = PCM_X_ADMIT_F_EXACT_REPLAY; + return result; +} + + +static PcmXQueueResult +pcm_x_master_terminal_lookup(const PcmXTicketRef *ref, PcmXSlotRef *ticket_ref_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXMasterTicketSlot *ticket; + PcmXQueueResult result; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, ticket_ref_out, &ticket); + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Prove that one terminal slot carries exactly one outcome authority. */ +static PcmXQueueResult +pcm_x_master_terminal_outcome_exact(const PcmXMasterTicketSlot *ticket, uint32 state) +{ + const uint64 allowed = PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM + | PCM_X_RESPONSE_TOMBSTONE_COMPLETE | PCM_X_RESPONSE_TOMBSTONE_CANCEL; + uint64 outcome; + uint64 expected; + uint32 flags; + + if (ticket == NULL) + return PCM_X_QUEUE_CORRUPT; + flags = pcm_x_slot_flags_read(&ticket->slot); + if (!pcm_x_master_pending_x_flags_valid(flags)) + return PCM_X_QUEUE_CORRUPT; + if ((ticket->reliable.response_tombstone_mask & ~allowed) != 0) + return PCM_X_QUEUE_CORRUPT; + outcome = ticket->reliable.response_tombstone_mask + & (PCM_X_RESPONSE_TOMBSTONE_COMPLETE | PCM_X_RESPONSE_TOMBSTONE_CANCEL); + if (state == PCM_XT_COMPLETE) + expected = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + else if (state == PCM_XT_CANCELLED) + expected = PCM_X_RESPONSE_TOMBSTONE_CANCEL; + else if (state == PCM_XT_RETIRE_CREDIT) { + if (outcome != PCM_X_RESPONSE_TOMBSTONE_COMPLETE + && outcome != PCM_X_RESPONSE_TOMBSTONE_CANCEL) + return PCM_X_QUEUE_CORRUPT; + expected = outcome; + } else + return PCM_X_QUEUE_BAD_STATE; + if (outcome != expected) + return PCM_X_QUEUE_CORRUPT; + if ((expected == PCM_X_RESPONSE_TOMBSTONE_COMPLETE && ticket->ref.grant_generation == 0) + || (expected == PCM_X_RESPONSE_TOMBSTONE_CANCEL && ticket->ref.grant_generation != 0)) + return PCM_X_QUEUE_CORRUPT; + if (flags + == (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING)) + return PCM_X_QUEUE_NOT_READY; + if (flags != 0) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_master_cancel_ack_snapshot_exact(const PcmXTicketRef *ref, + PcmXMasterAdmission *cancelled_out, + bool *prehandle_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 partition; + uint32 state; + bool prehandle; + + if (cancelled_out != NULL) + pcm_x_master_admission_clear(cancelled_out); + if (prehandle_out != NULL) + *prehandle_out = false; + if (header == NULL || ref == NULL || cancelled_out == NULL || prehandle_out == NULL + || ref->grant_generation != 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &ref->identity.tag, + PCM_X_STATE_BIT(PCM_XT_CANCELLED) | PCM_X_STATE_BIT(PCM_XT_RETIRE_CREDIT)); + if (tag_slot == NULL || ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if ((state = pcm_x_slot_state_read(&ticket->slot)) != PCM_XT_CANCELLED + && state != PCM_XT_RETIRE_CREDIT) + result = PCM_X_QUEUE_BAD_STATE; + else if ((result = pcm_x_master_terminal_outcome_exact(ticket, state)) == PCM_X_QUEUE_OK) { + prehandle + = (ticket->reliable.response_tombstone_mask & PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM) + == 0; + if (ticket->prehandle.sender_session_incarnation == 0 + || ticket->prehandle.prehandle_sequence == 0) + result = PCM_X_QUEUE_CORRUPT; + else { + pcm_x_master_admission_from_ticket(cancelled_out, ticket_ref, ticket); + *prehandle_out = prehandle; + } + } + LWLockRelease(&header->master_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +pcm_x_master_terminal_leg_is_clear(const PcmXReliableLegState *leg) +{ + return leg != NULL && leg->pending_opcode == PCM_X_TERMINAL_LEG_NONE && leg->phase == 0 + && leg->expected_responder_node == 0 && leg->expected_responder_session == 0; +} + + +/* Terminal progress is monotone within the fixed participant set. */ +static bool +pcm_x_master_terminal_bitmaps_valid(const PcmXMasterTicketSlot *ticket, uint32 state) +{ + uint32 involved; + + if (ticket == NULL) + return false; + involved = ticket->involved_nodes_bitmap; + if (involved == 0 || (ticket->drained_nodes_bitmap & ~involved) != 0 + || (ticket->retire_acked_nodes_bitmap & ~involved) != 0) + return false; + if (state == PCM_XT_COMPLETE || state == PCM_XT_CANCELLED) + return ticket->retire_acked_nodes_bitmap == 0; + if (state == PCM_XT_RETIRE_CREDIT) + return ticket->drained_nodes_bitmap == involved; + return false; +} + + +PcmXQueueResult +cluster_pcm_x_master_terminal_leg_arm_exact(const PcmXTicketRef *ref, PcmXTerminalLegKind kind, + int32 responder_node, uint64 responder_session, + PcmXTerminalLegToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXMasterTicketSlot *ticket; + PcmXDirectoryResult directory_result; + PcmXRetirePayload retire_key; + PcmXSlotRef existing; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint64 next_sequence; + uint32 responder_bit; + uint32 partition; + uint32 state; + bool fail_closed = false; + + if (token_out != NULL) + memset(token_out, 0, sizeof(*token_out)); + if (header == NULL || ref == NULL || token_out == NULL + || (kind != PCM_X_TERMINAL_LEG_DRAIN && kind != PCM_X_TERMINAL_LEG_RETIRE) + || responder_node < 0 || responder_node >= PCM_X_PROTOCOL_NODE_LIMIT + || responder_session == 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (kind == PCM_X_TERMINAL_LEG_DRAIN) { + result = pcm_x_master_terminal_lookup(ref, &ticket_ref); + if (result != PCM_X_QUEUE_OK) + return result; + } else { + /* Publish the durable wire-key authority before arming a RETIRE leg. */ + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result == PCM_X_QUEUE_OK + && pcm_x_slot_state_read(&ticket->slot) != PCM_XT_RETIRE_CREDIT) + result = PCM_X_QUEUE_NOT_READY; + if (result == PCM_X_QUEUE_OK) { + /* RETIRE_CREDIT is published only after grant_generation and the + * terminal outcome are stable. No earlier state may expose a key. */ + pg_read_barrier(); + pcm_x_retire_key_from_ticket(ticket, &retire_key); + directory_result = pcm_x_directory_insert_locked(PCM_X_DIR_MASTER_TICKET_RETIRE, + &retire_key, ticket_ref, &existing); + if (directory_result == PCM_X_DIRECTORY_EXISTS) { + result = existing.slot_index == ticket_ref.slot_index + && existing.slot_generation == ticket_ref.slot_generation + ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_CORRUPT; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + } else if (directory_result != PCM_X_DIRECTORY_OK) { + result = pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + } + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; + } + } + responder_bit = UINT32_C(1) << responder_node; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) { + result = PCM_X_QUEUE_STALE; + goto arm_done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto arm_done; + } + state = pcm_x_slot_state_read(&ticket->slot); + result = pcm_x_master_terminal_outcome_exact(ticket, state); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto arm_done; + } + if (!pcm_x_master_terminal_bitmaps_valid(ticket, state)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_done; + } + if ((ticket->involved_nodes_bitmap & responder_bit) == 0) { + result = PCM_X_QUEUE_STALE; + goto arm_done; + } + frontier = &header->peer_frontiers[responder_node]; + if (frontier->cluster_epoch != ref->identity.cluster_epoch + || frontier->sender_session_incarnation != responder_session) { + result = PCM_X_QUEUE_STALE; + goto arm_done; + } + if (!pcm_x_master_terminal_leg_is_clear(&ticket->reliable)) { + if (ticket->reliable.pending_opcode == (uint16)kind + && ticket->reliable.phase == (uint16)kind + && ticket->reliable.expected_responder_node == responder_node + && ticket->reliable.expected_responder_session == responder_session + && ticket->reliable.state_sequence != 0 + && (kind != PCM_X_TERMINAL_LEG_DRAIN || ticket->drain_generation != 0)) { + token_out->state_sequence = ticket->reliable.state_sequence; + token_out->expected_responder_session = responder_session; + token_out->drain_generation + = kind == PCM_X_TERMINAL_LEG_DRAIN ? ticket->drain_generation : 0; + token_out->expected_responder_node = responder_node; + token_out->kind = (uint16)kind; + result = PCM_X_QUEUE_DUPLICATE; + } else + result = PCM_X_QUEUE_BUSY; + goto arm_done; + } + if (kind == PCM_X_TERMINAL_LEG_DRAIN) { + if (state == PCM_XT_RETIRE_CREDIT || (ticket->drained_nodes_bitmap & responder_bit) != 0) { + result = PCM_X_QUEUE_NOT_READY; + goto arm_done; + } + } else { + if (state != PCM_XT_RETIRE_CREDIT + || (ticket->retire_acked_nodes_bitmap & responder_bit) != 0) { + result = PCM_X_QUEUE_NOT_READY; + goto arm_done; + } + } + if (!cluster_pcm_x_generation_next(ticket->reliable.state_sequence, &next_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto arm_done; + } + if (kind == PCM_X_TERMINAL_LEG_DRAIN && ticket->drain_generation == 0) + ticket->drain_generation = next_sequence; + ticket->reliable.state_sequence = next_sequence; + ticket->reliable.retry_deadline_ms = 0; + ticket->reliable.expected_responder_session = responder_session; + ticket->reliable.retry_count = 0; + ticket->reliable.expected_responder_node = responder_node; + ticket->reliable.pending_opcode = (uint16)kind; + ticket->reliable.phase = (uint16)kind; + ticket->reliable.flags = 0; + pg_write_barrier(); + token_out->state_sequence = next_sequence; + token_out->expected_responder_session = responder_session; + token_out->drain_generation = kind == PCM_X_TERMINAL_LEG_DRAIN ? ticket->drain_generation : 0; + token_out->expected_responder_node = responder_node; + token_out->kind = (uint16)kind; + result = PCM_X_QUEUE_OK; + +arm_done: + LWLockRelease(&header->master_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_master_terminal_leg_ack_resolved(const PcmXTicketRef *ref, PcmXTerminalLegKind actual_kind, + int32 authenticated_node, uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXQueueResult result; + uint32 responder_bit; + uint32 partition; + uint32 state; + bool fail_closed = false; + + if (header == NULL || ref == NULL + || (actual_kind != PCM_X_TERMINAL_LEG_DRAIN && actual_kind != PCM_X_TERMINAL_LEG_RETIRE) + || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_session == 0 || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_master_terminal_lookup(ref, &ticket_ref); + if (result != PCM_X_QUEUE_OK) + return result; + responder_bit = UINT32_C(1) << authenticated_node; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (ticket == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref)) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto ack_done; + } + state = pcm_x_slot_state_read(&ticket->slot); + result = pcm_x_master_terminal_outcome_exact(ticket, state); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto ack_done; + } + if (!pcm_x_master_terminal_bitmaps_valid(ticket, state)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto ack_done; + } + if ((ticket->involved_nodes_bitmap & responder_bit) == 0) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + frontier = &header->peer_frontiers[authenticated_node]; + if (frontier->cluster_epoch != ref->identity.cluster_epoch + || frontier->sender_session_incarnation != authenticated_session) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + if (actual_kind == PCM_X_TERMINAL_LEG_DRAIN && ticket->drain_generation == 0) { + result = PCM_X_QUEUE_NOT_READY; + goto ack_done; + } + if ((actual_kind == PCM_X_TERMINAL_LEG_DRAIN ? ticket->drained_nodes_bitmap + : ticket->retire_acked_nodes_bitmap) + & responder_bit) { + /* A closed exact key replays without touching another node's pending leg. */ + result = PCM_X_QUEUE_DUPLICATE; + goto ack_done; + } + if (pcm_x_master_terminal_leg_is_clear(&ticket->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto ack_done; + } + if (ticket->reliable.pending_opcode != (uint16)actual_kind + || ticket->reliable.phase != (uint16)actual_kind + || ticket->reliable.expected_responder_node != authenticated_node + || ticket->reliable.expected_responder_session != authenticated_session) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + if ((actual_kind == PCM_X_TERMINAL_LEG_DRAIN && state != PCM_XT_COMPLETE + && state != PCM_XT_CANCELLED && state != PCM_XT_RETIRE_CREDIT) + || (actual_kind == PCM_X_TERMINAL_LEG_RETIRE && state != PCM_XT_RETIRE_CREDIT)) { + result = PCM_X_QUEUE_NOT_READY; + goto ack_done; + } + + /* Clear the durable pending leg before publishing its bitmap outcome. */ + ticket->reliable.retry_deadline_ms = 0; + ticket->reliable.expected_responder_session = 0; + ticket->reliable.retry_count = 0; + ticket->reliable.last_responder_node = (uint32)authenticated_node; + ticket->reliable.expected_responder_node = 0; + ticket->reliable.pending_opcode = PCM_X_TERMINAL_LEG_NONE; + ticket->reliable.last_response_opcode = (uint16)actual_kind; + ticket->reliable.phase = 0; + ticket->reliable.flags = 0; + pg_write_barrier(); + if (actual_kind == PCM_X_TERMINAL_LEG_DRAIN) { + ticket->drained_nodes_bitmap |= responder_bit; + if (state != PCM_XT_RETIRE_CREDIT && cluster_pcm_x_ticket_drain_ready(ticket)) { + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_RETIRE_CREDIT); + } + } else + ticket->retire_acked_nodes_bitmap |= responder_bit; + result = PCM_X_QUEUE_OK; + +ack_done: + LWLockRelease(&header->master_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_terminal_leg_ack_exact(const PcmXTicketRef *ref, + PcmXTerminalLegKind actual_kind, + int32 authenticated_node, uint64 authenticated_session) +{ + if (actual_kind != PCM_X_TERMINAL_LEG_DRAIN) + return PCM_X_QUEUE_INVALID; + return pcm_x_master_terminal_leg_ack_resolved(ref, actual_kind, authenticated_node, + authenticated_session); +} + + +static PcmXQueueResult +pcm_x_master_retire_ack_common(const PcmXRetirePayload *ack, int32 authenticated_node, + uint64 authenticated_session, PcmXTicketRef *ref_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXDirectoryResult directory_result; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + PcmXTicketRef ref; + PcmXQueueResult result; + uint32 partition; + + if (ref_out != NULL) + memset(ref_out, 0, sizeof(*ref_out)); + if (header == NULL || ack == NULL || ack->master_session_incarnation == 0 + || ack->retire_through_ticket_id == 0 || ack->sender_node < 0 + || ack->sender_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || ack->flags != 0) + return PCM_X_QUEUE_INVALID; + if (ack->sender_node != authenticated_node) + return PCM_X_QUEUE_STALE; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_RETIRE, ack, &ticket_ref); + if (directory_result == PCM_X_DIRECTORY_OK) { + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + ticket_ref); + if (ticket == NULL) { + result = PCM_X_QUEUE_CORRUPT; + } else if (!pcm_x_directory_key_matches_slot(PCM_X_DIR_MASTER_TICKET_RETIRE, ack, + &ticket->slot)) { + result = PCM_X_QUEUE_CORRUPT; + } else { + /* Only immutable locator fields are readable under allocator_lock. */ + memset(&ref, 0, sizeof(ref)); + ref.identity = ticket->ref.identity; + ref.handle = ticket->ref.handle; + result = PCM_X_QUEUE_OK; + } + } else if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + result = PCM_X_QUEUE_STALE; + else + result = pcm_x_queue_result_from_directory(directory_result); + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + /* grant_generation is owned by the master domain. RETIRE_CREDIT proves + * that its preceding publication is complete before we copy the full ref. */ + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref.identity.tag, + PCM_X_STATE_BIT(PCM_XT_RETIRE_CREDIT)); + if (ticket == NULL || !pcm_x_ticket_locator_equal(&ticket->ref, &ref)) + result = PCM_X_QUEUE_STALE; + else { + pg_read_barrier(); + ref.grant_generation = ticket->ref.grant_generation; + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->master_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + return result; + result = pcm_x_master_terminal_leg_ack_resolved(&ref, PCM_X_TERMINAL_LEG_RETIRE, + authenticated_node, authenticated_session); + if ((result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) && ref_out != NULL) + *ref_out = ref; + return result; +} + + +PcmXQueueResult +cluster_pcm_x_master_retire_ack_exact(const PcmXRetirePayload *ack, int32 authenticated_node, + uint64 authenticated_session) +{ + return pcm_x_master_retire_ack_common(ack, authenticated_node, authenticated_session, NULL); +} + + +PcmXQueueResult +cluster_pcm_x_master_retire_ack_resolve_exact(const PcmXRetirePayload *ack, + int32 authenticated_node, + uint64 authenticated_session, PcmXTicketRef *ref_out) +{ + if (ref_out == NULL) + return PCM_X_QUEUE_INVALID; + return pcm_x_master_retire_ack_common(ack, authenticated_node, authenticated_session, ref_out); +} + + +/* Return the oldest terminal ticket so the LMON cold-path retry driver cannot + * starve the global contiguous retirement frontier behind a newer slot. */ +PcmXQueueResult +cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXAllocatorView view; + PcmXMasterTicketSlot *raw; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + BufferTag tag; + uint64 generation_after; + uint64 best_ticket_id = UINT64_MAX; + Size i; + bool fail_closed = false; + + if (ref_out != NULL) + memset(ref_out, 0, sizeof(*ref_out)); + if (header == NULL || ref_out == NULL) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (!pcm_x_allocator_entry_unlocked(header) + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view)) + return PCM_X_QUEUE_INVALID; + + for (i = 0; i < view.capacity; i++) { + uint32 partition; + uint32 flags; + uint32 state; + PcmXQueueResult result; + bool release_work; + + raw = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&view, i); + if (raw == NULL) + continue; + state = pcm_x_slot_state_read(&raw->slot); + if (state != PCM_XT_COMPLETE && state != PCM_XT_CANCELLED && state != PCM_XT_RETIRE_CREDIT) + continue; + if (!pcm_x_slot_generation_read(&raw->slot, &ticket_ref.slot_generation)) + continue; + ticket_ref.slot_index = i; + tag = raw->ref.identity.tag; + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&raw->slot, &generation_after) + || generation_after != ticket_ref.slot_generation + || pcm_x_slot_state_read(&raw->slot) != state) + continue; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TICKET, ticket_ref, &tag, + PCM_X_STATE_BIT(PCM_XT_COMPLETE) | PCM_X_STATE_BIT(PCM_XT_CANCELLED) + | PCM_X_STATE_BIT(PCM_XT_RETIRE_CREDIT)); + if (ticket == NULL) { + LWLockRelease(&header->master_locks[partition].lock); + continue; + } + state = pcm_x_slot_state_read(&ticket->slot); + result = pcm_x_master_terminal_outcome_exact(ticket, state); + flags = pcm_x_slot_flags_read(&ticket->slot); + release_work = result == PCM_X_QUEUE_NOT_READY && state == PCM_XT_CANCELLED + && flags + == (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING); + if ((result != PCM_X_QUEUE_OK && !release_work) + || !pcm_x_master_terminal_bitmaps_valid(ticket, state) + || ticket->ref.handle.ticket_id == 0) { + fail_closed = result == PCM_X_QUEUE_CORRUPT + || !pcm_x_master_terminal_bitmaps_valid(ticket, state) + || ticket->ref.handle.ticket_id == 0; + LWLockRelease(&header->master_locks[partition].lock); + if (fail_closed) + break; + continue; + } + if (ticket->ref.handle.ticket_id < best_ticket_id) { + best_ticket_id = ticket->ref.handle.ticket_id; + *ref_out = ticket->ref; + } + LWLockRelease(&header->master_locks[partition].lock); + } + if (fail_closed) { + memset(ref_out, 0, sizeof(*ref_out)); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + memset(ref_out, 0, sizeof(*ref_out)); + return PCM_X_QUEUE_NOT_READY; + } + return best_ticket_id == UINT64_MAX ? PCM_X_QUEUE_NOT_FOUND : PCM_X_QUEUE_OK; +} + + +/* Prove that a terminal ticket is no longer reachable from the hot FIFO. */ +static bool +pcm_x_master_terminal_unlinked_locked(PcmXMasterTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXMasterTicketSlot *terminal, PcmXSlotRef terminal_ref) +{ + PcmXAllocatorView view; + Size current; + Size previous = PCM_X_INVALID_SLOT_INDEX; + Size visited = 0; + + if (tag_slot == NULL || terminal == NULL + || !pcm_x_allocator_view(PCM_X_ALLOC_MASTER_TICKET, &view) + || terminal->next_index != PCM_X_INVALID_SLOT_INDEX + || terminal->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->head_index == terminal_ref.slot_index + || tag_slot->tail_index == terminal_ref.slot_index + || tag_slot->active_index == terminal_ref.slot_index) + return false; + + current = tag_slot->head_index; + while (current != PCM_X_INVALID_SLOT_INDEX) { + PcmXMasterTicketSlot *linked; + PcmXSlotRef linked_ref; + + if (current == terminal_ref.slot_index || visited >= PCM_X_PROTOCOL_NODE_LIMIT) + return false; + linked = (PcmXMasterTicketSlot *)pcm_x_allocator_slot(&view, current); + if (linked == NULL) + return false; + linked_ref.slot_index = current; + if (!pcm_x_slot_generation_read(&linked->slot, &linked_ref.slot_generation)) + return false; + linked = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, linked_ref, + &tag_slot->tag, + PCM_X_MASTER_TICKET_DOMAIN_STATES); + if (linked == NULL || linked->tag_slot_index != tag_ref.slot_index + || linked->tag_slot_generation != tag_ref.slot_generation + || linked->prev_index != previous) + return false; + previous = current; + current = linked->next_index; + visited++; + } + if (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + return tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX; + return tag_slot->tail_index == previous; +} + + +PcmXQueueResult +cluster_pcm_x_master_detach_terminal_exact(const PcmXTicketRef *ref) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef found; + PcmXSlotRef tag_ref; + PcmXSlotRef ticket_ref; + PcmXPrehandleKey prehandle; + PcmXRetirePayload retire_key; + PcmXTicketHandle blocker_owner; + PcmXQueueResult result; + Size blocker_head = PCM_X_INVALID_SLOT_INDEX; + uint64 blocker_generation = 0; + uint32 blocker_count = 0; + uint32 expected_gate; + uint32 partition; + bool detach_tag = false; + bool hot_queue_empty; + bool release_blockers = false; + + if (header == NULL || ref == NULL || !pcm_x_wait_identity_valid(&ref->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + /* First prove consecutive retirement and capture immutable slot identities. */ + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + result = pcm_x_master_ticket_lookup_locked(ref, &ticket_ref, &ticket); + if (result != PCM_X_QUEUE_OK) + goto allocator_preflight_done; + if (pcm_x_slot_state_read(&ticket->slot) != PCM_XT_RETIRE_CREDIT) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_preflight_done; + } + if (header->fully_retired_ticket_id == UINT64_MAX + || ref->handle.ticket_id != header->fully_retired_ticket_id + 1) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_preflight_done; + } + prehandle = ticket->prehandle; + frontier = &header->peer_frontiers[ref->identity.node_id]; + if (frontier->cluster_epoch != ref->identity.cluster_epoch + || frontier->sender_session_incarnation != prehandle.sender_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto allocator_preflight_done; + } + if (frontier->retired_prehandle_sequence == UINT64_MAX + || prehandle.prehandle_sequence != frontier->retired_prehandle_sequence + 1) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_preflight_done; + } + tag_ref.slot_index = ticket->tag_slot_index; + tag_ref.slot_generation = ticket->tag_slot_generation; + result = PCM_X_QUEUE_OK; + +allocator_preflight_done: + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + /* Quiesce under the tag domain before allocator-owned reclamation. */ + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + ticket = (PcmXMasterTicketSlot *)pcm_x_domain_slot(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, + &ref->identity.tag, + PCM_X_STATE_BIT(PCM_XT_RETIRE_CREDIT)); + tag_slot = (PcmXMasterTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_MASTER_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (ticket == NULL || tag_slot == NULL || !pcm_x_ticket_ref_equal(&ticket->ref, ref) + || ticket->tag_slot_index != tag_ref.slot_index + || ticket->tag_slot_generation != tag_ref.slot_generation) { + result = PCM_X_QUEUE_STALE; + goto domain_done; + } + if (!pcm_x_runtime_token_exact(&runtime, ticket->master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto domain_done; + } + result = pcm_x_master_terminal_outcome_exact(ticket, pcm_x_slot_state_read(&ticket->slot)); + if (result != PCM_X_QUEUE_OK) + goto domain_done; + if (!pcm_x_master_terminal_leg_is_clear(&ticket->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto domain_done; + } + if (!pcm_x_master_blocker_stage_metadata_valid(ticket) + || !pcm_x_master_blocker_stage_is_clear(ticket) + || !pcm_x_master_blocker_published_source_valid(ticket)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (ticket->blocker_set_generation == 0) { + if (ticket->blocker_head_index != PCM_X_INVALID_SLOT_INDEX || ticket->blocker_count != 0 + || ticket->blocker_set_crc32c != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + } else { + PcmXAllocatorView blocker_view; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_blocker_chain_validate( + &blocker_view, ticket->blocker_head_index, ticket->blocker_count, + &ticket->ref.handle, ticket_ref.slot_index, ticket_ref.slot_generation, + ticket->blocker_set_generation, PCM_XB_LIVE, NULL, 0, NULL)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + } + if (ticket->involved_nodes_bitmap == 0 + || ticket->drained_nodes_bitmap != ticket->involved_nodes_bitmap + || (ticket->retire_acked_nodes_bitmap & ~ticket->involved_nodes_bitmap) != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (!cluster_pcm_x_ticket_retire_ready(ticket)) { + result = PCM_X_QUEUE_NOT_READY; + goto domain_done; + } + if (!pcm_x_master_terminal_unlinked_locked(tag_slot, tag_ref, ticket, ticket_ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (tag_slot->outstanding_ticket_count == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (tag_slot->outstanding_ticket_count == 1) { + /* First exclude an allocator-stage admission, then inspect hot state. */ + expected_gate = 0; + if (!pg_atomic_compare_exchange_u32(&tag_slot->admission_gate, &expected_gate, 2)) { + result = PCM_X_QUEUE_BUSY; + goto domain_done; + } + pg_read_barrier(); + hot_queue_empty = tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_index == PCM_X_INVALID_SLOT_INDEX + && pg_atomic_read_u32(&tag_slot->queued_node_bitmap) == 0; + if (tag_slot->outstanding_ticket_count != 1 || !hot_queue_empty) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); + detach_tag = true; + } + if (ticket->blocker_count != 0) { + PcmXAllocatorView blocker_view; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + blocker_owner = ticket->ref.handle; + blocker_head = ticket->blocker_head_index; + blocker_count = ticket->blocker_count; + blocker_generation = ticket->blocker_set_generation; + pcm_x_blocker_chain_state_write(&blocker_view, blocker_head, blocker_count, + PCM_XB_DETACHING); + release_blockers = true; + } + tag_slot->outstanding_ticket_count--; + pg_write_barrier(); + pcm_x_slot_state_write(&ticket->slot, PCM_XT_DETACHING); + result = PCM_X_QUEUE_OK; + +domain_done: + /* A post-CAS jump reaches here only for CORRUPT and retains gate value 2. */ + LWLockRelease(&header->master_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + /* Advance replay authority first, then remove exact indexes and free slots. */ + pcm_x_master_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, tag_slot, + detach_tag ? 2 : 0); + ticket = (PcmXMasterTicketSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TICKET, + ticket_ref); + tag_slot = (PcmXMasterTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_MASTER_TAG, tag_ref); + frontier = &header->peer_frontiers[ref->identity.node_id]; + if (ticket == NULL || tag_slot == NULL + || pcm_x_slot_state_read(&ticket->slot) != PCM_XT_DETACHING + || (detach_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_DETACHING) + || (!detach_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE) + || (detach_tag && tag_slot->outstanding_ticket_count != 0) + || !pcm_x_ticket_ref_equal(&ticket->ref, ref) + || !pcm_x_prehandle_equal(&ticket->prehandle, &prehandle) + || header->fully_retired_ticket_id == UINT64_MAX + || ref->handle.ticket_id != header->fully_retired_ticket_id + 1 + || frontier->cluster_epoch != ref->identity.cluster_epoch + || frontier->sender_session_incarnation != prehandle.sender_session_incarnation + || frontier->retired_prehandle_sequence == UINT64_MAX + || prehandle.prehandle_sequence != frontier->retired_prehandle_sequence + 1) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_detach_done; + } + if (pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &prehandle, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != ticket_ref.slot_index + || found.slot_generation != ticket_ref.slot_generation + || pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, &ref->handle, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != ticket_ref.slot_index + || found.slot_generation != ticket_ref.slot_generation + || (detach_tag + && (pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TAG, &ref->identity.tag, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != tag_ref.slot_index + || found.slot_generation != tag_ref.slot_generation))) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_detach_done; + } + pcm_x_retire_key_from_ticket(ticket, &retire_key); + if (pcm_x_directory_find_locked(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire_key, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != ticket_ref.slot_index + || found.slot_generation != ticket_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_detach_done; + } + /* Preserve the allocator's validate-before-first-free discipline: every + * terminal directory locator must be exact before reclaiming a retained + * published blocker chain. */ + if (release_blockers + && !pcm_x_blocker_chain_release_locked_exact( + blocker_head, blocker_count, &blocker_owner, ticket_ref.slot_index, + ticket_ref.slot_generation, blocker_generation, PCM_XB_DETACHING)) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_detach_done; + } + header->fully_retired_ticket_id = ref->handle.ticket_id; + frontier->retired_prehandle_sequence = prehandle.prehandle_sequence; + pg_write_barrier(); + if (pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &prehandle, + ticket_ref) + != PCM_X_DIRECTORY_OK + || pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TICKET_HANDLE, &ref->handle, + ticket_ref) + != PCM_X_DIRECTORY_OK + || pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire_key, + ticket_ref) + != PCM_X_DIRECTORY_OK + || (detach_tag + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_MASTER_TAG, &ref->identity.tag, + tag_ref) + != PCM_X_DIRECTORY_OK) + || pcm_x_allocator_release_locked(PCM_X_ALLOC_MASTER_TICKET, ticket_ref, PCM_XT_DETACHING) + != PCM_X_ALLOC_OK + || (detach_tag + && pcm_x_allocator_release_locked(PCM_X_ALLOC_MASTER_TAG, tag_ref, PCM_X_TAG_DETACHING) + != PCM_X_ALLOC_OK)) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_detach_done; + } + result = PCM_X_QUEUE_OK; + +allocator_detach_done: + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool pcm_x_local_gate_release(PcmXLocalTagSlot *tag_slot); + + +static PcmXQueueResult +pcm_x_local_gate_try_acquire_mode(PcmXLocalTagSlot *tag_slot, bool retire_protocol) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + uint32 retire_gate; + uint32 desired; + uint32 flags; + + if (header == NULL || tag_slot == NULL) + return PCM_X_QUEUE_CORRUPT; + /* RETIRE publishes this global gate while holding allocator_lock. Ordinary + * tag-only mutators check on both sides of their tag CAS: if they win first, + * RETIRE's quiescence scan sees the tag gate; if RETIRE wins between checks, + * they release without touching domain state. */ + retire_gate = pg_atomic_read_u32(&header->local_retire_gate); + if ((!retire_protocol && retire_gate != 0) + || (retire_protocol + && (tag_slot->master_node < 0 || retire_gate != (uint32)tag_slot->master_node + 1))) + return retire_protocol && tag_slot->master_node < 0 ? PCM_X_QUEUE_CORRUPT + : PCM_X_QUEUE_BUSY; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + for (;;) { + if ((flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) + return PCM_X_QUEUE_BUSY; + desired = flags | PCM_X_LOCAL_TAG_F_ADMISSION_GATE; + if (pcm_x_slot_flags_compare_exchange(&tag_slot->slot, &flags, desired)) { + pg_read_barrier(); + retire_gate = pg_atomic_read_u32(&header->local_retire_gate); + if ((!retire_protocol && retire_gate != 0) + || (retire_protocol && retire_gate != (uint32)tag_slot->master_node + 1)) { + if (!pcm_x_local_gate_release(tag_slot)) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_BUSY; + } + return PCM_X_QUEUE_OK; + } + } +} + + +static PcmXQueueResult +pcm_x_local_gate_try_acquire(PcmXLocalTagSlot *tag_slot) +{ + return pcm_x_local_gate_try_acquire_mode(tag_slot, false); +} + + +static PcmXQueueResult +pcm_x_local_retire_gate_try_acquire(PcmXLocalTagSlot *tag_slot) +{ + return pcm_x_local_gate_try_acquire_mode(tag_slot, true); +} + + +static bool +pcm_x_local_gate_release(PcmXLocalTagSlot *tag_slot) +{ + uint32 desired; + uint32 flags; + + pg_write_barrier(); + flags = pcm_x_slot_flags_read(&tag_slot->slot); + for (;;) { + if ((flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) + return false; + desired = flags & ~PCM_X_LOCAL_TAG_F_ADMISSION_GATE; + if (pcm_x_slot_flags_compare_exchange(&tag_slot->slot, &flags, desired)) + return true; + } +} + + +/* RETIRE scans and detaches more than one local tag under a monotonic peer + * watermark. Cross-lock admissions publish ADMISSION_GATE while moving from + * allocator_lock to a tag lock; serialize those handoffs with the whole + * retirement episode so a legal register/join window can never be mistaken + * for post-preflight corruption. Caller holds allocator_lock. */ +static PcmXQueueResult +pcm_x_local_retire_episode_state_locked(PcmXShmemHeader *header) +{ + uint32 retire_gate; + int marker_node = -1; + int marker_count = 0; + int i; + + Assert(header != NULL); + Assert(LWLockHeldByMe(&header->allocator_lock.lock)); + retire_gate = pg_atomic_read_u32(&header->local_retire_gate); + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + if (header->peer_frontiers[i].local_retire_in_progress_ticket_id != 0) { + marker_node = i; + marker_count++; + } + } + if (retire_gate == 0 && marker_count == 0) + return PCM_X_QUEUE_OK; + if (retire_gate == 0 || retire_gate > PCM_X_PROTOCOL_NODE_LIMIT || marker_count != 1 + || marker_node != (int)retire_gate - 1 + || header->peer_frontiers[marker_node].local_retire_in_progress_ticket_id + <= header->peer_frontiers[marker_node].local_retired_ticket_id) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_BUSY; +} + + +/* Order every allocator<->tag handoff against RETIRE without nesting LWLocks. + * The caller holds allocator_lock; claiming the atomic tag gate before that + * lock is released means RETIRE must observe either this gate or its own + * already-published episode marker. */ +static PcmXQueueResult +pcm_x_local_handoff_gate_claim_locked(PcmXShmemHeader *header, PcmXLocalTagSlot *tag_slot, + bool retire_protocol, PcmXQueueResult contention_result) +{ + PcmXQueueResult result; + + Assert(header != NULL); + Assert(tag_slot != NULL); + Assert(LWLockHeldByMe(&header->allocator_lock.lock)); + Assert(contention_result == PCM_X_QUEUE_BUSY || contention_result == PCM_X_QUEUE_GATE_RETRY); + if (!retire_protocol) { + result = pcm_x_local_retire_episode_state_locked(header); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_BUSY ? contention_result : result; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + return result == PCM_X_QUEUE_BUSY ? contention_result : result; +} + + +/* Prove that no admission was already between allocator publication and its + * tag-domain commit before claiming a RETIRE episode. Once the marker is + * published, register/join refuse to create a new such window. */ +static PcmXQueueResult +pcm_x_local_admission_handoffs_quiescent_locked(PcmXShmemHeader *header) +{ + PcmXAllocatorView view; + PcmXLocalTagSlot *tag_slot; + Size i; + uint32 flags; + uint32 state; + + Assert(header != NULL); + Assert(LWLockHeldByMe(&header->allocator_lock.lock)); + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view)) + return PCM_X_QUEUE_CORRUPT; + for (i = 0; i < view.capacity; i++) { + tag_slot = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&view, i); + if (tag_slot == NULL) + return PCM_X_QUEUE_CORRUPT; + state = pcm_x_slot_state_read(&tag_slot->slot); + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (state == PCM_X_SLOT_FREE || state == PCM_X_SLOT_GENERATION_EXHAUSTED) { + if (flags != 0) + return PCM_X_QUEUE_CORRUPT; + continue; + } + if ((flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) + return PCM_X_QUEUE_BUSY; + } + return PCM_X_QUEUE_OK; +} + + +/* Local-domain twin of pcm_x_master_gate_acquire_guarded(). */ +static void +pcm_x_local_gate_acquire_guarded(LWLock *lock, LWLockMode mode, PcmXLocalTagSlot *tag_slot) +{ + PG_TRY(); + { + LWLockAcquire(lock, mode); + } + PG_CATCH(); + { + pcm_x_runtime_fail_closed(); + if (LWLockHeldByMe(lock)) + LWLockRelease(lock); + if (tag_slot != NULL) + (void)pcm_x_local_gate_release(tag_slot); + PG_RE_THROW(); + } + PG_END_TRY(); +} + + +/* + * The final rekey phase starts after the target directory key is durable. + * An acquire ERROR there must retain ADMISSION_GATE as recovery evidence; + * releasing it would expose the intentional target-key/old-resident window. + */ +static void +pcm_x_local_gate_acquire_guarded_retain_gate(LWLock *lock, LWLockMode mode) +{ + PG_TRY(); + { + LWLockAcquire(lock, mode); + } + PG_CATCH(); + { + pcm_x_runtime_fail_closed(); + if (LWLockHeldByMe(lock)) + LWLockRelease(lock); + PG_RE_THROW(); + } + PG_END_TRY(); +} + + +static bool +pcm_x_local_stage_rollback_locked(bool new_tag, bool tag_directory_published, + bool member_directory_published, const BufferTag *tag, + const PcmXWaitIdentity *identity, PcmXSlotRef tag_ref, + PcmXLocalMembershipSlot *member, PcmXSlotRef member_ref) +{ + bool ok = true; + + if (member_directory_published + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_WAIT, identity, member_ref) + != PCM_X_DIRECTORY_OK) + ok = false; + if (member != NULL + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + PCM_XL_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + ok = false; + if (tag_directory_published + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, tag, tag_ref) + != PCM_X_DIRECTORY_OK) + ok = false; + if (new_tag + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + PCM_X_TAG_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + ok = false; + return ok; +} + + +static PcmXQueueResult +pcm_x_local_member_lookup_locked(const PcmXWaitIdentity *identity, PcmXSlotRef *member_ref_out, + PcmXLocalMembershipSlot **member_out) +{ + PcmXDirectoryView directory; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXDirectoryMatch match; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || identity == NULL || member_ref_out == NULL || member_out == NULL + || !LWLockHeldByMe(&header->allocator_lock.lock) + || !cluster_pcm_x_directory_key_hash(PCM_X_DIR_LOCAL_WAIT, identity, &key_hash) + || !pcm_x_directory_view(PCM_X_DIR_LOCAL_WAIT, &directory)) + return PCM_X_QUEUE_INVALID; + bucket = key_hash % directory.capacity; + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) + return PCM_X_QUEUE_NOT_FOUND; + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return pcm_x_queue_result_from_directory(PCM_X_DIRECTORY_CORRUPT); + if (entry->key_hash != key_hash) + continue; + member_ref.slot_index = entry->slot_index; + member_ref.slot_generation = entry->slot_generation; + member = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_WAIT, + member_ref); + if (member != NULL) { + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + tag_slot + = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + /* Do not read the resident identity while the local-lock rekey + * writer may be replacing it. A hash collision is conservatively + * retryable until the gate clears and exact matching resumes. The + * release side publishes identity before clearing the gate; pair its + * write barrier here before dereferencing the resident checksum. */ + if (tag_slot != NULL) { + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) + != 0) + return PCM_X_QUEUE_BUSY; + pg_read_barrier(); + } + } + match + = pcm_x_directory_entry_match_locked(PCM_X_DIR_LOCAL_WAIT, &directory, entry, identity); + if (match == PCM_X_DIRECTORY_MATCH_EXACT) { + if (member == NULL) + return pcm_x_queue_result_from_directory(PCM_X_DIRECTORY_STALE_REF); + *member_ref_out = member_ref; + *member_out = member; + return PCM_X_QUEUE_OK; + } + if (match == PCM_X_DIRECTORY_MATCH_STALE) + return pcm_x_queue_result_from_directory(PCM_X_DIRECTORY_STALE_REF); + if (match == PCM_X_DIRECTORY_MATCH_CORRUPT) + return pcm_x_queue_result_from_directory(PCM_X_DIRECTORY_CORRUPT); + } + return PCM_X_QUEUE_NOT_FOUND; +} + + +static bool +pcm_x_local_handle_exact(const PcmXLocalHandle *handle, PcmXSlotRef member_ref, + const PcmXLocalMembershipSlot *member, const PcmXLocalTagSlot *tag_slot) +{ + uint32 round; + uint32 flags; + + if (tag_slot == NULL) + return false; + round = member->admitted_round; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) { + if (member->local_sequence <= tag_slot->cutoff_sequence) { + if (round != tag_slot->local_round) + return false; + } else if (tag_slot->local_round == UINT32_MAX || round != tag_slot->local_round + 1) + return false; + } else if (round != tag_slot->local_round) + return false; + return pcm_x_wait_identity_equal(&handle->identity, &member->identity) + && handle->tag_slot.slot_index == member->tag_slot_index + && handle->tag_slot.slot_generation == member->tag_slot_generation + && handle->membership_slot.slot_index == member_ref.slot_index + && handle->membership_slot.slot_generation == member_ref.slot_generation + && handle->local_sequence == member->local_sequence && handle->local_round == round + && handle->role == member->role; +} + + +/* Allocator cleanup normally reuses the execution-round validator above. A + * final terminal RETIRE is the one exception: its local-lock transaction has + * already advanced the tag to the next round while the old member remains + * DETACHING until allocator cleanup. Revalidate only the immutable locator + * facts for that exact tombstone; applying current-round semantics to it would + * misclassify every successful close as corruption. */ +static bool +pcm_x_local_detaching_handle_exact(const PcmXLocalHandle *handle, PcmXSlotRef member_ref, + const PcmXLocalMembershipSlot *member, + const PcmXLocalTagSlot *tag_slot) +{ + return handle != NULL && member != NULL && tag_slot != NULL + && BufferTagsEqual(&handle->identity.tag, &tag_slot->tag) + && handle->identity.cluster_epoch == tag_slot->cluster_epoch + && pcm_x_wait_identity_equal(&handle->identity, &member->identity) + && handle->tag_slot.slot_index == member->tag_slot_index + && handle->tag_slot.slot_generation == member->tag_slot_generation + && handle->membership_slot.slot_index == member_ref.slot_index + && handle->membership_slot.slot_generation == member_ref.slot_generation + && handle->local_sequence == member->local_sequence + && handle->local_round == member->admitted_round && handle->role == member->role; +} + + +/* A revoke barrier freezes one immutable cohort; it does not stop that + * cohort from draining in FIFO order. Only post-cutoff members wait for + * exact round retirement. */ +static PcmXQueueResult +pcm_x_local_execution_round_locked(const PcmXLocalTagSlot *tag_slot, + const PcmXLocalMembershipSlot *member) +{ + uint32 flags = pcm_x_slot_flags_read(&tag_slot->slot); + + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) + return member->admitted_round == tag_slot->local_round ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_CORRUPT; + if (tag_slot->local_round == UINT32_MAX + || (tag_slot->cutoff_sequence == 0 && tag_slot->closed_round_member_count != 0)) + return PCM_X_QUEUE_CORRUPT; + if (member->admitted_round == tag_slot->local_round) + return member->local_sequence <= tag_slot->cutoff_sequence ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_CORRUPT; + if (member->admitted_round == tag_slot->local_round + 1) + return member->local_sequence > tag_slot->cutoff_sequence ? PCM_X_QUEUE_BARRIER_CLOSED + : PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_CORRUPT; +} + + +static void +pcm_x_local_holder_handle_clear(PcmXLocalHolderHandle *handle) +{ + if (handle == NULL) + return; + memset(handle, 0, sizeof(*handle)); + handle->tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + handle->holder_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static void +pcm_x_local_holder_snapshot_clear(PcmXLocalHolderSnapshot *snapshot) +{ + if (snapshot == NULL) + return; + memset(snapshot, 0, sizeof(*snapshot)); + snapshot->tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; +} + + +static bool +pcm_x_local_holder_key_valid(const PcmXLocalHolderKey *key) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + + return header != NULL && key != NULL && key->reserved == 0 + && pcm_x_wait_identity_valid(&key->identity) && key->buffer_id >= 0 + && (Size)key->buffer_id < header->layout.n_buffers; +} + + +static bool +pcm_x_local_holder_key_equal(const PcmXLocalHolderKey *left, const PcmXLocalHolderKey *right) +{ + return left != NULL && right != NULL && left->reserved == 0 && right->reserved == 0 + && left->buffer_id == right->buffer_id + && pcm_x_wait_identity_equal(&left->identity, &right->identity); +} + + +static bool +pcm_x_local_holder_state_is_occupancy(uint32 state) +{ + return state < 32 && (PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES & PCM_X_STATE_BIT(state)) != 0; +} + + +static void +pcm_x_local_holder_handle_from_member(PcmXLocalHolderHandle *handle, PcmXSlotRef tag_ref, + PcmXSlotRef holder_ref, const PcmXLocalMembershipSlot *member) +{ + pcm_x_local_holder_handle_clear(handle); + handle->key.identity = member->identity; + handle->key.buffer_id = member->buffer_id; + handle->tag_slot = tag_ref; + handle->holder_slot = holder_ref; +} + + +static bool +pcm_x_local_holder_handle_exact(const PcmXLocalHolderHandle *handle, PcmXSlotRef tag_ref, + PcmXSlotRef holder_ref, const PcmXLocalMembershipSlot *member) +{ + PcmXLocalHolderKey resident; + + if (handle == NULL || member == NULL || handle->tag_slot.slot_index != tag_ref.slot_index + || handle->tag_slot.slot_generation != tag_ref.slot_generation + || handle->holder_slot.slot_index != holder_ref.slot_index + || handle->holder_slot.slot_generation != holder_ref.slot_generation + || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || member->partition != PCM_X_LOCAL_PARTITION_HOLDER) + return false; + memset(&resident, 0, sizeof(resident)); + resident.identity = member->identity; + resident.buffer_id = member->buffer_id; + return pcm_x_local_holder_key_equal(&handle->key, &resident); +} + + +static void +pcm_x_local_tag_init_common(PcmXLocalTagSlot *tag_slot, const BufferTag *tag, uint64 cluster_epoch, + int32 master_node, uint64 master_session_incarnation) +{ + tag_slot->tag = *tag; + tag_slot->master_node = master_node; + tag_slot->cluster_epoch = cluster_epoch; + tag_slot->master_session_incarnation = master_session_incarnation; + tag_slot->next_sequence = 1; + tag_slot->writer_claim_generation = 0; + tag_slot->head_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->tail_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->leader_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_writer_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_holder_head_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->blocker_snapshot_head_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->blocker_snapshot_count = 0; + tag_slot->blocker_snapshot_crc32c = 0; + tag_slot->blocker_snapshot_next_chunk = 0; + tag_slot->local_round = 1; + tag_slot->membership_count = 0; + tag_slot->closed_round_member_count = 0; + tag_slot->terminal_drain_generation = 0; + tag_slot->committed_own_generation = 0; + memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); + memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); + tag_slot->holder_terminal_drain_generation = 0; + memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); + memset(&tag_slot->blocker_snapshot_reliable, 0, sizeof(tag_slot->blocker_snapshot_reliable)); + pcm_x_slot_flags_write(&tag_slot->slot, PCM_X_LOCAL_TAG_F_ADMISSION_GATE); +} + + +static bool +pcm_x_local_holder_stage_rollback_locked(bool new_tag, bool tag_directory_published, + const PcmXLocalHolderKey *key, PcmXSlotRef tag_ref, + PcmXLocalMembershipSlot *holder, PcmXSlotRef holder_ref) +{ + bool ok = true; + + if (holder != NULL + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_HOLDER, holder_ref, + PCM_X_SLOT_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + ok = false; + if (tag_directory_published + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, &key->identity.tag, tag_ref) + != PCM_X_DIRECTORY_OK) + ok = false; + if (new_tag + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + PCM_X_TAG_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + ok = false; + return ok; +} + + +static PcmXLocalMembershipSlot * +pcm_x_local_holder_slot_resolve(const BufferTag *tag, PcmXSlotRef tag_ref, Size holder_index, + PcmXSlotRef *holder_ref_out) +{ + PcmXAllocatorView holder_view; + PcmXLocalMembershipSlot *holder; + uint64 generation; + + if (holder_ref_out == NULL || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_HOLDER, &holder_view)) + return NULL; + holder = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&holder_view, holder_index); + if (holder == NULL || !pcm_x_slot_generation_read(&holder->slot, &generation)) + return NULL; + holder_ref_out->slot_index = holder_index; + holder_ref_out->slot_generation = generation; + holder = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_HOLDER, *holder_ref_out, + tag, PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES); + if (holder == NULL || holder->partition != PCM_X_LOCAL_PARTITION_HOLDER + || holder->tag_slot_index != tag_ref.slot_index + || holder->tag_slot_generation != tag_ref.slot_generation || holder->buffer_id < 0 + || (Size)holder->buffer_id >= ClusterPcmXConvertShmem->layout.n_buffers + || !pcm_x_wait_identity_valid(&holder->identity)) + return NULL; + return holder; +} + + +/* Resolve one linked holder under the BufferTag local-domain lock. */ +static PcmXLocalMembershipSlot * +pcm_x_local_holder_link_resolve(const BufferTag *tag, PcmXSlotRef tag_ref, Size holder_index, + Size expected_previous, PcmXSlotRef *holder_ref_out) +{ + PcmXLocalMembershipSlot *holder; + + holder = pcm_x_local_holder_slot_resolve(tag, tag_ref, holder_index, holder_ref_out); + if (holder == NULL || holder->prev_index != expected_previous) + return NULL; + return holder; +} + + +static bool +pcm_x_local_writer_claim_well_formed(const PcmXLocalWriterClaim *claim) +{ + return claim != NULL && claim->flags == 0 && claim->writer.flags == 0 + && claim->active_slot.slot_index != PCM_X_INVALID_SLOT_INDEX + && claim->active_slot.slot_generation != 0 && claim->claim_generation != 0 + && claim->local_round != 0 && claim->role == claim->writer.role + && (claim->role == PCM_X_LOCAL_ROLE_NODE_LEADER + || claim->role == PCM_X_LOCAL_ROLE_FOLLOWER) + && pcm_x_wait_identity_valid(&claim->writer.identity); +} + + +static bool +pcm_x_local_writer_holder_key_bound(const PcmXLocalHolderKey *key, + const PcmXLocalWriterClaim *claim) +{ + return BufferTagsEqual(&key->identity.tag, &claim->writer.identity.tag) + && key->identity.node_id == claim->writer.identity.node_id + && key->identity.procno == claim->writer.identity.procno + && key->identity.xid == claim->writer.identity.xid + && key->identity.cluster_epoch == claim->writer.identity.cluster_epoch; +} + + +static PcmXQueueResult +pcm_x_local_writer_holder_authority_check(const PcmXLocalHolderKey *key, + const PcmXLocalWriterClaim *claim, PcmXSlotRef tag_ref, + PcmXLocalTagSlot *tag_slot, PcmXSlotRef member_ref, + PcmXLocalMembershipSlot *member, PcmXSlotRef leader_ref, + PcmXLocalMembershipSlot *leader, + uint64 *committed_own_generation_out) +{ + PcmXQueueResult result; + uint64 expected_generation; + uint32 leader_state; + uint32 member_state; + uint32 tag_flags; + + *committed_own_generation_out = 0; + if (!pcm_x_local_writer_holder_key_bound(key, claim)) + return PCM_X_QUEUE_STALE; + if (member == NULL || !pcm_x_local_handle_exact(&claim->writer, member_ref, member, tag_slot)) + return PCM_X_QUEUE_STALE; + if (claim->writer.tag_slot.slot_index != tag_ref.slot_index + || claim->writer.tag_slot.slot_generation != tag_ref.slot_generation + || claim->writer.membership_slot.slot_index != member_ref.slot_index + || claim->writer.membership_slot.slot_generation != member_ref.slot_generation + || claim->claim_generation != tag_slot->writer_claim_generation + || tag_slot->active_writer_index != member_ref.slot_index + || tag_slot->active_writer_slot_generation != member_ref.slot_generation + || claim->local_round != member->admitted_round || claim->role != member->role + || (pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) + return PCM_X_QUEUE_STALE; + result = pcm_x_local_execution_round_locked(tag_slot, member); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_CORRUPT ? result : PCM_X_QUEUE_STALE; + tag_flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((tag_flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) + return PCM_X_QUEUE_NOT_READY; + if (leader == NULL || leader->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || leader->tag_slot_index != tag_ref.slot_index + || leader->tag_slot_generation != tag_ref.slot_generation + || leader->admitted_round != tag_slot->local_round + || tag_slot->head_index != leader_ref.slot_index) + return PCM_X_QUEUE_CORRUPT; + member_state = pcm_x_slot_state_read(&member->slot); + if ((member->role == PCM_X_LOCAL_ROLE_NODE_LEADER + && (member_ref.slot_index != leader_ref.slot_index + || member_ref.slot_generation != leader_ref.slot_generation)) + || (member->role == PCM_X_LOCAL_ROLE_FOLLOWER + && (member_state != PCM_XL_JOINED_NONWAITABLE || member->graph_generation != 0))) + return PCM_X_QUEUE_STALE; + leader_state = pcm_x_slot_state_read(&leader->slot); + if (leader_state != PCM_XL_GRANTED) + return PCM_X_QUEUE_NOT_READY; + if (member->role == PCM_X_LOCAL_ROLE_NODE_LEADER && member_state != PCM_XL_GRANTED) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->ref.handle.ticket_id == 0 || tag_slot->ref.handle.queue_generation == 0 + || tag_slot->ref.grant_generation == 0 + || !pcm_x_wait_identity_equal(&tag_slot->ref.identity, &leader->identity) + || tag_slot->committed_own_generation == 0 + || !cluster_pcm_x_generation_next(leader->identity.base_own_generation, + &expected_generation) + || tag_slot->committed_own_generation != expected_generation) + return PCM_X_QUEUE_CORRUPT; + if (key->identity.base_own_generation != tag_slot->committed_own_generation) + return PCM_X_QUEUE_STALE; + *committed_own_generation_out = tag_slot->committed_own_generation; + return PCM_X_QUEUE_OK; +} + + +/* Validate the active writer while allocator ownership prevents slot reuse. */ +static PcmXQueueResult +pcm_x_local_writer_holder_authority_allocator_locked(const PcmXLocalHolderKey *key, + const PcmXLocalWriterClaim *claim, + PcmXSlotRef tag_ref, + PcmXLocalTagSlot *tag_slot, + uint64 *committed_own_generation_out) +{ + PcmXLocalMembershipSlot *leader; + PcmXLocalMembershipSlot *member; + PcmXSlotRef leader_ref; + PcmXSlotRef member_ref = claim->active_slot; + + *committed_own_generation_out = 0; + if ((tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_slot_generation == 0)) + return PCM_X_QUEUE_CORRUPT; + leader_ref.slot_index = tag_slot->leader_index; + leader_ref.slot_generation = tag_slot->leader_slot_generation; + member = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_WAIT, + member_ref); + leader = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_WAIT, + leader_ref); + return pcm_x_local_writer_holder_authority_check(key, claim, tag_ref, tag_slot, member_ref, + member, leader_ref, leader, + committed_own_generation_out); +} + + +/* Revalidate the same proof in the local-domain link transaction. */ +static PcmXQueueResult +pcm_x_local_writer_holder_authority_domain_locked(const PcmXLocalHolderKey *key, + const PcmXLocalWriterClaim *claim, + PcmXSlotRef tag_ref, PcmXLocalTagSlot *tag_slot, + uint64 *committed_own_generation_out) +{ + PcmXLocalMembershipSlot *leader; + PcmXLocalMembershipSlot *member; + PcmXSlotRef leader_ref; + PcmXSlotRef member_ref = claim->active_slot; + + *committed_own_generation_out = 0; + if ((tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_slot_generation == 0)) + return PCM_X_QUEUE_CORRUPT; + leader_ref.slot_index = tag_slot->leader_index; + leader_ref.slot_generation = tag_slot->leader_slot_generation; + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &key->identity.tag, PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + leader = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, leader_ref, &key->identity.tag, PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + return pcm_x_local_writer_holder_authority_check(key, claim, tag_ref, tag_slot, member_ref, + member, leader_ref, leader, + committed_own_generation_out); +} + + +/* + * Prepare one prospective content-lock holder before taking the content lock. + * Allocator, directory, and local-domain publication all complete here; the + * linked ACQUIRING slot is already barrier occupancy before this returns. + */ +static PcmXQueueResult +pcm_x_local_holder_register_common(const PcmXLocalHolderKey *key, + const PcmXLocalWriterClaim *writer_claim, + PcmXLocalHolderHandle *handle_out, + uint64 *committed_own_generation_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot = NULL; + PcmXLocalMembershipSlot *holder = NULL; + PcmXSlotHeader *raw_slot; + PcmXSlotRef tag_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef holder_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef existing; + PcmXDirectoryResult directory_result; + PcmXAllocatorResult allocator_result; + PcmXQueueResult result = PCM_X_QUEUE_OK; + uint64 committed_own_generation = 0; + uint64 next_holder_generation; + uint32 partition; + bool new_tag = false; + bool writer_authorized = false; + bool gate_claimed = false; + bool tag_directory_published = false; + bool fail_closed = false; + + pcm_x_local_holder_handle_clear(handle_out); + if (committed_own_generation_out != NULL) + *committed_own_generation_out = 0; + if (header == NULL || handle_out == NULL || !pcm_x_local_holder_key_valid(key) + || (writer_claim != NULL + && (committed_own_generation_out == NULL + || !pcm_x_local_writer_claim_well_formed(writer_claim)))) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_runtime_token_exact(&runtime, runtime.master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_done; + } + if (writer_claim != NULL) { + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &key->identity.tag, &tag_ref); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_NOT_FOUND + ? PCM_X_QUEUE_STALE + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + tag_slot + = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + result = pcm_x_resident_tag_state_result(tag_slot == NULL ? NULL : &tag_slot->slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + if (!BufferTagsEqual(&tag_slot->tag, &key->identity.tag) + || tag_slot->cluster_epoch != key->identity.cluster_epoch) { + result = PCM_X_QUEUE_STALE; + goto allocator_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_BUSY) + result = PCM_X_QUEUE_GATE_RETRY; + else + fail_closed = true; + goto allocator_done; + } + gate_claimed = true; + result = pcm_x_local_writer_holder_authority_allocator_locked( + key, writer_claim, tag_ref, tag_slot, &committed_own_generation); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + writer_authorized = true; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_HOLDER, key, &holder_ref); + if (directory_result == PCM_X_DIRECTORY_OK) { + uint32 holder_state; + + holder = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_HOLDER, + holder_ref); + if (holder == NULL) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + holder_state = pcm_x_slot_state_read(&holder->slot); + if (holder_state == PCM_X_SLOT_RESERVED_NONVISIBLE || holder_state == PCM_XL_DETACHING) { + result = PCM_X_QUEUE_GATE_RETRY; + goto allocator_done; + } + tag_ref.slot_index = holder->tag_slot_index; + tag_ref.slot_generation = holder->tag_slot_generation; + if (!pcm_x_local_holder_state_is_occupancy(holder_state) + || !pcm_x_local_holder_handle_exact( + &(PcmXLocalHolderHandle){ .key = *key, + .tag_slot = tag_ref, + .holder_slot = holder_ref }, + tag_ref, holder_ref, holder) + || pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &key->identity.tag, &existing) + != PCM_X_DIRECTORY_OK + || existing.slot_index != tag_ref.slot_index + || existing.slot_generation != tag_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + pcm_x_local_holder_handle_from_member(handle_out, tag_ref, holder_ref, holder); + if (writer_authorized) + *committed_own_generation_out = committed_own_generation; + result = PCM_X_QUEUE_DUPLICATE; + goto allocator_done; + } + if (directory_result != PCM_X_DIRECTORY_NOT_FOUND) { + result = pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + result = pcm_x_local_retire_episode_state_locked(header); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_BUSY) + result = PCM_X_QUEUE_GATE_RETRY; + else + fail_closed = true; + goto allocator_done; + } + + if (!writer_authorized) { + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &key->identity.tag, &tag_ref); + if (directory_result == PCM_X_DIRECTORY_OK) { + tag_slot + = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + result = pcm_x_resident_tag_state_result(tag_slot == NULL ? NULL : &tag_slot->slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + if (!BufferTagsEqual(&tag_slot->tag, &key->identity.tag) + || tag_slot->cluster_epoch != key->identity.cluster_epoch) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_BUSY) + result = PCM_X_QUEUE_GATE_RETRY; + else + fail_closed = true; + goto allocator_done; + } + gate_claimed = true; + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) { + result = PCM_X_QUEUE_BARRIER_CLOSED; + goto allocator_done; + } + } else if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_LOCAL_TAG, &tag_ref, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + new_tag = true; + gate_claimed = true; + tag_slot = (PcmXLocalTagSlot *)raw_slot; + pcm_x_local_tag_init_common(tag_slot, &key->identity.tag, key->identity.cluster_epoch, + -1, 0); + } else { + result = pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + } + if (!cluster_pcm_x_generation_next(tag_slot->holder_set_generation, &next_holder_generation)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_LOCAL_HOLDER, &holder_ref, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (new_tag + && !pcm_x_local_holder_stage_rollback_locked(true, false, key, tag_ref, NULL, + holder_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + holder = (PcmXLocalMembershipSlot *)raw_slot; + holder->identity = key->identity; + holder->tag_slot_index = tag_ref.slot_index; + holder->tag_slot_generation = tag_ref.slot_generation; + holder->next_index = PCM_X_INVALID_SLOT_INDEX; + holder->prev_index = PCM_X_INVALID_SLOT_INDEX; + holder->buffer_id = key->buffer_id; + holder->role = PCM_X_LOCAL_ROLE_NONE; + holder->partition = PCM_X_LOCAL_PARTITION_HOLDER; + + if (new_tag) { + directory_result = pcm_x_directory_insert_locked(PCM_X_DIR_LOCAL_TAG, &key->identity.tag, + tag_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (!pcm_x_local_holder_stage_rollback_locked(true, false, key, tag_ref, holder, + holder_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + tag_directory_published = true; + } + directory_result + = pcm_x_directory_insert_locked(PCM_X_DIR_LOCAL_HOLDER, key, holder_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (!pcm_x_local_holder_stage_rollback_locked(new_tag, tag_directory_published, key, + tag_ref, holder, holder_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + result = PCM_X_QUEUE_OK; + +allocator_done: + if (result != PCM_X_QUEUE_OK && gate_claimed && !new_tag) { + if (fail_closed) { + result = PCM_X_QUEUE_CORRUPT; + } else if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + } + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&key->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, tag_slot); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &key->identity.tag, + PCM_X_STATE_BIT(new_tag ? PCM_X_TAG_RESERVED_NONVISIBLE : PCM_X_TAG_LIVE)); + holder = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_HOLDER, holder_ref, &key->identity.tag, + PCM_X_STATE_BIT(PCM_X_SLOT_RESERVED_NONVISIBLE)); + if (tag_slot == NULL || holder == NULL + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0 + || (!writer_authorized + && (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) + || holder->partition != PCM_X_LOCAL_PARTITION_HOLDER + || holder->tag_slot_index != tag_ref.slot_index + || holder->tag_slot_generation != tag_ref.slot_generation + || !pcm_x_local_holder_key_equal( + key, + &(PcmXLocalHolderKey){ .identity = holder->identity, .buffer_id = holder->buffer_id }) + || tag_slot->holder_set_generation == UINT64_MAX) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (writer_authorized) { + uint64 revalidated_generation = 0; + + result = pcm_x_local_writer_holder_authority_domain_locked( + key, writer_claim, tag_ref, tag_slot, &revalidated_generation); + if (result != PCM_X_QUEUE_OK || revalidated_generation != committed_own_generation) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + } + if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) { + PcmXLocalMembershipSlot *head; + PcmXSlotRef head_ref; + + head = pcm_x_local_holder_link_resolve(&key->identity.tag, tag_ref, + tag_slot->active_holder_head_index, + PCM_X_INVALID_SLOT_INDEX, &head_ref); + if (head == NULL || head_ref.slot_index == holder_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + holder->next_index = head_ref.slot_index; + head->prev_index = holder_ref.slot_index; + } + tag_slot->active_holder_head_index = holder_ref.slot_index; + tag_slot->holder_set_generation = next_holder_generation; + pg_write_barrier(); + pcm_x_slot_state_write(&holder->slot, PCM_XL_HOLDER_ACQUIRING); + if (new_tag) + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + pcm_x_local_holder_handle_from_member(handle_out, tag_ref, holder_ref, holder); + if (writer_authorized) + *committed_own_generation_out = committed_own_generation; + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + result = PCM_X_QUEUE_OK; + +domain_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_register(const PcmXLocalHolderKey *key, + PcmXLocalHolderHandle *handle_out) +{ + return pcm_x_local_holder_register_common(key, NULL, handle_out, NULL); +} + + +PcmXQueueResult +cluster_pcm_x_local_writer_holder_register_exact(const PcmXLocalHolderKey *key, + const PcmXLocalWriterClaim *claim, + PcmXLocalHolderHandle *handle_out, + uint64 *committed_own_generation_out) +{ + return pcm_x_local_holder_register_common(key, claim, handle_out, committed_own_generation_out); +} + + +/* + * Content-window W1. All addressed fields are immutable between ACQUIRING + * publication and post-RELEASING unlink. The generation seqlock fences slot + * reuse; the state CAS is the only shared mutation here. In particular this + * path must never acquire a PCM-X LWLock or promote a retry to fail-closed. + */ +static PcmXQueueResult +pcm_x_local_holder_transition_exact(const PcmXLocalHolderHandle *handle, uint32 expected_state, + uint32 desired_state) +{ + PcmXAllocatorView holder_view; + PcmXLocalMembershipSlot *holder; + uint64 generation; + uint32 current; + uint32 desired; + uint32 state; + + if (ClusterPcmXConvertShmem == NULL || handle == NULL + || !pcm_x_local_holder_key_valid(&handle->key) + || handle->tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || handle->holder_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_HOLDER, &holder_view)) + return PCM_X_QUEUE_INVALID; + holder = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&holder_view, + handle->holder_slot.slot_index); + if (holder == NULL) + return PCM_X_QUEUE_STALE; + if (!pcm_x_slot_generation_read(&holder->slot, &generation)) + return PCM_X_QUEUE_BUSY; + if (generation != handle->holder_slot.slot_generation) + return PCM_X_QUEUE_STALE; + state = pcm_x_slot_state_read(&holder->slot); + if (state != expected_state && state != desired_state) + return PCM_X_QUEUE_BAD_STATE; + pg_read_barrier(); + if (!pcm_x_local_holder_handle_exact(handle, handle->tag_slot, handle->holder_slot, holder)) + return PCM_X_QUEUE_CORRUPT; + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&holder->slot, &generation)) + return PCM_X_QUEUE_BUSY; + if (generation != handle->holder_slot.slot_generation) + return PCM_X_QUEUE_STALE; + + current = pg_atomic_read_u32(&holder->slot.state_flags); + for (;;) { + state = current & PCM_X_SLOT_STATE_MASK; + if (state == desired_state) + return PCM_X_QUEUE_DUPLICATE; + if (state != expected_state) + return PCM_X_QUEUE_BAD_STATE; + desired = (current & PCM_X_SLOT_FLAGS_MASK) | desired_state; + pg_write_barrier(); + if (pg_atomic_compare_exchange_u32(&holder->slot.state_flags, ¤t, desired)) + return PCM_X_QUEUE_OK; + } +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_activate_exact(const PcmXLocalHolderHandle *handle) +{ + return pcm_x_local_holder_transition_exact(handle, PCM_XL_HOLDER_ACQUIRING, + PCM_XL_HOLDER_ACTIVE); +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_mark_releasing_exact(const PcmXLocalHolderHandle *handle) +{ + return pcm_x_local_holder_transition_exact(handle, PCM_XL_HOLDER_ACTIVE, + PCM_XL_HOLDER_RELEASING); +} + + +static PcmXQueueResult +pcm_x_local_holder_claim_detaching(PcmXLocalMembershipSlot *holder, uint32 allowed_holder_states) +{ + uint32 current; + uint32 desired; + uint32 holder_state; + + if (holder == NULL || allowed_holder_states == 0 + || (allowed_holder_states & ~PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES) != 0) + return PCM_X_QUEUE_INVALID; + current = pg_atomic_read_u32(&holder->slot.state_flags); + for (;;) { + holder_state = current & PCM_X_SLOT_STATE_MASK; + if (!pcm_x_local_holder_state_is_occupancy(holder_state) + || (allowed_holder_states & PCM_X_STATE_BIT(holder_state)) == 0) + return pcm_x_local_holder_state_is_occupancy(holder_state) ? PCM_X_QUEUE_BAD_STATE + : PCM_X_QUEUE_CORRUPT; + desired = (current & PCM_X_SLOT_FLAGS_MASK) | PCM_XL_DETACHING; + pg_write_barrier(); + if (pg_atomic_compare_exchange_u32(&holder->slot.state_flags, ¤t, desired)) + return PCM_X_QUEUE_OK; + } +} + + +static PcmXQueueResult +pcm_x_local_holder_detach_exact(const PcmXLocalHolderHandle *handle, uint32 allowed_holder_states) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *gated_tag = NULL; + PcmXLocalMembershipSlot *holder; + PcmXLocalMembershipSlot *previous = NULL; + PcmXLocalMembershipSlot *next = NULL; + PcmXImageToken zero_image; + PcmXSlotRef found; + PcmXSlotRef tag_ref; + PcmXSlotRef holder_ref; + PcmXSlotRef previous_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef next_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint64 next_holder_generation; + uint32 flags; + uint32 holder_state; + uint32 partition; + bool detach_tag = false; + bool gate_claimed = false; + bool last_holder; + + memset(&zero_image, 0, sizeof(zero_image)); + + if (allowed_holder_states == 0 + || (allowed_holder_states & ~PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES) != 0 || header == NULL + || handle == NULL || !pcm_x_local_holder_key_valid(&handle->key) + || handle->tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || handle->holder_slot.slot_index == PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + /* The exact runtime token is the authority linearization point for both + * normal unregister and exceptional owner cleanup. A fenced runtime must + * preserve the directory, holder slot, and tag links byte-for-byte for + * deferred recovery; a close ordered after this point may let the already + * authorized cross-lock detach finish or retain DETACHING evidence. */ + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + LWLockRelease(&header->allocator_lock.lock); + return PCM_X_QUEUE_NOT_READY; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_HOLDER, &handle->key, &found); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + LWLockRelease(&header->allocator_lock.lock); + return PCM_X_QUEUE_NOT_FOUND; + } + if (directory_result != PCM_X_DIRECTORY_OK) { + LWLockRelease(&header->allocator_lock.lock); + return pcm_x_queue_result_from_directory(directory_result); + } + if (found.slot_index != handle->holder_slot.slot_index + || found.slot_generation != handle->holder_slot.slot_generation) { + LWLockRelease(&header->allocator_lock.lock); + return PCM_X_QUEUE_STALE; + } + holder_ref = found; + holder = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_HOLDER, + holder_ref); + if (holder == NULL) { + LWLockRelease(&header->allocator_lock.lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + holder_state = pcm_x_slot_state_read(&holder->slot); + if (holder_state == PCM_XL_DETACHING) { + LWLockRelease(&header->allocator_lock.lock); + return PCM_X_QUEUE_BUSY; + } + tag_ref.slot_index = holder->tag_slot_index; + tag_ref.slot_generation = holder->tag_slot_generation; + if (!pcm_x_local_holder_handle_exact(handle, tag_ref, holder_ref, holder) + || pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &handle->key.identity.tag, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != tag_ref.slot_index + || found.slot_generation != tag_ref.slot_generation) { + LWLockRelease(&header->allocator_lock.lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + if (!pcm_x_local_holder_state_is_occupancy(holder_state) + || (allowed_holder_states & PCM_X_STATE_BIT(holder_state)) == 0) { + LWLockRelease(&header->allocator_lock.lock); + if (pcm_x_local_holder_state_is_occupancy(holder_state)) + return PCM_X_QUEUE_BAD_STATE; + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || !BufferTagsEqual(&tag_slot->tag, &handle->key.identity.tag) + || tag_slot->cluster_epoch != handle->key.identity.cluster_epoch) { + LWLockRelease(&header->allocator_lock.lock); + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + result = pcm_x_local_handoff_gate_claim_locked(header, tag_slot, false, PCM_X_QUEUE_GATE_RETRY); + if (result != PCM_X_QUEUE_OK) { + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + gate_claimed = true; + gated_tag = tag_slot; + LWLockRelease(&header->allocator_lock.lock); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&handle->key.identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, gated_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &handle->key.identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + holder = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_HOLDER, holder_ref, + &handle->key.identity.tag, + PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES); + if (tag_slot == NULL || holder == NULL + || !pcm_x_local_holder_handle_exact(handle, tag_ref, holder_ref, holder)) { + result = PCM_X_QUEUE_STALE; + goto domain_done; + } + holder_state = pcm_x_slot_state_read(&holder->slot); + if (!pcm_x_local_holder_state_is_occupancy(holder_state) + || (allowed_holder_states & PCM_X_STATE_BIT(holder_state)) == 0) { + result = pcm_x_local_holder_state_is_occupancy(holder_state) ? PCM_X_QUEUE_BAD_STATE + : PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (!cluster_pcm_x_generation_next(tag_slot->holder_set_generation, &next_holder_generation)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (holder->prev_index == holder_ref.slot_index || holder->next_index == holder_ref.slot_index + || (holder->prev_index != PCM_X_INVALID_SLOT_INDEX + && holder->prev_index == holder->next_index)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (holder->prev_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->active_holder_head_index != holder_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + } else { + previous = pcm_x_local_holder_slot_resolve(&handle->key.identity.tag, tag_ref, + holder->prev_index, &previous_ref); + if (previous == NULL || previous->next_index != holder_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + } + if (holder->next_index != PCM_X_INVALID_SLOT_INDEX) { + next + = pcm_x_local_holder_link_resolve(&handle->key.identity.tag, tag_ref, + holder->next_index, holder_ref.slot_index, &next_ref); + if (next == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + } + last_holder = holder->prev_index == PCM_X_INVALID_SLOT_INDEX + && holder->next_index == PCM_X_INVALID_SLOT_INDEX; + if (last_holder && tag_slot->membership_count == 0) { + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->closed_round_member_count != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + /* A holder may leave after PROBE has frozen the tag. Reclaim only a + * completely pristine holder-only tag; otherwise preserve the durable + * barrier/transfer evidence for exact ACK, DRAIN, and RETIRE. */ + detach_tag + = (flags + & (PCM_X_LOCAL_TAG_F_REVOKE_BARRIER | PCM_X_LOCAL_TAG_F_TERMINAL_MASK + | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK)) + == 0 + && pcm_x_ticket_ref_is_zero(&tag_slot->ref) + && pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + && pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && pcm_x_transfer_leg_is_clear(&tag_slot->reliable) + && pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + && pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) + && tag_slot->holder_terminal_drain_generation == 0 + && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 + && tag_slot->blocker_snapshot_next_chunk == 0 && tag_slot->blocker_set_generation == 0 + && pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable); + } + result = pcm_x_local_holder_claim_detaching(holder, allowed_holder_states); + if (result != PCM_X_QUEUE_OK) + goto domain_done; + if (previous == NULL) + tag_slot->active_holder_head_index = holder->next_index; + else + previous->next_index = holder->next_index; + if (next != NULL) + next->prev_index = holder->prev_index; + tag_slot->holder_set_generation = next_holder_generation; + if (detach_tag) + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); + holder->next_index = PCM_X_INVALID_SLOT_INDEX; + holder->prev_index = PCM_X_INVALID_SLOT_INDEX; + result = PCM_X_QUEUE_OK; + +domain_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) { + if (result != PCM_X_QUEUE_CORRUPT && gate_claimed) { + if (!pcm_x_local_gate_release(gated_tag)) + result = PCM_X_QUEUE_CORRUPT; + else + gate_claimed = false; + } + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, tag_slot); + holder = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_HOLDER, + holder_ref); + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (holder == NULL || tag_slot == NULL + || pcm_x_slot_state_read(&holder->slot) != PCM_XL_DETACHING + || (detach_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_DETACHING) + || (!detach_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE) + || !pcm_x_local_holder_handle_exact(handle, tag_ref, holder_ref, holder) + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0 + || pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_HOLDER, &handle->key, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != holder_ref.slot_index + || found.slot_generation != holder_ref.slot_generation + || (detach_tag + && (pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &handle->key.identity.tag, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != tag_ref.slot_index + || found.slot_generation != tag_ref.slot_generation))) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_cleanup_done; + } + if (pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_HOLDER, &handle->key, holder_ref) + != PCM_X_DIRECTORY_OK + || (detach_tag + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, &handle->key.identity.tag, + tag_ref) + != PCM_X_DIRECTORY_OK) + || pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_HOLDER, holder_ref, PCM_XL_DETACHING) + != PCM_X_ALLOC_OK + || (detach_tag + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, PCM_X_TAG_DETACHING) + != PCM_X_ALLOC_OK)) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_cleanup_done; + } + if (!detach_tag && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + goto allocator_cleanup_done; + } + result = PCM_X_QUEUE_OK; + +allocator_cleanup_done: + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + +/* Error cleanup may run beneath a PG_CATCH. Such a caller must first copy + * and flush its original ErrorData (LockBuffer does this) so this wrapper's + * FlushErrorState can discard only a nested detach failure. Exact detach may + * take its LWLocks once; on acquisition ERROR preserve every published + * identity, close the runtime, and return CORRUPT without a second throw. */ +static PcmXQueueResult +pcm_x_local_holder_cleanup_detach_exact(const PcmXLocalHolderHandle *handle, + uint32 allowed_holder_states) +{ + volatile PcmXQueueResult result = PCM_X_QUEUE_CORRUPT; + + PG_TRY(); + { + result = pcm_x_local_holder_detach_exact(handle, allowed_holder_states); + } + PG_CATCH(); + { + FlushErrorState(); + pcm_x_runtime_fail_closed(); + result = PCM_X_QUEUE_CORRUPT; + } + PG_END_TRY(); + return (PcmXQueueResult)result; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_abort_acquiring_exact(const PcmXLocalHolderHandle *handle) +{ + return pcm_x_local_holder_cleanup_detach_exact(handle, + PCM_X_STATE_BIT(PCM_XL_HOLDER_ACQUIRING)); +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_exceptional_detach_exact(const PcmXLocalHolderHandle *handle, + LWLock *content_lock) +{ + /* This is a production guard, not an assertion: ACTIVE evidence may be + * removed only after error cleanup has actually released the content + * LWLock. The saved exact handle, not the current BufferDesc mirror, + * provides tag/generation/slot ABA protection. */ + if (content_lock == NULL || LWLockHeldByMe(content_lock)) + return PCM_X_QUEUE_BAD_STATE; + return pcm_x_local_holder_cleanup_detach_exact(handle, PCM_X_LOCAL_HOLDER_OCCUPANCY_STATES); +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_unregister_exact(const PcmXLocalHolderHandle *handle) +{ + return pcm_x_local_holder_detach_exact(handle, PCM_X_STATE_BIT(PCM_XL_HOLDER_RELEASING)); +} + + +static int +pcm_x_local_holder_handle_compare(const void *left_ptr, const void *right_ptr) +{ + const PcmXLocalHolderHandle *left = (const PcmXLocalHolderHandle *)left_ptr; + const PcmXLocalHolderHandle *right = (const PcmXLocalHolderHandle *)right_ptr; + +#define PCM_X_COMPARE_FIELD(field) \ + do { \ + if (left->key.identity.field < right->key.identity.field) \ + return -1; \ + if (left->key.identity.field > right->key.identity.field) \ + return 1; \ + } while (0) + PCM_X_COMPARE_FIELD(node_id); + PCM_X_COMPARE_FIELD(procno); + PCM_X_COMPARE_FIELD(cluster_epoch); + PCM_X_COMPARE_FIELD(request_id); + PCM_X_COMPARE_FIELD(wait_seq); + PCM_X_COMPARE_FIELD(xid); + PCM_X_COMPARE_FIELD(base_own_generation); +#undef PCM_X_COMPARE_FIELD + if (left->key.buffer_id < right->key.buffer_id) + return -1; + if (left->key.buffer_id > right->key.buffer_id) + return 1; + if (left->holder_slot.slot_index < right->holder_slot.slot_index) + return -1; + if (left->holder_slot.slot_index > right->holder_slot.slot_index) + return 1; + if (left->holder_slot.slot_generation < right->holder_slot.slot_generation) + return -1; + if (left->holder_slot.slot_generation > right->holder_slot.slot_generation) + return 1; + return 0; +} + + +/* Caller holds the BufferTag local-domain lock. The count is established + * before any output byte is written, so NO_CAPACITY is a byte-stable + * count-first result. */ +static PcmXQueueResult +pcm_x_local_holder_snapshot_copy_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalHolderHandle *holders_out, Size holder_capacity, + PcmXLocalHolderSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalMembershipSlot *holder; + PcmXSlotRef holder_ref; + Size count = 0; + Size current; + Size previous; + Size i; + + if (header == NULL || tag_slot == NULL || snapshot_out == NULL + || (holder_capacity != 0 && holders_out == NULL)) + return PCM_X_QUEUE_INVALID; + current = tag_slot->active_holder_head_index; + previous = PCM_X_INVALID_SLOT_INDEX; + while (current != PCM_X_INVALID_SLOT_INDEX && count < header->layout.local_holder.capacity) { + holder = pcm_x_local_holder_link_resolve(&tag_slot->tag, tag_ref, current, previous, + &holder_ref); + if (holder == NULL || holder->next_index == current) + return PCM_X_QUEUE_CORRUPT; + previous = current; + current = holder->next_index; + count++; + } + if (current != PCM_X_INVALID_SLOT_INDEX || (count != 0 && tag_slot->holder_set_generation == 0)) + return PCM_X_QUEUE_CORRUPT; + snapshot_out->tag_slot = tag_ref; + snapshot_out->holder_set_generation = tag_slot->holder_set_generation; + snapshot_out->holder_count = count; + if (count > holder_capacity) + return PCM_X_QUEUE_NO_CAPACITY; + + current = tag_slot->active_holder_head_index; + previous = PCM_X_INVALID_SLOT_INDEX; + for (i = 0; i < count; i++) { + holder = pcm_x_local_holder_link_resolve(&tag_slot->tag, tag_ref, current, previous, + &holder_ref); + if (holder == NULL) + return PCM_X_QUEUE_CORRUPT; + pcm_x_local_holder_handle_from_member(&holders_out[i], tag_ref, holder_ref, holder); + previous = current; + current = holder->next_index; + } + if (count > 1) + qsort(holders_out, count, sizeof(*holders_out), pcm_x_local_holder_handle_compare); + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_snapshot(const BufferTag *tag, PcmXLocalHolderHandle *holders_out, + Size holder_capacity, PcmXLocalHolderSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result = PCM_X_QUEUE_OK; + uint32 partition; + + pcm_x_local_holder_snapshot_clear(snapshot_out); + if (header == NULL || tag == NULL || snapshot_out == NULL + || (holder_capacity != 0 && holders_out == NULL)) + return PCM_X_QUEUE_INVALID; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_OK; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_STALE; + goto done; + } + result = pcm_x_local_holder_snapshot_copy_locked(tag_slot, tag_ref, holders_out, + holder_capacity, snapshot_out); + +done: + LWLockRelease(&header->local_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_snapshot_revalidate(const BufferTag *tag, + const PcmXLocalHolderSnapshot *snapshot) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef found; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 partition; + + if (header == NULL || tag == NULL || snapshot == NULL) + return PCM_X_QUEUE_INVALID; + if (snapshot->holder_count > header->layout.local_holder.capacity + || (snapshot->holder_count != 0 && snapshot->holder_set_generation == 0)) + return PCM_X_QUEUE_INVALID; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &found); + LWLockRelease(&header->allocator_lock.lock); + if (snapshot->tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX) { + if (snapshot->tag_slot.slot_generation != 0 || snapshot->holder_set_generation != 0 + || snapshot->holder_count != 0) + return PCM_X_QUEUE_INVALID; + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_OK; + if (directory_result == PCM_X_DIRECTORY_OK) + return PCM_X_QUEUE_STALE; + return pcm_x_queue_result_from_directory(directory_result); + } + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + if (found.slot_index != snapshot->tag_slot.slot_index + || found.slot_generation != snapshot->tag_slot.slot_generation) + return PCM_X_QUEUE_STALE; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, found, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) + result = PCM_X_QUEUE_STALE; + else if (tag_slot->holder_set_generation != snapshot->holder_set_generation) + result = PCM_X_QUEUE_STALE; + else + result = PCM_X_QUEUE_OK; + LWLockRelease(&header->local_locks[partition].lock); + return result; +} + + +/* Reserve a tag-only holder participant without publishing transfer state + * under the allocator lock. Both an existing and a new tag return with the + * admission gate claimed until the caller publishes under the BufferTag + * domain lock. */ +static PcmXQueueResult +pcm_x_local_holder_transfer_tag_prepare(const PcmXTicketRef *ref, int32 master_node, + uint64 master_session, PcmXLocalTagSlot **prepared_tag_out, + PcmXSlotRef *tag_ref_out, bool *new_tag_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalTagSlot *tag_slot = NULL; + PcmXSlotHeader *raw_slot; + PcmXSlotRef existing; + PcmXDirectoryResult directory_result; + PcmXAllocatorResult allocator_result; + PcmXQueueResult result; + + *prepared_tag_out = NULL; + *tag_ref_out = (PcmXSlotRef){ PCM_X_INVALID_SLOT_INDEX, 0 }; + *new_tag_out = false; + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + result = pcm_x_local_retire_episode_state_locked(header); + if (result != PCM_X_QUEUE_OK) { + goto prepare_done; + } + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, tag_ref_out); + if (directory_result == PCM_X_DIRECTORY_OK) { + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, + *tag_ref_out); + result = pcm_x_resident_tag_state_result(tag_slot == NULL ? NULL : &tag_slot->slot); + if (result == PCM_X_QUEUE_OK + && (!BufferTagsEqual(&tag_slot->tag, &ref->identity.tag) + || tag_slot->cluster_epoch != ref->identity.cluster_epoch)) + result = PCM_X_QUEUE_CORRUPT; + if (result == PCM_X_QUEUE_OK) + result + = pcm_x_local_handoff_gate_claim_locked(header, tag_slot, false, PCM_X_QUEUE_BUSY); + goto prepare_done; + } + if (directory_result != PCM_X_DIRECTORY_NOT_FOUND) { + result = pcm_x_queue_result_from_directory(directory_result); + goto prepare_done; + } + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref_out, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + goto prepare_done; + } + tag_slot = (PcmXLocalTagSlot *)raw_slot; + pcm_x_local_tag_init_common(tag_slot, &ref->identity.tag, ref->identity.cluster_epoch, + master_node, master_session); + directory_result = pcm_x_directory_insert_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, + *tag_ref_out, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + if (pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, *tag_ref_out, + PCM_X_TAG_RESERVED_NONVISIBLE) + != PCM_X_ALLOC_OK) + result = PCM_X_QUEUE_CORRUPT; + *tag_ref_out = (PcmXSlotRef){ PCM_X_INVALID_SLOT_INDEX, 0 }; + goto prepare_done; + } + *new_tag_out = true; + result = PCM_X_QUEUE_OK; + +prepare_done: + if (result == PCM_X_QUEUE_OK) + *prepared_tag_out = tag_slot; + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +pcm_x_local_holder_transfer_tag_abort_new(const BufferTag *tag, PcmXSlotRef tag_ref, + PcmXLocalTagSlot *prepared_tag) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef found; + bool ok = false; + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, prepared_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (pcm_x_local_retire_episode_state_locked(header) == PCM_X_QUEUE_OK && tag_slot != NULL + && pcm_x_slot_state_read(&tag_slot->slot) == PCM_X_TAG_RESERVED_NONVISIBLE + && (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0 + && BufferTagsEqual(&tag_slot->tag, tag) + && pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &found) == PCM_X_DIRECTORY_OK + && found.slot_index == tag_ref.slot_index + && found.slot_generation == tag_ref.slot_generation + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, tag, tag_ref) + == PCM_X_DIRECTORY_OK + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + PCM_X_TAG_RESERVED_NONVISIBLE) + == PCM_X_ALLOC_OK) + ok = true; + LWLockRelease(&header->allocator_lock.lock); + return ok; +} + + +static bool +pcm_x_local_holder_transfer_peer_exact(const PcmXLocalTagSlot *tag_slot, int32 master_node, + uint64 master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + const PcmXPeerFrontier *frontier; + + if (header == NULL || tag_slot == NULL || master_node < 0 + || master_node >= PCM_X_PROTOCOL_NODE_LIMIT || master_session == 0 + || tag_slot->master_node != master_node + || tag_slot->master_session_incarnation != master_session) + return false; + frontier = &header->peer_frontiers[master_node]; + return frontier->cluster_epoch == tag_slot->cluster_epoch + && frontier->sender_session_incarnation == master_session; +} + + +static void +pcm_x_local_cutoff_fill(const PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalCutoff *cutoff_out) +{ + memset(cutoff_out, 0, sizeof(*cutoff_out)); + cutoff_out->tag_slot = tag_ref; + cutoff_out->cutoff_sequence = tag_slot->cutoff_sequence; + cutoff_out->master_session_incarnation = tag_slot->master_session_incarnation; + cutoff_out->closed_round = tag_slot->local_round; + cutoff_out->next_round = tag_slot->local_round + 1; + cutoff_out->master_node = tag_slot->master_node; +} + + +/* + * Close the current local writer round under the tag admission gate. The + * membership-free case is intentional: a holder-only (or freshly reserved) + * tag still needs the same barrier so a writer arriving after PROBE cannot be + * mistaken for a member of the sampled round. + */ +static PcmXQueueResult +pcm_x_local_freeze_round_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalCutoff *cutoff_out) +{ + PcmXLocalMembershipSlot *tail; + PcmXAllocatorView member_view; + uint32 flags; + + if (tag_slot == NULL || cutoff_out == NULL) + return PCM_X_QUEUE_INVALID; + memset(cutoff_out, 0, sizeof(*cutoff_out)); + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) { + if (tag_slot->local_round == UINT32_MAX + || tag_slot->closed_round_member_count > tag_slot->membership_count + || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || (tag_slot->cutoff_sequence == 0 && tag_slot->closed_round_member_count != 0)) + return PCM_X_QUEUE_CORRUPT; + pcm_x_local_cutoff_fill(tag_slot, tag_ref, cutoff_out); + return PCM_X_QUEUE_DUPLICATE; + } + if (tag_slot->local_round == UINT32_MAX) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + if (tag_slot->closed_round_member_count != 0 || tag_slot->next_sequence == 0) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->membership_count == 0) { + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_CORRUPT; + } else { + if (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) + return PCM_X_QUEUE_CORRUPT; + tail = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, tag_slot->tail_index); + if (tail == NULL || tail->tag_slot_index != tag_ref.slot_index + || tail->tag_slot_generation != tag_ref.slot_generation || tail->local_sequence == 0 + || tail->local_sequence >= tag_slot->next_sequence) + return PCM_X_QUEUE_CORRUPT; + } + + /* Sequence allocation is monotonic. Cancelled/detached identities leave + * legal holes but remain members of the round closed at this high-water. */ + tag_slot->cutoff_sequence = tag_slot->next_sequence - 1; + tag_slot->closed_round_member_count = tag_slot->membership_count; + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); + pcm_x_local_cutoff_fill(tag_slot, tag_ref, cutoff_out); + return PCM_X_QUEUE_OK; +} + + +static bool +pcm_x_local_blocker_freeze_reservation_exact(const PcmXLocalTagSlot *tag_slot) +{ + return tag_slot != NULL && !pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && tag_slot->blocker_set_generation == 0 + && pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) + && tag_slot->blocker_snapshot_reliable.state_sequence == 0 + && tag_slot->blocker_snapshot_reliable.response_tombstone_mask == 0 + && tag_slot->blocker_snapshot_reliable.last_responder_node == 0 + && tag_slot->blocker_snapshot_reliable.last_response_opcode == 0 + && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 + && tag_slot->blocker_snapshot_next_chunk == 0; +} + + +PcmXQueueResult +cluster_pcm_x_local_probe_freeze_snapshot_exact( + const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, + PcmXLocalHolderHandle *holders_out, Size holder_capacity, PcmXLocalHolderSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *prepared_tag; + PcmXSlotRef tag_ref; + PcmXLocalCutoff cutoff; + PcmXQueueResult result; + PcmXQueueResult freeze_result; + uint32 flags; + uint32 partition; + bool new_tag; + bool gate_claimed = false; + bool durable_freeze = false; + bool fail_closed = false; + + pcm_x_local_holder_snapshot_clear(snapshot_out); + if (header == NULL || ref == NULL || snapshot_out == NULL + || (holder_capacity != 0 && holders_out == NULL) || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity) || ref->handle.ticket_id == 0 + || ref->handle.queue_generation == 0 || authenticated_master_node < 0 + || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_holder_transfer_tag_prepare(ref, authenticated_master_node, + authenticated_master_session, &prepared_tag, + &tag_ref, &new_tag); + if (result != PCM_X_QUEUE_OK) + return result; + gate_claimed = true; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, + prepared_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &ref->identity.tag, + PCM_X_STATE_BIT(new_tag ? PCM_X_TAG_RESERVED_NONVISIBLE : PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + PcmXAllocatorView tag_view; + PcmXLocalTagSlot *raw_tag = NULL; + uint64 generation; + + /* A writer-in-progress generation is never normalized to "no tag". + * It is a retryable closed decision; every other impossible locator is + * stale evidence. */ + if (pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &tag_view)) + raw_tag = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&tag_view, tag_ref.slot_index); + result = raw_tag != NULL && !pcm_x_slot_generation_read(&raw_tag->slot, &generation) + ? PCM_X_QUEUE_BUSY + : PCM_X_QUEUE_STALE; + tag_slot = prepared_tag; + goto freeze_release_gate; + } + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto freeze_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto freeze_release_gate; + } + if (tag_slot->master_node == -1 && tag_slot->master_session_incarnation == 0) { + if (tag_slot->membership_count != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto freeze_release_gate; + } + tag_slot->master_node = authenticated_master_node; + tag_slot->master_session_incarnation = authenticated_master_session; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto freeze_release_gate; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (!new_tag && (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 + && tag_slot->membership_count != 0 + && (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) { + /* cancel_exact unlinks before terminal detach decrements membership. + * Wait for that evidence owner instead of treating the legal window as a + * broken FIFO or publishing a blocker reservation that cannot be frozen. */ + if (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->closed_round_member_count == 0) { + result = PCM_X_QUEUE_NOT_READY; + goto freeze_release_gate; + } + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto freeze_release_gate; + } + + if (!pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { + if (!pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, ref)) { + result = PCM_X_QUEUE_BUSY; + goto freeze_release_gate; + } + if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + if (tag_slot->blocker_set_generation == 0 + || !pcm_x_transfer_leg_exact( + &tag_slot->blocker_snapshot_reliable, PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto freeze_release_gate; + } + /* The previous set still awaits its exact ACK. The caller must + * replay it, not resample a possibly different holder set. */ + result = PCM_X_QUEUE_DUPLICATE; + goto freeze_release_gate; + } + if (tag_slot->blocker_set_generation == 0) { + if (!pcm_x_local_blocker_freeze_reservation_exact(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto freeze_release_gate; + } + } else if (tag_slot->blocker_set_generation == UINT64_MAX + || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 + || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto freeze_release_gate; + } + } else { + if (tag_slot->blocker_set_generation != 0 + || !pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) + || tag_slot->blocker_snapshot_reliable.state_sequence != 0 + || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0 + || tag_slot->blocker_snapshot_reliable.last_responder_node != 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || !pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { + result = PCM_X_QUEUE_BUSY; + goto freeze_release_gate; + } + tag_slot->blocker_snapshot_ref = *ref; + } + + freeze_result = pcm_x_local_freeze_round_locked(tag_slot, tag_ref, &cutoff); + if (freeze_result != PCM_X_QUEUE_OK && freeze_result != PCM_X_QUEUE_DUPLICATE) { + result = freeze_result; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto freeze_release_gate; + } + if (new_tag) + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + durable_freeze = true; + result = pcm_x_local_holder_snapshot_copy_locked(tag_slot, tag_ref, holders_out, + holder_capacity, snapshot_out); + if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + +freeze_release_gate: + if (gate_claimed && !fail_closed && (!new_tag || durable_freeze)) { + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + gate_claimed = false; + } +freeze_done: + LWLockRelease(&header->local_locks[partition].lock); + if (!durable_freeze && new_tag && !fail_closed + && !pcm_x_local_holder_transfer_tag_abort_new(&ref->identity.tag, tag_ref, prepared_tag)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +typedef struct PcmXHeldContentLockSnapshot { + BufferTag tags[LWLOCK_MAX_HELD_BY_PROC]; + Size count; + bool overflow; +} PcmXHeldContentLockSnapshot; + + +/* ForEachLWLockHeldByMe callback: reverse-map and copy only. Acquiring a + * PCM local lock here would invert content_lock -> PCM and deadlock against + * holder registration/freeze, so all shared-state checks happen later. */ +static void +pcm_x_collect_held_content_lock(LWLock *lock, LWLockMode mode, void *context) +{ + PcmXHeldContentLockSnapshot *snapshot = (PcmXHeldContentLockSnapshot *)context; + BufferDesc *buf; + Size i; + + (void)mode; + if (snapshot == NULL || snapshot->overflow) + return; + buf = BufferDescriptorFromContentLock(lock); + if (buf == NULL) + return; + for (i = 0; i < snapshot->count; i++) { + if (BufferTagsEqual(&snapshot->tags[i], &buf->tag)) + return; + } + if (snapshot->count >= lengthof(snapshot->tags)) { + snapshot->overflow = true; + return; + } + snapshot->tags[snapshot->count++] = buf->tag; +} + + +/* Check one copied tag after ForEachLWLockHeldByMe has returned. */ +static PcmXQueueResult +pcm_x_local_nested_wait_guard_find_locked(const BufferTag *tag, PcmXSlotRef *tag_ref_out) +{ + PcmXDirectoryView directory; + PcmXAllocatorView allocator; + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + uint64 key_hash; + Size bucket; + Size step; + + if (header == NULL || tag == NULL || tag_ref_out == NULL + || !LWLockHeldByMe(&header->allocator_lock.lock) + || !cluster_pcm_x_directory_key_hash(PCM_X_DIR_LOCAL_TAG, tag, &key_hash) + || !pcm_x_directory_view(PCM_X_DIR_LOCAL_TAG, &directory) + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &allocator)) + return PCM_X_QUEUE_INVALID; + tag_ref_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + tag_ref_out->slot_generation = 0; + bucket = key_hash % directory.capacity; + for (step = 0; step < directory.capacity; step++) { + PcmXDirectoryEntry *entry = &directory.entries[(bucket + step) % directory.capacity]; + PcmXLocalTagSlot *slot; + uint64 generation; + uint64 resident_key_hash; + uint32 state; + + if (entry->state == PCM_X_DIRECTORY_EMPTY) + return PCM_X_QUEUE_NOT_FOUND; + if (entry->state != PCM_X_DIRECTORY_OCCUPIED) + return PCM_X_QUEUE_CORRUPT; + if (entry->key_hash != key_hash) + continue; + slot = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&allocator, entry->slot_index); + if (slot == NULL) + return PCM_X_QUEUE_CORRUPT; + /* Unlike the generic directory lookup, the pre-sleep guard must + * preserve an in-progress generation write as BUSY. Treating it as + * NOT_FOUND would open the exact freeze-vs-sleep race this API closes. */ + if (!pcm_x_slot_generation_read(&slot->slot, &generation)) + return PCM_X_QUEUE_BUSY; + state = pcm_x_slot_state_read(&slot->slot); + if (generation != entry->slot_generation || state == PCM_X_SLOT_FREE + || state == PCM_X_SLOT_GENERATION_EXHAUSTED + || !pcm_x_directory_slot_key_hash(PCM_X_DIR_LOCAL_TAG, &slot->slot, &resident_key_hash) + || resident_key_hash != entry->key_hash) + return PCM_X_QUEUE_CORRUPT; + if (!BufferTagsEqual(&slot->tag, tag)) + continue; + tag_ref_out->slot_index = entry->slot_index; + tag_ref_out->slot_generation = entry->slot_generation; + return PCM_X_QUEUE_OK; + } + return PCM_X_QUEUE_NOT_FOUND; +} + + +static PcmXQueueResult +pcm_x_local_nested_wait_guard_tag(const BufferTag *tag) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + uint32 partition; + + if (header == NULL || tag == NULL) + return PCM_X_QUEUE_OK; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_nested_wait_guard_find_locked(tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_NOT_FOUND) + return PCM_X_QUEUE_OK; + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + /* PCM runtime formation is irrelevant until an exact local-tag entry + * proves that this held content lock participates in the protocol. */ + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + PcmXAllocatorView tag_view; + PcmXLocalTagSlot *raw_tag = NULL; + uint64 generation; + + if (pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &tag_view)) + raw_tag = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&tag_view, tag_ref.slot_index); + /* A generation writer in progress is retryable BUSY. A completed + * locator mismatch is stale evidence, but is still a closed result. */ + result = raw_tag != NULL && !pcm_x_slot_generation_read(&raw_tag->slot, &generation) + ? PCM_X_QUEUE_BUSY + : PCM_X_QUEUE_STALE; + } else if (!pcm_x_runtime_token_exact(&runtime, 0)) + result = PCM_X_QUEUE_NOT_READY; + else if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) + result = PCM_X_QUEUE_BARRIER_CLOSED; + else + result = PCM_X_QUEUE_OK; + LWLockRelease(&header->local_locks[partition].lock); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_nested_wait_guard_before_block(void) +{ + PcmXHeldContentLockSnapshot snapshot; + PcmXQueueResult result; + Size i; + + memset(&snapshot, 0, sizeof(snapshot)); + ForEachLWLockHeldByMe(pcm_x_collect_held_content_lock, &snapshot); + if (snapshot.overflow) { + pcm_x_runtime_fail_closed(); + return PCM_X_QUEUE_CORRUPT; + } + for (i = 0; i < snapshot.count; i++) { + result = pcm_x_local_nested_wait_guard_tag(&snapshot.tags[i]); + if (result != PCM_X_QUEUE_OK) + return result; + } + return PCM_X_QUEUE_OK; +} + + +static bool +pcm_x_local_blocker_chain_validate(const PcmXAllocatorView *view, Size head_index, uint32 count, + const PcmXTicketRef *ref, PcmXSlotRef tag_ref, + uint64 set_generation, uint32 expected_state, + uint32 expected_crc32c, PcmXBlockerChunkPayload *edges_out) +{ + ClusterLmdVertex previous; + pg_crc32c crc; + Size current = head_index; + uint32 i; + bool have_previous = false; + + if (view == NULL || ref == NULL || count > view->capacity) + return false; + if (count == 0) + return head_index == PCM_X_INVALID_SLOT_INDEX && expected_crc32c == 0; + if (head_index == PCM_X_INVALID_SLOT_INDEX) + return false; + INIT_CRC32C(crc); + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot; + uint64 generation; + + slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(view, current); + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &generation) || generation == 0 + || pcm_x_slot_state_read(&slot->slot) != expected_state + || !pcm_x_ticket_handle_equal(&slot->handle, &ref->handle) + || slot->owner_slot_index != tag_ref.slot_index + || slot->owner_slot_generation != tag_ref.slot_generation + || slot->set_generation != set_generation || slot->chunk_no != i + || slot->direction != PCM_X_BLOCKER_DIRECTION_LOCAL + || !pcm_x_lmd_vertex_valid(&slot->blocker) + || (have_previous && pcm_x_lmd_vertex_identity_compare(&previous, &slot->blocker) >= 0) + || (slot->next_index != PCM_X_INVALID_SLOT_INDEX && slot->next_index >= view->capacity)) + return false; + pcm_x_blocker_crc32c_update(&crc, &slot->blocker); + if (edges_out != NULL) { + PcmXBlockerChunkPayload *edge = &edges_out[i]; + + memset(edge, 0, sizeof(*edge)); + edge->tag = ref->identity.tag; + edge->requester_node = ref->identity.node_id; + edge->requester_procno = ref->identity.procno; + edge->chunk_no = i; + edge->cluster_epoch = ref->identity.cluster_epoch; + edge->request_id = ref->identity.request_id; + edge->handle = ref->handle; + edge->grant_generation = ref->grant_generation; + edge->set_generation = set_generation; + edge->blocker = slot->blocker; + } + previous = slot->blocker; + have_previous = true; + current = slot->next_index; + } + if (current != PCM_X_INVALID_SLOT_INDEX) + return false; + FIN_CRC32C(crc); + return (uint32)crc == expected_crc32c; +} + + +static bool +pcm_x_local_blocker_chain_release_locked(Size head_index, uint32 count, const PcmXTicketRef *ref, + PcmXSlotRef tag_ref, uint64 set_generation, + uint32 expected_state) +{ + PcmXAllocatorView view; + Size current = head_index; + uint32 i; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view) || ref == NULL || count > view.capacity) + return false; + if (count == 0) + return head_index == PCM_X_INVALID_SLOT_INDEX; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot; + uint64 generation; + + slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &generation) + || pcm_x_slot_state_read(&slot->slot) != expected_state + || !pcm_x_ticket_handle_equal(&slot->handle, &ref->handle) + || slot->owner_slot_index != tag_ref.slot_index + || slot->owner_slot_generation != tag_ref.slot_generation + || slot->set_generation != set_generation || slot->chunk_no != i + || slot->direction != PCM_X_BLOCKER_DIRECTION_LOCAL + || (slot->next_index != PCM_X_INVALID_SLOT_INDEX && slot->next_index >= view.capacity)) + return false; + current = slot->next_index; + } + if (current != PCM_X_INVALID_SLOT_INDEX) + return false; + current = head_index; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + PcmXSlotRef slot_ref; + Size next = slot->next_index; + + slot_ref.slot_index = current; + if (!pcm_x_slot_generation_read(&slot->slot, &slot_ref.slot_generation) + || pcm_x_allocator_release_locked(PCM_X_ALLOC_BLOCKER, slot_ref, expected_state) + != PCM_X_ALLOC_OK) + return false; + current = next; + } + return true; +} + + +static PcmXAllocatorResult +pcm_x_local_blocker_chain_reserve_locked(const PcmXTicketRef *ref, PcmXSlotRef tag_ref, + uint64 set_generation, const ClusterLmdVertex *blockers, + Size blocker_count, Size *head_index_out) +{ + PcmXAllocatorResult result; + PcmXBlockerSlot *previous = NULL; + Size head = PCM_X_INVALID_SLOT_INDEX; + Size i; + + *head_index_out = PCM_X_INVALID_SLOT_INDEX; + result = pcm_x_blocker_capacity_preflight_locked(blocker_count); + if (result != PCM_X_ALLOC_OK) + return result; + for (i = 0; i < blocker_count; i++) { + PcmXSlotHeader *raw_slot; + PcmXBlockerSlot *slot; + PcmXSlotRef slot_ref; + + result = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_BLOCKER, &slot_ref, &raw_slot); + if (result != PCM_X_ALLOC_OK) { + if (result == PCM_X_ALLOC_NO_CAPACITY) + result = PCM_X_ALLOC_CORRUPT; + break; + } + slot = (PcmXBlockerSlot *)raw_slot; + slot->handle = ref->handle; + slot->blocker = blockers[i]; + slot->owner_slot_generation = tag_ref.slot_generation; + slot->set_generation = set_generation; + slot->next_index = PCM_X_INVALID_SLOT_INDEX; + slot->owner_slot_index = tag_ref.slot_index; + slot->chunk_no = (uint32)i; + slot->direction = PCM_X_BLOCKER_DIRECTION_LOCAL; + if (previous == NULL) + head = slot_ref.slot_index; + else + previous->next_index = slot_ref.slot_index; + previous = slot; + } + if (result != PCM_X_ALLOC_OK) { + if (!pcm_x_local_blocker_chain_release_locked(head, (uint32)i, ref, tag_ref, set_generation, + PCM_XB_RESERVED_NONVISIBLE)) + result = PCM_X_ALLOC_CORRUPT; + return result; + } + *head_index_out = head; + return PCM_X_ALLOC_OK; +} + + +/* Reserved blocker slots are private to one in-flight arm. The allocator + * lock publishes only their RESERVED lifecycle; after the exact local-tag + * lock mints a durable set generation, relabel the whole private chain in two + * passes so corruption can never leave a partially relabeled chain. */ +static bool +pcm_x_local_blocker_chain_relabel_reserved(Size head_index, uint32 count, const PcmXTicketRef *ref, + PcmXSlotRef tag_ref, uint64 old_generation, + uint64 new_generation) +{ + PcmXAllocatorView view; + Size current = head_index; + uint32 i; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &view) || ref == NULL || new_generation == 0 + || new_generation == UINT64_MAX || count > view.capacity) + return false; + if (count == 0) + return head_index == PCM_X_INVALID_SLOT_INDEX; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + uint64 slot_generation; + + if (slot == NULL || !pcm_x_slot_generation_read(&slot->slot, &slot_generation) + || slot_generation == 0 + || pcm_x_slot_state_read(&slot->slot) != PCM_XB_RESERVED_NONVISIBLE + || !pcm_x_ticket_handle_equal(&slot->handle, &ref->handle) + || slot->owner_slot_index != tag_ref.slot_index + || slot->owner_slot_generation != tag_ref.slot_generation + || slot->set_generation != old_generation || slot->chunk_no != i + || slot->direction != PCM_X_BLOCKER_DIRECTION_LOCAL + || (slot->next_index != PCM_X_INVALID_SLOT_INDEX && slot->next_index >= view.capacity)) + return false; + current = slot->next_index; + } + if (current != PCM_X_INVALID_SLOT_INDEX) + return false; + current = head_index; + for (i = 0; i < count; i++) { + PcmXBlockerSlot *slot = (PcmXBlockerSlot *)pcm_x_allocator_slot(&view, current); + + slot->set_generation = new_generation; + current = slot->next_index; + } + return true; +} + + +static bool +pcm_x_local_holder_snapshot_exact_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + const PcmXLocalHolderSnapshot *snapshot, + const PcmXLocalHolderHandle *holders, Size holder_count) +{ + PcmXLocalMembershipSlot *holder; + PcmXSlotRef holder_ref; + Size current; + Size previous; + Size active_count = 0; + Size i; + + if (tag_slot == NULL || snapshot == NULL || snapshot->holder_count != holder_count + || snapshot->tag_slot.slot_index != tag_ref.slot_index + || snapshot->tag_slot.slot_generation != tag_ref.slot_generation + || snapshot->holder_set_generation != tag_slot->holder_set_generation) + return false; + current = tag_slot->active_holder_head_index; + previous = PCM_X_INVALID_SLOT_INDEX; + while (current != PCM_X_INVALID_SLOT_INDEX + && active_count < ClusterPcmXConvertShmem->layout.local_holder.capacity) { + holder = pcm_x_local_holder_link_resolve(&tag_slot->tag, tag_ref, current, previous, + &holder_ref); + if (holder == NULL || holder->next_index == current) + return false; + previous = current; + current = holder->next_index; + active_count++; + } + if (current != PCM_X_INVALID_SLOT_INDEX || active_count != holder_count) + return false; + for (i = 0; i < holder_count; i++) { + if (i != 0 && pcm_x_local_holder_handle_compare(&holders[i - 1], &holders[i]) >= 0) + return false; + holder = pcm_x_local_holder_slot_resolve(&tag_slot->tag, tag_ref, + holders[i].holder_slot.slot_index, &holder_ref); + if (holder == NULL + || !pcm_x_local_holder_handle_exact(&holders[i], tag_ref, holder_ref, holder)) + return false; + } + return true; +} + + +/* The holder registry is a pin/barrier fingerprint, not a WFG edge list. + * Outer code may publish only live nested waits observed for exact registered + * holder processes. One process can own at most one current nested wait, and + * passive holders legitimately produce an empty blocker set. */ +static bool +pcm_x_local_filtered_blockers_valid(const PcmXTicketRef *ref, const PcmXLocalHolderHandle *holders, + Size holder_count, const ClusterLmdVertex *blockers, + Size blocker_count, uint32 *set_crc32c_out) +{ + ClusterLmdVertex previous; + pg_crc32c crc; + Size i; + + if (ref == NULL || set_crc32c_out == NULL || blocker_count > holder_count + || (holder_count != 0 && holders == NULL) || (blocker_count != 0 && blockers == NULL)) + return false; + *set_crc32c_out = 0; + if (blocker_count == 0) + return true; + INIT_CRC32C(crc); + for (i = 0; i < blocker_count; i++) { + const ClusterLmdVertex *blocker = &blockers[i]; + Size holder_index; + bool bound = false; + + if (!pcm_x_lmd_vertex_valid(blocker) || blocker->node_id != cluster_node_id + || (blocker->node_id == ref->identity.node_id && blocker->procno == ref->identity.procno + && blocker->cluster_epoch == ref->identity.cluster_epoch + && blocker->request_id == ref->identity.request_id) + || (i != 0 && pcm_x_lmd_vertex_identity_compare(&previous, blocker) >= 0) + || (i != 0 && previous.node_id == blocker->node_id + && previous.procno == blocker->procno)) + return false; + for (holder_index = 0; holder_index < holder_count; holder_index++) { + const PcmXWaitIdentity *holder = &holders[holder_index].key.identity; + + if (holder->node_id == blocker->node_id && holder->procno == blocker->procno + && holder->cluster_epoch == blocker->cluster_epoch) { + bound = true; + break; + } + } + if (!bound) + return false; + pcm_x_blocker_crc32c_update(&crc, blocker); + previous = *blocker; + } + FIN_CRC32C(crc); + *set_crc32c_out = (uint32)crc; + return true; +} + + +static bool +pcm_x_local_blocker_ack_tombstone_exact(const PcmXLocalTagSlot *tag_slot, + int32 authenticated_master_node) +{ + return tag_slot != NULL && authenticated_master_node >= 0 + && authenticated_master_node < PCM_X_PROTOCOL_NODE_LIMIT + && tag_slot->blocker_set_generation != 0 + && tag_slot->blocker_set_generation != UINT64_MAX + && pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) + && tag_slot->blocker_snapshot_reliable.state_sequence != 0 + && tag_slot->blocker_snapshot_reliable.last_response_opcode + == PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + && tag_slot->blocker_snapshot_reliable.last_responder_node + == (uint32)authenticated_master_node + && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 + && tag_slot->blocker_snapshot_next_chunk == 0; +} + + +PcmXQueueResult +cluster_pcm_x_local_blocker_snapshot_arm_exact( + const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, + const PcmXLocalHolderSnapshot *holder_snapshot, const PcmXLocalHolderHandle *holders, + Size holder_count, const ClusterLmdVertex *blockers, Size blocker_count, + PcmXLocalBlockerSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *prepared_tag; + PcmXAllocatorView blocker_view; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + PcmXAllocatorResult allocator_result; + Size reserved_head = PCM_X_INVALID_SLOT_INDEX; + uint64 reserved_generation = 0; + uint64 set_generation = 0; + uint64 next_sequence; + uint32 set_crc32c = 0; + uint32 partition; + bool new_tag; + bool gate_claimed = false; + bool chain_published = false; + bool fail_closed = false; + + if (snapshot_out != NULL) + memset(snapshot_out, 0, sizeof(*snapshot_out)); + if (header == NULL || ref == NULL || holder_snapshot == NULL || snapshot_out == NULL + || ref->grant_generation != 0 || !pcm_x_wait_identity_valid(&ref->identity) + || holder_count != holder_snapshot->holder_count || (holder_count != 0 && holders == NULL) + || holder_count > header->layout.local_holder.capacity || blocker_count > holder_count + || blocker_count > header->layout.max_wait_edges || blocker_count > UINT32_MAX + || (blocker_count != 0 && blockers == NULL) || authenticated_master_node < 0 + || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (!pcm_x_local_filtered_blockers_valid(ref, holders, holder_count, blockers, blocker_count, + &set_crc32c)) + return PCM_X_QUEUE_INVALID; + result = pcm_x_local_holder_transfer_tag_prepare(ref, authenticated_master_node, + authenticated_master_session, &prepared_tag, + &tag_ref, &new_tag); + if (result != PCM_X_QUEUE_OK) + return result; + gate_claimed = true; + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, prepared_tag); + allocator_result = pcm_x_local_blocker_chain_reserve_locked( + ref, tag_ref, reserved_generation, blockers, blocker_count, &reserved_head); + LWLockRelease(&header->allocator_lock.lock); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto arm_cleanup_new_tag; + } + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, + prepared_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &ref->identity.tag, + PCM_X_STATE_BIT(new_tag ? PCM_X_TAG_RESERVED_NONVISIBLE : PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_STALE; + tag_slot = prepared_tag; + goto arm_release_gate; + } + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_domain_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_STALE; + goto arm_release_gate; + } + if (tag_slot->master_node == -1 && tag_slot->master_session_incarnation == 0) { + tag_slot->master_node = authenticated_master_node; + tag_slot->master_session_incarnation = authenticated_master_session; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto arm_release_gate; + } + if (new_tag) { + if (holder_count != 0 || blocker_count != 0 + || holder_snapshot->tag_slot.slot_index != PCM_X_INVALID_SLOT_INDEX + || holder_snapshot->tag_slot.slot_generation != 0 + || holder_snapshot->holder_set_generation != 0) { + result = PCM_X_QUEUE_STALE; + goto arm_release_gate; + } + } else if (!pcm_x_local_holder_snapshot_exact_locked(tag_slot, tag_ref, holder_snapshot, + holders, holder_count)) { + result = PCM_X_QUEUE_STALE; + goto arm_release_gate; + } + if (!pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { + if (!pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, ref)) { + result = PCM_X_QUEUE_BUSY; + goto arm_release_gate; + } + if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + if (tag_slot->blocker_set_generation != 0 + && tag_slot->blocker_set_generation != UINT64_MAX + && tag_slot->blocker_snapshot_count == blocker_count + && tag_slot->blocker_snapshot_crc32c == set_crc32c + && pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, + authenticated_master_node, authenticated_master_session) + && pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + && pcm_x_local_blocker_chain_validate( + &blocker_view, tag_slot->blocker_snapshot_head_index, + tag_slot->blocker_snapshot_count, ref, tag_ref, + tag_slot->blocker_set_generation, PCM_XB_LIVE, set_crc32c, NULL)) { + result = PCM_X_QUEUE_DUPLICATE; + goto arm_output; + } + result = PCM_X_QUEUE_BUSY; + goto arm_release_gate; + } + /* A type-48 freeze reserves this ref with generation zero. After an + * exact ACK, the same ref remains as a tombstone and a later PROBE + * advances N -> N+1; neither case conflicts with the new arm. The + * reservation must otherwise be byte-exact so torn or stale state cannot + * be normalized into a new blocker generation. */ + if (tag_slot->blocker_set_generation == 0) { + if (!pcm_x_local_blocker_freeze_reservation_exact(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_domain_done; + } + } else if (!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_domain_done; + } + } + if (!cluster_pcm_x_generation_next(tag_slot->blocker_set_generation, &set_generation) + || set_generation == UINT64_MAX) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto arm_domain_done; + } + if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) + || !cluster_pcm_x_generation_next(tag_slot->blocker_snapshot_reliable.state_sequence, + &next_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto arm_domain_done; + } + if (!pcm_x_local_blocker_chain_relabel_reserved(reserved_head, (uint32)blocker_count, ref, + tag_ref, reserved_generation, set_generation)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_domain_done; + } + reserved_generation = set_generation; + if (!pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_local_blocker_chain_validate(&blocker_view, reserved_head, (uint32)blocker_count, + ref, tag_ref, set_generation, + PCM_XB_RESERVED_NONVISIBLE, set_crc32c, NULL)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_domain_done; + } + pcm_x_blocker_chain_state_write(&blocker_view, reserved_head, (uint32)blocker_count, + PCM_XB_LIVE); + tag_slot->blocker_snapshot_ref = *ref; + tag_slot->blocker_set_generation = set_generation; + tag_slot->blocker_snapshot_head_index = reserved_head; + tag_slot->blocker_snapshot_count = (uint32)blocker_count; + tag_slot->blocker_snapshot_crc32c = set_crc32c; + tag_slot->blocker_snapshot_next_chunk = (uint32)blocker_count; + tag_slot->blocker_snapshot_reliable.state_sequence = next_sequence; + tag_slot->blocker_snapshot_reliable.retry_deadline_ms = 0; + tag_slot->blocker_snapshot_reliable.expected_responder_session = authenticated_master_session; + tag_slot->blocker_snapshot_reliable.retry_count = 0; + tag_slot->blocker_snapshot_reliable.last_responder_node = 0; + tag_slot->blocker_snapshot_reliable.expected_responder_node = authenticated_master_node; + tag_slot->blocker_snapshot_reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT; + tag_slot->blocker_snapshot_reliable.last_response_opcode = 0; + tag_slot->blocker_snapshot_reliable.phase = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT; + tag_slot->blocker_snapshot_reliable.flags = 0; + tag_slot->blocker_snapshot_reliable.reserved = 0; + if (new_tag) + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + chain_published = true; + result = PCM_X_QUEUE_OK; + +arm_output: + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + snapshot_out->ref = tag_slot->blocker_snapshot_ref; + snapshot_out->set_generation = tag_slot->blocker_set_generation; + snapshot_out->reliable_state_sequence = tag_slot->blocker_snapshot_reliable.state_sequence; + snapshot_out->blocker_count = tag_slot->blocker_snapshot_count; + snapshot_out->set_crc32c = tag_slot->blocker_snapshot_crc32c; + } +arm_release_gate: + if (gate_claimed && !fail_closed && (!new_tag || chain_published)) { + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + gate_claimed = false; + } +arm_domain_done: + LWLockRelease(&header->local_locks[partition].lock); + + if (!chain_published) { + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, + gate_claimed ? (tag_slot != NULL ? tag_slot : prepared_tag) + : NULL); + if (!pcm_x_local_blocker_chain_release_locked(reserved_head, (uint32)blocker_count, ref, + tag_ref, reserved_generation, + PCM_XB_RESERVED_NONVISIBLE)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->allocator_lock.lock); + } +arm_cleanup_new_tag: + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE && !new_tag && gate_claimed + && !fail_closed) { + if (!pcm_x_local_gate_release(prepared_tag)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + gate_claimed = false; + } + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE && new_tag && !fail_closed + && !pcm_x_local_holder_transfer_tag_abort_new(&ref->identity.tag, tag_ref, prepared_tag)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_blocker_snapshot_lookup_exact(const PcmXTicketRef *ref, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXLocalBlockerSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (snapshot_out != NULL) + memset(snapshot_out, 0, sizeof(*snapshot_out)); + if (header == NULL || ref == NULL || snapshot_out == NULL || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity) || authenticated_master_node < 0 + || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_NOT_FOUND; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || !pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_STALE; + goto lookup_done; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto lookup_done; + } + if (pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { + if (tag_slot->blocker_set_generation != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || !pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_NOT_FOUND; + goto lookup_done; + } + if (!pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, ref)) { + result = PCM_X_QUEUE_BUSY; + goto lookup_done; + } + if (pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + if (pcm_x_local_blocker_freeze_reservation_exact(tag_slot) + || pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node)) + result = PCM_X_QUEUE_NOT_FOUND; + else { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto lookup_done; + } + if (tag_slot->blocker_set_generation == 0 || tag_slot->blocker_set_generation == UINT64_MAX + || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_count > header->layout.max_wait_edges + || tag_slot->blocker_snapshot_next_chunk != tag_slot->blocker_snapshot_count) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto lookup_done; + } + if (!pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto lookup_done; + } + snapshot_out->ref = tag_slot->blocker_snapshot_ref; + snapshot_out->set_generation = tag_slot->blocker_set_generation; + snapshot_out->reliable_state_sequence = tag_slot->blocker_snapshot_reliable.state_sequence; + snapshot_out->blocker_count = tag_slot->blocker_snapshot_count; + snapshot_out->set_crc32c = tag_slot->blocker_snapshot_crc32c; + result = PCM_X_QUEUE_OK; + +lookup_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_blocker_snapshot_copy_exact(const PcmXLocalBlockerSnapshot *snapshot, + PcmXBlockerSetHeaderPayload *header_out, + PcmXBlockerChunkPayload *edges_out, + Size edge_capacity) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXAllocatorView blocker_view; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (header_out != NULL) + memset(header_out, 0, sizeof(*header_out)); + if (header == NULL || snapshot == NULL || header_out == NULL + || (snapshot->blocker_count != 0 && edges_out == NULL) + || snapshot->ref.grant_generation != 0 + || !pcm_x_wait_identity_valid(&snapshot->ref.identity) || snapshot->set_generation == 0 + || snapshot->set_generation == UINT64_MAX || snapshot->reliable_state_sequence == 0 + || snapshot->blocker_count > header->layout.max_wait_edges) + return PCM_X_QUEUE_INVALID; + if (snapshot->blocker_count > edge_capacity) + return PCM_X_QUEUE_NO_CAPACITY; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &snapshot->ref.identity.tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&snapshot->ref.identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + &snapshot->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || !pcm_x_runtime_token_exact(&runtime, 0)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, &snapshot->ref) + || tag_slot->blocker_set_generation != snapshot->set_generation + || tag_slot->blocker_snapshot_reliable.state_sequence + != snapshot->reliable_state_sequence + || tag_slot->blocker_snapshot_count != snapshot->blocker_count + || tag_slot->blocker_snapshot_crc32c != snapshot->set_crc32c) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, tag_slot->master_node, + tag_slot->master_session_incarnation)) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, tag_slot->master_node, + tag_slot->master_session_incarnation)) + result = PCM_X_QUEUE_NOT_READY; + else if (tag_slot->blocker_snapshot_next_chunk != tag_slot->blocker_snapshot_count + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_local_blocker_chain_validate( + &blocker_view, tag_slot->blocker_snapshot_head_index, + tag_slot->blocker_snapshot_count, &tag_slot->blocker_snapshot_ref, tag_ref, + tag_slot->blocker_set_generation, PCM_XB_LIVE, tag_slot->blocker_snapshot_crc32c, + edges_out)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else { + header_out->ref = tag_slot->blocker_snapshot_ref; + header_out->set_generation = tag_slot->blocker_set_generation; + header_out->nblockers = tag_slot->blocker_snapshot_count; + header_out->set_crc32c = tag_slot->blocker_snapshot_crc32c; + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_blocker_ack_exact(const PcmXTicketRef *ref, uint64 set_generation, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXAllocatorView blocker_view; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + Size detached_head = PCM_X_INVALID_SLOT_INDEX; + uint32 detached_count = 0; + uint32 partition; + bool gate_claimed = false; + bool release_chain = false; + bool fail_closed = false; + + if (header == NULL || ref == NULL || ref->grant_generation != 0 + || !pcm_x_wait_identity_valid(&ref->identity) || set_generation == 0 + || set_generation == UINT64_MAX || authenticated_master_node < 0 + || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto ack_done; + } + gate_claimed = true; + if (!pcm_x_runtime_token_exact(&runtime, 0) + || !pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto ack_release_gate; + } + if (!pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, ref) + || tag_slot->blocker_set_generation != set_generation) { + result = PCM_X_QUEUE_STALE; + goto ack_release_gate; + } + if (pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + if (tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_DUPLICATE; + goto ack_release_gate; + } + if (!pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, authenticated_master_node, + authenticated_master_session) + || tag_slot->blocker_snapshot_next_chunk != tag_slot->blocker_snapshot_count + || !pcm_x_allocator_view(PCM_X_ALLOC_BLOCKER, &blocker_view) + || !pcm_x_local_blocker_chain_validate( + &blocker_view, tag_slot->blocker_snapshot_head_index, tag_slot->blocker_snapshot_count, + ref, tag_ref, set_generation, PCM_XB_LIVE, tag_slot->blocker_snapshot_crc32c, NULL)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto ack_release_gate; + } + detached_head = tag_slot->blocker_snapshot_head_index; + detached_count = tag_slot->blocker_snapshot_count; + pcm_x_blocker_chain_state_write(&blocker_view, detached_head, detached_count, PCM_XB_DETACHING); + tag_slot->blocker_snapshot_head_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->blocker_snapshot_count = 0; + tag_slot->blocker_snapshot_crc32c = 0; + tag_slot->blocker_snapshot_next_chunk = 0; + pcm_x_transfer_leg_clear(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, authenticated_master_node); + release_chain = true; + result = PCM_X_QUEUE_OK; + +ack_release_gate: + if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +ack_done: + LWLockRelease(&header->local_locks[partition].lock); + if (release_chain) { + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, NULL); + if (!pcm_x_local_blocker_chain_release_locked(detached_head, detached_count, ref, tag_ref, + set_generation, PCM_XB_DETACHING)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->allocator_lock.lock); + } + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *prepared_tag; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + uint32 flags; + uint32 partition; + bool new_tag; + bool gate_claimed = false; + bool tag_published = false; + bool fail_closed = false; + + if (header == NULL || revoke == NULL + || !pcm_x_image_id_master_exact(revoke->image_id, authenticated_master_node) + || revoke->ref.grant_generation == 0 || !pcm_x_wait_identity_valid(&revoke->ref.identity) + || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (header->peer_frontiers[authenticated_master_node].cluster_epoch + != revoke->ref.identity.cluster_epoch + || header->peer_frontiers[authenticated_master_node].sender_session_incarnation + != authenticated_master_session) + return PCM_X_QUEUE_STALE; + result = pcm_x_local_holder_transfer_tag_prepare(&revoke->ref, authenticated_master_node, + authenticated_master_session, &prepared_tag, + &tag_ref, &new_tag); + if (result != PCM_X_QUEUE_OK) + return result; + gate_claimed = true; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&revoke->ref.identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, + prepared_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &revoke->ref.identity.tag, + PCM_X_STATE_BIT(new_tag ? PCM_X_TAG_RESERVED_NONVISIBLE : PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_STALE; + tag_slot = prepared_tag; + goto revoke_release_gate; + } + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto revoke_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto revoke_release_gate; + } + if (tag_slot->master_node == -1 && tag_slot->master_session_incarnation == 0) { + tag_slot->master_node = authenticated_master_node; + tag_slot->master_session_incarnation = authenticated_master_session; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto revoke_release_gate; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0) { + result = pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &revoke->ref) ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_BUSY; + goto revoke_release_gate; + } + if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { + if (!pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &revoke->ref) + || tag_slot->holder_image.image_id != revoke->image_id) { + result = PCM_X_QUEUE_STALE; + goto revoke_release_gate; + } + if ((!pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + && !pcm_x_transfer_leg_exact(&tag_slot->holder_reliable, + PGRAC_IC_MSG_PCM_X_IMAGE_READY, authenticated_master_node, + authenticated_master_session)) + || (pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + && tag_slot->holder_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_REVOKE)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto revoke_release_gate; + } + result = PCM_X_QUEUE_DUPLICATE; + goto revoke_release_gate; + } + /* Type 49 is the fail-closed boundary that replaces the legacy type-48 + * unconditional ACK. The same ticket locator's immutable blocker set must + * have received an exact type-48 ACK and reclaimed every blocker slot before + * holder revoke state can become visible. A premature 49 is retryable; an + * inconsistent tombstone is corruption. */ + if (pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { + result = PCM_X_QUEUE_NOT_READY; + goto revoke_release_gate; + } + if (!pcm_x_ticket_locator_equal(&tag_slot->blocker_snapshot_ref, &revoke->ref)) { + result = PCM_X_QUEUE_STALE; + goto revoke_release_gate; + } + if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto revoke_release_gate; + } + if (tag_slot->blocker_set_generation == 0 + || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto revoke_release_gate; + } + if (!pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + || tag_slot->holder_terminal_drain_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto revoke_release_gate; + } + tag_slot->holder_ref = revoke->ref; + memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + tag_slot->holder_image.image_id = revoke->image_id; + pcm_x_transfer_leg_clear(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_REVOKE, + authenticated_master_node); + memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); + tag_slot->blocker_set_generation = 0; + memset(&tag_slot->blocker_snapshot_reliable, 0, sizeof(tag_slot->blocker_snapshot_reliable)); + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); + if (new_tag) { + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + tag_published = true; + } + result = PCM_X_QUEUE_OK; + +revoke_release_gate: + if (gate_claimed && !fail_closed && (!new_tag || tag_published)) { + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + gate_claimed = false; + } +revoke_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE && new_tag && !fail_closed + && !pcm_x_local_holder_transfer_tag_abort_new(&revoke->ref.identity.tag, tag_ref, + prepared_tag)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_progress_exact(const BufferTag *tag, + PcmXLocalHolderProgress *progress_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (progress_out != NULL) + memset(progress_out, 0, sizeof(*progress_out)); + if (header == NULL || tag == NULL || progress_out == NULL) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_NOT_FOUND; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, 0)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) + result = PCM_X_QUEUE_NOT_FOUND; + else if (!pcm_x_local_terminal_ref_valid(&tag_slot->holder_ref) + || tag_slot->holder_ref.grant_generation == 0 || tag_slot->holder_image.image_id == 0 + || !pcm_x_local_holder_transfer_peer_exact(tag_slot, tag_slot->master_node, + tag_slot->master_session_incarnation)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else { + progress_out->ref = tag_slot->holder_ref; + progress_out->image = tag_slot->holder_image; + progress_out->reliable_state_sequence = tag_slot->holder_reliable.state_sequence; + progress_out->pending_opcode = tag_slot->holder_reliable.pending_opcode; + progress_out->last_response_opcode = tag_slot->holder_reliable.last_response_opcode; + progress_out->phase = tag_slot->holder_reliable.phase; + /* Expose terminal holder state, not the reliable-leg reserved flags. + * Consumers need this positive evidence after DRAIN clears the leg. */ + progress_out->flags = (uint16)(pcm_x_slot_flags_read(&tag_slot->slot) + & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + progress_out->master_session_incarnation = tag_slot->master_session_incarnation; + progress_out->master_node = tag_slot->master_node; + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + memset(progress_out, 0, sizeof(*progress_out)); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_queue_invalidate_authorize_exact(const BufferTag *tag, uint64 cluster_epoch, + uint64 request_id, int32 requester_node, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 flags; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || tag == NULL || request_id == 0 || requester_node < 0 + || requester_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_master_node < 0 + || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (header->peer_frontiers[authenticated_master_node].cluster_epoch != cluster_epoch + || header->peer_frontiers[authenticated_master_node].sender_session_incarnation + != authenticated_master_session) + return PCM_X_QUEUE_STALE; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_NOT_FOUND; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_runtime_token_exact(&runtime, 0)) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) + result = PCM_X_QUEUE_STALE; + else { + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 + || (flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) + result = PCM_X_QUEUE_NOT_READY; + else if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) + result = PCM_X_QUEUE_NOT_READY; + else if (pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) + result = PCM_X_QUEUE_NOT_READY; + else if (tag_slot->blocker_snapshot_ref.grant_generation != 0 + || !pcm_x_local_admission_ref_valid(&tag_slot->blocker_snapshot_ref, + &tag_slot->blocker_snapshot_ref.identity) + || !BufferTagsEqual(tag, &tag_slot->blocker_snapshot_ref.identity.tag) + || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != cluster_epoch + || tag_slot->blocker_snapshot_ref.identity.request_id != request_id + || tag_slot->blocker_snapshot_ref.identity.node_id != requester_node) + result = PCM_X_QUEUE_STALE; + else if (!pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) + || tag_slot->blocker_set_generation == 0) + result = PCM_X_QUEUE_NOT_READY; + else if (tag_slot->blocker_set_generation == UINT64_MAX + || tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_reliable.last_response_opcode + != PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK + || tag_slot->blocker_snapshot_reliable.last_responder_node + != (uint32)authenticated_master_node + || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + || tag_slot->holder_terminal_drain_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_holder_image_ready_arm_exact(const PcmXGrantPayload *image_ready, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXGrantPayload *replay_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint64 next_sequence; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + + if (replay_out != NULL) + memset(replay_out, 0, sizeof(*replay_out)); + if (header == NULL || image_ready == NULL || replay_out == NULL + || image_ready->ref.grant_generation == 0 + || !pcm_x_wait_identity_valid(&image_ready->ref.identity) + || !pcm_x_image_token_valid(&image_ready->image) + || !pcm_x_image_id_master_exact(image_ready->image.image_id, authenticated_master_node) + || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, + &image_ready->ref.identity.tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_NOT_FOUND; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + partition + = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&image_ready->ref.identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + &image_ready->ref.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_STALE; + goto ready_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto ready_done; + } + gate_claimed = true; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto ready_release_gate; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session) + || !pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &image_ready->ref) + || tag_slot->holder_image.image_id != image_ready->image.image_id) { + result = PCM_X_QUEUE_STALE; + goto ready_release_gate; + } + if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_NOT_READY; + goto ready_release_gate; + } + if (!pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable)) { + if (!pcm_x_transfer_leg_exact(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_IMAGE_READY, + authenticated_master_node, authenticated_master_session)) { + result = PCM_X_QUEUE_BUSY; + goto ready_release_gate; + } + result = pcm_x_image_token_equal(&tag_slot->holder_image, &image_ready->image) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + goto ready_output; + } + if (tag_slot->holder_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_REVOKE) { + result = PCM_X_QUEUE_BAD_STATE; + goto ready_release_gate; + } + if (!cluster_pcm_x_generation_next(tag_slot->holder_reliable.state_sequence, &next_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto ready_release_gate; + } + tag_slot->holder_image = image_ready->image; + tag_slot->holder_reliable.state_sequence = next_sequence; + tag_slot->holder_reliable.expected_responder_session = authenticated_master_session; + tag_slot->holder_reliable.expected_responder_node = authenticated_master_node; + tag_slot->holder_reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + tag_slot->holder_reliable.phase = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + tag_slot->holder_reliable.flags = 0; + tag_slot->holder_reliable.reserved = 0; + result = PCM_X_QUEUE_OK; + +ready_output: + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + replay_out->ref = tag_slot->holder_ref; + replay_out->image = tag_slot->holder_image; + } +ready_release_gate: + if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +ready_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_join_begin(const PcmXWaitIdentity *identity, int32 master_node, + uint64 master_session_incarnation, PcmXLocalHandle *handle_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXLocalTagSlot *tag_slot = NULL; + PcmXLocalMembershipSlot *member = NULL; + PcmXSlotHeader *raw_slot; + PcmXSlotRef tag_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef member_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef existing; + PcmXDirectoryResult directory_result; + PcmXAllocatorResult allocator_result; + PcmXQueueResult result = PCM_X_QUEUE_OK; + uint64 local_sequence = 0; + uint32 flags; + uint32 local_round; + uint32 partition; + bool new_tag = false; + bool gate_claimed = false; + bool tag_directory_published = false; + bool member_directory_published = false; + bool fail_closed = false; + bool adopt_holder_only_tag = false; + + pcm_x_local_handle_clear(handle_out); + if (header == NULL || handle_out == NULL || !pcm_x_wait_identity_valid(identity) + || master_node < 0 || master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || master_session_incarnation == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + if (!pcm_x_runtime_token_exact(&runtime, runtime.master_session_incarnation)) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_done; + } + frontier = &header->peer_frontiers[master_node]; + if (frontier->cluster_epoch == 0 && frontier->sender_session_incarnation == 0 + && frontier->next_expected_prehandle_sequence == 1 + && frontier->retired_prehandle_sequence == 0) { + result = PCM_X_QUEUE_NOT_READY; + goto allocator_done; + } + if (frontier->cluster_epoch != identity->cluster_epoch + || frontier->sender_session_incarnation != master_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto allocator_done; + } + result = pcm_x_local_member_lookup_locked(identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + goto allocator_duplicate; + } + if (result != PCM_X_QUEUE_NOT_FOUND) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto allocator_done; + } + result = pcm_x_local_retire_episode_state_locked(header); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &identity->tag, &tag_ref); + if (directory_result == PCM_X_DIRECTORY_OK) { + tag_slot + = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + result = pcm_x_resident_tag_state_result(tag_slot == NULL ? NULL : &tag_slot->slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + if (!BufferTagsEqual(&tag_slot->tag, &identity->tag) + || tag_slot->cluster_epoch != identity->cluster_epoch) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + gate_claimed = true; + if (tag_slot->master_node == -1 && tag_slot->master_session_incarnation == 0) + adopt_holder_only_tag = true; + else if (tag_slot->master_node != master_node + || tag_slot->master_session_incarnation != master_session_incarnation) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + if (adopt_holder_only_tag + && (tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + || tag_slot->membership_count != 0 + || tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->next_sequence != 1 + || tag_slot->reliable.pending_opcode != 0 || tag_slot->reliable.phase != 0 + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) + != 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto allocator_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (tag_slot->next_sequence == 0 || tag_slot->next_sequence == UINT64_MAX + || tag_slot->membership_count == SIZE_MAX + || (((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) + && tag_slot->local_round == UINT32_MAX)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + goto allocator_done; + } + local_sequence = tag_slot->next_sequence; + } else if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) { + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_LOCAL_TAG, &tag_ref, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto allocator_done; + } + new_tag = true; + gate_claimed = true; + tag_slot = (PcmXLocalTagSlot *)raw_slot; + pcm_x_local_tag_init_common(tag_slot, &identity->tag, identity->cluster_epoch, master_node, + master_session_incarnation); + local_sequence = 1; + } else { + result = pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto allocator_done; + } + + allocator_result + = pcm_x_allocator_reserve_locked(PCM_X_ALLOC_LOCAL_WAIT, &member_ref, &raw_slot); + if (allocator_result != PCM_X_ALLOC_OK) { + result = pcm_x_queue_result_from_allocator(allocator_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (new_tag + && !pcm_x_local_stage_rollback_locked(true, false, false, &identity->tag, identity, + tag_ref, NULL, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + member = (PcmXLocalMembershipSlot *)raw_slot; + member->identity = *identity; + member->local_sequence = local_sequence; + member->tag_slot_index = tag_ref.slot_index; + member->tag_slot_generation = tag_ref.slot_generation; + member->next_index = PCM_X_INVALID_SLOT_INDEX; + member->prev_index = PCM_X_INVALID_SLOT_INDEX; + member->buffer_id = -1; + member->partition = PCM_X_LOCAL_PARTITION_WAIT; + + if (new_tag) { + directory_result = pcm_x_directory_insert_locked(PCM_X_DIR_LOCAL_TAG, &identity->tag, + tag_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + if (!pcm_x_local_stage_rollback_locked(true, false, false, &identity->tag, identity, + tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + tag_directory_published = true; + } + directory_result + = pcm_x_directory_insert_locked(PCM_X_DIR_LOCAL_WAIT, identity, member_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + result = directory_result == PCM_X_DIRECTORY_EXISTS + ? PCM_X_QUEUE_CORRUPT + : pcm_x_queue_result_from_directory(directory_result); + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + if (!pcm_x_local_stage_rollback_locked(new_tag, tag_directory_published, false, + &identity->tag, identity, tag_ref, member, + member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + goto allocator_done; + } + member_directory_published = true; + (void)member_directory_published; + tag_slot->next_sequence = local_sequence + 1; + result = PCM_X_QUEUE_OK; + +allocator_done: + if (result != PCM_X_QUEUE_OK && gate_claimed && !new_tag) { + if (fail_closed) { + /* + * A post-claim allocator invariant failed. Keep the gate for + * recovery and report CORRUPT; retryable outcomes release below. + */ + result = PCM_X_QUEUE_CORRUPT; + } else { + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + gate_claimed = false; + } + } + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&identity->tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, tag_slot); + { + PcmXAllocatorView tag_view; + PcmXLocalTagSlot *raw_tag; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &tag_view)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + raw_tag = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&tag_view, tag_ref.slot_index); + if (raw_tag == NULL + || (pcm_x_slot_flags_read(&raw_tag->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + pg_read_barrier(); + } + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &identity->tag, + PCM_X_STATE_BIT(new_tag ? PCM_X_TAG_RESERVED_NONVISIBLE : PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &identity->tag, + PCM_X_STATE_BIT(PCM_XL_RESERVED_NONVISIBLE)); + if (tag_slot == NULL || member == NULL + || !pcm_x_wait_identity_equal(&member->identity, identity) + || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (tag_slot->membership_count == SIZE_MAX) { + /* The allocator phase checked this while holding the same gate. */ + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (adopt_holder_only_tag) { + if (tag_slot->master_node != -1 || tag_slot->master_session_incarnation != 0 + || tag_slot->membership_count != 0 || tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + tag_slot->master_node = master_node; + tag_slot->master_session_incarnation = master_session_incarnation; + } else if (tag_slot->master_node != master_node + || tag_slot->master_session_incarnation != master_session_incarnation) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + tag_slot->head_index = member_ref.slot_index; + } else { + PcmXAllocatorView member_view; + PcmXLocalMembershipSlot *tail; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + tail = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, tag_slot->tail_index); + if (tail == NULL || tail->next_index != PCM_X_INVALID_SLOT_INDEX + || tail->tag_slot_index != tag_ref.slot_index + || tail->tag_slot_generation != tag_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + member->prev_index = tag_slot->tail_index; + tail->next_index = member_ref.slot_index; + } + tag_slot->tail_index = member_ref.slot_index; + tag_slot->membership_count++; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + local_round = tag_slot->local_round; + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) + local_round++; + member->admitted_round = local_round; + /* A closed round has no leader by construction. New arrivals belong to + * next_round and must remain non-waitable followers until exact round + * retirement reopens the tag and promotes the oldest survivor. */ + if (tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + tag_slot->leader_index = member_ref.slot_index; + tag_slot->leader_slot_generation = member_ref.slot_generation; + member->role = PCM_X_LOCAL_ROLE_NODE_LEADER; + pcm_x_slot_state_write(&member->slot, PCM_XL_NODE_LEADER); + } else { + member->role = PCM_X_LOCAL_ROLE_FOLLOWER; + pcm_x_slot_state_write(&member->slot, PCM_XL_JOINED_NONWAITABLE); + } + if (new_tag) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_LIVE); + } + pcm_x_local_handle_from_member(handle_out, member_ref, member, (PcmXLocalRole)member->role); + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + goto domain_done; + } + if (member->role == PCM_X_LOCAL_ROLE_FOLLOWER) + pcm_x_stats_increment(&header->stats.coalesced_count); + result = PCM_X_QUEUE_OK; + +domain_done: + /* Any post-publication jump here is CORRUPT and retains the gate. */ + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + pcm_x_runtime_fail_closed(); + return result; + +allocator_duplicate: + LWLockRelease(&header->allocator_lock.lock); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&identity->tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &identity->tag, PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, &identity->tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (member != NULL && tag_slot != NULL + && pcm_x_wait_identity_equal(&member->identity, identity)) { + pcm_x_local_handle_from_member(handle_out, member_ref, member, (PcmXLocalRole)member->role); + result = PCM_X_QUEUE_DUPLICATE; + } else { + PcmXAllocatorView member_view; + PcmXAllocatorView tag_view; + PcmXLocalMembershipSlot *raw_member; + PcmXLocalTagSlot *raw_tag; + uint64 member_generation1; + uint64 member_generation2; + uint64 tag_generation; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view) + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &tag_view)) { + result = PCM_X_QUEUE_CORRUPT; + goto duplicate_done; + } + raw_member + = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, member_ref.slot_index); + raw_tag = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&tag_view, tag_ref.slot_index); + if (raw_member != NULL + && !pcm_x_slot_generation_read(&raw_member->slot, &member_generation1)) { + result = PCM_X_QUEUE_STALE; + goto duplicate_done; + } + if (raw_tag != NULL && !pcm_x_slot_generation_read(&raw_tag->slot, &tag_generation)) { + result = PCM_X_QUEUE_STALE; + goto duplicate_done; + } + if (raw_member != NULL && raw_tag != NULL + && member_generation1 == member_ref.slot_generation + && pcm_x_slot_state_read(&raw_member->slot) == PCM_XL_RESERVED_NONVISIBLE + && tag_generation == tag_ref.slot_generation + && (pcm_x_slot_flags_read(&raw_tag->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) { + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&raw_member->slot, &member_generation2)) { + result = PCM_X_QUEUE_STALE; + goto duplicate_done; + } + result + = member_generation2 == member_ref.slot_generation + && pcm_x_slot_state_read(&raw_member->slot) == PCM_XL_RESERVED_NONVISIBLE + && (pcm_x_slot_flags_read(&raw_tag->slot) + & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) + != 0 + ? PCM_X_QUEUE_BUSY + : PCM_X_QUEUE_STALE; + } else + result = PCM_X_QUEUE_STALE; + } + +duplicate_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Resolve one already-published local membership without admitting a new one. */ +PcmXQueueResult +cluster_pcm_x_local_lookup_exact(const PcmXWaitIdentity *identity, PcmXLocalHandle *handle_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 member_state; + uint32 partition; + bool fail_closed = false; + + pcm_x_local_handle_clear(handle_out); + if (header == NULL || identity == NULL || handle_out == NULL + || !pcm_x_wait_identity_valid(identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_member_lookup_locked(identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&identity->tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, &identity->tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &identity->tag, PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL) { + result = PCM_X_QUEUE_STALE; + goto lookup_done; + } + /* The directory may already carry a rekey target while this resident still + * carries the old identity. Check the cross-lock marker before reading any + * identity field. */ + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) { + result = PCM_X_QUEUE_BUSY; + goto lookup_done; + } + member_state = pcm_x_slot_state_read(&member->slot); + if (!pcm_x_wait_identity_equal(&member->identity, identity) + || member->partition != PCM_X_LOCAL_PARTITION_WAIT + || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || !BufferTagsEqual(&tag_slot->tag, &identity->tag) + || tag_slot->cluster_epoch != identity->cluster_epoch || tag_slot->master_node < 0 + || tag_slot->master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || tag_slot->master_session_incarnation == 0 || tag_slot->membership_count == 0 + || (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + && member->role != PCM_X_LOCAL_ROLE_FOLLOWER) + || (member->role == PCM_X_LOCAL_ROLE_NODE_LEADER && member_state != PCM_XL_NODE_LEADER + && member_state != PCM_XL_REMOTE_WAIT && member_state != PCM_XL_CONTENT_ACTIVE + && member_state != PCM_XL_GRANTED) + || (member->role == PCM_X_LOCAL_ROLE_FOLLOWER && member_state != PCM_XL_JOINED_NONWAITABLE + && member_state != PCM_XL_WAITABLE_FOLLOWER && member_state != PCM_XL_CANCELLED)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto lookup_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto lookup_done; + } + pcm_x_local_handle_from_member(handle_out, member_ref, member, (PcmXLocalRole)member->role); + result = PCM_X_QUEUE_OK; + +lookup_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static void +pcm_x_local_reliable_token_clear(PcmXLocalReliableToken *token) +{ + if (token != NULL) + memset(token, 0, sizeof(*token)); +} + + +static bool +pcm_x_local_reliable_leg_is_clear(const PcmXReliableLegState *leg) +{ + return leg != NULL && leg->retry_deadline_ms == 0 && leg->expected_responder_session == 0 + && leg->retry_count == 0 && leg->expected_responder_node == 0 && leg->pending_opcode == 0 + && leg->phase == 0 && leg->flags == 0 && leg->reserved == 0; +} + + +static bool +pcm_x_ticket_ref_is_zero(const PcmXTicketRef *ref) +{ + PcmXTicketRef zero; + + memset(&zero, 0, sizeof(zero)); + return ref != NULL && pcm_x_ticket_ref_equal(ref, &zero); +} + + +static bool +pcm_x_local_admission_ref_valid(const PcmXTicketRef *ref, const PcmXWaitIdentity *identity) +{ + return ref != NULL && identity != NULL && ref->handle.ticket_id != 0 + && ref->handle.queue_generation != 0 && ref->grant_generation == 0 + && pcm_x_wait_identity_equal(&ref->identity, identity); +} + + +/* Terminal refs keep the exact locator and the final non-sentinel grant. */ +static bool +pcm_x_local_terminal_ref_valid(const PcmXTicketRef *ref) +{ + return ref != NULL && pcm_x_wait_identity_valid(&ref->identity) && ref->handle.ticket_id != 0 + && ref->handle.queue_generation != 0 && ref->grant_generation != UINT64_MAX; +} + + +static PcmXQueueResult +pcm_x_local_refs_lookup(const PcmXLocalHandle *leader, PcmXSlotRef *tag_ref_out, + PcmXSlotRef *member_ref_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalMembershipSlot *member; + PcmXSlotRef member_ref; + PcmXQueueResult result; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_member_lookup_locked(&leader->identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref_out->slot_index = member->tag_slot_index; + tag_ref_out->slot_generation = member->tag_slot_generation; + *member_ref_out = member_ref; + } + LWLockRelease(&header->allocator_lock.lock); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_progress_exact(const PcmXLocalHandle *handle, PcmXLocalProgress *progress_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 member_state; + uint32 partition; + bool fail_closed = false; + + if (progress_out != NULL) + memset(progress_out, 0, sizeof(*progress_out)); + if (header == NULL || handle == NULL || progress_out == NULL || handle->flags != 0 + || !pcm_x_wait_identity_valid(&handle->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(handle, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&handle->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &handle->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &handle->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL) { + result = PCM_X_QUEUE_STALE; + goto progress_done; + } + /* See cluster_pcm_x_local_lookup_exact(): never dereference a resident + * identity while the gated rekey writer owns the local-lock phase. */ + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) { + result = PCM_X_QUEUE_BUSY; + goto progress_done; + } + member_state = pcm_x_slot_state_read(&member->slot); + if (!pcm_x_wait_identity_equal(&member->identity, &handle->identity) + || member->partition != PCM_X_LOCAL_PARTITION_WAIT + || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || !BufferTagsEqual(&tag_slot->tag, &handle->identity.tag) + || tag_slot->cluster_epoch != handle->identity.cluster_epoch || tag_slot->master_node < 0 + || tag_slot->master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || tag_slot->master_session_incarnation == 0 || tag_slot->membership_count == 0 + || (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + && member->role != PCM_X_LOCAL_ROLE_FOLLOWER) + || (member->role == PCM_X_LOCAL_ROLE_NODE_LEADER && member_state != PCM_XL_NODE_LEADER + && member_state != PCM_XL_REMOTE_WAIT && member_state != PCM_XL_CONTENT_ACTIVE + && member_state != PCM_XL_GRANTED) + || (member->role == PCM_X_LOCAL_ROLE_FOLLOWER && member_state != PCM_XL_JOINED_NONWAITABLE + && member_state != PCM_XL_WAITABLE_FOLLOWER && member_state != PCM_XL_CANCELLED)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto progress_done; + } + if (!pcm_x_local_handle_exact(handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto progress_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto progress_done; + } + + progress_out->identity = member->identity; + progress_out->ref = tag_slot->ref; + progress_out->image = tag_slot->image; + progress_out->local_sequence = member->local_sequence; + progress_out->reliable_state_sequence = tag_slot->reliable.state_sequence; + progress_out->local_round = member->admitted_round; + progress_out->member_state = member_state; + progress_out->role = member->role; + progress_out->pending_opcode = tag_slot->reliable.pending_opcode; + progress_out->last_response_opcode = tag_slot->reliable.last_response_opcode; + progress_out->phase = tag_slot->reliable.phase; + progress_out->master_session_incarnation = tag_slot->master_session_incarnation; + progress_out->master_node = tag_slot->master_node; + result = PCM_X_QUEUE_OK; + +progress_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + memset(progress_out, 0, sizeof(*progress_out)); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_follower_wfg_snapshot_exact(const PcmXLocalHandle *follower, + PcmXLocalFollowerWfgSnapshot *snapshot_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *blocker; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef blocker_ref; + PcmXQueueResult result; + uint32 member_state; + uint32 partition; + bool fail_closed = false; + + pcm_x_local_follower_wfg_snapshot_clear(snapshot_out); + if (header == NULL || follower == NULL || snapshot_out == NULL || follower->flags != 0 + || follower->role != PCM_X_LOCAL_ROLE_FOLLOWER + || !pcm_x_wait_identity_valid(&follower->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(follower, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&follower->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &follower->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &follower->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(follower, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto snapshot_done; + } + member_state = pcm_x_slot_state_read(&member->slot); + if (member->role != PCM_X_LOCAL_ROLE_FOLLOWER + || (member_state != PCM_XL_JOINED_NONWAITABLE && member_state != PCM_XL_WAITABLE_FOLLOWER) + || (pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto snapshot_done; + } + if ((member_state == PCM_XL_JOINED_NONWAITABLE) != (member->graph_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto snapshot_done; + } + result = pcm_x_local_execution_round_locked(tag_slot, member); + /* Next-round writers cannot claim yet, but they are real waiters and must + * remain visible to the cross-node WFG while the frozen cohort drains. */ + if (result == PCM_X_QUEUE_CORRUPT) { + fail_closed = true; + goto snapshot_done; + } + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0) + || ((tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_slot_generation == 0))) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + /* A holder-only freeze has no local writer blocker. The same empty-leader + * window is reachable after a closed-round leader drains and before exact + * round retirement promotes the oldest late writer. Keep the backend + * behind the barrier without inventing a WFG edge or halting the runtime. */ + if (tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) { + if (result == PCM_X_QUEUE_BARRIER_CLOSED + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_BARRIER_CLOSED; + goto snapshot_done; + } + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + blocker_ref.slot_index = tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + ? tag_slot->active_writer_index + : tag_slot->leader_index; + blocker_ref.slot_generation = tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + ? tag_slot->active_writer_slot_generation + : tag_slot->leader_slot_generation; + if (blocker_ref.slot_index == member_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + blocker = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, blocker_ref, + &follower->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (blocker == NULL || blocker->partition != PCM_X_LOCAL_PARTITION_WAIT + || blocker->tag_slot_index != tag_ref.slot_index + || blocker->tag_slot_generation != tag_ref.slot_generation + || !pcm_x_wait_identity_valid(&blocker->identity) + || blocker->local_sequence >= member->local_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + /* An empty active-writer slot is a transient scheduling window after + * completion. Do not publish a new edge to the completed node leader; + * the caller retries until the next FIFO writer is active. */ + if ((pcm_x_slot_flags_read(&blocker->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_BUSY; + goto snapshot_done; + } + + snapshot_out->waiter = *follower; + snapshot_out->blocker = blocker->identity; + snapshot_out->blocker_slot = blocker_ref; + snapshot_out->waiter_graph_generation = member->graph_generation; + snapshot_out->leader_index = tag_slot->leader_index; + snapshot_out->active_writer_index = tag_slot->active_writer_index; + snapshot_out->leader_slot_generation = tag_slot->leader_slot_generation; + snapshot_out->active_writer_slot_generation = tag_slot->active_writer_slot_generation; + snapshot_out->holder_set_generation = tag_slot->holder_set_generation; + snapshot_out->local_round = member->admitted_round; + result = PCM_X_QUEUE_OK; + +snapshot_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + pcm_x_local_follower_wfg_snapshot_clear(snapshot_out); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_follower_wfg_commit_exact(const PcmXLocalFollowerWfgSnapshot *snapshot, + uint64 graph_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *blocker; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef expected_blocker; + PcmXQueueResult result; + uint32 member_state; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + + if (header == NULL || snapshot == NULL || graph_generation == 0 || snapshot->reserved != 0 + || snapshot->waiter.flags != 0 || snapshot->waiter.role != PCM_X_LOCAL_ROLE_FOLLOWER + || snapshot->local_round == 0 || !pcm_x_wait_identity_valid(&snapshot->waiter.identity) + || !pcm_x_wait_identity_valid(&snapshot->blocker) + || snapshot->blocker_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || snapshot->blocker_slot.slot_generation == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(&snapshot->waiter, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition + = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&snapshot->waiter.identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + &snapshot->waiter.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &snapshot->waiter.identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(&snapshot->waiter, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto commit_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto commit_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto commit_done; + } + gate_claimed = true; + result = pcm_x_local_execution_round_locked(tag_slot, member); + /* Barrier-closed next-round writers still publish exact WFG edges; only + * execution/claim is held until round retirement. */ + if (result == PCM_X_QUEUE_CORRUPT) { + fail_closed = true; + goto commit_release_gate; + } + member_state = pcm_x_slot_state_read(&member->slot); + if (member->role != PCM_X_LOCAL_ROLE_FOLLOWER + || (member_state != PCM_XL_JOINED_NONWAITABLE && member_state != PCM_XL_WAITABLE_FOLLOWER) + || (pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto commit_release_gate; + } + if ((member_state == PCM_XL_JOINED_NONWAITABLE) != (member->graph_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto commit_done; + } + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0) + || tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_slot_generation == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto commit_done; + } + expected_blocker.slot_index = tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + ? tag_slot->active_writer_index + : tag_slot->leader_index; + expected_blocker.slot_generation = tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + ? tag_slot->active_writer_slot_generation + : tag_slot->leader_slot_generation; + if (snapshot->local_round != member->admitted_round + || snapshot->leader_index != tag_slot->leader_index + || snapshot->active_writer_index != tag_slot->active_writer_index + || snapshot->leader_slot_generation != tag_slot->leader_slot_generation + || snapshot->active_writer_slot_generation != tag_slot->active_writer_slot_generation + || snapshot->holder_set_generation != tag_slot->holder_set_generation + || snapshot->blocker_slot.slot_index != expected_blocker.slot_index + || snapshot->blocker_slot.slot_generation != expected_blocker.slot_generation) { + result = PCM_X_QUEUE_STALE; + goto commit_release_gate; + } + blocker = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, expected_blocker, + &snapshot->waiter.identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (blocker == NULL || blocker->partition != PCM_X_LOCAL_PARTITION_WAIT + || blocker->tag_slot_index != tag_ref.slot_index + || blocker->tag_slot_generation != tag_ref.slot_generation + || !pcm_x_wait_identity_equal(&blocker->identity, &snapshot->blocker) + || blocker->local_sequence >= member->local_sequence) { + result = PCM_X_QUEUE_STALE; + goto commit_release_gate; + } + /* Claim/release can ABA active_writer back to INVALID while leaving the + * same leader locator. COMPLETE closes that semantic ABA: the external + * graph must be removed and replaced, never committed against old proof. */ + if ((pcm_x_slot_flags_read(&blocker->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_STALE; + goto commit_release_gate; + } + if (member->graph_generation == graph_generation) { + result = member_state == PCM_XL_WAITABLE_FOLLOWER ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_CORRUPT; + if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + goto commit_release_gate; + } + if (snapshot->waiter_graph_generation != member->graph_generation) { + result = PCM_X_QUEUE_STALE; + goto commit_release_gate; + } + member->graph_generation = graph_generation; + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_WAITABLE_FOLLOWER); + result = PCM_X_QUEUE_OK; + +commit_release_gate: + if (!fail_closed && gate_claimed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +commit_done: + /* Structural failures retain the gate as recovery evidence. */ + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_follower_wfg_clear_exact(const PcmXLocalHandle *follower, + uint64 graph_generation) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 member_state; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + + if (header == NULL || follower == NULL || graph_generation == 0 || follower->flags != 0 + || follower->role != PCM_X_LOCAL_ROLE_FOLLOWER + || !pcm_x_wait_identity_valid(&follower->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(follower, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&follower->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &follower->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &follower->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(follower, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto clear_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto clear_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto clear_done; + } + gate_claimed = true; + if (member->role != PCM_X_LOCAL_ROLE_FOLLOWER + || (pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto clear_release_gate; + } + member_state = pcm_x_slot_state_read(&member->slot); + if (member_state == PCM_XL_JOINED_NONWAITABLE) { + if (member->graph_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto clear_done; + } + result = PCM_X_QUEUE_DUPLICATE; + goto clear_release_gate; + } + if (member_state != PCM_XL_WAITABLE_FOLLOWER) { + result = PCM_X_QUEUE_BAD_STATE; + goto clear_release_gate; + } + if (member->graph_generation == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto clear_done; + } + if (member->graph_generation != graph_generation) { + result = PCM_X_QUEUE_STALE; + goto clear_release_gate; + } + member->graph_generation = 0; + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_JOINED_NONWAITABLE); + result = PCM_X_QUEUE_OK; + +clear_release_gate: + if (!fail_closed && gate_claimed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +clear_done: + /* Structural failures retain the gate as recovery evidence. */ + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_local_follower_chain_ready(const PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + Size first_index, Size expected_prev_index, + uint64 expected_prev_sequence, PcmXLocalMembershipSlot **first_out, + PcmXSlotRef *first_ref_out, bool *first_promote_out); + + +/* A promoted successor is directory-visible before terminal image/ticket + * evidence is retired. Timed waiter wakeups must not let that successor + * claim content authority, refreeze the round, or mint a new ENQUEUE while + * either cleanup lane is still durable. */ +static PcmXQueueResult +pcm_x_local_terminal_cleanup_fence(const PcmXLocalTagSlot *tag_slot) +{ + uint32 flags; + uint32 holder_flags; + uint32 writer_flags; + + if (tag_slot == NULL) + return PCM_X_QUEUE_CORRUPT; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + writer_flags = flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK; + holder_flags = flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + if ((writer_flags == 0 && tag_slot->terminal_drain_generation != 0) + || (writer_flags != 0 && writer_flags != PCM_X_LOCAL_TAG_F_TERMINAL_READY + && writer_flags != PCM_X_LOCAL_TAG_F_TERMINAL_MASK) + || (writer_flags == PCM_X_LOCAL_TAG_F_TERMINAL_READY + && tag_slot->terminal_drain_generation != 0) + || (writer_flags == PCM_X_LOCAL_TAG_F_TERMINAL_MASK + && (tag_slot->terminal_drain_generation == 0 + || tag_slot->terminal_drain_generation == UINT64_MAX)) + || (holder_flags == 0) != (tag_slot->holder_terminal_drain_generation == 0) + || (holder_flags != 0 + && (holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + || tag_slot->holder_terminal_drain_generation == 0 + || tag_slot->holder_terminal_drain_generation == UINT64_MAX))) + return PCM_X_QUEUE_CORRUPT; + return writer_flags != 0 || holder_flags != 0 ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_OK; +} + + +/* + * Move one promoted local leader to the ownership generation sampled by its + * caller immediately before the first ENQUEUE. + * + * The tag admission gate is the cross-lock transaction marker. It freezes + * ENQUEUE, cancellation, writer claims, and terminal cleanup while the local + * lock proves that the leader has no graph or outbound authority. The + * allocator phase then replaces only the immutable directory key. While its + * target key names the still-old resident identity, lookup/progress observe + * the gate and return BUSY without reading that identity. A final local-lock + * phase commits the resident identity and then opens the gate. If target-key + * publication fails, the old key is restored without ever changing resident + * bytes. + */ +PcmXQueueResult +cluster_pcm_x_local_leader_rekey_generation_exact(const PcmXLocalHandle *leader, + uint64 base_own_generation, + PcmXLocalHandle *rekeyed_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXWaitIdentity target_identity; + PcmXWaitIdentity old_identity; + PcmXLocalHandle resident_handle; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *gated_tag = NULL; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef found; + PcmXSlotRef existing; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + PcmXImageToken zero_image; + PcmXTicketHandle zero_handle; + uint32 partition; + uint32 flags; + uint32 member_flags; + uint32 state; + bool fail_closed = false; + bool gate_claimed = false; + bool replay_target = false; + bool migrate = false; + + pcm_x_local_handle_clear(rekeyed_out); + if (header == NULL || leader == NULL || rekeyed_out == NULL || leader->flags != 0 + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + target_identity = leader->identity; + target_identity.base_own_generation = base_own_generation; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + /* Resolve either the original key or an already-committed target key. The + * latter makes a lost-return replay idempotent without reviving the old + * directory key. */ + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + result = pcm_x_local_member_lookup_locked(&leader->identity, &member_ref, &member); + if (result == PCM_X_QUEUE_NOT_FOUND + && !pcm_x_wait_identity_equal(&leader->identity, &target_identity)) { + result = pcm_x_local_member_lookup_locked(&target_identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) + replay_target = true; + } + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto rekey_allocator_lookup_done; + } + if (member_ref.slot_index != leader->membership_slot.slot_index + || member_ref.slot_generation != leader->membership_slot.slot_generation) { + result = replay_target ? PCM_X_QUEUE_CORRUPT : PCM_X_QUEUE_STALE; + fail_closed = replay_target; + goto rekey_allocator_lookup_done; + } + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + resident_handle = *leader; + if (replay_target) + resident_handle.identity = target_identity; + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || tag_ref.slot_index != leader->tag_slot.slot_index + || tag_ref.slot_generation != leader->tag_slot.slot_generation + || member->partition != PCM_X_LOCAL_PARTITION_WAIT + || !BufferTagsEqual(&tag_slot->tag, &leader->identity.tag) + || tag_slot->cluster_epoch != leader->identity.cluster_epoch + || !pcm_x_local_handle_exact(&resident_handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto rekey_allocator_lookup_done; + } + if (!replay_target && !pcm_x_wait_identity_equal(&leader->identity, &target_identity)) { + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_WAIT, &target_identity, &found); + if (directory_result == PCM_X_DIRECTORY_OK) { + /* request_id/wait_seq are immutable; another slot owning the target + * generation is an identity alias, never ordinary contention. */ + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_lookup_done; + } + if (directory_result != PCM_X_DIRECTORY_NOT_FOUND) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_lookup_done; + } + } + result = pcm_x_local_handoff_gate_claim_locked(header, tag_slot, false, PCM_X_QUEUE_BUSY); + if (result == PCM_X_QUEUE_OK) { + gate_claimed = true; + gated_tag = tag_slot; + } else if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + +rekey_allocator_lookup_done: + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (fail_closed) { + pcm_x_runtime_fail_closed(); + } + return result; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, gated_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &leader->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &leader->identity.tag, + PCM_X_STATE_BIT(PCM_XL_NODE_LEADER)); + resident_handle = *leader; + if (replay_target) + resident_handle.identity = target_identity; + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(&resident_handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto rekey_local_release_gate; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto rekey_local_release_gate; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0 + || (tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_done; + } + state = pcm_x_slot_state_read(&member->slot); + if (state != PCM_XL_NODE_LEADER) { + result = PCM_X_QUEUE_BAD_STATE; + goto rekey_local_release_gate; + } + member_flags = pcm_x_slot_flags_read(&member->slot); + if (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || tag_slot->head_index != member_ref.slot_index + || member->prev_index != PCM_X_INVALID_SLOT_INDEX + || (member_flags & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_done; + } + if (member->graph_generation != 0 + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_BUSY; + goto rekey_local_release_gate; + } + result = pcm_x_local_terminal_cleanup_fence(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto rekey_local_release_gate; + } + memset(&zero_handle, 0, sizeof(zero_handle)); + if (!pcm_x_ticket_ref_is_zero(&tag_slot->ref) + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || tag_slot->prehandle.sender_session_incarnation != 0 + || tag_slot->prehandle.prehandle_sequence != 0 + || !pcm_x_ticket_handle_equal(&member->handle, &zero_handle)) { + result = PCM_X_QUEUE_BUSY; + goto rekey_local_release_gate; + } + memset(&zero_image, 0, sizeof(zero_image)); + if (!pcm_x_image_token_equal(&tag_slot->image, &zero_image) + || tag_slot->committed_own_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_done; + } + if (replay_target || pcm_x_wait_identity_equal(&member->identity, &target_identity)) { + pcm_x_local_handle_from_member(rekeyed_out, member_ref, member, + PCM_X_LOCAL_ROLE_NODE_LEADER); + result = PCM_X_QUEUE_DUPLICATE; + goto rekey_local_release_gate; + } + if (base_own_generation < member->identity.base_own_generation) { + result = PCM_X_QUEUE_STALE; + goto rekey_local_release_gate; + } + old_identity = member->identity; + migrate = true; + result = PCM_X_QUEUE_OK; + goto rekey_local_done; + +rekey_local_release_gate: + if (!fail_closed && gate_claimed) { + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + gate_claimed = false; + } +rekey_local_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) { + pcm_x_runtime_fail_closed(); + return result; + } + if (!migrate) + return result; + + /* The gate remains set across the local-to-allocator handoff. No local + * operation can mint authority while the immutable directory key moves. */ + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, gated_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + member = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_WAIT, + member_ref); + if (tag_slot == NULL || member == NULL + || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || pcm_x_slot_state_read(&member->slot) != PCM_XL_NODE_LEADER + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0 + || !pcm_x_wait_identity_equal(&member->identity, &old_identity) + || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_commit_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + gate_claimed = false; + goto rekey_allocator_commit_done; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_WAIT, &old_identity, &found); + if (directory_result != PCM_X_DIRECTORY_OK || found.slot_index != member_ref.slot_index + || found.slot_generation != member_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_commit_done; + } + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_WAIT, &target_identity, &found); + if (directory_result != PCM_X_DIRECTORY_NOT_FOUND) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_commit_done; + } + if (pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_WAIT, &old_identity, member_ref) + != PCM_X_DIRECTORY_OK) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_commit_done; + } + directory_result = pcm_x_directory_insert_rekey_target_locked(&target_identity, &old_identity, + member_ref, &existing); + if (directory_result != PCM_X_DIRECTORY_OK) { + /* Resident bytes never changed. Republish exactly the old key under + * this same allocator lock before closing the runtime. */ + if (pcm_x_directory_insert_locked(PCM_X_DIR_LOCAL_WAIT, &old_identity, member_ref, + &existing) + != PCM_X_DIRECTORY_OK) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_commit_done; + } + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_allocator_commit_done; + } + result = PCM_X_QUEUE_OK; + +rekey_allocator_commit_done: + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + pcm_x_local_handle_clear(rekeyed_out); + if (fail_closed) { + pcm_x_runtime_fail_closed(); + } + return result; + } + + /* The directory now names target_identity while the resident still names + * old_identity. Only this local-lock phase may close that gated window. + * Any acquire or validation failure retains the gate and blocks recovery. */ + pcm_x_local_gate_acquire_guarded_retain_gate(&header->local_locks[partition].lock, + LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &old_identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &old_identity.tag, PCM_X_STATE_BIT(PCM_XL_NODE_LEADER)); + resident_handle = *leader; + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(&resident_handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_commit_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + fail_closed = true; + goto rekey_local_commit_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + member_flags = pcm_x_slot_flags_read(&member->slot); + state = pcm_x_slot_state_read(&member->slot); + if ((flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0 + || (tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0) + || state != PCM_XL_NODE_LEADER || member->partition != PCM_X_LOCAL_PARTITION_WAIT + || member->role != PCM_X_LOCAL_ROLE_NODE_LEADER || tag_slot->membership_count == 0 + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || tag_slot->head_index != member_ref.slot_index + || member->prev_index != PCM_X_INVALID_SLOT_INDEX + || (member_flags & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0 + || member->graph_generation != 0 + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_commit_done; + } + if (pcm_x_local_terminal_cleanup_fence(tag_slot) != PCM_X_QUEUE_OK + || !pcm_x_ticket_ref_is_zero(&tag_slot->ref) + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || tag_slot->prehandle.sender_session_incarnation != 0 + || tag_slot->prehandle.prehandle_sequence != 0 + || !pcm_x_ticket_handle_equal(&member->handle, &zero_handle) + || !pcm_x_image_token_equal(&tag_slot->image, &zero_image) + || tag_slot->committed_own_generation != 0 + || !pcm_x_wait_identity_equal(&member->identity, &old_identity)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_commit_done; + } + + member->identity = target_identity; + pcm_x_local_handle_from_member(rekeyed_out, member_ref, member, PCM_X_LOCAL_ROLE_NODE_LEADER); + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto rekey_local_commit_done; + } + gate_claimed = false; + result = PCM_X_QUEUE_OK; + +rekey_local_commit_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + pcm_x_local_handle_clear(rekeyed_out); + if (fail_closed) { + pcm_x_runtime_fail_closed(); + } + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_writer_claim_exact(const PcmXLocalHandle *writer, + PcmXLocalWriterClaim *claim_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *leader; + PcmXLocalMembershipSlot *candidate; + PcmXAllocatorView member_view; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef leader_ref; + PcmXQueueResult result; + uint32 state; + uint32 partition; + Size candidate_index; + Size previous_index = PCM_X_INVALID_SLOT_INDEX; + Size visited = 0; + uint64 previous_sequence = 0; + uint64 next_claim_generation; + bool gate_claimed = false; + bool fail_closed = false; + + pcm_x_local_writer_claim_clear(claim_out); + if (header == NULL || writer == NULL || claim_out == NULL || writer->flags != 0 + || (writer->role != PCM_X_LOCAL_ROLE_NODE_LEADER + && writer->role != PCM_X_LOCAL_ROLE_FOLLOWER) + || !pcm_x_wait_identity_valid(&writer->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(writer, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&writer->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &writer->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &writer->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(writer, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto claim_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto claim_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto claim_done; + } + gate_claimed = true; + result = pcm_x_local_terminal_cleanup_fence(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto claim_release_gate; + } + result = pcm_x_local_execution_round_locked(tag_slot, member); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + goto claim_release_gate; + } + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + if (tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_BUSY; + goto claim_release_gate; + } + state = pcm_x_slot_state_read(&member->slot); + if ((pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto claim_release_gate; + } + if (member->role == PCM_X_LOCAL_ROLE_NODE_LEADER) { + if (tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || tag_slot->head_index != member_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + if (state != PCM_XL_NODE_LEADER && state != PCM_XL_REMOTE_WAIT + && state != PCM_XL_CONTENT_ACTIVE && state != PCM_XL_GRANTED) { + result = PCM_X_QUEUE_BAD_STATE; + goto claim_release_gate; + } + } else { + if (state != PCM_XL_JOINED_NONWAITABLE || member->graph_generation != 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto claim_release_gate; + } + if (tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_slot_generation == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + leader_ref.slot_index = tag_slot->leader_index; + leader_ref.slot_generation = tag_slot->leader_slot_generation; + leader = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, leader_ref, + &writer->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (leader == NULL || leader->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || leader->local_sequence >= member->local_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + candidate_index = tag_slot->head_index; + while (candidate_index != PCM_X_INVALID_SLOT_INDEX) { + PcmXSlotRef candidate_ref; + uint32 candidate_flags; + + if (visited++ >= tag_slot->membership_count || visited > member_view.capacity) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + candidate + = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, candidate_index); + if (candidate == NULL + || !pcm_x_slot_generation_read(&candidate->slot, &candidate_ref.slot_generation)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + candidate_ref.slot_index = candidate_index; + candidate = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, candidate_ref, &writer->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (candidate == NULL || candidate->tag_slot_index != tag_ref.slot_index + || candidate->tag_slot_generation != tag_ref.slot_generation + || candidate->prev_index != previous_index + || candidate->admitted_round != member->admitted_round + || candidate->local_sequence <= previous_sequence + || candidate->local_sequence > member->local_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + if (candidate_index == member_ref.slot_index) + break; + state = pcm_x_slot_state_read(&candidate->slot); + candidate_flags = pcm_x_slot_flags_read(&candidate->slot); + if (candidate_index == tag_slot->leader_index) { + if (candidate->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || (state != PCM_XL_NODE_LEADER && state != PCM_XL_REMOTE_WAIT + && state != PCM_XL_CONTENT_ACTIVE && state != PCM_XL_GRANTED)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + } else if (candidate->role != PCM_X_LOCAL_ROLE_FOLLOWER + || (state != PCM_XL_JOINED_NONWAITABLE && state != PCM_XL_WAITABLE_FOLLOWER) + || ((state == PCM_XL_JOINED_NONWAITABLE) + != (candidate->graph_generation == 0))) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + if ((candidate_flags & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) == 0) { + result = PCM_X_QUEUE_BUSY; + goto claim_release_gate; + } + if (candidate_index != tag_slot->leader_index + && (state != PCM_XL_JOINED_NONWAITABLE || candidate->graph_generation != 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + previous_index = candidate_index; + previous_sequence = candidate->local_sequence; + candidate_index = candidate->next_index; + } + if (candidate_index != member_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + } + if (!cluster_pcm_x_generation_next(tag_slot->writer_claim_generation, &next_claim_generation)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto claim_done; + } + tag_slot->writer_claim_generation = next_claim_generation; + tag_slot->active_writer_index = member_ref.slot_index; + tag_slot->active_writer_slot_generation = member_ref.slot_generation; + pg_write_barrier(); + claim_out->writer = *writer; + claim_out->active_slot = member_ref; + claim_out->claim_generation = next_claim_generation; + claim_out->local_round = member->admitted_round; + claim_out->role = member->role; + result = PCM_X_QUEUE_OK; + +claim_release_gate: + if (!fail_closed && gate_claimed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +claim_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + pcm_x_local_writer_claim_clear(claim_out); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_local_writer_claim_finish_exact(const PcmXLocalWriterClaim *claim, bool complete, + PcmXWaitIdentity *successor_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXAllocatorView member_view; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *successor; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef successor_ref; + PcmXWaitIdentity successor_identity; + PcmXQueueResult result; + uint32 successor_flags; + uint32 successor_state; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + bool have_successor = false; + + if (successor_out != NULL) + memset(successor_out, 0, sizeof(*successor_out)); + memset(&successor_identity, 0, sizeof(successor_identity)); + + if (header == NULL || claim == NULL || claim->flags != 0 || claim->writer.flags != 0 + || claim->active_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || claim->active_slot.slot_generation == 0 || claim->claim_generation == 0 + || claim->local_round == 0 || claim->role != claim->writer.role + || !pcm_x_wait_identity_valid(&claim->writer.identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(&claim->writer, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&claim->writer.identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, + &claim->writer.identity.tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &claim->writer.identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(&claim->writer, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto release_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto release_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto release_done; + } + gate_claimed = true; + if (claim->active_slot.slot_index != member_ref.slot_index + || claim->active_slot.slot_generation != member_ref.slot_generation + || claim->claim_generation != tag_slot->writer_claim_generation + || tag_slot->active_writer_index != member_ref.slot_index + || tag_slot->active_writer_slot_generation != member_ref.slot_generation + || claim->local_round != member->admitted_round || claim->role != member->role) { + result = PCM_X_QUEUE_STALE; + goto release_gate; + } + if ((pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + /* Collect only the generation-exact direct successor in this closed local + * round. Validation precedes WRITER_COMPLETE, so a corrupt link cannot + * publish completion or a wake hint. A next-round follower remains asleep + * until the RETIRE watermark promotes that round. */ + if (complete && successor_out != NULL) { + if (member->next_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->tail_index != member_ref.slot_index) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + } else { + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + successor_ref.slot_index = member->next_index; + successor = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, + successor_ref.slot_index); + if (successor == NULL + || !pcm_x_slot_generation_read(&successor->slot, &successor_ref.slot_generation)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + successor = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, successor_ref, &claim->writer.identity.tag, + PCM_X_STATE_BIT(PCM_XL_JOINED_NONWAITABLE) + | PCM_X_STATE_BIT(PCM_XL_WAITABLE_FOLLOWER)); + if (successor == NULL || successor->tag_slot_index != tag_ref.slot_index + || successor->tag_slot_generation != tag_ref.slot_generation + || successor->prev_index != member_ref.slot_index + || successor->local_sequence <= member->local_sequence + || successor->local_sequence >= tag_slot->next_sequence + || successor->role != PCM_X_LOCAL_ROLE_FOLLOWER + || !pcm_x_wait_identity_valid(&successor->identity)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + successor_flags = pcm_x_slot_flags_read(&successor->slot); + successor_state = pcm_x_slot_state_read(&successor->slot); + if ((successor_flags & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0 + || (successor_state == PCM_XL_JOINED_NONWAITABLE + && successor->graph_generation != 0) + || (successor_state == PCM_XL_WAITABLE_FOLLOWER + && successor->graph_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + if (successor->admitted_round == claim->local_round) { + successor_identity = successor->identity; + have_successor = true; + } else if (claim->local_round == UINT32_MAX + || successor->admitted_round != claim->local_round + 1 + || (pcm_x_slot_flags_read(&tag_slot->slot) + & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) + == 0 + || member->local_sequence > tag_slot->cutoff_sequence + || successor->local_sequence <= tag_slot->cutoff_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto release_done; + } + } + } + if (complete) + (void)pcm_x_slot_flags_fetch_or(&member->slot, PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE); + pg_write_barrier(); + tag_slot->active_writer_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_writer_slot_generation = 0; + result = PCM_X_QUEUE_OK; + +release_gate: + if (!fail_closed && gate_claimed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +release_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result == PCM_X_QUEUE_OK && complete && successor_out != NULL && have_successor) + *successor_out = successor_identity; + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_writer_claim_release_collect_exact(const PcmXLocalWriterClaim *claim, + PcmXWaitIdentity *successor_out) +{ + if (successor_out == NULL) + return PCM_X_QUEUE_INVALID; + return pcm_x_local_writer_claim_finish_exact(claim, true, successor_out); +} + + +PcmXQueueResult +cluster_pcm_x_local_writer_claim_release_exact(const PcmXLocalWriterClaim *claim) +{ + return pcm_x_local_writer_claim_finish_exact(claim, true, NULL); +} + + +PcmXQueueResult +cluster_pcm_x_local_writer_claim_abort_exact(const PcmXLocalWriterClaim *claim) +{ + return pcm_x_local_writer_claim_finish_exact(claim, false, NULL); +} + + +static PcmXQueueResult +pcm_x_local_leader_slots_in_states_exact(const PcmXLocalHandle *leader, PcmXSlotRef tag_ref, + PcmXSlotRef member_ref, uint32 allowed_states, + PcmXLocalTagSlot **tag_slot_out, + PcmXLocalMembershipSlot **member_out) +{ + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + uint32 state; + + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &leader->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &leader->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(leader, member_ref, member, tag_slot)) + return PCM_X_QUEUE_STALE; + state = pcm_x_slot_state_read(&member->slot); + if ((allowed_states & PCM_X_STATE_BIT(state)) == 0) + return PCM_X_QUEUE_BAD_STATE; + if (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || tag_slot->master_node < 0 || tag_slot->master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || tag_slot->cluster_epoch != leader->identity.cluster_epoch + || tag_slot->master_session_incarnation == 0) + return PCM_X_QUEUE_CORRUPT; + *tag_slot_out = tag_slot; + *member_out = member; + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +pcm_x_local_leader_slots_exact(const PcmXLocalHandle *leader, PcmXSlotRef tag_ref, + PcmXSlotRef member_ref, PcmXLocalTagSlot **tag_slot_out, + PcmXLocalMembershipSlot **member_out) +{ + return pcm_x_local_leader_slots_in_states_exact(leader, tag_ref, member_ref, + PCM_X_STATE_BIT(PCM_XL_NODE_LEADER) + | PCM_X_STATE_BIT(PCM_XL_REMOTE_WAIT), + tag_slot_out, member_out); +} + + +/* Reliable transfer ledgers remain live after the queue leader progresses + * beyond REMOTE_WAIT. Snapshot/retry must therefore accept the two + * application phases as well; admission/cancel callers keep the narrower + * helper above and cannot accidentally operate on a granted writer. */ +static PcmXQueueResult +pcm_x_local_reliable_slots_exact(const PcmXLocalHandle *leader, PcmXSlotRef tag_ref, + PcmXSlotRef member_ref, PcmXLocalTagSlot **tag_slot_out, + PcmXLocalMembershipSlot **member_out) +{ + return pcm_x_local_leader_slots_in_states_exact( + leader, tag_ref, member_ref, + PCM_X_STATE_BIT(PCM_XL_NODE_LEADER) | PCM_X_STATE_BIT(PCM_XL_REMOTE_WAIT) + | PCM_X_STATE_BIT(PCM_XL_CONTENT_ACTIVE) | PCM_X_STATE_BIT(PCM_XL_GRANTED), + tag_slot_out, member_out); +} + + +static PcmXQueueResult +pcm_x_local_outbound_authority_exact(const PcmXLocalTagSlot *tag_slot, + PcmXOutboundTargetFrontier **outbound_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXOutboundTargetFrontier *outbound; + PcmXPeerFrontier *peer; + + if (tag_slot == NULL || tag_slot->master_node < 0 + || tag_slot->master_node >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_QUEUE_CORRUPT; + outbound = &header->outbound_targets[tag_slot->master_node]; + peer = &header->peer_frontiers[tag_slot->master_node]; + if ((outbound->flags & ~PCM_X_OUTBOUND_TARGET_KNOWN_FLAGS) != 0 + || (outbound->flags & PCM_X_OUTBOUND_TARGET_INITIALIZED) == 0 + || outbound->next_prehandle_sequence == 0 || peer->next_expected_prehandle_sequence == 0 + || peer->retired_prehandle_sequence >= peer->next_expected_prehandle_sequence) + return PCM_X_QUEUE_CORRUPT; + if (outbound->flags != (PCM_X_OUTBOUND_TARGET_INITIALIZED | PCM_X_OUTBOUND_TARGET_BOUND) + || outbound->cluster_epoch != tag_slot->cluster_epoch + || outbound->target_session_incarnation != tag_slot->master_session_incarnation + || peer->cluster_epoch != tag_slot->cluster_epoch + || peer->sender_session_incarnation != tag_slot->master_session_incarnation) + return PCM_X_QUEUE_STALE; + *outbound_out = outbound; + return PCM_X_QUEUE_OK; +} + + +static bool +pcm_x_local_reliable_leg_valid(const PcmXLocalTagSlot *tag_slot, + const PcmXLocalMembershipSlot *member) +{ + const PcmXReliableLegState *leg; + uint32 member_state; + + if (tag_slot == NULL || member == NULL) + return false; + leg = &tag_slot->reliable; + if (leg->state_sequence == 0 || leg->expected_responder_session == 0 + || leg->expected_responder_node != tag_slot->master_node + || leg->expected_responder_session != tag_slot->master_session_incarnation + || tag_slot->prehandle.sender_session_incarnation == 0 + || tag_slot->prehandle.prehandle_sequence == 0 || leg->flags != 0 || leg->reserved != 0) + return false; + member_state = pcm_x_slot_state_read(&member->slot); + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_ENQUEUE) + return leg->phase == PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE + && member_state == PCM_XL_NODE_LEADER && pcm_x_ticket_ref_is_zero(&tag_slot->ref); + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM) + return leg->phase == PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM + && member_state == PCM_XL_NODE_LEADER + && pcm_x_local_admission_ref_valid(&tag_slot->ref, &member->identity); + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL) + return leg->phase == PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL + && member_state == PCM_XL_NODE_LEADER && pcm_x_ticket_ref_is_zero(&tag_slot->ref); + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_CANCEL) + return leg->phase == PCM_X_LOCAL_RELIABLE_PHASE_CANCEL && member_state == PCM_XL_REMOTE_WAIT + && pcm_x_local_admission_ref_valid(&tag_slot->ref, &member->identity); + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY) + return leg->phase == PGRAC_IC_MSG_PCM_X_INSTALL_READY + && member_state == PCM_XL_CONTENT_ACTIVE && tag_slot->ref.grant_generation != 0 + && pcm_x_image_token_valid(&tag_slot->image); + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) + return leg->phase == PGRAC_IC_MSG_PCM_X_FINAL_ACK && member_state == PCM_XL_CONTENT_ACTIVE + && tag_slot->ref.grant_generation != 0 && pcm_x_image_token_valid(&tag_slot->image) + && tag_slot->committed_own_generation != 0; + if (leg->pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM) + return leg->phase == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM && member_state == PCM_XL_GRANTED + && tag_slot->ref.grant_generation != 0 && pcm_x_image_token_valid(&tag_slot->image) + && tag_slot->committed_own_generation != 0; + return false; +} + + +static void +pcm_x_local_reliable_token_from_slot(PcmXLocalReliableToken *token, + const PcmXLocalTagSlot *tag_slot, + const PcmXLocalMembershipSlot *member) +{ + pcm_x_local_reliable_token_clear(token); + token->identity = member->identity; + token->prehandle = tag_slot->prehandle; + token->ref = tag_slot->ref; + token->state_sequence = tag_slot->reliable.state_sequence; + token->retry_deadline_ms = tag_slot->reliable.retry_deadline_ms; + token->expected_responder_session = tag_slot->reliable.expected_responder_session; + token->retry_count = tag_slot->reliable.retry_count; + token->expected_responder_node = tag_slot->reliable.expected_responder_node; + token->pending_opcode = tag_slot->reliable.pending_opcode; + token->phase = tag_slot->reliable.phase; + token->flags = tag_slot->reliable.flags; +} + + +static bool +pcm_x_local_reliable_token_exact(const PcmXLocalReliableToken *expected, + const PcmXLocalTagSlot *tag_slot, + const PcmXLocalMembershipSlot *member) +{ + return expected != NULL && expected->reserved == 0 + && pcm_x_wait_identity_equal(&expected->identity, &member->identity) + && pcm_x_prehandle_equal(&expected->prehandle, &tag_slot->prehandle) + && pcm_x_ticket_ref_equal(&expected->ref, &tag_slot->ref) + && expected->state_sequence == tag_slot->reliable.state_sequence + && expected->expected_responder_session == tag_slot->reliable.expected_responder_session + && expected->expected_responder_node == tag_slot->reliable.expected_responder_node + && expected->pending_opcode == tag_slot->reliable.pending_opcode + && expected->phase == tag_slot->reliable.phase + && expected->flags == tag_slot->reliable.flags; +} + + +static bool +pcm_x_local_response_matches(uint16 pending_opcode, uint16 response_opcode) +{ + return (pending_opcode == PGRAC_IC_MSG_PCM_X_ENQUEUE + && response_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_ACK) + || (pending_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM + && response_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK) + || (pending_opcode == PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL + && response_opcode == PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK) + || (pending_opcode == PGRAC_IC_MSG_PCM_X_CANCEL + && response_opcode == PGRAC_IC_MSG_PCM_X_CANCEL_ACK) + || (pending_opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY + && response_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X) + || (pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK + && response_opcode == PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); +} + + +static void +pcm_x_local_reliable_clear(PcmXReliableLegState *leg, uint16 response_opcode, + int32 authenticated_node) +{ + leg->retry_deadline_ms = 0; + leg->expected_responder_session = 0; + leg->retry_count = 0; + leg->last_responder_node = (uint32)authenticated_node; + leg->expected_responder_node = 0; + leg->pending_opcode = 0; + leg->last_response_opcode = response_opcode; + leg->phase = 0; + leg->flags = 0; + leg->reserved = 0; +} + + +PcmXQueueResult +cluster_pcm_x_local_enqueue_arm_exact(const PcmXLocalHandle *leader, + PcmXEnqueuePayload *payload_out, + PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound = NULL; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + PcmXPrehandleKey prehandle; + uint64 next_state_sequence; + uint32 expected_gate = 0; + uint32 partition; + bool fail_closed = false; + bool mint_gate_claimed = false; + bool tag_gate_claimed = false; + + if (payload_out != NULL) + memset(payload_out, 0, sizeof(*payload_out)); + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || payload_out == NULL || token_out == NULL + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_leader_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto enqueue_done; + if (pcm_x_slot_state_read(&member->slot) != PCM_XL_NODE_LEADER) { + result = PCM_X_QUEUE_BAD_STATE; + goto enqueue_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto enqueue_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto enqueue_done; + } + tag_gate_claimed = true; + result = pcm_x_local_terminal_cleanup_fence(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto enqueue_done; + } + if (!pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto enqueue_done; + } + if (tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_ENQUEUE) { + result = PCM_X_QUEUE_BUSY; + goto enqueue_done; + } + payload_out->identity = member->identity; + payload_out->prehandle = tag_slot->prehandle; + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + result = PCM_X_QUEUE_DUPLICATE; + goto enqueue_done; + } + if (tag_slot->prehandle.sender_session_incarnation != 0 + || tag_slot->prehandle.prehandle_sequence != 0 + || !pcm_x_ticket_ref_is_zero(&tag_slot->ref)) { + if ((tag_slot->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK + && tag_slot->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_CANCEL_ACK) + || tag_slot->prehandle.sender_session_incarnation == 0 + || tag_slot->prehandle.prehandle_sequence == 0 + || !pcm_x_local_admission_ref_valid(&tag_slot->ref, &tag_slot->ref.identity) + || pcm_x_wait_identity_equal(&tag_slot->ref.identity, &member->identity)) { + result = PCM_X_QUEUE_BAD_STATE; + goto enqueue_done; + } + /* Retire the preceding leader's duplicate-ACK tombstone immediately + * before minting this successor's new prehandle. */ + memset(&tag_slot->prehandle, 0, sizeof(tag_slot->prehandle)); + memset(&tag_slot->ref, 0, sizeof(tag_slot->ref)); + } + outbound = &header->outbound_targets[tag_slot->master_node]; + if (!pg_atomic_compare_exchange_u32(&outbound->mint_gate, &expected_gate, 1)) { + result = PCM_X_QUEUE_BUSY; + goto enqueue_done; + } + mint_gate_claimed = true; + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto enqueue_done; + } + if (outbound->next_prehandle_sequence == UINT64_MAX + || !cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, + &next_state_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto enqueue_done; + } + prehandle.sender_session_incarnation = runtime.master_session_incarnation; + prehandle.prehandle_sequence = outbound->next_prehandle_sequence; + tag_slot->prehandle = prehandle; + tag_slot->reliable.state_sequence = next_state_sequence; + tag_slot->reliable.retry_deadline_ms = 0; + tag_slot->reliable.expected_responder_session = tag_slot->master_session_incarnation; + tag_slot->reliable.retry_count = 0; + tag_slot->reliable.expected_responder_node = tag_slot->master_node; + tag_slot->reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_ENQUEUE; + tag_slot->reliable.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + tag_slot->reliable.flags = 0; + tag_slot->reliable.reserved = 0; + /* The durable local resend authority precedes the no-hole counter mint. */ + pg_write_barrier(); + outbound->next_prehandle_sequence = prehandle.prehandle_sequence + 1; + payload_out->identity = member->identity; + payload_out->prehandle = prehandle; + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + result = PCM_X_QUEUE_OK; + +enqueue_done: + if (mint_gate_claimed) { + pg_write_barrier(); + pg_atomic_write_u32(&outbound->mint_gate, 0); + } + if (tag_gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_reliable_snapshot_exact(const PcmXLocalHandle *leader, + PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || token_out == NULL + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + result = pcm_x_local_reliable_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto snapshot_done; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto snapshot_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto snapshot_done; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto snapshot_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + result = PCM_X_QUEUE_OK; + +snapshot_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_reliable_retry_exact(const PcmXLocalHandle *leader, + const PcmXLocalReliableToken *expected, + uint64 retry_deadline_ms, + PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || expected == NULL || token_out == NULL + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_reliable_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto retry_done; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto retry_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto retry_done; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto retry_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto retry_done; + } + if (!pcm_x_local_reliable_token_exact(expected, tag_slot, member)) { + result = PCM_X_QUEUE_STALE; + goto retry_done; + } + if (tag_slot->reliable.retry_count != PG_UINT32_MAX) + tag_slot->reliable.retry_count++; + tag_slot->reliable.retry_deadline_ms = retry_deadline_ms; + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + result = PCM_X_QUEUE_OK; + +retry_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_local_duplicate_response_exact(const PcmXLocalHandle *leader, const PcmXTicketRef *actual_ref, + const PcmXPrehandleKey *actual_prehandle, + uint16 response_opcode, uint32 expected_member_state, + int32 authenticated_node, uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + result = pcm_x_local_leader_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto duplicate_done; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto duplicate_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto duplicate_done; + } + if (pcm_x_slot_state_read(&member->slot) != expected_member_state + || tag_slot->reliable.last_response_opcode != response_opcode) { + result = PCM_X_QUEUE_NOT_READY; + goto duplicate_done; + } + if (authenticated_node != tag_slot->master_node + || authenticated_session != tag_slot->master_session_incarnation + || !pcm_x_ticket_ref_equal(actual_ref, &tag_slot->ref) + || (actual_prehandle != NULL + && !pcm_x_prehandle_equal(actual_prehandle, &tag_slot->prehandle))) { + result = PCM_X_QUEUE_STALE; + goto duplicate_done; + } + result = PCM_X_QUEUE_DUPLICATE; + +duplicate_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_reliable_ack_exact(const PcmXLocalHandle *leader, + const PcmXLocalReliableToken *expected, + const PcmXTicketRef *actual_ref, + const PcmXPrehandleKey *actual_prehandle, + uint16 response_opcode, int32 authenticated_node, + uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || leader == NULL || expected == NULL || actual_ref == NULL + || !pcm_x_wait_identity_valid(&leader->identity) || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || (response_opcode != PGRAC_IC_MSG_PCM_X_ADMIT_ACK + && response_opcode != PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_leader_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto ack_done; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto ack_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto ack_done; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto ack_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto ack_done; + } + if (!pcm_x_local_reliable_token_exact(expected, tag_slot, member) + || !pcm_x_local_response_matches(tag_slot->reliable.pending_opcode, response_opcode) + || authenticated_node != tag_slot->reliable.expected_responder_node + || authenticated_session != tag_slot->reliable.expected_responder_session) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + if (tag_slot->reliable.pending_opcode == PGRAC_IC_MSG_PCM_X_ENQUEUE) { + if (actual_prehandle == NULL + || !pcm_x_prehandle_equal(actual_prehandle, &tag_slot->prehandle) + || !pcm_x_local_admission_ref_valid(actual_ref, &member->identity)) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + tag_slot->ref = *actual_ref; + member->handle = actual_ref->handle; + } else if (!pcm_x_ticket_ref_equal(actual_ref, &tag_slot->ref) + || (actual_prehandle != NULL + && !pcm_x_prehandle_equal(actual_prehandle, &tag_slot->prehandle))) { + result = PCM_X_QUEUE_STALE; + goto ack_done; + } + pg_write_barrier(); + pcm_x_local_reliable_clear(&tag_slot->reliable, response_opcode, authenticated_node); + result = PCM_X_QUEUE_OK; + +ack_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_apply_admit_ack_exact(const PcmXLocalHandle *leader, + const PcmXAdmitAckPayload *ack, int32 authenticated_node, + uint64 authenticated_session) +{ + const uint16 known_flags = (uint16)(PCM_X_ADMIT_F_EXACT_REPLAY | PCM_X_ADMIT_F_NODE_COALESCED + | PCM_X_ADMIT_F_QUEUE_HEAD); + PcmXLocalReliableToken token; + PcmXQueueResult result; + PcmXQueueResult duplicate_result; + + if (ClusterPcmXConvertShmem == NULL || leader == NULL || ack == NULL + || !pcm_x_wait_identity_valid(&leader->identity) || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || (ack->result != PCM_X_QUEUE_OK && ack->result != PCM_X_QUEUE_DUPLICATE) + || ack->phase != PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE || (ack->flags & ~known_flags) != 0 + || ack->prehandle.sender_session_incarnation == 0 || ack->prehandle.prehandle_sequence == 0 + || !pcm_x_wait_identity_valid(&ack->ref.identity) || ack->ref.handle.ticket_id == 0 + || ack->ref.handle.queue_generation == 0 || ack->ref.grant_generation != 0) + return PCM_X_QUEUE_INVALID; + result = cluster_pcm_x_local_reliable_snapshot_exact(leader, &token); + if (result == PCM_X_QUEUE_OK && token.pending_opcode == PGRAC_IC_MSG_PCM_X_ENQUEUE) { + if (!pcm_x_prehandle_equal(&token.prehandle, &ack->prehandle) + || !pcm_x_wait_identity_equal(&token.identity, &ack->ref.identity)) + return PCM_X_QUEUE_STALE; + result = cluster_pcm_x_local_reliable_ack_exact(leader, &token, &ack->ref, &ack->prehandle, + PGRAC_IC_MSG_PCM_X_ADMIT_ACK, + authenticated_node, authenticated_session); + if (result == PCM_X_QUEUE_OK) + return result; + if (result != PCM_X_QUEUE_NOT_READY && result != PCM_X_QUEUE_STALE) + return result; + duplicate_result = pcm_x_local_duplicate_response_exact( + leader, &ack->ref, &ack->prehandle, PGRAC_IC_MSG_PCM_X_ADMIT_ACK, PCM_XL_NODE_LEADER, + authenticated_node, authenticated_session); + return duplicate_result == PCM_X_QUEUE_DUPLICATE ? duplicate_result : result; + } + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_NOT_READY) + return result; + return pcm_x_local_duplicate_response_exact(leader, &ack->ref, &ack->prehandle, + PGRAC_IC_MSG_PCM_X_ADMIT_ACK, PCM_XL_NODE_LEADER, + authenticated_node, authenticated_session); +} + + +PcmXQueueResult +cluster_pcm_x_local_admit_confirm_arm_exact(const PcmXLocalHandle *leader, + PcmXPhasePayload *payload_out, + PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint64 next_state_sequence; + uint32 partition; + bool fail_closed = false; + + if (payload_out != NULL) + memset(payload_out, 0, sizeof(*payload_out)); + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || payload_out == NULL || token_out == NULL + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_leader_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto confirm_arm_done; + if (pcm_x_slot_state_read(&member->slot) != PCM_XL_NODE_LEADER) { + result = PCM_X_QUEUE_BAD_STATE; + goto confirm_arm_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto confirm_arm_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto confirm_arm_done; + } + if (!pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto confirm_arm_done; + } + if (tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM) { + result = PCM_X_QUEUE_BUSY; + goto confirm_arm_done; + } + payload_out->ref = tag_slot->ref; + payload_out->phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + result = PCM_X_QUEUE_DUPLICATE; + goto confirm_arm_done; + } + if (!pcm_x_local_admission_ref_valid(&tag_slot->ref, &member->identity) + || tag_slot->prehandle.sender_session_incarnation == 0 + || tag_slot->prehandle.prehandle_sequence == 0) { + result = PCM_X_QUEUE_BAD_STATE; + goto confirm_arm_done; + } + if (!cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, &next_state_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto confirm_arm_done; + } + tag_slot->reliable.state_sequence = next_state_sequence; + tag_slot->reliable.retry_deadline_ms = 0; + tag_slot->reliable.expected_responder_session = tag_slot->master_session_incarnation; + tag_slot->reliable.retry_count = 0; + tag_slot->reliable.expected_responder_node = tag_slot->master_node; + tag_slot->reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM; + tag_slot->reliable.phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + tag_slot->reliable.flags = 0; + tag_slot->reliable.reserved = 0; + pg_write_barrier(); + payload_out->ref = tag_slot->ref; + payload_out->phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + result = PCM_X_QUEUE_OK; + +confirm_arm_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_admit_confirm_ack_exact(const PcmXLocalHandle *leader, + const PcmXPhasePayload *ack, int32 authenticated_node, + uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 state; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || leader == NULL || ack == NULL + || !pcm_x_wait_identity_valid(&leader->identity) || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || ack->reason != 0 || ack->phase != PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM + || ack->flags != 0 || !pcm_x_wait_identity_valid(&ack->ref.identity) + || ack->ref.handle.ticket_id == 0 || ack->ref.handle.queue_generation == 0 + || ack->ref.grant_generation != 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_leader_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto confirm_ack_done; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto confirm_ack_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto confirm_ack_done; + } + state = pcm_x_slot_state_read(&member->slot); + if (state == PCM_XL_REMOTE_WAIT) { + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && tag_slot->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK + && authenticated_node == tag_slot->master_node + && authenticated_session == tag_slot->master_session_incarnation + && pcm_x_ticket_ref_equal(&ack->ref, &tag_slot->ref)) + result = PCM_X_QUEUE_DUPLICATE; + else + result = PCM_X_QUEUE_STALE; + goto confirm_ack_done; + } + if (state != PCM_XL_NODE_LEADER) { + result = PCM_X_QUEUE_BAD_STATE; + goto confirm_ack_done; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto confirm_ack_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto confirm_ack_done; + } + if (tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM + || tag_slot->reliable.phase != PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM + || authenticated_node != tag_slot->reliable.expected_responder_node + || authenticated_session != tag_slot->reliable.expected_responder_session + || !pcm_x_ticket_ref_equal(&ack->ref, &tag_slot->ref)) { + result = PCM_X_QUEUE_STALE; + goto confirm_ack_done; + } + pcm_x_local_reliable_clear(&tag_slot->reliable, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, + authenticated_node); + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_REMOTE_WAIT); + result = PCM_X_QUEUE_OK; + +confirm_ack_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_local_transfer_slots_exact(const PcmXLocalHandle *leader, PcmXSlotRef tag_ref, + PcmXSlotRef member_ref, PcmXLocalTagSlot **tag_slot_out, + PcmXLocalMembershipSlot **member_out) +{ + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + uint32 state; + + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &leader->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &leader->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(leader, member_ref, member, tag_slot)) + return PCM_X_QUEUE_STALE; + state = pcm_x_slot_state_read(&member->slot); + if (state != PCM_XL_REMOTE_WAIT && state != PCM_XL_CONTENT_ACTIVE && state != PCM_XL_GRANTED) + return PCM_X_QUEUE_BAD_STATE; + if (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || tag_slot->master_node < 0 || tag_slot->master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || tag_slot->cluster_epoch != leader->identity.cluster_epoch + || tag_slot->master_session_incarnation == 0) + return PCM_X_QUEUE_CORRUPT; + *tag_slot_out = tag_slot; + *member_out = member; + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_local_prepare_grant_exact(const PcmXLocalHandle *leader, + const PcmXGrantPayload *prepare, int32 authenticated_node, + uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 state; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || leader == NULL || prepare == NULL || prepare->ref.grant_generation == 0 + || !pcm_x_image_token_valid(&prepare->image) + || !pcm_x_image_id_master_exact(prepare->image.image_id, authenticated_node) + || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_session == 0 || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_transfer_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto prepare_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto prepare_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto prepare_done; + } + /* Outbound authority proves the peer frontier; bind the DATA source to it. */ + if (authenticated_node != tag_slot->master_node + || authenticated_session != tag_slot->master_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto prepare_done; + } + state = pcm_x_slot_state_read(&member->slot); + if (state == PCM_XL_REMOTE_WAIT || state == PCM_XL_CONTENT_ACTIVE || state == PCM_XL_GRANTED) { + result + = pcm_x_ticket_ref_equal(&tag_slot->ref, &prepare->ref) + && pcm_x_image_token_equal(&tag_slot->image, &prepare->image) + && (tag_slot->reliable.last_response_opcode + == PGRAC_IC_MSG_PCM_X_PREPARE_GRANT + || tag_slot->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X + || tag_slot->reliable.last_response_opcode + == PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK) + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + if (result == PCM_X_QUEUE_DUPLICATE + || tag_slot->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_PREPARE_GRANT) + goto prepare_done; + } + if (state != PCM_XL_REMOTE_WAIT) { + result = PCM_X_QUEUE_BAD_STATE; + goto prepare_done; + } + if (!pcm_x_ticket_locator_equal(&tag_slot->ref, &prepare->ref) + || !pcm_x_wait_identity_equal(&prepare->ref.identity, &member->identity)) { + result = PCM_X_QUEUE_STALE; + goto prepare_done; + } + if (!pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto prepare_done; + } + /* The master may cross ACTIVE_PROBE->ACTIVE_TRANSFER after the local + * requester has armed CANCEL. An exact PREPARE_GRANT is the durable + * proof that transfer won that race; consume only that CANCEL leg so + * neither side waits forever on the other terminal path. */ + if (tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_CANCEL + || tag_slot->reliable.phase != PCM_X_LOCAL_RELIABLE_PHASE_CANCEL) { + result = PCM_X_QUEUE_BUSY; + goto prepare_done; + } + pcm_x_local_reliable_clear(&tag_slot->reliable, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, + authenticated_node); + } else if (tag_slot->reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK) { + result = PCM_X_QUEUE_BAD_STATE; + goto prepare_done; + } + tag_slot->ref = prepare->ref; + tag_slot->image = prepare->image; + tag_slot->committed_own_generation = 0; + tag_slot->reliable.last_responder_node = (uint32)authenticated_node; + tag_slot->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; + pg_write_barrier(); + result = PCM_X_QUEUE_OK; + +prepare_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_local_transfer_arm_exact(const PcmXLocalHandle *leader, uint16 opcode, + const PcmXTicketRef *expected_ref, + const PcmXImageToken *expected_image, + uint64 committed_own_generation, + PcmXInstallReadyPayload *install_ready_out, + PcmXFinalAckPayload *final_ack_out, + PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint64 expected_generation; + uint64 next_sequence; + uint16 expected_response; + uint32 partition; + bool fail_closed = false; + + if (install_ready_out != NULL) + memset(install_ready_out, 0, sizeof(*install_ready_out)); + if (final_ack_out != NULL) + memset(final_ack_out, 0, sizeof(*final_ack_out)); + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || token_out == NULL + || (opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY + && (expected_ref == NULL || expected_image == NULL || install_ready_out == NULL + || final_ack_out != NULL || committed_own_generation != 0 + || expected_ref->grant_generation == 0 || !pcm_x_image_token_valid(expected_image))) + || (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK + && (expected_ref != NULL || expected_image != NULL || final_ack_out == NULL + || install_ready_out != NULL || committed_own_generation == 0)) + || (opcode != PGRAC_IC_MSG_PCM_X_INSTALL_READY && opcode != PGRAC_IC_MSG_PCM_X_FINAL_ACK) + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + if (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + if (!cluster_pcm_x_generation_next(leader->identity.base_own_generation, + &expected_generation)) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + if (committed_own_generation != expected_generation) + return PCM_X_QUEUE_STALE; + } + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_transfer_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto arm_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto arm_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto arm_done; + } + if (tag_slot->ref.grant_generation == 0 || !pcm_x_image_token_valid(&tag_slot->image)) { + result = PCM_X_QUEUE_BAD_STATE; + goto arm_done; + } + if (opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY) { + if (!pcm_x_ticket_ref_equal(&tag_slot->ref, expected_ref) + || !pcm_x_image_token_equal(&tag_slot->image, expected_image)) { + result = PCM_X_QUEUE_STALE; + goto arm_done; + } + if (pcm_x_slot_state_read(&member->slot) != PCM_XL_REMOTE_WAIT + && pcm_x_slot_state_read(&member->slot) != PCM_XL_CONTENT_ACTIVE) { + result = PCM_X_QUEUE_BAD_STATE; + goto arm_done; + } + } else if (pcm_x_slot_state_read(&member->slot) != PCM_XL_CONTENT_ACTIVE) { + result = PCM_X_QUEUE_BAD_STATE; + goto arm_done; + } + expected_response = opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY + ? PGRAC_IC_MSG_PCM_X_PREPARE_GRANT + : PGRAC_IC_MSG_PCM_X_COMMIT_X; + if (!pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto arm_done; + } + result = tag_slot->reliable.pending_opcode == opcode && tag_slot->reliable.phase == opcode + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_BUSY; + if (result == PCM_X_QUEUE_DUPLICATE && opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK + && tag_slot->committed_own_generation != committed_own_generation) + result = PCM_X_QUEUE_STALE; + goto arm_output; + } + if (opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY + && pcm_x_slot_state_read(&member->slot) != PCM_XL_REMOTE_WAIT) { + result = PCM_X_QUEUE_NOT_READY; + goto arm_done; + } + if (tag_slot->reliable.last_response_opcode != expected_response) { + result = PCM_X_QUEUE_BAD_STATE; + goto arm_done; + } + if (!cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, &next_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto arm_done; + } + tag_slot->reliable.state_sequence = next_sequence; + tag_slot->reliable.retry_deadline_ms = 0; + tag_slot->reliable.expected_responder_session = tag_slot->master_session_incarnation; + tag_slot->reliable.retry_count = 0; + tag_slot->reliable.expected_responder_node = tag_slot->master_node; + tag_slot->reliable.pending_opcode = opcode; + tag_slot->reliable.phase = opcode; + tag_slot->reliable.flags = 0; + tag_slot->reliable.reserved = 0; + if (opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) + tag_slot->committed_own_generation = committed_own_generation; + pg_write_barrier(); + if (opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY) + pcm_x_slot_state_write(&member->slot, PCM_XL_CONTENT_ACTIVE); + result = PCM_X_QUEUE_OK; + +arm_output: + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + if (opcode == PGRAC_IC_MSG_PCM_X_INSTALL_READY) { + install_ready_out->ref = tag_slot->ref; + install_ready_out->image_id = tag_slot->image.image_id; + install_ready_out->result = PCM_X_QUEUE_OK; + install_ready_out->phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + } else { + final_ack_out->ref = tag_slot->ref; + final_ack_out->image_id = tag_slot->image.image_id; + final_ack_out->committed_own_generation = tag_slot->committed_own_generation; + } + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + } + +arm_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_install_ready_arm_exact(const PcmXLocalHandle *leader, + const PcmXTicketRef *expected_ref, + const PcmXImageToken *expected_image, + PcmXInstallReadyPayload *install_ready_out, + PcmXLocalReliableToken *token_out) +{ + return pcm_x_local_transfer_arm_exact(leader, PGRAC_IC_MSG_PCM_X_INSTALL_READY, expected_ref, + expected_image, 0, install_ready_out, NULL, token_out); +} + + +PcmXQueueResult +cluster_pcm_x_local_commit_x_exact(const PcmXLocalHandle *leader, const PcmXPhasePayload *commit, + int32 authenticated_node, uint64 authenticated_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 partition; + bool fail_closed = false; + + if (header == NULL || leader == NULL || commit == NULL || commit->ref.grant_generation == 0 + || commit->reason != 0 || commit->phase != PGRAC_IC_MSG_PCM_X_COMMIT_X || commit->flags != 0 + || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_session == 0 || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_transfer_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto commit_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto commit_done; + } + if (authenticated_node != tag_slot->master_node + || authenticated_session != tag_slot->master_session_incarnation + || !pcm_x_ticket_ref_equal(&tag_slot->ref, &commit->ref)) { + result = PCM_X_QUEUE_STALE; + goto commit_done; + } + if (pcm_x_slot_state_read(&member->slot) != PCM_XL_CONTENT_ACTIVE) { + result = PCM_X_QUEUE_BAD_STATE; + goto commit_done; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + result = tag_slot->reliable.last_response_opcode == PGRAC_IC_MSG_PCM_X_COMMIT_X + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_NOT_READY; + goto commit_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto commit_done; + } + if (tag_slot->reliable.pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_ACK) { + result = PCM_X_QUEUE_DUPLICATE; + goto commit_done; + } + if (tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_INSTALL_READY + || tag_slot->reliable.phase != PGRAC_IC_MSG_PCM_X_INSTALL_READY + || !pcm_x_local_response_matches(tag_slot->reliable.pending_opcode, + PGRAC_IC_MSG_PCM_X_COMMIT_X)) { + result = PCM_X_QUEUE_STALE; + goto commit_done; + } + pcm_x_local_reliable_clear(&tag_slot->reliable, PGRAC_IC_MSG_PCM_X_COMMIT_X, + authenticated_node); + result = PCM_X_QUEUE_OK; + +commit_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_final_ack_arm_exact(const PcmXLocalHandle *leader, + uint64 committed_own_generation, + PcmXFinalAckPayload *final_ack_out, + PcmXLocalReliableToken *token_out) +{ + return pcm_x_local_transfer_arm_exact(leader, PGRAC_IC_MSG_PCM_X_FINAL_ACK, NULL, NULL, + committed_own_generation, NULL, final_ack_out, token_out); +} + + +PcmXQueueResult +cluster_pcm_x_local_final_commit_ack_exact(const PcmXLocalHandle *leader, + const PcmXPhasePayload *final_commit, + int32 authenticated_node, uint64 authenticated_session, + PcmXPhasePayload *final_confirm_out, + PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint64 next_sequence; + uint32 state; + uint32 partition; + bool fail_closed = false; + + if (final_confirm_out != NULL) + memset(final_confirm_out, 0, sizeof(*final_confirm_out)); + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || final_commit == NULL || final_confirm_out == NULL + || token_out == NULL || final_commit->ref.grant_generation == 0 || final_commit->reason != 0 + || final_commit->phase != PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK || final_commit->flags != 0 + || authenticated_node < 0 || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_session == 0 || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_transfer_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto final_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto final_done; + } + if (authenticated_node != tag_slot->master_node + || authenticated_session != tag_slot->master_session_incarnation + || !pcm_x_ticket_ref_equal(&tag_slot->ref, &final_commit->ref)) { + result = PCM_X_QUEUE_STALE; + goto final_done; + } + state = pcm_x_slot_state_read(&member->slot); + if (state == PCM_XL_GRANTED) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto final_done; + } + result = tag_slot->reliable.pending_opcode == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM + && tag_slot->reliable.phase == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM + && tag_slot->reliable.last_response_opcode + == PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + goto final_output; + } + if (state != PCM_XL_CONTENT_ACTIVE) { + result = PCM_X_QUEUE_BAD_STATE; + goto final_done; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + result = PCM_X_QUEUE_NOT_READY; + goto final_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto final_done; + } + if (tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_FINAL_ACK + || tag_slot->reliable.phase != PGRAC_IC_MSG_PCM_X_FINAL_ACK + || !pcm_x_local_response_matches(tag_slot->reliable.pending_opcode, + PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK)) { + result = PCM_X_QUEUE_STALE; + goto final_done; + } + if (!cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, &next_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto final_done; + } + tag_slot->reliable.state_sequence = next_sequence; + tag_slot->reliable.retry_deadline_ms = 0; + tag_slot->reliable.expected_responder_session = tag_slot->master_session_incarnation; + tag_slot->reliable.retry_count = 0; + tag_slot->reliable.last_responder_node = (uint32)authenticated_node; + tag_slot->reliable.expected_responder_node = tag_slot->master_node; + tag_slot->reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + tag_slot->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + tag_slot->reliable.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + tag_slot->reliable.flags = 0; + tag_slot->reliable.reserved = 0; + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_GRANTED); + result = PCM_X_QUEUE_OK; + +final_output: + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) { + final_confirm_out->ref = tag_slot->ref; + final_confirm_out->phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + } + +final_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_begin_revoke_cutoff_exact(const PcmXLocalHandle *leader, + PcmXLocalCutoff *cutoff_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef member_ref; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + uint32 partition; + + if (cutoff_out != NULL) + memset(cutoff_out, 0, sizeof(*cutoff_out)); + if (header == NULL || leader == NULL || cutoff_out == NULL + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_member_lookup_locked(&leader->identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &leader->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &leader->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(leader, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto cutoff_done; + } + if (pcm_x_slot_state_read(&member->slot) != PCM_XL_NODE_LEADER + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation) { + result = PCM_X_QUEUE_BAD_STATE; + goto cutoff_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto cutoff_done; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, tag_slot->master_node, + tag_slot->master_session_incarnation)) { + result = PCM_X_QUEUE_STALE; + goto cutoff_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + goto cutoff_done; + } + result = pcm_x_local_terminal_cleanup_fence(tag_slot); + if (result == PCM_X_QUEUE_OK) + result = pcm_x_local_freeze_round_locked(tag_slot, tag_ref, cutoff_out); + if (result == PCM_X_QUEUE_CORRUPT) + goto cutoff_gate_error; + if (!pcm_x_local_gate_release(tag_slot)) + result = PCM_X_QUEUE_CORRUPT; + if (result == PCM_X_QUEUE_OK) + pcm_x_stats_increment(&header->stats.revoke_count); + goto cutoff_done; + +cutoff_gate_error: + /* Preserve the gate as evidence once a linked queue invariant is broken. */ +cutoff_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +static PcmXQueueResult +pcm_x_local_blocker_lane_retire_state(const PcmXLocalTagSlot *tag_slot) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + bool empty_head; + + if (header == NULL || tag_slot == NULL) + return PCM_X_QUEUE_CORRUPT; + if (pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { + if (tag_slot->blocker_set_generation != 0 + || !pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable) + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_OK; + } + if (!pcm_x_wait_identity_valid(&tag_slot->blocker_snapshot_ref.identity) + || !pcm_x_local_admission_ref_valid(&tag_slot->blocker_snapshot_ref, + &tag_slot->blocker_snapshot_ref.identity) + || !BufferTagsEqual(&tag_slot->tag, &tag_slot->blocker_snapshot_ref.identity.tag) + || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != tag_slot->cluster_epoch + || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->blocker_set_generation == 0) + return pcm_x_local_blocker_freeze_reservation_exact(tag_slot) ? PCM_X_QUEUE_NOT_READY + : PCM_X_QUEUE_CORRUPT; + if (tag_slot->blocker_set_generation == UINT64_MAX) + return PCM_X_QUEUE_CORRUPT; + if (pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable)) + return pcm_x_local_blocker_ack_tombstone_exact(tag_slot, tag_slot->master_node) + ? PCM_X_QUEUE_NOT_READY + : PCM_X_QUEUE_CORRUPT; + empty_head = tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX; + if (tag_slot->blocker_snapshot_reliable.state_sequence == 0 + || tag_slot->blocker_snapshot_count > header->layout.max_wait_edges + || tag_slot->blocker_snapshot_next_chunk != tag_slot->blocker_snapshot_count + || (empty_head != (tag_slot->blocker_snapshot_count == 0)) + || !pcm_x_transfer_leg_exact(&tag_slot->blocker_snapshot_reliable, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, tag_slot->master_node, + tag_slot->master_session_incarnation)) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_NOT_READY; +} + + +static bool +pcm_x_local_blocker_lane_is_pristine(const PcmXLocalTagSlot *tag_slot) +{ + return tag_slot != NULL && pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && tag_slot->blocker_set_generation == 0 + && tag_slot->blocker_snapshot_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->blocker_snapshot_count == 0 && tag_slot->blocker_snapshot_crc32c == 0 + && tag_slot->blocker_snapshot_next_chunk == 0 + && pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable); +} + + +/* A local participant has exactly one external terminal authority: holder_ref + * for the selected image source, or blocker_snapshot_ref for a non-source + * holder. Locator mismatch must never mask the structurally impossible + * dual-lane shape as STALE. */ +static bool +pcm_x_local_external_terminal_lanes_collide(const PcmXLocalTagSlot *tag_slot) +{ + return tag_slot != NULL && !pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + && !pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref); +} + + +/* A closed writer round cannot reopen while the holder lane can still mutate + * or lose its source page. For the S3 core we use the stronger steady-state + * boundary: exact holder DRAIN, after immutable image publication and before + * terminal RETIRE. A pure writer round has no holder/blocker evidence and is + * immediately eligible once its own closed cohort is empty. */ +static PcmXQueueResult +pcm_x_local_holder_lane_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 flags) +{ + PcmXImageToken zero_image; + PcmXQueueResult blocker_state; + bool blocker_terminal; + uint32 holder_flags; + + if (tag_slot == NULL) + return PCM_X_QUEUE_CORRUPT; + holder_flags = flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + memset(&zero_image, 0, sizeof(zero_image)); + blocker_terminal = pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + && !pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && (tag_slot->blocker_snapshot_ref.grant_generation != 0 || holder_flags != 0 + || tag_slot->holder_terminal_drain_generation != 0); + + /* A non-source S-holder never owns an image-transfer lane. Its exact + * DRAIN upgrades the immutable type-48 admission locator in place to the + * final grant ref and consumes the blocker ACK tombstone. This is the + * external-participant terminal form; it is mutually exclusive with the + * source holder_ref form and adds no parallel pin lifecycle. */ + if (blocker_terminal) { + if ((tag_slot->blocker_snapshot_ref.grant_generation == 0 + && !pcm_x_local_admission_ref_valid(&tag_slot->blocker_snapshot_ref, + &tag_slot->blocker_snapshot_ref.identity)) + || (tag_slot->blocker_snapshot_ref.grant_generation != 0 + && !pcm_x_local_terminal_ref_valid(&tag_slot->blocker_snapshot_ref)) + || tag_slot->blocker_snapshot_ref.grant_generation == UINT64_MAX + || !BufferTagsEqual(&tag_slot->tag, &tag_slot->blocker_snapshot_ref.identity.tag) + || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != tag_slot->cluster_epoch + || tag_slot->blocker_set_generation != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || !pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable) + || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) + || holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + || tag_slot->holder_terminal_drain_generation == 0 + || tag_slot->holder_terminal_drain_generation == UINT64_MAX) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_OK; + } + + blocker_state = pcm_x_local_blocker_lane_retire_state(tag_slot); + + if (pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { + if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_NOT_READY; + if (holder_flags != 0 || tag_slot->holder_terminal_drain_generation != 0 + || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable)) + return PCM_X_QUEUE_CORRUPT; + return blocker_state; + } + /* REVOKE consumes and clears the blocker snapshot before publishing a + * holder ref; both lanes alive at once is not a retryable phase. */ + if (blocker_state != PCM_X_QUEUE_OK) + return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_local_terminal_ref_valid(&tag_slot->holder_ref) + || tag_slot->holder_ref.grant_generation == 0 + || !BufferTagsEqual(&tag_slot->tag, &tag_slot->holder_ref.identity.tag) + || tag_slot->holder_ref.identity.cluster_epoch != tag_slot->cluster_epoch + || !pcm_x_image_token_valid(&tag_slot->holder_image) + || !pcm_x_image_id_master_exact(tag_slot->holder_image.image_id, tag_slot->master_node)) + return PCM_X_QUEUE_CORRUPT; + if (holder_flags == 0) + return PCM_X_QUEUE_NOT_READY; + if (holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + || tag_slot->holder_terminal_drain_generation == 0 + || tag_slot->holder_terminal_drain_generation == UINT64_MAX + || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + || tag_slot->holder_reliable.state_sequence == 0 + || tag_slot->holder_reliable.response_tombstone_mask != 0 + || tag_slot->holder_reliable.last_responder_node != (uint32)tag_slot->master_node + || tag_slot->holder_reliable.last_response_opcode != PGRAC_IC_MSG_PCM_X_DRAIN_POLL) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_OK; +} + + +static const PcmXTicketRef * +pcm_x_local_external_terminal_ref(const PcmXLocalTagSlot *tag_slot) +{ + uint32 holder_flags; + + if (tag_slot == NULL) + return NULL; + if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) + return pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) ? &tag_slot->holder_ref + : NULL; + holder_flags = pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + if (!pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && (tag_slot->blocker_snapshot_ref.grant_generation != 0 || holder_flags != 0 + || tag_slot->holder_terminal_drain_generation != 0)) + return &tag_slot->blocker_snapshot_ref; + return NULL; +} + + +static bool pcm_x_local_terminal_pending_clear(const PcmXLocalTagSlot *tag_slot); + + +/* A requester may also be an exact external participant: either the source + * holder or a non-source S-holder recorded by the blocker ACK tombstone. One + * DRAIN then closes both terminal legs for the same ticket. This is the only + * case in which external evidence may be consumed while the writer-round + * barrier remains closed. */ +static PcmXQueueResult +pcm_x_local_same_ref_dual_retire_state(const PcmXLocalTagSlot *tag_slot, uint32 flags, + bool *dual_out, bool *cancel_dual_out) +{ + const PcmXTicketRef *external_ref; + PcmXImageToken zero_image; + bool source_holder; + uint32 holder_flags; + uint32 writer_flags; + + if (tag_slot == NULL || dual_out == NULL || cancel_dual_out == NULL) + return PCM_X_QUEUE_CORRUPT; + *dual_out = false; + *cancel_dual_out = false; + writer_flags = flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK; + holder_flags = flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + if (writer_flags == 0 || holder_flags == 0) + return PCM_X_QUEUE_OK; + source_holder = !pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref); + external_ref = source_holder ? &tag_slot->holder_ref : &tag_slot->blocker_snapshot_ref; + memset(&zero_image, 0, sizeof(zero_image)); + if (writer_flags != PCM_X_LOCAL_TAG_F_TERMINAL_MASK + || holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK || tag_slot->membership_count == 0 + || pcm_x_ticket_ref_is_zero(external_ref) + || !pcm_x_ticket_ref_equal(&tag_slot->ref, external_ref) + || tag_slot->terminal_drain_generation == 0 + || tag_slot->terminal_drain_generation == UINT64_MAX + || tag_slot->terminal_drain_generation != tag_slot->holder_terminal_drain_generation + || !pcm_x_local_terminal_pending_clear(tag_slot) + || pcm_x_local_holder_lane_retire_state(tag_slot, flags) != PCM_X_QUEUE_OK) + return PCM_X_QUEUE_CORRUPT; + if (external_ref->grant_generation == 0) { + if (source_holder || tag_slot->ref.grant_generation != 0 + || (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 + || tag_slot->closed_round_member_count == 0 + || tag_slot->closed_round_member_count > tag_slot->membership_count + || !pcm_x_image_token_equal(&tag_slot->image, &zero_image) + || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image)) + return PCM_X_QUEUE_CORRUPT; + *dual_out = true; + *cancel_dual_out = true; + return PCM_X_QUEUE_OK; + } + /* A successful grant keeps the original, stronger transfer-round proof. + * Cancellation never enters this arm merely because it shares a locator. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 || tag_slot->closed_round_member_count != 1 + || !pcm_x_image_token_valid(&tag_slot->image) + || (source_holder && !pcm_x_image_token_equal(&tag_slot->image, &tag_slot->holder_image)) + || (!source_holder + && tag_slot->image.source_node == (uint32)tag_slot->ref.identity.node_id)) + return PCM_X_QUEUE_CORRUPT; + *dual_out = true; + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_local_current_cutoff_snapshot_exact(const BufferTag *tag, uint64 cluster_epoch, + int32 master_node, + uint64 master_session_incarnation, + PcmXLocalCutoff *cutoff_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + uint32 flags; + uint32 partition; + bool fail_closed = false; + + if (cutoff_out != NULL) + memset(cutoff_out, 0, sizeof(*cutoff_out)); + if (header == NULL || tag == NULL || cutoff_out == NULL || master_node < 0 + || master_node >= PCM_X_PROTOCOL_NODE_LIMIT || master_session_incarnation == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_NOT_FOUND; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_SHARED); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || tag_slot->cluster_epoch != cluster_epoch + || tag_slot->master_node != master_node + || tag_slot->master_session_incarnation != master_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto snapshot_done; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, master_node, + master_session_incarnation)) { + result = PCM_X_QUEUE_STALE; + goto snapshot_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto snapshot_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0) { + result = PCM_X_QUEUE_BUSY; + goto snapshot_done; + } + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + result = PCM_X_QUEUE_NOT_READY; + goto snapshot_done; + } + if (tag_slot->local_round == 0 || tag_slot->local_round == UINT32_MAX + || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || (tag_slot->cutoff_sequence == 0 && tag_slot->closed_round_member_count != 0) + || tag_slot->closed_round_member_count > tag_slot->membership_count + || ((tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_slot_generation == 0)) + || ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0))) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto snapshot_done; + } + pcm_x_local_cutoff_fill(tag_slot, tag_ref, cutoff_out); + result = PCM_X_QUEUE_OK; + +snapshot_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) + memset(cutoff_out, 0, sizeof(*cutoff_out)); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_retire_round_exact(const BufferTag *tag, uint64 cluster_epoch, + const PcmXLocalCutoff *cutoff, + PcmXLocalHandle *next_leader_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *gated_tag = NULL; + PcmXLocalMembershipSlot *candidate = NULL; + PcmXSlotRef tag_ref; + PcmXSlotRef found; + PcmXSlotRef candidate_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + PcmXQueueResult holder_state; + PcmXAllocatorView member_view; + uint32 candidate_state; + uint32 flags; + uint32 partition; + bool candidate_promote = false; + bool detach_tag = false; + bool fail_closed = false; + bool gate_claimed = false; + + pcm_x_local_handle_clear(next_leader_out); + if (header == NULL || tag == NULL || cutoff == NULL || next_leader_out == NULL + || cutoff->closed_round == 0 || cutoff->closed_round == UINT32_MAX + || cutoff->next_round != cutoff->closed_round + 1 || cutoff->master_node < 0 + || cutoff->master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || cutoff->master_session_incarnation == 0 || cutoff->reserved != 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &tag_ref); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + result = PCM_X_QUEUE_NOT_FOUND; + else if (directory_result != PCM_X_DIRECTORY_OK) + result = pcm_x_queue_result_from_directory(directory_result); + else if (tag_ref.slot_index != cutoff->tag_slot.slot_index + || tag_ref.slot_generation != cutoff->tag_slot.slot_generation) + result = PCM_X_QUEUE_STALE; + else { + tag_slot + = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE) + result = PCM_X_QUEUE_STALE; + else if (!BufferTagsEqual(&tag_slot->tag, tag) + || tag_slot->cluster_epoch != cluster_epoch) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result + = pcm_x_local_handoff_gate_claim_locked(header, tag_slot, false, PCM_X_QUEUE_BUSY); + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; + } + gate_claimed = true; + gated_tag = tag_slot; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, gated_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL || tag_slot->cluster_epoch != cluster_epoch + || tag_slot->cutoff_sequence != cutoff->cutoff_sequence + || tag_slot->master_node != cutoff->master_node + || tag_slot->master_session_incarnation != cutoff->master_session_incarnation) { + result = PCM_X_QUEUE_STALE; + goto retire_round_done; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, cutoff->master_node, + cutoff->master_session_incarnation)) { + result = PCM_X_QUEUE_STALE; + goto retire_round_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto retire_round_done; + } + if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (((tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_slot_generation == 0)) + || ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) + || ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + || (tag_slot->membership_count == 0 && tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) + || ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + && tag_slot->cutoff_sequence >= tag_slot->next_sequence) + || tag_slot->closed_round_member_count > tag_slot->membership_count) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + if (tag_slot->local_round != cutoff->next_round) { + result = PCM_X_QUEUE_STALE; + goto retire_round_gate_release; + } + if (tag_slot->closed_round_member_count != 0 + || ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX)) + || (tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_index != tag_slot->head_index)) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + if (tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX) { + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + candidate = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, + tag_slot->leader_index); + if (candidate == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + candidate_ref.slot_index = tag_slot->leader_index; + candidate_ref.slot_generation = tag_slot->leader_slot_generation; + candidate = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, candidate_ref, tag, + PCM_X_STATE_BIT(PCM_XL_NODE_LEADER) | PCM_X_STATE_BIT(PCM_XL_REMOTE_WAIT) + | PCM_X_STATE_BIT(PCM_XL_CONTENT_ACTIVE) | PCM_X_STATE_BIT(PCM_XL_GRANTED)); + candidate_state + = candidate == NULL ? PCM_X_SLOT_FREE : pcm_x_slot_state_read(&candidate->slot); + if (candidate == NULL || candidate->tag_slot_index != tag_ref.slot_index + || candidate->tag_slot_generation != tag_ref.slot_generation + || candidate->admitted_round != cutoff->next_round + || candidate->local_sequence <= cutoff->cutoff_sequence + || candidate->identity.cluster_epoch != cluster_epoch + || candidate->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || candidate->graph_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + /* A replay can race the successor's normal ENQUEUE/GRANT progress. + * Its exact resident identity remains valid, but only a still-idle + * NODE_LEADER needs another wake token. */ + if (candidate_state == PCM_XL_NODE_LEADER + && (pcm_x_slot_flags_read(&candidate->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) + == 0) + pcm_x_local_handle_from_member(next_leader_out, candidate_ref, candidate, + PCM_X_LOCAL_ROLE_NODE_LEADER); + } + result = PCM_X_QUEUE_DUPLICATE; + goto retire_round_gate_release; + } + if (tag_slot->local_round != cutoff->closed_round + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || tag_slot->closed_round_member_count != 0) { + result = PCM_X_QUEUE_NOT_READY; + goto retire_round_gate_release; + } + holder_state = pcm_x_local_holder_lane_retire_state(tag_slot, flags); + if (holder_state != PCM_X_QUEUE_OK) { + result = holder_state; + if (result == PCM_X_QUEUE_CORRUPT) + goto retire_round_done; + goto retire_round_gate_release; + } + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, tag_slot->head_index, + PCM_X_INVALID_SLOT_INDEX, 0, &candidate, + &candidate_ref, &candidate_promote); + if (result == PCM_X_QUEUE_BUSY) { + result = PCM_X_QUEUE_NOT_READY; + goto retire_round_gate_release; + } + if (result != PCM_X_QUEUE_OK || candidate_promote) { + result = PCM_X_QUEUE_CORRUPT; + goto retire_round_done; + } + } + tag_slot->local_round = cutoff->next_round; + (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); + if (candidate != NULL) { + tag_slot->leader_index = candidate_ref.slot_index; + tag_slot->leader_slot_generation = candidate_ref.slot_generation; + candidate->role = PCM_X_LOCAL_ROLE_NODE_LEADER; + pg_write_barrier(); + pcm_x_slot_state_write(&candidate->slot, PCM_XL_NODE_LEADER); + pcm_x_local_handle_from_member(next_leader_out, candidate_ref, candidate, + PCM_X_LOCAL_ROLE_NODE_LEADER); + } + result = PCM_X_QUEUE_OK; + /* An empty pure-writer tag has no later backend that could reclaim it. + * Keep the admission gate across the local-to-allocator lock handoff and + * remove it here. Holder/terminal evidence keeps the tag live for its own + * exact RETIRE path. */ + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (candidate == NULL && tag_slot->membership_count == 0 + && tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + && pcm_x_ticket_ref_is_zero(&tag_slot->ref) + && pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + && pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) + && pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable) + && (flags & (PCM_X_LOCAL_TAG_F_TERMINAL_MASK | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK)) + == 0) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); + detach_tag = true; + goto retire_round_done; + } + +retire_round_gate_release: + if (!pcm_x_local_gate_release(tag_slot)) + result = PCM_X_QUEUE_CORRUPT; + else + gate_claimed = false; + goto retire_round_done; + +retire_round_done: + /* Any post-acquire direct jump here is CORRUPT and retains the gate. */ + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_CORRUPT && gate_claimed) { + if (!pcm_x_local_gate_release(gated_tag)) + result = PCM_X_QUEUE_CORRUPT; + else + gate_claimed = false; + } + if (result == PCM_X_QUEUE_CORRUPT) { + pcm_x_runtime_fail_closed(); + return result; + } + if (result != PCM_X_QUEUE_OK || !detach_tag) + return result; + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, tag_slot); + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_DETACHING + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0 + || pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, tag, &found) != PCM_X_DIRECTORY_OK + || found.slot_index != tag_ref.slot_index + || found.slot_generation != tag_ref.slot_generation + || pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, tag, tag_ref) + != PCM_X_DIRECTORY_OK + || pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, PCM_X_TAG_DETACHING) + != PCM_X_ALLOC_OK) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_OK; + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +static bool +pcm_x_local_unlink_member_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalMembershipSlot *member, PcmXSlotRef member_ref) +{ + PcmXAllocatorView member_view; + PcmXLocalMembershipSlot *next = NULL; + PcmXLocalMembershipSlot *previous = NULL; + + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) + return false; + if (member->prev_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->head_index != member_ref.slot_index) + return false; + } else { + if (member->prev_index == member_ref.slot_index + || tag_slot->head_index == member_ref.slot_index) + return false; + previous + = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, member->prev_index); + if (previous == NULL || previous->next_index != member_ref.slot_index + || previous->tag_slot_index != tag_ref.slot_index + || previous->tag_slot_generation != tag_ref.slot_generation) + return false; + } + if (member->next_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->tail_index != member_ref.slot_index) + return false; + } else { + if (member->next_index == member_ref.slot_index || member->next_index == member->prev_index + || tag_slot->tail_index == member_ref.slot_index) + return false; + next = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, member->next_index); + if (next == NULL || next->prev_index != member_ref.slot_index + || next->tag_slot_index != tag_ref.slot_index + || next->tag_slot_generation != tag_ref.slot_generation) + return false; + } + + /* Validate both directions before changing either one. */ + if (previous == NULL) + tag_slot->head_index = member->next_index; + else + previous->next_index = member->next_index; + if (next == NULL) + tag_slot->tail_index = member->prev_index; + else + next->prev_index = member->prev_index; + member->next_index = PCM_X_INVALID_SLOT_INDEX; + member->prev_index = PCM_X_INVALID_SLOT_INDEX; + return true; +} + + +/* Arm either cancel wire leg without dropping the currently authoritative + * local leader. PREHANDLE_CANCEL replaces the older ENQUEUE resend authority; + * full CANCEL starts only after ADMIT_CONFIRM_ACK made the leader REMOTE_WAIT. */ +static PcmXQueueResult +pcm_x_local_cancel_arm_common(const PcmXLocalHandle *leader, bool prehandle_cancel, + PcmXPrehandleCancelPayload *prehandle_out, + PcmXPhasePayload *phase_out, PcmXLocalReliableToken *token_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint64 next_state_sequence; + uint16 desired_opcode; + uint16 desired_phase; + uint32 member_state; + uint32 partition; + bool fail_closed = false; + + if (prehandle_out != NULL) + memset(prehandle_out, 0, sizeof(*prehandle_out)); + if (phase_out != NULL) + memset(phase_out, 0, sizeof(*phase_out)); + pcm_x_local_reliable_token_clear(token_out); + if (header == NULL || leader == NULL || token_out == NULL + || (prehandle_cancel && (prehandle_out == NULL || phase_out != NULL)) + || (!prehandle_cancel && (phase_out == NULL || prehandle_out != NULL)) + || !pcm_x_wait_identity_valid(&leader->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + result = pcm_x_local_leader_slots_exact(leader, tag_ref, member_ref, &tag_slot, &member); + if (result != PCM_X_QUEUE_OK) + goto cancel_arm_done; + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto cancel_arm_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto cancel_arm_done; + } + member_state = pcm_x_slot_state_read(&member->slot); + desired_opcode + = prehandle_cancel ? PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL : PGRAC_IC_MSG_PCM_X_CANCEL; + desired_phase = prehandle_cancel ? PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL + : PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + if ((prehandle_cancel && member_state != PCM_XL_NODE_LEADER) + || (!prehandle_cancel && member_state != PCM_XL_REMOTE_WAIT)) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_arm_done; + } + if (!pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_arm_done; + } + if (tag_slot->reliable.pending_opcode == desired_opcode + && tag_slot->reliable.phase == desired_phase) { + result = PCM_X_QUEUE_DUPLICATE; + goto cancel_arm_output; + } + if (!prehandle_cancel || tag_slot->reliable.pending_opcode != PGRAC_IC_MSG_PCM_X_ENQUEUE + || tag_slot->reliable.phase != PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE) { + result = PCM_X_QUEUE_BUSY; + goto cancel_arm_done; + } + } else if (prehandle_cancel) { + /* A prehandle exists only while ENQUEUE itself remains application-pending. */ + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_arm_done; + } + if ((prehandle_cancel && !pcm_x_ticket_ref_is_zero(&tag_slot->ref)) + || (!prehandle_cancel + && !pcm_x_local_admission_ref_valid(&tag_slot->ref, &member->identity))) { + result = PCM_X_QUEUE_BAD_STATE; + goto cancel_arm_done; + } + if (!cluster_pcm_x_generation_next(tag_slot->reliable.state_sequence, &next_state_sequence)) { + result = PCM_X_QUEUE_COUNTER_EXHAUSTED; + fail_closed = true; + goto cancel_arm_done; + } + tag_slot->reliable.state_sequence = next_state_sequence; + tag_slot->reliable.retry_deadline_ms = 0; + tag_slot->reliable.expected_responder_session = tag_slot->master_session_incarnation; + tag_slot->reliable.retry_count = 0; + tag_slot->reliable.expected_responder_node = tag_slot->master_node; + tag_slot->reliable.pending_opcode = desired_opcode; + tag_slot->reliable.phase = desired_phase; + tag_slot->reliable.flags = 0; + tag_slot->reliable.reserved = 0; + pg_write_barrier(); + result = PCM_X_QUEUE_OK; + +cancel_arm_output: + if (prehandle_cancel) { + prehandle_out->identity = member->identity; + prehandle_out->prehandle = tag_slot->prehandle; + } else { + phase_out->ref = tag_slot->ref; + phase_out->phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + } + pcm_x_local_reliable_token_from_slot(token_out, tag_slot, member); + +cancel_arm_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_prehandle_cancel_arm_exact(const PcmXLocalHandle *leader, + PcmXPrehandleCancelPayload *payload_out, + PcmXLocalReliableToken *token_out) +{ + return pcm_x_local_cancel_arm_common(leader, true, payload_out, NULL, token_out); +} + + +PcmXQueueResult +cluster_pcm_x_local_cancel_arm_exact(const PcmXLocalHandle *leader, PcmXPhasePayload *payload_out, + PcmXLocalReliableToken *token_out) +{ + return pcm_x_local_cancel_arm_common(leader, false, NULL, payload_out, token_out); +} + + +static PcmXQueueResult pcm_x_local_cancel_successor_state(const PcmXLocalTagSlot *tag_slot, + const PcmXLocalMembershipSlot *candidate, + bool *promote_out); + + +static PcmXQueueResult +pcm_x_local_cancel_current_leader_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalHandle *leader_out) +{ + PcmXAllocatorView member_view; + PcmXLocalMembershipSlot *candidate; + PcmXSlotRef candidate_ref; + PcmXQueueResult result; + bool promote; + + pcm_x_local_handle_clear(leader_out); + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) { + if (tag_slot->leader_slot_generation != 0) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + return tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX ? PCM_X_QUEUE_OK + : PCM_X_QUEUE_CORRUPT; + candidate_ref.slot_index = tag_slot->head_index; + candidate = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, + candidate_ref.slot_index); + if (candidate == NULL || candidate->tag_slot_index != tag_ref.slot_index + || candidate->tag_slot_generation != tag_ref.slot_generation + || candidate->prev_index != PCM_X_INVALID_SLOT_INDEX + || candidate->role != PCM_X_LOCAL_ROLE_FOLLOWER + || !pcm_x_slot_generation_read(&candidate->slot, &candidate_ref.slot_generation)) + return PCM_X_QUEUE_CORRUPT; + candidate = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, candidate_ref, &tag_slot->tag, + PCM_X_STATE_BIT(PCM_XL_JOINED_NONWAITABLE) | PCM_X_STATE_BIT(PCM_XL_WAITABLE_FOLLOWER)); + if (candidate == NULL) + return PCM_X_QUEUE_CORRUPT; + result = pcm_x_local_cancel_successor_state(tag_slot, candidate, &promote); + if (result != PCM_X_QUEUE_OK) + return result; + return promote ? PCM_X_QUEUE_CORRUPT : PCM_X_QUEUE_OK; + } + candidate + = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, tag_slot->leader_index); + if (candidate == NULL || candidate->tag_slot_index != tag_ref.slot_index + || candidate->tag_slot_generation != tag_ref.slot_generation + || !pcm_x_slot_generation_read(&candidate->slot, &candidate_ref.slot_generation)) + return PCM_X_QUEUE_CORRUPT; + candidate_ref.slot_index = tag_slot->leader_index; + candidate = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, candidate_ref, &tag_slot->tag, PCM_X_STATE_BIT(PCM_XL_NODE_LEADER)); + if (candidate == NULL || candidate->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || tag_slot->leader_slot_generation != candidate_ref.slot_generation) + return PCM_X_QUEUE_CORRUPT; + if ((pcm_x_slot_flags_read(&candidate->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) + return PCM_X_QUEUE_NOT_READY; + if (leader_out != NULL) + pcm_x_local_handle_from_member(leader_out, candidate_ref, candidate, + PCM_X_LOCAL_ROLE_NODE_LEADER); + return PCM_X_QUEUE_OK; +} + + +/* Classify the linked successor before an interrupt path unlinks a leader. + * An externally published follower edge must be removed exactly before role + * rotation. A post-cutoff follower remains in the next round and is never + * promoted while the revoke barrier is closed. */ +static PcmXQueueResult +pcm_x_local_cancel_successor_state(const PcmXLocalTagSlot *tag_slot, + const PcmXLocalMembershipSlot *candidate, bool *promote_out) +{ + uint32 candidate_state; + uint32 flags; + + if (tag_slot == NULL || candidate == NULL || promote_out == NULL) + return PCM_X_QUEUE_CORRUPT; + *promote_out = false; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (candidate->local_sequence == 0 || candidate->local_sequence >= tag_slot->next_sequence + || candidate->identity.cluster_epoch != tag_slot->cluster_epoch) + return PCM_X_QUEUE_CORRUPT; + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) { + if (tag_slot->local_round == UINT32_MAX + || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || tag_slot->closed_round_member_count > tag_slot->membership_count) + return PCM_X_QUEUE_CORRUPT; + if (candidate->local_sequence <= tag_slot->cutoff_sequence) { + if (candidate->admitted_round != tag_slot->local_round) + return PCM_X_QUEUE_CORRUPT; + *promote_out = true; + } else if (candidate->admitted_round != tag_slot->local_round + 1) + return PCM_X_QUEUE_CORRUPT; + } else { + if (candidate->admitted_round != tag_slot->local_round) + return PCM_X_QUEUE_CORRUPT; + *promote_out = true; + } + if ((pcm_x_slot_flags_read(&candidate->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) + return *promote_out ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_CORRUPT; + candidate_state = pcm_x_slot_state_read(&candidate->slot); + if (candidate_state == PCM_XL_WAITABLE_FOLLOWER) + return candidate->graph_generation != 0 ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_CORRUPT; + if (candidate_state != PCM_XL_JOINED_NONWAITABLE || candidate->graph_generation != 0) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_OK; +} + + +/* Prove that every linked follower has retired its external WFG edge before + * a leader identity can disappear or change roles. Checking only the head + * would leave later followers pointing at a terminal identity. */ +static PcmXQueueResult +pcm_x_local_follower_chain_ready(const PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + Size first_index, Size expected_prev_index, + uint64 expected_prev_sequence, PcmXLocalMembershipSlot **first_out, + PcmXSlotRef *first_ref_out, bool *first_promote_out) +{ + PcmXAllocatorView member_view; + PcmXLocalMembershipSlot *candidate; + PcmXSlotRef candidate_ref; + PcmXQueueResult result; + Size current = first_index; + Size previous = expected_prev_index; + uint64 previous_sequence = expected_prev_sequence; + Size visited = 0; + bool promote; + + if (tag_slot == NULL || first_out == NULL || first_ref_out == NULL || first_promote_out == NULL + || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &member_view)) + return PCM_X_QUEUE_CORRUPT; + *first_out = NULL; + first_ref_out->slot_index = PCM_X_INVALID_SLOT_INDEX; + first_ref_out->slot_generation = 0; + *first_promote_out = false; + while (current != PCM_X_INVALID_SLOT_INDEX) { + candidate = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&member_view, current); + if (candidate == NULL || visited >= tag_slot->membership_count + || visited >= member_view.capacity || candidate->tag_slot_index != tag_ref.slot_index + || candidate->tag_slot_generation != tag_ref.slot_generation + || candidate->prev_index != previous || candidate->local_sequence <= previous_sequence + || candidate->role != PCM_X_LOCAL_ROLE_FOLLOWER + || !pcm_x_slot_generation_read(&candidate->slot, &candidate_ref.slot_generation)) + return PCM_X_QUEUE_CORRUPT; + candidate_ref.slot_index = current; + candidate = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, candidate_ref, &tag_slot->tag, + PCM_X_STATE_BIT(PCM_XL_JOINED_NONWAITABLE) | PCM_X_STATE_BIT(PCM_XL_WAITABLE_FOLLOWER)); + if (candidate == NULL) + return PCM_X_QUEUE_CORRUPT; + result = pcm_x_local_cancel_successor_state(tag_slot, candidate, &promote); + if (result != PCM_X_QUEUE_OK) + return result; + if (visited == 0) { + *first_out = candidate; + *first_ref_out = candidate_ref; + *first_promote_out = promote; + } + previous = current; + previous_sequence = candidate->local_sequence; + current = candidate->next_index; + visited++; + } + return tag_slot->tail_index == previous ? PCM_X_QUEUE_OK : PCM_X_QUEUE_CORRUPT; +} + + +/* Apply a cancel ACK and rotate the local leader under one local-domain lock. + * The reliable leg remains intact until all queue/candidate invariants have + * been validated, so transport delivery can never masquerade as application + * completion. */ +static PcmXQueueResult +pcm_x_local_cancel_ack_common(const PcmXLocalHandle *leader, const PcmXTicketRef *actual_ref, + const PcmXPrehandleKey *actual_prehandle, uint16 response_opcode, + int32 authenticated_node, uint64 authenticated_session, + PcmXLocalHandle *new_leader_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXOutboundTargetFrontier *outbound; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *candidate = NULL; + PcmXImageToken zero_image = { 0 }; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef candidate_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXQueueResult result; + uint16 pending_opcode; + uint16 pending_phase; + uint32 expected_state; + uint32 state; + uint32 flags; + uint32 partition; + bool fail_closed = false; + bool gate_claimed = false; + bool promote_candidate = false; + + pcm_x_local_handle_clear(new_leader_out); + if (header == NULL || leader == NULL || actual_ref == NULL + || !pcm_x_wait_identity_valid(&leader->identity) || authenticated_node < 0 + || authenticated_node >= PCM_X_PROTOCOL_NODE_LIMIT || authenticated_session == 0 + || !pcm_x_local_admission_ref_valid(actual_ref, &leader->identity)) + return PCM_X_QUEUE_INVALID; + if (response_opcode == PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK) { + if (actual_prehandle == NULL || actual_prehandle->sender_session_incarnation == 0 + || actual_prehandle->prehandle_sequence == 0) + return PCM_X_QUEUE_INVALID; + pending_opcode = PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL; + pending_phase = PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL; + expected_state = PCM_XL_NODE_LEADER; + } else if (response_opcode == PGRAC_IC_MSG_PCM_X_CANCEL_ACK) { + if (actual_prehandle != NULL) + return PCM_X_QUEUE_INVALID; + pending_opcode = PGRAC_IC_MSG_PCM_X_CANCEL; + pending_phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + expected_state = PCM_XL_REMOTE_WAIT; + } else + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + result = pcm_x_local_refs_lookup(leader, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&leader->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &leader->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &leader->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(leader, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto cancel_ack_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto cancel_ack_done; + } + result = pcm_x_local_outbound_authority_exact(tag_slot, &outbound); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT || result == PCM_X_QUEUE_STALE; + goto cancel_ack_done; + } + state = pcm_x_slot_state_read(&member->slot); + if (state == PCM_XL_CANCELLED) { + if (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER || tag_slot->reliable.pending_opcode != 0 + || tag_slot->reliable.last_response_opcode != response_opcode + || authenticated_node != tag_slot->master_node + || authenticated_session != tag_slot->master_session_incarnation + || !pcm_x_ticket_handle_equal(&member->handle, &actual_ref->handle) + || !pcm_x_ticket_ref_equal(&tag_slot->ref, actual_ref) + || (actual_prehandle != NULL + && !pcm_x_prehandle_equal(&tag_slot->prehandle, actual_prehandle))) { + result = PCM_X_QUEUE_STALE; + goto cancel_ack_done; + } + result = pcm_x_local_cancel_current_leader_locked(tag_slot, tag_ref, new_leader_out); + if (result == PCM_X_QUEUE_OK) + result = PCM_X_QUEUE_DUPLICATE; + else + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto cancel_ack_done; + } + if (state != expected_state || member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || tag_slot->head_index != member_ref.slot_index) { + result = state == expected_state ? PCM_X_QUEUE_CORRUPT : PCM_X_QUEUE_BAD_STATE; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto cancel_ack_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto cancel_ack_done; + } + gate_claimed = true; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + if (tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_BUSY; + goto cancel_ack_release_gate; + } + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0) { + result = pcm_x_local_execution_round_locked(tag_slot, member); + if (result != PCM_X_QUEUE_OK || tag_slot->closed_round_member_count == 0 + || tag_slot->closed_round_member_count > tag_slot->membership_count + || tag_slot->cutoff_sequence >= tag_slot->next_sequence) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + } else if (tag_slot->closed_round_member_count != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + if (!pcm_x_image_token_equal(&tag_slot->image, &zero_image)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + if (member->graph_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + if (pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || !pcm_x_local_reliable_leg_valid(tag_slot, member)) { + result = pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) ? PCM_X_QUEUE_NOT_READY + : PCM_X_QUEUE_CORRUPT; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto cancel_ack_release_gate; + } + if (tag_slot->reliable.pending_opcode != pending_opcode + || tag_slot->reliable.phase != pending_phase + || !pcm_x_local_response_matches(pending_opcode, response_opcode) + || authenticated_node != tag_slot->reliable.expected_responder_node + || authenticated_session != tag_slot->reliable.expected_responder_session + || (actual_prehandle != NULL + && !pcm_x_prehandle_equal(actual_prehandle, &tag_slot->prehandle)) + || (pending_opcode == PGRAC_IC_MSG_PCM_X_CANCEL + && !pcm_x_ticket_ref_equal(actual_ref, &tag_slot->ref))) { + result = PCM_X_QUEUE_STALE; + goto cancel_ack_release_gate; + } + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, member->next_index, + member_ref.slot_index, member->local_sequence, + &candidate, &candidate_ref, &promote_candidate); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto cancel_ack_release_gate; + } + if (!pcm_x_local_unlink_member_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto cancel_ack_release_gate; + } + tag_slot->leader_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->leader_slot_generation = 0; + member->handle = actual_ref->handle; + member->graph_generation = 0; + if (pending_opcode == PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL) + tag_slot->ref = *actual_ref; + pcm_x_local_reliable_clear(&tag_slot->reliable, response_opcode, authenticated_node); + /* Keep the exact old prehandle/ref as a bounded duplicate-ACK tombstone. + * The successor clears it under this same local-domain lock before minting + * its own ENQUEUE authority. */ + if (candidate != NULL && promote_candidate) { + tag_slot->leader_index = candidate_ref.slot_index; + tag_slot->leader_slot_generation = candidate_ref.slot_generation; + candidate->role = PCM_X_LOCAL_ROLE_NODE_LEADER; + pg_write_barrier(); + pcm_x_slot_state_write(&candidate->slot, PCM_XL_NODE_LEADER); + if (new_leader_out != NULL) + pcm_x_local_handle_from_member(new_leader_out, candidate_ref, candidate, + PCM_X_LOCAL_ROLE_NODE_LEADER); + } + tag_slot->terminal_drain_generation = 0; + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_READY); + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_CANCELLED); + result = PCM_X_QUEUE_OK; + +cancel_ack_release_gate: + /* Retain a claimed gate as recovery evidence after an invariant failure; + * retryable/application mismatches release it for the durable resend. */ + if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + +cancel_ack_done: + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_prehandle_cancel_ack_exact(const PcmXLocalHandle *leader, + const PcmXAdmitAckPayload *ack, + int32 authenticated_node, + uint64 authenticated_session, + PcmXLocalHandle *new_leader_out) +{ + if (ack == NULL || ack->result != PCM_X_QUEUE_OK + || ack->phase != PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL || ack->flags != 0) + return PCM_X_QUEUE_INVALID; + return pcm_x_local_cancel_ack_common(leader, &ack->ref, &ack->prehandle, + PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK, + authenticated_node, authenticated_session, new_leader_out); +} + + +PcmXQueueResult +cluster_pcm_x_local_cancel_ack_exact(const PcmXLocalHandle *leader, const PcmXPhasePayload *ack, + int32 authenticated_node, uint64 authenticated_session, + PcmXLocalHandle *new_leader_out) +{ + if (ack == NULL || ack->reason != 0 || ack->phase != PCM_X_LOCAL_RELIABLE_PHASE_CANCEL + || ack->flags != 0) + return PCM_X_QUEUE_INVALID; + return pcm_x_local_cancel_ack_common(leader, &ack->ref, NULL, PGRAC_IC_MSG_PCM_X_CANCEL_ACK, + authenticated_node, authenticated_session, new_leader_out); +} + + +static PcmXQueueResult +pcm_x_local_terminal_refs_lookup(const PcmXTicketRef *ref, PcmXSlotRef *tag_ref_out, + PcmXSlotRef *member_ref_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalMembershipSlot *member; + PcmXSlotRef member_ref; + PcmXQueueResult result; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_member_lookup_locked(&ref->identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref_out->slot_index = member->tag_slot_index; + tag_ref_out->slot_generation = member->tag_slot_generation; + *member_ref_out = member_ref; + } + LWLockRelease(&header->allocator_lock.lock); + return result == PCM_X_QUEUE_NOT_FOUND ? PCM_X_QUEUE_STALE : result; +} + + +static bool +pcm_x_local_terminal_pending_clear(const PcmXLocalTagSlot *tag_slot) +{ + return tag_slot != NULL && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX; +} + + +PcmXQueueResult +cluster_pcm_x_local_terminal_publish_exact(const PcmXTicketRef *ref, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + uint32 flags; + uint32 member_state; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + + if (header == NULL || !pcm_x_local_terminal_ref_valid(ref) || authenticated_master_node < 0 + || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + frontier = &header->peer_frontiers[authenticated_master_node]; + if (frontier->cluster_epoch != ref->identity.cluster_epoch + || frontier->sender_session_incarnation != authenticated_master_session) + return PCM_X_QUEUE_STALE; + result = pcm_x_local_terminal_refs_lookup(ref, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &ref->identity.tag, + PCM_X_STATE_BIT(PCM_XL_CANCELLED) | PCM_X_STATE_BIT(PCM_XL_GRANTED)); + if (tag_slot == NULL || member == NULL || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || !pcm_x_wait_identity_equal(&member->identity, &ref->identity)) { + result = PCM_X_QUEUE_STALE; + goto publish_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto publish_done; + } + if (tag_slot->master_node != authenticated_master_node + || tag_slot->master_session_incarnation != authenticated_master_session + || tag_slot->cluster_epoch != ref->identity.cluster_epoch + || !pcm_x_ticket_handle_equal(&member->handle, &ref->handle)) { + result = PCM_X_QUEUE_STALE; + goto publish_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto publish_done; + } + gate_claimed = true; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) != 0 + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_READY) == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto publish_done; + } + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_READY) != 0) { + if (((flags & PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) != 0) + != (tag_slot->terminal_drain_generation != 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = pcm_x_ticket_ref_equal(&tag_slot->ref, ref) ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + goto publish_done; + } + member_state = pcm_x_slot_state_read(&member->slot); + if ((member_state != PCM_XL_CANCELLED && member_state != PCM_XL_GRANTED) + || !pcm_x_local_terminal_pending_clear(tag_slot)) { + result = PCM_X_QUEUE_NOT_READY; + goto publish_done; + } + if (member->next_index != PCM_X_INVALID_SLOT_INDEX + || member->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->head_index == member_ref.slot_index + || tag_slot->tail_index == member_ref.slot_index + || tag_slot->leader_index == member_ref.slot_index) { + result = PCM_X_QUEUE_NOT_READY; + goto publish_done; + } + if (!pcm_x_local_terminal_unlinked_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto publish_done; + } + if (!pcm_x_ticket_locator_equal(&tag_slot->ref, ref)) { + result = PCM_X_QUEUE_STALE; + goto publish_done; + } + tag_slot->ref = *ref; + tag_slot->terminal_drain_generation = 0; + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_READY); + result = PCM_X_QUEUE_OK; + +publish_done: + if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +typedef struct PcmXLocalRoundClosePlan { + PcmXLocalMembershipSlot *candidate; + PcmXSlotRef candidate_ref; + bool close_round; +} PcmXLocalRoundClosePlan; + +/* A holder-only PROBE freezes an empty current cohort; writers joining later + * are linked as next-round followers. Validate that whole shape before the + * first terminal mutation, then apply an infallible close under the same local + * lock. A terminal replay uses the same validator to finish a close stranded + * after the terminal evidence was published. */ +static PcmXQueueResult +pcm_x_local_empty_frozen_round_plan_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalRoundClosePlan *plan) +{ + PcmXQueueResult result; + uint32 flags; + bool candidate_promote = false; + + if (tag_slot == NULL || plan == NULL) + return PCM_X_QUEUE_CORRUPT; + memset(plan, 0, sizeof(*plan)); + plan->candidate_ref.slot_index = PCM_X_INVALID_SLOT_INDEX; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + /* Every first holder DRAIN closes beneath the revoke fence, including the + * case where a writer cohort is still terminal and therefore no round is + * advanced here. Check these common facts before the nonempty fast path; + * otherwise a torn barrier/drain can be overwritten as a successful first + * arrival merely because closed_round_member_count is nonzero. */ + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 + || (flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 + || tag_slot->holder_terminal_drain_generation != 0 || tag_slot->local_round == 0 + || tag_slot->local_round == UINT32_MAX || tag_slot->next_sequence == 0 + || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || tag_slot->closed_round_member_count > tag_slot->membership_count + || (tag_slot->closed_round_member_count != 0 && tag_slot->cutoff_sequence == 0)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->closed_round_member_count != 0) + return PCM_X_QUEUE_OK; + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0 + || ((tag_slot->membership_count == 0) != (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX)) + || ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_slot_generation != 0 + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_slot_generation != 0 + || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + || !pcm_x_ticket_ref_is_zero(&tag_slot->ref)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, tag_slot->head_index, + PCM_X_INVALID_SLOT_INDEX, 0, &plan->candidate, + &plan->candidate_ref, &candidate_promote); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_BUSY ? PCM_X_QUEUE_NOT_READY : result; + if (plan->candidate == NULL || candidate_promote) + return PCM_X_QUEUE_CORRUPT; + } + plan->close_round = true; + return PCM_X_QUEUE_OK; +} + + +static void +pcm_x_local_empty_frozen_round_apply_locked(PcmXLocalTagSlot *tag_slot, + const PcmXLocalRoundClosePlan *plan) +{ + if (tag_slot == NULL || plan == NULL || !plan->close_round) + return; + tag_slot->local_round++; + pg_write_barrier(); + (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); + if (plan->candidate != NULL) { + tag_slot->leader_index = plan->candidate_ref.slot_index; + tag_slot->leader_slot_generation = plan->candidate_ref.slot_generation; + plan->candidate->role = PCM_X_LOCAL_ROLE_NODE_LEADER; + pg_write_barrier(); + pcm_x_slot_state_write(&plan->candidate->slot, PCM_XL_NODE_LEADER); + } +} + + +/* Project the state immediately after RETIRE removes the final member of a + * closed writer cohort. The terminal member is already unlinked by DRAIN, so + * the remaining FIFO, if any, must consist entirely of next-round followers. + * This is a read-only plan: every retryable/structural result precedes the + * terminal/ref/count mutation performed by detach_terminal_common. */ +static PcmXQueueResult +pcm_x_local_final_member_round_plan_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalRoundClosePlan *plan) +{ + PcmXQueueResult holder_state; + PcmXQueueResult result; + uint32 flags; + bool candidate_promote = false; + + if (tag_slot == NULL || plan == NULL) + return PCM_X_QUEUE_CORRUPT; + memset(plan, 0, sizeof(*plan)); + plan->candidate_ref.slot_index = PCM_X_INVALID_SLOT_INDEX; + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0 || tag_slot->closed_round_member_count != 1 + || tag_slot->membership_count == 0 || tag_slot->local_round == 0 + || tag_slot->local_round == UINT32_MAX || tag_slot->cutoff_sequence == 0 + || tag_slot->next_sequence == 0 || tag_slot->cutoff_sequence >= tag_slot->next_sequence + || ((tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX)) + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_slot_generation != 0 + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_slot_generation != 0 + || tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) + return PCM_X_QUEUE_CORRUPT; + holder_state = pcm_x_local_holder_lane_retire_state(tag_slot, flags); + if (holder_state != PCM_X_QUEUE_OK) + return holder_state; + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX) { + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, tag_slot->head_index, + PCM_X_INVALID_SLOT_INDEX, 0, &plan->candidate, + &plan->candidate_ref, &candidate_promote); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_BUSY ? PCM_X_QUEUE_NOT_READY : result; + if (plan->candidate == NULL || candidate_promote) + return PCM_X_QUEUE_CORRUPT; + } + plan->close_round = true; + return PCM_X_QUEUE_OK; +} + + +/* Close the terminal leg for an S-holder node that was not selected as the + * image source. Type-48 leaves a generation-zero admission locator plus an + * exact ACK tombstone. The authenticated DRAIN carries the first final grant + * generation this node can know, so capture it exactly and make all replays + * generation- and drain-exact. Caller holds the local tag lock. */ +static PcmXQueueResult +pcm_x_local_blocker_participant_drain_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + const PcmXDrainPollPayload *poll, + int32 authenticated_master_node) +{ + PcmXImageToken zero_image; + PcmXLocalRoundClosePlan round_plan; + PcmXQueueResult blocker_state; + PcmXQueueResult holder_state; + uint32 holder_flags; + + if (tag_slot == NULL || poll == NULL) + return PCM_X_QUEUE_CORRUPT; + if (pcm_x_local_external_terminal_lanes_collide(tag_slot)) + return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + || pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) + return PCM_X_QUEUE_STALE; + if (!pcm_x_ticket_locator_equal(&tag_slot->blocker_snapshot_ref, &poll->ref)) + return PCM_X_QUEUE_STALE; + + holder_flags = pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + memset(&zero_image, 0, sizeof(zero_image)); + if (tag_slot->blocker_snapshot_ref.grant_generation != 0 || holder_flags != 0 + || tag_slot->holder_terminal_drain_generation != 0) { + /* Classify the resident terminal shape before the incoming replay. A + * wrong poll must not hide torn resident flags/drain evidence as STALE. */ + holder_state = pcm_x_local_holder_lane_retire_state(tag_slot, + pcm_x_slot_flags_read(&tag_slot->slot)); + if (holder_state != PCM_X_QUEUE_OK) + return holder_state; + if (!pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, &poll->ref) + || tag_slot->holder_terminal_drain_generation != poll->drain_generation) + return PCM_X_QUEUE_STALE; + return PCM_X_QUEUE_DUPLICATE; + } + if (poll->ref.grant_generation == 0 + && !pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, &poll->ref)) + return PCM_X_QUEUE_STALE; + if (poll->ref.grant_generation == 0 + && (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) + return PCM_X_QUEUE_CORRUPT; + + if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_NOT_READY; + if (holder_flags != 0 || tag_slot->holder_terminal_drain_generation != 0 + || !pcm_x_image_token_equal(&tag_slot->holder_image, &zero_image) + || !pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable)) + return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_local_admission_ref_valid(&tag_slot->blocker_snapshot_ref, + &tag_slot->blocker_snapshot_ref.identity) + || !BufferTagsEqual(&tag_slot->tag, &tag_slot->blocker_snapshot_ref.identity.tag) + || tag_slot->blocker_snapshot_ref.identity.cluster_epoch != tag_slot->cluster_epoch) + return PCM_X_QUEUE_CORRUPT; + if (!pcm_x_local_blocker_ack_tombstone_exact(tag_slot, authenticated_master_node) + || tag_slot->blocker_snapshot_reliable.response_tombstone_mask != 0) { + blocker_state = pcm_x_local_blocker_lane_retire_state(tag_slot); + return blocker_state == PCM_X_QUEUE_CORRUPT ? blocker_state : PCM_X_QUEUE_NOT_READY; + } + blocker_state = pcm_x_local_empty_frozen_round_plan_locked(tag_slot, tag_ref, &round_plan); + if (blocker_state != PCM_X_QUEUE_OK) + return blocker_state; + + tag_slot->blocker_snapshot_ref = poll->ref; + tag_slot->blocker_set_generation = 0; + memset(&tag_slot->blocker_snapshot_reliable, 0, sizeof(tag_slot->blocker_snapshot_reliable)); + tag_slot->holder_terminal_drain_generation = poll->drain_generation; + pg_write_barrier(); + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + pcm_x_local_empty_frozen_round_apply_locked(tag_slot, &round_plan); + return PCM_X_QUEUE_OK; +} + + +/* A holder node may have no local writer membership at all. Its type-50 + * resend ledger therefore closes directly through the local-tag directory. + * The exact DRAIN_POLL is positive proof that the master consumed IMAGE_READY; + * transport send success never reaches this function. */ +static PcmXQueueResult +pcm_x_local_holder_drain_poll_exact(const PcmXDrainPollPayload *poll, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalRoundClosePlan round_plan; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef tag_ref; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + PcmXQueueResult round_state; + uint32 flags; + uint32 holder_terminal_flags; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + + runtime = cluster_pcm_x_runtime_snapshot(); + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &poll->ref.identity.tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&poll->ref.identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &poll->ref.identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_STALE; + goto holder_drain_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto holder_drain_done; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto holder_drain_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto holder_drain_done; + } + gate_claimed = true; + if (pcm_x_local_external_terminal_lanes_collide(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto holder_drain_done; + } + if (pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref)) { + result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, + authenticated_master_node); + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto holder_drain_done; + } + if (!pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &poll->ref)) { + result = PCM_X_QUEUE_STALE; + goto holder_drain_done; + } + /* REVOKE publishes holder_ref and consumes the blocker lane under this same + * tag lock. Seeing both is structural corruption, never a transient wait. */ + if (!pcm_x_local_blocker_lane_is_pristine(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto holder_drain_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + holder_terminal_flags = flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + if ((holder_terminal_flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_DRAINED) != 0 + && (holder_terminal_flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_READY) == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto holder_drain_done; + } + if (holder_terminal_flags != 0) { + if (holder_terminal_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + || tag_slot->holder_terminal_drain_generation == 0 + || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else if (tag_slot->holder_terminal_drain_generation != poll->drain_generation) + result = PCM_X_QUEUE_STALE; + else + result = PCM_X_QUEUE_DUPLICATE; + goto holder_drain_done; + } + if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 || tag_slot->blocker_snapshot_crc32c != 0 + || tag_slot->blocker_snapshot_next_chunk != 0 + || !pcm_x_transfer_leg_is_clear(&tag_slot->blocker_snapshot_reliable) + || !pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref)) { + result = PCM_X_QUEUE_NOT_READY; + goto holder_drain_done; + } + if (!pcm_x_transfer_leg_exact(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_IMAGE_READY, + authenticated_master_node, authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto holder_drain_done; + } + round_state = pcm_x_local_empty_frozen_round_plan_locked(tag_slot, tag_ref, &round_plan); + if (round_state != PCM_X_QUEUE_OK) { + result = round_state; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto holder_drain_done; + } + pcm_x_transfer_leg_clear(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_DRAIN_POLL, + authenticated_master_node); + tag_slot->holder_terminal_drain_generation = poll->drain_generation; + pg_write_barrier(); + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + pcm_x_local_empty_frozen_round_apply_locked(tag_slot, &round_plan); + result = PCM_X_QUEUE_OK; + +holder_drain_done: + if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *candidate; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXSlotRef candidate_ref; + PcmXQueueResult external_result; + PcmXQueueResult result; + uint32 flags; + uint32 member_state; + uint32 partition; + bool gate_claimed = false; + bool fail_closed = false; + bool promote_candidate; + + if (header == NULL || poll == NULL || !pcm_x_local_terminal_ref_valid(&poll->ref) + || poll->drain_generation == 0 || poll->drain_generation == UINT64_MAX + || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + frontier = &header->peer_frontiers[authenticated_master_node]; + if (frontier->cluster_epoch != poll->ref.identity.cluster_epoch + || frontier->sender_session_incarnation != authenticated_master_session) + return PCM_X_QUEUE_STALE; + result = pcm_x_local_terminal_refs_lookup(&poll->ref, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_STALE) + return pcm_x_local_holder_drain_poll_exact(poll, authenticated_master_node, + authenticated_master_session); + return result; + } + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&poll->ref.identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &poll->ref.identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &poll->ref.identity.tag, + PCM_X_STATE_BIT(PCM_XL_CANCELLED) | PCM_X_STATE_BIT(PCM_XL_GRANTED)); + if (tag_slot == NULL || member == NULL || member->tag_slot_index != tag_ref.slot_index + || member->tag_slot_generation != tag_ref.slot_generation + || !pcm_x_wait_identity_equal(&member->identity, &poll->ref.identity)) { + result = PCM_X_QUEUE_STALE; + goto drain_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_done; + } + if (tag_slot->master_node != authenticated_master_node + || tag_slot->master_session_incarnation != authenticated_master_session + || tag_slot->cluster_epoch != poll->ref.identity.cluster_epoch + || !pcm_x_ticket_ref_equal(&tag_slot->ref, &poll->ref) + || !pcm_x_ticket_handle_equal(&member->handle, &poll->ref.handle)) { + result = PCM_X_QUEUE_STALE; + goto drain_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto drain_done; + } + gate_claimed = true; + if (pcm_x_local_external_terminal_lanes_collide(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) != 0 + && (flags & PCM_X_LOCAL_TAG_F_TERMINAL_READY) == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + /* FINAL_CONFIRM and backend unlock are independent arrivals. DRAIN must + * never unlink the leader or clear a claim owned by a backend that still + * has content authority; the master retry closes the terminal leg after + * exact claim release. */ + if (tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_done; + } + member_state = pcm_x_slot_state_read(&member->slot); + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_READY) == 0) { + if (member_state != PCM_XL_GRANTED || (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_done; + } + if (member->admitted_round != tag_slot->local_round + || member->local_sequence > tag_slot->cutoff_sequence + || tag_slot->closed_round_member_count == 0) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + if ((pcm_x_slot_flags_read(&member->slot) & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) == 0 + || tag_slot->closed_round_member_count != 1) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_done; + } + if (!pcm_x_local_reliable_leg_valid(tag_slot, member) + || !pcm_x_transfer_leg_exact(&tag_slot->reliable, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, + authenticated_master_node, authenticated_master_session) + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation + || member->role != PCM_X_LOCAL_ROLE_NODE_LEADER) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, member->next_index, + member_ref.slot_index, member->local_sequence, + &candidate, &candidate_ref, &promote_candidate); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (!fail_closed) + goto drain_release_gate; + goto drain_done; + } + if (!pcm_x_local_unlink_member_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + tag_slot->leader_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->leader_slot_generation = 0; + pcm_x_transfer_leg_clear(&tag_slot->reliable, PGRAC_IC_MSG_PCM_X_DRAIN_POLL, + authenticated_master_node); + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_READY); + flags |= PCM_X_LOCAL_TAG_F_TERMINAL_READY; + } + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) != 0) { + result = tag_slot->terminal_drain_generation == poll->drain_generation + ? PCM_X_QUEUE_DUPLICATE + : PCM_X_QUEUE_STALE; + goto drain_release_gate; + } + if ((member_state != PCM_XL_CANCELLED && member_state != PCM_XL_GRANTED) + || !pcm_x_local_terminal_pending_clear(tag_slot)) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_release_gate; + } + if (member->next_index != PCM_X_INVALID_SLOT_INDEX + || member->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->head_index == member_ref.slot_index + || tag_slot->tail_index == member_ref.slot_index + || tag_slot->leader_index == member_ref.slot_index) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_done; + } + if (!pcm_x_local_terminal_unlinked_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + /* The requester node may simultaneously be either the source S-holder or + * a non-source holder participant. Consume both durable legs under this + * one tag lock before publishing either drain. */ + if (pcm_x_ticket_ref_equal(&tag_slot->holder_ref, &poll->ref)) { + uint32 holder_flags = flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + + if (!pcm_x_local_blocker_lane_is_pristine(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + if (holder_flags != 0) { + if (holder_flags != PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK + || tag_slot->holder_terminal_drain_generation != poll->drain_generation + || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto drain_done; + } + } else if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX + || !pcm_x_transfer_leg_exact( + &tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_IMAGE_READY, + authenticated_master_node, authenticated_master_session)) { + result = PCM_X_QUEUE_NOT_READY; + goto drain_release_gate; + } else { + pcm_x_transfer_leg_clear(&tag_slot->holder_reliable, PGRAC_IC_MSG_PCM_X_DRAIN_POLL, + authenticated_master_node); + tag_slot->holder_terminal_drain_generation = poll->drain_generation; + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + } + } else if (!pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && pcm_x_ticket_locator_equal(&tag_slot->blocker_snapshot_ref, &poll->ref)) { + external_result = pcm_x_local_blocker_participant_drain_locked(tag_slot, tag_ref, poll, + authenticated_master_node); + if (external_result != PCM_X_QUEUE_OK && external_result != PCM_X_QUEUE_DUPLICATE) { + result = external_result; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto drain_release_gate; + } + } + tag_slot->terminal_drain_generation = poll->drain_generation; + pg_write_barrier(); + (void)pcm_x_slot_flags_fetch_or(&tag_slot->slot, PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED); + result = PCM_X_QUEUE_OK; + +drain_release_gate: +drain_done: + if (gate_claimed && !fail_closed && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->local_locks[partition].lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_cancel_exact(const PcmXLocalHandle *handle, PcmXLocalHandle *new_leader_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *candidate = NULL; + PcmXSlotRef member_ref; + PcmXSlotRef candidate_ref = { PCM_X_INVALID_SLOT_INDEX, 0 }; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + uint32 partition; + uint32 state; + bool promote_candidate = false; + + pcm_x_local_handle_clear(new_leader_out); + if (header == NULL || handle == NULL || !pcm_x_wait_identity_valid(&handle->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_member_lookup_locked(&handle->identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) + return result; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&handle->identity.tag)); + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &handle->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_WAIT, member_ref, + &handle->identity.tag, + PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto cancel_local_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto cancel_local_done; + } + result = pcm_x_local_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + goto cancel_local_done; + } + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + /* Cancellation may unlink or promote queue members. Serialize it after + * exact active-claim completion so no claim token can retain a dangling + * slot or a role changed underneath it. */ + if (tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) { + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + state = pcm_x_slot_state_read(&member->slot); + if (state == PCM_XL_CANCELLED) { + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_DUPLICATE : PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + if (state != PCM_XL_NODE_LEADER && state != PCM_XL_JOINED_NONWAITABLE + && state != PCM_XL_WAITABLE_FOLLOWER) { + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_BAD_STATE : PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + if ((state == PCM_XL_NODE_LEADER + && (member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || tag_slot->leader_index != member_ref.slot_index + || tag_slot->leader_slot_generation != member_ref.slot_generation)) + || (state != PCM_XL_NODE_LEADER + && (member->role != PCM_X_LOCAL_ROLE_FOLLOWER + || tag_slot->leader_index == member_ref.slot_index))) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + if (state == PCM_XL_WAITABLE_FOLLOWER && member->graph_generation != 0) { + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + if ((state == PCM_XL_JOINED_NONWAITABLE || state == PCM_XL_NODE_LEADER) + && member->graph_generation != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + /* Follower cancellation is local-only. Once a leader has minted any + * remote authority, it must keep the tag/member and durable resend leg + * until an exact 58/60 application ACK performs the atomic rotation. */ + if (state == PCM_XL_NODE_LEADER + && (tag_slot->prehandle.sender_session_incarnation != 0 + || tag_slot->prehandle.prehandle_sequence != 0 + || !pcm_x_ticket_ref_is_zero(&tag_slot->ref) + || !pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable))) { + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + /* A follower can have been an active blocker and then abort its claim + * without acquiring a persistent COMPLETE marker. Conservatively drain + * every later edge before any linked identity disappears; this also keeps + * cancellation sound across that former-active window. */ + result = pcm_x_local_follower_chain_ready(tag_slot, tag_ref, member->next_index, + member_ref.slot_index, member->local_sequence, + &candidate, &candidate_ref, &promote_candidate); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_BUSY) + result = pcm_x_local_gate_release(tag_slot) ? result : PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + if (!pcm_x_local_unlink_member_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto cancel_local_done; + } + if (state == PCM_XL_NODE_LEADER) { + tag_slot->leader_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->leader_slot_generation = 0; + if (candidate != NULL && promote_candidate) { + tag_slot->leader_index = candidate_ref.slot_index; + tag_slot->leader_slot_generation = candidate_ref.slot_generation; + candidate->role = PCM_X_LOCAL_ROLE_NODE_LEADER; + pg_write_barrier(); + pcm_x_slot_state_write(&candidate->slot, PCM_XL_NODE_LEADER); + if (new_leader_out != NULL) + pcm_x_local_handle_from_member(new_leader_out, candidate_ref, candidate, + PCM_X_LOCAL_ROLE_NODE_LEADER); + } + } + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_CANCELLED); + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_OK : PCM_X_QUEUE_CORRUPT; + +cancel_local_done: + /* Any post-acquire direct jump here is CORRUPT and retains the gate. */ + LWLockRelease(&header->local_locks[partition].lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; +} + + +/* Prove that a terminal membership is no longer reachable from the local FIFO. */ +static bool +pcm_x_local_terminal_unlinked_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXLocalMembershipSlot *terminal, PcmXSlotRef terminal_ref) +{ + PcmXAllocatorView view; + Size current; + Size previous = PCM_X_INVALID_SLOT_INDEX; + Size visited = 0; + + if (tag_slot == NULL || terminal == NULL || !pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_WAIT, &view) + || terminal->next_index != PCM_X_INVALID_SLOT_INDEX + || terminal->prev_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->head_index == terminal_ref.slot_index + || tag_slot->tail_index == terminal_ref.slot_index + || tag_slot->leader_index == terminal_ref.slot_index + || tag_slot->active_writer_index == terminal_ref.slot_index) + return false; + + current = tag_slot->head_index; + while (current != PCM_X_INVALID_SLOT_INDEX) { + PcmXLocalMembershipSlot *linked; + PcmXSlotRef linked_ref; + + if (current == terminal_ref.slot_index || visited >= tag_slot->membership_count + || visited >= view.capacity) + return false; + linked = (PcmXLocalMembershipSlot *)pcm_x_allocator_slot(&view, current); + if (linked == NULL) + return false; + linked_ref.slot_index = current; + if (!pcm_x_slot_generation_read(&linked->slot, &linked_ref.slot_generation)) + return false; + linked = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, linked_ref, &tag_slot->tag, PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (linked == NULL || linked->tag_slot_index != tag_ref.slot_index + || linked->tag_slot_generation != tag_ref.slot_generation + || linked->prev_index != previous) + return false; + previous = current; + current = linked->next_index; + visited++; + } + if (tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX) + return tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX; + return tag_slot->tail_index == previous; +} + + +/* Return an open, empty writer queue to the holder-only form. The holder + * chain and its generation remain authoritative; only queue-session state is + * cleared so a later writer can adopt the same tag under the admission gate. */ +static void +pcm_x_local_reset_holder_only_queue(PcmXLocalTagSlot *tag_slot) +{ + tag_slot->master_node = -1; + tag_slot->master_session_incarnation = 0; + tag_slot->next_sequence = 1; + tag_slot->cutoff_sequence = 0; + tag_slot->blocker_set_generation = 0; + memset(&tag_slot->prehandle, 0, sizeof(tag_slot->prehandle)); + memset(&tag_slot->ref, 0, sizeof(tag_slot->ref)); + memset(&tag_slot->image, 0, sizeof(tag_slot->image)); + memset(&tag_slot->reliable, 0, sizeof(tag_slot->reliable)); + tag_slot->leader_slot_generation = 0; + tag_slot->active_writer_slot_generation = 0; + tag_slot->closed_round_member_count = 0; + tag_slot->committed_own_generation = 0; +} + + +static PcmXQueueResult pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *tag_slot, + PcmXSlotRef tag_ref, + PcmXWaitIdentity *identity_out, + PcmXLocalHandle *handle_out, + bool *candidate_out); + + +static PcmXQueueResult +pcm_x_local_detach_terminal_common(const PcmXLocalHandle *handle, bool retire_protocol, + PcmXLocalHandle *promoted_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXLocalRoundClosePlan round_plan; + PcmXLocalHandle promoted; + PcmXLocalHandle ready_leader; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot *gated_tag = NULL; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot *promoted_member = NULL; + PcmXSlotRef found; + PcmXSlotRef member_ref; + PcmXSlotRef tag_ref; + PcmXQueueResult result; + PcmXWaitIdentity ready_identity; + uint32 flags; + uint32 partition; + bool cancel_same_ref_dual = false; + bool detach_after_close = false; + bool detach_tag = false; + bool gate_claimed = false; + bool same_ref_dual = false; + bool ready_leader_candidate = false; + bool target_in_closed_round; + bool close_round = false; + + pcm_x_local_handle_clear(promoted_out); + pcm_x_local_handle_clear(&promoted); + pcm_x_local_handle_clear(&ready_leader); + memset(&round_plan, 0, sizeof(round_plan)); + memset(&ready_identity, 0, sizeof(ready_identity)); + if (header == NULL || handle == NULL || !pcm_x_wait_identity_valid(&handle->identity)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + if (retire_protocol) + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_SHARED, NULL); + else + LWLockAcquire(&header->allocator_lock.lock, LW_SHARED); + result = pcm_x_local_member_lookup_locked(&handle->identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) { + tag_ref.slot_index = member->tag_slot_index; + tag_ref.slot_generation = member->tag_slot_generation; + if (!retire_protocol) { + tag_slot + = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE + || !BufferTagsEqual(&tag_slot->tag, &handle->identity.tag) + || tag_slot->cluster_epoch != handle->identity.cluster_epoch) + result = PCM_X_QUEUE_CORRUPT; + else + result = pcm_x_local_handoff_gate_claim_locked(header, tag_slot, false, + PCM_X_QUEUE_BUSY); + } + } + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + gate_claimed = !retire_protocol; + if (gate_claimed) + gated_tag = tag_slot; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&handle->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, + retire_protocol ? NULL : gated_tag); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &handle->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &handle->identity.tag, + PCM_X_STATE_BIT(PCM_XL_CANCELLED) + | (retire_protocol ? PCM_X_STATE_BIT(PCM_XL_GRANTED) : 0)); + if (tag_slot == NULL || member == NULL + || !pcm_x_local_handle_exact(handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_STALE; + goto detach_local_domain_done; + } + if (!pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + if (retire_protocol) { + result = pcm_x_local_retire_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) + goto detach_local_domain_done; + } + if (retire_protocol) { + gate_claimed = true; + gated_tag = tag_slot; + } else if ((pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + if (tag_slot->membership_count == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + if (!pcm_x_local_terminal_unlinked_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0) { + uint32 terminal_flags = flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK; + + if (!retire_protocol) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + if (terminal_flags == PCM_X_LOCAL_TAG_F_TERMINAL_READY) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + if (terminal_flags != PCM_X_LOCAL_TAG_F_TERMINAL_MASK + || tag_slot->terminal_drain_generation == 0 + || !pcm_x_local_terminal_ref_valid(&tag_slot->ref) + || !pcm_x_wait_identity_equal(&tag_slot->ref.identity, &handle->identity) + || !pcm_x_ticket_handle_equal(&tag_slot->ref.handle, &member->handle)) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + } else if (retire_protocol) { + result = PCM_X_QUEUE_STALE; + goto detach_local_domain_done; + } + if (retire_protocol) { + result = pcm_x_local_same_ref_dual_retire_state(tag_slot, flags, &same_ref_dual, + &cancel_same_ref_dual); + if (result != PCM_X_QUEUE_OK) + goto detach_local_domain_done; + if (cancel_same_ref_dual) { + /* Unlike a successful grant, cancellation has no positive generation + * that a holder-only detach can validate later. Prove the exact + * writer tombstone is CANCELLED, then consume both same-ref terminal + * legs atomically under this local lock. */ + if (!same_ref_dual || pcm_x_slot_state_read(&member->slot) != PCM_XL_CANCELLED + || tag_slot->ref.grant_generation != 0 + || !pcm_x_ticket_ref_equal(&tag_slot->blocker_snapshot_ref, &tag_slot->ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + } + } + target_in_closed_round = (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + && member->admitted_round == tag_slot->local_round + && member->local_sequence <= tag_slot->cutoff_sequence; + if ((cancel_same_ref_dual && !target_in_closed_round) + || (target_in_closed_round && tag_slot->closed_round_member_count == 0) + || tag_slot->closed_round_member_count > tag_slot->membership_count + || (((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) + && tag_slot->closed_round_member_count != 0) + || (tag_slot->membership_count == 1 + && tag_slot->closed_round_member_count != (target_in_closed_round ? 1 : 0))) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + if (retire_protocol && target_in_closed_round && tag_slot->closed_round_member_count == 1) { + result = pcm_x_local_final_member_round_plan_locked(tag_slot, tag_ref, &round_plan); + if (result != PCM_X_QUEUE_OK) + goto detach_local_domain_done; + close_round = true; + } + if (tag_slot->membership_count == 1) { + if (tag_slot->head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->reliable.pending_opcode != 0 || tag_slot->reliable.phase != 0) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + /* Writer membership and holder transfer are independent lifetimes on + * the same tag. Neither returning to holder-only form nor deleting the + * tag may erase the master binding while an exact holder ticket, send + * leg, or terminal-drain outcome remains authoritative. */ + if (!pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + || !pcm_x_transfer_leg_is_clear(&tag_slot->holder_reliable) + || ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0 && !cancel_same_ref_dual)) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + if (tag_slot->active_holder_head_index != PCM_X_INVALID_SLOT_INDEX) { + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + || tag_slot->blocker_snapshot_head_index != PCM_X_INVALID_SLOT_INDEX + || tag_slot->blocker_snapshot_count != 0 + || tag_slot->closed_round_member_count != 0) { + result = PCM_X_QUEUE_NOT_READY; + goto detach_local_domain_done; + } + pcm_x_local_reset_holder_only_queue(tag_slot); + } else if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); + detach_tag = true; + } + if (close_round) { + if (round_plan.candidate != NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + detach_after_close = true; + } + } + /* All retryable and structural checks precede the first mutation. A + * cancellation dual terminal is one local transaction: holder evidence, + * writer terminal, and membership either all survive or all advance. */ + if (cancel_same_ref_dual) { + memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); + tag_slot->blocker_set_generation = 0; + memset(&tag_slot->blocker_snapshot_reliable, 0, + sizeof(tag_slot->blocker_snapshot_reliable)); + tag_slot->holder_terminal_drain_generation = 0; + pg_write_barrier(); + (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + } + if (target_in_closed_round) { + tag_slot->closed_round_member_count--; + } + if (retire_protocol) { + uint64 reliable_state_sequence = tag_slot->reliable.state_sequence; + + memset(&tag_slot->prehandle, 0, sizeof(tag_slot->prehandle)); + memset(&tag_slot->ref, 0, sizeof(tag_slot->ref)); + /* image and committed_own_generation are writer-round evidence. They + * must disappear in the same local transaction as the exact terminal + * ref; otherwise the promoted next-round leader inherits a stale grant + * and its generation rekey correctly classifies the tag as corrupt. */ + memset(&tag_slot->image, 0, sizeof(tag_slot->image)); + tag_slot->committed_own_generation = 0; + /* DRAIN closes the old round by leaving its response opcode as a replay + * tombstone. RETIRE consumes that tombstone with the membership: a + * promoted successor must see a pristine application leg or the requester + * cannot mint its first ENQUEUE. Keep only the monotonic state sequence so + * a new round cannot recreate an old reliable token. */ + memset(&tag_slot->reliable, 0, sizeof(tag_slot->reliable)); + tag_slot->reliable.state_sequence = reliable_state_sequence; + tag_slot->terminal_drain_generation = 0; + (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_TERMINAL_MASK); + } + tag_slot->membership_count--; + if (close_round) { + pcm_x_local_empty_frozen_round_apply_locked(tag_slot, &round_plan); + if (round_plan.candidate != NULL) + pcm_x_local_handle_from_member(&promoted, round_plan.candidate_ref, + round_plan.candidate, PCM_X_LOCAL_ROLE_NODE_LEADER); + } + /* Cancellation ACK may have promoted a same-round successor before DRAIN, + * while the terminal fence still made that first wake unusable. Once this + * RETIRE mutation removes the last local terminal lane, report either that + * already-promoted leader or the final-member promotion above through the + * same exact output. An independent holder terminal keeps the fence and + * therefore deliberately suppresses the wake until its own RETIRE. */ + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if (retire_protocol + && (flags & (PCM_X_LOCAL_TAG_F_TERMINAL_MASK | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK)) + == 0) { + result = pcm_x_local_ready_leader_wake_locked(tag_slot, tag_ref, &ready_identity, + &ready_leader, &ready_leader_candidate); + if (result != PCM_X_QUEUE_OK) + goto detach_local_domain_done; + if (ready_leader_candidate) { + if (pcm_x_wait_identity_valid(&promoted.identity) + && (!pcm_x_wait_identity_equal(&promoted.identity, &ready_identity) + || promoted.membership_slot.slot_index + != ready_leader.membership_slot.slot_index + || promoted.membership_slot.slot_generation + != ready_leader.membership_slot.slot_generation)) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_domain_done; + } + promoted = ready_leader; + } + } + if (detach_after_close) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); + detach_tag = true; + } + pg_write_barrier(); + pcm_x_slot_state_write(&member->slot, PCM_XL_DETACHING); + result = PCM_X_QUEUE_OK; + +detach_local_domain_done: + /* Any post-acquire error reaching here is CORRUPT and retains the gate. */ + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK) { + if (result != PCM_X_QUEUE_CORRUPT && gate_claimed) { + if (!pcm_x_local_gate_release(gated_tag)) + result = PCM_X_QUEUE_CORRUPT; + else + gate_claimed = false; + } + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + return result; + } + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, gated_tag); + member = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_WAIT, + member_ref); + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (close_round && round_plan.candidate != NULL) + promoted_member = (PcmXLocalMembershipSlot *)pcm_x_slot_ref_resolve_locked( + PCM_X_ALLOC_LOCAL_WAIT, round_plan.candidate_ref); + if (member == NULL || tag_slot == NULL + || pcm_x_slot_state_read(&member->slot) != PCM_XL_DETACHING + || (detach_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_DETACHING) + || (!detach_tag && pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_LIVE) + || !(close_round ? pcm_x_local_detaching_handle_exact(handle, member_ref, member, tag_slot) + : pcm_x_local_handle_exact(handle, member_ref, member, tag_slot)) + || (close_round && round_plan.candidate != NULL + && (promoted_member == NULL + || pcm_x_slot_state_read(&promoted_member->slot) != PCM_XL_NODE_LEADER + || promoted_member->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || promoted_member->tag_slot_index != tag_ref.slot_index + || promoted_member->tag_slot_generation != tag_ref.slot_generation + || promoted_member->admitted_round != tag_slot->local_round + || promoted_member->local_sequence != promoted.local_sequence + || !pcm_x_wait_identity_equal(&promoted_member->identity, &promoted.identity))) + || (pcm_x_slot_flags_read(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) == 0) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_allocator_done; + } + if (pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_WAIT, &handle->identity, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != member_ref.slot_index + || found.slot_generation != member_ref.slot_generation + || (detach_tag + && (pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &handle->identity.tag, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != tag_ref.slot_index + || found.slot_generation != tag_ref.slot_generation))) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_allocator_done; + } + if (pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_WAIT, &handle->identity, member_ref) + != PCM_X_DIRECTORY_OK + || (detach_tag + && pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, &handle->identity.tag, + tag_ref) + != PCM_X_DIRECTORY_OK) + || pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_WAIT, member_ref, PCM_XL_DETACHING) + != PCM_X_ALLOC_OK + || (detach_tag + && pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, PCM_X_TAG_DETACHING) + != PCM_X_ALLOC_OK)) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_allocator_done; + } + if (!detach_tag && !pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + goto detach_local_allocator_done; + } + result = PCM_X_QUEUE_OK; + +detach_local_allocator_done: + /* Allocator-phase failures are CORRUPT and retain any surviving gate. */ + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + pcm_x_runtime_fail_closed(); + if (result == PCM_X_QUEUE_OK && promoted_out != NULL) + *promoted_out = promoted; + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_detach_terminal_exact(const PcmXLocalHandle *handle) +{ + return pcm_x_local_detach_terminal_common(handle, false, NULL); +} + + +/* + * RETIRE is deliberately a bounded cold-path scan over the existing local-tag + * pool. A live tag's BufferTag is immutable until allocator-owned release; + * generation/state are sampled before taking its partition lock and then + * revalidated exactly under that lock. No allocator/domain locks are nested. + */ +static PcmXQueueResult +pcm_x_local_ready_leader_wake_locked(PcmXLocalTagSlot *tag_slot, PcmXSlotRef tag_ref, + PcmXWaitIdentity *identity_out, PcmXLocalHandle *handle_out, + bool *candidate_out) +{ + PcmXLocalMembershipSlot *leader; + PcmXSlotRef leader_ref; + uint32 member_flags; + uint32 member_state; + + if (identity_out == NULL || candidate_out == NULL) + return PCM_X_QUEUE_CORRUPT; + memset(identity_out, 0, sizeof(*identity_out)); + pcm_x_local_handle_clear(handle_out); + *candidate_out = false; + if (tag_slot == NULL) + return PCM_X_QUEUE_CORRUPT; + if ((tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->leader_slot_generation == 0)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_OK; + if ((tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX) + != (tag_slot->active_writer_slot_generation == 0)) + return PCM_X_QUEUE_CORRUPT; + if (tag_slot->active_writer_index != PCM_X_INVALID_SLOT_INDEX) + return PCM_X_QUEUE_OK; + leader_ref.slot_index = tag_slot->leader_index; + leader_ref.slot_generation = tag_slot->leader_slot_generation; + leader = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, leader_ref, &tag_slot->tag, PCM_X_LOCAL_MEMBER_DOMAIN_STATES); + if (leader == NULL || tag_slot->membership_count == 0 + || tag_slot->head_index != leader_ref.slot_index + || leader->tag_slot_index != tag_ref.slot_index + || leader->tag_slot_generation != tag_ref.slot_generation + || leader->prev_index != PCM_X_INVALID_SLOT_INDEX + || leader->role != PCM_X_LOCAL_ROLE_NODE_LEADER + || leader->admitted_round != tag_slot->local_round || leader->graph_generation != 0 + || !BufferTagsEqual(&leader->identity.tag, &tag_slot->tag) + || leader->identity.cluster_epoch != tag_slot->cluster_epoch) + return PCM_X_QUEUE_CORRUPT; + member_flags = pcm_x_slot_flags_read(&leader->slot); + if ((member_flags & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE) != 0) + return PCM_X_QUEUE_CORRUPT; + member_state = pcm_x_slot_state_read(&leader->slot); + if (!pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable)) { + if (!pcm_x_local_reliable_leg_valid(tag_slot, leader)) + return PCM_X_QUEUE_CORRUPT; + return PCM_X_QUEUE_OK; + } + if (member_state != PCM_XL_NODE_LEADER) + return PCM_X_QUEUE_OK; + *identity_out = leader->identity; + if (handle_out != NULL) + pcm_x_local_handle_from_member(handle_out, leader_ref, leader, + PCM_X_LOCAL_ROLE_NODE_LEADER); + *candidate_out = true; + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +pcm_x_local_retire_candidate_at(Size slot_index, const PcmXRetirePayload *request, + int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXTicketRef *ref_out, + PcmXTicketRef *writer_ref_out, bool *writer_candidate_out, + bool *holder_candidate_out, bool *candidate_out, + bool *contains_watermark_out, PcmXWaitIdentity *holder_wake_out, + bool *holder_wake_candidate_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXAllocatorView view; + PcmXLocalTagSlot *raw; + PcmXLocalTagSlot *tag_slot; + const PcmXTicketRef *external_ref; + PcmXSlotRef tag_ref; + BufferTag tag; + uint64 generation_after; + uint32 flags; + uint32 partition; + PcmXQueueResult result = PCM_X_QUEUE_OK; + PcmXQueueResult holder_state; + bool cancel_same_ref_dual = false; + bool same_ref_dual = false; + + *candidate_out = false; + *writer_candidate_out = false; + *holder_candidate_out = false; + *contains_watermark_out = false; + *holder_wake_candidate_out = false; + memset(ref_out, 0, sizeof(*ref_out)); + memset(writer_ref_out, 0, sizeof(*writer_ref_out)); + memset(holder_wake_out, 0, sizeof(*holder_wake_out)); + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view) || slot_index >= view.capacity) + return PCM_X_QUEUE_INVALID; + raw = (PcmXLocalTagSlot *)pcm_x_allocator_slot(&view, slot_index); + if (raw == NULL || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE) + return PCM_X_QUEUE_OK; + if (!pcm_x_slot_generation_read(&raw->slot, &tag_ref.slot_generation)) + return PCM_X_QUEUE_BUSY; + tag_ref.slot_index = slot_index; + tag = raw->tag; + pg_read_barrier(); + if (!pcm_x_slot_generation_read(&raw->slot, &generation_after) + || generation_after != tag_ref.slot_generation + || pcm_x_slot_state_read(&raw->slot) != PCM_X_TAG_LIVE) + return PCM_X_QUEUE_BUSY; + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_SHARED, NULL); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot(PCM_X_ALLOC_LOCAL_TAG, tag_ref, &tag, + PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + if (tag_slot == NULL) { + result = PCM_X_QUEUE_BUSY; + goto candidate_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & (PCM_X_LOCAL_TAG_F_TERMINAL_MASK | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK)) == 0) + goto candidate_done; + if (tag_slot->master_node != authenticated_master_node) + goto candidate_done; + if (tag_slot->cluster_epoch != request->cluster_epoch + || tag_slot->master_session_incarnation != authenticated_master_session) { + result = PCM_X_QUEUE_STALE; + goto candidate_done; + } + result = pcm_x_local_same_ref_dual_retire_state(tag_slot, flags, &same_ref_dual, + &cancel_same_ref_dual); + if (result != PCM_X_QUEUE_OK) + goto candidate_done; + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != 0) { + holder_state = pcm_x_local_holder_lane_retire_state(tag_slot, flags); + if (holder_state != PCM_X_QUEUE_OK) { + result = holder_state; + goto candidate_done; + } + if (!pcm_x_local_terminal_ref_valid(&tag_slot->ref) + || !BufferTagsEqual(&tag_slot->tag, &tag_slot->ref.identity.tag)) { + result = PCM_X_QUEUE_CORRUPT; + goto candidate_done; + } + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != PCM_X_LOCAL_TAG_F_TERMINAL_MASK + || tag_slot->terminal_drain_generation == 0 + || !pcm_x_local_terminal_pending_clear(tag_slot)) { + result = PCM_X_QUEUE_NOT_READY; + goto candidate_done; + } + if (tag_slot->ref.handle.ticket_id == request->retire_through_ticket_id) + *contains_watermark_out = true; + if (tag_slot->ref.handle.ticket_id <= request->retire_through_ticket_id) { + *writer_ref_out = tag_slot->ref; + *writer_candidate_out = true; + *ref_out = tag_slot->ref; + *candidate_out = true; + } + } + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) != 0) { + holder_state = pcm_x_local_holder_lane_retire_state(tag_slot, flags); + if (holder_state != PCM_X_QUEUE_OK) { + result = holder_state; + goto candidate_done; + } + external_ref = pcm_x_local_external_terminal_ref(tag_slot); + if (external_ref == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto candidate_done; + } + if (external_ref->handle.ticket_id == request->retire_through_ticket_id) + *contains_watermark_out = true; + if (external_ref->handle.ticket_id <= request->retire_through_ticket_id) { + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual) { + result = PCM_X_QUEUE_NOT_READY; + goto candidate_done; + } + /* A generation-zero dual terminal must validate and consume the + * CANCELLED membership before its blocker evidence disappears. Keep + * the writer candidate selected above; detach_terminal_common closes + * both lanes under the same local lock. */ + if (cancel_same_ref_dual) { + if (!*candidate_out || !pcm_x_ticket_ref_equal(ref_out, external_ref)) + result = PCM_X_QUEUE_CORRUPT; + goto candidate_done; + } + if (!same_ref_dual && *candidate_out) + goto candidate_done; + *ref_out = *external_ref; + *holder_candidate_out = true; + *candidate_out = true; + } + } + if (result == PCM_X_QUEUE_OK && *holder_candidate_out) + result = pcm_x_local_ready_leader_wake_locked(tag_slot, tag_ref, holder_wake_out, NULL, + holder_wake_candidate_out); + +candidate_done: + LWLockRelease(&header->local_locks[partition].lock); + return result; +} + + +static PcmXQueueResult +pcm_x_local_holder_detach_terminal_exact(const PcmXTicketRef *ref, int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXWaitIdentity *ready_leader_out, + bool *ready_leader_candidate_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalTagSlot *tag_slot; + const PcmXTicketRef *external_ref; + PcmXSlotRef tag_ref; + PcmXSlotRef found; + PcmXDirectoryResult directory_result; + PcmXQueueResult result; + PcmXQueueResult holder_state; + PcmXWaitIdentity ready_leader; + uint32 flags; + uint32 partition; + bool detach_tag = false; + bool fail_closed = false; + bool blocker_terminal = false; + bool cancel_same_ref_dual = false; + bool same_ref_dual = false; + bool ready_leader_candidate = false; + + if (ready_leader_out == NULL || ready_leader_candidate_out == NULL) + return PCM_X_QUEUE_INVALID; + memset(ready_leader_out, 0, sizeof(*ready_leader_out)); + *ready_leader_candidate_out = false; + memset(&ready_leader, 0, sizeof(ready_leader)); + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_SHARED, NULL); + directory_result + = pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, &tag_ref); + LWLockRelease(&header->allocator_lock.lock); + if (directory_result == PCM_X_DIRECTORY_NOT_FOUND) + return PCM_X_QUEUE_STALE; + if (directory_result != PCM_X_DIRECTORY_OK) + return pcm_x_queue_result_from_directory(directory_result); + + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&ref->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_EXCLUSIVE, NULL); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &ref->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + external_ref = pcm_x_local_external_terminal_ref(tag_slot); + if (tag_slot == NULL || external_ref == NULL || !pcm_x_ticket_ref_equal(external_ref, ref)) { + result = PCM_X_QUEUE_STALE; + goto holder_detach_domain_done; + } + blocker_terminal = external_ref == &tag_slot->blocker_snapshot_ref; + result = pcm_x_local_retire_gate_try_acquire(tag_slot); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto holder_detach_domain_done; + } + if (!pcm_x_local_holder_transfer_peer_exact(tag_slot, authenticated_master_node, + authenticated_master_session)) { + result = PCM_X_QUEUE_STALE; + goto holder_detach_release_gate; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + result = pcm_x_local_same_ref_dual_retire_state(tag_slot, flags, &same_ref_dual, + &cancel_same_ref_dual); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto holder_detach_domain_done; + } + if (cancel_same_ref_dual) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto holder_detach_domain_done; + } + holder_state = pcm_x_local_holder_lane_retire_state(tag_slot, flags); + if (holder_state != PCM_X_QUEUE_OK) { + result = holder_state; + fail_closed = result == PCM_X_QUEUE_CORRUPT; + if (!fail_closed) + goto holder_detach_release_gate; + goto holder_detach_domain_done; + } + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 && !same_ref_dual) { + result = PCM_X_QUEUE_NOT_READY; + goto holder_detach_release_gate; + } + result = pcm_x_local_ready_leader_wake_locked(tag_slot, tag_ref, &ready_leader, NULL, + &ready_leader_candidate); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto holder_detach_domain_done; + } + if (blocker_terminal) { + memset(&tag_slot->blocker_snapshot_ref, 0, sizeof(tag_slot->blocker_snapshot_ref)); + tag_slot->blocker_set_generation = 0; + memset(&tag_slot->blocker_snapshot_reliable, 0, + sizeof(tag_slot->blocker_snapshot_reliable)); + } else { + memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); + memset(&tag_slot->holder_image, 0, sizeof(tag_slot->holder_image)); + memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); + } + tag_slot->holder_terminal_drain_generation = 0; + (void)pcm_x_slot_flags_fetch_and(&tag_slot->slot, ~PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + flags = pcm_x_slot_flags_read(&tag_slot->slot); + detach_tag + = tag_slot->membership_count == 0 && tag_slot->closed_round_member_count == 0 + && tag_slot->active_holder_head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->head_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->tail_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->leader_index == PCM_X_INVALID_SLOT_INDEX + && tag_slot->active_writer_index == PCM_X_INVALID_SLOT_INDEX + && pcm_x_ticket_ref_is_zero(&tag_slot->ref) + && pcm_x_ticket_ref_is_zero(&tag_slot->holder_ref) + && pcm_x_ticket_ref_is_zero(&tag_slot->blocker_snapshot_ref) + && pcm_x_local_reliable_leg_is_clear(&tag_slot->reliable) + && pcm_x_transfer_leg_is_pristine(&tag_slot->holder_reliable) + && pcm_x_transfer_leg_is_pristine(&tag_slot->blocker_snapshot_reliable) + && (flags & (PCM_X_LOCAL_TAG_F_REVOKE_BARRIER | PCM_X_LOCAL_TAG_F_TERMINAL_MASK)) == 0; + if (detach_tag) { + pg_write_barrier(); + pcm_x_slot_state_write(&tag_slot->slot, PCM_X_TAG_DETACHING); + result = PCM_X_QUEUE_OK; + goto holder_detach_domain_done; + } + result = pcm_x_local_gate_release(tag_slot) ? PCM_X_QUEUE_OK : PCM_X_QUEUE_CORRUPT; + if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + goto holder_detach_domain_done; + +holder_detach_release_gate: + if (!pcm_x_local_gate_release(tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } +holder_detach_domain_done: + LWLockRelease(&header->local_locks[partition].lock); + if (result != PCM_X_QUEUE_OK || !detach_tag) { + if (fail_closed) + pcm_x_runtime_fail_closed(); + if (result == PCM_X_QUEUE_OK && ready_leader_candidate) { + *ready_leader_out = ready_leader; + *ready_leader_candidate_out = true; + } + return result; + } + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, tag_slot); + tag_slot = (PcmXLocalTagSlot *)pcm_x_slot_ref_resolve_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref); + if (tag_slot == NULL || pcm_x_slot_state_read(&tag_slot->slot) != PCM_X_TAG_DETACHING + || pcm_x_directory_find_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, &found) + != PCM_X_DIRECTORY_OK + || found.slot_index != tag_ref.slot_index + || found.slot_generation != tag_ref.slot_generation + || pcm_x_directory_delete_exact_locked(PCM_X_DIR_LOCAL_TAG, &ref->identity.tag, tag_ref) + != PCM_X_DIRECTORY_OK + || pcm_x_allocator_release_locked(PCM_X_ALLOC_LOCAL_TAG, tag_ref, PCM_X_TAG_DETACHING) + != PCM_X_ALLOC_OK) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else + result = PCM_X_QUEUE_OK; + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + if (result == PCM_X_QUEUE_OK && ready_leader_candidate) { + *ready_leader_out = ready_leader; + *ready_leader_candidate_out = true; + } + return result; +} + + +static PcmXQueueResult +pcm_x_local_terminal_handle_lookup(const PcmXWaitIdentity *identity, PcmXLocalHandle *handle_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalMembershipSlot *member; + PcmXSlotRef member_ref; + PcmXQueueResult result; + + pcm_x_local_handle_clear(handle_out); + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_SHARED, NULL); + result = pcm_x_local_member_lookup_locked(identity, &member_ref, &member); + if (result == PCM_X_QUEUE_OK) + pcm_x_local_handle_from_member(handle_out, member_ref, member, (PcmXLocalRole)member->role); + LWLockRelease(&header->allocator_lock.lock); + return result == PCM_X_QUEUE_NOT_FOUND ? PCM_X_QUEUE_STALE : result; +} + + +/* RETIRE's first pass must prove every projected final-member round close + * before any older tag is reclaimed. In particular a late follower can + * still own an external WFG edge; that is ordinary NOT_READY, not a reason to + * apply earlier tags and then turn the second-pass surprise into corruption. */ +static PcmXQueueResult +pcm_x_local_final_member_round_preflight_exact(const PcmXLocalHandle *handle, + PcmXWaitIdentity *projected_wake_out, + bool *projected_wake_candidate_out) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXLocalRoundClosePlan plan; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXSlotRef tag_ref; + PcmXSlotRef member_ref; + PcmXQueueResult result; + PcmXWaitIdentity existing_wake; + uint32 flags; + uint32 partition; + bool cancel_same_ref_dual = false; + bool existing_wake_candidate = false; + bool same_ref_dual = false; + bool target_in_closed_round; + + if (projected_wake_out == NULL || projected_wake_candidate_out == NULL) + return PCM_X_QUEUE_INVALID; + memset(projected_wake_out, 0, sizeof(*projected_wake_out)); + memset(&existing_wake, 0, sizeof(existing_wake)); + *projected_wake_candidate_out = false; + if (header == NULL || handle == NULL || !pcm_x_wait_identity_valid(&handle->identity)) + return PCM_X_QUEUE_INVALID; + result = pcm_x_local_refs_lookup(handle, &tag_ref, &member_ref); + if (result != PCM_X_QUEUE_OK) + return result == PCM_X_QUEUE_NOT_FOUND ? PCM_X_QUEUE_STALE : result; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&handle->identity.tag)); + pcm_x_local_gate_acquire_guarded(&header->local_locks[partition].lock, LW_SHARED, NULL); + tag_slot = (PcmXLocalTagSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_TAG, tag_ref, &handle->identity.tag, PCM_X_STATE_BIT(PCM_X_TAG_LIVE)); + member = (PcmXLocalMembershipSlot *)pcm_x_domain_slot( + PCM_X_ALLOC_LOCAL_WAIT, member_ref, &handle->identity.tag, + PCM_X_STATE_BIT(PCM_XL_CANCELLED) | PCM_X_STATE_BIT(PCM_XL_GRANTED)); + if (tag_slot == NULL || member == NULL) { + result = PCM_X_QUEUE_STALE; + goto preflight_done; + } + if (!pcm_x_local_handle_exact(handle, member_ref, member, tag_slot)) { + result = PCM_X_QUEUE_CORRUPT; + goto preflight_done; + } + flags = pcm_x_slot_flags_read(&tag_slot->slot); + if ((flags & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) != PCM_X_LOCAL_TAG_F_TERMINAL_MASK + || tag_slot->terminal_drain_generation == 0 + || !pcm_x_wait_identity_equal(&tag_slot->ref.identity, &member->identity) + || !pcm_x_ticket_handle_equal(&tag_slot->ref.handle, &member->handle) + || !pcm_x_local_terminal_unlinked_locked(tag_slot, tag_ref, member, member_ref)) { + result = PCM_X_QUEUE_CORRUPT; + goto preflight_done; + } + result = pcm_x_local_same_ref_dual_retire_state(tag_slot, flags, &same_ref_dual, + &cancel_same_ref_dual); + if (result != PCM_X_QUEUE_OK) + goto preflight_done; + if (cancel_same_ref_dual + && (!same_ref_dual || pcm_x_slot_state_read(&member->slot) != PCM_XL_CANCELLED)) { + result = PCM_X_QUEUE_CORRUPT; + goto preflight_done; + } + /* A cancel ACK may already have rotated leader_index while its terminal + * fence kept the successor asleep. Project that exact leader only when + * this writer RETIRE also clears every holder terminal lane; otherwise the + * independent holder RETIRE remains the sole future wake authority. */ + if ((flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) == 0 || same_ref_dual) { + result = pcm_x_local_ready_leader_wake_locked(tag_slot, tag_ref, &existing_wake, NULL, + &existing_wake_candidate); + if (result != PCM_X_QUEUE_OK) + goto preflight_done; + if (existing_wake_candidate) { + *projected_wake_out = existing_wake; + *projected_wake_candidate_out = true; + } + } + target_in_closed_round = (flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0 + && member->admitted_round == tag_slot->local_round + && member->local_sequence <= tag_slot->cutoff_sequence; + if ((flags & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) == 0) { + result = tag_slot->closed_round_member_count == 0 ? PCM_X_QUEUE_OK : PCM_X_QUEUE_CORRUPT; + goto preflight_done; + } + if (!target_in_closed_round || tag_slot->closed_round_member_count == 0 + || tag_slot->closed_round_member_count > tag_slot->membership_count) { + result = PCM_X_QUEUE_CORRUPT; + goto preflight_done; + } + if (tag_slot->closed_round_member_count != 1) { + result = PCM_X_QUEUE_OK; + goto preflight_done; + } + result = pcm_x_local_final_member_round_plan_locked(tag_slot, tag_ref, &plan); + if (result == PCM_X_QUEUE_OK && plan.candidate != NULL) { + if (*projected_wake_candidate_out) { + result = PCM_X_QUEUE_CORRUPT; + goto preflight_done; + } + *projected_wake_out = plan.candidate->identity; + *projected_wake_candidate_out = true; + } + +preflight_done: + LWLockRelease(&header->local_locks[partition].lock); + return result; +} + + +static PcmXQueueResult +pcm_x_local_wake_project(PcmXLocalWakeBatch *wake_batch, const PcmXWaitIdentity *identity, + Size *projected_count) +{ + Size i; + + if (projected_count == NULL || identity == NULL || !pcm_x_wait_identity_valid(identity)) + return PCM_X_QUEUE_CORRUPT; + if (wake_batch == NULL) + return PCM_X_QUEUE_OK; + for (i = 0; i < *projected_count; i++) { + if (pcm_x_wait_identity_equal(&wake_batch->items[i], identity)) + return PCM_X_QUEUE_CORRUPT; + } + if (*projected_count >= wake_batch->capacity) + return PCM_X_QUEUE_NO_CAPACITY; + wake_batch->items[*projected_count] = *identity; + (*projected_count)++; + return PCM_X_QUEUE_OK; +} + + +static PcmXQueueResult +pcm_x_local_wake_actual_match(const PcmXLocalWakeBatch *wake_batch, + const PcmXWaitIdentity *identity, Size projected_count, + Size *actual_count) +{ + if (actual_count == NULL || identity == NULL || !pcm_x_wait_identity_valid(identity)) + return PCM_X_QUEUE_CORRUPT; + if (wake_batch == NULL) + return PCM_X_QUEUE_OK; + if (*actual_count >= projected_count + || !pcm_x_wait_identity_equal(&wake_batch->items[*actual_count], identity)) + return PCM_X_QUEUE_CORRUPT; + (*actual_count)++; + return PCM_X_QUEUE_OK; +} + + +PcmXQueueResult +cluster_pcm_x_local_retire_up_to_collect_exact(const PcmXRetirePayload *request, + int32 authenticated_master_node, + uint64 authenticated_master_session, + PcmXLocalWakeBatch *wake_batch) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXRuntimeSnapshot runtime; + PcmXPeerFrontier *frontier; + PcmXAllocatorView view; + PcmXTicketRef candidate_ref; + PcmXTicketRef writer_candidate_ref; + PcmXLocalHandle handle; + PcmXLocalHandle promoted; + PcmXWaitIdentity holder_wake; + PcmXWaitIdentity writer_wake; + PcmXQueueResult result = PCM_X_QUEUE_OK; + Size actual_wake_count = 0; + Size i; + Size projected_wake_count = 0; + uint32 expected_retire_gate; + bool candidate; + bool contains_watermark; + bool holder_candidate; + bool holder_wake_candidate; + bool writer_candidate; + bool writer_wake_candidate; + bool found_exact = false; + bool fail_closed = false; + + if (wake_batch != NULL) + wake_batch->count = 0; + if (header == NULL || request == NULL || request->master_session_incarnation == 0 + || request->retire_through_ticket_id == 0 || request->sender_node < 0 + || request->sender_node >= PCM_X_PROTOCOL_NODE_LIMIT || request->flags != 0 + || authenticated_master_node < 0 || authenticated_master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authenticated_master_session == 0 + || (wake_batch != NULL && wake_batch->capacity != 0 && wake_batch->items == NULL)) + return PCM_X_QUEUE_INVALID; + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return PCM_X_QUEUE_NOT_READY; + + /* Claim one exact monotonic retirement episode before any local evidence is + * removed. A stranded nonzero marker is intentionally fail-closed evidence. */ + LWLockAcquire(&header->allocator_lock.lock, LW_EXCLUSIVE); + frontier = &header->peer_frontiers[authenticated_master_node]; + if (request->master_session_incarnation != authenticated_master_session + || frontier->cluster_epoch != request->cluster_epoch + || frontier->sender_session_incarnation != authenticated_master_session) { + result = PCM_X_QUEUE_STALE; + goto claim_done; + } + if (request->retire_through_ticket_id <= frontier->local_retired_ticket_id) { + result = PCM_X_QUEUE_DUPLICATE; + goto claim_done; + } + result = pcm_x_local_retire_episode_state_locked(header); + if (result != PCM_X_QUEUE_OK) { + fail_closed = result == PCM_X_QUEUE_CORRUPT; + goto claim_done; + } + expected_retire_gate = 0; + if (!pg_atomic_compare_exchange_u32(&header->local_retire_gate, &expected_retire_gate, + (uint32)authenticated_master_node + 1)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto claim_done; + } + pg_write_barrier(); + frontier->local_retire_in_progress_ticket_id = request->retire_through_ticket_id; + result = pcm_x_local_admission_handoffs_quiescent_locked(header); + if (result != PCM_X_QUEUE_OK) { + if (result == PCM_X_QUEUE_BUSY) { + frontier->local_retire_in_progress_ticket_id = 0; + pg_write_barrier(); + pg_atomic_write_u32(&header->local_retire_gate, 0); + } else + fail_closed = true; + goto claim_done; + } + result = PCM_X_QUEUE_OK; + +claim_done: + LWLockRelease(&header->allocator_lock.lock); + if (result != PCM_X_QUEUE_OK) { + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; + } + if (!pcm_x_allocator_view(PCM_X_ALLOC_LOCAL_TAG, &view)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto retire_failed; + } + + /* First pass is read-only: an undrained lower/equal ticket must not cause + * partial reclamation of any older terminal evidence. */ + for (i = 0; i < view.capacity; i++) { + result = pcm_x_local_retire_candidate_at( + i, request, authenticated_master_node, authenticated_master_session, &candidate_ref, + &writer_candidate_ref, &writer_candidate, &holder_candidate, &candidate, + &contains_watermark, &holder_wake, &holder_wake_candidate); + if (result != PCM_X_QUEUE_OK) + goto retire_preflight_failed; + if (holder_wake_candidate) { + result = pcm_x_local_wake_project(wake_batch, &holder_wake, &projected_wake_count); + if (result != PCM_X_QUEUE_OK) + goto retire_preflight_failed; + } + if (writer_candidate) { + result = pcm_x_local_terminal_handle_lookup(&writer_candidate_ref.identity, &handle); + if (result == PCM_X_QUEUE_OK) + result = pcm_x_local_final_member_round_preflight_exact(&handle, &writer_wake, + &writer_wake_candidate); + if (result != PCM_X_QUEUE_OK) + goto retire_preflight_failed; + if (writer_wake_candidate) { + result = pcm_x_local_wake_project(wake_batch, &writer_wake, &projected_wake_count); + if (result != PCM_X_QUEUE_OK) + goto retire_preflight_failed; + } + } + if (contains_watermark) + found_exact = true; + } + if (!found_exact) { + result = PCM_X_QUEUE_STALE; + goto retire_preflight_failed; + } + + /* Second pass revalidates each exact terminal membership while detaching. + * Any post-first-detach surprise retains the in-progress marker and blocks + * the runtime; retrying a partially applied watermark would be ambiguous. */ + for (i = 0; i < view.capacity; i++) { + do { + result = pcm_x_local_retire_candidate_at( + i, request, authenticated_master_node, authenticated_master_session, &candidate_ref, + &writer_candidate_ref, &writer_candidate, &holder_candidate, &candidate, + &contains_watermark, &holder_wake, &holder_wake_candidate); + if (result != PCM_X_QUEUE_OK) { + fail_closed = true; + result = PCM_X_QUEUE_CORRUPT; + goto retire_failed; + } + if (!candidate) + break; + memset(&promoted, 0, sizeof(promoted)); + memset(&writer_wake, 0, sizeof(writer_wake)); + writer_wake_candidate = false; + if (holder_candidate) + result = pcm_x_local_holder_detach_terminal_exact( + &candidate_ref, authenticated_master_node, authenticated_master_session, + &writer_wake, &writer_wake_candidate); + else { + result = pcm_x_local_terminal_handle_lookup(&candidate_ref.identity, &handle); + if (result == PCM_X_QUEUE_OK) + result = pcm_x_local_detach_terminal_common(&handle, true, &promoted); + if (result == PCM_X_QUEUE_OK && pcm_x_wait_identity_valid(&promoted.identity)) { + writer_wake = promoted.identity; + writer_wake_candidate = true; + } + } + if (result != PCM_X_QUEUE_OK) { + fail_closed = true; + result = PCM_X_QUEUE_CORRUPT; + goto retire_failed; + } + if (writer_wake_candidate) { + result = pcm_x_local_wake_actual_match(wake_batch, &writer_wake, + projected_wake_count, &actual_wake_count); + if (result != PCM_X_QUEUE_OK) { + fail_closed = true; + result = PCM_X_QUEUE_CORRUPT; + goto retire_failed; + } + } + } while (candidate); + } + if (wake_batch != NULL && actual_wake_count != projected_wake_count) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + goto retire_failed; + } + + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, NULL); + frontier = &header->peer_frontiers[authenticated_master_node]; + if (pcm_x_local_retire_episode_state_locked(header) != PCM_X_QUEUE_BUSY + || frontier->cluster_epoch != request->cluster_epoch + || frontier->sender_session_incarnation != authenticated_master_session + || frontier->local_retire_in_progress_ticket_id != request->retire_through_ticket_id + || frontier->local_retired_ticket_id >= request->retire_through_ticket_id + || pg_atomic_read_u32(&header->local_retire_gate) != (uint32)authenticated_master_node + 1 + || !pcm_x_runtime_token_exact(&runtime, 0)) { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } else { + frontier->local_retired_ticket_id = request->retire_through_ticket_id; + pg_write_barrier(); + frontier->local_retire_in_progress_ticket_id = 0; + pg_write_barrier(); + pg_atomic_write_u32(&header->local_retire_gate, 0); + result = PCM_X_QUEUE_OK; + } + LWLockRelease(&header->allocator_lock.lock); + if (fail_closed) + pcm_x_runtime_fail_closed(); + if (result == PCM_X_QUEUE_OK && wake_batch != NULL) { + pg_write_barrier(); + wake_batch->count = projected_wake_count; + } + return result; + +retire_preflight_failed: + /* No tag was mutated, so a retryable/stale preflight can release the claim. + * Structural corruption retains both exact episode witnesses. */ + if (result == PCM_X_QUEUE_CORRUPT) { + fail_closed = true; + goto retire_failed; + } + pcm_x_local_gate_acquire_guarded(&header->allocator_lock.lock, LW_EXCLUSIVE, NULL); + frontier = &header->peer_frontiers[authenticated_master_node]; + if (pcm_x_local_retire_episode_state_locked(header) == PCM_X_QUEUE_BUSY + && frontier->local_retire_in_progress_ticket_id == request->retire_through_ticket_id + && pg_atomic_read_u32(&header->local_retire_gate) + == (uint32)authenticated_master_node + 1) { + frontier->local_retire_in_progress_ticket_id = 0; + pg_write_barrier(); + pg_atomic_write_u32(&header->local_retire_gate, 0); + } else { + result = PCM_X_QUEUE_CORRUPT; + fail_closed = true; + } + LWLockRelease(&header->allocator_lock.lock); + if (result == PCM_X_QUEUE_CORRUPT) + fail_closed = true; + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; + +retire_failed: + /* Keep the in-progress marker as the durable ambiguity witness. */ + if (fail_closed) + pcm_x_runtime_fail_closed(); + return result; +} + + +PcmXQueueResult +cluster_pcm_x_local_retire_up_to_exact(const PcmXRetirePayload *request, + int32 authenticated_master_node, + uint64 authenticated_master_session) +{ + return cluster_pcm_x_local_retire_up_to_collect_exact(request, authenticated_master_node, + authenticated_master_session, NULL); +} + + +/* Validate signed runtime inputs before converting them to Size. */ +static void +pcm_x_compute_runtime_layout(PcmXShmemLayout *layout) +{ + if (MaxBackends <= 0 || NBuffers <= 0 || cluster_lmd_max_wait_edges <= 0) + ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid PCM-X shared memory capacity inputs"), + errhint("Use positive MaxBackends, shared_buffers, and " + "cluster.lmd_max_wait_edges values."))); + + cluster_pcm_x_layout_compute((Size)MaxBackends, (Size)NUM_AUXILIARY_PROCS, (Size)NBuffers, + (Size)cluster_lmd_max_wait_edges, layout); +} + + +/* + * cluster_pcm_x_convert_shmem_size -- Return the checked runtime region size. + * + * Inputs: + * None. Capacity inputs come from postmaster configuration. + * + * Returns: + * The exact byte size required by the PCM-X conversion region. + * + * Side Effects: + * Raises ERROR for invalid capacity inputs or arithmetic overflow. + * + * Author: SqlRush + */ +Size +cluster_pcm_x_convert_shmem_size(void) +{ + PcmXShmemLayout layout; + + pcm_x_compute_runtime_layout(&layout); + return layout.total_size; +} + + +/* Initialize one index-based free list without storing process pointers. */ +static void +pcm_x_init_free_list(char *base, Size slots_offset, Size slot_size, Size first_slot_index, + Size capacity, PcmXAllocatorState *allocator) +{ + Size i; + + allocator->free_head = capacity == 0 ? PCM_X_INVALID_SLOT_INDEX : first_slot_index; + allocator->used = 0; + allocator->high_water = 0; + allocator->generation_exhausted = 0; + for (i = 0; i < capacity;) { + Size local_next = add_size(i, 1); + Size next = local_next < capacity ? add_size(first_slot_index, local_next) + : PCM_X_INVALID_SLOT_INDEX; + Size offset = add_size(slots_offset, mul_size(i, slot_size)); + PcmXSlotHeader *slot = (PcmXSlotHeader *)(base + offset); + + slot->next_free = next; + pg_atomic_init_u32(&slot->generation_change_seq, 0); + slot->slot_generation_lo = 0; + slot->slot_generation_hi = 0; + pg_atomic_init_u32(&slot->state_flags, PCM_X_SLOT_FREE); + i = local_next; + } +} + + +/* Build all pool allocators, keeping membership wait/holder lists disjoint. */ +static void +pcm_x_init_allocators(PcmXShmemHeader *header) +{ + PcmXShmemLayout *layout = &header->layout; + char *base = (char *)header; + + pcm_x_init_free_list(base, layout->pools[PCM_X_POOL_MASTER_TAG].slots_offset, + layout->pools[PCM_X_POOL_MASTER_TAG].slot_size, 0, + layout->pools[PCM_X_POOL_MASTER_TAG].capacity, + &header->allocator[PCM_X_ALLOC_MASTER_TAG]); + pcm_x_init_free_list(base, layout->pools[PCM_X_POOL_MASTER_TICKET].slots_offset, + layout->pools[PCM_X_POOL_MASTER_TICKET].slot_size, 0, + layout->pools[PCM_X_POOL_MASTER_TICKET].capacity, + &header->allocator[PCM_X_ALLOC_MASTER_TICKET]); + pcm_x_init_free_list(base, layout->pools[PCM_X_POOL_BLOCKER].slots_offset, + layout->pools[PCM_X_POOL_BLOCKER].slot_size, 0, + layout->pools[PCM_X_POOL_BLOCKER].capacity, + &header->allocator[PCM_X_ALLOC_BLOCKER]); + pcm_x_init_free_list(base, layout->pools[PCM_X_POOL_LOCAL_TAG].slots_offset, + layout->pools[PCM_X_POOL_LOCAL_TAG].slot_size, 0, + layout->pools[PCM_X_POOL_LOCAL_TAG].capacity, + &header->allocator[PCM_X_ALLOC_LOCAL_TAG]); + pcm_x_init_free_list(base, layout->local_wait.slots_offset, sizeof(PcmXLocalMembershipSlot), + layout->local_wait.first_slot_index, layout->local_wait.capacity, + &header->allocator[PCM_X_ALLOC_LOCAL_WAIT]); + pcm_x_init_free_list(base, layout->local_holder.slots_offset, sizeof(PcmXLocalMembershipSlot), + layout->local_holder.first_slot_index, layout->local_holder.capacity, + &header->allocator[PCM_X_ALLOC_LOCAL_HOLDER]); +} + + +/* + * cluster_pcm_x_convert_shmem_init -- Create or attach the conversion region. + * + * Inputs: + * None. The expected layout comes from postmaster configuration. + * + * Returns: + * Nothing. On success, ClusterPcmXConvertShmem is published locally. + * + * Side Effects: + * Creates and initializes the shared region on first use. An attach + * preserves mutable allocator state. A ProcGlobal or layout mismatch is + * fatal, and the process-local pointer remains unpublished. + * + * Author: SqlRush + */ +void +cluster_pcm_x_convert_shmem_init(void) +{ + PcmXShmemLayout expected; + PcmXShmemHeader *header; + PcmXAttachResult attach_result; + bool found; + + ClusterPcmXConvertShmem = NULL; + pcm_x_compute_runtime_layout(&expected); + if (ProcGlobal == NULL || (Size)ProcGlobal->allProcCount != expected.process_capacity) + ereport(FATAL, + (errcode(ERRCODE_INTERNAL_ERROR), + errmsg("PCM-X process capacity does not match ProcGlobal"), + errhint("Restart the postmaster with a consistent backend process layout."))); + Assert((Size)ProcGlobal->allProcCount == expected.process_capacity); + + header = ShmemInitStruct(PCM_X_SHMEM_REGION_NAME, expected.total_size, &found); + if (found) { + attach_result = cluster_pcm_x_validate_attach(header, &expected); + if (attach_result != PCM_X_ATTACH_OK) + ereport(FATAL, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("PCM-X shared memory layout is incompatible (validation result %d)", + (int)attach_result), + errhint("Restart the postmaster with one binary and one PCM-X capacity " + "configuration."))); + } else { + int i; + + memset(header, 0, expected.total_size); + header->layout = expected; + header->next_ticket_id = 1; + header->next_image_id = 1; + header->master_session_incarnation = 0; + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + header->peer_frontiers[i].next_expected_prehandle_sequence = 1; + pg_atomic_init_u32(&header->outbound_targets[i].mint_gate, 0); + } + pg_atomic_init_u32(&header->runtime_gate, + pcm_x_runtime_gate_pack(0, PCM_X_RUNTIME_RECOVERY_BLOCKED)); + pg_atomic_init_u32(&header->activation_retry_generation, 0); + pg_atomic_init_u32(&header->local_retire_gate, 0); + pcm_x_init_stats(&header->stats); + pcm_x_init_allocators(header); + LWLockInitialize(&header->allocator_lock.lock, LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR); + for (i = 0; i < PCM_X_LOCK_PARTITIONS; i++) { + LWLockInitialize(&header->master_locks[i].lock, LWTRANCHE_CLUSTER_PCM_X_MASTER); + LWLockInitialize(&header->local_locks[i].lock, LWTRANCHE_CLUSTER_PCM_X_LOCAL); + } + } + ClusterPcmXConvertShmem = header; +} + + +static const ClusterShmemRegion pcm_x_convert_region = { + .name = PCM_X_SHMEM_REGION_NAME, + .size_fn = cluster_pcm_x_convert_shmem_size, + .init_fn = cluster_pcm_x_convert_shmem_init, + .lwlock_count = PCM_X_LWLOCK_COUNT, + .owner_subsys = "pcm_x_convert", + .reserved_flags = 0, +}; + + +/* + * cluster_pcm_x_convert_shmem_register -- Register the one external region. + * + * Inputs: + * None. + * + * Returns: + * Nothing. + * + * Side Effects: + * Adds one immutable descriptor to the cluster shared-memory registry. + * + * Author: SqlRush + */ +void +cluster_pcm_x_convert_shmem_register(void) +{ + cluster_shmem_register_region(&pcm_x_convert_region); +} diff --git a/src/backend/cluster/cluster_pcm_x_image_fetch.c b/src/backend/cluster/cluster_pcm_x_image_fetch.c new file mode 100644 index 00000000000..a73f60b6c0a --- /dev/null +++ b/src/backend/cluster/cluster_pcm_x_image_fetch.c @@ -0,0 +1,169 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_x_image_fetch.c + * Dependency-light validation layer for the PCM-X holder-image adapter. + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_pcm_x_image_fetch.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#ifdef USE_PGRAC_CLUSTER + +#include "cluster/cluster_pcm_x_image_fetch.h" +#include "storage/bufpage.h" + + +static bool +pcm_x_image_fetch_wait_identity_equal(const PcmXWaitIdentity *left, const PcmXWaitIdentity *right) +{ + return left != NULL && right != NULL && BufferTagsEqual(&left->tag, &right->tag) + && left->node_id == right->node_id && left->procno == right->procno + && left->xid == right->xid && left->cluster_epoch == right->cluster_epoch + && left->request_id == right->request_id && left->wait_seq == right->wait_seq + && left->base_own_generation == right->base_own_generation; +} + + +static bool +pcm_x_image_fetch_token_valid(const PcmXImageToken *image, int32 expected_master_node) +{ + int32 encoded_master_node; + + return image != NULL && image->source_node < PCM_X_PROTOCOL_NODE_LIMIT + && expected_master_node >= 0 && expected_master_node < PCM_X_PROTOCOL_NODE_LIMIT + && cluster_pcm_x_image_id_decode(image->image_id, &encoded_master_node, NULL) + && encoded_master_node == expected_master_node; +} + + +static bool +pcm_x_image_fetch_progress_valid(const PcmXLocalProgress *progress, int32 requester_node, + int32 requester_backend_id) +{ + int32 decoded_backend_id; + int32 decoded_node; + + return progress != NULL && requester_node >= 0 && requester_node < PCM_X_PROTOCOL_NODE_LIMIT + && requester_backend_id > 0 && progress->identity.node_id == requester_node + && pcm_x_image_fetch_wait_identity_equal(&progress->identity, &progress->ref.identity) + && progress->ref.handle.ticket_id != 0 && progress->ref.handle.queue_generation != 0 + && progress->ref.grant_generation != 0 && progress->identity.wait_seq != 0 + && cluster_gcs_requester_id_decode(progress->identity.request_id, &decoded_node, + &decoded_backend_id, NULL) + && decoded_node == requester_node && decoded_backend_id == requester_backend_id + && progress->role == PCM_X_LOCAL_ROLE_NODE_LEADER + && progress->member_state == PCM_XL_REMOTE_WAIT && progress->pending_opcode == 0 + && progress->last_response_opcode == PGRAC_IC_MSG_PCM_X_PREPARE_GRANT + && progress->master_session_incarnation != 0 + && pcm_x_image_fetch_token_valid(&progress->image, progress->master_node); +} + + +bool +cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, int32 requester_node, + int32 requester_backend_id, + GcsBlockRequestPayload *request_out) +{ + if (request_out != NULL) + memset(request_out, 0, sizeof(*request_out)); + if (request_out == NULL + || !pcm_x_image_fetch_progress_valid(progress, requester_node, requester_backend_id)) + return false; + + request_out->request_id = progress->image.image_id; + request_out->epoch = progress->identity.cluster_epoch; + request_out->tag = progress->identity.tag; + request_out->sender_node = requester_node; + request_out->requester_backend_id = requester_backend_id; + request_out->transition_id = (uint8)PCM_TRANS_N_TO_S; + return true; +} + + +bool +cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env, + const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, int32 holder_node, + int32 current_master_node, uint64 current_epoch) +{ + static const uint8 zero_reserved[sizeof(request->reserved_0)] = { 0 }; + int32 decoded_backend_id; + int32 decoded_master_node; + int32 decoded_requester_node; + + return env != NULL && request != NULL && holder != NULL && holder_node >= 0 + && holder_node < PCM_X_PROTOCOL_NODE_LIMIT && current_master_node >= 0 + && current_master_node < PCM_X_PROTOCOL_NODE_LIMIT + && env->source_node_id == (uint32)request->sender_node + && env->dest_node_id == (uint32)holder_node && env->payload_length == sizeof(*request) + && request->sender_node >= 0 && request->sender_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->requester_backend_id > 0 && request->epoch == current_epoch + && request->transition_id == (uint8)PCM_TRANS_N_TO_S + && memcmp(request->reserved_0, zero_reserved, sizeof(zero_reserved)) == 0 + && holder->master_node == current_master_node && holder->master_session_incarnation != 0 + && holder->ref.grant_generation != 0 && holder->ref.handle.ticket_id != 0 + && holder->ref.handle.queue_generation != 0 + && holder->ref.identity.cluster_epoch == current_epoch + && holder->ref.identity.wait_seq != 0 + && BufferTagsEqual(&holder->ref.identity.tag, &request->tag) + && holder->ref.identity.node_id == request->sender_node + && holder->pending_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY + && holder->phase == PGRAC_IC_MSG_PCM_X_IMAGE_READY && holder->flags == 0 + && pcm_x_image_fetch_token_valid(&holder->image, current_master_node) + && holder->image.image_id == request->request_id + && holder->image.source_node == (uint32)holder_node + && cluster_pcm_x_image_id_decode(request->request_id, &decoded_master_node, NULL) + && decoded_master_node == current_master_node + && cluster_gcs_requester_id_decode(holder->ref.identity.request_id, + &decoded_requester_node, &decoded_backend_id, NULL) + && decoded_requester_node == request->sender_node + && decoded_backend_id == request->requester_backend_id; +} + + +bool +cluster_pcm_x_image_fetch_reply_exact(const GcsBlockReplyHeader *reply, const char *block_data, + const PcmXLocalProgress *progress, int32 requester_node, + int32 requester_backend_id) +{ + static const uint8 zero_reserved[sizeof(reply->reserved_0)] = { 0 }; + PageHeaderData page_header; + + if (reply == NULL || block_data == NULL + || !pcm_x_image_fetch_progress_valid(progress, requester_node, requester_backend_id)) + return false; + if (reply->request_id != progress->image.image_id || reply->page_lsn != progress->image.page_lsn + || reply->epoch != progress->identity.cluster_epoch + || reply->checksum != progress->image.page_checksum + || reply->sender_node != (int32)progress->image.source_node + || reply->requester_backend_id != requester_backend_id + || reply->transition_id != (uint8)PCM_TRANS_N_TO_S + || reply->status != (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER + || GcsBlockReplyHeaderGetForwardingMasterNode(reply) != GCS_BLOCK_REPLY_NO_FORWARDING_MASTER + || memcmp(reply->reserved_0, zero_reserved, sizeof(zero_reserved)) != 0 + || cluster_gcs_block_compute_checksum(block_data) != progress->image.page_checksum) + return false; + + memcpy(&page_header, block_data, sizeof(page_header)); + return (uint64)PageXLogRecPtrGet(page_header.pd_lsn) == progress->image.page_lsn + && (uint64)page_header.pd_block_scn == progress->image.page_scn; +} + + +bool +cluster_pcm_x_image_fetch_reservation_exact(const ClusterPcmOwnSnapshot *live, + const ClusterPcmOwnSnapshot *base, + uint64 reservation_token) +{ + /* Self-source S handoff validates its already-resident immutable bytes and + * never enters the network fetch/copy path. */ + return cluster_pcm_x_grant_reservation_kind(live, base, reservation_token) + == CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW; +} + +#endif /* USE_PGRAC_CLUSTER */ diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 69b01b83be4..bcefca80938 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -125,7 +125,9 @@ * 52..55 uint32 poll_cycle_count * 56..59 uint32 torn_write_detect_count * 60..63 uint32 _pad - * 64..127 uint8[64] _reserved (future expansion) + * 64..71 uint64 self_incarnation (canonical boot session) + * 72..75 uint32 prior_unclean_death (crash-rejoin barrier) + * 76..127 uint8[52] _reserved (future expansion) * ============================================================ */ typedef struct ClusterQvotecShmem { pg_atomic_uint32 state; /* ClusterQvotecStatus */ @@ -140,15 +142,7 @@ typedef struct ClusterQvotecShmem { pg_atomic_uint32 poll_cycle_count; pg_atomic_uint32 torn_write_detect_count; pg_atomic_uint32 _pad; - /* - * Merge-order reservation (守门 07-15): the convert-queue lane claims - * offset 64..71 for its self_incarnation (pg_atomic_uint64, commit - * ee536b5bb7, StaticAssert-pinned). Queue merges first; this lane rebases - * after and drops this placeholder so self_incarnation occupies 64..71 and - * prior_unclean_death stays at 72. Keeping the byte layout identical now - * makes that rebase a no-op on the wire/shmem image. - */ - uint8 _reserved_queue_self_incarnation[8]; /* offset 64..71 */ + pg_atomic_uint64 self_incarnation; /* * Crash-rejoin re-declare barrier (Shape A) — set ONCE at qvotec startup * (before the READY publish), read-only thereafter: 1 iff this node's @@ -166,6 +160,8 @@ typedef struct ClusterQvotecShmem { StaticAssertDecl(sizeof(ClusterQvotecShmem) == 128, "ClusterQvotecShmem must be exactly 128 bytes (2 cache lines)"); +StaticAssertDecl(offsetof(ClusterQvotecShmem, self_incarnation) == 64, + "ClusterQvotecShmem self incarnation offset"); StaticAssertDecl(offsetof(ClusterQvotecShmem, prior_unclean_death) == 72, "prior_unclean_death must sit at offset 72 (queue lane owns 64..71)"); @@ -290,8 +286,7 @@ cluster_qvotec_shmem_init(void) pg_atomic_init_u32(&QvotecShmem->poll_cycle_count, 0); pg_atomic_init_u32(&QvotecShmem->torn_write_detect_count, 0); pg_atomic_init_u32(&QvotecShmem->_pad, 0); - memset(QvotecShmem->_reserved_queue_self_incarnation, 0, - sizeof(QvotecShmem->_reserved_queue_self_incarnation)); + pg_atomic_init_u64(&QvotecShmem->self_incarnation, 0); pg_atomic_init_u32(&QvotecShmem->prior_unclean_death, 0); memset(QvotecShmem->_reserved, 0, sizeof(QvotecShmem->_reserved)); } @@ -423,6 +418,21 @@ cluster_qvotec_get_current_epoch_at_boot(void) return pg_atomic_read_u64(&QvotecShmem->current_epoch_at_boot); } +void +cluster_qvotec_publish_self_incarnation(uint64 incarnation) +{ + if (QvotecShmem != NULL) + pg_atomic_write_u64(&QvotecShmem->self_incarnation, incarnation); +} + +uint64 +cluster_qvotec_get_self_incarnation(void) +{ + if (QvotecShmem == NULL) + return 0; + return pg_atomic_read_u64(&QvotecShmem->self_incarnation); +} + const char * cluster_qvotec_get_collision_state_name(void) { @@ -1758,6 +1768,7 @@ ClusterQvotecMain(void) * resets and is freed automatically at proc_exit. */ qvotec_self_incarnation = (uint64)GetCurrentTimestamp(); + cluster_qvotec_publish_self_incarnation(qvotec_self_incarnation); qvotec_slot_generation = 0; qvotec_slot_matrix = (ClusterVotingSlot *)MemoryContextAllocZero( TopMemoryContext, sizeof(ClusterVotingSlot) * CLUSTER_MAX_VOTING_DISKS * CLUSTER_MAX_NODES); @@ -2142,6 +2153,14 @@ cluster_qvotec_get_current_epoch_at_boot(void) { return 0; } +void +cluster_qvotec_publish_self_incarnation(uint64 incarnation pg_attribute_unused()) +{} +uint64 +cluster_qvotec_get_self_incarnation(void) +{ + return 0; +} const char * cluster_qvotec_get_collision_state_name(void) { diff --git a/src/backend/cluster/cluster_sf_dep.c b/src/backend/cluster/cluster_sf_dep.c index 9e4d51b6fd7..f52774731c8 100644 --- a/src/backend/cluster/cluster_sf_dep.c +++ b/src/backend/cluster/cluster_sf_dep.c @@ -355,6 +355,43 @@ cluster_sf_peer_supports_gcs_inval_busy(int32 peer_id) return (capabilities & PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1) != 0; } +/* PCM-X conversion frames are never sent until the current connection's + * verified HELLO advertises the complete byte-exact protocol family. */ +bool +cluster_sf_peer_supports_pcm_x_convert(int32 peer_id) +{ + uint32 capabilities; + + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + capabilities = cluster_sf_peer_cap_bits(&ClusterSfDep->peer_capabilities[peer_id]); + LWLockRelease(&ClusterSfDep->lock); + return (capabilities & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) != 0; +} + +/* Return one lock-coherent snapshot of the capability-record generation whose + * verified HELLO advertised PCM-X. Tier1 owns this record on CONTROL and can + * sample it around another authority read to reject reconnect ABA; RDMA keeps + * its existing registered generation-0 lifecycle. */ +bool +cluster_sf_peer_pcm_x_connection_generation(int32 peer_id, uint32 *generation) +{ + bool supported; + + if (generation != NULL) + *generation = 0; + if (ClusterSfDep == NULL || peer_id < 0 || peer_id >= CLUSTER_MAX_NODES) + return false; + + LWLockAcquire(&ClusterSfDep->lock, LW_SHARED); + supported = cluster_sf_peer_cap_generation_for_bits( + &ClusterSfDep->peer_capabilities[peer_id], PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, generation); + LWLockRelease(&ClusterSfDep->lock); + return supported; +} + /* * cluster_sf_note_peer_disconnected_gen * diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index 902f98140d3..e3f16d27bdf 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -102,6 +102,7 @@ #include "cluster/cluster_gcs_block.h" /* cluster_gcs_block_module_init (spec-2.33 D3) */ #include "cluster/cluster_gcs_block_dedup.h" /* cluster_gcs_block_dedup_module_init (spec-2.34 D2) */ #include "cluster/cluster_pcm_lock.h" /* cluster_pcm_lock_module_init (stage 1.7) */ +#include "cluster/cluster_pcm_x_convert.h" /* PCM-X external shmem substrate */ #include "cluster/cluster_sinval.h" /* cluster_sinval_module_init (spec-2.38 D2/D3) */ #include "cluster/cluster_tt_status.h" /* cluster_tt_status_shmem_register (spec-3.1 D2) */ #include "cluster/cluster_tt_local.h" /* cluster_tt_local_shmem_register (spec-3.1 D5) */ @@ -412,6 +413,10 @@ cluster_init_shmem_module(void) if (cluster_shmem_lookup_region("pgrac cluster pcm grd") == NULL) cluster_pcm_lock_module_init(); + /* spec-2.36a S3-core: one external region for the five PCM-X pools. */ + if (cluster_shmem_lookup_region(PCM_X_SHMEM_REGION_NAME) == NULL) + cluster_pcm_x_convert_shmem_register(); + /* * spec-2.32 D2: register cluster_gcs shmem region (outstanding-request * table per-backend + module counters). Single-source-of-truth for diff --git a/src/backend/cluster/cluster_tx_enqueue.c b/src/backend/cluster/cluster_tx_enqueue.c index 17c8ee70e85..e03cb8e164e 100644 --- a/src/backend/cluster/cluster_tx_enqueue.c +++ b/src/backend/cluster/cluster_tx_enqueue.c @@ -52,6 +52,7 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_lmd.h" #include "cluster/cluster_lmd_wait_state.h" +#include "cluster/cluster_pcm_x_convert.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_tt_status.h" #include "cluster/cluster_tx_enqueue.h" @@ -271,39 +272,45 @@ cluster_tx_enqueue_wait(const ClusterTTStatusKey *holder_key, int effective_time */ PG_TRY(); { - for (;;) { - ClusterTTStatusResult cres; - bool found; - TimestampTz now; - long wait_ms; - - ResetLatch(MyLatch); - - /* Re-check the holder's TT status (closes the register/wake race: - * a terminal status published before we slept is seen here). */ - found = cluster_tt_status_lookup_exact(holder_key, &cres); - if (found && cres.authoritative && txw_status_is_terminal(cres.status)) { - result = CLUSTER_TXW_RESOLVED; - break; + PcmXQueueResult guard_result; + + guard_result = cluster_pcm_x_nested_wait_guard_before_block(); + if (guard_result != PCM_X_QUEUE_OK) + result = CLUSTER_TXW_RETRY; + else + for (;;) { + ClusterTTStatusResult cres; + bool found; + TimestampTz now; + long wait_ms; + + ResetLatch(MyLatch); + + /* Re-check the holder's TT status (closes the register/wake race: + * a terminal status published before we slept is seen here). */ + found = cluster_tt_status_lookup_exact(holder_key, &cres); + if (found && cres.authoritative && txw_status_is_terminal(cres.status)) { + result = CLUSTER_TXW_RESOLVED; + break; + } + + now = GetCurrentTimestamp(); + if (now >= deadline) { + result = CLUSTER_TXW_TIMEOUT; + pg_atomic_fetch_add_u64(&ClusterTxw->timeout_count, 1); + break; + } + + wait_ms = (long)((deadline - now) / 1000); + if (wait_ms <= 0) + wait_ms = 1; + if (wait_ms > CLUSTER_TXW_TICK_MS) + wait_ms = CLUSTER_TXW_TICK_MS; + + (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, wait_ms, + WAIT_EVENT_GES_TX_ENQUEUE_WAIT); + CHECK_FOR_INTERRUPTS(); } - - now = GetCurrentTimestamp(); - if (now >= deadline) { - result = CLUSTER_TXW_TIMEOUT; - pg_atomic_fetch_add_u64(&ClusterTxw->timeout_count, 1); - break; - } - - wait_ms = (long)((deadline - now) / 1000); - if (wait_ms <= 0) - wait_ms = 1; - if (wait_ms > CLUSTER_TXW_TICK_MS) - wait_ms = CLUSTER_TXW_TICK_MS; - - (void)WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, wait_ms, - WAIT_EVENT_GES_TX_ENQUEUE_WAIT); - CHECK_FOR_INTERRUPTS(); - } } PG_CATCH(); { diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index 478c94c9e13..f4838ffbaca 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -56,7 +56,10 @@ #ifdef USE_PGRAC_CLUSTER #include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b — catalog buf hit/miss */ #include "cluster/cluster_mode.h" /* PGRAC (spec-6.14 D8): storage-mode gate */ +#include "cluster/cluster_pcm_direct_init.h" +#include "cluster/cluster_gcs_block.h" #include "cluster/cluster_pcm_lock.h" +#include "cluster/cluster_grd.h" /* existing block-path fail-closed counter */ #include "cluster/cluster_guc.h" /* spec-4.7a D2 — cluster_gcs_block_local_cache */ #include "cluster/cluster_block_recovery.h" /* spec-4.10 D1 — online block recovery */ #include "cluster/cluster_conf.h" /* spec-5.7 HW — cluster_conf_node_count */ @@ -65,230 +68,2313 @@ #include "cluster/cluster_pi_shadow.h" /* spec-6.12h D-h3a — PI ship-SCN stamp */ #include "cluster/cluster_hw_lease.h" /* spec-6.12d — per-node HW space leases */ #include "cluster/cluster_extend_gate.h" /* spec-5.7 §3.1d — liveness engage gate */ +#include "cluster/cluster_epoch.h" #include "cluster/cluster_sf_dep.h" /* spec-6.2 Smart Fusion DBWR brake */ #include "cluster/cluster_xnode_profile.h" /* spec-5.59 D3/D4 — read probe + relkind hint */ #include "cluster/cluster_itl.h" /* spec-6.12a — quiescent check for X->S downgrade */ #include "cluster/cluster_inject.h" /* GCS-race round-4c P1 — yield-notify-drop point */ #include "cluster/cluster_pcm_own.h" /* ownership-generation wave — per-buffer gen + flags */ +#include "cluster/cluster_pcm_x_bufmgr.h" /* spec-2.36a C1 opaque reservation API */ +#include "cluster/cluster_pcm_x_convert.h" + +/* + * PGRAC (spec-4.10 D1): ignore_checksum_failure is defined in bufpage.c with + * no public extern; the online-recovery read hook consults it to honor Q3 + * (online recovery takes precedence over ignore_checksum_failure). Reading + * this bool short-circuits the Q3 re-check on the healthy path (default off), + * so no extra per-read checksum work. + */ +extern bool ignore_checksum_failure; +#endif +#include "storage/bufmgr.h" +#include "storage/ipc.h" +#include "storage/latch.h" +#include "storage/lmgr.h" +#include "storage/proc.h" +#include "storage/smgr.h" +#include "storage/standby.h" +#include "utils/memdebug.h" +#include "utils/memutils.h" +#include "utils/ps_status.h" +#include "utils/rel.h" +#include "utils/resowner_private.h" +#include "utils/timestamp.h" +#include "utils/wait_event.h" + + +/* Note: these two macros only work on shared buffers, not local ones! */ +#define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ)) +#define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) + +/* Note: this macro only works on local buffers, not shared ones! */ +#define LocalBufHdrGetBlock(bufHdr) \ + LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] + +/* Bits in SyncOneBuffer's return value */ +#define BUF_WRITTEN 0x01 +#define BUF_REUSABLE 0x02 + +#define RELS_BSEARCH_THRESHOLD 20 + +#ifdef USE_PGRAC_CLUSTER +/* + * spec-6.14 D4: single PCM-tracking criterion by relfilenumber. Under + * cluster.shared_catalog the catalog lives in the single shared tree, so its + * pages need the same N/S/X + CR + PI lost-write coherency as user pages + * (routing a catalog page into the shared tree without PCM tracking would be a + * double-write hazard -- pairs with the D3 smgr flip, INV-14-1). All shared + * buffers under shared_catalog=on hold permanent (non-temp) relations: temp + * relations use local buffers and never reach these paths, and unlogged + * permanent relations are rejected at DDL time (Q12). Off mode keeps the + * historic FirstNormalObjectId user-only boundary. + * + * The eviction/release hooks (HC112) MUST use this SAME criterion as the + * acquire path, otherwise a tracked catalog page's PCM lock would leak on + * eviction (spec-6.14 R6 -- do not let a gate drift). + */ +static inline bool +cluster_bufmgr_reln_pcm_tracked(RelFileNumber relnum) +{ + return cluster_shared_catalog || relnum >= (RelFileNumber) FirstNormalObjectId; +} + +static inline bool +cluster_bufmgr_should_pcm_track(BufferDesc *buf) +{ + return cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&buf->tag)); +} + +/* Defined with the GCS copy/drop substrate below. The queue ownership + * adapters live earlier so LockBuffer can share them without exposing raw + * buffer-manager pin mechanics outside this file. */ +static XLogRecPtr cluster_gcs_clamp_ship_flush_lsn(XLogRecPtr page_lsn); +static void cluster_bufmgr_pin_for_gcs_locked(BufferDesc *buf, uint32 buf_state); +static void cluster_bufmgr_unpin_for_gcs(BufferDesc *buf); + +static ClusterPcmOwnResult +cluster_pcm_own_bump_failure(BufferDesc *buf, uint64 generation, uint32 *out_flags) +{ + ClusterPcmOwnResult live_result; + uint64 reservation_token; + uint32 flags; + + reservation_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + if (out_flags != NULL) + *out_flags = flags; + live_result = cluster_pcm_own_classify_live_flags(flags, reservation_token); + if (live_result != CLUSTER_PCM_OWN_OK) + return live_result; + if (generation == UINT64_MAX) + return CLUSTER_PCM_OWN_EXHAUSTED; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + return CLUSTER_PCM_OWN_CORRUPT; +} + +static void +cluster_pcm_own_report_bump_failure(BufferDesc *buf, ClusterPcmOwnResult result, + uint64 generation, uint32 flags, const char *context) +{ + if (result == CLUSTER_PCM_OWN_BUSY) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cluster PCM ownership transition conflicts with an active reservation"), + errdetail("context=%s buffer=%d generation=%llu flags=0x%x", context, + buf->buf_id, (unsigned long long) generation, flags))); + if (result == CLUSTER_PCM_OWN_EXHAUSTED) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster PCM ownership generation exhausted for buffer %d", buf->buf_id), + errdetail("context=%s generation=%llu flags=0x%x", context, + (unsigned long long) generation, flags))); + + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM ownership generation bump failed for buffer %d", buf->buf_id), + errdetail("context=%s generation=%llu flags=0x%x result=%d", context, + (unsigned long long) generation, flags, (int) result))); +} + +static inline ClusterPcmOwnResult cluster_pcm_own_bump_locked(BufferDesc *buf, + uint32 set_flags, uint32 clear_flags, + uint64 *out_generation, + uint32 *out_flags); + +/* + * PGRAC ownership-generation wave — coherent + * (pcm_state, generation, reservation_token, flags) mutation / read helpers. + * + * The complete tuple must move together under ONE lock so a reader never sees + * a torn view (TOCTOU). We use the buffer header spinlock as that lock. + * pcm_state lives on the BufferDesc; generation + reservation_token + flags + * live in the parallel ClusterPcmOwnArray indexed by buf_id (cluster_pcm_own.h). + * + * cluster_pcm_own_transition: for callers that do NOT already hold the header + * spinlock (e.g. the LockBuffer grant mirror + the X->S downgrade, which hold + * only the content lock). Sets pcm_state, bumps the generation, and applies + * flag changes atomically under the header spinlock. Every COMMITTED local + * ownership transition routes through here (or the *_locked bump below) so the + * generation increments exactly once per ownership change. + */ +static inline void +cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flags, + uint32 clear_flags) +{ + ClusterPcmOwnResult bump_result; + uint32 buf_state; + uint32 observed_flags = 0; + uint64 new_generation; + + buf_state = LockBufHdr(buf); + bump_result = cluster_pcm_own_bump_locked(buf, set_flags, clear_flags, + &new_generation, &observed_flags); + if (bump_result != CLUSTER_PCM_OWN_OK) { + UnlockBufHdr(buf, buf_state); + cluster_pcm_own_report_bump_failure(buf, bump_result, new_generation, + observed_flags, "ownership transition"); + } + buf->pcm_state = new_pcm_state; + UnlockBufHdr(buf, buf_state); +} + +/* + * cluster_pcm_own_transition_locked: for callers that ALREADY hold the header + * spinlock (the N-flip invalidate/drop/evict sites). Same effect minus the + * lock/unlock. Caller sets pcm_state itself (it already does); this bumps the + * generation and applies flags in the same tuple-critical section. The + * monotonic reservation token is intentionally unchanged. + */ +static inline ClusterPcmOwnResult +cluster_pcm_own_bump_locked(BufferDesc *buf, uint32 set_flags, uint32 clear_flags, + uint64 *out_generation, uint32 *out_flags) +{ + uint64 generation; + + if (!cluster_pcm_own_gen_bump_checked(buf->buf_id, &generation)) { + if (out_generation != NULL) + *out_generation = generation; + return cluster_pcm_own_bump_failure(buf, generation, out_flags); + } + if (set_flags != 0 || clear_flags != 0) + cluster_pcm_own_flags_apply(buf->buf_id, set_flags, clear_flags); + if (out_generation != NULL) + *out_generation = generation; + if (out_flags != NULL) + *out_flags = cluster_pcm_own_flags_get(buf->buf_id); + return CLUSTER_PCM_OWN_OK; +} + +static inline void +cluster_pcm_own_snapshot_locked(BufferDesc *buf, ClusterPcmOwnSnapshot *out) +{ + memset(out, 0, sizeof(*out)); + out->tag = buf->tag; + out->generation = cluster_pcm_own_gen_get(buf->buf_id); + out->reservation_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + out->flags = cluster_pcm_own_flags_get(buf->buf_id); + out->pcm_state = buf->pcm_state; +} + +/* A normal D-h1 PI is !BM_VALID. PCM-X retained-image transfer deliberately + * uses the otherwise-unoccupied PI+BM_VALID shape while N+REVOKING keeps the + * exact descriptor pinned to the image record. Write/reuse gates recognize + * the conservative PI+VALID prefix: malformed sidecar metadata must remain + * blocked rather than making old bytes flushable. Call with header authority + * or while holding the buffer content lock (retain/release take EXCLUSIVE). */ +static inline bool +cluster_bufmgr_pcm_x_retained_image_locked(BufferDesc *buf, uint32 buf_state) +{ + return buf != NULL && buf->buffer_type == (uint8) BUF_TYPE_PI + && (buf_state & BM_VALID) != 0; +} + +/* A resident mapping is GCS-shippable only while its node owns matching PCM + * authority. In particular, PI+BM_VALID N mirrors kept solely to break a + * passive-pin invalidate ring must never be mistaken for a current source. */ +static inline bool +cluster_bufmgr_pcm_current_image_locked(BufferDesc *buf, uint32 buf_state) +{ + return buf != NULL + && cluster_pcm_x_current_image_shape(buf->pcm_state, buf->buffer_type, + (buf_state & BM_VALID) != 0); +} + +static inline bool +cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(BufferDesc *buf, + uint32 buf_state) +{ + uint32 flags; + + if (!cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state)) + return false; + flags = cluster_pcm_own_flags_get(buf->buf_id); + + /* + * A clean N remnant with no live reservation is ordinary eviction + * material. Its monotonic token may legitimately still be zero before + * the first reservation, so flags -- not token value -- are the active + * lifecycle authority. Every live or malformed shape remains blocked. */ + return buf->pcm_state != (uint8) PCM_STATE_N + || (buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0 + || flags != 0; +} + +static inline bool cluster_pcm_own_snapshot_matches_locked( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected); + +static inline void +cluster_pcm_own_eviction_capture_locked(BufferDesc *buf, ClusterPcmOwnEvictionCapture *out) +{ + cluster_pcm_own_snapshot_locked(buf, out); +} + +static ClusterPcmOwnResult +cluster_pcm_own_eviction_commit_locked(BufferDesc *buf, + const ClusterPcmOwnEvictionCapture *capture, + uint64 *out_generation, uint32 *out_flags) +{ + ClusterPcmOwnResult result; + + if (!cluster_pcm_own_snapshot_matches_locked(buf, capture)) + return CLUSTER_PCM_OWN_STALE; + if (!cluster_pcm_own_eviction_reuse_allowed(capture)) { + result = cluster_pcm_own_classify_live_flags(capture->flags, + capture->reservation_token); + return result != CLUSTER_PCM_OWN_OK ? result : CLUSTER_PCM_OWN_EXHAUSTED; + } + + result = cluster_pcm_own_bump_locked(buf, 0, 0, out_generation, out_flags); + if (result == CLUSTER_PCM_OWN_OK) { + buf->pcm_state = (uint8)PCM_STATE_N; + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + } + return result; +} + +static inline bool +cluster_pcm_own_snapshot_matches_locked(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected) +{ + return BufferTagsEqual(&buf->tag, &expected->tag) + && cluster_pcm_own_gen_get(buf->buf_id) == expected->generation + && cluster_pcm_own_reservation_token_get(buf->buf_id) == expected->reservation_token + && cluster_pcm_own_flags_get(buf->buf_id) == expected->flags + && buf->pcm_state == expected->pcm_state; +} + +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_snapshot(BufferDesc *buf, ClusterPcmOwnSnapshot *out_snapshot) +{ + uint32 buf_state; + + if (buf == NULL || out_snapshot == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_snapshot, 0, sizeof(*out_snapshot)); + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + buf_state = LockBufHdr(buf); + cluster_pcm_own_snapshot_locked(buf, out_snapshot); + UnlockBufHdr(buf, buf_state); + return CLUSTER_PCM_OWN_OK; +} + +bool +cluster_bufmgr_pcm_x_content_write_permitted(BufferDesc *buf) +{ + bool permitted; + uint32 buf_state; + uint32 flags; + + if (buf == NULL) + return false; + buf_state = LockBufHdr(buf); + flags = cluster_pcm_own_flags_get(buf->buf_id); + permitted = (flags & PCM_OWN_FLAG_REVOKING) == 0 + && (!cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) + || flags == PCM_OWN_FLAG_GRANT_PENDING); + UnlockBufHdr(buf, buf_state); + return permitted; +} + +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, + ClusterPcmOwnSnapshot *out_snapshot) +{ + BufferDesc *buf; + BufferTag lookup_tag; + LWLock *partition_lock; + uint32 hashcode; + uint32 buf_state; + int buf_id; + ClusterPcmOwnResult result; + + if (out_buffer_id != NULL) + *out_buffer_id = -1; + if (out_snapshot != NULL) + memset(out_snapshot, 0, sizeof(*out_snapshot)); + if (tag == NULL || out_buffer_id == NULL || out_snapshot == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + lookup_tag = *tag; + hashcode = BufTableHashCode(&lookup_tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_SHARED); + buf_id = BufTableLookup(&lookup_tag, hashcode); + if (buf_id < 0) + { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } + buf = GetBufferDescriptor(buf_id); + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else + { + cluster_pcm_own_snapshot_locked(buf, out_snapshot); + *out_buffer_id = buf_id; + result = CLUSTER_PCM_OWN_OK; + } + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + return result; +} + +/* + * Complete the descriptor side of an exact S->N queue normalization. + * + * The caller owns the wire/application proof: the authoritative master has + * already ACKed this node's exact S release. This boundary owns only the + * local coherent tuple. It therefore accepts one byte-exact S snapshot, + * changes state and generation together under the buffer header spinlock, + * and returns the fresh N snapshot that ACTIVE_TRANSFER/PREPARE must later + * present. Queue JOIN/WAIT never sets a reservation flag here. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_s_release_to_n( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_n_snapshot) +{ + ClusterPcmOwnResult result; + uint32 buf_state; + + if (buf == NULL || expected_s == NULL || out_n_snapshot == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_n_snapshot, 0, sizeof(*out_n_snapshot)); + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + if (expected_s->pcm_state != (uint8) PCM_STATE_S || expected_s->flags != 0) + return CLUSTER_PCM_OWN_STALE; + + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_s)) + result = CLUSTER_PCM_OWN_STALE; + else + { + result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); + if (result == CLUSTER_PCM_OWN_OK) + { + buf->pcm_state = (uint8) PCM_STATE_N; + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + cluster_pcm_own_snapshot_locked(buf, out_n_snapshot); + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +/* + * Release a passively pinned MAIN/INIT S mirror for a queue INVALIDATE. + * + * A PG pin protects descriptor identity but is not content authority. The + * ordinary invalidate path nevertheless cannot unmap a pinned descriptor, + * which makes every hot-block writer wait on its own pin. Bind tag->buffer + * under the mapping lock, add one raw pin, then serialize all byte users with + * content EXCLUSIVE. Two exact tuple checks around WAL flush close retag, + * generation, reservation, and page-LSN ABA windows. Success keeps the clean + * bytes BM_VALID as an N mirror; every later LockBuffer must reacquire PCM + * before consuming them. VM/FSM never enter this shape because their + * pin-only readers may consume bytes without that content-lock gate. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, + XLogRecPtr *out_page_lsn, + uint64 *out_page_scn) +{ + BufferTag lookup_tag; + ClusterPcmOwnSnapshot expected_s; + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + BufferDesc *buf; + LWLock *content_lock; + LWLock *partition_lock; + Page page; + XLogRecPtr page_lsn = InvalidXLogRecPtr; + uint64 page_scn = 0; + uint32 buf_state; + uint32 hashcode; + uint32 shared_refcount; + uint32 flags; + uint64 token; + int buf_id; + bool dirty = false; + volatile bool content_locked = false; + + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_page_scn != NULL) + *out_page_scn = 0; + if (tag == NULL || out_page_lsn == NULL || out_page_scn == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + lookup_tag = *tag; + hashcode = BufTableHashCode(&lookup_tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_SHARED); + buf_id = BufTableLookup(&lookup_tag, hashcode); + if (buf_id < 0) + { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } + buf = GetBufferDescriptor(buf_id); + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, tag) || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else + { + shared_refcount = BUF_STATE_GET_REFCOUNT(buf_state); + token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (buf->pcm_state != (uint8) PCM_STATE_S) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if (shared_refcount == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (cluster_pcm_x_revoke_finish_mode(tag, shared_refcount) + != CLUSTER_PCM_X_REVOKE_FINISH_RETAIN) + result = CLUSTER_PCM_OWN_BUSY; + else + { + cluster_pcm_own_snapshot_locked(buf, &expected_s); + cluster_bufmgr_pin_for_gcs_locked(buf, buf_state); + buf_state = 0; + } + } + if (buf_state != 0) + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + if (result != CLUSTER_PCM_OWN_OK) + return result; + + content_lock = BufferDescriptorGetContentLock(buf); + PG_TRY(); + { + LWLockAcquire(content_lock, LW_EXCLUSIVE); + content_locked = true; + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) + || (buf_state & BM_VALID) == 0 + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + || (buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_STALE; + else + { + page = (Page) BufHdrGetBlock(buf); + page_lsn = PageGetLSN(page); + page_scn = (uint64) ((PageHeader) page)->pd_block_scn; + dirty = (buf_state & BM_DIRTY) != 0; + } + UnlockBufHdr(buf, buf_state); + + if (result == CLUSTER_PCM_OWN_OK && dirty && !XLogRecPtrIsInvalid(page_lsn)) + XLogFlush(cluster_gcs_clamp_ship_flush_lsn(page_lsn)); + + if (result == CLUSTER_PCM_OWN_OK) + { + buf_state = LockBufHdr(buf); + page = (Page) BufHdrGetBlock(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, &expected_s) + || (buf_state & BM_VALID) == 0 + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state) + || (buf_state & BM_IO_IN_PROGRESS) != 0 + || PageGetLSN(page) != page_lsn) + result = CLUSTER_PCM_OWN_STALE; + else + { + result = cluster_pcm_own_bump_locked(buf, 0, 0, NULL, NULL); + if (result == CLUSTER_PCM_OWN_OK) + { + buf->pcm_state = (uint8) PCM_STATE_N; + /* PI+BM_VALID is the established never-write/never-serve shape. + * Unlike a source retain it has no live reservation flag (the + * monotonic token remains idle): the next exact GRANT_PENDING + * install may overwrite it, then republishes CURRENT. */ + buf->buffer_type = (uint8) BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED + | BM_IO_ERROR); + } + } + UnlockBufHdr(buf, buf_state); + } + + LWLockRelease(content_lock); + content_locked = false; + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + PG_RE_THROW(); + } + PG_END_TRY(); + + cluster_bufmgr_unpin_for_gcs(buf); + if (result == CLUSTER_PCM_OWN_OK) + { + *out_page_lsn = page_lsn; + *out_page_scn = page_scn; + } + return result; +} + +/* Make freshly installed bytes visible only while the same queue + * reservation still owns both the descriptor and content EXCLUSIVE. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_publish_installed_x_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, uint64 reservation_token) +{ + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + uint32 buf_state; + + if (buf == NULL || expected == NULL || reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (!LWLockHeldByMe(BufferDescriptorGetContentLock(buf))) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + if (expected->pcm_state != (uint8) PCM_STATE_N || expected->flags != 0 + || expected->reservation_token == UINT64_MAX + || reservation_token != expected->reservation_token + 1) + return CLUSTER_PCM_OWN_STALE; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected->tag) + || cluster_pcm_own_gen_get(buf->buf_id) != expected->generation + || cluster_pcm_own_reservation_token_get(buf->buf_id) != reservation_token + || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_GRANT_PENDING + || buf->pcm_state != (uint8) PCM_STATE_N) + result = CLUSTER_PCM_OWN_STALE; + else + { + buf_state |= BM_VALID; + buf->buffer_type = (uint8) BUF_TYPE_CURRENT; + } + UnlockBufHdr(buf, buf_state); + return result; +} + +static ClusterPcmOwnResult +cluster_pcm_own_begin_grant_reservation(BufferDesc *buf, ClusterPcmOwnSnapshot *out_base, + uint64 *out_token) +{ + ClusterPcmOwnResult result; + uint32 buf_state; + + if (buf == NULL || out_base == NULL || out_token == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_base, 0, sizeof(*out_base)); + *out_token = 0; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + cluster_pcm_own_snapshot_locked(buf, out_base); + result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, out_base->generation, + PCM_OWN_FLAG_GRANT_PENDING, out_token); + UnlockBufHdr(buf, buf_state); + return result; +} + +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_begin_x_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, + uint64 *out_token) +{ + ClusterPcmOwnResult result; + uint32 buf_state; + + if (buf == NULL || expected == NULL || out_token == NULL) + return CLUSTER_PCM_OWN_INVALID; + *out_token = 0; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + if (expected->pcm_state != (uint8)PCM_STATE_N || expected->flags != 0) + return CLUSTER_PCM_OWN_STALE; + + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected)) + result = CLUSTER_PCM_OWN_STALE; + else + result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected->generation, + PCM_OWN_FLAG_GRANT_PENDING, out_token); + UnlockBufHdr(buf, buf_state); + return result; +} + +/* Reuse an exact requester-as-source N/S/X revoke lifecycle for its X grant. + * The caller holds content EXCLUSIVE across immutable-page validation and + * this linearization; no second token or generation is allocated here. An + * exact duplicate PREPARE observes GRANT_PENDING and is idempotent. After + * the handoff ACTIVE_TRANSFER is irreversible: cleanup must preserve + * GRANT_PENDING evidence and fail closed, never toggle it back. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, uint64 *out_token) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + LWLock *content_lock; + uint64 live_token; + uint32 flags; + uint32 buf_state; + bool source_is_n; + bool source_is_s; + bool source_is_x; + + if (out_token != NULL) + *out_token = 0; + if (buf == NULL || expected_revoking == NULL || out_token == NULL) + return CLUSTER_PCM_OWN_INVALID; + source_is_n = expected_revoking->pcm_state == (uint8)PCM_STATE_N; + source_is_s = expected_revoking->pcm_state == (uint8)PCM_STATE_S; + source_is_x = expected_revoking->pcm_state == (uint8)PCM_STATE_X; + if ((!source_is_n && !source_is_s && !source_is_x) + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_STALE; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + content_lock = BufferDescriptorGetContentLock(buf); + if (!LWLockHeldByMe(content_lock)) + return CLUSTER_PCM_OWN_INVALID; + + buf_state = LockBufHdr(buf); + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != expected_revoking->pcm_state + || live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if ((buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if (source_is_n + && ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) + != 0 + || buf->buffer_type != (uint8)BUF_TYPE_CURRENT)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if (!source_is_n && !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if (flags == PCM_OWN_FLAG_GRANT_PENDING) + result = CLUSTER_PCM_OWN_OK; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else + result = cluster_pcm_own_revoke_to_grant_handoff_exact( + buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token); + UnlockBufHdr(buf, buf_state); + if (result == CLUSTER_PCM_OWN_OK) + *out_token = expected_revoking->reservation_token; + return result; +} + +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, uint64 *out_token) +{ + if (expected_revoking == NULL || expected_revoking->pcm_state != (uint8)PCM_STATE_S) + return CLUSTER_PCM_OWN_STALE; + return cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(buf, expected_revoking, + out_token); +} + +static ClusterPcmOwnResult +cluster_pcm_own_finish_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, + uint64 reservation_token, uint8 new_pcm_state, + uint64 *out_committed_generation) +{ + ClusterPcmOwnSnapshot live; + ClusterPcmXGrantReservationKind kind; + ClusterPcmOwnResult result; + uint32 buf_state; + + if (buf == NULL || base == NULL || out_committed_generation == NULL) + return CLUSTER_PCM_OWN_INVALID; + *out_committed_generation = 0; + if (new_pcm_state != (uint8)PCM_STATE_S && new_pcm_state != (uint8)PCM_STATE_X) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + cluster_pcm_own_snapshot_locked(buf, &live); + kind = cluster_pcm_x_grant_reservation_kind(&live, base, reservation_token); + if (kind == CLUSTER_PCM_X_GRANT_RESERVATION_INVALID) { + ClusterPcmOwnResult live_shape + = cluster_pcm_own_classify_live_flags(live.flags, live.reservation_token); + + result = live_shape == CLUSTER_PCM_OWN_CORRUPT ? live_shape : CLUSTER_PCM_OWN_STALE; + } else if (kind != CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW + && (new_pcm_state != (uint8)PCM_STATE_X + || !LWLockHeldByMe(BufferDescriptorGetContentLock(buf)))) + result = CLUSTER_PCM_OWN_INVALID; + else { + result = cluster_pcm_own_grant_commit_exact(buf->buf_id, base->generation, + reservation_token, out_committed_generation); + if (result == CLUSTER_PCM_OWN_OK) { + buf->buffer_type + = new_pcm_state == (uint8)PCM_STATE_S ? (uint8)BUF_TYPE_SCUR : (uint8)BUF_TYPE_XCUR; + buf->pcm_state = new_pcm_state; + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_x_commit(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, + uint64 reservation_token, uint64 *out_committed_generation) +{ + if (expected == NULL + || (expected->pcm_state != (uint8)PCM_STATE_N && expected->pcm_state != (uint8)PCM_STATE_S + && expected->pcm_state != (uint8)PCM_STATE_X)) + return CLUSTER_PCM_OWN_STALE; + return cluster_pcm_own_finish_grant_reservation(buf, expected, reservation_token, + (uint8)PCM_STATE_X, out_committed_generation); +} + +static ClusterPcmOwnResult +cluster_pcm_own_abort_grant_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, + uint64 reservation_token) +{ + ClusterPcmOwnResult result; + uint32 buf_state; + + if (buf == NULL || base == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (base->flags != 0 || base->reservation_token == UINT64_MAX + || reservation_token != base->reservation_token + 1) + return CLUSTER_PCM_OWN_STALE; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &base->tag) + || cluster_pcm_own_gen_get(buf->buf_id) != base->generation + || buf->pcm_state != base->pcm_state) + result = CLUSTER_PCM_OWN_STALE; + else + result = cluster_pcm_own_reservation_abort_exact( + buf->buf_id, base->generation, reservation_token, PCM_OWN_FLAG_GRANT_PENDING); + UnlockBufHdr(buf, buf_state); + return result; +} + +static void +cluster_pcm_own_abort_grant_or_error(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, + uint64 reservation_token, const char *context) +{ + ClusterPcmOwnResult result; + + result = cluster_pcm_own_abort_grant_reservation(buf, base, reservation_token); + if (result != CLUSTER_PCM_OWN_OK) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not abort exact cluster PCM grant reservation: result=%d", + (int)result), + errdetail("context=%s buffer=%d generation=%llu token=%llu", context, + buf != NULL ? buf->buf_id : -1, + (unsigned long long)(base != NULL ? base->generation : 0), + (unsigned long long)reservation_token))); +} + +static void +cluster_pcm_own_abort_grant_after_error(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, + uint64 reservation_token, const char *context) +{ + ClusterPcmOwnResult result; + + result = cluster_pcm_own_abort_grant_reservation(buf, base, reservation_token); + if (result != CLUSTER_PCM_OWN_OK) + elog(LOG, + "failed to abort exact cluster PCM grant reservation while handling " + "another error: context=%s buffer=%d generation=%llu token=%llu result=%d; " + "reservation remains fail-closed", + context, buf != NULL ? buf->buf_id : -1, + (unsigned long long)(base != NULL ? base->generation : 0), + (unsigned long long)reservation_token, (int)result); +} + +static ClusterPcmOwnResult +cluster_pcm_own_abort_grant_after_master_rollback(BufferDesc *buf, + const ClusterPcmOwnSnapshot *base, + uint64 reservation_token) +{ + ClusterPcmOwnResult result; + uint32 buf_state; + uint64 new_generation; + + if (buf == NULL || base == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (base->flags != 0 || base->reservation_token == UINT64_MAX + || reservation_token != base->reservation_token + 1) + return CLUSTER_PCM_OWN_STALE; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (base->pcm_state != (uint8)PCM_STATE_N && base->pcm_state != (uint8)PCM_STATE_S) + result = CLUSTER_PCM_OWN_INVALID; + else if (!BufferTagsEqual(&buf->tag, &base->tag) + || cluster_pcm_own_gen_get(buf->buf_id) != base->generation + || cluster_pcm_own_reservation_token_get(buf->buf_id) != reservation_token + || cluster_pcm_own_flags_get(buf->buf_id) != PCM_OWN_FLAG_GRANT_PENDING + || buf->pcm_state != base->pcm_state) + result = CLUSTER_PCM_OWN_STALE; + else if (base->pcm_state == (uint8)PCM_STATE_S && base->generation == UINT64_MAX) + result = CLUSTER_PCM_OWN_EXHAUSTED; + else { + result = cluster_pcm_own_reservation_abort_exact( + buf->buf_id, base->generation, reservation_token, PCM_OWN_FLAG_GRANT_PENDING); + if (result == CLUSTER_PCM_OWN_OK && base->pcm_state == (uint8)PCM_STATE_S) { + if (!cluster_pcm_own_gen_bump_checked(buf->buf_id, &new_generation)) { + /* + * The MAX check above and this header-lock hold make failure + * unreachable for a coherent entry. Restore the same token's + * live marker if corruption/not-ready is nevertheless + * observed, so the caller fails closed instead of advertising + * completed cleanup. + */ + cluster_pcm_own_flags_apply(buf->buf_id, PCM_OWN_FLAG_GRANT_PENDING, 0); + result = CLUSTER_PCM_OWN_CORRUPT; + } else + buf->pcm_state = (uint8)PCM_STATE_N; + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +pg_attribute_noreturn() static void +cluster_pcm_own_rollback_grant_after_error_and_rethrow( + BufferDesc *buf, const ClusterPcmOwnSnapshot *base, uint64 reservation_token, + PcmLockMode acquired_mode, bool has_reservation, ErrorData *original_error, + MemoryContext caller_context) +{ + ClusterPcmOwnResult rollback_result; + ErrorData *cleanup_error; + volatile bool master_released = false; + + /* + * LockBuffer saved and flushed the original error before either holder or + * master cleanup. Work in that caller-owned context so a throwing remote + * release can be copied, logged, and discarded without replacing it. + */ + Assert(original_error != NULL); + MemoryContextSwitchTo(caller_context); + + PG_TRY(); + { + cluster_pcm_lock_release_buffer_for_eviction(buf, acquired_mode); + master_released = true; + } + PG_CATCH(); + { + MemoryContextSwitchTo(caller_context); + cleanup_error = CopyErrorData(); + FlushErrorState(); + elog(LOG, + "failed to roll back cluster PCM master grant while preserving an earlier " + "content-lock error: buffer=%d token=%llu cleanup_error=%s; reservation " + "remains fail-closed", + buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token, + cleanup_error->message != NULL ? cleanup_error->message : "unknown"); + FreeErrorData(cleanup_error); + } + PG_END_TRY(); + + if (master_released) { + if (has_reservation) { + rollback_result = cluster_pcm_own_abort_grant_after_master_rollback( + buf, base, reservation_token); + if (rollback_result != CLUSTER_PCM_OWN_OK) + elog(LOG, + "failed to converge local cluster PCM state after content-lock error " + "master rollback: buffer=%d token=%llu result=%d; reservation remains " + "fail-closed", + buf != NULL ? buf->buf_id : -1, + (unsigned long long)reservation_token, (int)rollback_result); + } else + elog(LOG, + "cluster PCM content-lock error rolled back a master grant without local " + "reservation evidence for buffer %d", + buf != NULL ? buf->buf_id : -1); + } + + ReThrowError(original_error); +} + +static void +cluster_pcm_own_finish_grant_or_rollback(BufferDesc *buf, const ClusterPcmOwnSnapshot *base, + uint64 reservation_token, uint8 new_pcm_state, + PcmLockMode acquired_mode, + uint64 *out_committed_generation) +{ + ClusterPcmOwnResult finish_result; + ClusterPcmOwnResult rollback_result; + + finish_result = cluster_pcm_own_finish_grant_reservation( + buf, base, reservation_token, new_pcm_state, out_committed_generation); + if (finish_result == CLUSTER_PCM_OWN_OK) + return; + + /* + * The master grant is already durable. Do not clear local reservation + * evidence until its X/S holder has been released successfully. + */ + PG_TRY(); + { + cluster_pcm_lock_release_buffer_for_eviction(buf, acquired_mode); + } + PG_CATCH(); + { + elog(LOG, + "cluster PCM exact finish failed (result=%d) and master grant rollback " + "also failed for buffer %d token=%llu; reservation remains fail-closed", + (int)finish_result, buf != NULL ? buf->buf_id : -1, + (unsigned long long)reservation_token); + PG_RE_THROW(); + } + PG_END_TRY(); + + rollback_result + = cluster_pcm_own_abort_grant_after_master_rollback(buf, base, reservation_token); + if (rollback_result != CLUSTER_PCM_OWN_OK) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not converge local cluster PCM state after master grant rollback: " + "result=%d", + (int)rollback_result), + errdetail("buffer=%d base_state=%u generation=%llu token=%llu finish_result=%d", + buf != NULL ? buf->buf_id : -1, + base != NULL ? (unsigned int)base->pcm_state : 0, + (unsigned long long)(base != NULL ? base->generation : 0), + (unsigned long long)reservation_token, (int)finish_result))); + + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("could not finish exact cluster PCM grant reservation: result=%d", + (int)finish_result), + errdetail("master grant was rolled back for buffer %d token=%llu", + buf != NULL ? buf->buf_id : -1, (unsigned long long)reservation_token))); +} + +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_x_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, + uint64 reservation_token) +{ + if (expected == NULL || expected->pcm_state != (uint8)PCM_STATE_N) + return CLUSTER_PCM_OWN_STALE; + return cluster_pcm_own_abort_grant_reservation(buf, expected, reservation_token); +} + +/* + * W2 RED substrate — process-local marker: true while a GCS drop's + * InvalidateBufferTry attempt is in flight, so the drop-prepin inject inside + * InvalidateBufferTry fires ONLY for GCS drops (never for plain evictions). + * The drop runs to completion in one call chain in the pump process, so a + * plain static is race-free. + */ +static bool cluster_bufmgr_in_gcs_drop = false; + +/* + * cluster_pcm_own_read: read the (pcm_state, generation, flags) projection of + * the coherent ownership tuple under the header spinlock. These consumers do + * not act on a reservation identity, so they intentionally omit its token. + * Used by the cached-X writer re-verify and the drop-restore ABA check. Any + * *out may be NULL. + */ +static inline void +cluster_pcm_own_read(BufferDesc *buf, uint8 *out_state, uint64 *out_gen, uint32 *out_flags) +{ + uint32 buf_state; + + buf_state = LockBufHdr(buf); + if (out_state != NULL) + *out_state = buf->pcm_state; + if (out_gen != NULL) + *out_gen = cluster_pcm_own_gen_get(buf->buf_id); + if (out_flags != NULL) + *out_flags = cluster_pcm_own_flags_get(buf->buf_id); + UnlockBufHdr(buf, buf_state); +} + +/* One backend-local entry per held shared-buffer content LWLock. This is + * bounded by PG's own held-LWLock ceiling and stores the complete exact + * handle returned by PCM-X; error cleanup never reconstructs identity from a + * possibly changed BufferDesc ownership mirror. */ +typedef enum ClusterPcmXHolderLedgerPhase +{ + PCM_X_HOLDER_LEDGER_UNUSED = 0, + PCM_X_HOLDER_LEDGER_ACQUIRING, + PCM_X_HOLDER_LEDGER_ACTIVE, + PCM_X_HOLDER_LEDGER_RELEASING, + PCM_X_HOLDER_LEDGER_DEFERRED +} ClusterPcmXHolderLedgerPhase; + +typedef struct ClusterPcmXHolderLedgerEntry +{ + ClusterPcmXHolderLedgerPhase phase; + int32 buffer_id; + LWLock *content_lock; + PcmXLocalHolderHandle handle; +} ClusterPcmXHolderLedgerEntry; + +static ClusterPcmXHolderLedgerEntry + cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]; +static uint64 cluster_bufmgr_pcm_x_holder_identity = 0; + +StaticAssertDecl(lengthof(cluster_bufmgr_pcm_x_holder_ledger) == LWLOCK_MAX_HELD_BY_PROC, + "PCM-X holder ledger must match the process held-LWLock bound"); + +/* Writer execution authority is deliberately not folded into the holder + * ledger. A holder describes one content-lock occupant for remote revoke; + * a writer claim is the FIFO right that must survive from queue grant through + * content unlock even when holder registration is retried independently. */ +typedef enum ClusterPcmXWriterLedgerPhase { + PCM_X_WRITER_LEDGER_UNUSED = 0, + PCM_X_WRITER_LEDGER_HANDOFF, + PCM_X_WRITER_LEDGER_ACQUIRING, + PCM_X_WRITER_LEDGER_ACTIVE, + PCM_X_WRITER_LEDGER_RELEASING, + PCM_X_WRITER_LEDGER_DEFERRED +} ClusterPcmXWriterLedgerPhase; + +typedef struct ClusterPcmXWriterLedgerEntry { + ClusterPcmXWriterLedgerPhase phase; + int32 buffer_id; + LWLock *content_lock; + PcmXLocalWriterClaim claim; + ClusterPcmOwnSnapshot granted; + bool claim_handed_off; +} ClusterPcmXWriterLedgerEntry; + +static ClusterPcmXWriterLedgerEntry cluster_bufmgr_pcm_x_writer_ledger[LWLOCK_MAX_HELD_BY_PROC]; + +StaticAssertDecl(lengthof(cluster_bufmgr_pcm_x_writer_ledger) == LWLOCK_MAX_HELD_BY_PROC, + "PCM-X writer ledger must match the process held-LWLock bound"); + +static ClusterPcmXWriterLedgerEntry *cluster_bufmgr_pcm_x_writer_find(BufferDesc *buf); +static bool cluster_bufmgr_pcm_x_writer_claim_entry_exact(const ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf); +static bool cluster_bufmgr_pcm_x_writer_entry_exact(const ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf); +static void cluster_bufmgr_pcm_x_writer_report_failure(PcmXQueueResult result, BufferDesc *buf, + const char *operation); +static void cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state, + bool page_is_new, + ClusterPcmDirectInitSnapshot *out); + +static void +cluster_bufmgr_pcm_x_holder_retry_wait(LWLock *content_lock, int32 buffer_id, + uint32 wait_index) +{ + long delay_ms; + + /* + * This is a production lock-order guard. Waiting while the page content + * lock is held would make RETIRE depend on the holder it is draining. + */ + if (content_lock == NULL || LWLockHeldByMe(content_lock)) + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cannot wait for a cluster PCM-X holder gate while holding content authority"), + errdetail("buffer=%d wait_index=%u", buffer_id, wait_index))); + delay_ms = cluster_pcm_x_holder_retry_delay_ms(wait_index); + CHECK_FOR_INTERRUPTS(); + (void) WaitLatch(MyLatch, WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, delay_ms, + WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT); + CHECK_FOR_INTERRUPTS(); +} + +static ClusterPcmXHolderLedgerEntry * +cluster_bufmgr_pcm_x_holder_find(BufferDesc *buf) +{ + int i; + + if (buf == NULL) + return NULL; + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { + ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; + + if (entry->phase != PCM_X_HOLDER_LEDGER_UNUSED && entry->buffer_id == buf->buf_id) + return entry; + } + return NULL; +} + +static ClusterPcmXHolderLedgerEntry * +cluster_bufmgr_pcm_x_holder_free_entry(void) +{ + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + if (cluster_bufmgr_pcm_x_holder_ledger[i].phase == PCM_X_HOLDER_LEDGER_UNUSED) + return &cluster_bufmgr_pcm_x_holder_ledger[i]; + return NULL; +} + +static bool +cluster_bufmgr_pcm_x_holder_entry_exact(const ClusterPcmXHolderLedgerEntry *entry, + BufferDesc *buf) +{ + PcmXRuntimeSnapshot runtime; + uint64 cluster_epoch; + + if (entry == NULL || buf == NULL) + return false; + if (entry->buffer_id != buf->buf_id + || entry->content_lock != BufferDescriptorGetContentLock(buf) + || entry->handle.key.buffer_id != buf->buf_id + || !BufferTagsEqual(&entry->handle.key.identity.tag, &buf->tag)) + return false; + if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || entry->handle.key.identity.node_id != cluster_node_id || MyProc == NULL + || MyProc->pgprocno < 0 + || entry->handle.key.identity.procno != (uint32) MyProc->pgprocno + || entry->handle.key.identity.request_id == 0 + || entry->handle.key.identity.wait_seq != entry->handle.key.identity.request_id + || entry->handle.tag_slot.slot_index == PCM_X_INVALID_SLOT_INDEX + || entry->handle.holder_slot.slot_index == PCM_X_INVALID_SLOT_INDEX) + return false; + cluster_epoch = cluster_epoch_get_current(); + runtime = cluster_pcm_x_runtime_snapshot(); + if (entry->handle.key.identity.cluster_epoch != cluster_epoch + || runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) + return false; + return true; +} + +static void +cluster_bufmgr_pcm_x_holder_report_failure(PcmXQueueResult result, BufferDesc *buf, + const char *operation) +{ + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED + || result == PCM_X_QUEUE_NO_CAPACITY) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cluster PCM-X holder operation is not ready"), + errdetail("operation=%s buffer=%d result=%d", operation, + buf != NULL ? buf->buf_id : -1, (int) result))); + + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM-X holder operation failed"), + errdetail("operation=%s buffer=%d result=%d", operation, + buf != NULL ? buf->buf_id : -1, (int) result))); +} + +static void +cluster_bufmgr_pcm_x_holder_clear(ClusterPcmXHolderLedgerEntry *entry) +{ + if (entry != NULL) + MemSet(entry, 0, sizeof(*entry)); +} + +static void +cluster_bufmgr_pcm_x_holder_defer_fail_closed(ClusterPcmXHolderLedgerEntry *entry) +{ + if (entry != NULL) + entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); +} + +/* Error and transaction-tail cleanup never enters explicit WaitLatch/backoff, + * CHECK_FOR_INTERRUPTS, or ereport. Exact detach may acquire its internal + * LWLocks once; that API converts a lock-acquire ERROR to CORRUPT. A retryable + * gate leaves the complete backend-local handle in DEFERRED for a later safe + * LockBuffer entrance. */ +static void +cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(void) +{ + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { + ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; + PcmXQueueResult result; + + if (entry->phase != PCM_X_HOLDER_LEDGER_DEFERRED) + continue; + if (entry->content_lock == NULL) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "could not drain deferred cluster PCM-X holder with no content lock: buffer=%d", + entry->buffer_id); + continue; + } + if (LWLockHeldByMe(entry->content_lock)) + { + elog(LOG, "could not drain deferred cluster PCM-X holder while content lock is held: buffer=%d", + entry->buffer_id); + continue; + } + result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, + entry->content_lock); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) + cluster_bufmgr_pcm_x_holder_clear(entry); + else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "could not drain deferred exact cluster PCM-X holder: buffer=%d result=%d", + entry->buffer_id, (int) result); + } + } +} + +/* Same-buffer reuse cannot proceed around old RELEASING evidence. Each wait + * batch is bounded (2+4+8+16+32ms) but healthy registration may start the + * next batch; CHECK_FOR_INTERRUPTS remains the user-visible cancellation + * boundary. */ +static void +cluster_bufmgr_pcm_x_holder_drain_deferred(ClusterPcmXHolderLedgerEntry *entry) +{ + PcmXRuntimeSnapshot runtime; + uint32 wait_index = 0; + + if (entry == NULL || entry->phase != PCM_X_HOLDER_LEDGER_DEFERRED) + return; + for (;;) + { + PcmXQueueResult result; + ClusterPcmXHolderRetryAction action; + + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "deferred detach lock order"); + } + result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, + entry->content_lock); + action = cluster_pcm_x_holder_unregister_retry_action(result, 0); + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) + { + cluster_bufmgr_pcm_x_holder_clear(entry); + return; + } + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + cluster_bufmgr_pcm_x_holder_report_failure(result, + GetBufferDescriptor(entry->buffer_id), + "deferred detach"); + } + cluster_bufmgr_pcm_x_holder_retry_wait( + entry->content_lock, entry->buffer_id, + wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + wait_index++; + if (wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0) + { + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE + || runtime.master_session_incarnation == 0) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + cluster_bufmgr_pcm_x_holder_report_failure( + PCM_X_QUEUE_NOT_READY, GetBufferDescriptor(entry->buffer_id), + "deferred detach runtime"); + } + } + } +} + +static ClusterPcmXHolderLedgerEntry * +cluster_bufmgr_pcm_x_holder_prepare(BufferDesc *buf) +{ + ClusterPcmXHolderLedgerEntry *entry; + ClusterPcmXWriterLedgerEntry *writer_entry; + PcmXRuntimeSnapshot runtime; + PcmXLocalHolderKey key; + PcmXLocalHolderHandle handle; + PcmXQueueResult result; + TransactionId xid; + uint64 cluster_epoch; + uint64 committed_own_generation = 0; + uint64 identity; + uint64 own_generation; + LWLock *content_lock; + uint32 wait_index = 0; + bool writer_authorized = false; + + cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(); + entry = cluster_bufmgr_pcm_x_holder_find(buf); + if (entry != NULL && entry->phase == PCM_X_HOLDER_LEDGER_DEFERRED) + { + cluster_bufmgr_pcm_x_holder_drain_deferred(entry); + entry = NULL; + } + if (entry != NULL) + { + if (entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING + || !cluster_bufmgr_pcm_x_holder_entry_exact(entry, buf)) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, buf, + "reuse existing holder ledger"); + } + if (LWLockHeldByMe(entry->content_lock)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cannot register a cluster PCM-X holder for a pre-held content lock"), + errdetail("buffer=%d", buf->buf_id))); + return entry; + } + if (!cluster_bufmgr_should_pcm_track(buf)) + return NULL; + writer_entry = cluster_bufmgr_pcm_x_writer_find(buf); + if (writer_entry != NULL) { + if (writer_entry->phase != PCM_X_WRITER_LEDGER_ACQUIRING || !writer_entry->claim_handed_off + || !cluster_bufmgr_pcm_x_writer_entry_exact(writer_entry, buf)) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, + "holder writer authority"); + writer_authorized = true; + } + + runtime = cluster_pcm_x_runtime_snapshot(); + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { + /* + * A queued writer already owns exact FIFO execution authority. It + * may never cross this runtime transition without the matching holder + * lane; ERROR unwinds through writer_abort_acquiring and completes + * the turn. + */ + if (writer_authorized) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_NOT_READY, buf, + "holder runtime"); + return NULL; + } + content_lock = BufferDescriptorGetContentLock(buf); + if (LWLockHeldByMe(content_lock)) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cannot register a cluster PCM-X holder for a pre-held content lock"), + errdetail("buffer=%d", buf->buf_id))); + entry = cluster_bufmgr_pcm_x_holder_free_entry(); + if (entry == NULL) + { + int i; + + /* + * Deferred exact handles do not consume the ledger forever. If every + * slot is occupied, wait for one safe detach before declaring a real + * held-LWLock bound violation. */ + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { + ClusterPcmXHolderLedgerEntry *candidate + = &cluster_bufmgr_pcm_x_holder_ledger[i]; + + if (candidate->phase != PCM_X_HOLDER_LEDGER_DEFERRED) + continue; + cluster_bufmgr_pcm_x_holder_drain_deferred(candidate); + entry = candidate; + break; + } + } + if (entry == NULL) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster PCM-X holder ledger is full"), + errdetail("maximum entries=%d", LWLOCK_MAX_HELD_BY_PROC))); + + cluster_pcm_own_read(buf, NULL, &own_generation, NULL); + cluster_epoch = cluster_epoch_get_current(); + if (cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT || MyProc == NULL + || MyProc->pgprocno < 0) + ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), + errmsg("cluster PCM-X holder identity is unavailable"), + errdetail("buffer=%d node=%d procno=%d epoch=%llu", buf->buf_id, + cluster_node_id, MyProc != NULL ? MyProc->pgprocno : -1, + (unsigned long long)cluster_epoch))); + if (!writer_authorized && cluster_bufmgr_pcm_x_holder_identity == UINT64_MAX) + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster PCM-X holder identity exhausted"), + errdetail("buffer=%d", buf->buf_id))); + MemSet(&key, 0, sizeof(key)); + if (writer_authorized) { + key.identity = writer_entry->claim.writer.identity; + key.identity.base_own_generation = writer_entry->granted.generation; + } else { + identity = ++cluster_bufmgr_pcm_x_holder_identity; + xid = GetTopTransactionIdIfAny(); + key.identity.tag = buf->tag; + key.identity.node_id = cluster_node_id; + key.identity.procno = (uint32)MyProc->pgprocno; + key.identity.xid = xid; + key.identity.cluster_epoch = cluster_epoch; + key.identity.request_id = identity; + key.identity.wait_seq = identity; + key.identity.base_own_generation = own_generation; + } + key.buffer_id = buf->buf_id; + for (;;) + { + ClusterPcmXHolderRetryAction action; + PcmXRuntimeSnapshot current_runtime; + + if (writer_authorized) + result = cluster_pcm_x_local_writer_holder_register_exact( + &key, &writer_entry->claim, &handle, &committed_own_generation); + else + result = cluster_pcm_x_local_holder_register(&key, &handle); + current_runtime = cluster_pcm_x_runtime_snapshot(); + action = cluster_pcm_x_holder_register_retry_action( + result, current_runtime.state == PCM_X_RUNTIME_ACTIVE + && current_runtime.master_session_incarnation != 0); + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) { + /* + * Publish the exact handle before any validation that can ERROR. + * From this point UnlockBuffers/owner-exit owns the registration + * and can detach it even if the committed-generation proof is + * corrupt. + */ + entry->buffer_id = buf->buf_id; + entry->content_lock = content_lock; + entry->handle = handle; + entry->phase = PCM_X_HOLDER_LEDGER_ACQUIRING; + if (writer_authorized && committed_own_generation != writer_entry->granted.generation) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, + "holder generation"); + break; + } + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) + cluster_bufmgr_pcm_x_holder_report_failure(result, buf, "register"); + cluster_bufmgr_pcm_x_holder_retry_wait( + content_lock, buf->buf_id, + wait_index++ % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + } + + return entry; +} + +static void +cluster_bufmgr_pcm_x_holder_activate(ClusterPcmXHolderLedgerEntry *entry) +{ + PcmXQueueResult result; + + if (entry == NULL) + return; + if (entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING) + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), "activate phase"); + result = cluster_pcm_x_local_holder_activate_exact(&entry->handle); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + cluster_bufmgr_pcm_x_holder_report_failure(result, + GetBufferDescriptor(entry->buffer_id), "activate"); + entry->phase = PCM_X_HOLDER_LEDGER_ACTIVE; +} + +static void +cluster_bufmgr_pcm_x_holder_mark_releasing(ClusterPcmXHolderLedgerEntry *entry) +{ + PcmXQueueResult result; + + if (entry == NULL) + return; + if (entry->phase != PCM_X_HOLDER_LEDGER_ACTIVE) + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "mark releasing phase"); + result = cluster_pcm_x_local_holder_mark_releasing_exact(&entry->handle); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + cluster_bufmgr_pcm_x_holder_report_failure(result, + GetBufferDescriptor(entry->buffer_id), "mark releasing"); + entry->phase = PCM_X_HOLDER_LEDGER_RELEASING; +} + +static void +cluster_bufmgr_pcm_x_holder_unregister(ClusterPcmXHolderLedgerEntry *entry) +{ + PcmXQueueResult result; + uint32 waits_used = 0; + + if (entry == NULL) + return; + if (entry->phase != PCM_X_HOLDER_LEDGER_RELEASING) + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "unregister phase"); + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) + cluster_bufmgr_pcm_x_holder_report_failure(PCM_X_QUEUE_BAD_STATE, + GetBufferDescriptor(entry->buffer_id), + "unregister lock order"); + for (;;) + { + ClusterPcmXHolderRetryAction action; + + result = cluster_pcm_x_local_holder_unregister_exact(&entry->handle); + action = cluster_pcm_x_holder_unregister_retry_action(result, waits_used); + if (action == CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE) + { + cluster_bufmgr_pcm_x_holder_clear(entry); + return; + } + if (action == CLUSTER_PCM_X_HOLDER_RETRY_DEFER) + { + entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; + return; + } + if (action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + cluster_bufmgr_pcm_x_holder_report_failure( + result, GetBufferDescriptor(entry->buffer_id), "unregister"); + } + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, + waits_used++); + } +} + +static void +cluster_bufmgr_pcm_x_holder_abort_acquiring(ClusterPcmXHolderLedgerEntry *entry) +{ + PcmXQueueResult result; + + if (entry == NULL) + return; + if (entry->content_lock == NULL) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "could not abort cluster PCM-X ACQUIRING holder with no content lock: buffer=%d", + entry->buffer_id); + return; + } + if (LWLockHeldByMe(entry->content_lock)) + return; + result = cluster_pcm_x_local_holder_abort_acquiring_exact(&entry->handle); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) + cluster_bufmgr_pcm_x_holder_clear(entry); + else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) + entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; + else + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "could not abort exact cluster PCM-X ACQUIRING holder: buffer=%d result=%d", + entry->buffer_id, (int) result); + } +} + +static void +cluster_bufmgr_pcm_x_holder_exception_cleanup_all(void) +{ + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) + { + ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; + PcmXQueueResult result; + + if (entry->phase == PCM_X_HOLDER_LEDGER_UNUSED) + continue; + if (entry->content_lock == NULL) + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "could not detach cluster PCM-X holder after error with no content lock: buffer=%d", + entry->buffer_id); + continue; + } + if (LWLockHeldByMe(entry->content_lock)) + continue; + result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, + entry->content_lock); + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) + cluster_bufmgr_pcm_x_holder_clear(entry); + else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) + entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; + else + { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "could not detach exact cluster PCM-X holder after error: buffer=%d result=%d", + entry->buffer_id, (int) result); + } + } +} + +static void +cluster_bufmgr_pcm_x_holder_reset(void) +{ + MemSet(cluster_bufmgr_pcm_x_holder_ledger, 0, + sizeof(cluster_bufmgr_pcm_x_holder_ledger)); + cluster_bufmgr_pcm_x_holder_identity = 0; +} + +static ClusterPcmXWriterLedgerEntry * +cluster_bufmgr_pcm_x_writer_find(BufferDesc *buf) +{ + int i; + + if (buf == NULL) + return NULL; + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_writer_ledger); i++) { + ClusterPcmXWriterLedgerEntry *entry = &cluster_bufmgr_pcm_x_writer_ledger[i]; + + if (entry->phase != PCM_X_WRITER_LEDGER_UNUSED && entry->buffer_id == buf->buf_id) + return entry; + } + return NULL; +} + +static ClusterPcmXWriterLedgerEntry * +cluster_bufmgr_pcm_x_writer_free_entry(void) +{ + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_writer_ledger); i++) + if (cluster_bufmgr_pcm_x_writer_ledger[i].phase == PCM_X_WRITER_LEDGER_UNUSED) + return &cluster_bufmgr_pcm_x_writer_ledger[i]; + return NULL; +} + +static bool +cluster_bufmgr_pcm_x_writer_claim_entry_exact(const ClusterPcmXWriterLedgerEntry *entry, + BufferDesc *buf) +{ + return entry != NULL && buf != NULL && entry->buffer_id == buf->buf_id + && entry->content_lock == BufferDescriptorGetContentLock(buf) && entry->claim.flags == 0 + && entry->claim.writer.flags == 0 + && entry->claim.active_slot.slot_index == entry->claim.writer.membership_slot.slot_index + && entry->claim.active_slot.slot_generation + == entry->claim.writer.membership_slot.slot_generation + && entry->claim.active_slot.slot_index != PCM_X_INVALID_SLOT_INDEX + && entry->claim.active_slot.slot_generation != 0 && entry->claim.claim_generation != 0 + && entry->claim.local_round != 0 + && entry->claim.local_round == entry->claim.writer.local_round + && entry->claim.role == entry->claim.writer.role + && (entry->claim.role == PCM_X_LOCAL_ROLE_NODE_LEADER + || entry->claim.role == PCM_X_LOCAL_ROLE_FOLLOWER); +} + +static bool +cluster_bufmgr_pcm_x_writer_entry_exact(const ClusterPcmXWriterLedgerEntry *entry, BufferDesc *buf) +{ + return cluster_bufmgr_pcm_x_writer_claim_entry_exact(entry, buf) + && BufferTagsEqual(&entry->claim.writer.identity.tag, &buf->tag) + && cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &entry->granted, + &entry->granted); +} + +static void +cluster_bufmgr_pcm_x_writer_report_failure(PcmXQueueResult result, BufferDesc *buf, + const char *operation) +{ + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED + || result == PCM_X_QUEUE_NO_CAPACITY) + ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cluster PCM-X writer operation is not ready"), + errdetail("operation=%s buffer=%d result=%d", operation, + buf != NULL ? buf->buf_id : -1, (int)result))); + + ereport(ERROR, + (errcode(ERRCODE_DATA_CORRUPTED), errmsg("cluster PCM-X writer operation failed"), + errdetail("operation=%s buffer=%d result=%d", operation, + buf != NULL ? buf->buf_id : -1, (int)result))); +} + +static void +cluster_bufmgr_pcm_x_writer_clear(ClusterPcmXWriterLedgerEntry *entry) +{ + if (entry != NULL) + MemSet(entry, 0, sizeof(*entry)); +} + +static void cluster_bufmgr_pcm_x_writer_release(ClusterPcmXWriterLedgerEntry *entry); + +static void +cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(void) +{ + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_writer_ledger); i++) { + ClusterPcmXWriterLedgerEntry *entry = &cluster_bufmgr_pcm_x_writer_ledger[i]; + PcmXQueueResult result; + + if (entry->phase != PCM_X_WRITER_LEDGER_DEFERRED) + continue; + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) { + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, + "could not drain deferred cluster PCM-X writer with content authority: buffer=%d", + entry->buffer_id); + continue; + } + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) + cluster_bufmgr_pcm_x_writer_clear(entry); + else if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) { + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, "could not drain deferred exact cluster PCM-X writer: buffer=%d result=%d", + entry->buffer_id, (int)result); + } + } +} + +static ClusterPcmXWriterLedgerEntry * +cluster_bufmgr_pcm_x_writer_prepare(BufferDesc *buf, PcmLockMode mode) +{ + ClusterPcmXWriterLedgerEntry *entry; + ClusterPcmDirectInitSnapshot observed; + ClusterPcmOwnSnapshot granted; + ClusterPcmOwnResult own_result; + PcmXQueueResult release_result; + PcmXQueueResult result; + LWLock *content_lock; + uint32 buf_state; + + if (mode != PCM_LOCK_MODE_X || buf == NULL || !cluster_bufmgr_should_pcm_track(buf)) + return NULL; + content_lock = BufferDescriptorGetContentLock(buf); + if (LWLockHeldByMe(content_lock)) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, + "prepare lock order"); + + cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(); + entry = cluster_bufmgr_pcm_x_writer_find(buf); + if (entry != NULL && entry->phase == PCM_X_WRITER_LEDGER_DEFERRED) { + cluster_bufmgr_pcm_x_writer_release(entry); + entry = NULL; + } + if (entry != NULL) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, + "reuse writer ledger"); + entry = cluster_bufmgr_pcm_x_writer_free_entry(); + if (entry == NULL) + ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("cluster PCM-X writer ledger is full"), + errdetail("maximum entries=%d", LWLOCK_MAX_HELD_BY_PROC))); + entry->buffer_id = buf->buf_id; + entry->content_lock = content_lock; + entry->phase = PCM_X_WRITER_LEDGER_HANDOFF; + entry->claim_handed_off = false; + + buf_state = LockBufHdr(buf); + cluster_bufmgr_pcm_direct_init_snapshot_locked(buf, buf_state, false, &observed); + UnlockBufHdr(buf, buf_state); + if ((observed.buf_state & BM_VALID) == 0) { + cluster_bufmgr_pcm_x_writer_clear(entry); + return NULL; + } + MemSet(&entry->claim, 0, sizeof(entry->claim)); + entry->claim_handed_off = false; + pg_write_barrier(); + result = cluster_gcs_pcm_x_acquire_writer(buf, &entry->claim, &entry->claim_handed_off); + if (result != PCM_X_QUEUE_OK) { + if (entry->claim_handed_off) { + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, + "failed claim handoff"); + } + cluster_bufmgr_pcm_x_writer_clear(entry); + cluster_bufmgr_pcm_x_writer_report_failure(result, buf, "queue acquire"); + } + if (!entry->claim_handed_off) { + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, "claim handoff"); + } + + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &granted); + if (own_result != CLUSTER_PCM_OWN_OK + || !cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &granted, &granted)) { + release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (release_result == PCM_X_QUEUE_OK) + cluster_bufmgr_pcm_x_writer_clear(entry); + else { + /* Keep the exact claim reachable for owner-exit/deferred retry. */ + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(release_result, buf, + "grant snapshot cleanup"); + } + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, "grant snapshot"); + } + + entry->granted = granted; + entry->phase = PCM_X_WRITER_LEDGER_ACQUIRING; + return entry; +} + +static void +cluster_bufmgr_pcm_x_writer_activate(ClusterPcmXWriterLedgerEntry *entry) +{ + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult own_result; + BufferDesc *buf; + + if (entry == NULL) + return; + buf = GetBufferDescriptor(entry->buffer_id); + if (entry->phase != PCM_X_WRITER_LEDGER_ACQUIRING + || !cluster_bufmgr_pcm_x_writer_entry_exact(entry, buf) || entry->content_lock == NULL + || !LWLockHeldByMe(entry->content_lock)) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, "activate phase"); + own_result = cluster_bufmgr_pcm_own_snapshot(buf, &live); + if (own_result != CLUSTER_PCM_OWN_OK + || !cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &entry->granted, &live)) { + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf, "activate ownership"); + } + entry->phase = PCM_X_WRITER_LEDGER_ACTIVE; +} + +static void +cluster_bufmgr_pcm_x_writer_mark_releasing(ClusterPcmXWriterLedgerEntry *entry) +{ + BufferDesc *buf; -/* - * PGRAC (spec-4.10 D1): ignore_checksum_failure is defined in bufpage.c with - * no public extern; the online-recovery read hook consults it to honor Q3 - * (online recovery takes precedence over ignore_checksum_failure). Reading - * this bool short-circuits the Q3 re-check on the healthy path (default off), - * so no extra per-read checksum work. - */ -extern bool ignore_checksum_failure; -#endif -#include "storage/bufmgr.h" -#include "storage/ipc.h" -#include "storage/lmgr.h" -#include "storage/proc.h" -#include "storage/smgr.h" -#include "storage/standby.h" -#include "utils/memdebug.h" -#include "utils/memutils.h" -#include "utils/ps_status.h" -#include "utils/rel.h" -#include "utils/resowner_private.h" -#include "utils/timestamp.h" + if (entry == NULL) + return; + buf = GetBufferDescriptor(entry->buffer_id); + if (entry->phase != PCM_X_WRITER_LEDGER_ACTIVE + || !cluster_bufmgr_pcm_x_writer_entry_exact(entry, buf) || entry->content_lock == NULL + || !LWLockHeldByMe(entry->content_lock)) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, "mark releasing"); + entry->phase = PCM_X_WRITER_LEDGER_RELEASING; +} +static void +cluster_bufmgr_pcm_x_writer_release(ClusterPcmXWriterLedgerEntry *entry) +{ + BufferDesc *buf; + uint32 waits_used = 0; -/* Note: these two macros only work on shared buffers, not local ones! */ -#define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ)) -#define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr))) + if (entry == NULL) + return; + buf = GetBufferDescriptor(entry->buffer_id); + if ((entry->phase != PCM_X_WRITER_LEDGER_RELEASING + && entry->phase != PCM_X_WRITER_LEDGER_DEFERRED) + || !cluster_bufmgr_pcm_x_writer_claim_entry_exact(entry, buf) || entry->content_lock == NULL + || LWLockHeldByMe(entry->content_lock)) + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_BAD_STATE, buf, "release phase"); + for (;;) { + ClusterPcmXWriterRetryAction action; + PcmXQueueResult result; + + result = cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(&entry->claim); + action = cluster_pcm_x_writer_release_retry_action(result, waits_used); + if (action == CLUSTER_PCM_X_WRITER_RETRY_COMPLETE) { + cluster_bufmgr_pcm_x_writer_clear(entry); + return; + } + if (action == CLUSTER_PCM_X_WRITER_RETRY_FAIL) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(result, buf, "release"); + } + if (action == CLUSTER_PCM_X_WRITER_RETRY_DEFER) { + PcmXRuntimeSnapshot runtime = cluster_pcm_x_runtime_snapshot(); + + if (runtime.state != PCM_X_RUNTIME_ACTIVE || runtime.master_session_incarnation == 0) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_NOT_READY, buf, + "release runtime"); + } + waits_used = 0; + } + cluster_bufmgr_pcm_x_holder_retry_wait(entry->content_lock, entry->buffer_id, + waits_used++ + % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS); + } +} -/* Note: this macro only works on local buffers, not shared ones! */ -#define LocalBufHdrGetBlock(bufHdr) \ - LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)] +static void +cluster_bufmgr_pcm_x_writer_abort_acquiring(ClusterPcmXWriterLedgerEntry *entry) +{ + PcmXQueueResult result; -/* Bits in SyncOneBuffer's return value */ -#define BUF_WRITTEN 0x01 -#define BUF_REUSABLE 0x02 + if (entry == NULL || entry->phase != PCM_X_WRITER_LEDGER_ACQUIRING) + return; + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) + return; + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) + cluster_bufmgr_pcm_x_writer_clear(entry); + else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + else { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, "could not abort exact cluster PCM-X ACQUIRING writer: buffer=%d result=%d", + entry->buffer_id, (int)result); + } +} -#define RELS_BSEARCH_THRESHOLD 20 +static void +cluster_bufmgr_pcm_x_writer_exception_cleanup_all(void) +{ + int i; -#ifdef USE_PGRAC_CLUSTER -/* - * spec-6.14 D4: single PCM-tracking criterion by relfilenumber. Under - * cluster.shared_catalog the catalog lives in the single shared tree, so its - * pages need the same N/S/X + CR + PI lost-write coherency as user pages - * (routing a catalog page into the shared tree without PCM tracking would be a - * double-write hazard -- pairs with the D3 smgr flip, INV-14-1). All shared - * buffers under shared_catalog=on hold permanent (non-temp) relations: temp - * relations use local buffers and never reach these paths, and unlogged - * permanent relations are rejected at DDL time (Q12). Off mode keeps the - * historic FirstNormalObjectId user-only boundary. - * - * The eviction/release hooks (HC112) MUST use this SAME criterion as the - * acquire path, otherwise a tracked catalog page's PCM lock would leak on - * eviction (spec-6.14 R6 -- do not let a gate drift). - */ -static inline bool -cluster_bufmgr_reln_pcm_tracked(RelFileNumber relnum) + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_writer_ledger); i++) { + ClusterPcmXWriterLedgerEntry *entry = &cluster_bufmgr_pcm_x_writer_ledger[i]; + PcmXQueueResult result; + + if (entry->phase == PCM_X_WRITER_LEDGER_UNUSED) + continue; + if (entry->phase == PCM_X_WRITER_LEDGER_HANDOFF && !entry->claim_handed_off) { + /* requester cleanup still owns any in-flight claim */ + cluster_bufmgr_pcm_x_writer_clear(entry); + continue; + } + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)) + continue; + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + if (result == PCM_X_QUEUE_OK) + cluster_bufmgr_pcm_x_writer_clear(entry); + else if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY) + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + else { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, + "could not release exact cluster PCM-X writer after error: buffer=%d result=%d", + entry->buffer_id, (int)result); + } + } +} + +/* on_shmem_exit has no later backend entrance that could consume a DEFERRED + * ledger. Once LWLockReleaseAll has dropped content authority, retry both + * queue lanes to an exact shared terminal while the same runtime remains + * active. No CHECK_FOR_INTERRUPTS, ereport, or latch wait is legal here. */ +static bool +cluster_bufmgr_pcm_x_writer_owner_exit_drain_once(bool runtime_active) { - return cluster_shared_catalog || relnum >= (RelFileNumber) FirstNormalObjectId; + bool retry = false; + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_writer_ledger); i++) { + ClusterPcmXWriterLedgerEntry *entry = &cluster_bufmgr_pcm_x_writer_ledger[i]; + ClusterPcmXOwnerExitAction action; + PcmXQueueResult result; + BufferDesc *buf; + + if (entry->phase == PCM_X_WRITER_LEDGER_UNUSED) + continue; + if (entry->phase == PCM_X_WRITER_LEDGER_HANDOFF && !entry->claim_handed_off) { + /* + * The before_shmem requester callback remained the sole owner. + * It either completed exact cleanup or preserved shared evidence + * behind RECOVERY_BLOCKED; guessing a second release here would + * be unsafe. + */ + continue; + } + if (entry->buffer_id < 0 || entry->buffer_id >= NBuffers) { + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, + "could not owner-exit drain cluster PCM-X writer with invalid buffer: buffer=%d", + entry->buffer_id); + continue; + } + buf = GetBufferDescriptor(entry->buffer_id); + if (entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock) + || !cluster_bufmgr_pcm_x_writer_claim_entry_exact(entry, buf)) { + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, + "could not owner-exit drain malformed cluster PCM-X writer: buffer=%d phase=%d", + entry->buffer_id, (int)entry->phase); + continue; + } + result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim); + action = cluster_pcm_x_owner_exit_action(result, false, runtime_active); + if (action == CLUSTER_PCM_X_OWNER_EXIT_COMPLETE) + cluster_bufmgr_pcm_x_writer_clear(entry); + else if (action == CLUSTER_PCM_X_OWNER_EXIT_RETRY) { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + retry = true; + } else { + entry->phase = PCM_X_WRITER_LEDGER_DEFERRED; + cluster_pcm_x_runtime_fail_closed(); + elog(LOG, "preserving cluster PCM-X writer at owner exit: buffer=%d result=%d", + entry->buffer_id, (int)result); + } + } + return retry; } -static inline bool -cluster_bufmgr_should_pcm_track(BufferDesc *buf) +static bool +cluster_bufmgr_pcm_x_holder_owner_exit_drain_once(bool runtime_active) { - return cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&buf->tag)); + bool retry = false; + int i; + + for (i = 0; i < lengthof(cluster_bufmgr_pcm_x_holder_ledger); i++) { + ClusterPcmXHolderLedgerEntry *entry = &cluster_bufmgr_pcm_x_holder_ledger[i]; + ClusterPcmXOwnerExitAction action; + PcmXQueueResult result; + BufferDesc *buf; + + if (entry->phase == PCM_X_HOLDER_LEDGER_UNUSED) + continue; + if (entry->buffer_id < 0 || entry->buffer_id >= NBuffers) { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, + "could not owner-exit drain cluster PCM-X holder with invalid buffer: buffer=%d", + entry->buffer_id); + continue; + } + buf = GetBufferDescriptor(entry->buffer_id); + if (entry->content_lock == NULL + || entry->content_lock != BufferDescriptorGetContentLock(buf) + || LWLockHeldByMe(entry->content_lock)) { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, + "could not owner-exit drain malformed cluster PCM-X holder: buffer=%d phase=%d", + entry->buffer_id, (int)entry->phase); + continue; + } + result = cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle, + entry->content_lock); + action = cluster_pcm_x_owner_exit_action(result, true, runtime_active); + if (action == CLUSTER_PCM_X_OWNER_EXIT_COMPLETE) + cluster_bufmgr_pcm_x_holder_clear(entry); + else if (action == CLUSTER_PCM_X_OWNER_EXIT_RETRY) { + entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED; + retry = true; + } else { + cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry); + elog(LOG, "preserving cluster PCM-X holder at owner exit: buffer=%d result=%d", + entry->buffer_id, (int)result); + } + } + return retry; } -/* - * PGRAC ownership-generation wave — coherent (pcm_state, generation, flags) - * mutation / read helpers. - * - * The three fields must move together under ONE lock so a reader never sees a - * torn triple (TOCTOU). We use the buffer header spinlock as that lock. - * pcm_state lives on the BufferDesc; generation + flags live in the parallel - * ClusterPcmOwnArray indexed by buf_id (cluster_pcm_own.h). - * - * cluster_pcm_own_transition: for callers that do NOT already hold the header - * spinlock (e.g. the LockBuffer grant mirror + the X->S downgrade, which hold - * only the content lock). Sets pcm_state, bumps the generation, and applies - * flag changes atomically under the header spinlock. Every COMMITTED local - * ownership transition routes through here (or the *_locked bump below) so the - * generation increments exactly once per ownership change. - */ -static inline void -cluster_pcm_own_transition(BufferDesc *buf, uint8 new_pcm_state, uint32 set_flags, - uint32 clear_flags) +static void +cluster_bufmgr_pcm_x_owner_exit_drain(void) { - uint32 buf_state; + for (;;) { + PcmXRuntimeSnapshot runtime = cluster_pcm_x_runtime_snapshot(); + bool runtime_active + = runtime.state == PCM_X_RUNTIME_ACTIVE && runtime.master_session_incarnation != 0; + bool writer_retry; + bool holder_retry; - buf_state = LockBufHdr(buf); - buf->pcm_state = new_pcm_state; - cluster_pcm_own_gen_bump(buf->buf_id); - if (set_flags != 0 || clear_flags != 0) - cluster_pcm_own_flags_apply(buf->buf_id, set_flags, clear_flags); - UnlockBufHdr(buf, buf_state); + writer_retry = cluster_bufmgr_pcm_x_writer_owner_exit_drain_once(runtime_active); + + /* + * Content LWLocks are already gone. The holder lane may therefore + * detach even when the writer's short admission gate asks for another + * pass; active_writer still blocks both a successor claim and DRAIN + * until WRITER_COMPLETE commits on a later iteration. + */ + holder_retry = cluster_bufmgr_pcm_x_holder_owner_exit_drain_once(runtime_active); + if (!writer_retry && !holder_retry) + return; + if (!runtime_active) { + cluster_pcm_x_runtime_fail_closed(); + return; + } + pg_usleep(1000L); + } } -/* - * cluster_pcm_own_transition_locked: for callers that ALREADY hold the header - * spinlock (the N-flip invalidate/drop/evict sites). Same effect minus the - * lock/unlock. Caller sets pcm_state itself (it already does); this bumps the - * generation + flags in the same critical section. - */ -static inline void -cluster_pcm_own_bump_locked(BufferDesc *buf, uint32 set_flags, uint32 clear_flags) +static void +cluster_bufmgr_pcm_x_writer_reset(void) { - cluster_pcm_own_gen_bump(buf->buf_id); - if (set_flags != 0 || clear_flags != 0) - cluster_pcm_own_flags_apply(buf->buf_id, set_flags, clear_flags); + MemSet(cluster_bufmgr_pcm_x_writer_ledger, 0, sizeof(cluster_bufmgr_pcm_x_writer_ledger)); } -/* - * cluster_pcm_own_set_flags: apply flag bits WITHOUT changing pcm_state or - * bumping the generation (a GRANT_PENDING / REVOKING marker is not itself an - * ownership transition; the finalize that follows is). Under the header - * spinlock so a lock-free reader of the triple sees a coherent flags value. - */ -static inline void -cluster_pcm_own_set_flags(BufferDesc *buf, uint32 set_flags, uint32 clear_flags) +/* Defined with the process-local pin table below. Direct-init proofs call it + * while holding the matching buffer header spinlock so pin + mapping identity + * are captured in one critical section. */ +static inline int32 GetPrivateRefCount(Buffer buffer); + +/* Capture the proof projection while tag, ownership, and shared refcount are + * serialized by the BufferDesc header lock. The private refcount table is + * process-local and cannot change concurrently inside this backend. */ +static void +cluster_bufmgr_pcm_direct_init_snapshot_locked(BufferDesc *buf, uint32 buf_state, + bool page_is_new, + ClusterPcmDirectInitSnapshot *out) { - uint32 buf_state; + memset(out, 0, sizeof(*out)); + out->tag = buf->tag; + out->generation = cluster_pcm_own_gen_get(buf->buf_id); + out->reservation_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + out->flags = cluster_pcm_own_flags_get(buf->buf_id); + out->buf_state = buf_state; + out->buf_id = buf->buf_id; + out->private_refcount = GetPrivateRefCount(BufferDescriptorGetBuffer(buf)); + out->buffer_type = buf->buffer_type; + out->pcm_state = buf->pcm_state; + out->page_is_new = page_is_new; +} - buf_state = LockBufHdr(buf); - cluster_pcm_own_flags_apply(buf->buf_id, set_flags, clear_flags); - UnlockBufHdr(buf, buf_state); +pg_attribute_noreturn() static void +cluster_bufmgr_pcm_direct_init_report_failure(BufferDesc *buf, ClusterPcmOwnResult result, + const ClusterPcmDirectInitSnapshot *observed, + const char *context) +{ + uint64 generation = observed != NULL ? observed->generation : 0; + uint32 flags = observed != NULL ? observed->flags : 0; + + cluster_grd_inc_block_path_failclosed(); + if (result == CLUSTER_PCM_OWN_BUSY || result == CLUSTER_PCM_OWN_CORRUPT || + result == CLUSTER_PCM_OWN_EXHAUSTED || result == CLUSTER_PCM_OWN_NOT_READY) + cluster_pcm_own_report_bump_failure(buf, result, generation, flags, context); + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cluster PCM direct initialization lacks an exact live proof"), + errdetail("context=%s buffer=%d result=%d generation=%llu flags=0x%x", + context, buf != NULL ? buf->buf_id : -1, (int) result, + (unsigned long long) generation, flags))); } -/* - * W2 RED substrate — process-local marker: true while a GCS drop's - * InvalidateBufferTry attempt is in flight, so the drop-prepin inject inside - * InvalidateBufferTry fires ONLY for GCS drops (never for plain evictions). - * The drop runs to completion in one call chain in the pump process, so a - * plain static is race-free. - */ -static bool cluster_bufmgr_in_gcs_drop = false; +pg_attribute_noreturn() static void +cluster_bufmgr_pcm_direct_init_no_grant_failclosed(BufferDesc *buf, + ClusterPcmDirectInitKind kind) +{ + cluster_grd_inc_block_path_failclosed(); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cluster PCM direct initialization did not obtain X ownership"), + errdetail("buffer=%d operation=%d returned a one-shot image without a durable grant", + buf != NULL ? buf->buf_id : -1, (int) kind))); +} -/* - * cluster_pcm_own_read: read the coherent (pcm_state, generation, flags) triple - * under the header spinlock. Used by the cached-X writer re-verify and the - * drop-restore ABA check. Any *out may be NULL. - */ -static inline void -cluster_pcm_own_read(BufferDesc *buf, uint8 *out_state, uint64 *out_gen, uint32 *out_flags) +/* Arm only at a source-authorized operation site. This does not reserve the + * buffer: the consumer revalidates the complete identity before publishing + * GRANT_PENDING, so a changed/reused descriptor fails closed. */ +static void +cluster_bufmgr_pcm_arm_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind kind, + ClusterPcmDirectInitProof *proof) { + ClusterPcmDirectInitSnapshot observed; + ClusterPcmOwnResult result; uint32 buf_state; + memset(proof, 0, sizeof(*proof)); + if (!cluster_pcm_is_active() || !cluster_bufmgr_should_pcm_track(buf)) + return; + buf_state = LockBufHdr(buf); - if (out_state != NULL) - *out_state = buf->pcm_state; - if (out_gen != NULL) - *out_gen = cluster_pcm_own_gen_get(buf->buf_id); - if (out_flags != NULL) - *out_flags = cluster_pcm_own_flags_get(buf->buf_id); + cluster_bufmgr_pcm_direct_init_snapshot_locked( + buf, buf_state, PageIsNew((Page) BufHdrGetBlock(buf)), &observed); + result = cluster_pcm_direct_init_proof_arm(kind, &observed, proof); UnlockBufHdr(buf, buf_state); + + if (result != CLUSTER_PCM_OWN_OK) + cluster_bufmgr_pcm_direct_init_report_failure(buf, result, &observed, + "direct-init arm"); } /* - * spec-6.14 D8 (A1 blocker): PCM write gate for the two content-lock - * acquisitions that bypass LockBuffer() -- ReadBuffer_common's - * RBM_ZERO_AND_LOCK branch and ExtendBufferedRelShared's EB_LOCK_FIRST - * branch (both LWLockAcquire the content lock directly because the buffer - * is not BM_VALID yet, which LockBuffer asserts). - * - * Without this, a freshly EXTENDED heap block is written under a content - * lock that never registered PCM X with the block's GCS master. A remote - * reader whose master-side GRD then shows state N takes the lawful HC88 - * "N => storage is current" tag-only + storage read -- and reads a stale - * zero page from the shared tree, caching it under durable S (the t/337 A1 - * false-invisible: heap extends use EB_LOCK_FIRST, so the INSERT's page - * never crossed; btree extends go through LockBuffer and did). Registering - * X here restores the HC88 premise for extended blocks. - * - * Mirrors the LockBuffer gate exactly (active + tracked + not covered), - * and runs BEFORE the content-lock LWLockAcquire: the acquire path may - * install a shipped image, which itself takes the content lock (deadlock - * if we held it), and LockBuffer establishes the same acquire-then-lock - * order. The caller zero-fills/PageInits the page afterwards, so an - * installed image being overwritten is fine -- the point is the master-side - * X registration, not the bytes. + * PCM write gate for the few operations which initialize known-new bytes and + * therefore cannot use the ordinary N/S -> X convert queue. The proof is + * stack-local and single-use. Its live recheck and GRANT_PENDING publication + * share one BufferDesc header-lock interval; no wire request can be emitted + * from an unproved or stale descriptor. */ static void -cluster_bufmgr_pcm_gate_direct_lock(BufferDesc *buf) +cluster_bufmgr_pcm_gate_direct_init(BufferDesc *buf, ClusterPcmDirectInitKind kind, + ClusterPcmDirectInitProof *proof) { - if (cluster_pcm_is_active() && - cluster_bufmgr_should_pcm_track(buf) && - !(cluster_gcs_block_local_cache && - cluster_pcm_mode_covers((PcmLockMode) buf->pcm_state, PCM_LOCK_MODE_X))) + ClusterPcmDirectInitSnapshot observed; + ClusterPcmOwnSnapshot pending_base; + ClusterPcmOwnResult pending_result; + bool grant_acquired = false; + uint32 buf_state; + uint64 pending_token = 0; + uint64 committed_generation; + + if (!cluster_pcm_is_active() || !cluster_bufmgr_should_pcm_track(buf)) + return; + if (proof == NULL) + cluster_bufmgr_pcm_direct_init_report_failure(buf, CLUSTER_PCM_OWN_INVALID, NULL, + "direct-init missing proof"); + + buf_state = LockBufHdr(buf); + cluster_bufmgr_pcm_direct_init_snapshot_locked( + buf, buf_state, PageIsNew((Page) BufHdrGetBlock(buf)), &observed); + pending_result = cluster_pcm_direct_init_proof_consume(kind, &observed, proof); + if (pending_result == CLUSTER_PCM_OWN_OK) + pending_result = cluster_pcm_own_reservation_begin_exact( + buf->buf_id, observed.generation, PCM_OWN_FLAG_GRANT_PENDING, &pending_token); + if (pending_result == CLUSTER_PCM_OWN_OK) + { + memset(&pending_base, 0, sizeof(pending_base)); + pending_base.tag = observed.tag; + pending_base.generation = observed.generation; + pending_base.reservation_token = observed.reservation_token; + pending_base.flags = observed.flags; + pending_base.pcm_state = observed.pcm_state; + } + UnlockBufHdr(buf, buf_state); + + if (pending_result != CLUSTER_PCM_OWN_OK) + cluster_bufmgr_pcm_direct_init_report_failure(buf, pending_result, &observed, + "direct-init consume"); + + PG_TRY(); { - /* - * PGRAC ownership-generation wave (W3 cover) — this direct-lock gate is - * an X install that BYPASSES LockBuffer (RBM_ZERO_AND_LOCK / - * EB_LOCK_FIRST), so it needs the same in-flight-grant protocol: mark - * GRANT_PENDING before the request goes out so a same-tag INVALIDATE - * arriving while pcm_state is still N parks (does not ack the grant - * away), then finalize through the coherent transition so pcm_state=X, - * the ownership generation bumps, and PENDING clears under one lock. - */ - cluster_pcm_own_set_flags(buf, PCM_OWN_FLAG_GRANT_PENDING, 0); + grant_acquired = cluster_pcm_lock_acquire_buffer(buf, PCM_LOCK_MODE_X); + } + PG_CATCH(); + { + cluster_pcm_own_abort_grant_after_error(buf, &pending_base, pending_token, + "direct-init acquire"); + PG_RE_THROW(); + } + PG_END_TRY(); - PG_TRY(); - { - if (cluster_pcm_lock_acquire_buffer(buf, PCM_LOCK_MODE_X)) - { - buf->buffer_type = (uint8) BUF_TYPE_XCUR; - cluster_pcm_own_transition(buf, (uint8) PCM_STATE_X, 0, - PCM_OWN_FLAG_GRANT_PENDING); - } - else - cluster_pcm_own_set_flags(buf, 0, PCM_OWN_FLAG_GRANT_PENDING); - } - PG_CATCH(); - { - /* W3 — never leak GRANT_PENDING past a throwing acquire (a stale - * marker parks every later same-tag INVALIDATE; liveness). */ - cluster_pcm_own_set_flags(buf, 0, PCM_OWN_FLAG_GRANT_PENDING); - PG_RE_THROW(); - } - PG_END_TRY(); + if (grant_acquired) + cluster_pcm_own_finish_grant_or_rollback(buf, &pending_base, pending_token, + (uint8) PCM_STATE_X, PCM_LOCK_MODE_X, + &committed_generation); + else + { + cluster_pcm_own_abort_grant_or_error(buf, &pending_base, pending_token, + "direct-init read-image"); + cluster_bufmgr_pcm_direct_init_no_grant_failclosed(buf, kind); } } + #endif /* @@ -692,9 +2778,11 @@ static void BufferSync(int flags); #ifdef USE_PGRAC_CLUSTER /* PGRAC: spec-6.12h D-h1 — Past Image conversion (definition below). */ static bool cluster_bufmgr_convert_to_pi_locked(BufferDesc *buf, uint32 buf_state); + /* PGRAC: spec-6.12h D-h2 — FlushBuffer write-note (cluster_gcs_block.h is * included mid-file, after FlushBuffer; redeclare the one symbol it needs). */ extern void cluster_gcs_block_pi_write_note(BufferTag tag, SCN page_scn); + /* PGRAC ownership-generation wave (W3) — RED delivery shim, used by LockBuffer * (before the mid-file include; same redeclare pattern as above). */ extern bool cluster_gcs_block_test_deliver_self_invalidate(BufferTag tag); @@ -708,10 +2796,14 @@ static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits); static void shared_buffer_write_error_callback(void *arg); static void local_buffer_write_error_callback(void *arg); + /* PGRAC: GCS serve-stall round-5 — InvalidateBuffer tail extraction + the * bounded non-waiting variant for the IC dispatch pump. */ -static void InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, +static bool InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, LWLock *oldPartitionLock, uint32 buf_state); +static void InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, + LWLock *oldPartitionLock, uint32 buf_state, + uint8 old_pcm_mode, bool release_pcm_holder); static bool InvalidateBufferTry(BufferDesc *buf); static BufferDesc *BufferAlloc(SMgrRelation smgr, char relpersistence, @@ -1006,6 +3098,7 @@ ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum, errmsg("cannot access temporary tables of other sessions"))); #ifdef USE_PGRAC_CLUSTER + /* * PGRAC: spec-5.59 D4 — relation-kind hint for the cross-node profiling * index axis. This is the last point on the read path that still has @@ -1256,6 +3349,9 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, IOContext io_context; IOObject io_object; bool isLocalBuf = SmgrIsTemp(smgr); +#ifdef USE_PGRAC_CLUSTER + ClusterPcmDirectInitProof direct_init_proof; +#endif *hit = false; @@ -1323,12 +3419,13 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, pgBufferUsage.shared_blks_read++; #ifdef USE_PGRAC_CLUSTER + /* * spec-6.14 D10b: catalog-page buffer traffic under the shared - * catalog (dump keys catalog.buf_hit_count / buf_miss_count). A - * miss is a catalog page sourced from shared storage or a - * cross-node CF transfer -- the cache-locality signal the A-L9 - * observability leg watches. + * catalog (dump keys catalog.buf_hit_count / buf_miss_count). A miss + * is a catalog page sourced from shared storage or a cross-node CF + * transfer -- the cache-locality signal the A-L9 observability leg + * watches. */ if (cluster_shared_catalog && smgr->smgr_rlocator.locator.relNumber < @@ -1369,8 +3466,7 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, if (!isLocalBuf) { if (mode == RBM_ZERO_AND_LOCK) - LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), - LW_EXCLUSIVE); + LockBuffer(BufferDescriptorGetBuffer(bufHdr), BUFFER_LOCK_EXCLUSIVE); else if (mode == RBM_ZERO_AND_CLEANUP_LOCK) LockBufferForCleanup(BufferDescriptorGetBuffer(bufHdr)); } @@ -1392,7 +3488,14 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, * wants us to allocate a buffer. */ if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) + { MemSet((char *) bufBlock, 0, BLCKSZ); +#ifdef USE_PGRAC_CLUSTER + if (!isLocalBuf) + cluster_bufmgr_pcm_arm_direct_init( + bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, &direct_init_proof); +#endif + } else { instr_time io_start = pgstat_prepare_io_time(); @@ -1408,14 +3511,15 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, PIV_LOG_WARNING | PIV_REPORT_STAT); #ifdef USE_PGRAC_CLUSTER + /* * PGRAC (spec-4.10 D1/D4): try to rebuild a corrupt block from WAL * before applying the zero/error policy. PERMANENT shared-buffer * relations only (relpersistence == RELPERSISTENCE_PERMANENT && * !isLocalBuf): temp / unlogged relations have no authoritative WAL - * chain -- a reused relfilenode could even match stale WAL of a dropped - * relation -- so they are never WAL-reconstructed. Single-node / - * own-thread only (cross-node forward Stage 5). Q3 + * chain -- a reused relfilenode could even match stale WAL of a + * dropped relation -- so they are never WAL-reconstructed. + * Single-node / own-thread only (cross-node forward Stage 5). Q3 * (recovery-precedence): with online recovery on, also rebuild when * ignore_checksum_failure masked a real checksum mismatch * (PageIsVerifiedExtended returned true); if it cannot rebuild, @@ -1452,10 +3556,10 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, { /* * Online recovery on but the block could not be rebuilt. - * PANIC escalation (operator opt-in) is restricted to the main - * data fork: VM/FSM and other auxiliary forks are - * reconstructible from scratch, so an unrebuildable one should - * fail the query (ERROR), not crash the instance. + * PANIC escalation (operator opt-in) is restricted to the + * main data fork: VM/FSM and other auxiliary forks are + * reconstructible from scratch, so an unrebuildable one + * should fail the query (ERROR), not crash the instance. */ int elevel = (cluster_block_recovery_on_unrecoverable == CLUSTER_BLKREC_ACTION_PANIC && forkNum == MAIN_FORKNUM) @@ -1495,9 +3599,8 @@ ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum, !isLocalBuf) { #ifdef USE_PGRAC_CLUSTER - /* spec-6.14 D8: register PCM X for this LockBuffer-bypassing - * exclusive acquisition (see cluster_bufmgr_pcm_gate_direct_lock). */ - cluster_bufmgr_pcm_gate_direct_lock(bufHdr); + cluster_bufmgr_pcm_gate_direct_init( + bufHdr, CLUSTER_PCM_DIRECT_INIT_READ_MISS, &direct_init_proof); #endif LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE); } @@ -1826,21 +3929,79 @@ InvalidateBuffer(BufferDesc *buf) goto retry; } - InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state); + if (!InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state)) + { + pg_usleep(1000L); + goto retry; + } +} + +/* + * InvalidateBufferCommitLocked -- shared commit tail of InvalidateBuffer + * and InvalidateBufferTry. + * + * Entry: buffer header spinlock held, oldPartitionLock held EXCLUSIVE, + * tag verified equal to *oldTag, refcount known zero. Releases both + * locks. This is the original InvalidateBuffer tail, extracted verbatim + * (PGRAC GCS serve-stall round-5; see the InvalidateBuffer header note). + */ +static bool +InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, + LWLock *oldPartitionLock, uint32 buf_state) +{ + uint8 old_pcm_mode = 0; + bool release_pcm_holder = false; +#ifdef USE_PGRAC_CLUSTER + ClusterPcmOwnEvictionCapture eviction_capture; + ClusterPcmOwnResult eviction_result; + uint32 observed_flags = 0; + uint64 observed_generation = 0; + + /* + * D5a: descriptor reuse is an exact ownership-tuple commit. Refuse to + * erase the tag while a reservation/revoke is live, at generation MAX, or + * after any tuple drift. The buffer cannot enter StrategyFreeBuffer + * until this commit and (when applicable) the saved-tag master release + * succeed. + */ + cluster_pcm_own_eviction_capture_locked(buf, &eviction_capture); + eviction_result = cluster_pcm_own_eviction_commit_locked( + buf, &eviction_capture, &observed_generation, &observed_flags); + if (eviction_result != CLUSTER_PCM_OWN_OK) + { + UnlockBufHdr(buf, buf_state); + LWLockRelease(oldPartitionLock); + if (eviction_result == CLUSTER_PCM_OWN_BUSY || eviction_result == CLUSTER_PCM_OWN_STALE) + return false; + cluster_pcm_own_report_bump_failure(buf, eviction_result, + observed_generation != 0 ? observed_generation + : eviction_capture.generation, + observed_flags != 0 ? observed_flags : eviction_capture.flags, + "buffer eviction"); + } + old_pcm_mode = eviction_capture.pcm_state; + release_pcm_holder = cluster_pcm_is_active() + && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) + && (old_pcm_mode == (uint8) PCM_LOCK_MODE_S || old_pcm_mode == (uint8) PCM_LOCK_MODE_X); +#endif + + InvalidateBufferCommitTailLocked(buf, oldTag, oldHash, oldPartitionLock, buf_state, + old_pcm_mode, release_pcm_holder); + return true; } /* - * InvalidateBufferCommitLocked -- shared commit tail of InvalidateBuffer - * and InvalidateBufferTry. + * InvalidateBufferCommitTailLocked -- remove a mapping after its ownership + * tuple has already been committed under the current header-lock hold. * - * Entry: buffer header spinlock held, oldPartitionLock held EXCLUSIVE, - * tag verified equal to *oldTag, refcount known zero. Releases both - * locks. This is the original InvalidateBuffer tail, extracted verbatim - * (PGRAC GCS serve-stall round-5; see the InvalidateBuffer header note). + * The normal D5a path above and the exact queue revoke path share this tail. + * The latter passes N/false because its master-visible release is driven by + * the queue protocol, not the cache-eviction wire. */ static void -InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, - LWLock *oldPartitionLock, uint32 buf_state) +InvalidateBufferCommitTailLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, + LWLock *oldPartitionLock, uint32 buf_state, + uint8 old_pcm_mode, bool release_pcm_holder) { uint32 oldFlags; @@ -1852,7 +4013,9 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, ClearBufferTag(&buf->tag); buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK); #ifdef USE_PGRAC_CLUSTER - /* PGRAC: spec-6.12h D-h3a — the mapping dies here; reset the cluster + + /* + * PGRAC: spec-6.12h D-h3a — the mapping dies here; reset the cluster * copy label under the same header-lock hold so a later retag can never * inherit a stale BUF_TYPE_PI (the PI shape must stay reachable ONLY * through cluster_bufmgr_convert_to_pi_locked, or the D-h3 shadow stamp @@ -1861,34 +4024,6 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, #endif UnlockBufHdr(buf, buf_state); -#ifdef USE_PGRAC_CLUSTER - /* - * PGRAC: spec-2.35 D4 (HC112) — cache eviction hook. Buffer is being - * invalidated (DROP / TRUNCATE relation, DropDatabaseBuffers, - * DropRelationsAllBuffers). Drop the cache-residency bit + propagate - * master_holder lifecycle (HC110). Uses the saved oldTag because the - * BufferDesc.tag was just cleared above. - */ - if (cluster_pcm_is_active() - && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(oldTag)) - && buf->pcm_state != (uint8) PCM_STATE_N) - { - PcmLockMode old_mode = (PcmLockMode) buf->pcm_state; - - buf->tag = *oldTag; /* temporary restore so the release helper sees - * the right tag — release sets pcm_state to N - * but does not write tag */ - cluster_pcm_lock_release_buffer_for_eviction(buf, old_mode); - ClearBufferTag(&buf->tag); - buf->pcm_state = (uint8) PCM_STATE_N; - /* PGRAC ownership-gen: bump under the held header spinlock so a buf_id - * reuse after this eviction cannot alias a stale captured generation - * (the classic ABA across tag reuse); also clear any lingering markers. */ - cluster_pcm_own_bump_locked(buf, 0, - PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); - } -#endif - /* * Remove the buffer from the lookup hashtable, if it was in there. */ @@ -1899,6 +4034,30 @@ InvalidateBufferCommitLocked(BufferDesc *buf, BufferTag *oldTag, uint32 oldHash, * Done with mapping lock. */ LWLockRelease(oldPartitionLock); +#ifdef USE_PGRAC_CLUSTER + + /* + * The descriptor tag is intentionally already cleared. Releasing by the + * immutable capture avoids the old temporary tag restoration race. On a + * throwing release the buffer is absent from the mapping table but is not + * yet reusable, which is the fail-closed state. */ + if (release_pcm_holder) + { + PG_TRY(); + { + cluster_pcm_lock_release_saved_tag_for_eviction(*oldTag, (PcmLockMode) old_pcm_mode); + } + PG_CATCH(); + { + elog(LOG, + "cluster PCM saved-tag eviction release failed for buffer %d mode=%d; " + "old mapping is removed and descriptor was not returned to the freelist", + buf->buf_id, (int) old_pcm_mode); + PG_RE_THROW(); + } + PG_END_TRY(); + } +#endif /* * Insert the buffer at the head of the list of free buffers. @@ -1940,15 +4099,16 @@ InvalidateBufferTry(BufferDesc *buf) oldPartitionLock = BufMappingPartitionLock(oldHash); #ifdef USE_PGRAC_CLUSTER + /* - * PGRAC ownership-generation wave (W2) — drop-prepin window. The header - * spinlock was released above and the partition lock is not yet taken, so - * this is the only gap where a foreign pin can slip in and fail the - * refcount recheck below (a continuously-held pin is caught by the drop's - * entry gates BEFORE staging N and never reaches this function). The - * sleep inject holds the gap open so the W2 RED can place that pin - * deterministically. Gated to GCS drops only — plain evictions must not - * stall. No lock is held across the sleep. + * PGRAC ownership-generation wave (W2) — drop-prepin window. The + * header spinlock was released above and the partition lock is not yet + * taken, so this is the only gap where a foreign pin can slip in and fail + * the refcount recheck below (a continuously-held pin is caught by the + * drop's entry gates BEFORE staging N and never reaches this function). + * The sleep inject holds the gap open so the W2 RED can place that pin + * deterministically. Gated to GCS drops only — plain evictions must + * not stall. No lock is held across the sleep. */ if (cluster_bufmgr_in_gcs_drop && BufTagGetForkNum(&oldTag) == MAIN_FORKNUM) @@ -1981,8 +4141,7 @@ InvalidateBufferTry(BufferDesc *buf) return false; /* foreign pin — caller parks / fail-closes */ } - InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state); - return true; + return InvalidateBufferCommitLocked(buf, &oldTag, oldHash, oldPartitionLock, buf_state); } /* @@ -2023,6 +4182,18 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); Assert(BufferTagsEqual(&buf_hdr->tag, &tag)); +#ifdef USE_PGRAC_CLUSTER + /* PCM-X retained images deliberately keep BM_VALID and may still have this + * victim pin. The live REVOKING token, not passive refcount, is the reuse + * authority; never let the clock-sweep shortcut bypass D5a. */ + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(buf_hdr, buf_state)) + { + UnlockBufHdr(buf_hdr, buf_state); + LWLockRelease(partition_lock); + return false; + } +#endif + /* * If somebody else pinned the buffer since, or even worse, dirtied it, * give up on this buffer: It's clearly in use. @@ -2047,7 +4218,9 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) ClearBufferTag(&buf_hdr->tag); buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK); #ifdef USE_PGRAC_CLUSTER - /* PGRAC: spec-6.12h D-h3a — victim reuse retags this buffer; reset the + + /* + * PGRAC: spec-6.12h D-h3a — victim reuse retags this buffer; reset the * cluster copy label under the header lock (same stale-BUF_TYPE_PI * containment as InvalidateBuffer above). */ buf_hdr->buffer_type = (uint8) BUF_TYPE_CURRENT; @@ -2057,11 +4230,12 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0); #ifdef USE_PGRAC_CLUSTER + /* * PGRAC: spec-2.35 D4 (HC112) — cache eviction hook for victim buffer. - * LRU selected this buffer + we already confirmed it is reusable. - * Drop the cache-residency bit + propagate master_holder lifecycle - * (HC110) before BufTableDelete completes the eviction. + * LRU selected this buffer + we already confirmed it is reusable. Drop + * the cache-residency bit + propagate master_holder lifecycle (HC110) + * before BufTableDelete completes the eviction. */ if (cluster_pcm_is_active() && cluster_bufmgr_reln_pcm_tracked(BufTagGetRelNumber(&tag)) @@ -2072,7 +4246,9 @@ InvalidateVictimBuffer(BufferDesc *buf_hdr) buf_hdr->tag = tag; /* restore for release helper (cleared above) */ cluster_pcm_lock_release_buffer_for_eviction(buf_hdr, old_mode); ClearBufferTag(&buf_hdr->tag); - /* PGRAC ownership-gen: coherent N-flip + generation bump (the header + + /* + * PGRAC ownership-gen: coherent N-flip + generation bump (the header * spinlock was dropped above at UnlockBufHdr) so a buf_id reuse after * this victim eviction cannot alias a stale captured generation. */ cluster_pcm_own_transition(buf_hdr, (uint8) PCM_STATE_N, 0, @@ -2387,10 +4563,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, { /* * Decide whether to engage the cross-node authority from runtime - * liveness, not the static configured node count (spec-5.7 §3.1d, v1.5 - * amend). HW passes wait_for_lms = true so a DML waits out a brief LMS - * warmup instead of failing. Temp relations are excluded above (always - * PG-local). + * liveness, not the static configured node count (spec-5.7 §3.1d, + * v1.5 amend). HW passes wait_for_lms = true so a DML waits out a + * brief LMS warmup instead of failing. Temp relations are excluded + * above (always PG-local). */ ClusterExtendEngage engage = cluster_extend_liveness_engage(true); @@ -2404,9 +4580,10 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, if (engage == CLUSTER_EXTEND_ENGAGE_COORDINATE) { /* - * A peer is alive and the substrate is ready: a permanent relation is - * GLOBALIZE (authority-owned from block 0); an unlogged relation has no - * WAL authority and cannot be coordinated -> fail closed. + * A peer is alive and the substrate is ready: a permanent + * relation is GLOBALIZE (authority-owned from block 0); an + * unlogged relation has no WAL authority and cannot be + * coordinated -> fail closed. */ hwc = cluster_hw_classify_persistence(bmr.rel->rd_rel->relpersistence, true); if (hwc == CLUSTER_HW_FAIL_CLOSED) @@ -2416,6 +4593,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, RelationGetRelationName(bmr.rel)), errhint("Unlogged relations have no WAL authority to coordinate cross-node extension."))); } + /* * engage == NATIVE: no alive peer to coordinate with -> hwc stays * CLUSTER_HW_NATIVE_LOCAL and the relation extends PG-natively. @@ -2460,14 +4638,16 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * we get the lock. */ #ifdef USE_PGRAC_CLUSTER + /* - * §3.1a M1a lock order: HW(X) (cross-node) BEFORE LockRelationForExtension - * (local), so a backend never holds the local extension lock while waiting - * cross-node. DL (the bulk-load lease, spec-5.7 D4) is already held by the - * caller above when this is a bulk load. HW is held only across the - * allocate; it is a synthetic resid (not auto-released by LockReleaseAll), - * so the brief window relies on reconfig-revoke (AD-013) + backend-exit - * cleanup as the backstop on an error path, mirroring CF (spec-5.6). + * §3.1a M1a lock order: HW(X) (cross-node) BEFORE + * LockRelationForExtension (local), so a backend never holds the local + * extension lock while waiting cross-node. DL (the bulk-load lease, + * spec-5.7 D4) is already held by the caller above when this is a bulk + * load. HW is held only across the allocate; it is a synthetic resid + * (not auto-released by LockReleaseAll), so the brief window relies on + * reconfig-revoke (AD-013) + backend-exit cleanup as the backstop on an + * error path, mirroring CF (spec-5.6). */ if (hwc == CLUSTER_HW_GLOBALIZE) { @@ -2513,25 +4693,27 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, BlockNumber seed_nblocks; /* - * §3.1c Q14/Q17 establishment seed: a FORCED re-stat of the fork's real - * size (invalidate the smgr_cached_nblocks cache first so smgrnblocks - * re-stats the shared file, not a stale cache). We hold HW(X), so the - * size is quiescent (no concurrent authority extend) and -- on cluster_fs - * -- the shared file's EOF is physically coherent (Q14). The master uses - * this ONLY at first establishment, to own a relation whose first blocks - * were created by a non-authority path (private build, sequence create) - * from its true EOF; for an established resid the master ignores it and - * the authority counter is the sole source (FileSize never read again). + * §3.1c Q14/Q17 establishment seed: a FORCED re-stat of the fork's + * real size (invalidate the smgr_cached_nblocks cache first so + * smgrnblocks re-stats the shared file, not a stale cache). We hold + * HW(X), so the size is quiescent (no concurrent authority extend) + * and -- on cluster_fs -- the shared file's EOF is physically + * coherent (Q14). The master uses this ONLY at first establishment, + * to own a relation whose first blocks were created by a + * non-authority path (private build, sequence create) from its true + * EOF; for an established resid the master ignores it and the + * authority counter is the sole source (FileSize never read again). */ bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber; seed_nblocks = smgrnblocks(bmr.smgr, fork); /* - * The block range is the cluster authority's, not the file size. HW(X) - * is released right after the allocate: the granted ranges are disjoint - * across nodes, so smgrzeroextend of distinct ranges is safe without it - * held. The authority may grant fewer blocks than requested; release - * the surplus victim buffers (mirroring the extend_upto path below). + * The block range is the cluster authority's, not the file size. + * HW(X) is released right after the allocate: the granted ranges are + * disjoint across nodes, so smgrzeroextend of distinct ranges is safe + * without it held. The authority may grant fewer blocks than + * requested; release the surplus victim buffers (mirroring the + * extend_upto path below). * * PGRAC: spec-6.12d -- with static space affinity on, a plain-heap * extend over-asks the master (up to cluster.space_lease_blocks) so @@ -2724,6 +4906,22 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, * BM_VALID between our clearing it and StartBufferIO inspecting * it. */ +#ifdef USE_PGRAC_CLUSTER + { + uint32 cluster_state = LockBufHdr(existing_hdr); + bool blocked + = cluster_bufmgr_pcm_x_retained_image_locked(existing_hdr, cluster_state) + || (cluster_pcm_own_flags_get(existing_hdr->buf_id) + & PCM_OWN_FLAG_REVOKING) != 0; + + UnlockBufHdr(existing_hdr, cluster_state); + if (blocked) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_IN_USE), + errmsg("cannot reuse retained cluster PCM image during " + "relation extension"))); + } +#endif do { uint32 buf_state = LockBufHdr(existing_hdr); @@ -2772,6 +4970,7 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, smgrzeroextend(bmr.smgr, fork, first_block, extend_by, false); #ifdef USE_PGRAC_CLUSTER + /* * PGRAC: spec-6.12d -- physically materialize the lease tail of the HW * grant as zero pages (the file MUST stay dense up to the durably @@ -2823,12 +5022,14 @@ ExtendBufferedRelShared(BufferManagerRelation bmr, if (lock) { #ifdef USE_PGRAC_CLUSTER - /* spec-6.14 D8: register PCM X for this LockBuffer-bypassing - * exclusive acquisition -- heap extends land here, and without - * the registration the new block's master stays at N and remote - * readers lawfully storage-read a stale zero page (see - * cluster_bufmgr_pcm_gate_direct_lock). */ - cluster_bufmgr_pcm_gate_direct_lock(buf_hdr); + ClusterPcmDirectInitProof direct_init_proof; + + /* This exact buffer is in the range successfully zeroextended above + * and still owns BM_IO_IN_PROGRESS + !BM_VALID. */ + cluster_bufmgr_pcm_arm_direct_init( + buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, &direct_init_proof); + cluster_bufmgr_pcm_gate_direct_init( + buf_hdr, CLUSTER_PCM_DIRECT_INIT_EXTEND, &direct_init_proof); #endif LWLockAcquire(BufferDescriptorGetContentLock(buf_hdr), LW_EXCLUSIVE); } @@ -2874,6 +5075,25 @@ MarkBufferDirty(Buffer buffer) Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE)); +#ifdef USE_PGRAC_CLUSTER + + /* + * A retained source page is immutable transfer evidence, not a current + * writable copy. Content EXCLUSIVE makes this check stable against the + * retain/release boundary. + */ + buf_state = LockBufHdr(bufHdr); + if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, buf_state)) + { + UnlockBufHdr(bufHdr, buf_state); + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("cannot dirty a retained cluster PCM image"), + errdetail("buffer=%d", bufHdr->buf_id))); + } + UnlockBufHdr(bufHdr, buf_state); +#endif + old_buf_state = pg_atomic_read_u32(&bufHdr->state); for (;;) { @@ -3823,6 +6043,19 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) */ buf_state = LockBufHdr(bufHdr); +#ifdef USE_PGRAC_CLUSTER + + /* + * A live retained image is neither reusable nor writable. Test before + * advertising BUF_REUSABLE; finish/release serialize the later race with + * the content-lock recheck below. */ + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) + { + UnlockBufHdr(bufHdr, buf_state); + return 0; + } +#endif + if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 && BUF_STATE_GET_USAGECOUNT(buf_state) == 0) { @@ -3849,6 +6082,23 @@ SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context) PinBuffer_Locked(bufHdr); LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); +#ifdef USE_PGRAC_CLUSTER + + /* + * If retain won after the optimistic header probe, content SHARE proves + * the transition is now complete and stable. + */ + buf_state = LockBufHdr(bufHdr); + if (cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked(bufHdr, buf_state)) + { + UnlockBufHdr(bufHdr, buf_state); + LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); + UnpinBuffer(bufHdr); + return result & ~BUF_REUSABLE; + } + UnlockBufHdr(bufHdr, buf_state); +#endif + FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); @@ -3880,6 +6130,17 @@ AtEOXact_Buffers(bool isCommit) AtEOXact_LocalBuffers(isCommit); +#ifdef USE_PGRAC_CLUSTER + + /* + * Normal UNLOCK may defer an exact RELEASING holder when RETIRE owns the + * short gate. Transaction tail gets one non-waiting drain opportunity; + * unresolved evidence remains backend-local for the next safe entrance. + */ + cluster_bufmgr_pcm_x_writer_drain_deferred_nowait(); + cluster_bufmgr_pcm_x_holder_drain_deferred_nowait(); +#endif + Assert(PrivateRefCountOverflowed == 0); } @@ -3896,6 +6157,10 @@ InitBufferPoolAccess(void) HASHCTL hash_ctl; memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray)); +#ifdef USE_PGRAC_CLUSTER + cluster_bufmgr_pcm_x_writer_reset(); + cluster_bufmgr_pcm_x_holder_reset(); +#endif hash_ctl.keysize = sizeof(int32); hash_ctl.entrysize = sizeof(PrivateRefCountEntry); @@ -3920,6 +6185,10 @@ AtProcExit_Buffers(int code, Datum arg) { UnlockBuffers(); +#ifdef USE_PGRAC_CLUSTER + cluster_bufmgr_pcm_x_owner_exit_drain(); +#endif + CheckForBufferLeaks(); /* localbuf.c needs a chance too */ @@ -4000,51 +6269,35 @@ AssertNotCatalogBufferLock(LWLock *lock, LWLockMode mode, { BufferDesc *bufHdr; BufferTag tag; - Oid relid; -#ifdef USE_PGRAC_CLUSTER - ptrdiff_t delta; -#endif + Oid relid; if (mode != LW_EXCLUSIVE) return; - if (!((BufferDescPadded *) lock > BufferDescriptors && - (BufferDescPadded *) lock < BufferDescriptors + NBuffers)) - return; /* not a buffer lock */ - -#ifdef USE_PGRAC_CLUSTER /* * PGRAC (stage 1.6 hardening, codex review 2026-05-02 P1/P2 #1): * - * BufferDesc now contains both content_lock (offset 36 on PG 16.13) - * and pcm_lock (offset 104). The address-range check above matches - * both -- pcm_lock is also "in BufferDescriptors range". + * BufferDesc now contains both content_lock (offset 36 on PG 16.13) and + * pcm_lock (offset 104). The address-range check above matches both -- + * pcm_lock is also "in BufferDescriptors range". * - * Without this guard, when Stage 2 真值激活 acquires pcm_lock - * exclusively, this debug helper would interpret it as content_lock - * and reverse-deref using offsetof(content_lock) -- yielding a - * BufferDesc pointer pointing to garbage (pcm_lock_addr - 36 falls - * in the middle of a previous slot's cold-section fields). Reading - * bufHdr->tag would then return random memory, with arbitrary - * consequences ranging from spurious assertion fail to silent - * garbage relid checks against system catalog. + * Without this guard, when Stage 2 真值激活 acquires pcm_lock + * exclusively, this debug helper would interpret it as content_lock and + * reverse-deref using offsetof(content_lock) -- yielding a BufferDesc + * pointer pointing to garbage (pcm_lock_addr - 36 falls in the middle of + * a previous slot's cold-section fields). Reading bufHdr->tag would then + * return random memory, with arbitrary consequences ranging from spurious + * assertion fail to silent garbage relid checks against system catalog. * - * Verify the lock is precisely at content_lock offset within its - * slot; otherwise it's pcm_lock (or a future second embedded - * LWLock) and reverse-deref does not apply. Spec-1.6 5th - * StaticAssertDecl already locks "cluster fields after content_lock" - * semantically; this runtime check is the matching debug-time - * enforcement. + * Use the shared exact reverse-map; otherwise pcm_lock (or a future + * embedded LWLock) could be misidentified as content_lock. * - * Spec: spec-stage1-codex-fixes.md §1.2 Deliverable 5 + spec-1.6 §11 hardening + * Spec: spec-stage1-codex-fixes.md §1.2 Deliverable 5 + spec-1.6 §11 + * hardening */ - delta = (char *) lock - (char *) BufferDescriptors; - if (delta % sizeof(BufferDescPadded) != offsetof(BufferDesc, content_lock)) - return; /* embedded LWLock other than content_lock (e.g. pcm_lock) */ -#endif - - bufHdr = (BufferDesc *) - ((char *) lock - offsetof(BufferDesc, content_lock)); + bufHdr = BufferDescriptorFromContentLock(lock); + if (bufHdr == NULL) + return; /* not an exact shared-buffer content lock */ tag = bufHdr->tag; /* @@ -4196,6 +6449,22 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, char *bufToWrite; uint32 buf_state; +#ifdef USE_PGRAC_CLUSTER + + /* + * Caller holds content SHARE. Retain/release require EXCLUSIVE, so this + * coherent PI+VALID observation cannot change before return. Never even + * enter StartBufferIO for old transfer bytes. + */ + buf_state = LockBufHdr(buf); + if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state)) + { + UnlockBufHdr(buf, buf_state); + return; + } + UnlockBufHdr(buf, buf_state); +#endif + /* * Try to start an I/O operation. If StartBufferIO returns false, then * someone else flushed the buffer before we could, so we need not do @@ -4205,6 +6474,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, return; #ifdef USE_PGRAC_CLUSTER + /* * PGRAC spec-6.2 D7: a Smart Fusion dependent buffer must not be written * to shared storage before every origin dependency is observed durable. @@ -4313,6 +6583,7 @@ FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object, false); #ifdef USE_PGRAC_CLUSTER + /* * PGRAC spec-6.12h D-h2 (Q25-A "写盘成功" face): a cluster-tracked block's * CURRENT copy was just written toward shared storage. Record (tag, @@ -5440,6 +7711,9 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) { BufferDesc *bufHdr; Page page = BufferGetPage(buffer); +#ifdef USE_PGRAC_CLUSTER + uint32 retained_state; +#endif if (!BufferIsValid(buffer)) elog(ERROR, "bad buffer ID: %d", buffer); @@ -5456,6 +7730,22 @@ MarkBufferDirtyHint(Buffer buffer, bool buffer_std) /* here, either share or exclusive lock is OK */ Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr))); +#ifdef USE_PGRAC_CLUSTER + + /* + * Hint changes are optional. A retained image may already have had an + * in-memory hint bit touched by its caller, but it must never regain + * dirty state and become eligible for stale output. + */ + retained_state = LockBufHdr(bufHdr); + if (cluster_bufmgr_pcm_x_retained_image_locked(bufHdr, retained_state)) + { + UnlockBufHdr(bufHdr, retained_state); + return; + } + UnlockBufHdr(bufHdr, retained_state); +#endif + /* * This routine might get called many times on the same page, if we are * making the first scan after commit of an xact that added/deleted many @@ -5601,6 +7891,10 @@ UnlockBuffers(void) PinCountWaitBuf = NULL; } +#ifdef USE_PGRAC_CLUSTER + cluster_bufmgr_pcm_x_writer_exception_cleanup_all(); + cluster_bufmgr_pcm_x_holder_exception_cleanup_all(); +#endif } /* @@ -5620,6 +7914,14 @@ LockBuffer(Buffer buffer, int mode) bool pcm_covered = false; /* took the cached-cover fast path */ uint64 pcm_covered_gen = 0; /* ownership generation captured at cover */ bool pcm_pending_set = false; /* we set GRANT_PENDING (W3), must clear */ + MemoryContext pcm_error_context = CurrentMemoryContext; + ClusterPcmOwnSnapshot pcm_pending_base; + ClusterPcmOwnResult pcm_pending_result = CLUSTER_PCM_OWN_OK; + uint64 pcm_pending_token = 0; + uint64 pcm_committed_generation = 0; + ClusterPcmXHolderLedgerEntry *pcm_x_holder = NULL; + ClusterPcmXWriterLedgerEntry *pcm_x_writer = NULL; + bool pcm_x_writer_managed = false; #endif Assert(BufferIsPinned(buffer)); @@ -5638,126 +7940,190 @@ LockBuffer(Buffer buffer, int mode) cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(buf)) { + uint8 pcm_initial_state = (uint8)PCM_STATE_N; + uint32 pcm_initial_flags = 0; + pcm_mode = (mode == BUFFER_LOCK_SHARE) ? PCM_LOCK_MODE_S : PCM_LOCK_MODE_X; - /* - * PGRAC: spec-4.7a D2 — hold-until-revoked acquire gate. - * - * If this node already holds a PCM mode that covers the requested - * mode (X ⊇ {S,X}, S ⊇ S), skip the remote master round-trip - * entirely. buf->pcm_state is the node-level cache: it is finalized - * after a successful acquire (below) and cleared by the INVALIDATE - * handler + eviction/drop paths, so a covering value means the node - * still genuinely holds the lock (Rule 8.A — no stale grant). This - * is what stops a bulk single-node workload from issuing one PCM - * round-trip per LockBuffer (the spec-4.7a D0 request storm that - * floods the dedup HTAB → DENIED_DEDUP_FULL → 53R90). - * - * spec-2.33 D7 — when an acquire IS needed, the buffer-aware entry - * point lets the GCS data-plane sender install received block bytes - * directly into this BufferDesc on GRANTED (HC84 PageSetLSN + memcpy - * under content_lock EXCLUSIVE). - */ - if (cluster_gcs_block_local_cache && - cluster_pcm_mode_covers((PcmLockMode) buf->pcm_state, pcm_mode)) - { - /* Already covered locally — no master round-trip, nothing to - * finalize or roll back (pcm_acquired stays false). The cover - * decision was made on the raw pcm_state read; a concurrent BAST - * downgrade (X->S) can still fire before we take the content lock - * below, so this cover must be RE-VERIFIED after the content lock - * (ownership-generation P0). */ - pcm_covered = true; - /* Capture the ownership generation NOW (header spinlock), so the - * post-content-lock re-verify can detect any ownership round that - * raced the content-lock window (a BAST X->S, or an N->X->N ABA) - * even when pcm_state looks unchanged (ownership-generation P0). */ - cluster_pcm_own_read(buf, NULL, &pcm_covered_gen, NULL); - - /* PGRAC: spec-5.59 §3.6 read amortization probe — a share - * acquire served entirely from locally-held PCM state is the - * "S-holder hit" the probe counts (GUC-gated inside). */ - if (pcm_mode == PCM_LOCK_MODE_S) - cluster_xp_note_read(false); + + /* A queue grant leaves node X cached until revoke. Preserve that + * authority before deciding that this LockBuffer is a new writer + * conversion; otherwise every local write needlessly retires and + * re-enqueues the same node X. The post-content-lock generation + * check below remains the race-closing authority. */ + if (pcm_mode == PCM_LOCK_MODE_X && cluster_gcs_block_local_cache) { + cluster_pcm_own_read(buf, &pcm_initial_state, &pcm_covered_gen, + &pcm_initial_flags); + pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(cluster_gcs_block_local_cache, + pcm_mode == PCM_LOCK_MODE_X, pcm_initial_state, pcm_initial_flags); } - else - { - /* PGRAC ownership-generation wave (W3): mark GRANT_PENDING before - * the request goes out. Between the install (inside acquire, done - * under its own content lock) and this backend's finalize below, - * pcm_state is still N; a same-tag INVALIDATE arriving in that - * window must NOT treat N as already-invalidated and ACK away this - * in-flight grant (which would strand a stale X after finalize). - * The invalidate handler parks/denies while PENDING is set. */ - cluster_pcm_own_set_flags(buf, PCM_OWN_FLAG_GRANT_PENDING, 0); - pcm_pending_set = true; - - /* PGRAC: spec-5.2 D2 — a one-shot READ_IMAGE returns false (no - * durable grant); pcm_acquired stays false so the ownership - * mirror below is skipped and buf->pcm_state is left at N (the - * next access re-fetches — a cached copy with no invalidation - * path would go stale, Rule 8.A). */ - PG_TRY(); - { - pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); - } - PG_CATCH(); - { + + if (!pcm_covered) + pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode); + if (pcm_x_writer == NULL && !pcm_covered) { + /* + * PGRAC: spec-4.7a D2 — hold-until-revoked acquire gate. + * + * If this node already holds a PCM mode that covers the requested + * mode (X ⊇ {S,X}, S ⊇ S), skip the remote master round-trip + * entirely. buf->pcm_state is the node-level cache: it is + * finalized after a successful acquire (below) and cleared by the + * INVALIDATE handler + eviction/drop paths, so a covering value + * means the node still genuinely holds the lock (Rule 8.A — no + * stale grant). This is what stops a bulk single-node workload + * from issuing one PCM round-trip per LockBuffer (the spec-4.7a + * D0 request storm that floods the dedup HTAB → + * DENIED_DEDUP_FULL → 53R90). + * + * spec-2.33 D7 — when an acquire IS needed, the buffer-aware + * entry point lets the GCS data-plane sender install received + * block bytes directly into this BufferDesc on GRANTED (HC84 + * PageSetLSN + memcpy under content_lock EXCLUSIVE). + */ + if (cluster_gcs_block_local_cache + && cluster_pcm_mode_covers((PcmLockMode)buf->pcm_state, pcm_mode)) { + /* + * Already covered locally — no master round-trip, nothing + * to finalize or roll back (pcm_acquired stays false). The + * cover decision was made on the raw pcm_state read; a + * concurrent BAST downgrade (X->S) can still fire before we + * take the content lock below, so this cover must be + * RE-VERIFIED after the content lock (ownership-generation + * P0). + */ + pcm_covered = true; + + /* + * Capture the ownership generation NOW (header spinlock), so + * the post-content-lock re-verify can detect any ownership + * round that raced the content-lock window (a BAST X->S, or + * an N->X->N ABA) even when pcm_state looks unchanged + * (ownership-generation P0). + */ + cluster_pcm_own_read(buf, NULL, &pcm_covered_gen, NULL); + + /* + * PGRAC: spec-5.59 §3.6 read amortization probe — a share + * acquire served entirely from locally-held PCM state is the + * "S-holder hit" the probe counts (GUC-gated inside). + */ + if (pcm_mode == PCM_LOCK_MODE_S) + cluster_xp_note_read(false); + } else { + /* + * PGRAC ownership-generation wave (W3): mark GRANT_PENDING + * before the request goes out. Between the install (inside + * acquire, done under its own content lock) and this + * backend's finalize below, pcm_state is still N; a same-tag + * INVALIDATE arriving in that window must NOT treat N as + * already-invalidated and ACK away this in-flight grant + * (which would strand a stale X after finalize). The + * invalidate handler parks/denies while PENDING is set. + */ + /* + * Legacy acquire path: token-exact, but still pre-request. + * The convert-queue path must bypass this point during + * JOIN/WAIT and call begin_x_reservation only from + * ACTIVE_TRANSFER/PREPARE. + */ + pcm_pending_result = cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base, + &pcm_pending_token); + if (pcm_pending_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_own_report_bump_failure( + buf, pcm_pending_result, pcm_pending_base.generation, + pcm_pending_base.flags, "LockBuffer master reservation"); + pcm_pending_set = true; + /* - * W3 — an acquire that THREW (upgrade-invalidate timeout, - * transfer deny, ...) must not leak GRANT_PENDING: a stale - * marker parks every later same-tag INVALIDATE on this node - * (never ACKed -> remote upgrades wedge, liveness). The - * later PG_CATCH below only covers the content-lock window, - * not this acquire. + * PGRAC: spec-5.2 D2 — a one-shot READ_IMAGE returns false + * (no durable grant); pcm_acquired stays false so the + * ownership mirror below is skipped and buf->pcm_state is + * left at N (the next access re-fetches — a cached copy + * with no invalidation path would go stale, Rule 8.A). */ - cluster_pcm_own_set_flags(buf, 0, PCM_OWN_FLAG_GRANT_PENDING); - PG_RE_THROW(); + PG_TRY(); + { + pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); + } + PG_CATCH(); + { + /* + * W3 — an acquire that THREW (upgrade-invalidate + * timeout, transfer deny, ...) must not leak + * GRANT_PENDING: a stale marker parks every later + * same-tag INVALIDATE on this node (never ACKed -> remote + * upgrades wedge, liveness). The later PG_CATCH below + * only covers the content-lock window, not this acquire. + */ + cluster_pcm_own_abort_grant_after_error( + buf, &pcm_pending_base, pcm_pending_token, "LockBuffer master acquire"); + PG_RE_THROW(); + } + PG_END_TRY(); } - PG_END_TRY(); } } #endif if (mode == BUFFER_LOCK_UNLOCK) + { +#ifdef USE_PGRAC_CLUSTER + pcm_x_writer = cluster_bufmgr_pcm_x_writer_find(buf); + pcm_x_writer_managed = pcm_x_writer != NULL; + pcm_x_holder = cluster_bufmgr_pcm_x_holder_find(buf); + cluster_bufmgr_pcm_x_writer_mark_releasing(pcm_x_writer); + cluster_bufmgr_pcm_x_holder_mark_releasing(pcm_x_holder); +#endif LWLockRelease(BufferDescriptorGetContentLock(buf)); +#ifdef USE_PGRAC_CLUSTER + cluster_bufmgr_pcm_x_writer_release(pcm_x_writer); + cluster_bufmgr_pcm_x_holder_unregister(pcm_x_holder); +#endif + } else { #ifdef USE_PGRAC_CLUSTER /* - * GCS serve-stall round-6 (ownership P0) — cached-X BAST window. When - * the cover fast path above decided we already hold X, a concurrent - * BAST X->S self-downgrade can still grab the content lock and flip - * pcm_state to S in the window before we acquire it. This inject holds - * that window open so the RED can drive the downgrade deterministically; - * the post-content-lock re-verify below is what closes it. + * GCS serve-stall round-6 (ownership P0) — cached-X BAST window. + * When the cover fast path above decided we already hold X, a + * concurrent BAST X->S self-downgrade can still grab the content lock + * and flip pcm_state to S in the window before we acquire it. This + * inject holds that window open so the RED can drive the downgrade + * deterministically; the post-content-lock re-verify below is what + * closes it. + */ + /* + * MAIN-fork gate: the visibilitymap_pin X prefetch also runs a + * covered-X LockBuffer on the VM page and would otherwise consume the + * stall ahead of the heap block under test. */ - /* MAIN-fork gate: the visibilitymap_pin X prefetch also runs a - * covered-X LockBuffer on the VM page and would otherwise consume - * the stall ahead of the heap block under test. */ if (pcm_covered && pcm_mode == PCM_LOCK_MODE_X && BufTagGetForkNum(&buf->tag) == MAIN_FORKNUM) CLUSTER_INJECTION_POINT("cluster-pcm-writer-cached-x-stall"); PG_TRY(); { + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf); #endif if (mode == BUFFER_LOCK_SHARE) LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); else LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); #ifdef USE_PGRAC_CLUSTER + cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer); + /* - * PGRAC ownership-generation wave (W1) — cached-cover re-verify. - * The cover fast path decided we already held the mode on a raw, - * unlocked pcm_state read. A BAST X->S downgrade (or any ownership - * round) can have raced this content-lock window and revoked the - * cover. Re-read the coherent (pcm_state, generation, flags) triple - * under the header spinlock: if the generation moved, the mode no - * longer covers, or a grant/revoke is in flight, the cover is STALE - * -- writing/reading a block we no longer own is the Rule 8.A + * PGRAC ownership-generation wave (W1) — cached-cover + * re-verify. The cover fast path decided we already held the mode + * on a raw, unlocked pcm_state read. A BAST X->S downgrade (or + * any ownership round) can have raced this content-lock window + * and revoked the cover. Re-read the state/generation/flags + * projection of the coherent ownership tuple under the header + * spinlock: if the generation moved, the mode no longer covers, + * or a grant/revoke is in flight, the cover is STALE -- + * writing/reading a block we no longer own is the Rule 8.A * violation. Release, do a real master re-acquire, re-take the - * content lock. Bounded: after a real acquire we hold the content - * lock and the downgrade path serializes under it, so no further - * downgrade can intervene -- at most one fallback. + * content lock. Bounded: after a real acquire we hold the + * content lock and the downgrade path serializes under it, so no + * further downgrade can intervene -- at most one fallback. */ if (pcm_covered) { @@ -5773,9 +8139,15 @@ LockBuffer(Buffer buffer, int mode) cluster_pcm_note_writer_cover_stale_detected(); pcm_covered = false; LWLockRelease(BufferDescriptorGetContentLock(buf)); - cluster_pcm_own_set_flags(buf, PCM_OWN_FLAG_GRANT_PENDING, 0); + pcm_pending_result = cluster_pcm_own_begin_grant_reservation( + buf, &pcm_pending_base, &pcm_pending_token); + if (pcm_pending_result != CLUSTER_PCM_OWN_OK) + cluster_pcm_own_report_bump_failure( + buf, pcm_pending_result, pcm_pending_base.generation, + pcm_pending_base.flags, "LockBuffer revalidate reservation"); pcm_pending_set = true; pcm_acquired = cluster_pcm_lock_acquire_buffer(buf, pcm_mode); + pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf); if (mode == BUFFER_LOCK_SHARE) LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED); else @@ -5783,48 +8155,74 @@ LockBuffer(Buffer buffer, int mode) cluster_pcm_note_writer_reverify_reacquire(); } } + cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder); } PG_CATCH(); { - if (pcm_acquired) - { - /* PGRAC: spec-2.35 D4 (HC112) — acquire-then-LWLock-fail + ErrorData *original_error; + + /* + * Holder detach is itself allowed to encounter an LWLock ERROR. + * Preserve the content-lock failure outside ErrorContext and + * flush it before any cleanup begins; nested cleanup may then + * discard only its own error without destroying the user-visible + * original. + */ + MemoryContextSwitchTo(pcm_error_context); + original_error = CopyErrorData(); + FlushErrorState(); + if (pcm_x_writer != NULL && !LWLockHeldByMe(BufferDescriptorGetContentLock(buf))) + cluster_bufmgr_pcm_x_writer_abort_acquiring(pcm_x_writer); + if (pcm_x_holder != NULL && !LWLockHeldByMe(BufferDescriptorGetContentLock(buf))) + cluster_bufmgr_pcm_x_holder_abort_acquiring(pcm_x_holder); + if (pcm_acquired) { + /* + * PGRAC: spec-2.35 D4 (HC112) — acquire-then-LWLock-fail * rollback path: PCM acquire succeeded but LWLockAcquire * threw. Roll back the cache-residency claim using the * eviction release (it fully clears the bitmap bit so the - * partial acquire does not leak a stale holder). */ - cluster_pcm_lock_release_buffer_for_eviction(buf, pcm_mode); + * partial acquire does not leak a stale holder). + */ + cluster_pcm_own_rollback_grant_after_error_and_rethrow( + buf, &pcm_pending_base, pcm_pending_token, pcm_mode, pcm_pending_set, + original_error, pcm_error_context); } - /* W3: an acquire that threw leaves GRANT_PENDING set -- clear it so - * a later invalidate is not blocked by a phantom in-flight grant. */ - if (pcm_pending_set) - cluster_pcm_own_set_flags(buf, 0, PCM_OWN_FLAG_GRANT_PENDING); - PG_RE_THROW(); + + /* + * W3: an error before a durable acquire leaves GRANT_PENDING set + * -- clear it exactly so a later invalidate is not blocked by a + * phantom in-flight grant. + */ + else if (pcm_pending_set) + cluster_pcm_own_abort_grant_after_error(buf, &pcm_pending_base, pcm_pending_token, + "LockBuffer content-lock acquire"); + ReThrowError(original_error); } PG_END_TRY(); /* - * PGRAC ownership-generation wave (W3) — grant-finalize window. A real - * X acquire has installed the grant but pcm_state is still N with - * GRANT_PENDING set (finalize below flips it to X). This inject holds - * that window open so the RED can land a peer INVALIDATE and prove the - * GRANT_PENDING consult parks it instead of acking the grant away. + * PGRAC ownership-generation wave (W3) — grant-finalize window. A + * real X acquire has installed the grant but pcm_state is still N + * with GRANT_PENDING set (finalize below flips it to X). This inject + * holds that window open so the RED can land a peer INVALIDATE and + * prove the GRANT_PENDING consult parks it instead of acking the + * grant away. */ if (pcm_acquired && pcm_pending_set && pcm_mode == PCM_LOCK_MODE_X) { CLUSTER_INJECTION_POINT("cluster-pcm-grant-finalize-window"); /* - * W3 RED delivery — when armed (:skip, one-shot), drive the REAL - * invalidate handler with a synthetic same-tag directive right - * here, while pcm_state is still N and GRANT_PENDING is set. A - * master INVALIDATE cannot be steered into this window from SQL - * (it targets S-holders; a mirror-N node is the X-grantee and is - * served by X-forward instead — the real producers are - * master/mirror asymmetry races, e.g. a deferred eviction - * release). The shim must observe the park (return false + - * parked counter); an already_invalidated ACK here is the W3 - * defect. + * W3 RED delivery — when armed (:skip, one-shot), drive the + * REAL invalidate handler with a synthetic same-tag directive + * right here, while pcm_state is still N and GRANT_PENDING is + * set. A master INVALIDATE cannot be steered into this window + * from SQL (it targets S-holders; a mirror-N node is the + * X-grantee and is served by X-forward instead — the real + * producers are master/mirror asymmetry races, e.g. a deferred + * eviction release). The shim must observe the park (return + * false + parked counter); an already_invalidated ACK here is the + * W3 defect. */ CLUSTER_INJECTION_POINT("cluster-pcm-grant-finalize-deliver-invalidate"); if (cluster_injection_should_skip( @@ -5840,27 +8238,29 @@ LockBuffer(Buffer buffer, int mode) { /* * Finalize the grant under the header spinlock: set buffer_type + - * pcm_state, bump the ownership generation, and clear GRANT_PENDING - * atomically. Consumers that captured the generation before this - * point now observe the change (ownership-generation P0). + * pcm_state, bump the ownership generation, and clear + * GRANT_PENDING atomically. Consumers that captured the + * generation before this point now observe the change + * (ownership-generation P0). */ - buf->buffer_type = (pcm_mode == PCM_LOCK_MODE_S) ? (uint8) BUF_TYPE_SCUR - : (uint8) BUF_TYPE_XCUR; - cluster_pcm_own_transition(buf, - (pcm_mode == PCM_LOCK_MODE_S) ? (uint8) PCM_STATE_S - : (uint8) PCM_STATE_X, - 0, PCM_OWN_FLAG_GRANT_PENDING); + cluster_pcm_own_finish_grant_or_rollback( + buf, &pcm_pending_base, pcm_pending_token, + (pcm_mode == PCM_LOCK_MODE_S) ? (uint8) PCM_STATE_S : (uint8) PCM_STATE_X, + pcm_mode, + &pcm_committed_generation); } else if (pcm_pending_set) { /* No durable grant (one-shot READ_IMAGE): clear the PENDING marker * we set before the acquire so it does not linger. */ - cluster_pcm_own_set_flags(buf, 0, PCM_OWN_FLAG_GRANT_PENDING); + cluster_pcm_own_abort_grant_or_error(buf, &pcm_pending_base, pcm_pending_token, + "LockBuffer read-image"); } #endif } #ifdef USE_PGRAC_CLUSTER + /* * PGRAC: spec-5.2 §3.5 D11 — the deferred-writer read-image marker * (PCM_STATE_READ_IMAGE) is transient: it lives only for the @@ -5889,25 +8289,89 @@ LockBuffer(Buffer buffer, int mode) PcmLockMode old_mode = (PcmLockMode) buf->pcm_state; /* - * PGRAC: spec-4.7a D2 — hold-until-revoked. With the node-level cache - * on, the PCM lock (S residency per HC111, AND X per spec-4.7a) is kept - * across content-lock unlock; buf->pcm_state is preserved so the next - * acquire is served locally with no master round-trip. The lock and - * buf->pcm_state are cleared together by the INVALIDATE handler and the - * eviction/drop hooks (Rule 8.A — no stale grant). With the cache off, - * fall back to the spec-2.33/2.35 behavior: release X here and mirror - * the queried state (X → N, S residency preserved). + * PGRAC: spec-4.7a D2 — hold-until-revoked. With the node-level + * cache on, the PCM lock (S residency per HC111, AND X per spec-4.7a) + * is kept across content-lock unlock; buf->pcm_state is preserved so + * the next acquire is served locally with no master round-trip. The + * lock and buf->pcm_state are cleared together by the INVALIDATE + * handler and the eviction/drop hooks (Rule 8.A — no stale grant). + * With the cache off, a legacy writer falls back to the + * spec-2.33/2.35 behavior: release X here and mirror the queried + * state (X → N, S residency preserved). A queue-managed writer + * must retain node X; DRAIN/RETIRE owns that release after every + * same-round FIFO successor records WRITER_COMPLETE. */ - if (!cluster_gcs_block_local_cache) - { + if (cluster_pcm_x_should_release_legacy_on_unlock(cluster_gcs_block_local_cache, + pcm_x_writer_managed)) { cluster_pcm_lock_unlock_content_buffer(buf, old_mode); - /* PGRAC ownership-gen: coherent state mirror + generation bump for - * the cache-off X-release (X -> queried state). */ - cluster_pcm_own_transition(buf, - (uint8) cluster_pcm_lock_query(buf->tag), - 0, 0); + + /* + * PGRAC ownership-gen: coherent state mirror + generation bump + * for the cache-off X-release (X -> queried state). + */ + cluster_pcm_own_transition(buf, (uint8)cluster_pcm_lock_query(buf->tag), 0, 0); + } + } +#endif +} + +/* VM/FSM pages read with RBM_ZERO_ON_ERROR become BM_VALID before their + * PageIsNew initialization. Dedicated wrappers are the provenance: a plain + * LockBuffer(X) on the same N buffer remains queue-only. */ +#ifdef USE_PGRAC_CLUSTER +static void +LockBufferForAuxiliaryPageInit(Buffer buffer, ClusterPcmDirectInitKind kind) +{ + BufferDesc *buf; + + Assert(BufferIsPinned(buffer)); + Assert(kind == CLUSTER_PCM_DIRECT_INIT_VM || kind == CLUSTER_PCM_DIRECT_INIT_FSM); + if (BufferIsLocal(buffer)) + return; + + buf = GetBufferDescriptor(buffer - 1); + if (cluster_pcm_is_active() && cluster_bufmgr_should_pcm_track(buf)) + { + ClusterPcmDirectInitProof direct_init_proof; + uint8 pcm_state; + uint32 flags; + + /* + * A concurrent initializer may already have established X while the + * caller's unlocked PageIsNew probe was true. The ordinary cached-X + * path revalidates that grant after taking the content lock. + */ + cluster_pcm_own_read(buf, &pcm_state, NULL, &flags); + if (pcm_state == (uint8) PCM_STATE_X && flags == 0) + { + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); + return; } + + cluster_bufmgr_pcm_arm_direct_init(buf, kind, &direct_init_proof); + cluster_bufmgr_pcm_gate_direct_init(buf, kind, &direct_init_proof); } + LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); +} +#endif + +void +LockBufferForVisibilityMapPageInit(Buffer buffer) +{ +#ifdef USE_PGRAC_CLUSTER + LockBufferForAuxiliaryPageInit(buffer, CLUSTER_PCM_DIRECT_INIT_VM); +#else + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); +#endif +} + +void +LockBufferForFreeSpaceMapPageInit(Buffer buffer) +{ +#ifdef USE_PGRAC_CLUSTER + LockBufferForAuxiliaryPageInit(buffer, CLUSTER_PCM_DIRECT_INIT_FSM); +#else + LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); #endif } @@ -5918,18 +8382,24 @@ LockBuffer(Buffer buffer, int mode) * * PGRAC ownership-generation audit — this is NOT a PCM bypass entrance for the * ownership-generation wave: it acquires only the content lock and never reads - * or mutates the (pcm_state, ownership generation, flags) triple, so it cannot - * create a stale-generation TOCTOU (W1) nor an install/drop window (W2/W3). It - * deliberately does NOT register a PCM X grant: a caller that then WRITES the - * page is backstopped by cluster_bufmgr_block_write_permitted() at write time - * (which fails closed on a non-X pcm_state) regardless of how the content lock - * was obtained, and adding a (blocking) PCM acquire here would violate the - * don't-wait contract. Out-of-window by construction. + * or mutates the (pcm_state, generation, reservation_token, flags) tuple, so + * it cannot create a stale-generation TOCTOU (W1) nor an install/drop window + * (W2/W3). It deliberately does NOT register a PCM X grant: a caller that + * then WRITES the page is backstopped by + * cluster_bufmgr_block_write_permitted() at write time. The post-acquire + * reservation check below additionally refuses protocol-owned bytes without + * adding a blocking PCM round trip, which would violate this API's don't-wait + * contract. */ bool ConditionalLockBuffer(Buffer buffer) { BufferDesc *buf; + bool acquired; +#ifdef USE_PGRAC_CLUSTER + bool blocked; + uint32 buf_state; +#endif Assert(BufferIsPinned(buffer)); if (BufferIsLocal(buffer)) @@ -5937,8 +8407,28 @@ ConditionalLockBuffer(Buffer buffer) buf = GetBufferDescriptor(buffer - 1); - return LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf), - LW_EXCLUSIVE); + acquired = LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf), + LW_EXCLUSIVE); +#ifdef USE_PGRAC_CLUSTER + if (acquired) + { + /* This API bypasses LockBuffer/W1. Recreate its reservation boundary + * after acquiring content authority: an already-running user wins and + * serializes before revoke finish; a user arriving after REVOKING or + * GRANT_PENDING must not modify protocol-owned bytes. + */ + buf_state = LockBufHdr(buf); + blocked = cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) + || cluster_pcm_own_flags_get(buf->buf_id) != 0; + UnlockBufHdr(buf, buf_state); + if (blocked) + { + LWLockRelease(BufferDescriptorGetContentLock(buf)); + return false; + } + } +#endif + return acquired; } /* @@ -6861,9 +9351,9 @@ TestForOldSnapshot_impl(Snapshot snapshot, Relation relation) * ============================================================================ */ -#include "access/xlog.h" /* XLogFlush */ -#include "cluster/cluster_gcs.h" /* spec-6.12a ㉕ — downgrade notify (nowait) */ -#include "cluster/cluster_gcs_block.h" +#include "access/xlog.h" /* XLogFlush */ +#include "cluster/cluster_gcs.h" /* spec-6.12a ㉕ — downgrade notify + * (nowait) */ /* * cluster_gcs_clamp_ship_flush_lsn -- spec-6.14 D11, the ship-path sibling of @@ -6985,7 +9475,8 @@ cluster_bufmgr_probe_block_for_gcs(BufferTag tag) uint32 buf_state; buf_state = LockBufHdr(buf); - valid = ((buf_state & BM_VALID) != 0) && BufferTagsEqual(&buf->tag, &tag); + valid = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); UnlockBufHdr(buf, buf_state); } LWLockRelease(partition_lock); @@ -7081,7 +9572,7 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char LWLockRelease(partition_lock); return false; } - if ((buf_state & BM_VALID) == 0) + if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -7095,10 +9586,9 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char page = (Page) BufHdrGetBlock(buf); /* - * HC89: revalidation loop with single retry budget. - * retries = 0 — first attempt - * retries = 1 — single retry permitted - * retries = 2 — give up, fail-closed + * HC89: revalidation loop with single retry budget. retries = 0 — first + * attempt retries = 1 — single retry permitted retries = 2 — give up, + * fail-closed */ stable = false; for (retries = 0; retries < 2; retries++) @@ -7109,10 +9599,10 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char LWLockRelease(content_lock); /* - * HC82: flush WAL up to the page LSN before shipping the bytes. - * Use XLogFlush(page_lsn) specifically — NOT XLogFlush(insert - * pointer) which would be correct but over-flushes and doesn't - * express the "flush this page before ship" safety contract. + * HC82: flush WAL up to the page LSN before shipping the bytes. Use + * XLogFlush(page_lsn) specifically — NOT XLogFlush(insert pointer) + * which would be correct but over-flushes and doesn't express the + * "flush this page before ship" safety contract. */ #ifdef USE_CLUSTER_UNIT if (cluster_gcs_block_test_xlog_flush_hook != NULL) @@ -7122,19 +9612,32 @@ cluster_bufmgr_copy_block_for_gcs(BufferTag tag, XLogRecPtr *out_page_lsn, char XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); /* - * Reacquire content_lock SHARED and revalidate that the page LSN - * has not advanced past first_lsn AND the buffer tag still matches. + * Reacquire content_lock SHARED and revalidate that the page LSN has + * not advanced past first_lsn AND the buffer tag still matches. * Either signals concurrent mutation that would break HC82's "ship * the bytes that I just flushed WAL for" contract. */ LWLockAcquire(content_lock, LW_SHARED); + { + uint32 buf_state = LockBufHdr(buf); + bool current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + + UnlockBufHdr(buf, buf_state); + if (!current) + { + LWLockRelease(content_lock); + break; + } + } second_lsn = PageGetLSN(page); #ifdef USE_CLUSTER_UNIT + /* - * Spec L25/L26 hook: if the test injection returns N > 0 we mimic - * N consecutive LSN drift events. retries 0 + drift available - * means second_lsn != first_lsn synthetically. + * Spec L25/L26 hook: if the test injection returns N > 0 we mimic N + * consecutive LSN drift events. retries 0 + drift available means + * second_lsn != first_lsn synthetically. */ if (cluster_gcs_block_test_lsn_drift_hook != NULL) { @@ -7210,7 +9713,8 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page uint32 buf_state; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + if (!BufferTagsEqual(&buf->tag, &tag) + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -7237,6 +9741,18 @@ cluster_bufmgr_borrow_block_for_gcs_live_sge(BufferTag tag, XLogRecPtr *out_page XLogFlush(cluster_gcs_clamp_ship_flush_lsn(first_lsn)); LWLockAcquire(content_lock, LW_SHARED); + { + uint32 buf_state = LockBufHdr(buf); + bool current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + + UnlockBufHdr(buf, buf_state); + if (!current) + { + LWLockRelease(content_lock); + break; + } + } second_lsn = PageGetLSN(page); #ifdef USE_CLUSTER_UNIT @@ -7399,6 +9915,12 @@ cluster_bufmgr_downgrade_x_to_s_for_gcs(BufferTag tag) content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) + { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return false; + } /* * Re-verify under the content lock: still our tag, still PCM X, and @@ -7472,6 +9994,7 @@ cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forkn { BufferTag tag; uint32 hashcode; + uint32 buf_state; LWLock *partition_lock; int buf_id; BufferDesc *buf; @@ -7493,33 +10016,45 @@ cluster_bufmgr_lock_resident_for_stamp(RelFileLocator rlocator, ForkNumber forkn } buf = GetBufferDescriptor(buf_id); - /* Pin under the header spinlock, then drop the partition lock (the pin + /* + * Pin under the header spinlock, then drop the partition lock (the pin * keeps the identity stable), and take the content lock EXCLUSIVE for the - * GenericXLog stamp mutation. */ + * GenericXLog stamp mutation. + */ + /* - * PGRAC: spec-6.12g bugfix — PinBuffer (NOT PinBuffer_Locked) because the - * committing backend may ALREADY hold a pin on this block (a same-txn DML - * that has not yet released it). PinBuffer_Locked asserts a first-time - * pin (Assert(GetPrivateRefCountEntry(...) == NULL)) and crashes the whole - * postmaster under concurrent write load; PinBuffer handles the - * already-pinned case (bumps the existing private refcount). The standard - * buffer-hit path (BufferAlloc) pins the same way while holding the - * partition lock, so this is a legal call sequence. + * PGRAC: spec-6.12g bugfix — PinBuffer (NOT PinBuffer_Locked) because + * the committing backend may ALREADY hold a pin on this block (a same-txn + * DML that has not yet released it). PinBuffer_Locked asserts a + * first-time pin (Assert(GetPrivateRefCountEntry(...) == NULL)) and + * crashes the whole postmaster under concurrent write load; PinBuffer + * handles the already-pinned case (bumps the existing private refcount). + * The standard buffer-hit path (BufferAlloc) pins the same way while + * holding the partition lock, so this is a legal call sequence. */ (void)PinBuffer(buf, NULL); LWLockRelease(partition_lock); LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE); - /* Re-validate under the content lock: a concurrent transfer could have - * evicted + reused this descriptor between the pin and the lock. */ + /* + * Re-validate under the content lock: a concurrent transfer could have + * evicted + reused this descriptor between the pin and the lock. A + * retained/revoking source is still resident by design but is no longer a + * legal commit-stamp target. + */ + buf_state = LockBufHdr(buf); if (!BufferTagsEqual(&buf->tag, &tag) - || (pg_atomic_read_u32(&buf->state) & BM_VALID) == 0) + || (buf_state & BM_VALID) == 0 + || cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) + || (cluster_pcm_own_flags_get(buf->buf_id) & PCM_OWN_FLAG_REVOKING) != 0) { + UnlockBufHdr(buf, buf_state); LWLockRelease(BufferDescriptorGetContentLock(buf)); ReleaseBuffer(BufferDescriptorGetBuffer(buf)); return InvalidBuffer; } + UnlockBufHdr(buf, buf_state); return BufferDescriptorGetBuffer(buf); } @@ -7604,6 +10139,12 @@ cluster_bufmgr_downgrade_x_to_s_remote_for_gcs(BufferTag tag, int32 master_node) content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); + if (!cluster_bufmgr_pcm_x_content_write_permitted(buf)) + { + LWLockRelease(content_lock); + cluster_bufmgr_unpin_for_gcs(buf); + return false; + } /* Same re-verify as the local variant: tag, PCM X, quiescent. */ if (!BufferTagsEqual(&buf->tag, &tag) @@ -7751,7 +10292,8 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa uint32 buf_state; buf_state = LockBufHdr(buf); - if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0) + if (!BufferTagsEqual(&buf->tag, &tag) + || !cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) { UnlockBufHdr(buf, buf_state); LWLockRelease(partition_lock); @@ -7768,6 +10310,18 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa for (retries = 0; retries < 2; retries++) { LWLockAcquire(content_lock, LW_SHARED); + { + uint32 buf_state = LockBufHdr(buf); + bool current = BufferTagsEqual(&buf->tag, &tag) + && cluster_bufmgr_pcm_current_image_locked(buf, buf_state); + + UnlockBufHdr(buf, buf_state); + if (!current) + { + LWLockRelease(content_lock); + break; + } + } first_lsn = PageGetLSN(page); memcpy(dst, page, BLCKSZ); second_lsn = PageGetLSN(page); @@ -7792,9 +10346,9 @@ cluster_bufmgr_copy_block_for_gcs_smart_fusion(BufferTag tag, XLogRecPtr *out_pa /* * Keep lock ordering compatible with receiver install: * cluster_sf_dep_install_vec() takes the Smart Fusion dep lock before - * page install may take content_lock EXCLUSIVE. We therefore read the - * dep store after releasing content_lock, while the raw pin keeps this - * descriptor/tag from being recycled. + * page install may take content_lock EXCLUSIVE. We therefore read + * the dep store after releasing content_lock, while the raw pin keeps + * this descriptor/tag from being recycled. */ has_existing_deps = cluster_sf_dep_vec_for_ship(BufferDescriptorGetBuffer(buf), &existing_vec); @@ -8012,17 +10566,17 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode XLogFlush(cluster_gcs_clamp_ship_flush_lsn(page_lsn)); /* - * PGRAC: spec-6.12a ㉕ (latent-bug fix) — InvalidateBuffer requires the - * buffer header spinlock held at ENTRY (it Assert()s BM_LOCKED) and + * PGRAC: spec-6.12a ㉕ (latent-bug fix) — InvalidateBuffer requires + * the buffer header spinlock held at ENTRY (it Assert()s BM_LOCKED) and * releases it internally. The bare call here previously ran with the * header unlocked; it never fired because the invalidate handler's * remote-holder path short-circuited before reaching the drop (see the * matching handler fix), so the first real caller tripped the assert. * Re-lock, re-validate the mapping (the buffer may have been evicted / - * retagged while unpinned during XLogFlush), and clear the residency - * mode under the lock so InvalidateBuffer's eviction hook sees - * PCM_STATE_N and skips the remote-master release wire (the §3.5 LMON - * context has no backend slot to send one — same BLOCKER A contract as + * retagged while unpinned during XLogFlush), and clear the residency mode + * under the lock so InvalidateBuffer's eviction hook sees PCM_STATE_N and + * skips the remote-master release wire (the §3.5 LMON context has no + * backend slot to send one — same BLOCKER A contract as * cluster_bufmgr_drop_block_for_gcs_no_wire below). */ buf_state = LockBufHdr(buf); @@ -8054,9 +10608,9 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode /* * PGRAC: GCS serve-stall round-5 (A2) — bounded drop. A pinner racing * in between the refcount check above and the Try variant's own - * partition-ordered recheck fails the call; restore the residency - * mode (it was cleared for the eviction hook) so the still-resident - * copy keeps its true PCM state. + * partition-ordered recheck fails the call; restore the residency mode + * (it was cleared for the eviction hook) so the still-resident copy keeps + * its true PCM state. */ cluster_bufmgr_in_gcs_drop = true; /* gates the drop-prepin inject */ if (!InvalidateBufferTry(buf)) /* releases the header spinlock */ @@ -8087,16 +10641,18 @@ cluster_bufmgr_invalidate_block_for_gcs(BufferTag tag, PcmLockMode expected_mode cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); buf_state = LockBufHdr(buf); + /* - * PGRAC ownership-generation wave (W2) — restore the staged N ONLY when - * it is still the same ownership epoch we staged. The ==N guard alone - * blocks a plain overwrite but NOT an N->X->N ABA: a concurrent round - * that granted X (finalize bumped the generation) and then dropped back - * to N leaves pcm_state==N again with a NEW generation. Restoring - * saved_pcm_state (the stale pre-drop X/S) over that re-owned block - * would resurrect a dead grant (Rule 8.A double/stale holder). Gate the - * restore on the generation being unchanged; if it moved, the block was - * legitimately re-owned and dropped in between, so leave it at N. + * PGRAC ownership-generation wave (W2) — restore the staged N ONLY + * when it is still the same ownership epoch we staged. The ==N guard + * alone blocks a plain overwrite but NOT an N->X->N ABA: a concurrent + * round that granted X (finalize bumped the generation) and then + * dropped back to N leaves pcm_state==N again with a NEW generation. + * Restoring saved_pcm_state (the stale pre-drop X/S) over that + * re-owned block would resurrect a dead grant (Rule 8.A double/stale + * holder). Gate the restore on the generation being unchanged; if it + * moved, the block was legitimately re-owned and dropped in between, + * so leave it at N. */ if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8) PCM_STATE_N && @@ -8181,6 +10737,7 @@ cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page LWLock *content_lock; uint32 buf_state; bool dirty; + bool write_permitted; if (out_page_scn != NULL) *out_page_scn = InvalidScn; @@ -8204,10 +10761,11 @@ cluster_bufmgr_refresh_block_from_storage_for_gcs(BufferDesc *buf, SCN *out_page content_lock = BufferDescriptorGetContentLock(buf); LWLockAcquire(content_lock, LW_EXCLUSIVE); + write_permitted = cluster_bufmgr_pcm_x_content_write_permitted(buf); buf_state = LockBufHdr(buf); dirty = (buf_state & BM_DIRTY) != 0; UnlockBufHdr(buf, buf_state); - if (dirty) + if (dirty || !write_permitted) { LWLockRelease(content_lock); return false; @@ -8452,17 +11010,17 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn } /* - * PGRAC: GCS serve-stall round-6 — generation gate. A local writer that - * held a cached X grant could commit to this page in the window between - * the transfer's ship-image copy and this drop (gaps (a) cached-X + * PGRAC: GCS serve-stall round-6 — generation gate. A local writer + * that held a cached X grant could commit to this page in the window + * between the transfer's ship-image copy and this drop (gaps (a) cached-X * no-reverify + (b) copy->drop admission). Its pin is already gone (the * refcount check above passed), so PINNED cannot catch it; the page LSN, * read here under the same header spinlock that the writer's unpin * released, is the generation token. A LSN past the caller's copy-time * expected_lsn means the captured ship image is STALE: refuse the drop so - * the caller fail-closes with a retryable deny and the re-serve copies the - * current page (Rule 8.A — never grant a stale image over a committed - * write). expected_lsn == Invalid skips the gate. + * the caller fail-closes with a retryable deny and the re-serve copies + * the current page (Rule 8.A — never grant a stale image over a + * committed write). expected_lsn == Invalid skips the gate. */ if (!XLogRecPtrIsInvalid(expected_lsn) && page_lsn != expected_lsn) { @@ -8490,8 +11048,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn * InvalidateBuffer requires the buffer header spinlock held at ENTRY and * releases it internally (it Assert()s BM_LOCKED). We dropped the header * lock for the (blocking) XLogFlush above, so re-lock and re-validate the - * buffer still maps our tag; if it was evicted / reused meanwhile there is - * nothing to drop. Clear the cache-residency mode under this lock so + * buffer still maps our tag; if it was evicted / reused meanwhile there + * is nothing to drop. Clear the cache-residency mode under this lock so * InvalidateBuffer's eviction hook sees PCM_STATE_N and skips the remote- * master release wire (the §3.5 LMON context has no backend slot to send * one -- BLOCKER A). @@ -8511,7 +11069,8 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn return CLUSTER_BUFMGR_GCS_DROP_PINNED; } - /* PGRAC: GCS serve-stall round-6 — re-check the generation gate under + /* + * PGRAC: GCS serve-stall round-6 — re-check the generation gate under * this second header-lock hold: a local writer could have committed * during the (blocking) XLogFlush window above. A LSN past expected_lsn * means the ship image is stale; refuse without touching pcm_state @@ -8527,8 +11086,10 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn staged_gen = cluster_pcm_own_gen_get(buf->buf_id); /* PGRAC W2 */ buf->pcm_state = (uint8) PCM_STATE_N; - /* PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping (the - * shipped current went to the new holder; our copy becomes the PI). */ + /* + * PGRAC: spec-6.12h D-h1 — keep a Past Image instead of dropping (the + * shipped current went to the new holder; our copy becomes the PI). + */ if (cluster_bufmgr_convert_to_pi_locked(buf, buf_state)) return CLUSTER_BUFMGR_GCS_DROP_DROPPED; @@ -8556,11 +11117,13 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn cluster_pcm_own_transition(buf, (uint8) PCM_STATE_N, 0, 0); buf_state = LockBufHdr(buf); + /* - * PGRAC ownership-generation wave (W2) — see the twin arm above: restore - * the staged N only when the ownership generation is unchanged, so an - * N->X->N ABA that re-owned and dropped the block in the InvalidateBuffer - * window does not get a stale pre-drop state restored over it. + * PGRAC ownership-generation wave (W2) — see the twin arm above: + * restore the staged N only when the ownership generation is + * unchanged, so an N->X->N ABA that re-owned and dropped the block in + * the InvalidateBuffer window does not get a stale pre-drop state + * restored over it. */ if (BufferTagsEqual(&buf->tag, &tag) && buf->pcm_state == (uint8) PCM_STATE_N && @@ -8578,6 +11141,741 @@ cluster_bufmgr_drop_block_for_gcs_no_wire(BufferTag tag, XLogRecPtr expected_lsn return CLUSTER_BUFMGR_GCS_DROP_DROPPED; } +/* Abort only the matching requester-as-source N staging reservation. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_n_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint32 buf_state; + + if (buf == NULL || expected_revoking == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (expected_revoking->pcm_state != (uint8)PCM_STATE_N + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_revoking)) + result = CLUSTER_PCM_OWN_STALE; + else { + live_result = cluster_pcm_own_classify_live_flags(expected_revoking->flags, + expected_revoking->reservation_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else + result = cluster_pcm_own_reservation_abort_exact( + buf->buf_id, expected_revoking->generation, expected_revoking->reservation_token, + PCM_OWN_FLAG_REVOKING); + } + UnlockBufHdr(buf, buf_state); + return result; +} + +/* + * Fence a clean N descriptor, refresh it from shared storage, and return one + * immutable requester-as-source image. + * + * REVOKING binds descriptor identity before the physical read. Storage is + * read and verified into scratch without holding the content lock; content + * EXCLUSIVE plus a second full tuple/header check then makes the resident + * copy and caller copy derive from that exact scratch. No READY state is + * published here. Ordinary failures remove only this exact reservation; + * inability to do so is corruption and retains the evidence fail-closed. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_prepare_n_source_image(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_n, + ClusterPcmOwnSnapshot *out_revoking, + char block_data[BLCKSZ], XLogRecPtr *out_page_lsn, + uint64 *out_page_scn) +{ + PGAlignedBlock scratch; + BufferTag tag; + ClusterPcmOwnSnapshot live; + ClusterPcmOwnResult abort_result; + volatile ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + SMgrRelation reln; + LWLock *content_lock; + uint64 reservation_token = 0; + uint32 buf_state; + volatile bool content_locked = false; + + if (out_revoking != NULL) + memset(out_revoking, 0, sizeof(*out_revoking)); + if (block_data != NULL) + memset(block_data, 0, BLCKSZ); + if (out_page_lsn != NULL) + *out_page_lsn = InvalidXLogRecPtr; + if (out_page_scn != NULL) + *out_page_scn = 0; + if (buf == NULL || expected_n == NULL || out_revoking == NULL || block_data == NULL + || out_page_lsn == NULL || out_page_scn == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (expected_n->pcm_state != (uint8)PCM_STATE_N || expected_n->flags != 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + tag = expected_n->tag; + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, expected_n) || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else { + result = cluster_pcm_own_reservation_begin_exact(buf->buf_id, expected_n->generation, + PCM_OWN_FLAG_REVOKING, &reservation_token); + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_own_snapshot_locked(buf, &live); + } + UnlockBufHdr(buf, buf_state); + if (result != CLUSTER_PCM_OWN_OK) + return result; + + content_lock = BufferDescriptorGetContentLock(buf); + PG_TRY(); + { + reln = smgropen(BufTagGetRelFileLocator(&tag), InvalidBackendId); + smgrread(reln, BufTagGetForkNum(&tag), tag.blockNum, scratch.data); + if (!PageIsVerifiedExtended((Page)scratch.data, tag.blockNum, + PIV_LOG_WARNING | PIV_REPORT_STAT)) + result = CLUSTER_PCM_OWN_CORRUPT; + + if (result == CLUSTER_PCM_OWN_OK) { + LWLockAcquire(content_lock, LW_EXCLUSIVE); + content_locked = true; + buf_state = LockBufHdr(buf); + if (!cluster_pcm_own_snapshot_matches_locked(buf, &live) + || live.flags != PCM_OWN_FLAG_REVOKING || live.pcm_state != (uint8)PCM_STATE_N + || (buf_state & BM_VALID) == 0) + result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) + != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else { + memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ); + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + memcpy(block_data, scratch.data, BLCKSZ); + *out_page_lsn = PageGetLSN((Page)scratch.data); + *out_page_scn = (uint64)((PageHeader)scratch.data)->pd_block_scn; + *out_revoking = live; + } + UnlockBufHdr(buf, buf_state); + LWLockRelease(content_lock); + content_locked = false; + } + } + PG_CATCH(); + { + if (content_locked && LWLockHeldByMe(content_lock)) + LWLockRelease(content_lock); + abort_result = cluster_bufmgr_pcm_own_abort_n_revoke(buf, &live); + if (abort_result != CLUSTER_PCM_OWN_OK) + elog(LOG, + "could not abort exact PCM-X N-source reservation after storage read error: " + "buffer=%d generation=%llu token=%llu result=%d; evidence retained", + buf->buf_id, (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, (int)abort_result); + PG_RE_THROW(); + } + PG_END_TRY(); + + if (result == CLUSTER_PCM_OWN_OK) + return CLUSTER_PCM_OWN_OK; + abort_result = cluster_bufmgr_pcm_own_abort_n_revoke(buf, &live); + if (abort_result != CLUSTER_PCM_OWN_OK) { + elog(LOG, + "could not abort exact PCM-X N-source reservation after refresh failure: " + "buffer=%d generation=%llu token=%llu result=%d; evidence retained", + buf->buf_id, (unsigned long long)live.generation, + (unsigned long long)live.reservation_token, (int)abort_result); + return CLUSTER_PCM_OWN_CORRUPT; + } + memset(out_revoking, 0, sizeof(*out_revoking)); + memset(block_data, 0, BLCKSZ); + *out_page_lsn = InvalidXLogRecPtr; + *out_page_scn = 0; + return result; +} + +/* + * Begin an exact source-holder S revoke for PCM-X image staging. + * + * The S source remains S while REVOKING is live. The flag closes the local + * cached-cover window without inventing an X grant. Passive PG pins are not + * ownership holders and therefore do not block this protocol reservation; + * the final byte boundary is serialized by the content lock instead. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_revoking) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint64 live_token; + uint64 reservation_token = 0; + uint32 flags; + uint32 buf_state; + + if (buf == NULL || expected_s == NULL || out_revoking == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_revoking, 0, sizeof(*out_revoking)); + if (expected_s->pcm_state != (uint8) PCM_STATE_S || expected_s->flags != 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected_s->tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_s->generation + || buf->pcm_state != (uint8) PCM_STATE_S) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result != CLUSTER_PCM_OWN_OK) + result = live_result; + else if (live_token != expected_s->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else + { + result = cluster_pcm_own_reservation_begin_exact( + buf->buf_id, expected_s->generation, PCM_OWN_FLAG_REVOKING, + &reservation_token); + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_own_snapshot_locked(buf, out_revoking); + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +/* Abort only the matching S-source staging reservation. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint64 live_token; + uint32 flags; + uint32 buf_state; + + if (buf == NULL || expected_revoking == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (expected_revoking->pcm_state != (uint8) PCM_STATE_S + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != (uint8) PCM_STATE_S) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else + result = cluster_pcm_own_reservation_abort_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, PCM_OWN_FLAG_REVOKING); + } + UnlockBufHdr(buf, buf_state); + return result; +} + +/* + * Begin an exact source-holder X revoke. + * + * This is deliberately an opaque BufferDesc operation. The returned + * snapshot is the complete live REVOKING identity consumed by byte staging, + * abort, and retained finish; callers never inspect the ownership sidecar. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_x, + ClusterPcmOwnSnapshot *out_revoking) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint64 live_token; + uint64 reservation_token = 0; + uint32 flags; + uint32 buf_state; + + if (buf == NULL || expected_x == NULL || out_revoking == NULL) + return CLUSTER_PCM_OWN_INVALID; + memset(out_revoking, 0, sizeof(*out_revoking)); + if (expected_x->pcm_state != (uint8) PCM_STATE_X || expected_x->flags != 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected_x->tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_x->generation + || buf->pcm_state != (uint8) PCM_STATE_X) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result != CLUSTER_PCM_OWN_OK) + result = live_result; + else if (live_token != expected_x->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else + { + result = cluster_pcm_own_reservation_begin_exact( + buf->buf_id, expected_x->generation, PCM_OWN_FLAG_REVOKING, + &reservation_token); + if (result == CLUSTER_PCM_OWN_OK) + cluster_pcm_own_snapshot_locked(buf, out_revoking); + } + } + UnlockBufHdr(buf, buf_state); + return result; +} + +/* + * Abort one exact source-holder revoke before any ownership transfer. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result; + uint64 live_token; + uint32 flags; + uint32 buf_state; + + if (buf == NULL || expected_revoking == NULL) + return CLUSTER_PCM_OWN_INVALID; + if (expected_revoking->pcm_state != (uint8) PCM_STATE_X + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &expected_revoking->tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != (uint8) PCM_STATE_X) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else + result = cluster_pcm_own_reservation_abort_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, PCM_OWN_FLAG_REVOKING); + } + UnlockBufHdr(buf, buf_state); + return result; +} + +/* Commit a staged VM/FSM source without leaving a BM_VALID mapping behind. + * The exclusive mapping lock closes the lookup-to-pin window around the + * zero-refcount proof. The immutable A-record already owns the copied bytes, + * so the ordinary revoke commit may clear REVOKING and the descriptor may be + * removed with the same generation bump as its reuse linearization. */ +static ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking, + XLogRecPtr expected_lsn, + ClusterPcmOwnSnapshot *out_finished) +{ + ClusterPcmXRevokeFinishMode finish_mode; + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + BufferTag tag = expected_revoking->tag; + LWLock *partition_lock; + uint64 committed_generation = 0; + uint64 live_token; + uint32 hashcode; + uint32 flags; + uint32 buf_state; + int mapped_buf_id; + + hashcode = BufTableHashCode(&tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_EXCLUSIVE); + mapped_buf_id = BufTableLookup(&tag, hashcode); + if (mapped_buf_id != buf->buf_id) { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } + + buf_state = LockBufHdr(buf); + finish_mode = cluster_pcm_x_revoke_finish_mode(&tag, BUF_STATE_GET_REFCOUNT(buf_state)); + if (!BufferTagsEqual(&buf->tag, &tag) || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != expected_revoking->pcm_state) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_BUSY) + result = CLUSTER_PCM_OWN_BUSY; + else if (finish_mode != CLUSTER_PCM_X_REVOKE_FINISH_DROP) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else if (PageGetLSN((Page)BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + } + + if (result == CLUSTER_PCM_OWN_OK) + result = cluster_pcm_own_revoke_commit_exact(buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, + &committed_generation); + if (result != CLUSTER_PCM_OWN_OK) { + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + return result; + } + + Assert(expected_revoking->generation != UINT64_MAX); + Assert(committed_generation == expected_revoking->generation + 1); + buf->pcm_state = (uint8)PCM_STATE_N; + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + cluster_pcm_own_snapshot_locked(buf, out_finished); + InvalidateBufferCommitTailLocked(buf, &tag, hashcode, partition_lock, buf_state, + (uint8)PCM_STATE_N, false); + return CLUSTER_PCM_OWN_OK; +} + +/* + * Commit one staged S/X source as a retained image, except for VM/FSM. + * + * REVOKING already binds the descriptor to its tag through D5a and the victim + * guard, so this path deliberately does not nest a mapping lock with the + * content lock. Content EXCLUSIVE is the byte/I/O linearization: a flush + * that already owns SHARE completes first; once this lock is acquired, no + * writer or output I/O can overlap the exact LSN check and transition. + * Passive pins remain untouched. The descriptor keeps tag, BM_VALID, + * refcount, and bytes, while PI+N marks those bytes non-current and the exact + * REVOKING token prevents reuse until DRAIN. + */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_revoke_retain( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, + XLogRecPtr expected_lsn, ClusterPcmOwnSnapshot *out_retained) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + BufferTag tag; + LWLock *content_lock; + uint64 committed_generation; + uint64 live_token; + uint32 flags; + uint32 buf_state; + bool source_is_s; + bool source_is_x; + ClusterPcmXRevokeFinishMode finish_mode; + + if (out_retained != NULL) + memset(out_retained, 0, sizeof(*out_retained)); + if (buf == NULL || expected_revoking == NULL || out_retained == NULL) + return CLUSTER_PCM_OWN_INVALID; + source_is_s = expected_revoking->pcm_state == (uint8) PCM_STATE_S; + source_is_x = expected_revoking->pcm_state == (uint8) PCM_STATE_X; + if ((!source_is_s && !source_is_x) + || expected_revoking->flags != PCM_OWN_FLAG_REVOKING + || expected_revoking->reservation_token == 0) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + + tag = expected_revoking->tag; + finish_mode = cluster_pcm_x_revoke_finish_mode(&tag, 0); + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_INVALID) + return CLUSTER_PCM_OWN_INVALID; + if (finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_DROP) + return cluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(buf, expected_revoking, + expected_lsn, out_retained); + Assert(finish_mode == CLUSTER_PCM_X_REVOKE_FINISH_RETAIN); + content_lock = BufferDescriptorGetContentLock(buf); + LWLockAcquire(content_lock, LW_EXCLUSIVE); + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, &tag) + || (buf_state & BM_VALID) == 0 + || cluster_pcm_own_gen_get(buf->buf_id) != expected_revoking->generation + || buf->pcm_state != expected_revoking->pcm_state) + result = CLUSTER_PCM_OWN_STALE; + else if (!cluster_bufmgr_pcm_current_image_locked(buf, buf_state)) + result = CLUSTER_PCM_OWN_CORRUPT; + else if ((buf_state & BM_IO_IN_PROGRESS) != 0) + result = CLUSTER_PCM_OWN_BUSY; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != expected_revoking->reservation_token) + result = CLUSTER_PCM_OWN_STALE; + else if (PageGetLSN((Page) BufHdrGetBlock(buf)) != expected_lsn) + result = CLUSTER_PCM_OWN_STALE; + } + + if (result == CLUSTER_PCM_OWN_OK) + { + result = cluster_pcm_own_revoke_retain_commit_exact( + buf->buf_id, expected_revoking->generation, + expected_revoking->reservation_token, &committed_generation); + if (result == CLUSTER_PCM_OWN_OK) + { + buf->pcm_state = (uint8) PCM_STATE_N; + buf->buffer_type = (uint8) BUF_TYPE_PI; + buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED | + BM_CHECKPOINT_NEEDED | BM_IO_ERROR); + cluster_pcm_own_snapshot_locked(buf, out_retained); + } + } + + UnlockBufHdr(buf, buf_state); + LWLockRelease(content_lock); + return result; +} + +/* Release only the retained descriptor created from source_generation. The + * generation+1 and current token bind a DRAIN to one image round; a delayed + * DRAIN cannot clear REVOKING on a later transfer. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, + uint64 source_generation) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + BufferDesc *buf; + BufferTag lookup_tag; + LWLock *content_lock; + LWLock *partition_lock; + uint64 committed_generation; + uint64 live_token; + uint64 retained_token = 0; + uint32 hashcode; + uint32 flags; + uint32 buf_state; + int buf_id; + + if (tag == NULL || source_generation == UINT64_MAX) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + committed_generation = source_generation + 1; + lookup_tag = *tag; + hashcode = BufTableHashCode(&lookup_tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_SHARED); + buf_id = BufTableLookup(&lookup_tag, hashcode); + if (buf_id < 0) + { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } + buf = GetBufferDescriptor(buf_id); + + /* + * Phase 1 binds tag -> descriptor under the mapping lock and proves the + * live retention token before dropping mapping authority. REVOKING then + * prevents every reuse path until phase 2 revalidates under content + * EXCLUSIVE. Never hold mapping and content together (avoids ABBA with + * ordinary content-lock callers). + */ + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, tag) + || (buf_state & BM_VALID) == 0 + || buf->buffer_type != (uint8) BUF_TYPE_PI + || buf->pcm_state != (uint8) PCM_STATE_N + || cluster_pcm_own_gen_get(buf->buf_id) != committed_generation) + result = CLUSTER_PCM_OWN_STALE; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else + retained_token = live_token; + } + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + if (result != CLUSTER_PCM_OWN_OK) + return result; + + content_lock = BufferDescriptorGetContentLock(buf); + LWLockAcquire(content_lock, LW_EXCLUSIVE); + + buf_state = LockBufHdr(buf); + if (!BufferTagsEqual(&buf->tag, tag) + || (buf_state & BM_VALID) == 0 + || buf->buffer_type != (uint8) BUF_TYPE_PI + || buf->pcm_state != (uint8) PCM_STATE_N + || cluster_pcm_own_gen_get(buf->buf_id) != committed_generation) + result = CLUSTER_PCM_OWN_STALE; + else if ((buf_state + & (BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR)) != 0) + result = CLUSTER_PCM_OWN_CORRUPT; + else + { + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags == 0) + result = CLUSTER_PCM_OWN_STALE; + else if (flags != PCM_OWN_FLAG_REVOKING) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token != retained_token) + result = CLUSTER_PCM_OWN_STALE; + else { + result = cluster_pcm_own_revoke_retain_release_exact( + buf->buf_id, committed_generation, retained_token); + if (result == CLUSTER_PCM_OWN_OK) { + /* + * The exact token is the last stale-write fence. Release it + * before making the retained PI reloadable, while content + * EXCLUSIVE and the header lock keep page bytes invalid. + * Keep the mapping so any pre-existing pin can leave + * normally; the next ordinary read reloads the current page + * image. + */ + buf_state &= ~BM_VALID; + buf->buffer_type = (uint8)BUF_TYPE_CURRENT; + } + } + } + + UnlockBufHdr(buf, buf_state); + LWLockRelease(content_lock); + return result; +} + +/* Prove that a sole-requester source image was adopted by the exact S->X + * handoff. This is intentionally read-only: a delayed DRAIN may release its + * immutable A-record, but it must never mutate the current X descriptor. */ +ClusterPcmOwnResult +cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, uint64 source_generation) +{ + ClusterPcmOwnResult live_result; + ClusterPcmOwnResult result = CLUSTER_PCM_OWN_OK; + BufferDesc *buf; + BufferTag lookup_tag; + LWLock *partition_lock; + uint64 live_token; + uint32 flags; + uint32 hashcode; + uint32 buf_state; + int buf_id; + + if (tag == NULL || source_generation == UINT64_MAX) + return CLUSTER_PCM_OWN_INVALID; + if (ClusterPcmOwnArray == NULL) + return CLUSTER_PCM_OWN_NOT_READY; + lookup_tag = *tag; + hashcode = BufTableHashCode(&lookup_tag); + partition_lock = BufMappingPartitionLock(hashcode); + LWLockAcquire(partition_lock, LW_SHARED); + buf_id = BufTableLookup(&lookup_tag, hashcode); + if (buf_id < 0) { + LWLockRelease(partition_lock); + return CLUSTER_PCM_OWN_STALE; + } + buf = GetBufferDescriptor(buf_id); + buf_state = LockBufHdr(buf); + live_token = cluster_pcm_own_reservation_token_get(buf->buf_id); + flags = cluster_pcm_own_flags_get(buf->buf_id); + live_result = cluster_pcm_own_classify_live_flags(flags, live_token); + if (!BufferTagsEqual(&buf->tag, tag) + || cluster_pcm_own_gen_get(buf->buf_id) != source_generation + 1) + result = CLUSTER_PCM_OWN_STALE; + else if (live_result == CLUSTER_PCM_OWN_CORRUPT) + result = live_result; + else if (flags != 0) + result = CLUSTER_PCM_OWN_BUSY; + else if (live_token == 0 || (buf_state & BM_VALID) == 0 || buf->pcm_state != (uint8)PCM_STATE_X + || buf->buffer_type != (uint8)BUF_TYPE_XCUR) + result = CLUSTER_PCM_OWN_STALE; + UnlockBufHdr(buf, buf_state); + LWLockRelease(partition_lock); + return result; +} + /* ======================================================================== * PGRAC MODIFICATIONS by SqlRush — spec-6.12h D-h2 (PI discard helpers). * @@ -8653,7 +11951,8 @@ cluster_bufmgr_discard_pi_block(BufferTag tag) return false; } - /* A PI carries no residency claim (D-h1 set pcm_state N at conversion), + /* + * A PI carries no residency claim (D-h1 set pcm_state N at conversion), * but clear it again under the lock so InvalidateBuffer's eviction hook * can never see a stale mode and emit a release wire from LMON. */ buf->pcm_state = (uint8) PCM_STATE_N; @@ -8831,7 +12130,10 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) /* CL-I9: must run with a real ResourceOwner (backend/checkpointer). */ Assert(CurrentResourceOwner != NULL); - /* Make sure we can handle the pin inside the loop (as FlushDatabaseBuffers). */ + /* + * Make sure we can handle the pin inside the loop (as + * FlushDatabaseBuffers). + */ ResourceOwnerEnlargeBuffers(CurrentResourceOwner); for (i = 0; i < NBuffers; i++) @@ -8845,20 +12147,33 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) if ((buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY) && (PcmState) bufHdr->pcm_state == PCM_STATE_X) { + ClusterPcmOwnResult bump_result; + uint32 observed_flags = 0; + uint64 observed_generation = 0; + PinBuffer_Locked(bufHdr); /* releases the header spinlock */ LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED); FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL); LWLockRelease(BufferDescriptorGetContentLock(bufHdr)); /* - * Release X only now that the block is durable + current on shared - * storage. The pin held it valid across the unlocked window above. + * Release X only now that the block is durable + current on + * shared storage. The pin held it valid across the unlocked + * window above. */ buf_state = LockBufHdr(bufHdr); - bufHdr->pcm_state = (uint8) PCM_STATE_N; /* PGRAC ownership-gen: bump under the held spinlock (X -> N release * after the block is durable on shared storage). */ - cluster_pcm_own_bump_locked(bufHdr, 0, 0); + bump_result = cluster_pcm_own_bump_locked(bufHdr, 0, 0, + &observed_generation, &observed_flags); + if (bump_result != CLUSTER_PCM_OWN_OK) + { + UnlockBufHdr(bufHdr, buf_state); + UnpinBuffer(bufHdr); + cluster_pcm_own_report_bump_failure(bufHdr, bump_result, observed_generation, + observed_flags, "clean-leave X release"); + } + bufHdr->pcm_state = (uint8) PCM_STATE_N; UnlockBufHdr(bufHdr, buf_state); UnpinBuffer(bufHdr); @@ -8876,15 +12191,14 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) * active-ITL hard boundary, multi-row fail-closed leg). * * cluster_bufmgr_block_write_permitted(buffer) - * — false ONLY when buffer's block is held by this backend as a DEFERRED - * READ-IMAGE (pcm_state == PCM_STATE_READ_IMAGE): a remote node holds X and, - * because it still has an uncommitted ITL slot, deferred its writer-transfer - * (spec-5.2 §3.5 D11) and shipped us a read-image. Every other state - * permits the write: PCM_STATE_X (we own X), PCM_STATE_S (re-acquired to X - * by LockBuffer before reaching here), and PCM_STATE_N (a freshly-extended / - * storage-fallback / local-master block — legitimately writable without a - * recorded X grant). pcm_state != X is therefore NOT a valid "no write" - * test; only the explicit READ_IMAGE marker is. + * — false when buffer's block is held by this backend as a DEFERRED + * READ-IMAGE (pcm_state == PCM_STATE_READ_IMAGE), or when PCM-X has + * published REVOKING/retained-image authority. In the former case a remote + * node holds X and deferred its writer-transfer because it still has an + * uncommitted ITL slot. In the latter, immutable transfer bytes must remain + * non-writable through exact DRAIN. Outside those explicit states, + * pcm_state != X alone is not a valid "no write" test: freshly extended and + * storage-fallback N buffers can be legitimately writable. * * In the contended-row case the heap AM waits and re-acquires real X * (overwriting the marker) before writing, so it passes here; this catches @@ -8906,11 +12220,24 @@ cluster_bufmgr_flush_and_release_x_for_leave(void) bool cluster_bufmgr_block_write_permitted(Buffer buffer) { + BufferDesc *buf; PcmState state; + uint32 buf_state; + uint32 own_flags; if (BufferIsLocal(buffer)) return true; /* temp / local buffers are never CF-shared */ - state = (PcmState) GetBufferDescriptor(buffer - 1)->pcm_state; + buf = GetBufferDescriptor(buffer - 1); + buf_state = LockBufHdr(buf); + state = (PcmState) buf->pcm_state; + own_flags = cluster_pcm_own_flags_get(buf->buf_id); + if (cluster_bufmgr_pcm_x_retained_image_locked(buf, buf_state) + || (own_flags & PCM_OWN_FLAG_REVOKING) != 0) + { + UnlockBufHdr(buf, buf_state); + return false; + } + UnlockBufHdr(buf, buf_state); if (state == PCM_STATE_READ_IMAGE) return false; if (cluster_read_scache && state == PCM_STATE_S) diff --git a/src/backend/storage/freespace/freespace.c b/src/backend/storage/freespace/freespace.c index e27e8ca8ca4..1e43ca91641 100644 --- a/src/backend/storage/freespace/freespace.c +++ b/src/backend/storage/freespace/freespace.c @@ -619,7 +619,9 @@ fsm_readbuf(Relation rel, FSMAddress addr, bool extend) */ if (PageIsNew(BufferGetPage(buf))) { - LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); + /* PGRAC: authorize this known FSM zero-page initialization without + * admitting an ordinary N->X writer outside the PCM-X queue. */ + LockBufferForFreeSpaceMapPageInit(buf); if (PageIsNew(BufferGetPage(buf))) PageInit(BufferGetPage(buf), BLCKSZ, 0); LockBuffer(buf, BUFFER_LOCK_UNLOCK); diff --git a/src/backend/storage/lmgr/lwlock.c b/src/backend/storage/lmgr/lwlock.c index 9dfc0712046..effbbfbdf1b 100644 --- a/src/backend/storage/lmgr/lwlock.c +++ b/src/backend/storage/lmgr/lwlock.c @@ -74,6 +74,18 @@ * quick, before we're queued, since after Phase 2 we're already queued. * ------------------------------------------------------------------------- */ +/*------------------------------------------------------------------------- + * + * PGRAC MODIFICATIONS + * Modified by: SqlRush + * + * Use the held-LWLock ceiling exported by storage/lwlock.h so cluster + * shared-memory sizing and the lock manager share one definition. + * + * Spec: spec-2.36a-gcs-writer-convert-queue.md + * + *------------------------------------------------------------------------- + */ #include "postgres.h" #include "miscadmin.h" @@ -258,6 +270,12 @@ static const char *const BuiltinTrancheNames[] = { "ClusterOidLease", /* PGRAC LWTRANCHE_CLUSTER_NATIVE_PREHISTORY: GCS-race round-3b drain */ "ClusterNativePrehistory", + /* PGRAC LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR: spec-2.36a five-pool metadata */ + "ClusterPcmXAllocator", + /* PGRAC LWTRANCHE_CLUSTER_PCM_X_MASTER: spec-2.36a master queue partitions */ + "ClusterPcmXMaster", + /* PGRAC LWTRANCHE_CLUSTER_PCM_X_LOCAL: spec-2.36a local coordinator */ + "ClusterPcmXLocal", #endif }; @@ -286,8 +304,6 @@ LWLockPadded *MainLWLockArray = NULL; * occasionally the number can be much higher; for example, the pg_buffercache * extension locks all buffer partitions simultaneously. */ -#define MAX_SIMUL_LWLOCKS 200 - /* struct representing the LWLocks we're holding */ typedef struct LWLockHandle { LWLock *lock; @@ -295,7 +311,7 @@ typedef struct LWLockHandle { } LWLockHandle; static int num_held_lwlocks = 0; -static LWLockHandle held_lwlocks[MAX_SIMUL_LWLOCKS]; +static LWLockHandle held_lwlocks[LWLOCK_MAX_HELD_BY_PROC]; /* struct representing the LWLock tranche request for named tranche */ typedef struct NamedLWLockTrancheRequest { @@ -1244,7 +1260,7 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) Assert(!(proc == NULL && IsUnderPostmaster)); /* Ensure we will have room to remember the lock */ - if (num_held_lwlocks >= MAX_SIMUL_LWLOCKS) + if (num_held_lwlocks >= LWLOCK_MAX_HELD_BY_PROC) elog(ERROR, "too many LWLocks taken"); /* @@ -1389,7 +1405,7 @@ LWLockConditionalAcquire(LWLock *lock, LWLockMode mode) PRINT_LWDEBUG("LWLockConditionalAcquire", lock, mode); /* Ensure we will have room to remember the lock */ - if (num_held_lwlocks >= MAX_SIMUL_LWLOCKS) + if (num_held_lwlocks >= LWLOCK_MAX_HELD_BY_PROC) elog(ERROR, "too many LWLocks taken"); /* @@ -1450,7 +1466,7 @@ LWLockAcquireOrWait(LWLock *lock, LWLockMode mode) PRINT_LWDEBUG("LWLockAcquireOrWait", lock, mode); /* Ensure we will have room to remember the lock */ - if (num_held_lwlocks >= MAX_SIMUL_LWLOCKS) + if (num_held_lwlocks >= LWLOCK_MAX_HELD_BY_PROC) elog(ERROR, "too many LWLocks taken"); /* diff --git a/src/include/cluster/cluster_gcs_block.h b/src/include/cluster/cluster_gcs_block.h index f71ddd9c118..d4f02ce8fc4 100644 --- a/src/include/cluster/cluster_gcs_block.h +++ b/src/include/cluster/cluster_gcs_block.h @@ -69,13 +69,1043 @@ #define CLUSTER_GCS_BLOCK_H #include "c.h" +#include "cluster/cluster_gcs_reqid.h" +#include "cluster/cluster_lmd_wait_state.h" #include "cluster/cluster_pcm_lock.h" /* PcmLockTransition */ -#include "cluster/cluster_sf_dep.h" /* ClusterSfDepVec / max origins */ -#include "storage/block.h" /* BLCKSZ */ -#include "storage/buf_internals.h" /* BufferTag, BufferDesc */ +#include "cluster/cluster_pcm_x_bufmgr.h" +#include "cluster/cluster_pcm_x_convert.h" +#include "cluster/cluster_sf_dep.h" /* ClusterSfDepVec / max origins */ +#include "storage/block.h" /* BLCKSZ */ +#include "storage/buf_internals.h" /* BufferTag, BufferDesc */ #ifdef USE_PGRAC_CLUSTER +/* A RETIRE wake is only a latency hint, but it still must never target a + * recycled PGPROC wait generation. Keep the tuple classifier pure so the + * seqlock reader and GCS adapter share one executable definition of exact. */ +static inline bool +cluster_gcs_pcm_x_wait_identity_matches(const PcmXWaitIdentity *identity, int32 local_node, + uint32 all_proc_count, + ClusterLmdWaitStateReadResult read_result, + const ClusterLmdWaitStateSnapshot *snapshot) +{ + return identity != NULL && snapshot != NULL && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && identity->node_id == local_node + && identity->procno < all_proc_count && identity->request_id != 0 + && identity->wait_seq != 0 && read_result == CLUSTER_LMD_WAIT_STATE_READ_ACTIVE + && snapshot->active && snapshot->kind == CLUSTER_LMD_WAIT_PCM_CONVERT + && snapshot->request_id == identity->request_id + && snapshot->cluster_epoch == identity->cluster_epoch + && snapshot->wait_seq == identity->wait_seq && snapshot->xid == identity->xid; +} + +/* One lock-free authority observation. The qvotec slot tuple and the + * connection capability record have different publishers, so consumers must + * sample both ends and reject a change as retryable rather than accepting a + * half-old identity. Zero is a valid value for INITIAL epoch and for the + * registered RDMA connection generation; the explicit *_valid bits carry + * presence. */ +typedef struct ClusterGcsPcmXAuthSample { + uint64 session_before; + uint64 session_after; + uint64 slot_generation_before; + uint64 slot_generation_after; + uint64 observed_epoch_before; + uint64 observed_epoch_after; + uint32 connection_generation_before; + uint32 connection_generation_after; + bool connection_before_valid; + bool connection_after_valid; + bool slot_before_valid; + bool slot_after_valid; + bool fresh_before; + bool fresh_after; +} ClusterGcsPcmXAuthSample; + +typedef enum PcmXSessionAuthResult { + PCM_X_SESSION_AUTH_INVALID = 0, + PCM_X_SESSION_AUTH_OK, + PCM_X_SESSION_AUTH_CONNECTION_NOT_READY, + PCM_X_SESSION_AUTH_SLOT_NOT_READY, + PCM_X_SESSION_AUTH_EPOCH_NOT_READY, + PCM_X_SESSION_AUTH_FRESH_NOT_READY, + PCM_X_SESSION_AUTH_SLOT_TORN, + PCM_X_SESSION_AUTH_EPOCH_TORN, + PCM_X_SESSION_AUTH_CONNECTION_TORN +} PcmXSessionAuthResult; + +static inline PcmXSessionAuthResult +cluster_gcs_pcm_x_auth_sample_classify(const ClusterGcsPcmXAuthSample *sample, + uint64 expected_epoch) +{ + if (sample == NULL) + return PCM_X_SESSION_AUTH_INVALID; + if (!sample->connection_before_valid || !sample->connection_after_valid) + return PCM_X_SESSION_AUTH_CONNECTION_NOT_READY; + if (!sample->slot_before_valid || !sample->slot_after_valid || sample->session_before == 0 + || sample->session_after == 0 || sample->slot_generation_before == 0 + || sample->slot_generation_after == 0) + return PCM_X_SESSION_AUTH_SLOT_NOT_READY; + if (sample->observed_epoch_before != expected_epoch + || sample->observed_epoch_after != expected_epoch) + return sample->observed_epoch_before == sample->observed_epoch_after + ? PCM_X_SESSION_AUTH_EPOCH_NOT_READY + : PCM_X_SESSION_AUTH_EPOCH_TORN; + if (!sample->fresh_before || !sample->fresh_after) + return PCM_X_SESSION_AUTH_FRESH_NOT_READY; + if (sample->session_before != sample->session_after + || sample->slot_generation_before != sample->slot_generation_after) + return PCM_X_SESSION_AUTH_SLOT_TORN; + if (sample->observed_epoch_before != sample->observed_epoch_after) + return PCM_X_SESSION_AUTH_EPOCH_TORN; + if (sample->connection_generation_before != sample->connection_generation_after) + return PCM_X_SESSION_AUTH_CONNECTION_TORN; + return PCM_X_SESSION_AUTH_OK; +} + +typedef enum GcsBlockPcmXRequesterSite { + GCS_BLOCK_PCM_X_RETRY_SITE_JOIN = 0, + GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, + GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, + GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, + GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF, + GCS_BLOCK_PCM_X_RETRY_SITE_FOLLOWER_SNAPSHOT, + GCS_BLOCK_PCM_X_RETRY_SITE_WFG_COMMIT, + GCS_BLOCK_PCM_X_RETRY_SITE_WFG_CLEAR, + GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, + GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, + GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, + GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM, + GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM +} GcsBlockPcmXRequesterSite; + +typedef enum GcsBlockPcmXRetryAction { + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED = 0, + GCS_BLOCK_PCM_X_RETRY_WAIT, + GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS, + GCS_BLOCK_PCM_X_RETRY_RESNAPSHOT_WFG +} GcsBlockPcmXRetryAction; + +typedef enum GcsBlockPcmXFormationAction { + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED = 0, + GCS_BLOCK_PCM_X_FORMATION_WAIT, + GCS_BLOCK_PCM_X_FORMATION_PROCEED +} GcsBlockPcmXFormationAction; + +/* A pristine startup has never published a queue authority token, so a + * writer may wait interruptibly for LMON formation. Once any generation or + * session has been published, a non-ACTIVE state is fail-stop evidence and + * must never be mistaken for startup or routed through the legacy protocol. */ +static inline GcsBlockPcmXFormationAction +cluster_gcs_pcm_x_requester_formation_action(const PcmXRuntimeSnapshot *runtime) +{ + if (runtime == NULL) + return GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED; + if (runtime->state == PCM_X_RUNTIME_ACTIVE && runtime->gate_generation != 0 + && runtime->master_session_incarnation != 0) + return GCS_BLOCK_PCM_X_FORMATION_PROCEED; + if (runtime->state == PCM_X_RUNTIME_RECOVERY_BLOCKED && runtime->gate_generation == 0 + && runtime->master_session_incarnation == 0) + return GCS_BLOCK_PCM_X_FORMATION_WAIT; + return GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED; +} + +/* Every retry in one admitted request remains subordinate to the exact + * formation token captured before publishing PCM_CONVERT. */ +static inline bool +cluster_gcs_pcm_x_requester_runtime_exact(const PcmXRuntimeSnapshot *start, + const PcmXRuntimeSnapshot *current) +{ + return start != NULL && current != NULL && start->state == PCM_X_RUNTIME_ACTIVE + && current->state == PCM_X_RUNTIME_ACTIVE && start->gate_generation != 0 + && start->gate_generation == current->gate_generation + && start->master_session_incarnation != 0 + && start->master_session_incarnation == current->master_session_incarnation; +} + +/* A generation/locator handoff while validating already-held content locks is + * a normal transient. It is safe to retry only while the requester's + * PCM_CONVERT wait identity remains published. BARRIER_CLOSED is deliberately + * excluded: a barrier frozen before this wait cannot acquire the nested edge + * retroactively, so the requester must unwind without sleeping. */ +static inline bool +cluster_gcs_pcm_x_nested_guard_retryable(PcmXQueueResult result) +{ + return result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_GATE_RETRY + || result == PCM_X_QUEUE_STALE; +} + +/* A local promotion changes only the role byte. Every locator and immutable + * waiter field must still name the exact same membership; accepting a fresh + * handle with any other change would turn an alias/ABA into writer authority. */ +static inline bool +cluster_gcs_pcm_x_role_refresh_exact(const PcmXLocalHandle *follower, + const PcmXLocalHandle *promoted) +{ + return follower != NULL && promoted != NULL && follower->flags == 0 && promoted->flags == 0 + && follower->role == PCM_X_LOCAL_ROLE_FOLLOWER + && promoted->role == PCM_X_LOCAL_ROLE_NODE_LEADER + && BufferTagsEqual(&follower->identity.tag, &promoted->identity.tag) + && follower->identity.node_id == promoted->identity.node_id + && follower->identity.procno == promoted->identity.procno + && follower->identity.xid == promoted->identity.xid + && follower->identity.cluster_epoch == promoted->identity.cluster_epoch + && follower->identity.request_id == promoted->identity.request_id + && follower->identity.wait_seq == promoted->identity.wait_seq + && follower->identity.base_own_generation == promoted->identity.base_own_generation + && follower->tag_slot.slot_index == promoted->tag_slot.slot_index + && follower->tag_slot.slot_generation == promoted->tag_slot.slot_generation + && follower->membership_slot.slot_index == promoted->membership_slot.slot_index + && follower->membership_slot.slot_generation == promoted->membership_slot.slot_generation + && follower->local_sequence == promoted->local_sequence + && follower->local_round == promoted->local_round; +} + +/* A queue result is meaningful only at the operation that produced it. + * Immutable-token failures (especially STALE) must never enter a generic + * retry loop. An arm may race a newer progress publication; a WFG commit + * may race its exact blocker; both have an explicit recovery action. */ +static inline GcsBlockPcmXRetryAction +cluster_gcs_pcm_x_requester_retry_action(GcsBlockPcmXRequesterSite site, PcmXQueueResult result) +{ + switch (site) { + case GCS_BLOCK_PCM_X_RETRY_SITE_JOIN: + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_NO_CAPACITY) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY: + case GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH: + case GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM: + case GCS_BLOCK_PCM_X_RETRY_SITE_FOLLOWER_SNAPSHOT: + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF: + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_WFG_COMMIT: + if (result == PCM_X_QUEUE_STALE || result == PCM_X_QUEUE_NOT_READY + || result == PCM_X_QUEUE_BUSY || result == PCM_X_QUEUE_GATE_RETRY + || result == PCM_X_QUEUE_BARRIER_CLOSED) + return GCS_BLOCK_PCM_X_RETRY_RESNAPSHOT_WFG; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS: + /* local_progress reports BUSY while a promoted leader's admission + * rekey owns the cross-lock gate. The resident identity is deliberately + * unreadable in that window; wait and reload instead of treating the + * fail-closed snapshot discipline itself as corruption. */ + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM: + if (result == PCM_X_QUEUE_BAD_STATE) + return GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS; + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH: + if (result == PCM_X_QUEUE_NOT_READY || result == PCM_X_QUEUE_BUSY + || result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_NO_CAPACITY) + return GCS_BLOCK_PCM_X_RETRY_WAIT; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM: + if (result == PCM_X_QUEUE_BAD_STATE) + return GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS; + break; + case GCS_BLOCK_PCM_X_RETRY_SITE_WFG_CLEAR: + case GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM: + break; + } + return GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED; +} + +static inline uint32 +cluster_gcs_pcm_x_requester_wait_index_advance(uint32 wait_index) +{ + const uint32 maximum = (uint32)CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS - 1; + + return wait_index < maximum ? wait_index + 1 : maximum; +} + +typedef enum GcsBlockPcmXCleanupAction { + GCS_BLOCK_PCM_X_CLEANUP_NONE = 0, + GCS_BLOCK_PCM_X_CLEANUP_CANCEL_LOCAL, + GCS_BLOCK_PCM_X_CLEANUP_PRESERVE_FAIL_CLOSED +} GcsBlockPcmXCleanupAction; + +/* Before cohort freeze/PREPARE, an exact local membership (and its active + * writer claim) can be unwound without inventing remote state. Once an + * irreversible local or ownership boundary exists, only recovery resolves it. */ +static inline GcsBlockPcmXCleanupAction +cluster_gcs_pcm_x_requester_cleanup_action(bool handle_live, bool claim_live, + bool irreversible_started, bool ownership_committed) +{ + if (irreversible_started || ownership_committed || (claim_live && !handle_live)) + return GCS_BLOCK_PCM_X_CLEANUP_PRESERVE_FAIL_CLOSED; + if (handle_live) + return GCS_BLOCK_PCM_X_CLEANUP_CANCEL_LOCAL; + return GCS_BLOCK_PCM_X_CLEANUP_NONE; +} + +/* PREPARE may publish GRANT_PENDING only from the exact clean N tuple named + * by the active queue identity. A nonzero idle reservation token is valid; + * only live flags denote an overlapping ownership lifecycle. */ +static inline PcmXQueueResult +cluster_gcs_pcm_x_remote_reservation_preflight(const ClusterPcmOwnSnapshot *live, + const PcmXWaitIdentity *identity) +{ + ClusterPcmOwnResult live_result; + + if (live == NULL || identity == NULL) + return PCM_X_QUEUE_INVALID; + if (!BufferTagsEqual(&live->tag, &identity->tag) + || live->generation != identity->base_own_generation) + return PCM_X_QUEUE_STALE; + if (live->generation == UINT64_MAX) + return PCM_X_QUEUE_COUNTER_EXHAUSTED; + live_result = cluster_pcm_own_classify_live_flags(live->flags, live->reservation_token); + if (live_result == CLUSTER_PCM_OWN_CORRUPT) + return PCM_X_QUEUE_CORRUPT; + if (live_result == CLUSTER_PCM_OWN_BUSY) + return PCM_X_QUEUE_BUSY; + if (live->pcm_state != (uint8)PCM_STATE_N) + return PCM_X_QUEUE_STALE; + return PCM_X_QUEUE_OK; +} + +/* A requester-domain id carries the authenticated backend ordinal used by + * the existing per-backend block-reply table. */ +static inline bool +cluster_gcs_requester_id_decode(uint64 request_id, int32 *node_out, int32 *backend_id_out, + uint64 *sequence_out) +{ + uint64 sequence; + int32 node; + int32 backend_id; + + if ((request_id & GCS_REQID_LOCAL_DOMAIN_FLAG) != 0) + return false; + sequence = request_id & GCS_REQID_REQUESTER_SEQ_MASK; + if (sequence == 0) + return false; + node = (int32)((request_id >> GCS_REQID_NODE_SHIFT) & GCS_REQID_NODE_MASK); + backend_id = (int32)(((request_id >> GCS_REQID_BACKEND_SHIFT) & GCS_REQID_BACKEND_MASK) + 1); + if (node < 0 || node >= PCM_X_PROTOCOL_NODE_LIMIT || backend_id <= 0) + return false; + if (node_out != NULL) + *node_out = node; + if (backend_id_out != NULL) + *backend_id_out = backend_id; + if (sequence_out != NULL) + *sequence_out = sequence; + return true; +} + +typedef struct GcsBlockPcmXImageIdentity { + PcmXTicketRef ref; + PcmXImageToken image; +} GcsBlockPcmXImageIdentity; + +typedef struct GcsBlockPcmXImageBinding { + GcsBlockPcmXImageIdentity identity; + uint64 master_session; +} GcsBlockPcmXImageBinding; + +StaticAssertDecl(sizeof(GcsBlockPcmXImageIdentity) == 128, + "PCM-X image identity must fit the existing dedup 128-byte metadata cell"); +StaticAssertDecl(sizeof(GcsBlockPcmXImageBinding) == 136, + "PCM-X image binding is identity plus authenticated master session"); + +static inline bool +GcsBlockPcmXImageIdentityEqual(const GcsBlockPcmXImageIdentity *left, + const GcsBlockPcmXImageIdentity *right) +{ + return left != NULL && right != NULL && memcmp(left, right, sizeof(*left)) == 0; +} + +static inline bool +GcsBlockPcmXImageBindingEqual(const GcsBlockPcmXImageBinding *left, + const GcsBlockPcmXImageBinding *right) +{ + return left != NULL && right != NULL && left->master_session == right->master_session + && GcsBlockPcmXImageIdentityEqual(&left->identity, &right->identity); +} + +/* + * A formation tick may compare the active runtime only after two complete, + * byte-identical authority samples. A membership/session read that changes + * while either sample is collected is transient evidence, not proof of drift; + * the caller leaves ACTIVE unchanged and samples again on the next tick. + */ +static inline bool +cluster_gcs_pcm_x_formation_samples_stable(bool before_complete, + const PcmXPeerBinding before[PCM_X_PROTOCOL_NODE_LIMIT], + bool after_complete, + const PcmXPeerBinding after[PCM_X_PROTOCOL_NODE_LIMIT]) +{ + return before_complete && after_complete && before != NULL && after != NULL + && memcmp(before, after, sizeof(PcmXPeerBinding) * PCM_X_PROTOCOL_NODE_LIMIT) == 0; +} + + +/* Derive the complete holder set and one image source from a coherent GRD + * snapshot. The active queue requester must still own the pending-X gate. + * A canonical N authority and an ordered self-X round use the requester as a + * synthetic source. They still materialize a normal immutable A-record and + * therefore do not introduce a no-image wire arm or bypass the FIFO ticket. */ +static inline PcmXQueueResult +cluster_gcs_pcm_x_authority_holders_exact(const PcmAuthoritySnapshot *authority, + int32 requester_node, uint64 ticket_id, + uint32 *holders_out, int32 *source_out) +{ + uint64 pending_x_value; + + if (holders_out != NULL) + *holders_out = 0; + if (source_out != NULL) + *source_out = -1; + if (authority == NULL || holders_out == NULL || source_out == NULL || requester_node < 0 + || requester_node >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_QUEUE_INVALID; + if (!PcmPendingXQueueValue(ticket_id, &pending_x_value)) + return PCM_X_QUEUE_INVALID; + if (authority->reserved[0] != 0 || authority->reserved[1] != 0) + return PCM_X_QUEUE_CORRUPT; + if (authority->pending_x_requester_node != requester_node + || authority->pending_x_since_lsn != pending_x_value) + return PCM_X_QUEUE_STALE; + if (authority->state == PCM_STATE_N) { + if (authority->x_holder_node != -1 || authority->s_holders_bitmap != 0 + || authority->master_holder.node_id != UINT32_MAX) + return PCM_X_QUEUE_CORRUPT; + *holders_out = UINT32_C(1) << requester_node; + *source_out = requester_node; + return PCM_X_QUEUE_OK; + } + if (authority->state == PCM_STATE_X) { + if (authority->x_holder_node < 0 || authority->x_holder_node >= PCM_X_PROTOCOL_NODE_LIMIT + || authority->s_holders_bitmap != 0 + || authority->master_holder.node_id != (uint32)authority->x_holder_node) + return PCM_X_QUEUE_CORRUPT; + *holders_out = UINT32_C(1) << authority->x_holder_node; + *source_out = authority->x_holder_node; + return PCM_X_QUEUE_OK; + } + if (authority->state != PCM_STATE_S) + return PCM_X_QUEUE_CORRUPT; + if (authority->x_holder_node != -1 || authority->s_holders_bitmap == 0 + || authority->master_holder.node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || (authority->s_holders_bitmap & (UINT32_C(1) << authority->master_holder.node_id)) == 0) + return PCM_X_QUEUE_CORRUPT; + + *holders_out = authority->s_holders_bitmap; + /* A requester that already owns S must be the image source. Invalidating + * its resident S mirror would bump the immutable identity base before the + * later X commit, while FINAL_ACK is exact at base+1. The source handoff + * instead reuses S+REVOKING as GRANT_PENDING and performs the sole bump at + * COMMIT_X. A non-holder requester uses the canonical GRD master_holder. */ + if ((authority->s_holders_bitmap & (UINT32_C(1) << requester_node)) != 0) + *source_out = requester_node; + else + *source_out = (int32)authority->master_holder.node_id; + return PCM_X_QUEUE_OK; +} + + +/* Select one deterministic holder while preserving an exact bitmap ledger. */ +static inline PcmXQueueResult +cluster_gcs_pcm_x_next_unacked_holder(uint32 pending, uint32 acked, int32 *holder_out) +{ + uint32 unacked; + int32 node; + + if (holder_out == NULL) + return PCM_X_QUEUE_INVALID; + *holder_out = -1; + if ((acked & ~pending) != 0) + return PCM_X_QUEUE_CORRUPT; + unacked = pending & ~acked; + if (unacked == 0) + return PCM_X_QUEUE_NOT_FOUND; + for (node = 0; node < PCM_X_PROTOCOL_NODE_LIMIT; node++) { + if ((unacked & (UINT32_C(1) << node)) != 0) { + *holder_out = node; + return PCM_X_QUEUE_OK; + } + } + return PCM_X_QUEUE_CORRUPT; +} + + +/* Build the process-local GRD commit token from an already validated engine + * FINAL_ACK barrier. No wire or shared-memory layout is changed here. */ +static inline bool +cluster_gcs_pcm_x_grd_handoff_token_build(const PcmXMasterFinalAckToken *final, + const PcmAuthoritySnapshot *authority, + PcmXGrdHandoffToken *handoff_out) +{ + const PcmXWaitIdentity *identity; + + if (handoff_out != NULL) + memset(handoff_out, 0, sizeof(*handoff_out)); + if (final == NULL || authority == NULL || handoff_out == NULL) + return false; + identity = &final->final_ack.ref.identity; + if (identity->node_id < 0 || identity->node_id >= PCM_X_PROTOCOL_NODE_LIMIT + || identity->request_id == 0 || final->final_ack.ref.handle.ticket_id == 0 + || final->final_ack.ref.grant_generation == 0 || final->final_ack.image_id == 0 + || final->final_ack.committed_own_generation == 0 || final->image.image_id == 0 + || final->image.image_id != final->final_ack.image_id + || final->image.source_node >= PCM_X_PROTOCOL_NODE_LIMIT) + return false; + handoff_out->tag = identity->tag; + handoff_out->authority = *authority; + handoff_out->cluster_epoch = identity->cluster_epoch; + handoff_out->request_id = identity->request_id; + handoff_out->ticket_id = final->final_ack.ref.handle.ticket_id; + handoff_out->grant_generation = final->final_ack.ref.grant_generation; + handoff_out->image_id = final->image.image_id; + handoff_out->source_own_generation = final->image.source_own_generation; + handoff_out->page_scn = (SCN) final->image.page_scn; + handoff_out->page_lsn = final->image.page_lsn; + handoff_out->requester_node = identity->node_id; + handoff_out->source_node = (int32) final->image.source_node; + handoff_out->requester_procno = identity->procno; + handoff_out->page_checksum = final->image.page_checksum; + return true; +} + + +static inline bool +cluster_gcs_pcm_x_holder_image_identity_exact(const PcmXLocalHolderProgress *progress, + const PcmXTicketRef *ref, uint64 image_id, + int32 source_node) +{ + return progress != NULL && ref != NULL && image_id != 0 && source_node >= 0 + && source_node < PCM_X_PROTOCOL_NODE_LIMIT + && memcmp(&progress->ref, ref, sizeof(*ref)) == 0 && progress->image.image_id == image_id + && progress->image.source_node == (uint32)source_node; +} + + +/* image_id exists from REVOKE onward. The reliable leg, not generation, + * proves that the rest of the image token has been published. */ +static inline bool +cluster_gcs_pcm_x_holder_image_ready_exact(const PcmXLocalHolderProgress *progress, + const PcmXTicketRef *ref, uint64 image_id, + int32 source_node) +{ + return cluster_gcs_pcm_x_holder_image_identity_exact(progress, ref, image_id, source_node) + && progress->pending_opcode == PGRAC_IC_MSG_PCM_X_IMAGE_READY + && progress->phase == PGRAC_IC_MSG_PCM_X_IMAGE_READY + && progress->last_response_opcode == PGRAC_IC_MSG_PCM_X_REVOKE + && (progress->flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) == 0; +} + + +/* DRAIN clears the reliable leg but retains the exact holder image until + * RETIRE. This is the post-transition capture used to release the dedup and + * retained-image substrates without treating generation zero as absence. */ +static inline bool +cluster_gcs_pcm_x_holder_image_drained_exact(const PcmXLocalHolderProgress *progress, + const PcmXTicketRef *ref, uint64 image_id, + int32 source_node) +{ + return cluster_gcs_pcm_x_holder_image_identity_exact(progress, ref, image_id, source_node) + && progress->pending_opcode == 0 && progress->phase == 0 + && progress->last_response_opcode == PGRAC_IC_MSG_PCM_X_DRAIN_POLL + && (progress->flags & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK) + == PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; +} + + +/* Before the GRD handoff, 0/REVOKE is the only transfer phase that still + * depends on the pending-X cookie. Later reliable phases carry their own + * exact replay evidence after the handoff has atomically cleared that cookie. */ +static inline bool +cluster_gcs_pcm_x_transfer_pre_handoff_phase(uint16 pending_opcode) +{ + return pending_opcode == 0 || pending_opcode == PGRAC_IC_MSG_PCM_X_REVOKE; +} + + +/* A graph published for this confirm must be removed by the full waiter + * identity if engine revalidation does not accept that exact publication. */ +static inline bool +cluster_gcs_pcm_x_confirm_compensation_required(uint64 graph_generation, + PcmXQueueResult revalidate_result) +{ + return graph_generation != 0 && revalidate_result != PCM_X_QUEUE_OK + && revalidate_result != PCM_X_QUEUE_DUPLICATE; +} + + +/* + * Authenticate a tag-bearing PCM-X admission request before it can touch the + * master queue. Session/frontier and process-capacity checks remain in the + * queue engine; this boundary binds the immutable identity to the DATA + * connection, current epoch, and authoritative tag master. + */ +static inline bool +cluster_gcs_pcm_x_enqueue_ingress_valid(const PcmXEnqueuePayload *request, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->identity.node_id == authenticated_node + && request->identity.cluster_epoch == current_epoch && tag_master == local_node + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->identity.request_id != 0 && request->identity.wait_seq != 0 + && request->prehandle.sender_session_incarnation != 0 + && request->prehandle.prehandle_sequence != 0; +} + +static inline bool +cluster_gcs_pcm_x_admit_confirm_ingress_valid(const PcmXPhasePayload *request, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->ref.identity.node_id == authenticated_node + && request->ref.identity.cluster_epoch == current_epoch && tag_master == local_node + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->ref.identity.request_id != 0 && request->ref.identity.wait_seq != 0 + && request->ref.handle.ticket_id != 0 && request->ref.handle.queue_generation != 0 + && request->ref.grant_generation == 0 && request->reason == 0 + && request->phase == PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM && request->flags == 0; +} + +static inline bool +cluster_gcs_pcm_x_admit_confirm_ack_ingress_valid(const PcmXPhasePayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && ack->ref.identity.node_id == local_node + && ack->ref.identity.cluster_epoch == current_epoch && tag_master == authenticated_node + && ack->ref.identity.request_id != 0 && ack->ref.identity.wait_seq != 0 + && ack->ref.handle.ticket_id != 0 && ack->ref.handle.queue_generation != 0 + && ack->ref.grant_generation == 0 && ack->reason == 0 + && ack->phase == PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM && ack->flags == 0; +} + +static inline bool +cluster_gcs_pcm_x_ticket_ref_wire_valid(const PcmXTicketRef *ref, uint64 current_epoch) +{ + return ref != NULL && ref->identity.node_id >= 0 + && ref->identity.node_id < PCM_X_PROTOCOL_NODE_LIMIT + && ref->identity.cluster_epoch == current_epoch && ref->identity.request_id != 0 + && ref->identity.wait_seq != 0 && ref->handle.ticket_id != 0 + && ref->handle.queue_generation != 0 && ref->grant_generation == 0; +} + +/* BLOCKER_SET originates at the probed holder, not at the ticket requester. */ +static inline bool +cluster_gcs_pcm_x_blocker_header_ingress_valid(const PcmXBlockerSetHeaderPayload *header, + Size payload_length, int32 authenticated_node, + uint64 current_epoch, int32 tag_master, + int32 local_node) +{ + return header != NULL && payload_length == sizeof(*header) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && cluster_gcs_pcm_x_ticket_ref_wire_valid(&header->ref, current_epoch) + && header->set_generation != 0 && header->set_generation != UINT64_MAX + && header->nblockers <= INT_MAX; +} + +static inline bool +cluster_gcs_pcm_x_blocker_edge_ingress_valid(const PcmXBlockerChunkPayload *edge, + Size payload_length, int32 authenticated_node, + uint64 current_epoch, int32 tag_master, + int32 local_node) +{ + return edge != NULL && payload_length == sizeof(*edge) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && edge->requester_node >= 0 && edge->requester_node < PCM_X_PROTOCOL_NODE_LIMIT + && edge->cluster_epoch == current_epoch && edge->request_id != 0 + && edge->handle.ticket_id != 0 && edge->handle.queue_generation != 0 + && edge->grant_generation == 0 && edge->set_generation != 0 + && edge->set_generation != UINT64_MAX && edge->blocker.node_id == authenticated_node + && edge->blocker.cluster_epoch == current_epoch && edge->blocker.wait_seq != 0 + && (edge->blocker.request_id != 0 || TransactionIdIsValid(edge->blocker.xid)); +} + +/* + * PcmXPhasePayload has exactly 64 non-ref bits at bytes [88,96). Type 48 + * uses all of them as one lossless set_generation: zero is the + * master-to-holder PROBE arm; nonzero is the generation-exact ACK arm, so a + * delayed ACK can never acknowledge a newer set on the same ticket. + */ +static inline void +cluster_gcs_pcm_x_blocker_ack_set_generation(PcmXPhasePayload *ack, uint64 set_generation) +{ + if (ack == NULL) + return; + ack->reason = (uint32)set_generation; + ack->phase = (uint16)(set_generation >> 32); + ack->flags = (uint16)(set_generation >> 48); +} + +static inline uint64 +cluster_gcs_pcm_x_blocker_ack_generation(const PcmXPhasePayload *ack) +{ + if (ack == NULL) + return 0; + return (uint64)ack->reason | ((uint64)ack->phase << 32) | ((uint64)ack->flags << 48); +} + +/* Build only the nonzero ACK arm. Invalid generations leave a zeroed + * payload, so production callers cannot accidentally emit the reserved + * all-zero PROBE encoding in non-assert builds. */ +static inline bool +cluster_gcs_pcm_x_blocker_ack_build(const PcmXTicketRef *ref, uint64 set_generation, + PcmXPhasePayload *ack) +{ + if (ack == NULL) + return false; + memset(ack, 0, sizeof(*ack)); + if (ref == NULL || set_generation == 0 || set_generation == UINT64_MAX) + return false; + ack->ref = *ref; + cluster_gcs_pcm_x_blocker_ack_set_generation(ack, set_generation); + return true; +} + +static inline bool +cluster_gcs_pcm_x_blocker_control_ingress_valid(const PcmXPhasePayload *control, + Size payload_length, int32 authenticated_node, + uint64 current_epoch, int32 tag_master, + int32 local_node) +{ + return control != NULL && payload_length == sizeof(*control) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && cluster_gcs_pcm_x_ticket_ref_wire_valid(&control->ref, current_epoch); +} + +static inline bool +cluster_gcs_pcm_x_blocker_probe_ingress_valid(const PcmXPhasePayload *probe, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return cluster_gcs_pcm_x_blocker_control_ingress_valid( + probe, payload_length, authenticated_node, current_epoch, tag_master, local_node) + && cluster_gcs_pcm_x_blocker_ack_generation(probe) == 0; +} + +static inline bool +cluster_gcs_pcm_x_blocker_ack_ingress_valid(const PcmXPhasePayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + uint64 set_generation = cluster_gcs_pcm_x_blocker_ack_generation(ack); + + return cluster_gcs_pcm_x_blocker_control_ingress_valid(ack, payload_length, authenticated_node, + current_epoch, tag_master, local_node) + && set_generation != 0 && set_generation != UINT64_MAX; +} + +/* Transfer messages are legal only after the master assigns a grant generation. */ +static inline bool +cluster_gcs_pcm_x_transfer_ref_wire_valid(const PcmXTicketRef *ref, uint64 current_epoch) +{ + return ref != NULL && ref->identity.node_id >= 0 + && ref->identity.node_id < PCM_X_PROTOCOL_NODE_LIMIT + && ref->identity.cluster_epoch == current_epoch && ref->identity.request_id != 0 + && ref->identity.wait_seq != 0 && ref->handle.ticket_id != 0 + && ref->handle.queue_generation != 0 && ref->grant_generation != 0; +} + +static inline bool +cluster_gcs_pcm_x_image_id_master_wire_valid(uint64 image_id, int32 expected_master_node) +{ + int32 encoded_master_node; + + return expected_master_node >= 0 && expected_master_node < PCM_X_PROTOCOL_NODE_LIMIT + && cluster_pcm_x_image_id_decode(image_id, &encoded_master_node, NULL) + && encoded_master_node == expected_master_node; +} + +static inline bool +cluster_gcs_pcm_x_image_token_wire_valid(const PcmXImageToken *image, int32 expected_master_node) +{ + return image != NULL && image->source_node < PCM_X_PROTOCOL_NODE_LIMIT + && cluster_gcs_pcm_x_image_id_master_wire_valid(image->image_id, expected_master_node); +} + +/* REVOKE is master -> current holder; the requester identity is third-party. */ +static inline bool +cluster_gcs_pcm_x_revoke_ingress_valid(const PcmXRevokePayload *request, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&request->ref, current_epoch) + && cluster_gcs_pcm_x_image_id_master_wire_valid(request->image_id, authenticated_node); +} + +/* IMAGE_READY is current holder -> master and binds the image to that holder. */ +static inline bool +cluster_gcs_pcm_x_image_ready_ingress_valid(const PcmXGrantPayload *ready, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return ready != NULL && payload_length == sizeof(*ready) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&ready->ref, current_epoch) + && cluster_gcs_pcm_x_image_token_wire_valid(&ready->image, local_node) + && ready->image.source_node == (uint32)authenticated_node; +} + +/* PREPARE_GRANT is master -> exact requester and may carry a third-party image. */ +static inline bool +cluster_gcs_pcm_x_prepare_grant_ingress_valid(const PcmXGrantPayload *grant, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return grant != NULL && payload_length == sizeof(*grant) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&grant->ref, current_epoch) + && grant->ref.identity.node_id == local_node + && cluster_gcs_pcm_x_image_token_wire_valid(&grant->image, authenticated_node); +} + +/* INSTALL_READY is one canonical requester -> master application ACK. */ +static inline bool +cluster_gcs_pcm_x_install_ready_ingress_valid(const PcmXInstallReadyPayload *ready, + Size payload_length, int32 authenticated_node, + uint64 current_epoch, int32 tag_master, + int32 local_node) +{ + return ready != NULL && payload_length == sizeof(*ready) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&ready->ref, current_epoch) + && ready->ref.identity.node_id == authenticated_node + && cluster_gcs_pcm_x_image_id_master_wire_valid(ready->image_id, local_node) + && ready->result == PCM_X_QUEUE_OK && ready->phase == PGRAC_IC_MSG_PCM_X_INSTALL_READY + && ready->flags == 0; +} + +/* COMMIT_X is one canonical master -> exact requester application command. */ +static inline bool +cluster_gcs_pcm_x_commit_x_ingress_valid(const PcmXPhasePayload *commit, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return commit != NULL && payload_length == sizeof(*commit) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&commit->ref, current_epoch) + && commit->ref.identity.node_id == local_node && commit->reason == 0 + && commit->phase == PGRAC_IC_MSG_PCM_X_COMMIT_X && commit->flags == 0; +} + +/* FINAL_ACK proves the requester committed exactly one ownership generation. */ +static inline bool +cluster_gcs_pcm_x_final_ack_ingress_valid(const PcmXFinalAckPayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&ack->ref, current_epoch) + && ack->ref.identity.node_id == authenticated_node + && cluster_gcs_pcm_x_image_id_master_wire_valid(ack->image_id, local_node) + && ack->ref.identity.base_own_generation != UINT64_MAX + && ack->committed_own_generation == ack->ref.identity.base_own_generation + 1; +} + +/* FINAL_COMMIT_ACK is the canonical master -> requester application ACK. */ +static inline bool +cluster_gcs_pcm_x_final_commit_ack_ingress_valid(const PcmXPhasePayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&ack->ref, current_epoch) + && ack->ref.identity.node_id == local_node && ack->reason == 0 + && ack->phase == PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK && ack->flags == 0; +} + +/* FINAL_CONFIRM is the canonical requester -> master terminal confirmation. */ +static inline bool +cluster_gcs_pcm_x_final_confirm_ingress_valid(const PcmXPhasePayload *confirm, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return confirm != NULL && payload_length == sizeof(*confirm) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && cluster_gcs_pcm_x_transfer_ref_wire_valid(&confirm->ref, current_epoch) + && confirm->ref.identity.node_id == authenticated_node && confirm->reason == 0 + && confirm->phase == PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM && confirm->flags == 0; +} + +static inline bool +cluster_gcs_pcm_x_prehandle_cancel_ingress_valid(const PcmXPrehandleCancelPayload *request, + Size payload_length, int32 authenticated_node, + uint64 current_epoch, int32 tag_master, + int32 local_node) +{ + return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->identity.node_id == authenticated_node + && request->identity.cluster_epoch == current_epoch && tag_master == local_node + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->identity.request_id != 0 && request->identity.wait_seq != 0 + && request->prehandle.sender_session_incarnation != 0 + && request->prehandle.prehandle_sequence != 0; +} + +static inline bool +cluster_gcs_pcm_x_prehandle_cancel_ack_ingress_valid(const PcmXAdmitAckPayload *ack, + Size payload_length, int32 authenticated_node, + uint64 current_epoch, int32 tag_master, + int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && ack->ref.identity.node_id == local_node + && ack->ref.identity.cluster_epoch == current_epoch && tag_master == authenticated_node + && ack->ref.identity.request_id != 0 && ack->ref.identity.wait_seq != 0 + && ack->ref.handle.ticket_id != 0 && ack->ref.handle.queue_generation != 0 + && ack->ref.grant_generation == 0 && ack->prehandle.sender_session_incarnation != 0 + && ack->prehandle.prehandle_sequence != 0 && ack->result == PCM_X_QUEUE_OK + && ack->phase == PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL && ack->flags == 0; +} + +static inline bool +cluster_gcs_pcm_x_cancel_ingress_valid(const PcmXPhasePayload *request, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->ref.identity.node_id == authenticated_node + && request->ref.identity.cluster_epoch == current_epoch && tag_master == local_node + && local_node >= 0 && local_node < PCM_X_PROTOCOL_NODE_LIMIT + && request->ref.identity.request_id != 0 && request->ref.identity.wait_seq != 0 + && request->ref.handle.ticket_id != 0 && request->ref.handle.queue_generation != 0 + && request->ref.grant_generation == 0 && request->reason == 0 + && request->phase == PCM_X_LOCAL_RELIABLE_PHASE_CANCEL && request->flags == 0; +} + +static inline bool +cluster_gcs_pcm_x_cancel_ack_ingress_valid(const PcmXPhasePayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && ack->ref.identity.node_id == local_node + && ack->ref.identity.cluster_epoch == current_epoch && tag_master == authenticated_node + && ack->ref.identity.request_id != 0 && ack->ref.identity.wait_seq != 0 + && ack->ref.handle.ticket_id != 0 && ack->ref.handle.queue_generation != 0 + && ack->ref.grant_generation == 0 && ack->reason == 0 + && ack->phase == PCM_X_LOCAL_RELIABLE_PHASE_CANCEL && ack->flags == 0; +} + +/* Terminal messages bind the final, non-sentinel grant generation as well. */ +static inline bool +cluster_gcs_pcm_x_terminal_ref_wire_valid(const PcmXTicketRef *ref, uint64 current_epoch) +{ + return ref != NULL && ref->identity.node_id >= 0 + && ref->identity.node_id < PCM_X_PROTOCOL_NODE_LIMIT + && ref->identity.cluster_epoch == current_epoch && ref->identity.request_id != 0 + && ref->identity.wait_seq != 0 && ref->handle.ticket_id != 0 + && ref->handle.queue_generation != 0 && ref->grant_generation != UINT64_MAX; +} + +/* DRAIN_POLL is authoritative only from the current master of this tag. */ +static inline bool +cluster_gcs_pcm_x_drain_poll_ingress_valid(const PcmXDrainPollPayload *poll, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return poll != NULL && payload_length == sizeof(*poll) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == authenticated_node + && cluster_gcs_pcm_x_terminal_ref_wire_valid(&poll->ref, current_epoch) + && poll->drain_generation != 0 && poll->drain_generation != UINT64_MAX; +} + +/* DRAIN_ACK may come from any involved participant, but only to this master. */ +static inline bool +cluster_gcs_pcm_x_drain_ack_ingress_valid(const PcmXPhasePayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + int32 tag_master, int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && tag_master == local_node + && cluster_gcs_pcm_x_terminal_ref_wire_valid(&ack->ref, current_epoch) + && ack->reason == 0 && ack->phase == 0 && ack->flags == 0; +} + +/* RETIRE_UP_TO is tagless; bind it to the DATA peer session and local target. */ +static inline bool +cluster_gcs_pcm_x_retire_request_ingress_valid(const PcmXRetirePayload *request, + Size payload_length, int32 authenticated_node, + uint64 authenticated_session, uint64 current_epoch, + int32 local_node) +{ + return request != NULL && payload_length == sizeof(*request) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && authenticated_session != 0 + && request->cluster_epoch == current_epoch + && request->master_session_incarnation == authenticated_session + && request->retire_through_ticket_id != 0 && request->sender_node == local_node + && request->flags == 0; +} + +/* RETIRE_ACK names its authenticated responder and this master's session. */ +static inline bool +cluster_gcs_pcm_x_retire_ack_ingress_valid(const PcmXRetirePayload *ack, Size payload_length, + int32 authenticated_node, uint64 current_epoch, + uint64 master_session, int32 local_node) +{ + return ack != NULL && payload_length == sizeof(*ack) && authenticated_node >= 0 + && authenticated_node < PCM_X_PROTOCOL_NODE_LIMIT && local_node >= 0 + && local_node < PCM_X_PROTOCOL_NODE_LIMIT && ack->cluster_epoch == current_epoch + && master_session != 0 && ack->master_session_incarnation == master_session + && ack->retire_through_ticket_id != 0 && ack->sender_node == authenticated_node + && ack->flags == 0; +} + +static inline void +cluster_gcs_pcm_x_vertex_from_identity(const PcmXWaitIdentity *identity, ClusterLmdVertex *vertex) +{ + if (vertex == NULL) + return; + memset(vertex, 0, sizeof(*vertex)); + if (identity == NULL) + return; + vertex->node_id = identity->node_id; + vertex->procno = identity->procno; + vertex->cluster_epoch = identity->cluster_epoch; + vertex->request_id = identity->request_id; + vertex->xid = identity->xid; + vertex->wait_seq = identity->wait_seq; +} + /* ============================================================ * GCS_BLOCK_DATA_SIZE -- block bytes carried in every reply. * @@ -561,6 +1591,51 @@ GcsBlockInvalidateAckPayloadGetPageScn(const GcsBlockInvalidateAckPayload *p) * to the round-5 park). */ #define GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY 5 +/* Slot occupancy, not epoch value, proves that an outstanding request exists. + * The first legal reconfiguration advances epoch 0 to 1 and must wake those + * initial-formation requests just like every later epoch transition. */ +static inline bool +cluster_gcs_block_epoch_advance_stales_slot(bool in_use, uint64 request_epoch, uint64 new_epoch) +{ + return in_use && request_epoch < new_epoch; +} + +/* + * Classify an INVALIDATE_ACK against one exact queue transfer round. A + * nonmatching tag/epoch/request belongs to the legacy invalidate machinery; + * once that identity matches, however, an unexpected or already-credited + * sender is consumed here so it cannot mutate GRD through the legacy + * slotless path. The caller authenticates the transport/session separately. + */ +static inline PcmXQueueResult +cluster_gcs_pcm_x_invalidate_ack_match_exact(const PcmXMasterDriveSnapshot *snapshot, + const GcsBlockInvalidateAckPayload *ack, + uint64 current_epoch, int32 authenticated_source) +{ + uint32 source_bit; + + if (snapshot == NULL || ack == NULL || authenticated_source < 0 + || authenticated_source >= PCM_X_PROTOCOL_NODE_LIMIT) + return PCM_X_QUEUE_INVALID; + if (snapshot->ticket_state != PCM_XT_ACTIVE_TRANSFER + || snapshot->ref.identity.cluster_epoch != current_epoch || ack->epoch != current_epoch + || ack->request_id != snapshot->ref.identity.request_id + || !BufferTagsEqual(&ack->tag, &snapshot->ref.identity.tag)) + return PCM_X_QUEUE_NOT_FOUND; + if (ack->sender_node != authenticated_source) + return PCM_X_QUEUE_STALE; + source_bit = UINT32_C(1) << (uint32)authenticated_source; + if ((snapshot->pending_s_holders_bitmap & source_bit) == 0) + return PCM_X_QUEUE_STALE; + if ((snapshot->acked_s_holders_bitmap & source_bit) != 0) + return PCM_X_QUEUE_DUPLICATE; + if (ack->ack_status == GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY) + return PCM_X_QUEUE_BUSY; + if (ack->ack_status == 0 || ack->ack_status == 2) + return PCM_X_QUEUE_OK; + return PCM_X_QUEUE_BAD_STATE; +} + /* ============================================================ * GcsBlockRedeclarePayload -- wire ABI for PGRAC_IC_MSG_GCS_BLOCK_REDECLARE @@ -2257,6 +3332,9 @@ extern uint64 cluster_gcs_get_invalidate_parked_count(void); extern uint64 cluster_gcs_get_invalidate_park_expired_count(void); extern uint64 cluster_gcs_get_invalidate_busy_sent_count(void); extern uint64 cluster_gcs_get_invalidate_busy_received_count(void); +extern uint64 cluster_gcs_get_invalidate_passive_s_release_count(void); +extern uint64 cluster_gcs_get_pcm_x_self_handoff_count(void); +extern uint64 cluster_gcs_get_pcm_x_self_handoff_drain_count(void); extern uint64 cluster_gcs_get_invalidate_park_overflow_count(void); extern uint64 cluster_gcs_get_drop_pinned_deny_count(void); extern uint64 cluster_gcs_get_xfer_stale_deny_count(void); @@ -2338,9 +3416,13 @@ extern uint64 cluster_gcs_get_block_dedup_done_marked_count(void); /* RC-F DON extern uint64 cluster_gcs_get_block_dedup_done_mismatch_count(void); /* RC-F DONE */ extern uint64 cluster_gcs_get_block_dedup_hint_violation_count(void); /* review F5 */ extern uint64 cluster_gcs_get_block_dedup_legacy_pin_count(void); /* review F5 */ -extern uint64 cluster_gcs_get_fallback_scn_verify_pass_count(void); /* round-4c FUNC-1 */ -extern uint64 cluster_gcs_get_fallback_scn_refresh_count(void); /* round-4c FUNC-1 */ -extern uint64 cluster_gcs_get_fallback_scn_failclosed_count(void); /* round-4c FUNC-1 */ +extern uint64 cluster_gcs_get_block_dedup_pcm_x_stage_count(void); +extern uint64 cluster_gcs_get_block_dedup_pcm_x_replay_count(void); +extern uint64 cluster_gcs_get_block_dedup_pcm_x_release_count(void); +extern uint64 cluster_gcs_get_block_dedup_pcm_x_failclosed_count(void); +extern uint64 cluster_gcs_get_fallback_scn_verify_pass_count(void); /* round-4c FUNC-1 */ +extern uint64 cluster_gcs_get_fallback_scn_refresh_count(void); /* round-4c FUNC-1 */ +extern uint64 cluster_gcs_get_fallback_scn_failclosed_count(void); /* round-4c FUNC-1 */ /* * PGRAC: spec-2.35 D12 — 7 NEW reliability/lifecycle counter accessors @@ -2478,6 +3560,34 @@ extern void cluster_gcs_block_lmon_handle_direct_land_completion(int32 peer_node extern void cluster_gcs_block_lmon_abort_direct_land_peer(int32 peer_node, ClusterGcsBlockDirectAbortReason reason); extern int cluster_gcs_block_lmon_drain_direct_land_aborts(void); +extern void cluster_gcs_block_pcm_x_formation_tick(void); +extern void cluster_gcs_block_pcm_x_owner_start(int worker_id); +extern void cluster_gcs_block_pcm_x_image_pump_tick(int worker_id); +extern void cluster_gcs_pcm_x_terminal_kick(const PcmXTicketRef *ref); +/* Type-48 zero-generation arm. One call stages at most one exact PROBE; + * duplicate calls replay the same durable master leg. */ +extern void cluster_gcs_pcm_x_blocker_probe_kick(const PcmXTicketRef *ref, int32 holder_node); + +/* Stage one PCM-X application frame onto the BufferTag-owning DATA worker. + * This is also the loopback path when the target is this node: the LMS worker + * dispatches the frame locally instead of relying on the IC self-send no-op. */ +extern bool cluster_gcs_pcm_x_stage_frame(uint8 msg_type, int32 dest_node_id, const void *payload, + uint16 payload_len); +/* Join the node-local FIFO, drive the sole remote queue request when this + * backend is leader, and return one exact writer claim. claim_handed_off is + * published before requester cleanup authority is dropped, closing the + * owner-exit gap while bufmgr adopts the claim. The caller must keep the + * claim through content-lock ownership and release it after UNLOCK. */ +extern PcmXQueueResult cluster_gcs_pcm_x_acquire_writer(BufferDesc *buf, + PcmXLocalWriterClaim *claim_out, + bool *claim_handed_off); +extern PcmXQueueResult +cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(const PcmXLocalWriterClaim *claim); +/* Error/owner-exit cleanup adapter: never propagates ERROR. A caught ERROR + * closes the runtime and returns CORRUPT while leaving the exact claim in the + * caller's ledger for fail-closed evidence/retry. */ +extern PcmXQueueResult +cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(const PcmXLocalWriterClaim *claim); /* ============================================================ diff --git a/src/include/cluster/cluster_gcs_block_dedup.h b/src/include/cluster/cluster_gcs_block_dedup.h index 67123546208..3d8e220fb64 100644 --- a/src/include/cluster/cluster_gcs_block_dedup.h +++ b/src/include/cluster/cluster_gcs_block_dedup.h @@ -88,13 +88,15 @@ StaticAssertDecl(sizeof(GcsBlockDedupKey) == 24, "spec-2.34 D2 GcsBlockDedupKey * [ 24, 44) tag BufferTag (20B) HC91 * [ 44, 45) transition_id uint8 HC91 * [ 45, 46) status uint8 (GcsBlockReplyStatus) - * [ 46, 56) _pad0[10] explicit pad to 8-align + * [ 46, 47) entry_kind uint8 + * [ 47, 48) _pad0 explicit pad to 8-align + * [ 48, 56) pcm_x_master_session uint64 (PCM-X kind only) * [ 56, 104) reply_header GcsBlockReplyHeader (48B) * [ 104, 105) has_sf_dep bool spec-6.2 * [ 105, 106) sf_flags uint8 spec-6.2 * [ 106, 107) sf_dep_count uint8 spec-6.2 * [ 107, 112) _pad1[5] explicit pad to 8-align - * [ 112, 240) sf_dep_vec ClusterSfDepVec spec-6.2 + * [ 112, 240) payload_meta sf_dep_vec or PCM-X identity * [ 240, 8432) block_data char[GCS_BLOCK_DATA_SIZE] * [ 8432, 8440) completed_at_ts TimestampTz (TTL sweep — replied) * [ 8440, 8448) registered_at_ts TimestampTz (TTL sweep — in-flight) @@ -127,6 +129,12 @@ StaticAssertDecl(sizeof(GcsBlockDedupKey) == 24, "spec-2.34 D2 GcsBlockDedupKey * reclaim-SAFE under cap pressure immediately * (the completion proof is exactly what the * §3.1 in-window whitelist was waiting for). + * For PCM-X IMAGE entries only, the same existing + * cell is an outbound-admission marker: nonzero + * suppresses type-50 resends until exact type 49 + * clears it. It is never application completion; + * exact DRAIN removes the dedicated entry, and all + * generic reclaim paths reject non-GENERIC kinds. * pinned_lifetime_us the legal-request-lifetime threshold, pinned * at REGISTRATION from the requester's wire * hint (2x margin applied) or, absent a hint, @@ -138,31 +146,85 @@ StaticAssertDecl(sizeof(GcsBlockDedupKey) == 24, "spec-2.34 D2 GcsBlockDedupKey * registration (2x the reply-timeout then in * force). * ============================================================ */ +typedef enum GcsBlockDedupEntryKind { + GCS_BLOCK_DEDUP_ENTRY_GENERIC = 0, + GCS_BLOCK_DEDUP_ENTRY_PCM_X_RESERVED = 1, + GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE = 2, + /* Stable bytes exist, but X->N has not yet been positively committed. + * This state is retained across LMS death and is never sendable. */ + GCS_BLOCK_DEDUP_ENTRY_PCM_X_MATERIALIZED_UNCOMMITTED = 3 +} GcsBlockDedupEntryKind; + +typedef union GcsBlockDedupPayloadMeta { + ClusterSfDepVec sf_dep_vec; + GcsBlockPcmXImageIdentity pcm_x_identity; +} GcsBlockDedupPayloadMeta; + +StaticAssertDecl(sizeof(GcsBlockDedupPayloadMeta) == 128, + "dedup payload metadata remains the established 128-byte cell"); + typedef struct GcsBlockDedupEntry { - GcsBlockDedupKey key; /* 24B — HTAB key */ - BufferTag tag; /* 20B — HC91 collision check */ - uint8 transition_id; /* 1B — HC91 collision check */ - uint8 status; /* 1B — GcsBlockReplyStatus */ - uint8 _pad0[10]; /* 10B — explicit pad; header @ 56 */ - GcsBlockReplyHeader reply_header; /* 48B — full reply header (HC99) */ - bool has_sf_dep; /* 1B — spec-6.2 v2 dep vector present */ - uint8 sf_flags; /* 1B — GCS_BLOCK_REPLY_SF_* */ - uint8 sf_dep_count; /* 1B — non-empty dep vector entries */ - uint8 _pad1[5]; /* 5B — dep_vec @ 112 */ - ClusterSfDepVec sf_dep_vec; /* 128B — spec-6.2 cached v2 deps */ - char block_data[GCS_BLOCK_DATA_SIZE]; /* 8192B — full page payload */ - TimestampTz completed_at_ts; /* 8B — TTL sweep replied */ - TimestampTz registered_at_ts; /* 8B — TTL sweep in-flight */ - TimestampTz done_at_ts; /* 8B — round-2: DONE proof consumed */ - int64 pinned_lifetime_us; /* 8B — round-2: TTL pinned at register */ - int64 pinned_done_linger_us; /* 8B — round-2: quarantine pinned */ + GcsBlockDedupKey key; /* 24B — HTAB key */ + BufferTag tag; /* 20B — HC91 collision check */ + uint8 transition_id; /* 1B — HC91 collision check */ + uint8 status; /* 1B — GcsBlockReplyStatus */ + uint8 entry_kind; /* 1B — GcsBlockDedupEntryKind */ + uint8 _pad0; /* 1B — session @ 48 */ + uint64 pcm_x_master_session; /* 8B — PCM-X kind only */ + GcsBlockReplyHeader reply_header; /* 48B — full reply header (HC99) */ + bool has_sf_dep; /* 1B — spec-6.2 v2 dep vector present */ + uint8 sf_flags; /* 1B — GCS_BLOCK_REPLY_SF_* */ + uint8 sf_dep_count; /* 1B — non-empty dep vector entries */ + uint8 _pad1[5]; /* 5B — dep_vec @ 112 */ + GcsBlockDedupPayloadMeta payload_meta; /* 128B — deps or exact PCM-X identity */ + char block_data[GCS_BLOCK_DATA_SIZE]; /* 8192B — full page payload */ + TimestampTz completed_at_ts; /* 8B — TTL sweep replied */ + TimestampTz registered_at_ts; /* 8B — TTL sweep in-flight */ + TimestampTz done_at_ts; /* 8B — round-2: DONE proof consumed */ + int64 pinned_lifetime_us; /* 8B — round-2: TTL pinned at register */ + int64 pinned_done_linger_us; /* 8B — round-2: quarantine pinned */ } GcsBlockDedupEntry; -StaticAssertDecl(offsetof(GcsBlockDedupEntry, sf_dep_vec) == 112, - "spec-6.2 GcsBlockDedupEntry dep vector offset must be 112"); +StaticAssertDecl(offsetof(GcsBlockDedupEntry, entry_kind) == 46, + "dedup entry kind occupies established padding at offset 46"); +StaticAssertDecl(offsetof(GcsBlockDedupEntry, pcm_x_master_session) == 48, + "PCM-X master session occupies established padding at offset 48"); +StaticAssertDecl(offsetof(GcsBlockDedupEntry, reply_header) == 56, + "dedup reply header offset remains 56"); +StaticAssertDecl(offsetof(GcsBlockDedupEntry, payload_meta) == 112, + "dedup payload metadata offset remains 112"); StaticAssertDecl(sizeof(GcsBlockDedupEntry) == 8472, "GcsBlockDedupEntry 8472B (8448 spec-6.2 + 24 round-2 DONE lifecycle)"); +typedef enum GcsBlockPcmXImageResult { + GCS_BLOCK_PCM_X_IMAGE_RESERVED = 0, + GCS_BLOCK_PCM_X_IMAGE_STORED, + GCS_BLOCK_PCM_X_IMAGE_DUPLICATE, + GCS_BLOCK_PCM_X_IMAGE_REPLAY, + GCS_BLOCK_PCM_X_IMAGE_RELEASED, + GCS_BLOCK_PCM_X_IMAGE_STAGED, + GCS_BLOCK_PCM_X_IMAGE_REARMED, + GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND, + GCS_BLOCK_PCM_X_IMAGE_NOT_READY, + GCS_BLOCK_PCM_X_IMAGE_STALE, + GCS_BLOCK_PCM_X_IMAGE_FULL, + GCS_BLOCK_PCM_X_IMAGE_INVALID +} GcsBlockPcmXImageResult; + +/* By-value LMS work descriptor. It deliberately carries no page bytes: + * RESERVED work materializes from the buffer manager, while READY work only + * needs the exact ticket/image/session binding to arm or replay type 50. */ +typedef struct GcsBlockPcmXImageWork { + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + BufferTag tag; + uint8 entry_kind; + uint8 _reserved[3]; +} GcsBlockPcmXImageWork; + +StaticAssertDecl(sizeof(GcsBlockPcmXImageWork) == 184, + "PCM-X LMS work descriptor remains a by-value 184-byte token"); + /* ============================================================ * GcsBlockDedupResult — outcome of lookup_or_register. @@ -307,6 +369,8 @@ GcsBlockDedupEntryIsReclaimSafe(const GcsBlockDedupEntry *entry, TimestampTz now int64 age_us; int64 out_of_window_us; + if (entry->entry_kind != GCS_BLOCK_DEDUP_ENTRY_GENERIC) + return false; if (entry->completed_at_ts == 0) return false; @@ -389,6 +453,45 @@ extern GcsBlockDedupResult cluster_gcs_block_dedup_lookup_or_register( uint32 requester_lifetime_hint_ms, bool requester_done_capable, GcsBlockDedupEntry *cached_reply_out); +/* Dedicated PCM-X image storage over the existing dedup entry pool. Reserve + * claims capacity before the revoke lifecycle starts. Materialize publishes + * immutable bytes only after the caller has established its exact revoke + * barrier. Generic dedup APIs never mutate or retire these entries. */ +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_reserve(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *reserved_binding); +extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_materialize( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding, const GcsBlockReplyHeader *reply_header, + const char *block_data); +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_publish_ready_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding); +extern GcsBlockPcmXImageResult cluster_gcs_block_dedup_pcm_x_lookup( + int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *expected_binding, GcsBlockDedupEntry *cached_reply_out); +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_release_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding); +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_next_work(int worker_id, GcsBlockPcmXImageWork *work_out); +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_mark_staged_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding); +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *ready_binding); +extern GcsBlockPcmXImageResult +cluster_gcs_block_dedup_pcm_x_rearm_exact(int worker_id, const GcsBlockDedupKey *key, + const BufferTag *tag, + const GcsBlockPcmXImageBinding *reserved_binding); +extern bool cluster_gcs_block_dedup_pcm_x_restart_audit(int worker_id); + /* * cluster_gcs_block_dedup_mark_done — consume a requester completion proof * (GCS_BLOCK_DONE). Under the shard's exclusive lock, verifies the FULL @@ -497,6 +600,10 @@ extern uint64 cluster_gcs_block_dedup_get_done_marked_count(void); /* RC-F DON extern uint64 cluster_gcs_block_dedup_get_done_mismatch_count(void); /* RC-F DONE */ extern uint64 cluster_gcs_block_dedup_get_hint_violation_count(void); /* review F5 */ extern uint64 cluster_gcs_block_dedup_get_legacy_pin_count(void); /* review F5 */ +extern uint64 cluster_gcs_block_dedup_get_pcm_x_stage_count(void); +extern uint64 cluster_gcs_block_dedup_get_pcm_x_replay_count(void); +extern uint64 cluster_gcs_block_dedup_get_pcm_x_release_count(void); +extern uint64 cluster_gcs_block_dedup_get_pcm_x_failclosed_count(void); /* * PGRAC: spec-7.3 D5 — count of dedup accesses rejected because worker_id diff --git a/src/include/cluster/cluster_ic.h b/src/include/cluster/cluster_ic.h index fa74019724c..4f10bf143fb 100644 --- a/src/include/cluster/cluster_ic.h +++ b/src/include/cluster/cluster_ic.h @@ -364,6 +364,10 @@ typedef enum ClusterICPlane { * the sender wakes and reports a real value again, so old peers keep * receiving the conservative clock sample instead. */ #define PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 ((uint32)0x00000100U) +/* This binary registers and enforces the byte-exact PCM-X conversion message + * family. A sender must not transmit any PCM-X frame to a peer that did not + * advertise this bit; an old peer would treat the type as unregistered. */ +#define PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1 ((uint32)0x00000200U) /* * PGRAC: spec-7.2 D2 — plane + connection-epoch ride the documented-zero * pad region (capabilities precedent: occupy pad bytes, do not resize V1). diff --git a/src/include/cluster/cluster_ic_envelope.h b/src/include/cluster/cluster_ic_envelope.h index 1f9e7031f38..1c838ea084b 100644 --- a/src/include/cluster/cluster_ic_envelope.h +++ b/src/include/cluster/cluster_ic_envelope.h @@ -198,77 +198,105 @@ typedef enum ClusterICMsgType { PGRAC_IC_MSG_KO_FLUSH_ACK = 25, /* PGRAC: spec-5.7 D6 — KO apply-after-drop ACK * (KoFlushAckHeader: batch_id + acker_node + status), peer -> the dropping node, * sent ONLY after the peer has really dropped the relfilenode's buffers. */ - PGRAC_IC_MSG_CLEAN_LEAVE_ANNOUNCE = 26, /* PGRAC: spec-5.13 D8 — leaving node -> + PGRAC_IC_MSG_CLEAN_LEAVE_ANNOUNCE = 26, /* PGRAC: spec-5.13 D8 — leaving node -> * survivors (ClusterLeaveAnnouncePayload: leaving_node + leave_epoch + * preflight flag). preflight=1 probes peer clean_leave_enabled; preflight=0 * is the real "I am leaving" announce that enters survivors into leave-aware * reconfig. Consumed in the membership layer (NOT gated by clean_leave_enabled * GUC) so a disabled survivor still replies LEAVE_DRAIN_NAK rather than going * silent (§3.4 mixed-mode). */ - PGRAC_IC_MSG_LEAVE_DRAIN_ACK = 27, /* PGRAC: spec-5.13 D8 — survivor -> leaving node + PGRAC_IC_MSG_LEAVE_DRAIN_ACK = 27, /* PGRAC: spec-5.13 D8 — survivor -> leaving node * (ClusterLeaveAckPayload, nak=0): "dropped all refs to the leaving node + * accepted remaster handoff + ready-to-commit". PRE-epoch readiness ACK * (does NOT wait for / include cache invalidate — §3.1 F1 non-cycle). */ - PGRAC_IC_MSG_LEAVE_DRAIN_NAK = 28, /* PGRAC: spec-5.13 D8 — survivor -> leaving node + PGRAC_IC_MSG_LEAVE_DRAIN_NAK = 28, /* PGRAC: spec-5.13 D8 — survivor -> leaving node * (ClusterLeaveAckPayload, nak=1): refuse the clean leave (peer disabled / * not in quorum). Leaving node CLUSTER_LEAVE_ABORTED (clean abort, no * escalate, no epoch bump) on any NAK (§3.4). */ - PGRAC_IC_MSG_LEAVE_COMMIT_READY = 29, /* PGRAC: spec-5.13 D6 — leaving node -> + PGRAC_IC_MSG_LEAVE_COMMIT_READY = 29, /* PGRAC: spec-5.13 D6 — leaving node -> * survivor coordinator (ClusterLeaveAnnouncePayload, preflight=0): "I have * drained + every survivor acked; bump the leave epoch now". The leaving * node self-drives the drain but the survivor coordinator owns the epoch * bump (Q6-A); this is the readiness handoff that triggers the coordinator's * two-phase commit. Idempotent (re-sent each tick until the CLEAN_LEAVE * commit is observed; the coordinator ignores it once clean_departed). */ - PGRAC_IC_MSG_LEAVE_COMMITTED = 30, /* PGRAC: spec-5.13 Hardening v1.0.1 (P1-V0.7) — + PGRAC_IC_MSG_LEAVE_COMMITTED = 30, /* PGRAC: spec-5.13 Hardening v1.0.1 (P1-V0.7) — * survivor coordinator -> leaving node (ClusterLeaveAnnouncePayload, preflight=0): * "the COMMITTED marker is majority-durable; the durable truth-source exists, * you may exit". Gates the leaving node's BARRIER_WAIT -> COMMITTED transition * so it never departs before the marker is durable (§2.5 exit gate); re-sent * each tick while the leaver is alive (idempotent; best-effort delivery). */ - PGRAC_IC_MSG_NODE_REMOVE_ANNOUNCE = 31, /* PGRAC: spec-5.18 D10 — removal coordinator -> + PGRAC_IC_MSG_NODE_REMOVE_ANNOUNCE = 31, /* PGRAC: spec-5.18 D10 — removal coordinator -> * survivors (ClusterNodeRemoveAnnouncePayload: coordinator + target + remove_epoch + * removal_event_id). Survivors drop their refs to the removed node + reply * REMOVE_CLEANUP_ACK. */ - PGRAC_IC_MSG_REMOVE_CLEANUP_ACK = 32, /* PGRAC: spec-5.18 D10 — survivor -> removal + PGRAC_IC_MSG_REMOVE_CLEANUP_ACK = 32, /* PGRAC: spec-5.18 D10 — survivor -> removal * coordinator (ClusterNodeRemoveCleanupAckPayload): "I dropped all refs to the removed * node + accepted the permanent remaster"; sets the survivor's bit in the coordinator's * cleanup ACK barrier. */ - PGRAC_IC_MSG_BACKUP_REQUEST = 33, /* PGRAC: spec-6.5 D1/D4 — backup coordinator -> + PGRAC_IC_MSG_BACKUP_REQUEST = 33, /* PGRAC: spec-6.5 D1/D4 — backup coordinator -> * peers (ClusterBackupWireRequest): START / STOP / ABORT / RESTORE_POINT request. * LMON-mediated; peer LMON executes the local native backup/restore-point leg and * replies with BACKUP_ACK. */ - PGRAC_IC_MSG_BACKUP_ACK = 34, /* PGRAC: spec-6.5 D1/D4 — peer -> backup + PGRAC_IC_MSG_BACKUP_ACK = 34, /* PGRAC: spec-6.5 D1/D4 — peer -> backup * coordinator (ClusterBackupWireAck): local thread REDO/checkpoint/stop-cut * metadata or fail-closed NAK reason. */ - PGRAC_IC_MSG_SMART_FUSION_DURABLE = 35, /* PGRAC: spec-6.2 D8 — origin durable-LSN + PGRAC_IC_MSG_SMART_FUSION_DURABLE = 35, /* PGRAC: spec-6.2 D8 — origin durable-LSN * gossip for Smart Fusion dependency release. */ - PGRAC_IC_MSG_UNDO_HORIZON = 36, /* PGRAC: spec-5.22e D5-2 — per-peer undo + PGRAC_IC_MSG_UNDO_HORIZON = 36, /* PGRAC: spec-5.22e D5-2 — per-peer undo * retention horizon report (LMON-only producer; p2p, never * broadcast; capability-gated on UNDO_HORIZON_V1 so an old * peer never sees an unregistered msg_type). */ - PGRAC_IC_MSG_PEER_CAPS_REPLY = 37, /* PGRAC: spec-2.2 additive amendment + PGRAC_IC_MSG_PEER_CAPS_REPLY = 37, /* PGRAC: spec-2.2 additive amendment * (spec-5.22e D5 prereq) — acceptor -> dialer capability * reply carrying the acceptor's own standard 64-byte HELLO * as payload (LMON-only producer; p2p, never broadcast; * capability-gated on the dialer's CAPS_REPLY_V1 HELLO bit * so an old dialer never sees an unregistered msg_type). */ - PGRAC_IC_MSG_GCS_BLOCK_DONE = 38, /* PGRAC: GCS-race round-2 RC-F — + PGRAC_IC_MSG_GCS_BLOCK_DONE = 38, /* PGRAC: GCS-race round-2 RC-F — * requester -> master completion proof for an accepted * terminal block reply; the master verifies full identity * and stamps the dedup entry done (advisory: loss is * absorbed by the pinned TTL backstop). */ - PGRAC_IC_MSG_XID_NATIVE_DISABLE = 39, /* PGRAC: GCS-race round-3 P0-1 — + PGRAC_IC_MSG_XID_NATIVE_DISABLE = 39, /* PGRAC: GCS-race round-3 P0-1 — * wrap-barrier coordinator -> member: clear your * native-prehistory coverage latch (one-way) and ACK; the * NATIVE_RAW_REUSED authority stamp is already durable * (LMON-only producer; p2p per alive member; * capability-gated on XID_NATIVE_DISABLE_V1). */ - PGRAC_IC_MSG_XID_NATIVE_DISABLE_ACK = 40 /* PGRAC: GCS-race round-3 P0-1 — + PGRAC_IC_MSG_XID_NATIVE_DISABLE_ACK = 40, /* PGRAC: GCS-race round-3 P0-1 — * member -> coordinator: my latch is off; sets the member's * bit in the barrier ack bitmap (LMON-only producer; p2p, * never broadcast). */ - /* values 41..255 available for future sub-spec; never reuse 0..40 */ + /* PCM-X conversion messages use one stable type per exact payload/phase. + * All are point-to-point DATA-plane frames. */ + PGRAC_IC_MSG_PCM_X_ENQUEUE = 41, + PGRAC_IC_MSG_PCM_X_ADMIT_ACK = 42, + PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM = 43, + PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK = 44, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_BEGIN = 45, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_EDGE = 46, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT = 47, + /* PcmXPhasePayload [88,96) generation: zero = master->holder PROBE; + * nonzero = generation-exact master->holder BLOCKER_SET ACK. */ + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK = 48, + PGRAC_IC_MSG_PCM_X_REVOKE = 49, + PGRAC_IC_MSG_PCM_X_IMAGE_READY = 50, + PGRAC_IC_MSG_PCM_X_PREPARE_GRANT = 51, + PGRAC_IC_MSG_PCM_X_INSTALL_READY = 52, + PGRAC_IC_MSG_PCM_X_COMMIT_X = 53, + PGRAC_IC_MSG_PCM_X_FINAL_ACK = 54, + PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK = 55, + PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM = 56, + PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL = 57, + PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK = 58, + PGRAC_IC_MSG_PCM_X_CANCEL = 59, + PGRAC_IC_MSG_PCM_X_CANCEL_ACK = 60, + PGRAC_IC_MSG_PCM_X_DRAIN_POLL = 61, + PGRAC_IC_MSG_PCM_X_DRAIN_ACK = 62, + PGRAC_IC_MSG_PCM_X_RETIRE_UP_TO = 63, + PGRAC_IC_MSG_PCM_X_RETIRE_ACK = 64 + /* values 65..255 available for future sub-spec; never reuse 0..64 */ } ClusterICMsgType; diff --git a/src/include/cluster/cluster_lmd.h b/src/include/cluster/cluster_lmd.h index 1b8e7019c46..cd4d3f93311 100644 --- a/src/include/cluster/cluster_lmd.h +++ b/src/include/cluster/cluster_lmd.h @@ -444,6 +444,16 @@ extern uint64 cluster_lmd_revalidate_fail_count_get(void); extern uint64 cluster_lmd_cross_node_victim_pending_count_get(void); extern uint64 cluster_lmd_inject_call_count_get(void); +/* PCM-X connector-owned WFG accounting; generic graph calls do not bump it. */ +extern uint64 cluster_lmd_pcm_convert_wfg_replace_count_get(void); +extern uint64 cluster_lmd_pcm_convert_wfg_remove_count_get(void); +extern uint64 cluster_lmd_pcm_convert_wfg_replace_fail_count_get(void); +extern uint64 cluster_lmd_pcm_convert_wfg_exact_remove_stale_count_get(void); +extern void cluster_lmd_pcm_convert_wfg_note_replace(void); +extern void cluster_lmd_pcm_convert_wfg_note_remove(void); +extern void cluster_lmd_pcm_convert_wfg_note_replace_fail(void); +extern void cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(void); + /* spec-2.23 D8 — coordinator probe counters. */ extern uint64 cluster_lmd_probe_broadcast_count_get(void); extern uint64 cluster_lmd_probe_partial_count_get(void); @@ -613,8 +623,36 @@ extern void cluster_lmd_graph_shmem_init(void); /* Graph low-level mutators (used by cluster_lmd_tarjan.c + D16 SRF). */ extern bool cluster_lmd_graph_add_edge(const ClusterLmdWaitEdge *edge); +/* + * spec-2.36a C2 -- atomically replace one waiter's complete blocker set. + * The graph either publishes exactly the distinct blocker identities supplied + * here, or leaves the previous set untouched (including on capacity failure). + * A zero-length set is an atomic remove-all. Callers must hold no queue, + * buffer, or GRD locks while invoking this API. + */ +extern uint64 cluster_lmd_graph_replace_waiter_edges_exact(const ClusterLmdVertex *waiter, + const ClusterLmdVertex *blockers, + int nblockers, uint64 request_id); +extern bool cluster_lmd_graph_replace_waiter_edges(const ClusterLmdVertex *waiter, + const ClusterLmdVertex *blockers, int nblockers, + uint64 request_id); extern bool cluster_lmd_graph_has_waiter(const ClusterLmdVertex *waiter); extern bool cluster_lmd_graph_remove_edge_by_waiter(const ClusterLmdVertex *waiter); + +/* + * An exact removal distinguishes a stable no-edge observation from an ABA + * mismatch. STALE never mutates the newer wait instance. The legacy bool + * wrapper below remains true only for REMOVED. + */ +typedef enum ClusterLmdGraphRemoveResult { + CLUSTER_LMD_GRAPH_REMOVE_REMOVED = 0, + CLUSTER_LMD_GRAPH_REMOVE_ABSENT, + CLUSTER_LMD_GRAPH_REMOVE_STALE +} ClusterLmdGraphRemoveResult; + +extern ClusterLmdGraphRemoveResult +cluster_lmd_graph_remove_edge_by_waiter_exact_result(const ClusterLmdVertex *waiter); +extern bool cluster_lmd_graph_remove_edge_by_waiter_exact(const ClusterLmdVertex *waiter); extern int cluster_lmd_graph_snapshot_copy(ClusterLmdWaitEdge *out_buf, int max_edges, uint64 *out_gen_at_snapshot); diff --git a/src/include/cluster/cluster_lmd_wait_state.h b/src/include/cluster/cluster_lmd_wait_state.h index 75e1a2ec7e6..3ed84e4ca68 100644 --- a/src/include/cluster/cluster_lmd_wait_state.h +++ b/src/include/cluster/cluster_lmd_wait_state.h @@ -46,28 +46,51 @@ */ typedef enum ClusterLmdWaitKind { CLUSTER_LMD_WAIT_NONE = 0, - CLUSTER_LMD_WAIT_GES = 1, /* GES REQUEST / CONVERT reply wait */ - CLUSTER_LMD_WAIT_TX = 2, /* cross-node TX enqueue wait */ + CLUSTER_LMD_WAIT_GES = 1, /* GES REQUEST / CONVERT reply wait */ + CLUSTER_LMD_WAIT_TX = 2, /* cross-node TX enqueue wait */ + CLUSTER_LMD_WAIT_PCM_CONVERT = 3, /* PCM X convert-queue wait */ } ClusterLmdWaitKind; /* * Per-proc cluster wait-state, embedded in PGPROC. Written only by the - * owning backend; read cross-process by the LMD resolver. `active` is the - * publish gate: the owner writes the data fields, issues a write barrier, - * then sets active. A reader observing active == 1 (with a read barrier) - * sees a consistent record; a torn read can only fail the resolver's exact - * (request_id, cluster_epoch, wait_seq) match and skip — never a false kill — - * because wait_seq is strictly monotonic. + * owning backend; read cross-process by the LMD resolver. change_seq is a + * seqlock counter: the owner makes it odd before changing any payload field + * and even after the complete tuple is stable. A reader accepts a snapshot + * only when equal even values bracket its payload copy. Therefore clear + + * republish cannot combine fields from different wait generations. + * + * active consumes the four alignment bytes that already preceded wait_seq, + * keeping this PGPROC-embedded record at 48 bytes without moving any payload + * field. wait_seq remains the monotonic per-wait ABA identity and is + * separate from the structural seqlock counter. */ typedef struct ClusterLmdProcWaitState { - pg_atomic_uint32 active; /* 1 = blocked in a cluster wait */ - pg_atomic_uint64 wait_seq; /* monotonic publish counter (ABA guard) */ - uint8 kind; /* ClusterLmdWaitKind */ - uint64 request_id; /* GES request id / TX waiter id */ - uint64 cluster_epoch; /* epoch at wait-enter */ - TransactionId xid; /* waiter xid (InvalidTransactionId allowed) */ + pg_atomic_uint32 change_seq; /* odd = payload update in progress */ + uint32 active; /* 1 = blocked in a cluster wait */ + pg_atomic_uint64 wait_seq; /* monotonic publish counter (ABA guard) */ + uint8 kind; /* ClusterLmdWaitKind */ + uint64 request_id; /* GES request id / TX waiter id */ + uint64 cluster_epoch; /* epoch at wait-enter */ + TransactionId xid; /* waiter xid (InvalidTransactionId allowed) */ } ClusterLmdProcWaitState; +StaticAssertDecl(sizeof(ClusterLmdProcWaitState) == 48, + "ClusterLmdProcWaitState PGPROC ABI must stay 48 bytes"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, change_seq) == 0, + "ClusterLmdProcWaitState change_seq offset"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, active) == 4, + "ClusterLmdProcWaitState active offset"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, wait_seq) == 8, + "ClusterLmdProcWaitState wait_seq offset"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, kind) == 16, + "ClusterLmdProcWaitState kind offset"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, request_id) == 24, + "ClusterLmdProcWaitState request_id offset"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, cluster_epoch) == 32, + "ClusterLmdProcWaitState cluster_epoch offset"); +StaticAssertDecl(offsetof(ClusterLmdProcWaitState, xid) == 40, + "ClusterLmdProcWaitState xid offset"); + /* Read-side snapshot handed to the resolver. */ typedef struct ClusterLmdWaitStateSnapshot { bool active; @@ -78,6 +101,17 @@ typedef struct ClusterLmdWaitStateSnapshot { uint64 wait_seq; } ClusterLmdWaitStateSnapshot; +/* + * Exact read outcome. INACTIVE means a stable even seqlock snapshot proved + * active=false. BUSY means the bounded reader could not obtain any stable + * snapshot and must not be treated as evidence that the proc is inactive. + */ +typedef enum ClusterLmdWaitStateReadResult { + CLUSTER_LMD_WAIT_STATE_READ_INACTIVE = 0, + CLUSTER_LMD_WAIT_STATE_READ_ACTIVE = 1, + CLUSTER_LMD_WAIT_STATE_READ_BUSY = 2, +} ClusterLmdWaitStateReadResult; + /* * InitProcGlobal — full zero of the record (once per proc slot at postmaster * start). Establishes wait_seq = 0 so the first publish yields wait_seq = 1. @@ -85,10 +119,10 @@ typedef struct ClusterLmdWaitStateSnapshot { extern void cluster_lmd_wait_state_init(ClusterLmdProcWaitState *ws); /* - * InitProcess — clear `active` for a freshly assigned backend WITHOUT - * resetting wait_seq. Preserving wait_seq across backend reuse is what makes - * the ABA guard work; the active clear backstops a predecessor that died mid - * wait without running its explicit clear. + * InitProcess — publish an inactive stable tuple for a freshly assigned + * backend WITHOUT resetting wait_seq. Preserving wait_seq across backend + * reuse is what makes the ABA guard work; reset also closes an odd seqlock + * left by a predecessor that died midway through publishing. */ extern void cluster_lmd_wait_state_reset(ClusterLmdProcWaitState *ws); @@ -104,9 +138,19 @@ extern uint64 cluster_lmd_wait_state_publish(ClusterLmdProcWaitState *ws, uint8 /* Clear on every wait exit path (grant / timeout / ERROR / cancel). */ extern void cluster_lmd_wait_state_clear(ClusterLmdProcWaitState *ws); +/* + * Tri-state cross-process read. ACTIVE fills the complete stable tuple; + * INACTIVE is returned only after a stable even read proves active=false; + * BUSY reports bounded retry exhaustion on an in-progress/torn writer. + * *out is zeroed for both non-ACTIVE results. + */ +extern ClusterLmdWaitStateReadResult +cluster_lmd_wait_state_read_exact(ClusterLmdProcWaitState *ws, ClusterLmdWaitStateSnapshot *out); + /* * Cross-process read by the resolver. Returns true and fills *out when the - * proc is actively waiting; returns false (out->active = false) otherwise. + * proc is actively waiting; returns false (out->active = false) when inactive + * or when a bounded number of retries cannot obtain a stable even snapshot. */ extern bool cluster_lmd_wait_state_read(ClusterLmdProcWaitState *ws, ClusterLmdWaitStateSnapshot *out); diff --git a/src/include/cluster/cluster_pcm_direct_init.h b/src/include/cluster/cluster_pcm_direct_init.h new file mode 100644 index 00000000000..230c8f193db --- /dev/null +++ b/src/include/cluster/cluster_pcm_direct_init.h @@ -0,0 +1,58 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_direct_init.h + * Process-local exact proof for PCM direct initialization. + * + * This is not a wire, shared-memory, or persistent ABI. A proof lives on a + * backend stack, binds one buffer mapping/ownership generation to one known + * initialization operation, and is consumed exactly once. bufmgr captures + * and revalidates snapshots under the BufferDesc header spinlock; this module + * owns the dependency-light decision table used by unit tests. + * + * Author: SqlRush + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_PCM_DIRECT_INIT_H +#define CLUSTER_PCM_DIRECT_INIT_H + +#include "cluster/cluster_pcm_own.h" +#include "storage/buf_internals.h" + +typedef enum ClusterPcmDirectInitKind { + CLUSTER_PCM_DIRECT_INIT_INVALID = 0, + CLUSTER_PCM_DIRECT_INIT_READ_MISS, + CLUSTER_PCM_DIRECT_INIT_EXTEND, + CLUSTER_PCM_DIRECT_INIT_VM, + CLUSTER_PCM_DIRECT_INIT_FSM +} ClusterPcmDirectInitKind; + +typedef struct ClusterPcmDirectInitSnapshot { + BufferTag tag; + uint64 generation; + uint64 reservation_token; + uint32 flags; + uint32 buf_state; + int32 buf_id; + int32 private_refcount; + uint8 buffer_type; + uint8 pcm_state; + bool page_is_new; +} ClusterPcmDirectInitSnapshot; + +typedef struct ClusterPcmDirectInitProof { + ClusterPcmDirectInitSnapshot identity; + ClusterPcmDirectInitKind kind; + bool armed; +} ClusterPcmDirectInitProof; + +extern ClusterPcmOwnResult +cluster_pcm_direct_init_proof_arm(ClusterPcmDirectInitKind kind, + const ClusterPcmDirectInitSnapshot *snapshot, + ClusterPcmDirectInitProof *out_proof); +extern ClusterPcmOwnResult +cluster_pcm_direct_init_proof_consume(ClusterPcmDirectInitKind kind, + const ClusterPcmDirectInitSnapshot *snapshot, + ClusterPcmDirectInitProof *proof); + +#endif /* CLUSTER_PCM_DIRECT_INIT_H */ diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index e94a9851988..70235256f28 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -53,6 +53,7 @@ #include "c.h" #include "cluster/cluster_buffer_desc.h" /* PcmState (1.6), INVALID_NODE_ID */ #include "cluster/cluster_conf.h" /* cluster_conf_has_peers */ +#include "cluster/cluster_grd.h" /* ClusterGrdHolderId */ #include "cluster/cluster_scn.h" /* spec-2.41 D2 — SCN dual watermark */ #include "storage/buf_internals.h" /* BufferTag */ @@ -115,6 +116,88 @@ typedef enum PcmLockTransition { typedef struct GrdEntry GrdEntry; +/* + * Coherent, process-local view of one PCM authority entry. + * + * Every field is copied while the entry lock is held. Callers may retain the + * value as an optimistic token and later require an exact match before an + * ownership handoff. This is not a wire structure and adds no GrdEntry + * fields. + */ +typedef struct PcmAuthoritySnapshot { + ClusterGrdHolderId master_holder; + uint64 transition_count; + uint64 pending_x_since_lsn; + PcmState state; + int32 x_holder_node; + uint32 s_holders_bitmap; + int32 pending_x_requester_node; + uint32 reserved[2]; +} PcmAuthoritySnapshot; + +StaticAssertDecl(sizeof(PcmAuthoritySnapshot) == 64, + "PcmAuthoritySnapshot process-local layout must remain 64 bytes"); + +/* + * Exact queue-to-GRD X handoff token. The queue engine builds this only + * after the source image and every required invalidation are acknowledged. + * It is deliberately process-local: changing it does not change wire ABI. + */ +typedef struct PcmXGrdHandoffToken { + BufferTag tag; + PcmAuthoritySnapshot authority; + uint64 cluster_epoch; + uint64 request_id; + uint64 ticket_id; + uint64 grant_generation; + uint64 image_id; + uint64 source_own_generation; + SCN page_scn; + uint64 page_lsn; + int32 requester_node; + int32 source_node; + uint32 requester_procno; + uint32 page_checksum; +} PcmXGrdHandoffToken; + +StaticAssertDecl(sizeof(PcmXGrdHandoffToken) == 168, + "PcmXGrdHandoffToken process-local layout must remain 168 bytes"); + +typedef enum PcmXGrdHandoffResult { + PCM_X_GRD_HANDOFF_OK = 0, + PCM_X_GRD_HANDOFF_DUPLICATE, + PCM_X_GRD_HANDOFF_NOT_FOUND, + PCM_X_GRD_HANDOFF_STALE, + PCM_X_GRD_HANDOFF_BAD_STATE, + PCM_X_GRD_HANDOFF_INVALID +} PcmXGrdHandoffResult; + +typedef enum PcmPendingXReserveResult { + PCM_PENDING_X_RESERVE_INVALID = -2, + PCM_PENDING_X_RESERVE_NO_CAPACITY = -1, + PCM_PENDING_X_RESERVE_OCCUPIED = 0, + PCM_PENDING_X_RESERVE_OK = 1 +} PcmPendingXReserveResult; + +/* Process-local queue cookie stored in GrdEntry.pending_x_since_lsn. Keep + * the encoder shared by the PCM owner and coherent authority snapshots so a + * same-node legacy mark or successor ticket can never satisfy an active + * convert round. */ +#define PCM_PENDING_X_QUEUE_KIND UINT64_C(0x8000000000000000) +#define PCM_PENDING_X_VALUE_MASK UINT64_C(0x7fffffffffffffff) + +static inline bool +PcmPendingXQueueValue(uint64 ticket_id, uint64 *value_out) +{ + if (value_out != NULL) + *value_out = 0; + if (ticket_id == 0 || ticket_id > PCM_PENDING_X_VALUE_MASK || value_out == NULL) + return false; + *value_out = PCM_PENDING_X_QUEUE_KIND | ticket_id; + return true; +} + + /* * GUC cluster.pcm_grd_max_entries -- maximum number of GrdEntry slots * in the cluster_pcm_grd shmem region. @@ -286,6 +369,7 @@ extern bool cluster_pcm_clean_page_xfer_consume(void); * caller does not need to remember which mode was last held. */ extern void cluster_pcm_lock_unlock_content_buffer(BufferDesc *buf, PcmLockMode mode); +extern void cluster_pcm_lock_release_saved_tag_for_eviction(BufferTag tag, PcmLockMode mode); extern void cluster_pcm_lock_release_buffer_for_eviction(BufferDesc *buf, PcmLockMode mode); /* PGRAC: spec-5.2 D11 — local master records self as new X holder after a * writer-transfer (the remote holder shipped + released its X). S3 forensics @@ -334,6 +418,9 @@ extern void cluster_pcm_lock_downgrade(BufferTag tag, PcmLockMode target_mode, b * initializes the header, HTAB, HTAB lock, and per-entry locks lazily. */ extern PcmLockMode cluster_pcm_lock_query(BufferTag tag); +extern bool cluster_pcm_lock_authority_snapshot(BufferTag tag, PcmAuthoritySnapshot *out); +extern PcmXGrdHandoffResult +cluster_pcm_lock_queue_handoff_x_exact(const PcmXGrdHandoffToken *token); extern int cluster_pcm_grd_count(void); extern void cluster_pcm_grd_get_summary(int *n_count, int *s_count, int *x_count, int *pi_holders_total, int *convert_queue_active); @@ -426,7 +513,18 @@ extern void cluster_pcm_lock_module_init(void); * pending_x_requester_node matching the dead node. Must * be idempotent under concurrent X grant clear races. * ============================================================ */ -extern void cluster_pcm_lock_set_pending_x(BufferTag tag, int32 requester_node, uint64 current_lsn); +/* Legacy request-scoped producer. OCCUPIED means a live barrier is already + * present; callers must retry/deny without overwriting it. */ +extern PcmPendingXReserveResult cluster_pcm_lock_set_pending_x(BufferTag tag, int32 requester_node, + uint64 current_lsn); +/* Queue-head reservation: claim only an idle barrier. Even the same node is + * occupied without separate ticket-exact queue-engine proof. */ +extern PcmPendingXReserveResult +cluster_pcm_lock_try_reserve_pending_x(BufferTag tag, int32 requester_node, uint64 ticket_id); +extern bool cluster_pcm_lock_queue_pending_x_exact(BufferTag tag, int32 requester_node, + uint64 ticket_id); +extern bool cluster_pcm_lock_clear_queue_pending_x_exact(BufferTag tag, int32 requester_node, + uint64 ticket_id); extern void cluster_pcm_lock_clear_pending_x(BufferTag tag); /* Identity-safe compare-and-clear: clears only while the mark still names * expected_requester (request-scoped clears MUST use this form; round-2 diff --git a/src/include/cluster/cluster_pcm_own.h b/src/include/cluster/cluster_pcm_own.h index 41e92da9da0..158d6477c3d 100644 --- a/src/include/cluster/cluster_pcm_own.h +++ b/src/include/cluster/cluster_pcm_own.h @@ -1,7 +1,7 @@ /*------------------------------------------------------------------------- * * cluster_pcm_own.h - * pgrac per-buffer node-local ownership generation + flags. + * pgrac per-buffer node-local ownership generation + token + flags. * * The buffer manager's BufferDesc.pcm_state is this node's cached belief * of the PCM mode it holds for a block. spec-2.31/4.7a mirror it across @@ -12,21 +12,27 @@ * generation, so it cannot serve either. * * This module adds an INDEPENDENT monotonic per-buffer ownership - * generation plus two transient flags, stored in a shmem array indexed by - * buf_id (parallel to BufferDescriptors, so no BufferDesc ABI change). + * generation, a monotonic reservation token, and two transient flags, + * stored in a shmem array indexed by buf_id (parallel to BufferDescriptors, + * so no BufferDesc ABI change). * The generation is bumped on every COMMITTED local ownership transition; * a consumer that captured the generation before a window and finds it * changed after knows an ownership round happened even if pcm_state looks * identical. The flags mark a grant that is in flight to install * (GRANT_PENDING) or a revoke in progress (REVOKING) so an invalidate / * BAST handler does not mistake a not-yet-finalized N for an idle block, - * and a cached writer re-verifies against them. + * and a cached writer re-verifies against them. reservation_token is one + * monotonic sequence: zero means "never issued" and active ownership is + * represented exclusively by the flags. The token remains at its last + * value after finish/abort, preventing delayed cleanup from aliasing a + * later reservation. * - * The (pcm_state, generation, flags) triple MUST be read and written - * under the SAME lock -- the buffer header spinlock -- to be free of a - * read-check-act race (TOCTOU). The transition/read helpers that enforce - * that live in bufmgr.c (they need BufferDesc + LockBufHdr); this header - * owns only the shmem array and the by-buf_id raw atomic accessors. + * The (pcm_state, generation, reservation_token, flags) tuple MUST be read + * and written under the SAME lock -- the buffer header spinlock -- to be + * free of a read-check-act race (TOCTOU). The transition/read helpers + * that enforce that live in bufmgr.c (they need BufferDesc + LockBufHdr); + * this header owns only the shmem array and the by-buf_id raw atomic + * accessors. * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California @@ -55,11 +61,39 @@ #define PCM_OWN_FLAG_REVOKING ((uint32)0x2) /* a revoke (downgrade/invalidate) started */ typedef struct ClusterPcmOwnEntry { - pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ - pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ - uint32 _pad; /* keep 16B aligned */ + pg_atomic_uint64 generation; /* monotone; bumped on every committed transition */ + pg_atomic_uint64 reservation_token; /* monotone; active iff a transient flag is set */ + pg_atomic_uint32 flags; /* PCM_OWN_FLAG_* */ + uint32 _pad; /* keep 24B aligned */ } ClusterPcmOwnEntry; +StaticAssertDecl(sizeof(ClusterPcmOwnEntry) == 24, "ClusterPcmOwnEntry must remain 24 bytes"); + +typedef enum ClusterPcmOwnResult { + CLUSTER_PCM_OWN_OK = 0, + CLUSTER_PCM_OWN_STALE, + CLUSTER_PCM_OWN_BUSY, + CLUSTER_PCM_OWN_EXHAUSTED, + CLUSTER_PCM_OWN_NOT_READY, + CLUSTER_PCM_OWN_CORRUPT, + CLUSTER_PCM_OWN_INVALID +} ClusterPcmOwnResult; + +/* Classify the sidecar's active-lifecycle shape. Token zero is valid only + * while idle (flags == 0), before the first reservation. Once a singleton + * live flag is published its exact token must be nonzero; combined or unknown + * flag bits are corruption, never ordinary contention. */ +static inline ClusterPcmOwnResult +cluster_pcm_own_classify_live_flags(uint32 flags, uint64 reservation_token) +{ + if (flags == 0) + return CLUSTER_PCM_OWN_OK; + if ((flags == PCM_OWN_FLAG_GRANT_PENDING || flags == PCM_OWN_FLAG_REVOKING) + && reservation_token != 0) + return CLUSTER_PCM_OWN_BUSY; + return CLUSTER_PCM_OWN_CORRUPT; +} + /* Shmem array, NBuffers entries, indexed by buf_id. */ extern PGDLLIMPORT ClusterPcmOwnEntry *ClusterPcmOwnArray; @@ -68,12 +102,54 @@ extern void cluster_pcm_own_shmem_init(void); extern void cluster_pcm_own_shmem_register(void); /* - * Raw by-buf_id atomic accessors. Callers that need the (pcm_state,gen,flags) - * triple coherent must hold the buffer header spinlock around these AND the - * BufferDesc.pcm_state access (see bufmgr.c cluster_pcm_own_* helpers); the - * atomics here are individually safe but the triple's coherence is the lock's - * job. NULL-safe (returns 0) before shmem init, so the enable-cluster-off and - * bootstrap paths are unaffected. + * Raw exact helpers. PRECONDITION: the caller holds the matching + * BufferDesc header spinlock for buf_id. That lock serializes token + flags + * publication and binds the sidecar mutation to tag/pcm_state authority. + * Cross-module callers must use the opaque bufmgr wrappers instead. + */ +extern ClusterPcmOwnResult cluster_pcm_own_reservation_begin_exact(int buf_id, + uint64 expected_generation, + uint32 reservation_flag, + uint64 *out_token); +extern ClusterPcmOwnResult cluster_pcm_own_reservation_abort_exact(int buf_id, + uint64 expected_generation, + uint64 reservation_token, + uint32 reservation_flag); +/* Requester-as-source N/S/X conversion reuses one source revoke lifecycle + * instead of allocating a parallel grant token. This exact, idempotent role + * handoff changes only the singleton live flag; generation and token remain + * unchanged until grant_commit_exact performs the sole ownership bump. */ +extern ClusterPcmOwnResult cluster_pcm_own_revoke_to_grant_handoff_exact(int buf_id, + uint64 expected_generation, + uint64 reservation_token); +extern ClusterPcmOwnResult cluster_pcm_own_grant_commit_exact(int buf_id, + uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation); +extern ClusterPcmOwnResult cluster_pcm_own_revoke_commit_exact(int buf_id, + uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation); +/* PCM-X retained-image revoke: commit advances the ownership generation but + * deliberately leaves the exact REVOKING token live until DRAIN proves the + * immutable image record is no longer needed. Release clears only that + * committed generation/token pair. Caller holds the BufferDesc header lock. */ +extern ClusterPcmOwnResult +cluster_pcm_own_revoke_retain_commit_exact(int buf_id, uint64 expected_generation, + uint64 reservation_token, + uint64 *out_committed_generation); +extern ClusterPcmOwnResult cluster_pcm_own_revoke_retain_release_exact(int buf_id, + uint64 committed_generation, + uint64 reservation_token); +extern bool cluster_pcm_own_gen_bump_checked(int buf_id, uint64 *out_generation); + +/* + * Raw by-buf_id atomic accessors. Callers that need the + * (pcm_state,generation,reservation_token,flags) tuple coherent must hold the + * buffer header spinlock around these AND the BufferDesc.pcm_state access (see + * bufmgr.c cluster_pcm_own_* helpers); the atomics here are individually safe + * but tuple coherence is the lock's job. NULL-safe (returns 0) before shmem + * init, so the enable-cluster-off and bootstrap paths are unaffected. */ static inline uint64 cluster_pcm_own_gen_get(int buf_id) @@ -91,12 +167,12 @@ cluster_pcm_own_flags_get(int buf_id) return pg_atomic_read_u32(&ClusterPcmOwnArray[buf_id].flags); } -static inline void -cluster_pcm_own_gen_bump(int buf_id) +static inline uint64 +cluster_pcm_own_reservation_token_get(int buf_id) { if (ClusterPcmOwnArray == NULL || buf_id < 0) - return; - (void)pg_atomic_fetch_add_u64(&ClusterPcmOwnArray[buf_id].generation, 1); + return 0; + return pg_atomic_read_u64(&ClusterPcmOwnArray[buf_id].reservation_token); } static inline void diff --git a/src/include/cluster/cluster_pcm_x_bufmgr.h b/src/include/cluster/cluster_pcm_x_bufmgr.h new file mode 100644 index 00000000000..904789b55dc --- /dev/null +++ b/src/include/cluster/cluster_pcm_x_bufmgr.h @@ -0,0 +1,352 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_x_bufmgr.h + * Opaque buffer-manager boundary for PCM-X ownership reservations. + * + * Queue JOIN/WAIT code must not call begin_x_reservation. Only the + * ACTIVE_TRANSFER PREPARE leg may begin the short GRANT_PENDING interval; + * the returned token is then mandatory for exact finish or abort. A remote + * source S->X conversion must first complete the protocol's exact S->N + ACK + * and capture a fresh N tuple; begin_x_reservation deliberately rejects an S + * snapshot. A requester acting as its own N/S/X source instead hands the + * existing exact REVOKING token to GRANT_PENDING at PREPARE, so it still has + * only one ownership lifecycle and one committed generation bump. All + * exported functions acquire the BufferDesc header spinlock themselves. No + * caller may access ClusterPcmOwnEntry directly across this boundary. + * + * Author: SqlRush + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_PCM_X_BUFMGR_H +#define CLUSTER_PCM_X_BUFMGR_H + +#include "access/xlogdefs.h" +#include "cluster/cluster_pcm_own.h" +#include "cluster/cluster_pcm_x_convert.h" +#include "storage/buf_internals.h" + +/* A retry batch is bounded and cancellable. Registration may begin another + * batch while the runtime remains healthy because bypassing a closed holder + * barrier would expose untracked page bytes. Post-content-lock unregister + * instead defers its exact handle after one batch so ordinary UNLOCK never + * becomes a transaction ERROR merely because RETIRE owns the short gate. */ +#define CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS 5 + +typedef enum ClusterPcmXHolderRetryAction { + CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE = 0, + CLUSTER_PCM_X_HOLDER_RETRY_WAIT, + CLUSTER_PCM_X_HOLDER_RETRY_DEFER, + CLUSTER_PCM_X_HOLDER_RETRY_FAIL +} ClusterPcmXHolderRetryAction; + +typedef enum ClusterPcmXWriterRetryAction { + CLUSTER_PCM_X_WRITER_RETRY_COMPLETE = 0, + CLUSTER_PCM_X_WRITER_RETRY_WAIT, + CLUSTER_PCM_X_WRITER_RETRY_DEFER, + CLUSTER_PCM_X_WRITER_RETRY_FAIL +} ClusterPcmXWriterRetryAction; + +typedef enum ClusterPcmXOwnerExitAction { + CLUSTER_PCM_X_OWNER_EXIT_COMPLETE = 0, + CLUSTER_PCM_X_OWNER_EXIT_RETRY, + CLUSTER_PCM_X_OWNER_EXIT_PRESERVE +} ClusterPcmXOwnerExitAction; + +/* Exit callbacks have no later safe entrance: a short admission/retire gate + * must be waited out while this exact runtime is still active. Once the + * runtime/incarnation changes, evidence is preserved behind RECOVERY_BLOCKED + * rather than being retried against a different authority generation. */ +static inline ClusterPcmXOwnerExitAction +cluster_pcm_x_owner_exit_action(PcmXQueueResult result, bool not_found_is_complete, + bool runtime_active) +{ + if (result == PCM_X_QUEUE_OK || (not_found_is_complete && result == PCM_X_QUEUE_NOT_FOUND)) + return CLUSTER_PCM_X_OWNER_EXIT_COMPLETE; + if (runtime_active && (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BUSY)) + return CLUSTER_PCM_X_OWNER_EXIT_RETRY; + return CLUSTER_PCM_X_OWNER_EXIT_PRESERVE; +} + +static inline ClusterPcmXHolderRetryAction +cluster_pcm_x_holder_register_retry_action(PcmXQueueResult result, bool runtime_active) +{ + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE) + return CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE; + if (result == PCM_X_QUEUE_GATE_RETRY || result == PCM_X_QUEUE_BARRIER_CLOSED + || (result == PCM_X_QUEUE_NOT_READY && runtime_active)) + return CLUSTER_PCM_X_HOLDER_RETRY_WAIT; + return CLUSTER_PCM_X_HOLDER_RETRY_FAIL; +} + +static inline ClusterPcmXHolderRetryAction +cluster_pcm_x_holder_unregister_retry_action(PcmXQueueResult result, uint32 waits_used) +{ + if (result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_NOT_FOUND) + return CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE; + if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) + return CLUSTER_PCM_X_HOLDER_RETRY_FAIL; + return waits_used < CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS ? CLUSTER_PCM_X_HOLDER_RETRY_WAIT + : CLUSTER_PCM_X_HOLDER_RETRY_DEFER; +} + +static inline ClusterPcmXWriterRetryAction +cluster_pcm_x_writer_release_retry_action(PcmXQueueResult result, uint32 waits_used) +{ + if (result == PCM_X_QUEUE_OK) + return CLUSTER_PCM_X_WRITER_RETRY_COMPLETE; + if (result != PCM_X_QUEUE_GATE_RETRY && result != PCM_X_QUEUE_BUSY) + return CLUSTER_PCM_X_WRITER_RETRY_FAIL; + return waits_used < CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS ? CLUSTER_PCM_X_WRITER_RETRY_WAIT + : CLUSTER_PCM_X_WRITER_RETRY_DEFER; +} + +static inline long +cluster_pcm_x_holder_retry_delay_ms(uint32 wait_index) +{ + uint32 bounded_index = Min(wait_index, (uint32)CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS - 1); + + return 2L << bounded_index; +} + +typedef struct ClusterPcmOwnSnapshot { + BufferTag tag; + uint64 generation; + uint64 reservation_token; + uint32 flags; + uint8 pcm_state; + uint8 _reserved[3]; +} ClusterPcmOwnSnapshot; + +StaticAssertDecl(sizeof(ClusterPcmOwnSnapshot) == 48, "ClusterPcmOwnSnapshot must remain 48 bytes"); + +/* The queue returns execution authority before bufmgr takes content EXCLUSIVE. + * Bind that claim to the one committed ownership generation and require the + * same complete tuple again after the content-lock window. */ +static inline bool +cluster_pcm_x_writer_grant_snapshot_exact(const PcmXLocalWriterClaim *claim, + const ClusterPcmOwnSnapshot *granted, + const ClusterPcmOwnSnapshot *live) +{ + return claim != NULL && granted != NULL && live != NULL && claim->flags == 0 + && claim->writer.flags == 0 && claim->claim_generation != 0 + && claim->writer.identity.base_own_generation != UINT64_MAX + && granted->generation == claim->writer.identity.base_own_generation + 1 + && granted->reservation_token != 0 && granted->flags == 0 + && granted->pcm_state == (uint8)PCM_STATE_X + && BufferTagsEqual(&granted->tag, &claim->writer.identity.tag) + && claim->active_slot.slot_index == claim->writer.membership_slot.slot_index + && claim->active_slot.slot_generation == claim->writer.membership_slot.slot_generation + && claim->local_round == claim->writer.local_round && claim->role == claim->writer.role + && (claim->role == PCM_X_LOCAL_ROLE_NODE_LEADER + || claim->role == PCM_X_LOCAL_ROLE_FOLLOWER) + && BufferTagsEqual(&live->tag, &granted->tag) && live->generation == granted->generation + && live->reservation_token == granted->reservation_token && live->flags == granted->flags + && live->pcm_state == granted->pcm_state; +} + +/* A queue-managed X remains node-owned until its exact DRAIN/RETIRE lane + * releases it. The legacy cache-off unlock is legal only when no queue claim + * governed this content-lock interval. */ +static inline bool +cluster_pcm_x_should_release_legacy_on_unlock(bool local_cache, bool queue_managed) +{ + return !local_cache && !queue_managed; +} + +/* A committed node X is cache residency authority, not a fresh writer + * conversion. Consult the coherent ownership tuple before JOIN so repeated + * local LockBuffer(X) calls keep the spec-4.7a hold-until-revoked fast path. + * Any transition flag forces the caller through normal arbitration/recheck. */ +static inline bool +cluster_pcm_x_cached_cover_bypasses_queue(bool local_cache, bool requested_x, uint8 pcm_state, + uint32 flags) +{ + return local_cache && requested_x && pcm_state == (uint8)PCM_STATE_X && flags == 0; +} + +typedef ClusterPcmOwnSnapshot ClusterPcmOwnEvictionCapture; + +typedef enum ClusterPcmXGrantReservationKind { + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID = 0, + CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW, + CLUSTER_PCM_X_GRANT_RESERVATION_N_REVOKE_HANDOFF, + CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF, + CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF +} ClusterPcmXGrantReservationKind; + +/* Classify every legal PREPARE reservation shape. Ordinary remote-image N + * acquisition allocates the next monotonic token. A requester acting as its + * own N/S/X image source instead reuses the already-live revoke token and + * changes only its role. Every shape retains tag/generation/pcm_state until + * the single grant commit. */ +static inline ClusterPcmXGrantReservationKind +cluster_pcm_x_grant_reservation_kind(const ClusterPcmOwnSnapshot *live, + const ClusterPcmOwnSnapshot *base, uint64 reservation_token) +{ + if (live == NULL || base == NULL || reservation_token == 0 || base->generation == UINT64_MAX + || !BufferTagsEqual(&live->tag, &base->tag) || live->generation != base->generation + || live->reservation_token != reservation_token || live->flags != PCM_OWN_FLAG_GRANT_PENDING + || live->pcm_state != base->pcm_state) + return CLUSTER_PCM_X_GRANT_RESERVATION_INVALID; + + if (base->pcm_state == (uint8)PCM_STATE_N && base->flags == 0 + && base->reservation_token != UINT64_MAX + && reservation_token == base->reservation_token + 1) + return CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW; + if (base->flags == PCM_OWN_FLAG_REVOKING && base->reservation_token != 0 + && reservation_token == base->reservation_token) { + if (base->pcm_state == (uint8)PCM_STATE_N) + return CLUSTER_PCM_X_GRANT_RESERVATION_N_REVOKE_HANDOFF; + if (base->pcm_state == (uint8)PCM_STATE_S) + return CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF; + if (base->pcm_state == (uint8)PCM_STATE_X) + return CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF; + } + return CLUSTER_PCM_X_GRANT_RESERVATION_INVALID; +} + +/* VM and FSM callers may consume a pinned buffer without taking its content + * lock. They therefore cannot coexist with the PI+BM_VALID retained shape: + * an already-pinned or newly-pinned reader could otherwise observe source + * bytes after ownership moved to N. Main/init forks keep the passive-pin + * tolerant retained path that breaks the S3 writer pin ring. */ +typedef enum ClusterPcmXRevokeFinishMode { + CLUSTER_PCM_X_REVOKE_FINISH_INVALID = 0, + CLUSTER_PCM_X_REVOKE_FINISH_RETAIN, + CLUSTER_PCM_X_REVOKE_FINISH_DROP, + CLUSTER_PCM_X_REVOKE_FINISH_BUSY +} ClusterPcmXRevokeFinishMode; + +static inline ClusterPcmXRevokeFinishMode +cluster_pcm_x_revoke_finish_mode(const BufferTag *tag, uint32 shared_refcount) +{ + if (tag == NULL || tag->forkNum < MAIN_FORKNUM || tag->forkNum > MAX_FORKNUM) + return CLUSTER_PCM_X_REVOKE_FINISH_INVALID; + if (tag->forkNum == FSM_FORKNUM || tag->forkNum == VISIBILITYMAP_FORKNUM) + return shared_refcount == 0 ? CLUSTER_PCM_X_REVOKE_FINISH_DROP + : CLUSTER_PCM_X_REVOKE_FINISH_BUSY; + return CLUSTER_PCM_X_REVOKE_FINISH_RETAIN; +} + +/* buffer_type is a monotone last-grant hint, not the live PCM authority. + * X->S yield therefore legitimately leaves S+XCUR. X acquisition always + * publishes XCUR, so X+SCUR remains malformed. N and PI/CURRENT/CR are never + * shippable even when BM_VALID remains set. */ +static inline bool +cluster_pcm_x_current_image_shape(uint8 pcm_state, uint8 buffer_type, bool valid) +{ + return valid + && ((pcm_state == (uint8)PCM_STATE_S + && (buffer_type == (uint8)BUF_TYPE_SCUR || buffer_type == (uint8)BUF_TYPE_XCUR)) + || (pcm_state == (uint8)PCM_STATE_X && buffer_type == (uint8)BUF_TYPE_XCUR)); +} + +/* Token zero is valid before the first reservation; a completed reservation + * leaves a nonzero monotonic token idle. In both cases flags alone say + * whether a lifecycle is currently active. */ +static inline bool +cluster_pcm_own_eviction_reuse_allowed(const ClusterPcmOwnEvictionCapture *capture) +{ + return capture != NULL && capture->generation != UINT64_MAX && capture->flags == 0; +} + +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_snapshot(BufferDesc *buf, + ClusterPcmOwnSnapshot *out_snapshot); +/* Resolve one resident descriptor and snapshot its ownership tuple while the + * mapping partition and buffer header still bind the same BufferTag. The + * returned buffer id is only a locator; every later lifecycle call rechecks + * the complete snapshot and therefore remains safe across descriptor reuse. */ +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_snapshot_by_tag(const BufferTag *tag, int *out_buffer_id, + ClusterPcmOwnSnapshot *out_snapshot); +/* Backstop direct content-lock mutation entrances that do not pass through + * LockBuffer/W1. GRANT_PENDING image installation is permitted; a live + * source REVOKING lifecycle or retained PI+VALID descriptor is not. */ +extern bool cluster_bufmgr_pcm_x_content_write_permitted(BufferDesc *buf); +/* Called only after the requester has proved the exact remote master's S->N + * RELEASE application ACK. Atomically normalizes the matching descriptor + * tuple and returns the fresh N snapshot used by the later PREPARE leg. */ +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_s_release_to_n(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_n_snapshot); +/* A queue INVALIDATE may target a MAIN/INIT S mirror that has passive PG + * pins, including the waiting writer's own pin. This by-tag boundary drains + * content authority, snapshots/flushes its page evidence, and atomically + * normalizes the exact S tuple to a clean BM_VALID N mirror. VM/FSM remain + * pin-intolerant because their readers may consume bytes without LockBuffer. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_release_pinned_s_for_gcs(const BufferTag *tag, + XLogRecPtr *out_page_lsn, + uint64 *out_page_scn); +/* Publish installed bytes only while content EXCLUSIVE and the exact + * N+GRANT_PENDING reservation still bind this descriptor. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_publish_installed_x_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, uint64 reservation_token); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_begin_x_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, + uint64 *out_token); +/* Reclassify an exact requester-as-source N/S/X revoke without allocating a + * second token or advancing generation. The S-named entry remains as a + * compatibility wrapper for callers that have not yet generalized. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, uint64 *out_token); +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, uint64 *out_token); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_finish_x_commit(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, + uint64 reservation_token, uint64 *out_committed_generation); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_x_reservation(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected, + uint64 reservation_token); + +/* An ordinary remote source-holder revoke is separate from the requester + * grant lifecycle, but uses the same ownership generation/token substrate. + * Begin accepts passive PG pins: they are not PCM holders. Finish is legal + * only after the caller has staged the immutable source bytes identified by + * expected_lsn. It changes exact S/X to N and bumps generation once. Main + * and init forks keep a valid PI-shaped descriptor plus the exact REVOKING + * token; the matching DRAIN releases that retained copy. VM/FSM instead + * require refcount zero under exclusive mapping authority and drop the + * mapping at commit, because their pin-only readers do not take a content + * lock and could otherwise observe stale retained bytes. */ +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_begin_x_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_x, + ClusterPcmOwnSnapshot *out_revoking); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_x_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking); + +/* Build an immutable requester-as-source image from a clean N descriptor and + * shared storage. Success leaves the exact same-generation REVOKING token + * live, installs the verified scratch page into the resident descriptor, and + * returns byte/LSN/SCN evidence from that one scratch image. It never emits + * protocol READY; the caller computes the GCS checksum and publishes READY + * only after its dedup record is complete. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_prepare_n_source_image( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_n, ClusterPcmOwnSnapshot *out_revoking, + char block_data[BLCKSZ], XLogRecPtr *out_page_lsn, uint64 *out_page_scn); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_n_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking); + +/* A shared source uses the same generation/token reservation and is never + * promoted to, or represented as, X. */ +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_begin_s_revoke(BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_s, + ClusterPcmOwnSnapshot *out_revoking); +extern ClusterPcmOwnResult +cluster_bufmgr_pcm_own_abort_s_revoke(BufferDesc *buf, + const ClusterPcmOwnSnapshot *expected_revoking); +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_finish_revoke_retain( + BufferDesc *buf, const ClusterPcmOwnSnapshot *expected_revoking, XLogRecPtr expected_lsn, + ClusterPcmOwnSnapshot *out_retained); +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_release_retained_image(const BufferTag *tag, + uint64 source_generation); +/* Read-only DRAIN proof for a sole-requester source that committed S->X in + * place. Exact base+1 X/XCUR authority means DRAIN may release only the + * immutable dedup record and must not invalidate this descriptor. */ +extern ClusterPcmOwnResult cluster_bufmgr_pcm_own_self_handoff_x_exact(const BufferTag *tag, + uint64 source_generation); + +#endif /* CLUSTER_PCM_X_BUFMGR_H */ diff --git a/src/include/cluster/cluster_pcm_x_convert.h b/src/include/cluster/cluster_pcm_x_convert.h new file mode 100644 index 00000000000..068ddee7cbc --- /dev/null +++ b/src/include/cluster/cluster_pcm_x_convert.h @@ -0,0 +1,1644 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_x_convert.h + * PCM-X conversion wire ABI and external shared-memory substrate. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_pcm_x_convert.h + * + * NOTES + * This is a pgrac-original file. Exported symbols use the PcmX or + * cluster_pcm_x prefix. Shared-memory references are offsets or slot + * indexes; no process-local pointer is stored in the region. + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_PCM_X_CONVERT_H +#define CLUSTER_PCM_X_CONVERT_H + +#include "access/transam.h" +#include "cluster/cluster_lmd.h" +#include "storage/buf_internals.h" +#include "storage/lwlock.h" + + +#define PCM_X_PROTOCOL_NODE_LIMIT 32 +#define PCM_X_SHMEM_REGION_NAME "pgrac cluster pcm convert queue" +#define PCM_X_SHMEM_MAGIC ((uint32)0x50435851) /* "PCXQ" */ +#define PCM_X_SHMEM_LAYOUT_VERSION ((uint32)12) +#define PCM_X_INVALID_SLOT_INDEX ((Size) - 1) +#define PCM_X_LOCK_PARTITIONS NUM_BUFFER_PARTITIONS +#define PCM_X_LWLOCK_COUNT (1 + 2 * PCM_X_LOCK_PARTITIONS) +#define PCM_X_HEADER_LOCK_PADDING 92 +/* Ticket ids share the low 63 bits of the zero-growth GrdEntry pending-X + * cookie; the high bit is the queue-vs-legacy namespace discriminator. */ +#define PCM_X_MASTER_TICKET_ID_MAX UINT64CONST(0x7fffffffffffffff) + +/* Image ids are opaque on the wire. Reserve high bits 110 for PCM-X so an + * image pull carried by the legacy block request/reply transport cannot + * collide with requester ids (0x) or supported-node local-upgrade ids (10x). + * Bit 61 is canonical zero; decoders reject the 111 alias. The allocator + * passes its raw monotonic sequence without masking; values past the 56-bit + * domain are exhausted, never wrapped. */ +#define PCM_X_IMAGE_ID_DOMAIN UINT64CONST(0xc000000000000000) +#define PCM_X_IMAGE_ID_DOMAIN_MASK UINT64CONST(0xe000000000000000) +#define PCM_X_IMAGE_ID_MASTER_SHIFT 56 +#define PCM_X_IMAGE_ID_MASTER_MASK UINT64CONST(0x1f) +#define PCM_X_IMAGE_ID_SEQ_MASK ((UINT64CONST(1) << PCM_X_IMAGE_ID_MASTER_SHIFT) - 1) + +static inline bool +cluster_pcm_x_image_id_encode(int32 master_node, uint64 raw_sequence, uint64 *image_id_out) +{ + if (image_id_out != NULL) + *image_id_out = 0; + if (image_id_out == NULL || master_node < 0 || master_node >= PCM_X_PROTOCOL_NODE_LIMIT + || raw_sequence == 0 || raw_sequence > PCM_X_IMAGE_ID_SEQ_MASK) + return false; + *image_id_out = PCM_X_IMAGE_ID_DOMAIN | ((uint64)master_node << PCM_X_IMAGE_ID_MASTER_SHIFT) + | raw_sequence; + return true; +} + +static inline bool +cluster_pcm_x_image_id_decode(uint64 image_id, int32 *master_node_out, uint64 *raw_sequence_out) +{ + uint64 raw_sequence; + + if ((image_id & PCM_X_IMAGE_ID_DOMAIN_MASK) != PCM_X_IMAGE_ID_DOMAIN) + return false; + raw_sequence = image_id & PCM_X_IMAGE_ID_SEQ_MASK; + if (raw_sequence == 0) + return false; + if (master_node_out != NULL) + *master_node_out + = (int32)((image_id >> PCM_X_IMAGE_ID_MASTER_SHIFT) & PCM_X_IMAGE_ID_MASTER_MASK); + if (raw_sequence_out != NULL) + *raw_sequence_out = raw_sequence; + return true; +} + +/* Every pool shares these lifecycle values in its persistent slot header. */ +#define PCM_X_SLOT_FREE ((uint32)0) +#define PCM_X_SLOT_RESERVED_NONVISIBLE ((uint32)1) +#define PCM_X_SLOT_DETACHING ((uint32)9) +#define PCM_X_SLOT_RECOVERY_BLOCKED ((uint32)10) +#define PCM_X_SLOT_GENERATION_EXHAUSTED ((uint32)11) +/* state_flags packs the lifecycle value below local queue flag bits. */ +#define PCM_X_SLOT_STATE_BITS 8 +#define PCM_X_SLOT_STATE_MASK UINT32_C(0xff) +#define PCM_X_SLOT_FLAGS_SHIFT PCM_X_SLOT_STATE_BITS +#define PCM_X_SLOT_FLAGS_MASK (~PCM_X_SLOT_STATE_MASK) + +/* Immutable waiter and ticket identities carried by the PCM-X protocol. */ +typedef struct PcmXWaitIdentity { + BufferTag tag; + int32 node_id; + uint32 procno; + TransactionId xid; + uint64 cluster_epoch; + uint64 request_id; + uint64 wait_seq; + uint64 base_own_generation; +} PcmXWaitIdentity; + +/* Exact local-holder identity; the same buffer may have many backend holders. */ +typedef struct PcmXLocalHolderKey { + PcmXWaitIdentity identity; + int32 buffer_id; + uint32 reserved; +} PcmXLocalHolderKey; + +/* Process-local exact reference to one persistent shared-memory slot. */ +typedef struct PcmXSlotRef { + Size slot_index; + uint64 slot_generation; +} PcmXSlotRef; + +typedef struct PcmXTicketHandle { + uint64 ticket_id; + uint64 queue_generation; +} PcmXTicketHandle; + +typedef struct PcmXPrehandleKey { + uint64 sender_session_incarnation; + uint64 prehandle_sequence; +} PcmXPrehandleKey; + +typedef struct PcmXTicketRef { + PcmXWaitIdentity identity; + PcmXTicketHandle handle; + uint64 grant_generation; +} PcmXTicketRef; + +typedef struct PcmXImageToken { + uint64 image_id; + uint64 source_own_generation; + uint64 page_scn; + uint64 page_lsn; + uint32 source_node; + uint32 page_checksum; +} PcmXImageToken; + +StaticAssertDecl(sizeof(PcmXWaitIdentity) == 64, "PCM-X wait identity ABI"); +StaticAssertDecl(sizeof(PcmXLocalHolderKey) == 72, "PCM-X local holder key ABI"); +StaticAssertDecl(sizeof(PcmXSlotRef) == 16, "PCM-X slot ref ABI"); +StaticAssertDecl(sizeof(PcmXTicketHandle) == 16, "PCM-X ticket handle ABI"); +StaticAssertDecl(sizeof(PcmXPrehandleKey) == 16, "PCM-X prehandle key ABI"); +StaticAssertDecl(sizeof(PcmXTicketRef) == 88, "PCM-X ticket ref ABI"); +StaticAssertDecl(sizeof(PcmXImageToken) == 40, "PCM-X image token ABI"); + + +/* Exact-length DATA payloads. Envelope framing is owned by the IC layer. */ +typedef struct PcmXEnqueuePayload { + PcmXWaitIdentity identity; + PcmXPrehandleKey prehandle; +} PcmXEnqueuePayload; + +typedef struct PcmXPrehandleCancelPayload { + PcmXWaitIdentity identity; + PcmXPrehandleKey prehandle; +} PcmXPrehandleCancelPayload; + +typedef struct PcmXAdmitAckPayload { + PcmXTicketRef ref; + PcmXPrehandleKey prehandle; + uint32 result; + uint16 phase; + uint16 flags; +} PcmXAdmitAckPayload; + +/* + * Generic 96-byte phase payload. For type 48, bytes [88,96) are one + * lossless blocker-set generation encoded across reason/phase/flags: + * all-zero is the master->holder PROBE request; any nonzero value is the + * generation-exact master->holder ACK for a completed 45-47 set. Both arms + * keep the same direction, payload size and capability gate. + */ +typedef struct PcmXPhasePayload { + PcmXTicketRef ref; + uint32 reason; + uint16 phase; + uint16 flags; +} PcmXPhasePayload; + +typedef struct PcmXRevokePayload { + PcmXTicketRef ref; + uint64 image_id; +} PcmXRevokePayload; + +typedef struct PcmXGrantPayload { + PcmXTicketRef ref; + PcmXImageToken image; +} PcmXGrantPayload; + +typedef struct PcmXInstallReadyPayload { + PcmXTicketRef ref; + uint64 image_id; + uint32 result; + uint16 phase; + uint16 flags; +} PcmXInstallReadyPayload; + +typedef struct PcmXFinalAckPayload { + PcmXTicketRef ref; + uint64 image_id; + uint64 committed_own_generation; +} PcmXFinalAckPayload; + +typedef struct PcmXBlockerSetHeaderPayload { + PcmXTicketRef ref; + uint64 set_generation; + uint32 nblockers; + uint32 set_crc32c; +} PcmXBlockerSetHeaderPayload; + +typedef struct PcmXBlockerChunkPayload { + BufferTag tag; + int32 requester_node; + uint32 requester_procno; + uint32 chunk_no; + uint64 cluster_epoch; + uint64 request_id; + PcmXTicketHandle handle; + uint64 grant_generation; + uint64 set_generation; + ClusterLmdVertex blocker; +} PcmXBlockerChunkPayload; + +typedef struct PcmXRetirePayload { + uint64 cluster_epoch; + uint64 master_session_incarnation; + uint64 retire_through_ticket_id; + int32 sender_node; + uint32 flags; +} PcmXRetirePayload; + +typedef struct PcmXDrainPollPayload { + PcmXTicketRef ref; + uint64 drain_generation; +} PcmXDrainPollPayload; + +StaticAssertDecl(sizeof(PcmXEnqueuePayload) == 80, "PCM-X ENQUEUE ABI"); +StaticAssertDecl(sizeof(PcmXPrehandleCancelPayload) == 80, "PCM-X pre-handle CANCEL ABI"); +StaticAssertDecl(sizeof(PcmXAdmitAckPayload) == 112, "PCM-X ADMIT_ACK ABI"); +StaticAssertDecl(sizeof(PcmXPhasePayload) == 96, "PCM-X phase ABI"); +StaticAssertDecl(sizeof(PcmXRevokePayload) == 96, "PCM-X revoke ABI"); +StaticAssertDecl(sizeof(PcmXGrantPayload) == 128, "PCM-X grant ABI"); +StaticAssertDecl(sizeof(PcmXInstallReadyPayload) == 104, "PCM-X INSTALL_READY ABI"); +StaticAssertDecl(sizeof(PcmXFinalAckPayload) == 104, "PCM-X FINAL_ACK ABI"); +StaticAssertDecl(sizeof(PcmXBlockerSetHeaderPayload) == 104, "PCM-X blocker header ABI"); +StaticAssertDecl(sizeof(PcmXBlockerChunkPayload) == 128, "PCM-X blocker chunk ABI"); +StaticAssertDecl(sizeof(PcmXRetirePayload) == 32, "PCM-X tombstone retire ABI"); +StaticAssertDecl(sizeof(PcmXDrainPollPayload) == 96, "PCM-X drain-poll ABI"); + + +/* The region contains exactly these five logical pools. */ +typedef enum PcmXPoolKind { + PCM_X_POOL_MASTER_TAG = 0, + PCM_X_POOL_MASTER_TICKET, + PCM_X_POOL_BLOCKER, + PCM_X_POOL_LOCAL_TAG, + PCM_X_POOL_LOCAL_MEMBERSHIP, + PCM_X_POOL_COUNT +} PcmXPoolKind; + +/* One membership pool has two fixed, independently managed partitions. */ +typedef enum PcmXAllocatorKind { + PCM_X_ALLOC_MASTER_TAG = 0, + PCM_X_ALLOC_MASTER_TICKET, + PCM_X_ALLOC_BLOCKER, + PCM_X_ALLOC_LOCAL_TAG, + PCM_X_ALLOC_LOCAL_WAIT, + PCM_X_ALLOC_LOCAL_HOLDER, + PCM_X_ALLOC_COUNT +} PcmXAllocatorKind; + +/* Seven bounded key spaces over the five logical pools. */ +typedef enum PcmXDirectoryKind { + PCM_X_DIR_MASTER_TAG = 0, + PCM_X_DIR_MASTER_TICKET_PREHANDLE, + PCM_X_DIR_MASTER_TICKET_HANDLE, + PCM_X_DIR_MASTER_TICKET_RETIRE, + PCM_X_DIR_LOCAL_TAG, + PCM_X_DIR_LOCAL_WAIT, + PCM_X_DIR_LOCAL_HOLDER, + PCM_X_DIR_COUNT +} PcmXDirectoryKind; + +typedef enum PcmXAllocatorResult { + PCM_X_ALLOC_OK = 0, + PCM_X_ALLOC_NO_CAPACITY, + PCM_X_ALLOC_STALE_REF, + PCM_X_ALLOC_BAD_STATE, + PCM_X_ALLOC_CORRUPT, + PCM_X_ALLOC_INVALID +} PcmXAllocatorResult; + +typedef enum PcmXDirectoryResult { + PCM_X_DIRECTORY_OK = 0, + PCM_X_DIRECTORY_NOT_FOUND, + PCM_X_DIRECTORY_EXISTS, + PCM_X_DIRECTORY_FULL, + PCM_X_DIRECTORY_STALE_REF, + PCM_X_DIRECTORY_CORRUPT, + PCM_X_DIRECTORY_INVALID +} PcmXDirectoryResult; + + +typedef enum PcmXRuntimeState { + PCM_X_RUNTIME_RECOVERY_BLOCKED = 0, + PCM_X_RUNTIME_ACTIVE, + PCM_X_RUNTIME_SHUTTING_DOWN +} PcmXRuntimeState; + +/* The low 3 bits are the gate phase; the upper 29 bits are an ABA generation. */ +#define PCM_X_RUNTIME_GATE_STATE_BITS 3 +#define PCM_X_RUNTIME_GATE_STATE_MASK 0x7U +#define PCM_X_RUNTIME_GATE_GENERATION_MAX (PG_UINT32_MAX >> PCM_X_RUNTIME_GATE_STATE_BITS) + +/* A gate state and the master-session authority published with it. */ +typedef struct PcmXRuntimeSnapshot { + uint64 master_session_incarnation; + uint32 gate_generation; + PcmXRuntimeState state; +} PcmXRuntimeSnapshot; + +StaticAssertDecl(sizeof(PcmXRuntimeSnapshot) == 16, "PCM-X runtime snapshot ABI"); + +typedef enum PcmXMasterTicketState { + PCM_XT_FREE = PCM_X_SLOT_FREE, + PCM_XT_RESERVED_NONVISIBLE = PCM_X_SLOT_RESERVED_NONVISIBLE, + PCM_XT_ADMITTING = 2, + PCM_XT_QUEUED, + PCM_XT_ACTIVE_PROBE, + PCM_XT_ACTIVE_TRANSFER, + PCM_XT_COMPLETE, + PCM_XT_CANCELLED, + PCM_XT_RETIRE_CREDIT, + PCM_XT_DETACHING = PCM_X_SLOT_DETACHING, + PCM_XT_RECOVERY_BLOCKED = PCM_X_SLOT_RECOVERY_BLOCKED, + PCM_XT_GENERATION_EXHAUSTED = PCM_X_SLOT_GENERATION_EXHAUSTED +} PcmXMasterTicketState; + +typedef enum PcmXTagState { + PCM_X_TAG_FREE = PCM_X_SLOT_FREE, + PCM_X_TAG_RESERVED_NONVISIBLE = PCM_X_SLOT_RESERVED_NONVISIBLE, + PCM_X_TAG_LIVE = 2, + PCM_X_TAG_DETACHING = PCM_X_SLOT_DETACHING, + PCM_X_TAG_RECOVERY_BLOCKED = PCM_X_SLOT_RECOVERY_BLOCKED, + PCM_X_TAG_GENERATION_EXHAUSTED = PCM_X_SLOT_GENERATION_EXHAUSTED +} PcmXTagState; + +typedef enum PcmXLocalMembershipState { + PCM_XL_FREE = PCM_X_SLOT_FREE, + PCM_XL_RESERVED_NONVISIBLE = PCM_X_SLOT_RESERVED_NONVISIBLE, + PCM_XL_JOINED_NONWAITABLE = 2, + PCM_XL_WAITABLE_FOLLOWER, + PCM_XL_NODE_LEADER, + PCM_XL_REMOTE_WAIT, + PCM_XL_CONTENT_ACTIVE, + PCM_XL_GRANTED, + PCM_XL_CANCELLED, + PCM_XL_DETACHING = PCM_X_SLOT_DETACHING, + PCM_XL_RECOVERY_BLOCKED = PCM_X_SLOT_RECOVERY_BLOCKED, + PCM_XL_GENERATION_EXHAUSTED = PCM_X_SLOT_GENERATION_EXHAUSTED, + /* Holder-only states; values are local shmem lifecycle, never wire ABI. */ + PCM_XL_HOLDER_ACQUIRING = 12, + PCM_XL_HOLDER_ACTIVE = PCM_XL_CONTENT_ACTIVE, + PCM_XL_HOLDER_RELEASING = 13 +} PcmXLocalMembershipState; + +typedef enum PcmXBlockerState { + PCM_XB_FREE = PCM_X_SLOT_FREE, + PCM_XB_RESERVED_NONVISIBLE = PCM_X_SLOT_RESERVED_NONVISIBLE, + PCM_XB_LIVE = 2, + PCM_XB_DETACHING = PCM_X_SLOT_DETACHING, + PCM_XB_RECOVERY_BLOCKED = PCM_X_SLOT_RECOVERY_BLOCKED, + PCM_XB_GENERATION_EXHAUSTED = PCM_X_SLOT_GENERATION_EXHAUSTED +} PcmXBlockerState; + +#define PCM_X_BLOCKER_DIRECTION_MASTER UINT32_C(1) +#define PCM_X_BLOCKER_DIRECTION_LOCAL UINT32_C(2) + +typedef enum PcmXMasterEvent { + PCM_X_EVENT_PUBLISH_ADMIT = 0, + PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_EVENT_PROMOTE_HEAD, + PCM_X_EVENT_BLOCKERS_COMMITTED, + PCM_X_EVENT_COMMIT_COMPLETE, + PCM_X_EVENT_CANCEL_EXACT, + PCM_X_EVENT_DRAIN_EXACT, + PCM_X_EVENT_RETIRE_ACK_EXACT, + PCM_X_EVENT_DETACH_EXACT, + PCM_X_EVENT_FAULT +} PcmXMasterEvent; + +typedef enum PcmXStepResult { + PCM_X_STEP_APPLIED = 0, + PCM_X_STEP_DUPLICATE, + PCM_X_STEP_STALE, + PCM_X_STEP_BLOCKED +} PcmXStepResult; + +/* Composite queue operations never expose a partially published admission. */ +typedef enum PcmXQueueResult { + PCM_X_QUEUE_OK = 0, + PCM_X_QUEUE_DUPLICATE, + PCM_X_QUEUE_RETIRED, + PCM_X_QUEUE_NOT_FOUND, + PCM_X_QUEUE_STALE, + PCM_X_QUEUE_NO_CAPACITY, + PCM_X_QUEUE_COUNTER_EXHAUSTED, + PCM_X_QUEUE_NOT_READY, + PCM_X_QUEUE_BUSY, + PCM_X_QUEUE_BAD_STATE, + PCM_X_QUEUE_CORRUPT, + PCM_X_QUEUE_INVALID, + /* Holder admission classifications; process-local only, never wire. */ + PCM_X_QUEUE_GATE_RETRY, + PCM_X_QUEUE_BARRIER_CLOSED +} PcmXQueueResult; + +/* A generation-exact type-47 replay can prove the blocker set DUPLICATE + * after the ticket has already advanced beyond ACTIVE_PROBE. Its type-48 + * ACK is still safe to replay; only structural corruption warrants closing + * the cluster runtime. A first application retains the stricter rule + * because its ACK may otherwise outrun an unconsumed probe leg. */ +static inline bool +cluster_pcm_x_blocker_commit_completion_requires_recovery(PcmXQueueResult stage_result, + PcmXQueueResult completion_result) +{ + if (completion_result == PCM_X_QUEUE_OK || completion_result == PCM_X_QUEUE_DUPLICATE) + return false; + if (stage_result == PCM_X_QUEUE_DUPLICATE && completion_result != PCM_X_QUEUE_CORRUPT) + return false; + return true; +} + +/* One reliable terminal application leg persisted in the master ticket. */ +typedef enum PcmXTerminalLegKind { + PCM_X_TERMINAL_LEG_NONE = 0, + PCM_X_TERMINAL_LEG_DRAIN, + PCM_X_TERMINAL_LEG_RETIRE +} PcmXTerminalLegKind; + +/* Process-local send token. Slot indexes and pointers never cross the wire. */ +typedef struct PcmXTerminalLegToken { + uint64 state_sequence; + uint64 expected_responder_session; + uint64 drain_generation; + int32 expected_responder_node; + uint16 kind; + uint16 reserved; +} PcmXTerminalLegToken; + +StaticAssertDecl(sizeof(PcmXTerminalLegToken) == 32, "PCM-X terminal leg token ABI"); + +#define PCM_X_ADMIT_F_EXACT_REPLAY UINT32_C(0x01) +#define PCM_X_ADMIT_F_NODE_COALESCED UINT32_C(0x02) +#define PCM_X_ADMIT_F_QUEUE_HEAD UINT32_C(0x04) + +/* Process-local admission result. Slot references are never sent on wire. */ +typedef struct PcmXMasterAdmission { + PcmXTicketRef ref; + PcmXPrehandleKey prehandle; + PcmXSlotRef tag_slot; + PcmXSlotRef ticket_slot; + uint64 master_session_incarnation; + uint64 admission_sequence; + uint32 flags; + uint32 reserved; +} PcmXMasterAdmission; + +/* + * Process-local two-phase WFG token. The caller publishes blockers after the + * snapshot returns, then presents this token to prove that the queue structure + * used by that graph replace is still exact. Slot indexes never cross wire. + */ +#define PCM_X_MASTER_WFG_MAX_BLOCKERS 2 +typedef struct PcmXMasterWfgToken { + PcmXTicketRef ticket; + PcmXSlotRef tag_slot; + PcmXSlotRef ticket_slot; + PcmXSlotRef predecessor_slot; + PcmXSlotRef active_slot; + uint64 queue_state_sequence; +} PcmXMasterWfgToken; + +typedef struct PcmXMasterWfgSnapshot { + PcmXMasterWfgToken token; + ClusterLmdVertex blockers[PCM_X_MASTER_WFG_MAX_BLOCKERS]; + uint32 blocker_count; + uint32 reserved; +} PcmXMasterWfgSnapshot; + +typedef enum PcmXLocalRole { + PCM_X_LOCAL_ROLE_NONE = 0, + PCM_X_LOCAL_ROLE_NODE_LEADER, + PCM_X_LOCAL_ROLE_FOLLOWER +} PcmXLocalRole; + +/* Fixed suballocator identity persisted in each local-membership slot. */ +typedef enum PcmXLocalMembershipPartition { + PCM_X_LOCAL_PARTITION_WAIT = 0, + PCM_X_LOCAL_PARTITION_HOLDER +} PcmXLocalMembershipPartition; + +/* Exact local queue locator plus the immutable waiter identity it names. */ +typedef struct PcmXLocalHandle { + PcmXWaitIdentity identity; + PcmXSlotRef tag_slot; + PcmXSlotRef membership_slot; + uint64 local_sequence; + uint32 local_round; + uint16 role; + uint16 flags; +} PcmXLocalHandle; + +/* Caller-owned, process-local wake hints published only after an entire local + * RETIRE watermark commits. Items written while preflighting are invisible + * until count is atomically published by the engine's return boundary. */ +typedef struct PcmXLocalWakeBatch { + PcmXWaitIdentity *items; + Size capacity; + Size count; +} PcmXLocalWakeBatch; + +StaticAssertDecl(sizeof(PcmXLocalWakeBatch) == 24, "PCM-X local wake batch ABI"); + +/* + * Process-local proof that one follower's WFG edge still names the exact + * active local writer, or the node leader when no writer is active, sampled + * before graph publication. No slot address and no shared/wire ABI crosses + * this boundary. + */ +typedef struct PcmXLocalFollowerWfgSnapshot { + PcmXLocalHandle waiter; + PcmXWaitIdentity blocker; + PcmXSlotRef blocker_slot; + uint64 waiter_graph_generation; + Size leader_index; + Size active_writer_index; + uint64 leader_slot_generation; + uint64 active_writer_slot_generation; + uint64 holder_set_generation; + uint32 local_round; + uint32 reserved; +} PcmXLocalFollowerWfgSnapshot; + +/* One-shot local active-writer claim. Exact release records durable local + * completion on the membership, so later writers can prove FIFO order and + * the same request identity can never be re-armed. */ +typedef struct PcmXLocalWriterClaim { + PcmXLocalHandle writer; + PcmXSlotRef active_slot; + uint64 claim_generation; + uint32 local_round; + uint16 role; + uint16 flags; +} PcmXLocalWriterClaim; + +/* + * Read-only, process-local view of one exact queue membership. This is the + * only supported polling surface for pcm_lock/bufmgr callers: shared-memory + * slot addresses and allocator indexes never escape the queue engine. + */ +typedef struct PcmXLocalProgress { + PcmXWaitIdentity identity; + PcmXTicketRef ref; + PcmXImageToken image; + uint64 local_sequence; + uint64 reliable_state_sequence; + uint32 local_round; + uint32 member_state; + uint16 role; + uint16 pending_opcode; + uint16 last_response_opcode; + uint16 phase; + /* Process-local authority snapshot; never persisted or sent on wire. */ + uint64 master_session_incarnation; + int32 master_node; + uint32 reserved; +} PcmXLocalProgress; + +StaticAssertDecl(sizeof(PcmXLocalProgress) == 240, "PCM-X local progress ABI"); + +/* Read-only process-local view of the independent holder transfer lane. */ +typedef struct PcmXLocalHolderProgress { + PcmXTicketRef ref; + PcmXImageToken image; + uint64 reliable_state_sequence; + uint16 pending_opcode; + uint16 last_response_opcode; + uint16 phase; + uint16 flags; + /* Process-local authority snapshot; never persisted or sent on wire. */ + uint64 master_session_incarnation; + int32 master_node; + uint32 reserved; +} PcmXLocalHolderProgress; + +StaticAssertDecl(sizeof(PcmXLocalHolderProgress) == 160, "PCM-X local holder progress ABI"); + +typedef struct PcmXLocalCutoff { + PcmXSlotRef tag_slot; + uint64 cutoff_sequence; + uint64 master_session_incarnation; + uint32 closed_round; + uint32 next_round; + int32 master_node; + uint32 reserved; +} PcmXLocalCutoff; + +/* Process-local exact holder locator; never stored in shmem or sent on wire. */ +typedef struct PcmXLocalHolderHandle { + PcmXLocalHolderKey key; + PcmXSlotRef tag_slot; + PcmXSlotRef holder_slot; +} PcmXLocalHolderHandle; + +/* Exact holder-set token returned with one canonical bounded enumeration. */ +typedef struct PcmXLocalHolderSnapshot { + PcmXSlotRef tag_slot; + uint64 holder_set_generation; + Size holder_count; +} PcmXLocalHolderSnapshot; + +/* Immutable local outbound blocker-set token used for whole-set replay. */ +typedef struct PcmXLocalBlockerSnapshot { + PcmXTicketRef ref; + uint64 set_generation; + uint64 reliable_state_sequence; + uint32 blocker_count; + uint32 set_crc32c; +} PcmXLocalBlockerSnapshot; + +typedef enum PcmXLocalReliablePhase { + PCM_X_LOCAL_RELIABLE_PHASE_NONE = 0, + PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE, + PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM, + PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL, + PCM_X_LOCAL_RELIABLE_PHASE_CANCEL +} PcmXLocalReliablePhase; + +/* Exact process-local resend token; scheduling fields never identify a leg. */ +typedef struct PcmXLocalReliableToken { + PcmXWaitIdentity identity; + PcmXPrehandleKey prehandle; + PcmXTicketRef ref; + uint64 state_sequence; + uint64 retry_deadline_ms; + uint64 expected_responder_session; + uint32 retry_count; + int32 expected_responder_node; + uint16 pending_opcode; + uint16 phase; + uint16 flags; + uint16 reserved; +} PcmXLocalReliableToken; + +/* One exact blocker-slot observation returned in canonical chunk order. */ +typedef struct PcmXMasterBlockerEntry { + PcmXSlotRef slot_ref; + ClusterLmdVertex blocker; + uint32 chunk_no; + uint32 reserved; +} PcmXMasterBlockerEntry; + +/* Immutable token for bounded copy + exact generation revalidation. */ +typedef struct PcmXMasterBlockerSnapshot { + PcmXTicketRef ref; + uint64 set_generation; + uint64 graph_generation; + uint32 blocker_count; + uint32 set_crc32c; +} PcmXMasterBlockerSnapshot; + +/* Exact current-X probe destination; never stored on wire. */ +typedef struct PcmXMasterProbeToken { + PcmXTicketRef ref; + uint64 state_sequence; + uint64 expected_responder_session; + int32 expected_responder_node; + uint32 reserved; +} PcmXMasterProbeToken; + +/* + * Immutable process-local view used by the tag-sharded master driver. It + * carries every mutable field that can change a holder-probe or invalidation + * decision; bitmap replacement must present the exact snapshot back to the + * engine. No allocator address or slot index escapes this API. + */ +typedef struct PcmXMasterDriveSnapshot { + PcmXTicketRef ref; + PcmXImageToken image; + uint64 master_session_incarnation; + uint64 blocker_set_generation; + uint64 blocker_stage_set_generation; + uint64 graph_generation; + uint64 state_sequence; + uint64 retry_deadline_ms; + uint64 response_tombstone_mask; + uint64 expected_responder_session; + uint32 pending_s_holders_bitmap; + uint32 acked_s_holders_bitmap; + uint32 involved_nodes_bitmap; + uint32 blocker_count; + uint32 retry_count; + uint32 last_responder_node; + int32 expected_responder_node; + uint16 ticket_state; + uint16 pending_opcode; + uint16 last_response_opcode; + uint16 phase; + uint16 flags; + uint16 reserved; +} PcmXMasterDriveSnapshot; + +/* Immutable barrier token: FINAL_ACK validation is separated from the + * engine mutation so the caller can commit external GRD authority first. */ +typedef struct PcmXMasterFinalAckToken { + PcmXFinalAckPayload final_ack; + PcmXImageToken image; + uint64 master_session_incarnation; + uint64 state_sequence; + uint64 authenticated_session; + int32 authenticated_node; + uint32 reserved; +} PcmXMasterFinalAckToken; + +/* Process-local continuation for the master-lock -> GRD -> master-lock + * cancellation choreography. It never crosses wire or shmem boundaries. */ +typedef struct PcmXMasterPendingXReleaseToken { + PcmXTicketRef ref; + uint64 master_session_incarnation; + uint64 state_sequence; +} PcmXMasterPendingXReleaseToken; + +StaticAssertDecl(sizeof(PcmXMasterAdmission) == 160, "PCM-X master admission ABI"); +StaticAssertDecl(sizeof(PcmXMasterWfgToken) == 160, "PCM-X master WFG token ABI"); +StaticAssertDecl(sizeof(PcmXMasterWfgSnapshot) == 264, "PCM-X master WFG snapshot ABI"); +StaticAssertDecl(sizeof(PcmXLocalHandle) == 112, "PCM-X local handle ABI"); +StaticAssertDecl(sizeof(PcmXLocalFollowerWfgSnapshot) == 248, + "PCM-X local follower WFG snapshot process-local ABI"); +StaticAssertDecl(offsetof(PcmXLocalFollowerWfgSnapshot, waiter_graph_generation) == 192, + "PCM-X local follower sampled graph offset"); +StaticAssertDecl(sizeof(PcmXLocalWriterClaim) == 144, "PCM-X local writer claim process-local ABI"); +StaticAssertDecl(offsetof(PcmXLocalWriterClaim, claim_generation) == 128, + "PCM-X local writer claim generation offset"); +StaticAssertDecl(sizeof(PcmXLocalCutoff) == 48, "PCM-X local cutoff process-local ABI"); +StaticAssertDecl(offsetof(PcmXLocalCutoff, master_session_incarnation) == 24, + "PCM-X local cutoff session offset"); +StaticAssertDecl(offsetof(PcmXLocalCutoff, master_node) == 40, "PCM-X local cutoff master offset"); +StaticAssertDecl(sizeof(PcmXLocalHolderHandle) == 104, "PCM-X local holder handle ABI"); +StaticAssertDecl(sizeof(PcmXLocalHolderSnapshot) == 32, "PCM-X local holder snapshot ABI"); +StaticAssertDecl(sizeof(PcmXLocalBlockerSnapshot) == 112, "PCM-X local blocker snapshot ABI"); +StaticAssertDecl(sizeof(PcmXLocalReliableToken) == 208, "PCM-X local reliable token ABI"); +StaticAssertDecl(sizeof(PcmXMasterBlockerEntry) == 72, "PCM-X master blocker entry ABI"); +StaticAssertDecl(sizeof(PcmXMasterBlockerSnapshot) == 112, "PCM-X master blocker snapshot ABI"); +StaticAssertDecl(sizeof(PcmXMasterProbeToken) == 112, "PCM-X master probe token ABI"); +StaticAssertDecl(sizeof(PcmXMasterDriveSnapshot) == 232, + "PCM-X master drive snapshot process-local ABI"); +StaticAssertDecl(sizeof(PcmXMasterFinalAckToken) == 176, + "PCM-X master FINAL_ACK token process-local ABI"); +StaticAssertDecl(sizeof(PcmXMasterPendingXReleaseToken) == 104, + "PCM-X pending-X release token process-local ABI"); + +#define PCM_X_G_EXACT UINT32_C(0x01) +#define PCM_X_G_WFG_READY UINT32_C(0x02) +#define PCM_X_G_HEAD UINT32_C(0x04) +#define PCM_X_G_REVERSIBLE UINT32_C(0x08) +#define PCM_X_G_DRAINED UINT32_C(0x10) +#define PCM_X_G_RETIRED UINT32_C(0x20) +#define PCM_X_G_CONFIRM_TOMBSTONE UINT32_C(0x40) +#define PCM_X_G_COMPLETE_TOMBSTONE UINT32_C(0x80) +#define PCM_X_G_CANCEL_TOMBSTONE UINT32_C(0x100) + +/* Durable application outcomes carried by PcmXReliableLegState. */ +#define PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM UINT64_C(0x01) +#define PCM_X_RESPONSE_TOMBSTONE_COMPLETE UINT64_C(0x02) +#define PCM_X_RESPONSE_TOMBSTONE_CANCEL UINT64_C(0x04) + +/* Local-tag queue bits stored above the lifecycle value in state_flags. */ +#define PCM_X_LOCAL_TAG_F_REVOKE_BARRIER UINT32_C(0x01) +#define PCM_X_LOCAL_TAG_F_ADMISSION_GATE UINT32_C(0x02) +#define PCM_X_LOCAL_TAG_F_TERMINAL_READY UINT32_C(0x04) +#define PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED UINT32_C(0x08) +#define PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_READY UINT32_C(0x10) +#define PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_DRAINED UINT32_C(0x20) +#define PCM_X_LOCAL_TAG_F_TERMINAL_MASK \ + (PCM_X_LOCAL_TAG_F_TERMINAL_READY | PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) +#define PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK \ + (PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_READY | PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_DRAINED) + +/* Local-membership bits use the same packed flag field in a disjoint slot + * type. COMPLETE is written only by exact active-claim release; it is the + * durable one-shot/FIFO proof for later local writers in the same round. */ +#define PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE UINT32_C(0x01) + +/* Master-ticket bits use the same packed flag field in a disjoint slot type. + * PENDING_X_CLAIMED is the durable ticket-exact proof that this ACTIVE_PROBE + * owns the external GRD pending-X barrier. It survives ACTIVE_TRANSFER and + * is cleared only after exact FINAL confirmation or the two-phase cancel has + * proved that the WFG edge and GRD cookie are no longer this ticket's. + * CANCEL_REQUESTED persists a requester-death/cancel intent while an already + * armed holder PROBE reliable leg finishes its application ACK; it forbids + * ACTIVE_TRANSFER but does not discard the holder's FIFO evidence. */ +#define PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED UINT32_C(0x01) +#define PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING UINT32_C(0x02) +#define PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED UINT32_C(0x04) +#define PCM_X_MASTER_TICKET_F_PENDING_X_KNOWN \ + (PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING \ + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED) + +/* Open-address directory entries refer to pool indexes, never pointers. */ +#define PCM_X_DIRECTORY_EMPTY ((uint32)0) +#define PCM_X_DIRECTORY_OCCUPIED ((uint32)1) + +typedef struct PcmXDirectoryEntry { + uint64 key_hash; + Size slot_index; + uint64 slot_generation; + uint32 state; + uint32 reserved; +} PcmXDirectoryEntry; + +StaticAssertDecl(sizeof(PcmXDirectoryEntry) == 32, "PCM-X directory entry ABI"); +StaticAssertDecl(offsetof(PcmXDirectoryEntry, slot_index) == 8, + "PCM-X directory slot index offset"); +StaticAssertDecl(offsetof(PcmXDirectoryEntry, slot_generation) == 16, + "PCM-X directory slot generation offset"); +StaticAssertDecl(offsetof(PcmXDirectoryEntry, state) == 24, "PCM-X directory state offset"); + +/* + * Persistent slot prefix. Generation and state survive EXEC_BACKEND attach; + * next_free is meaningful only in FREE state. Every pool keeps this prefix at + * offset zero so allocator code never needs a process-local pointer in shmem. + * + * generation_change_seq protects the two generation halves: a writer makes it + * odd before replacing either half and even after both halves are stable. A + * reader accepts only equal even values bracketing its copy. Slot writers are + * serialized by the allocator lock. state_flags atomically packs + * the lifecycle value in its low bits and pool-specific flags in the remainder. + */ +typedef struct PcmXSlotHeader { + Size next_free; + pg_atomic_uint32 generation_change_seq; + uint32 slot_generation_lo; + uint32 slot_generation_hi; + pg_atomic_uint32 state_flags; +} PcmXSlotHeader; + +/* One application request/ACK leg. retry_deadline_ms is scheduling only. */ +typedef struct PcmXReliableLegState { + uint64 state_sequence; + uint64 retry_deadline_ms; + uint64 response_tombstone_mask; + uint64 expected_responder_session; + uint32 retry_count; + uint32 last_responder_node; + int32 expected_responder_node; + uint16 pending_opcode; + uint16 last_response_opcode; + uint16 phase; + uint16 flags; + uint32 reserved; +} PcmXReliableLegState; + +typedef struct PcmXMasterTagSlot { + PcmXSlotHeader slot; + BufferTag tag; + pg_atomic_uint32 queued_node_bitmap; + uint64 cluster_epoch; + uint64 next_admission_sequence; + uint64 queue_state_sequence; + uint64 active_slot_generation; + Size head_index; + Size tail_index; + Size active_index; + pg_atomic_uint32 admission_gate; + uint32 reserved; + Size outstanding_ticket_count; +} PcmXMasterTagSlot; + +typedef struct PcmXMasterTicketSlot { + PcmXSlotHeader slot; + PcmXTicketRef ref; + PcmXPrehandleKey prehandle; + PcmXImageToken image; + PcmXReliableLegState reliable; + uint64 master_session_incarnation; + uint64 admission_sequence; + uint64 blocker_set_generation; + uint64 graph_generation; + uint64 drain_generation; + uint64 tag_slot_generation; + Size next_index; + Size prev_index; + Size tag_slot_index; + Size blocker_head_index; + Size blocker_stage_head_index; + uint64 blocker_stage_set_generation; + uint64 blocker_stage_source_session; + uint64 blocker_set_source_session; + uint32 blocker_count; + uint32 blocker_set_crc32c; + uint32 blocker_stage_count; + uint32 blocker_stage_crc32c; + uint32 blocker_stage_next_chunk; + int32 blocker_stage_source_node; + int32 blocker_set_source_node; + uint32 pending_s_holders_bitmap; + uint32 acked_s_holders_bitmap; + uint32 involved_nodes_bitmap; + uint32 drained_nodes_bitmap; + uint32 retire_acked_nodes_bitmap; +} PcmXMasterTicketSlot; + +typedef struct PcmXBlockerSlot { + PcmXSlotHeader slot; + PcmXTicketHandle handle; + ClusterLmdVertex blocker; + uint64 owner_slot_generation; + uint64 set_generation; + Size next_index; + Size owner_slot_index; + uint32 chunk_no; + uint32 direction; +} PcmXBlockerSlot; + +typedef struct PcmXLocalTagSlot { + PcmXSlotHeader slot; + BufferTag tag; + int32 master_node; + uint64 cluster_epoch; + uint64 next_sequence; + uint64 cutoff_sequence; + uint64 blocker_set_generation; + uint64 writer_claim_generation; + uint64 master_session_incarnation; + PcmXPrehandleKey prehandle; + PcmXTicketRef ref; + PcmXImageToken image; + PcmXReliableLegState reliable; + uint64 leader_slot_generation; + uint64 active_writer_slot_generation; + uint64 holder_set_generation; + Size head_index; + Size tail_index; + Size leader_index; + Size active_writer_index; + Size active_holder_head_index; + Size blocker_snapshot_head_index; + uint32 local_round; + uint32 blocker_snapshot_count; + uint32 blocker_snapshot_crc32c; + uint32 blocker_snapshot_next_chunk; + Size membership_count; + Size closed_round_member_count; + uint64 terminal_drain_generation; + uint64 committed_own_generation; + PcmXTicketRef holder_ref; + PcmXImageToken holder_image; + PcmXReliableLegState holder_reliable; + uint64 holder_terminal_drain_generation; + PcmXTicketRef blocker_snapshot_ref; + PcmXReliableLegState blocker_snapshot_reliable; +} PcmXLocalTagSlot; + +typedef struct PcmXLocalMembershipSlot { + PcmXSlotHeader slot; + PcmXWaitIdentity identity; + PcmXTicketHandle handle; + uint64 local_sequence; + uint64 tag_slot_generation; + uint64 graph_generation; + Size tag_slot_index; + Size next_index; + Size prev_index; + int32 buffer_id; + uint16 role; + uint16 partition; + uint32 admitted_round; +} PcmXLocalMembershipSlot; + +StaticAssertDecl(sizeof(PcmXSlotHeader) == 24, "PCM-X slot header ABI"); +StaticAssertDecl(offsetof(PcmXSlotHeader, next_free) == 0, "PCM-X free-link offset"); +StaticAssertDecl(offsetof(PcmXSlotHeader, generation_change_seq) == 8, + "PCM-X generation seqlock offset"); +StaticAssertDecl(offsetof(PcmXSlotHeader, slot_generation_lo) == 12, + "PCM-X generation low-half offset"); +StaticAssertDecl(offsetof(PcmXSlotHeader, slot_generation_hi) == 16, + "PCM-X generation high-half offset"); +StaticAssertDecl(offsetof(PcmXSlotHeader, state_flags) == 20, "PCM-X lifecycle state/flags offset"); +StaticAssertDecl(sizeof(PcmXReliableLegState) == 56, "PCM-X reliable leg ABI"); +StaticAssertDecl(sizeof(PcmXMasterTagSlot) == 120, "PCM-X master tag slot ABI"); +StaticAssertDecl(offsetof(PcmXMasterTagSlot, outstanding_ticket_count) == 112, + "PCM-X master outstanding-ticket count offset"); +StaticAssertDecl(sizeof(PcmXMasterTicketSlot) == 384, "PCM-X master ticket slot ABI"); +StaticAssertDecl(sizeof(PcmXBlockerSlot) == 128, "PCM-X blocker slot ABI"); +StaticAssertDecl(sizeof(PcmXLocalTagSlot) == 752, "PCM-X local tag slot ABI"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, membership_count) == 384, + "PCM-X local membership count offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, closed_round_member_count) == 392, + "PCM-X local closed-round count offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, terminal_drain_generation) == 400, + "PCM-X local terminal drain generation offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, committed_own_generation) == 408, + "PCM-X local committed ownership generation offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_ref) == 416, "PCM-X local holder ticket offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_image) == 504, + "PCM-X local holder image offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_reliable) == 544, + "PCM-X local holder reliable-leg offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation) == 600, + "PCM-X local holder terminal drain generation offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref) == 608, + "PCM-X local blocker snapshot ticket offset"); +StaticAssertDecl(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable) == 696, + "PCM-X local blocker snapshot reliable-leg offset"); +StaticAssertDecl(sizeof(PcmXLocalMembershipSlot) == 168, "PCM-X local membership slot ABI"); +StaticAssertDecl(offsetof(PcmXLocalMembershipSlot, admitted_round) == 160, + "PCM-X local admitted round offset"); + + +typedef struct PcmXPoolLayout { + Size slots_offset; + Size slot_size; + Size capacity; + Size directory_offset; + Size directory_capacity; +} PcmXPoolLayout; + +/* Master tickets have three independent key spaces over the same slot pool. */ +typedef struct PcmXMasterTicketDirectoryLayout { + Size prehandle_offset; + Size prehandle_capacity; + Size handle_offset; + Size handle_capacity; + Size retire_offset; + Size retire_capacity; +} PcmXMasterTicketDirectoryLayout; + +typedef struct PcmXMembershipPartitionLayout { + Size first_slot_index; + Size slots_offset; + Size capacity; + Size directory_offset; + Size directory_capacity; +} PcmXMembershipPartitionLayout; + +typedef struct PcmXShmemLayout { + uint32 magic; + uint32 version; + uint32 pool_count; + uint32 allocator_count; + uint32 lock_partition_count; + uint32 lwlock_count; + Size header_size; + Size total_size; + Size max_backends; + Size auxiliary_procs; + Size n_buffers; + Size max_wait_edges; + Size process_capacity; + Size locks_per_process; + Size active_holder_capacity; + Size holder_tag_capacity; + Size node_ticket_capacity; + PcmXPoolLayout pools[PCM_X_POOL_COUNT]; + PcmXMasterTicketDirectoryLayout master_ticket_directories; + PcmXMembershipPartitionLayout local_wait; + PcmXMembershipPartitionLayout local_holder; +} PcmXShmemLayout; + +typedef struct PcmXAllocatorState { + Size free_head; + Size used; + Size high_water; + Size generation_exhausted; +} PcmXAllocatorState; + +/* Bounded steady-state replay/retirement authority for one requester node. */ +typedef struct PcmXPeerFrontier { + uint64 cluster_epoch; + uint64 sender_session_incarnation; + uint64 next_expected_prehandle_sequence; + uint64 retired_prehandle_sequence; + uint64 local_retired_ticket_id; + uint64 local_retire_in_progress_ticket_id; +} PcmXPeerFrontier; + +/* Lifetime counters and exact current queue depth in the existing PCM-X region. */ +typedef struct PcmXStats { + pg_atomic_uint64 enqueue_count; + pg_atomic_uint64 admit_count; + pg_atomic_uint64 confirm_count; + pg_atomic_uint64 promotion_count; + pg_atomic_uint64 transfer_count; + pg_atomic_uint64 complete_count; + pg_atomic_uint64 cancel_count; + pg_atomic_uint64 revoke_count; + pg_atomic_uint64 coalesced_count; + pg_atomic_uint64 wait_count; + pg_atomic_uint64 full_count; + pg_atomic_uint64 stale_count; + pg_atomic_uint64 miss_count; + pg_atomic_uint64 recovery_blocked_count; + pg_atomic_uint64 activating_reset_count; + pg_atomic_uint64 depth; + pg_atomic_uint64 depth_high_water; + pg_atomic_uint64 own_begin_count; + pg_atomic_uint64 own_commit_count; + pg_atomic_uint64 own_abort_count; + pg_atomic_uint64 own_busy_count; + pg_atomic_uint64 own_corrupt_count; +} PcmXStats; + +/* Process-local copy used by debug views and acceptance gates. */ +typedef struct PcmXStatsSnapshot { + uint64 enqueue_count; + uint64 admit_count; + uint64 confirm_count; + uint64 promotion_count; + uint64 transfer_count; + uint64 complete_count; + uint64 cancel_count; + uint64 revoke_count; + uint64 coalesced_count; + uint64 wait_count; + uint64 full_count; + uint64 stale_count; + uint64 miss_count; + uint64 recovery_blocked_count; + uint64 activating_reset_count; + uint64 depth; + uint64 depth_high_water; + uint64 active_tags; + uint64 live_tickets; + uint64 live_slots; + /* Cold-path RETIRE episode gauges. local_retire_gate is the raw + * owner-plus-one value; marker_ticket_id is the first nonzero peer marker + * and marker_count exposes orphan/duplicate evidence. */ + uint64 local_retire_gate; + uint64 local_retire_marker_count; + uint64 local_retire_marker_ticket_id; + uint64 own_begin_count; + uint64 own_commit_count; + uint64 own_abort_count; + uint64 own_busy_count; + uint64 own_corrupt_count; +} PcmXStatsSnapshot; + +/* Optional peer authority installed while the runtime gate is ACTIVATING. */ +typedef struct PcmXPeerBinding { + uint64 cluster_epoch; + uint64 peer_session_incarnation; +} PcmXPeerBinding; + +/* Per-target outbound pre-handle authority; no process-local pointer lives here. */ +#define PCM_X_OUTBOUND_TARGET_INITIALIZED UINT32_C(0x01) +#define PCM_X_OUTBOUND_TARGET_BOUND UINT32_C(0x02) +#define PCM_X_OUTBOUND_TARGET_KNOWN_FLAGS \ + (PCM_X_OUTBOUND_TARGET_INITIALIZED | PCM_X_OUTBOUND_TARGET_BOUND) + +typedef struct PcmXOutboundTargetFrontier { + pg_atomic_uint32 mint_gate; + uint32 flags; + uint64 cluster_epoch; + uint64 target_session_incarnation; + uint64 next_prehandle_sequence; +} PcmXOutboundTargetFrontier; + +typedef struct PcmXShmemHeader { + PcmXShmemLayout layout; + PcmXAllocatorState allocator[PCM_X_ALLOC_COUNT]; + volatile uint64 master_session_incarnation; + uint64 next_ticket_id; + uint64 next_image_id; + uint64 fully_retired_ticket_id; + pg_atomic_uint32 runtime_gate; + /* Nonzero only after exact recovery of a stranded ACTIVATING publisher. */ + pg_atomic_uint32 activation_retry_generation; + /* Serializes a multi-tag local RETIRE watermark against every ordinary + * local-domain mutator without adding a sixth pool or moving lock offsets. */ + pg_atomic_uint32 local_retire_gate; + uint8 lock_padding[PCM_X_HEADER_LOCK_PADDING]; + LWLockPadded allocator_lock; + LWLockPadded master_locks[PCM_X_LOCK_PARTITIONS]; + LWLockPadded local_locks[PCM_X_LOCK_PARTITIONS]; + PcmXPeerFrontier peer_frontiers[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXStats stats; + PcmXOutboundTargetFrontier outbound_targets[PCM_X_PROTOCOL_NODE_LIMIT]; +} PcmXShmemHeader; + +StaticAssertDecl(sizeof(PcmXShmemLayout) == 440, "PCM-X shmem layout ABI"); +StaticAssertDecl(sizeof(PcmXAllocatorState) == 32, "PCM-X allocator state ABI"); +StaticAssertDecl(sizeof(PcmXPeerFrontier) == 48, "PCM-X peer frontier ABI"); +StaticAssertDecl(sizeof(PcmXStats) == 176, "PCM-X stats ABI"); +StaticAssertDecl(sizeof(PcmXStatsSnapshot) == 224, "PCM-X stats snapshot ABI"); +StaticAssertDecl(sizeof(PcmXPeerBinding) == 16, "PCM-X peer binding ABI"); +StaticAssertDecl(sizeof(PcmXOutboundTargetFrontier) == 32, "PCM-X outbound target frontier ABI"); +StaticAssertDecl(offsetof(PcmXOutboundTargetFrontier, mint_gate) == 0, + "PCM-X outbound target short gate offset"); +StaticAssertDecl(offsetof(PcmXOutboundTargetFrontier, next_prehandle_sequence) == 24, + "PCM-X outbound pre-handle counter offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, activation_retry_generation) == 668, + "PCM-X activation retry marker offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, local_retire_gate) == 672, + "PCM-X local retire gate offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, allocator_lock) == 768, + "PCM-X allocator lock cache-line offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, master_locks) == 896, "PCM-X master lock array offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, local_locks) == 17280, "PCM-X local lock array offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, peer_frontiers) == 33664, + "PCM-X peer frontier array offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, stats) == 35200, "PCM-X stats offset"); +StaticAssertDecl(offsetof(PcmXShmemHeader, outbound_targets) == 35376, + "PCM-X outbound target frontier array offset"); +StaticAssertDecl(sizeof(PcmXShmemHeader) == 36400, "PCM-X shmem header ABI"); + +typedef enum PcmXAttachResult { + PCM_X_ATTACH_OK = 0, + PCM_X_ATTACH_NULL, + PCM_X_ATTACH_MAGIC_MISMATCH, + PCM_X_ATTACH_VERSION_MISMATCH, + PCM_X_ATTACH_LAYOUT_MISMATCH +} PcmXAttachResult; + + +extern PcmXShmemHeader *ClusterPcmXConvertShmem; + +extern void cluster_pcm_x_layout_compute(Size max_backends, Size auxiliary_procs, Size n_buffers, + Size max_wait_edges, PcmXShmemLayout *layout); +extern PcmXAttachResult cluster_pcm_x_validate_attach(const PcmXShmemHeader *header, + const PcmXShmemLayout *expected); +extern bool cluster_pcm_x_generation_next(uint64 current, uint64 *next); +extern PcmXAllocatorResult cluster_pcm_x_allocator_reserve(PcmXAllocatorKind kind, + PcmXSlotRef *ref_out, + PcmXSlotHeader **slot_out); +extern PcmXAllocatorResult cluster_pcm_x_allocator_release_exact(PcmXAllocatorKind kind, + PcmXSlotRef expected, + uint32 expected_state); +extern PcmXSlotHeader *cluster_pcm_x_slot_ref_revalidate(PcmXAllocatorKind kind, + PcmXSlotRef expected, + uint32 expected_state, + const BufferTag *expected_tag, + uint32 tag_hash); +extern bool cluster_pcm_x_directory_key_hash(PcmXDirectoryKind kind, const void *key, + uint64 *hash_out); +extern PcmXDirectoryResult cluster_pcm_x_directory_find(PcmXDirectoryKind kind, const void *key, + PcmXSlotRef *found_out); +extern PcmXDirectoryResult cluster_pcm_x_directory_insert(PcmXDirectoryKind kind, const void *key, + PcmXSlotRef candidate, + PcmXSlotRef *existing_out); +extern PcmXDirectoryResult +cluster_pcm_x_directory_delete_exact(PcmXDirectoryKind kind, const void *key, PcmXSlotRef expected); +extern uint32 cluster_pcm_x_tag_hash(const BufferTag *tag); +extern uint32 cluster_pcm_x_lock_partition(uint32 tag_hash); +extern PcmXRuntimeSnapshot cluster_pcm_x_runtime_snapshot(void); +extern bool cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out); +extern void cluster_pcm_x_stats_note_enqueue(void); +extern void cluster_pcm_x_stats_note_wait(void); +extern void cluster_pcm_x_stats_note_queue_result(PcmXQueueResult result); +extern void cluster_pcm_x_stats_note_own_begin(void); +extern void cluster_pcm_x_stats_note_own_commit(void); +extern void cluster_pcm_x_stats_note_own_abort(void); +extern void cluster_pcm_x_stats_note_own_busy(void); +extern void cluster_pcm_x_stats_note_own_corrupt(void); +extern bool cluster_pcm_x_runtime_activate(uint64 master_session_incarnation); +extern bool +cluster_pcm_x_runtime_activate_bound(uint64 master_session_incarnation, + const PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]); +extern PcmXQueueResult cluster_pcm_x_runtime_peer_binding_revalidate_exact(int32 peer_node, + uint64 cluster_epoch, + uint64 peer_session); +extern bool cluster_pcm_x_runtime_reset_activating(uint32 expected_gate_generation); +extern bool cluster_pcm_x_runtime_transition(PcmXRuntimeState expected, PcmXRuntimeState desired); +/* Cross-layer fail-closed seam for adapters that have already consumed an + * irreversible external side effect and cannot safely retry it as ordinary + * queue work. */ +extern void cluster_pcm_x_runtime_fail_closed(void); +extern PcmXStepResult cluster_pcm_x_master_step(PcmXMasterTicketState current, + PcmXMasterEvent event, uint32 guards, + PcmXMasterTicketState *next); +extern bool cluster_pcm_x_ownership_reservation_allowed(PcmXMasterTicketState state); +extern bool cluster_pcm_x_ticket_drain_ready(const PcmXMasterTicketSlot *ticket); +extern bool cluster_pcm_x_ticket_retire_ready(const PcmXMasterTicketSlot *ticket); + +/* Atomic composite queue operations; none may leak allocator/directory state. */ +extern PcmXQueueResult cluster_pcm_x_peer_bind_ack_publish(int32 peer_node, uint64 cluster_epoch, + uint64 peer_session); +extern PcmXQueueResult cluster_pcm_x_master_admit_begin(const PcmXEnqueuePayload *request, + PcmXMasterAdmission *admission_out); +extern PcmXQueueResult +cluster_pcm_x_master_admit_wfg_snapshot_exact(const PcmXTicketRef *ref, + PcmXMasterWfgSnapshot *snapshot_out); +extern PcmXQueueResult +cluster_pcm_x_master_admit_confirm_revalidate_exact(const PcmXMasterWfgToken *token, + uint64 graph_generation); +extern PcmXQueueResult cluster_pcm_x_master_admit_confirm_exact(const PcmXTicketRef *ref, + uint64 graph_generation); +extern PcmXQueueResult cluster_pcm_x_master_promote_head_exact(const BufferTag *tag, + uint64 cluster_epoch, + PcmXTicketRef *active_out); +extern PcmXQueueResult cluster_pcm_x_master_pending_x_claim_state_exact(const PcmXTicketRef *ref, + bool *claimed_out); +extern PcmXQueueResult cluster_pcm_x_master_pending_x_claim_exact(const PcmXTicketRef *ref); +extern PcmXQueueResult +cluster_pcm_x_master_pending_x_cancel_prepare_exact(const PcmXTicketRef *ref, + PcmXMasterPendingXReleaseToken *token_out); +extern PcmXQueueResult +cluster_pcm_x_master_pending_x_cancel_finalize_exact(const PcmXMasterPendingXReleaseToken *token); +extern PcmXQueueResult cluster_pcm_x_master_cancel_ack_snapshot_exact( + const PcmXTicketRef *ref, PcmXMasterAdmission *cancelled_out, bool *prehandle_out); +extern PcmXQueueResult cluster_pcm_x_master_drive_work_next(Size *cursor_io, Size scan_budget, + BufferTag *tag_out, + uint64 *cluster_epoch_out); +extern PcmXQueueResult +cluster_pcm_x_master_drive_snapshot_exact(const BufferTag *tag, uint64 cluster_epoch, + PcmXMasterDriveSnapshot *snapshot_out); +extern PcmXQueueResult cluster_pcm_x_master_drive_bitmap_replace_exact( + const PcmXMasterDriveSnapshot *expected, uint32 pending_s_holders_bitmap, + uint32 acked_s_holders_bitmap, PcmXMasterDriveSnapshot *snapshot_out); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_set_replace_exact(const PcmXTicketRef *ref, uint64 set_generation, + const ClusterLmdVertex *blockers, Size nblockers, + uint32 set_crc32c); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_probe_arm_exact(const PcmXTicketRef *ref, int32 responder_node, + uint64 responder_session, PcmXPhasePayload *probe_out, + PcmXMasterProbeToken *token_out); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_stage_begin_exact(const PcmXBlockerSetHeaderPayload *begin, + int32 authenticated_source_node, + uint64 authenticated_source_session); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_stage_edge_exact(const PcmXBlockerChunkPayload *edge, + int32 authenticated_source_node, + uint64 authenticated_source_session); +extern PcmXQueueResult cluster_pcm_x_master_blocker_stage_commit_exact( + const PcmXBlockerSetHeaderPayload *commit, int32 authenticated_source_node, + uint64 authenticated_source_session, PcmXMasterBlockerEntry *entries_out, Size entry_capacity, + PcmXMasterBlockerSnapshot *snapshot_out); +extern PcmXQueueResult cluster_pcm_x_master_blocker_stage_abort_exact( + const PcmXTicketRef *ref, uint64 set_generation, int32 authenticated_source_node, + uint64 authenticated_source_session, uint64 expected_state_sequence); +extern uint32 cluster_pcm_x_blocker_set_crc32c(const ClusterLmdVertex *blockers, Size nblockers); +extern PcmXQueueResult cluster_pcm_x_master_blocker_snapshot_exact( + const PcmXTicketRef *ref, PcmXMasterBlockerEntry *entries_out, Size entry_capacity, + PcmXMasterBlockerSnapshot *snapshot_out); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_snapshot_revalidate_exact(const PcmXMasterBlockerSnapshot *snapshot, + const PcmXMasterBlockerEntry *entries, + Size entry_count); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_graph_commit_exact(const PcmXMasterBlockerSnapshot *snapshot, + const PcmXMasterBlockerEntry *entries, + Size entry_count, uint64 graph_generation); +extern PcmXQueueResult +cluster_pcm_x_master_blocker_probe_complete_exact(const PcmXTicketRef *ref, uint64 set_generation, + int32 authenticated_source_node, + uint64 authenticated_source_session); +extern PcmXQueueResult cluster_pcm_x_master_begin_transfer_exact(const PcmXTicketRef *ref, + uint64 graph_generation, + PcmXTicketRef *transfer_out); +extern PcmXQueueResult cluster_pcm_x_master_revoke_arm_exact(const PcmXTicketRef *ref, + int32 responder_node, + uint64 responder_session, + PcmXRevokePayload *revoke_out); +extern PcmXQueueResult cluster_pcm_x_master_image_ready_exact(const PcmXGrantPayload *image_ready, + int32 authenticated_node, + uint64 authenticated_session, + PcmXGrantPayload *prepare_out); +extern PcmXQueueResult +cluster_pcm_x_master_install_ready_exact(const PcmXInstallReadyPayload *install_ready, + int32 authenticated_node, uint64 authenticated_session, + PcmXPhasePayload *commit_out); +extern PcmXQueueResult cluster_pcm_x_master_final_ack_exact(const PcmXFinalAckPayload *final_ack, + int32 authenticated_node, + uint64 authenticated_session, + PcmXPhasePayload *final_commit_out); +extern PcmXQueueResult +cluster_pcm_x_master_final_ack_prepare_exact(const PcmXFinalAckPayload *final_ack, + int32 authenticated_node, uint64 authenticated_session, + PcmXMasterFinalAckToken *token_out); +extern PcmXQueueResult +cluster_pcm_x_master_final_ack_finalize_exact(const PcmXMasterFinalAckToken *token, + PcmXPhasePayload *final_commit_out); +extern PcmXQueueResult +cluster_pcm_x_master_final_confirm_exact(const PcmXPhasePayload *final_confirm, + int32 authenticated_node, uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_master_complete_exact(const PcmXTicketRef *ref); +extern PcmXQueueResult cluster_pcm_x_master_cancel_exact(const PcmXTicketRef *ref); +extern PcmXQueueResult cluster_pcm_x_master_cancel_reversible_exact(const PcmXTicketRef *ref); +extern PcmXQueueResult +cluster_pcm_x_master_prehandle_cancel_exact(const PcmXPrehandleCancelPayload *request, + PcmXMasterAdmission *cancelled_out); +extern PcmXQueueResult cluster_pcm_x_master_terminal_leg_arm_exact(const PcmXTicketRef *ref, + PcmXTerminalLegKind kind, + int32 responder_node, + uint64 responder_session, + PcmXTerminalLegToken *token_out); +/* + * Wire ACKs do not echo send-side state_sequence or drain_generation. The + * DRAIN carries an immutable ref; this API proves its one pending leg against + * authenticated peer context. RETIRE uses the byte-exact payload API below. + */ +extern PcmXQueueResult cluster_pcm_x_master_terminal_leg_ack_exact(const PcmXTicketRef *ref, + PcmXTerminalLegKind actual_kind, + int32 authenticated_node, + uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_master_retire_ack_exact(const PcmXRetirePayload *ack, + int32 authenticated_node, + uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_master_retire_ack_resolve_exact(const PcmXRetirePayload *ack, + int32 authenticated_node, + uint64 authenticated_session, + PcmXTicketRef *ref_out); +extern PcmXQueueResult cluster_pcm_x_master_terminal_work_next(PcmXTicketRef *ref_out); +extern PcmXQueueResult cluster_pcm_x_master_detach_terminal_exact(const PcmXTicketRef *ref); +extern PcmXQueueResult cluster_pcm_x_local_join_begin(const PcmXWaitIdentity *identity, + int32 master_node, + uint64 master_session_incarnation, + PcmXLocalHandle *handle_out); +extern PcmXQueueResult cluster_pcm_x_local_lookup_exact(const PcmXWaitIdentity *identity, + PcmXLocalHandle *handle_out); +/* A follower promoted after a completed round keeps its FIFO position but + * must bind the next ENQUEUE to the caller's freshly revalidated ownership + * generation. This moves only the exact local-wait directory identity; no + * slot, sequence, round, or role is reallocated. */ +extern PcmXQueueResult cluster_pcm_x_local_leader_rekey_generation_exact( + const PcmXLocalHandle *leader, uint64 base_own_generation, PcmXLocalHandle *rekeyed_out); +extern PcmXQueueResult cluster_pcm_x_local_progress_exact(const PcmXLocalHandle *handle, + PcmXLocalProgress *progress_out); +extern PcmXQueueResult +cluster_pcm_x_local_follower_wfg_snapshot_exact(const PcmXLocalHandle *follower, + PcmXLocalFollowerWfgSnapshot *snapshot_out); +extern PcmXQueueResult +cluster_pcm_x_local_follower_wfg_commit_exact(const PcmXLocalFollowerWfgSnapshot *snapshot, + uint64 graph_generation); +extern PcmXQueueResult cluster_pcm_x_local_follower_wfg_clear_exact(const PcmXLocalHandle *follower, + uint64 graph_generation); +extern PcmXQueueResult cluster_pcm_x_local_writer_claim_exact(const PcmXLocalHandle *writer, + PcmXLocalWriterClaim *claim_out); +extern PcmXQueueResult +cluster_pcm_x_local_writer_claim_release_collect_exact(const PcmXLocalWriterClaim *claim, + PcmXWaitIdentity *successor_out); +extern PcmXQueueResult +cluster_pcm_x_local_writer_claim_release_exact(const PcmXLocalWriterClaim *claim); +extern PcmXQueueResult +cluster_pcm_x_local_writer_claim_abort_exact(const PcmXLocalWriterClaim *claim); +extern PcmXQueueResult +cluster_pcm_x_local_holder_progress_exact(const BufferTag *tag, + PcmXLocalHolderProgress *progress_out); +/* Authorize a generation-less legacy INVALIDATE only when it names the exact + * frozen non-source queue participant whose blocker set has been ACKed by the + * currently bound master. This is a read-only predicate for the passive-pin + * S->N adapter; ordinary/late INVALIDATEs retain their pin-intolerant path. */ +extern PcmXQueueResult cluster_pcm_x_local_queue_invalidate_authorize_exact( + const BufferTag *tag, uint64 cluster_epoch, uint64 request_id, int32 requester_node, + int32 authenticated_master_node, uint64 authenticated_master_session); +/* `holders` proves the complete exact pin/barrier registry. `blockers` + * contains only separately observed, stable live nested waits bound to those + * holder processes; a torn outer PGPROC snapshot must be retried as BUSY and + * must never be normalized to blocker_count=0. */ +extern PcmXQueueResult cluster_pcm_x_local_blocker_snapshot_arm_exact( + const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, + const PcmXLocalHolderSnapshot *holder_snapshot, const PcmXLocalHolderHandle *holders, + Size holder_count, const ClusterLmdVertex *blockers, Size blocker_count, + PcmXLocalBlockerSnapshot *snapshot_out); +extern PcmXQueueResult cluster_pcm_x_local_blocker_snapshot_lookup_exact( + const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, + PcmXLocalBlockerSnapshot *snapshot_out); +extern PcmXQueueResult cluster_pcm_x_local_blocker_snapshot_copy_exact( + const PcmXLocalBlockerSnapshot *snapshot, PcmXBlockerSetHeaderPayload *header_out, + PcmXBlockerChunkPayload *edges_out, Size edge_capacity); +extern PcmXQueueResult cluster_pcm_x_local_blocker_ack_exact(const PcmXTicketRef *ref, + uint64 set_generation, + int32 authenticated_master_node, + uint64 authenticated_master_session); +extern PcmXQueueResult +cluster_pcm_x_local_holder_revoke_apply_exact(const PcmXRevokePayload *revoke, + int32 authenticated_master_node, + uint64 authenticated_master_session); +extern PcmXQueueResult cluster_pcm_x_local_holder_image_ready_arm_exact( + const PcmXGrantPayload *image_ready, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXGrantPayload *replay_out); +extern PcmXQueueResult cluster_pcm_x_local_enqueue_arm_exact(const PcmXLocalHandle *leader, + PcmXEnqueuePayload *payload_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_apply_admit_ack_exact(const PcmXLocalHandle *leader, + const PcmXAdmitAckPayload *ack, + int32 authenticated_node, + uint64 authenticated_session); +extern PcmXQueueResult +cluster_pcm_x_local_admit_confirm_arm_exact(const PcmXLocalHandle *leader, + PcmXPhasePayload *payload_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_admit_confirm_ack_exact(const PcmXLocalHandle *leader, + const PcmXPhasePayload *ack, + int32 authenticated_node, + uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_local_prepare_grant_exact(const PcmXLocalHandle *leader, + const PcmXGrantPayload *prepare, + int32 authenticated_node, + uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_local_install_ready_arm_exact( + const PcmXLocalHandle *leader, const PcmXTicketRef *expected_ref, + const PcmXImageToken *expected_image, PcmXInstallReadyPayload *install_ready_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_commit_x_exact(const PcmXLocalHandle *leader, + const PcmXPhasePayload *commit, + int32 authenticated_node, + uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_local_final_ack_arm_exact(const PcmXLocalHandle *leader, + uint64 committed_own_generation, + PcmXFinalAckPayload *final_ack_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_final_commit_ack_exact( + const PcmXLocalHandle *leader, const PcmXPhasePayload *final_commit, int32 authenticated_node, + uint64 authenticated_session, PcmXPhasePayload *final_confirm_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult +cluster_pcm_x_local_prehandle_cancel_arm_exact(const PcmXLocalHandle *leader, + PcmXPrehandleCancelPayload *payload_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_cancel_arm_exact(const PcmXLocalHandle *leader, + PcmXPhasePayload *payload_out, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_prehandle_cancel_ack_exact( + const PcmXLocalHandle *leader, const PcmXAdmitAckPayload *ack, int32 authenticated_node, + uint64 authenticated_session, PcmXLocalHandle *new_leader_out); +extern PcmXQueueResult cluster_pcm_x_local_cancel_ack_exact(const PcmXLocalHandle *leader, + const PcmXPhasePayload *ack, + int32 authenticated_node, + uint64 authenticated_session, + PcmXLocalHandle *new_leader_out); +/* + * Cancellation may publish after its exact application ACK. Successful + * transfer must not publish merely because type 50 or 56 was sent: the 49-56 + * lane first consumes an exact DRAIN_POLL as positive proof that the master + * applied that message, clears the corresponding holder/requester resend leg, + * and unlinks its terminal participant. DRAIN and RETIRE then retain the + * evidence in the existing pools; no sixth pool is introduced. + */ +extern PcmXQueueResult cluster_pcm_x_local_terminal_publish_exact( + const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session); +extern PcmXQueueResult cluster_pcm_x_local_drain_poll_exact(const PcmXDrainPollPayload *poll, + int32 authenticated_master_node, + uint64 authenticated_master_session); +extern PcmXQueueResult cluster_pcm_x_local_retire_up_to_exact(const PcmXRetirePayload *request, + int32 authenticated_master_node, + uint64 authenticated_master_session); +extern PcmXQueueResult cluster_pcm_x_local_retire_up_to_collect_exact( + const PcmXRetirePayload *request, int32 authenticated_master_node, + uint64 authenticated_master_session, PcmXLocalWakeBatch *wake_batch); +extern PcmXQueueResult +cluster_pcm_x_local_reliable_snapshot_exact(const PcmXLocalHandle *leader, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_reliable_retry_exact( + const PcmXLocalHandle *leader, const PcmXLocalReliableToken *expected, uint64 retry_deadline_ms, + PcmXLocalReliableToken *token_out); +extern PcmXQueueResult cluster_pcm_x_local_reliable_ack_exact( + const PcmXLocalHandle *leader, const PcmXLocalReliableToken *expected, + const PcmXTicketRef *actual_ref, const PcmXPrehandleKey *actual_prehandle, + uint16 response_opcode, int32 authenticated_node, uint64 authenticated_session); +extern PcmXQueueResult cluster_pcm_x_local_begin_revoke_cutoff_exact(const PcmXLocalHandle *leader, + PcmXLocalCutoff *cutoff_out); +extern PcmXQueueResult cluster_pcm_x_local_current_cutoff_snapshot_exact( + const BufferTag *tag, uint64 cluster_epoch, int32 master_node, + uint64 master_session_incarnation, PcmXLocalCutoff *cutoff_out); +extern PcmXQueueResult cluster_pcm_x_local_retire_round_exact(const BufferTag *tag, + uint64 cluster_epoch, + const PcmXLocalCutoff *cutoff, + PcmXLocalHandle *next_leader_out); +extern PcmXQueueResult cluster_pcm_x_local_cancel_exact(const PcmXLocalHandle *handle, + PcmXLocalHandle *new_leader_out); +extern PcmXQueueResult cluster_pcm_x_local_detach_terminal_exact(const PcmXLocalHandle *handle); +/* Holder caller order is strict: register/prepare before BufferDesc content + * LWLock; activate while content is held; mark_releasing before unlock; exact + * unregister only after unlock. The middle two calls are lock-free W1+CAS. */ +extern PcmXQueueResult cluster_pcm_x_local_holder_register(const PcmXLocalHolderKey *key, + PcmXLocalHolderHandle *handle_out); +/* A live FIFO writer claim may register its content holder inside the closed + * revoke cohort. The returned generation is sampled from the same exact + * committed authority that admitted the holder. */ +extern PcmXQueueResult cluster_pcm_x_local_writer_holder_register_exact( + const PcmXLocalHolderKey *key, const PcmXLocalWriterClaim *claim, + PcmXLocalHolderHandle *handle_out, uint64 *committed_own_generation_out); +/* Roll back a successfully published registration only while it is still in + * ACQUIRING (for example, if the following content-LWLock acquire throws). + * Both cleanup APIs below swallow an internal detach ERROR and return CORRUPT; + * a caller already inside PG_CATCH must therefore copy and flush its original + * ErrorData before entry, then explicitly ReThrowError that saved copy. */ +extern PcmXQueueResult +cluster_pcm_x_local_holder_abort_acquiring_exact(const PcmXLocalHolderHandle *handle); +/* Error cleanup may detach any holder content state, but only after the + * caller's content LWLock is demonstrably no longer held. */ +extern PcmXQueueResult +cluster_pcm_x_local_holder_exceptional_detach_exact(const PcmXLocalHolderHandle *handle, + LWLock *content_lock); +extern PcmXQueueResult +cluster_pcm_x_local_holder_activate_exact(const PcmXLocalHolderHandle *handle); +extern PcmXQueueResult +cluster_pcm_x_local_holder_mark_releasing_exact(const PcmXLocalHolderHandle *handle); +extern PcmXQueueResult +cluster_pcm_x_local_holder_unregister_exact(const PcmXLocalHolderHandle *handle); +extern PcmXQueueResult cluster_pcm_x_local_holder_snapshot(const BufferTag *tag, + PcmXLocalHolderHandle *holders_out, + Size holder_capacity, + PcmXLocalHolderSnapshot *snapshot_out); +/* + * Freeze one master-authenticated type-48 PROBE before sampling holder + * wait-state. The existing local round barrier and blocker_snapshot_ref are + * the only durable reservation; no parallel pin/barrier lifecycle is created. + * A count-only call may return NO_CAPACITY after persisting the freeze, and an + * exact same-ref replay may then copy the canonical holder set. + */ +extern PcmXQueueResult cluster_pcm_x_local_probe_freeze_snapshot_exact( + const PcmXTicketRef *ref, int32 authenticated_master_node, uint64 authenticated_master_session, + PcmXLocalHolderHandle *holders_out, Size holder_capacity, + PcmXLocalHolderSnapshot *snapshot_out); +/* Called after publishing MyProc's wait-state and before the first sleep. + * It snapshots held shared-buffer content locks without taking a lock in the + * ForEachLWLockHeldByMe callback, then rejects any tag whose local PROBE + * barrier is closed. BUSY/CORRUPT are closed decisions, never "inactive". */ +extern PcmXQueueResult cluster_pcm_x_nested_wait_guard_before_block(void); +extern PcmXQueueResult +cluster_pcm_x_local_holder_snapshot_revalidate(const BufferTag *tag, + const PcmXLocalHolderSnapshot *snapshot); + +extern Size cluster_pcm_x_convert_shmem_size(void); +extern void cluster_pcm_x_convert_shmem_init(void); +extern void cluster_pcm_x_convert_shmem_register(void); + + +#endif /* CLUSTER_PCM_X_CONVERT_H */ diff --git a/src/include/cluster/cluster_pcm_x_image_fetch.h b/src/include/cluster/cluster_pcm_x_image_fetch.h new file mode 100644 index 00000000000..d86b18f1700 --- /dev/null +++ b/src/include/cluster/cluster_pcm_x_image_fetch.h @@ -0,0 +1,72 @@ +/*------------------------------------------------------------------------- + * + * cluster_pcm_x_image_fetch.h + * PCM-X requester adapter for fetching a staged holder image through the + * existing GCS BLOCK request/reply family. + * + * No page bytes are added to PCM-X types 50/51. Their image_id is a + * generation-exact handle into the holder's dedicated dedup record; the + * established type-14/type-15 GCS channel carries the 48+8192 byte replay. + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_pcm_x_image_fetch.h + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_PCM_X_IMAGE_FETCH_H +#define CLUSTER_PCM_X_IMAGE_FETCH_H + +#include "cluster/cluster_gcs_block.h" +#include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_pcm_x_bufmgr.h" + +#ifdef USE_PGRAC_CLUSTER + +extern bool cluster_pcm_x_image_fetch_build_request(const PcmXLocalProgress *progress, + int32 requester_node, + int32 requester_backend_id, + GcsBlockRequestPayload *request_out); +extern bool cluster_pcm_x_image_fetch_request_exact(const ClusterICEnvelope *env, + const GcsBlockRequestPayload *request, + const PcmXLocalHolderProgress *holder, + int32 holder_node, int32 current_master_node, + uint64 current_epoch); +extern bool cluster_pcm_x_image_fetch_reply_exact(const GcsBlockReplyHeader *reply, + const char *block_data, + const PcmXLocalProgress *progress, + int32 requester_node, int32 requester_backend_id); +extern bool cluster_pcm_x_image_fetch_reservation_exact(const ClusterPcmOwnSnapshot *live, + const ClusterPcmOwnSnapshot *base, + uint64 reservation_token); + +/* + * Called after ACTIVE_TRANSFER has begun the exact GRANT_PENDING reservation + * and before INSTALL_READY is armed. request_runtime pins every internal + * backoff/reply wait and the install window to the formation that admitted the + * writer. The helper never commits or aborts the reservation; the queue + * driver remains the sole owner of its lifecycle. + */ +extern PcmXQueueResult cluster_gcs_pcm_x_fetch_image_and_install( + BufferDesc *buf, const PcmXLocalHandle *leader, const ClusterPcmOwnSnapshot *reservation_base, + uint64 reservation_token, const PcmXRuntimeSnapshot *request_runtime); + +/* Sole-requester S source: validate the already-resident immutable image and + * atomically hand its exact REVOKING token to GRANT_PENDING. No page bytes + * are copied and no new reservation token is issued. */ +extern PcmXQueueResult +cluster_gcs_pcm_x_adopt_self_image(BufferDesc *buf, const PcmXLocalHandle *leader, + const ClusterPcmOwnSnapshot *revoking_base, + uint64 *out_reservation_token); +/* COMMIT_X half of the same fused lifecycle. Revalidate the page under + * content EXCLUSIVE, then commit S->X with the protocol's sole generation + * bump. Once handoff occurred, failures preserve evidence and fail closed. */ +extern PcmXQueueResult +cluster_gcs_pcm_x_finish_self_image_x(BufferDesc *buf, const PcmXLocalHandle *leader, + const ClusterPcmOwnSnapshot *revoking_base, + uint64 reservation_token, uint64 *out_committed_generation); + +#endif /* USE_PGRAC_CLUSTER */ + +#endif /* CLUSTER_PCM_X_IMAGE_FETCH_H */ diff --git a/src/include/cluster/cluster_qvotec.h b/src/include/cluster/cluster_qvotec.h index a17279bb29e..cf9576b8f9a 100644 --- a/src/include/cluster/cluster_qvotec.h +++ b/src/include/cluster/cluster_qvotec.h @@ -309,6 +309,10 @@ extern int cluster_qvotec_get_quorum_state(void); extern int cluster_qvotec_get_disks_ok_count(void); extern int cluster_qvotec_get_disks_total_count(void); extern uint64 cluster_qvotec_get_current_epoch_at_boot(void); +/* Canonical boot session published by QVOTEC into its existing shared-memory + * region. QVOTEC is the sole writer; LMS/backends are read-only consumers. */ +extern void cluster_qvotec_publish_self_incarnation(uint64 incarnation); +extern uint64 cluster_qvotec_get_self_incarnation(void); extern const char *cluster_qvotec_get_collision_state_name(void); diff --git a/src/include/cluster/cluster_sf_dep.h b/src/include/cluster/cluster_sf_dep.h index a83cae98afe..a02d0e4ef39 100644 --- a/src/include/cluster/cluster_sf_dep.h +++ b/src/include/cluster/cluster_sf_dep.h @@ -86,6 +86,23 @@ cluster_sf_peer_cap_bits(const ClusterSfPeerCap *cap) return cap->valid ? cap->bits : 0; } +/* Snapshot the capability-record generation that authenticated every + * requested bit. Tier1 records are CONTROL-owned; RDMA records use their + * registered generation 0 convention. False always zeroes the output. */ +static inline bool +cluster_sf_peer_cap_generation_for_bits(const ClusterSfPeerCap *cap, uint32 required_bits, + uint32 *generation) +{ + if (generation != NULL) + *generation = 0; + if (cap == NULL || !cap->valid || required_bits == 0 + || (cap->bits & required_bits) != required_bits) + return false; + if (generation != NULL) + *generation = cap->generation; + return true; +} + /* Returns true iff the record was live for exactly this generation and got * cleared; a mismatch (older/newer generation, already invalid) is a no-op. */ static inline bool @@ -218,6 +235,8 @@ extern bool cluster_sf_peer_supports_gcs_done(int32 peer_id); extern bool cluster_sf_peer_supports_xid_native_disable(int32 peer_id); extern bool cluster_sf_peer_supports_xid_authority_flock(int32 peer_id); extern bool cluster_sf_peer_supports_gcs_inval_busy(int32 peer_id); +extern bool cluster_sf_peer_supports_pcm_x_convert(int32 peer_id); +extern bool cluster_sf_peer_pcm_x_connection_generation(int32 peer_id, uint32 *generation); extern void cluster_sf_note_peer_disconnected_gen(int32 peer_id, uint32 generation); extern void cluster_sf_note_peer_disconnected(int32 peer_id); extern const char *cluster_sf_peer_capabilities_summary(void); diff --git a/src/include/cluster/cluster_tx_enqueue.h b/src/include/cluster/cluster_tx_enqueue.h index a22d1140af1..9ad6e1a3171 100644 --- a/src/include/cluster/cluster_tx_enqueue.h +++ b/src/include/cluster/cluster_tx_enqueue.h @@ -48,7 +48,10 @@ typedef enum ClusterTxwResult { CLUSTER_TXW_RESOLVED = 0, /* holder committed/aborted — re-judge */ CLUSTER_TXW_TIMEOUT, /* finite timeout elapsed — caller ERRORs (53R70) */ - CLUSTER_TXW_DEAD_HOLDER /* holder node fenced / TT ABORTED — re-judge */ + CLUSTER_TXW_DEAD_HOLDER, /* holder node fenced / TT ABORTED — re-judge */ + /* A PCM-X freeze made sleeping under another held content lock unsafe. + * Process-local only: callers must retry/fail before any terminal handler. */ + CLUSTER_TXW_RETRY } ClusterTxwResult; /* @@ -59,7 +62,9 @@ typedef enum ClusterTxwResult { * effective_timeout_ms: finite wait budget; values <= 0 are clamped to a * finite default (perpetual -1 is forbidden, Q6 / 5.1b clause 8). * - * Returns RESOLVED / TIMEOUT / DEAD_HOLDER. Does not change any tuple. + * Returns RESOLVED / TIMEOUT / DEAD_HOLDER / RETRY. RETRY means the caller + * must leave the current PCM-X freeze window before attempting another wait. + * Does not change any tuple. */ extern ClusterTxwResult cluster_tx_enqueue_wait(const ClusterTTStatusKey *holder_key, int effective_timeout_ms); diff --git a/src/include/storage/buf_internals.h b/src/include/storage/buf_internals.h index c369d954248..20bf0eb08fb 100644 --- a/src/include/storage/buf_internals.h +++ b/src/include/storage/buf_internals.h @@ -524,6 +524,40 @@ GetBufferDescriptor(uint32 id) return &(BufferDescriptors[id]).bufferdesc; } +/* + * Reverse-map only the exact content_lock embedded in a shared BufferDesc. + * A coarse BufferDescriptors address-range test is insufficient in cluster + * builds because the same padded slot also embeds pcm_lock. Keep this helper + * shared by bufmgr diagnostics and the PCM-X nested-wait guard so neither can + * reinterpret pcm_lock (or a future embedded LWLock) as a content lock. + */ +static inline BufferDesc * +BufferDescriptorFromContentLock(LWLock *lock) +{ + uintptr_t addr; + uintptr_t end; + uintptr_t start; + Size offset; + Size span; + + if (lock == NULL || BufferDescriptors == NULL || NBuffers <= 0) + return NULL; + if ((Size)NBuffers > SIZE_MAX / sizeof(BufferDescPadded)) + return NULL; + span = (Size)NBuffers * sizeof(BufferDescPadded); + start = (uintptr_t)BufferDescriptors; + addr = (uintptr_t)lock; + if (span > UINTPTR_MAX - start) + return NULL; + end = start + span; + if (addr < start || addr >= end) + return NULL; + offset = (Size)(addr - start); + if (offset % sizeof(BufferDescPadded) != offsetof(BufferDesc, content_lock)) + return NULL; + return &BufferDescriptors[offset / sizeof(BufferDescPadded)].bufferdesc; +} + static inline BufferDesc * GetLocalBufferDescriptor(uint32 id) { diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 166524c50e0..2a0a63e2e70 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -243,6 +243,9 @@ extern void MarkBufferDirtyHint(Buffer buffer, bool buffer_std); extern void UnlockBuffers(void); extern void LockBuffer(Buffer buffer, int mode); +/* PGRAC: operation-scoped PCM-X direct-init entrances for zero VM/FSM pages. */ +extern void LockBufferForVisibilityMapPageInit(Buffer buffer); +extern void LockBufferForFreeSpaceMapPageInit(Buffer buffer); extern bool ConditionalLockBuffer(Buffer buffer); extern void LockBufferForCleanup(Buffer buffer); extern bool ConditionalLockBufferForCleanup(Buffer buffer); diff --git a/src/include/storage/lwlock.h b/src/include/storage/lwlock.h index 22b90b10911..1c9f8581bc9 100644 --- a/src/include/storage/lwlock.h +++ b/src/include/storage/lwlock.h @@ -11,6 +11,18 @@ * *------------------------------------------------------------------------- */ +/*------------------------------------------------------------------------- + * + * PGRAC MODIFICATIONS + * Modified by: SqlRush + * + * Export the per-process held-LWLock ceiling for checked cluster shared + * memory capacity calculations. + * + * Spec: spec-2.36a-gcs-writer-convert-queue.md + * + *------------------------------------------------------------------------- + */ #ifndef LWLOCK_H #define LWLOCK_H @@ -23,6 +35,9 @@ struct PGPROC; +/* PGRAC: shared ceiling for per-process held-LWLock capacity calculations. */ +#define LWLOCK_MAX_HELD_BY_PROC 200 + /* what state of the wait process is a backend in */ typedef enum LWLockWaitState { LW_WS_NOT_WAITING, /* not currently waiting / woken up */ @@ -361,6 +376,12 @@ typedef enum BuiltinTrancheIds { * the wrap-barrier DISABLE clears the latch under EXCLUSIVE and only * ACKs after release, so no reader can straddle the epoch rollover. */ LWTRANCHE_CLUSTER_NATIVE_PREHISTORY, + /* PGRAC (spec-2.36a): PCM-X external queue allocator/directory lock. */ + LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR, + /* PGRAC (spec-2.36a): PCM-X BufferTag master-queue partitions. */ + LWTRANCHE_CLUSTER_PCM_X_MASTER, + /* PGRAC (spec-2.36a): PCM-X node-local coordinator partitions. */ + LWTRANCHE_CLUSTER_PCM_X_LOCAL, #endif LWTRANCHE_FIRST_USER_DEFINED } BuiltinTrancheIds; diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 5d37f2dd970..1f217dabb4e 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -66,6 +66,11 @@ 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', 'all 56 categories appear (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); +is( $node->safe_psql('postgres', + q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), + '119', + 'gcs category has 119 keys (PCM-X queue observability +3)'); + # ---------- # Test 3: Column types are (text, text, text). diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index 04ea6ed9995..3f20d6ce910 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -112,9 +112,9 @@ # ownership-gen wave: +1 "pgrac cluster pcm ownership" (per-buffer ownership # generation + flags array, by buf_id; sorts between "pgrac cluster pcm grd" and # "pgrac cluster pi shadow": 'pcm g' < 'pcm o' < 'pi ' in C locale). -my $expected_region_count = $has_visibility_inject ? '83' : '82'; +my $expected_region_count = $has_visibility_inject ? '84' : '83'; my $expected_regions = - 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lms data outbound,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm grd,pgrac cluster pcm ownership,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo gcs,pgrac cluster undo horizon,pgrac cluster undo record cursor'; + 'pgrac block recovery,pgrac cluster advisory,pgrac cluster backup,pgrac cluster catalog stats,pgrac cluster cf stats,pgrac cluster clean_leave,pgrac cluster conf,pgrac cluster control,pgrac cluster cr admit stats,pgrac cluster cr coordinator,pgrac cluster cr counters,pgrac cluster cr pool,pgrac cluster cr relgen,pgrac cluster cr server,pgrac cluster cr tuple stats,pgrac cluster cssd,pgrac cluster diag,pgrac cluster dl,pgrac cluster durable tt counters,pgrac cluster epoch,pgrac cluster fence,pgrac cluster gcs,pgrac cluster gcs block,pgrac cluster gcs block dedup,pgrac cluster ges,pgrac cluster ges dedup,pgrac cluster ges reply wait,pgrac cluster grd,pgrac cluster grd outbound,pgrac cluster grd pending,pgrac cluster grd work queue,pgrac cluster hw,pgrac cluster hw lease,pgrac cluster ir,pgrac cluster ko,pgrac cluster lck,pgrac cluster lmd,pgrac cluster lmd graph,pgrac cluster lmd probe,pgrac cluster lmon,pgrac cluster lms,pgrac cluster lms data outbound,pgrac cluster lock-path counters,pgrac cluster mrp,pgrac cluster multixact overlay,pgrac cluster node_remove,pgrac cluster oid lease,pgrac cluster pcm convert queue,pgrac cluster pcm grd,pgrac cluster pcm ownership,pgrac cluster pi shadow,pgrac cluster qvotec,pgrac cluster reconfig,pgrac cluster resolver cache,pgrac cluster scn,pgrac cluster sequence,pgrac cluster sinval ack outbound,pgrac cluster sinval ack wait,pgrac cluster sinval inbound,pgrac cluster sinval outbound,pgrac cluster smart fusion deps,pgrac cluster smgr,pgrac cluster startup phase,pgrac cluster stats,pgrac cluster subtrans state,pgrac cluster ts,pgrac cluster tt local seq,pgrac cluster tt slot allocator,pgrac cluster tt status hint outbound,pgrac cluster tt status overlay,pgrac cluster tx enqueue,pgrac cluster undo cleaner,pgrac cluster undo gcs,pgrac cluster undo horizon,pgrac cluster undo record cursor'; $expected_regions .= ',pgrac cluster visibility inject' if $has_visibility_inject; # spec-4.12 D7: cooperative write-fence region; always registered. Sorts after @@ -283,8 +283,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '173', - 'L15 total injection registry size is 173 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '181', + 'L15 total injection registry size is 181 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 cluster-lms-data-dispatch + cluster-lms-conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index 2f0f51b4a99..877706341a2 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -56,7 +56,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '83' : '82'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound; ownership-gen wave +1 pgrac cluster pcm ownership + my $expected_region_count = $has_visibility_inject ? '84' : '83'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound; ownership-gen wave +1 pgrac cluster pcm ownership # ---------- @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '173', - 'L11 pg_stat_cluster_injections is 173 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '181', + 'L11 pg_stat_cluster_injections is 181 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index f1dc96d1d4b..8150abff731 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -71,7 +71,7 @@ # and +1 for the unconditional "pgrac cluster cr admit stats" region (spec-5.52 D9; # and +1 for the unconditional "pgrac cluster cr relgen" region (spec-5.56 D4; # full enumerated region list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '83' : '82'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound; ownership-gen wave +1 pgrac cluster pcm ownership + my $expected_region_count = $has_visibility_inject ? '84' : '83'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound; ownership-gen wave +1 pgrac cluster pcm ownership # ---------- @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '173', - 'L12a pg_stat_cluster_injections is 173 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '181', + 'L12a pg_stat_cluster_injections is 181 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index bac8c98e691..8302759b62a 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -61,7 +61,7 @@ # admission reason counters; +1 "pgrac cluster clean_leave" (spec-5.13); +1 # "pgrac cluster cr relgen" (spec-5.56 D4); +1 "pgrac cluster cr tuple stats" # (spec-5.54 D5); full list + count lives in t/020). - my $expected_region_count = $has_visibility_inject ? '83' : '82'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound; ownership-gen wave +1 pgrac cluster pcm ownership + my $expected_region_count = $has_visibility_inject ? '84' : '83'; # spec-5.22e D5-2 +1 undo horizon; spec-5.22b D2-6 +1 undo gcs; spec-6.2 +1 smart fusion deps; spec-6.4 +1 mrp; spec-6.12 +1 xnode lever +1 hw lease +1 cr server (6.12b); full list lives in t/020 +1 pi shadow (6.12h D-h3a); spec-7.2 D4 +1 lms data outbound; ownership-gen wave +1 pgrac cluster pcm ownership # ---------- @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '173', - 'L11 pg_stat_cluster_injections is 173 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '181', + 'L11 pg_stat_cluster_injections is 181 (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; spec-5.22e D5-7 +2; spec-5.22d D4-8 +1; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index 246d9bd3d8f..4d8520341ef 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -3,8 +3,8 @@ # 024_pcm_lock.pl # End-to-end regression for the PCM lock framework surface. This # started as the stage-1.7 scaffolding test, but spec-2.30 activates -# the local PCM state machine and expands dump_pcm from the 6-row -# stub surface to the active 20-row diagnostic surface. +# the local PCM state machine; the current active diagnostic surface +# is 58 rows after the 30-key PCM-X FIFO/ownership/runtime addition. # # Verifies the SQL surface backed by spec-1.7 Deliverable 4 (pcm # category) + Deliverable 5 (4 PCM inject points, registry 24->28) @@ -55,15 +55,14 @@ # ---------- -# L1: pg_cluster_state.pcm category has 23 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5 -# + spec-4.6a D12 dead_cleanup_entries) +# L1: pg_cluster_state.pcm category has 58 keys, including the 30-key PCM-X FIFO surface. # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '28', - 'L1 pg_cluster_state.pcm category has 28 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5 + spec-4.6a D12 + ownership-gen wave 4: writer_cover_stale_detected + writer_reverify_reacquire + restore_aba_detected + invalidate_parked_grant_pending + S3-forensics wm_prov_insert_fail_count)'); + '58', + 'L1 pg_cluster_state.pcm category has 58 keys (existing 28 + PCM-X FIFO/ownership/runtime 30)'); # ---------- @@ -161,10 +160,10 @@ # release-post -> release-pre). # ---------- is($node->safe_psql( - 'postgres', - 'SELECT count(*) FROM pg_stat_cluster_injections'), - '181', - 'L6a pg_stat_cluster_injections has 181 entries (stale-baseline catch-up, t/017-family: ownership-gen wave +6 cluster-pcm-*; +1 earlier-wave undercount; S3 forensics step 1a +1 cluster-ges-master-work-queue-full) (gcs-race-round4c +2 cluster-gcs-block-fallback-refresh-stale + cluster-gcs-block-yield-notify-drop; gcs-race-fix-2 +1 cluster-gcs-block-done-drop; gcs-race-fix +1 cluster-gcs-block-duplicate-grant-reply; spec-5.22d Hardening +1 cluster-undo-authority-scan; D5 lane 漏更本文件含 spec-5.22d D4-8 +1 + spec-5.22e D5-7 +2; spec-7.3 review P1-1 +1 cluster-lms-cr-fence-recheck; spec-7.1 D3-a +1 cluster-mxid-halfspace-hard-limit; spec-7.3 D7 +1 cluster-lms-cr-fence-refuse; spec-7.2 D6 +2 lms data-dispatch/conn-reset; spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); + 'postgres', + 'SELECT count(*) FROM pg_stat_cluster_injections'), + '183', + 'L6a pg_stat_cluster_injections has 183 entries (matches t/015 registry authority)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/106_lmd_smoke.pl b/src/test/cluster_tap/t/106_lmd_smoke.pl index 17ff268c20f..072fb36dd93 100644 --- a/src/test/cluster_tap/t/106_lmd_smoke.pl +++ b/src/test/cluster_tap/t/106_lmd_smoke.pl @@ -20,9 +20,8 @@ # L7: L122 alphabetic order — pg_cluster_state ORDER BY category # emits 'lmd' between 'lck' and 'lmon' (actual psql output # verify;**禁止凭字典直觉 sed** — ASCII `d` 0x64 < `o` 0x6F) -# L8: dump_lmd contributes 24 rows under category='lmd' -# (spec-2.19 daemon rows + spec-2.22 graph/Tarjan rows + spec- -# 2.24 D + cleanup rows) +# L8: dump_lmd contributes 51 rows under category='lmd' +# (existing daemon/graph/cancel/recovery rows + 4 PCM-X WFG rows) # in pg_cluster_state # L9: pg_cluster_lmd view explicit column count = 10 # (regression防御 against view schema drift) @@ -169,11 +168,12 @@ # ---------- -# L8: dump_lmd contributes 45 rows under category='lmd' in +# L8: dump_lmd contributes 51 rows under category='lmd' in # pg_cluster_state: # - spec-2.19 daemon state surface: state + ready_at_us + 5 counters # - spec-2.22 graph/Tarjan surface: 9 graph/deadlock rows # - spec-2.23 probe: 2 rows; spec-2.24 cancel/cleanup: 6 rows +# - PCM-X connector WFG replace/remove/failure/exact-remove-stale accounting: 4 rows # - spec-5.8 D6: 3 rows (deadlock_confirmed / confirm_unconfirmed / # reconfig_discard — coordinator two-round confirm + reconfig gate) # - spec-5.8 D8: 5 rows (probe_report_enqueue / probe_drop_stale / @@ -188,8 +188,8 @@ # ---------- my $lmd_rows = $node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='lmd'}); -is($lmd_rows, '47', - "L8 dump_lmd emits 47 rows under category='lmd' (daemon + graph/Tarjan/probe + spec-2.24 cancel/cleanup + spec-5.8 D6 confirm/reconfig + D8 shmem collector + spec-5.9 D10 victim-policy/cancel-robustness surface + Hardening v1.0.1 cancel_ack_mismatch + spec-5.8 Hardening v1.0.1 member_incomplete_count)"); +is($lmd_rows, '51', + "L8 dump_lmd emits 51 rows under category='lmd' (existing 47 + PCM-X connector WFG 4)"); # ---------- diff --git a/src/test/cluster_tap/t/108_pcm_state_machine.pl b/src/test/cluster_tap/t/108_pcm_state_machine.pl index 8a68afef215..0081dafc7b5 100644 --- a/src/test/cluster_tap/t/108_pcm_state_machine.pl +++ b/src/test/cluster_tap/t/108_pcm_state_machine.pl @@ -13,11 +13,11 @@ # L1 default cluster.pcm_grd_max_entries=-1 → auto NBuffers + pcm # category visible in pg_cluster_state # L2 explicit cluster.pcm_grd_max_entries=0 → PCM disabled surface -# L3 dump_pcm existing 6 row + 14 NEW row = total 20 + api_state -# string value "active"/"stub" +# L3 dump_pcm emits the exact 58-row PCM surface, including PCM-X +# FIFO, ownership, and runtime gauges # L4 9 transition counter rows present + non-negative -# L5 ClusterPcmTransitionApply wait event in pg_stat_cluster_wait_events -# L6 wait event baseline 75 → 77 → 78 → 79 (spec-2.32 +GCS_REPLY_WAIT;PCM_GRD_INIT + PCM_TRANSITION_APPLY) +# L5 ClusterPcmTransitionApply and PcmBlockConvertWait are registered +# L6 wait event baseline is exactly 123 # L7 no PCM wire opcode smoke (L107 N+5 严守:no PCM wire enum added) # # Spec: spec-2.30-pcm-9-state-machine-activation.md (FROZEN v0.3) @@ -48,8 +48,8 @@ my $pcm_category_rows = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_cluster_state WHERE category = 'pcm'"); -is($pcm_category_rows, '28', - 'L1 pg_cluster_state pcm category has 28 rows (spec-2.30 D9 surface + spec-6.14a D5 + spec-6.14 D5 KO-aux-defer counter + spec-4.6a D12 dead_cleanup_entries + ownership-gen wave 4: writer_cover_stale_detected + writer_reverify_reacquire + restore_aba_detected + invalidate_parked_grant_pending + S3-forensics wm_prov_insert_fail_count)'); +is($pcm_category_rows, '58', + 'L1 pg_cluster_state pcm category has 58 rows (existing 28 + PCM-X FIFO/ownership/runtime 30)'); # L3 — api_state shows "active" when GUC=-1 default my $api_state_default = $node_default->safe_psql( @@ -76,13 +76,19 @@ 'postgres', "SELECT count(*) FROM pg_stat_cluster_wait_events WHERE name = 'ClusterPcmTransitionApply'"); is($apply_event, '1', - 'L5 ClusterPcmTransitionApply wait event registered'); + 'L5 ClusterPcmTransitionApply wait event registered'); + +my $convert_wait_event = $node_default->safe_psql( + 'postgres', + "SELECT count(*) FROM pg_stat_cluster_wait_events WHERE name = 'PcmBlockConvertWait'"); +is($convert_wait_event, '1', + 'L5b PcmBlockConvertWait is the registered PCM-X FIFO wait event'); # L6 — wait event count baseline through spec-6.1. my $wait_event_count = $node_default->safe_psql( 'postgres', "SELECT count(*) FROM pg_stat_cluster_wait_events"); is($wait_event_count, '123', - 'L6 wait event baseline 118 (spec-6.13 RDMA wait surface)'); + 'L6 wait event baseline is exactly 123'); # L7 — no PCM wire opcode smoke (no SQL-visible PCM wire opcode enum surface) my $pcm_grd_init_event = $node_default->safe_psql( diff --git a/src/test/cluster_tap/t/109_lmd_deadlock_smoke.pl b/src/test/cluster_tap/t/109_lmd_deadlock_smoke.pl index 25e82e71b1e..30bc07e666b 100755 --- a/src/test/cluster_tap/t/109_lmd_deadlock_smoke.pl +++ b/src/test/cluster_tap/t/109_lmd_deadlock_smoke.pl @@ -294,12 +294,12 @@ sub wait_one_scan_tick { }); is($d6_sum, '0', 'L18 spec-5.8 D6 counters all start at 0 (single-node: coordinator scan no-op)'); -# L19: total lmd category row count is now 45 (24 + 3 D6 + 5 D8 + 13 spec-5.9 D10), +# L19: total lmd category row count is now 51 (existing 47 + PCM-X connector WFG 4), # matching dump_lmd. my $lmd_total = $node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category = 'lmd'}); -is($lmd_total, '47', - 'L19 dump_lmd emits 47 rows under category=lmd (incl. 3 spec-5.8 D6 + 5 D8 + 13 spec-5.9 D10 + 1 Hardening v1.0.1 cancel_ack_mismatch + 1 spec-5.8 Hardening v1.0.1 member_incomplete_count)'); +is($lmd_total, '51', + 'L19 dump_lmd emits 51 rows under category=lmd (existing 47 + PCM-X connector WFG 4)'); $node->stop; diff --git a/src/test/cluster_tap/t/110_gcs_loopback.pl b/src/test/cluster_tap/t/110_gcs_loopback.pl index 6e46e469689..8f9e2299d1f 100644 --- a/src/test/cluster_tap/t/110_gcs_loopback.pl +++ b/src/test/cluster_tap/t/110_gcs_loopback.pl @@ -65,12 +65,12 @@ sub gcs_value { $node->start; -# L1 — pg_cluster_state.gcs surface has 112 keys (branch-1 +1 master-direct rescue; round-4c fallback-scn + spec-7.2 D6 hist). +# L1 — pg_cluster_state.gcs surface has 119 keys (PCM-X queue observability +3). is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows; round-6 +1 xfer_stale_deny; ownership-gen ruling② +2 invalidate_busy_sent/received) (spec-7.2 D6)'); + '119', + 'L1 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); # L2 — api_state = "active" after postmaster phase 1 init. diff --git a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl index 54e13d39cb6..4584011bbf8 100644 --- a/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl +++ b/src/test/cluster_tap/t/111_gcs_block_ship_2node.pl @@ -8,7 +8,7 @@ # # L1 ClusterPair startup — both postmasters healthy + tier1 connected # L2 fresh baseline gcs counters on both nodes (block_* counters = 0) -# L3 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) (cumulative through +# L3 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3) (cumulative through # spec-6.13 direct-land observability) # L4 4 NEW wait events registered in pg_stat_cluster_wait_events: # ClusterGCSBlockShipWait, ClusterGCSBlockRequestDispatch, @@ -114,19 +114,19 @@ sub gcs_int_value { # ============================================================ -# L3: pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) +# L3: pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3) # (cumulative GCS surface through spec-6.13 direct-land observability). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '119', + 'L3 node0 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '119', + 'L3 node1 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); # ============================================================ diff --git a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl index 3f4f462fd9c..6e016070fe3 100644 --- a/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl +++ b/src/test/cluster_tap/t/112_gcs_block_retransmit_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 9 NEW reliability counters all 0 on both nodes -# L3 pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; cumulative through spec-7.2 D6 hist) +# L3 pg_cluster_state.gcs has 119 keys (PCM-X queue observability +3) # L4 2 NEW wait events registered (ClusterGCSBlockRetransmitWait + # ClusterGCSBlockEpochStaleRetry) # L5 CLUSTER_WAIT_EVENTS_COUNT = 120 (spec-7.2 +2) @@ -111,18 +111,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; cumulative through spec-7.2 D6 hist). +# L3: pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); + '119', + 'L3 node0 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)'); + '119', + 'L3 node1 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); # ============================================================ diff --git a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl index e18093b8b0b..b81c3164fd0 100644 --- a/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl +++ b/src/test/cluster_tap/t/113_gcs_block_2way_2node.pl @@ -9,7 +9,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 7 NEW counters all 0 + catversion >= 202605420 -# L3 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) (cumulative through spec-6.13) +# L3 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3) # L4 cross-node forward path: node A read first → master_holder = A; # force same tag on node B via test-only injection → master # chooses forward path → A direct-ships to B → block_forward_sent @@ -109,18 +109,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 119 keys (PCM-X queue observability +3). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '119', + 'L3 node0 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '119', + 'L3 node1 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); # ============================================================ diff --git a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl index 3c494a3f01a..88cca599f64 100644 --- a/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl +++ b/src/test/cluster_tap/t/114_gcs_block_3way_2node.pl @@ -12,7 +12,7 @@ # # L1 ClusterPair startup baseline (both postmasters healthy) # L2 fresh baseline: 6 NEW spec-2.36 counters all 0 -# L3 pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip; was 67 cumulative through spec-6.14a) +# L3 pg_cluster_state.gcs has 119 keys (PCM-X queue observability +3) # L4 catversion lower-bound >= 202605430; wait event count == 123 # L5 S barrier injection — DENIED_PENDING_X surfaces under # cluster-gcs-block-starvation-force-denied inject; reader @@ -107,18 +107,18 @@ sub gcs_int # ============================================================ -# L3: pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip; was 67 cumulative through spec-6.14a). +# L3: pg_cluster_state.gcs has 119 keys (PCM-X queue observability +3). # ============================================================ is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node0 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '119', + 'L3 node0 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L3 node1 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows) (spec-7.2 D6+flip)'); + '119', + 'L3 node1 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); # ============================================================ diff --git a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl index a08eaf6945d..646770944e8 100644 --- a/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl +++ b/src/test/cluster_tap/t/115_gcs_block_3way_3node.pl @@ -130,8 +130,8 @@ sub gcs_int is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - "L4 node$i pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; round-4c +3 fallback-scn rows; gcs-race-fix-2 +6 rows; serve-stall round-5 +6 send-admission +4 bounded-drop rows)"); + '119', + "L4 node$i pg_cluster_state.gcs has 119 keys (PCM-X queue observability +3)"); } diff --git a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl index b1223f72f18..9b396b50b71 100644 --- a/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl +++ b/src/test/cluster_tap/t/116_gcs_block_lost_write_2node.pl @@ -20,7 +20,7 @@ # L7 SQLSTATE 53R93 ERRCODE_CLUSTER_LOST_WRITE_DETECTED literal- # encodable in PG SQL (catalog 形式 verification) # L8 GUC switch back to 'error' SHOW returns 'error' -# L9 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; spec-7.2 D6+flip) (cumulative through spec-6.14a) +# L9 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3) # L10 Reply status enum value 12 (DENIED_LOST_WRITE) is新增的 # 最大 value (baseline workload must not trigger lost-write) # L11 spec-2.41 D / P1-C — behavioral lost-write inject: a @@ -108,8 +108,8 @@ sub gcs_int is($pair->node0->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L2 pg_cluster_state.gcs category has 112 keys (branch-1 +1 master-direct rescue; gcs-race-fix-2 +6 rows) (spec-7.2 D6+flip) (cumulative through spec-6.14a)'); + '119', + 'L2 pg_cluster_state.gcs category has 119 keys (PCM-X queue observability +3)'); # ============================================================ @@ -194,13 +194,13 @@ sub gcs_int # ============================================================ -# L9 (alias of L2): gcs key count = 89. +# L9 (alias of L2): gcs key count = 119. # ============================================================ is($pair->node1->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='gcs'}), - '112', - 'L9 node1 pg_cluster_state.gcs has 112 keys (branch-1 +1 master-direct rescue; gcs-race-fix-2 +6 rows) (cross-node parity)'); + '119', + 'L9 node1 pg_cluster_state.gcs has 119 keys (PCM-X queue observability +3)'); # ============================================================ diff --git a/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl new file mode 100644 index 00000000000..014bfa05290 --- /dev/null +++ b/src/test/cluster_tap/t/400_pcm_x_queue_4node_liveness.pl @@ -0,0 +1,588 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 400_pcm_x_queue_4node_liveness.pl +# spec-2.36a S3-core RED/GREEN: four nodes concurrently update four +# different tuples that occupy the same heap BufferTag. Every writer +# must make progress without surfacing a client error. +# +# Author: SqlRush +# +#------------------------------------------------------------------------- + +use strict; +use warnings FATAL => 'all'; + +use IPC::Run qw(start finish timeout); +use PostgreSQL::Test::ClusterQuad; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(time usleep); + +sub state_int +{ + my ($node, $category, $key) = @_; + my $value = $node->safe_psql('postgres', + qq{SELECT value FROM pg_cluster_state WHERE category='$category' AND key='$key'}); + unless (defined($value) && $value =~ /\A\d+\z/) + { + my $shown = defined($value) ? $value : ''; + die "missing or non-integer pg_cluster_state key $category.$key: [$shown]"; + } + return $value + 0; +} + +my @pcm_x_pcm_keys = qw( + pcm_x_runtime_state + pcm_x_runtime_generation + pcm_x_queue_enqueue_count + pcm_x_queue_admit_count + pcm_x_queue_confirm_count + pcm_x_queue_promotion_count + pcm_x_queue_transfer_count + pcm_x_queue_complete_count + pcm_x_queue_cancel_count + pcm_x_queue_revoke_count + pcm_x_queue_coalesced_count + pcm_x_queue_wait_count + pcm_x_queue_full_count + pcm_x_queue_stale_count + pcm_x_queue_miss_count + pcm_x_queue_recovery_blocked_count + pcm_x_queue_activating_reset_count + pcm_x_queue_depth + pcm_x_queue_depth_high_water + pcm_x_queue_active_tags + pcm_x_queue_live_tickets + pcm_x_queue_live_slots + pcm_x_local_retire_gate + pcm_x_local_retire_marker_count + pcm_x_local_retire_marker_ticket_id + pcm_x_own_begin_count + pcm_x_own_commit_count + pcm_x_own_abort_count + pcm_x_own_busy_count + pcm_x_own_corrupt_count +); + +my @pcm_x_lmd_keys = qw( + pcm_convert_wfg_replace_count + pcm_convert_wfg_remove_count + pcm_convert_wfg_replace_fail_count + pcm_convert_wfg_exact_remove_stale_count +); + +my @pcm_x_final_gauge_keys = qw( + pcm_x_queue_depth + pcm_x_queue_active_tags + pcm_x_queue_live_tickets + pcm_x_queue_live_slots + pcm_x_local_retire_gate + pcm_x_local_retire_marker_count + pcm_x_local_retire_marker_ticket_id +); + +my @positive_pcm_lifecycle_keys = qw( + pcm_x_queue_enqueue_count + pcm_x_queue_admit_count + pcm_x_queue_confirm_count + pcm_x_queue_promotion_count + pcm_x_queue_transfer_count + pcm_x_queue_complete_count + pcm_x_queue_revoke_count + pcm_x_queue_wait_count + pcm_x_own_begin_count + pcm_x_own_commit_count +); + +my @zero_pcm_failure_keys = qw( + pcm_x_queue_cancel_count + pcm_x_queue_full_count + pcm_x_queue_stale_count + pcm_x_queue_miss_count + pcm_x_queue_recovery_blocked_count + pcm_x_queue_activating_reset_count + pcm_x_own_abort_count + pcm_x_own_corrupt_count +); + +my @positive_wfg_keys = qw( + pcm_convert_wfg_replace_count + pcm_convert_wfg_remove_count +); + +sub exact_key_count +{ + my ($node, $category, $keys) = @_; + my $quoted = join(',', map { "'$_'" } @{$keys}); + return $node->safe_psql('postgres', + qq{SELECT count(*) FROM pg_cluster_state WHERE category='$category' AND key IN ($quoted)}); +} + +sub state_snapshot +{ + my ($node, $category, $keys) = @_; + my $quoted = join(',', map { "'$_'" } @{$keys}); + my $rows = $node->safe_psql('postgres', + qq{SELECT key || E'\\t' || value FROM pg_cluster_state } + . qq{WHERE category='$category' AND key IN ($quoted) ORDER BY key}, + timeout => 5); + my %expected = map { $_ => 1 } @{$keys}; + my %snapshot; + + for my $row (grep { $_ ne '' } split(/\n/, $rows)) + { + my ($key, $value) = split(/\t/, $row, 2); + my $shown_key = defined($key) ? $key : ''; + + die "unexpected pg_cluster_state key $category.$shown_key" + unless defined($key) && exists($expected{$key}); + die "duplicate pg_cluster_state key $category.$key" + if exists($snapshot{$key}); + my $shown = defined($value) ? $value : ''; + die "missing or non-integer pg_cluster_state key $category.$key: [$shown]" + unless defined($value) && $value =~ /\A\d+\z/; + $snapshot{$key} = $value + 0; + } + for my $key (@{$keys}) + { + die "missing pg_cluster_state key $category.$key" + unless exists($snapshot{$key}); + } + return \%snapshot; +} + +sub aggregate_snapshots +{ + my ($snapshots, $keys) = @_; + my %aggregate = map { $_ => 0 } @{$keys}; + + for my $snapshot (@{$snapshots}) + { + $aggregate{$_} += $snapshot->{$_} for @{$keys}; + } + return \%aggregate; +} + +sub wait_for_pcm_gauges_zero +{ + my ($quad, $all_keys, $gauge_keys, $timeout_seconds) = @_; + my $gauge_deadline = time() + $timeout_seconds; + my @snapshots; + + do + { + @snapshots = map { + state_snapshot($quad->node($_), 'pcm', $all_keys) + } (0 .. 3); + my $all_zero = 1; + + for my $snapshot (@snapshots) + { + for my $key (@{$gauge_keys}) + { + $all_zero = 0 if $snapshot->{$key} != 0; + } + } + return (1, \@snapshots) if $all_zero; + usleep(250_000); + } while (time() < $gauge_deadline); + + return (0, \@snapshots); +} + +my $warmup_error_count = 0; + +sub write_retry +{ + my ($node, $sql, $attempts) = @_; + $attempts //= 20; + my ($last_rc, $last_out, $last_err); + for (1 .. $attempts) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 30); + ($last_rc, $last_out, $last_err) = ($rc, $out, $err); + return 1 if defined($rc) && $rc == 0; + $warmup_error_count++; + diag('warmup retry error: rc=' . (defined($rc) ? $rc : 'undef') + . ' stdout=[' . ($out // '') . '] stderr=[' . ($err // '') . ']'); + usleep(300_000); + } + diag('write_retry exhausted: rc=' . (defined($last_rc) ? $last_rc : 'undef') + . ' stdout=[' . ($last_out // '') . '] stderr=[' . ($last_err // '') . ']'); + return 0; +} + +sub write_file +{ + my ($path, $contents) = @_; + open(my $fh, '>', $path) or die "open $path: $!"; + print {$fh} $contents; + close($fh) or die "close $path: $!"; +} + +my $quad = PostgreSQL::Test::ClusterQuad->new_quad( + 'pcm_xq_liveness', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.read_scache = on', + 'cluster.online_join = on', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.join_convergence_timeout_ms = 30000', + 'cluster.xid_striping = on', + 'cluster.crossnode_runtime_visibility = on', + 'cluster.page_scn_shortcut = on', + 'cluster.crossnode_write_write = on', + 'cluster.undo_gcs_coherence = on', + 'cluster.crossnode_cr_data_plane = on', + 'cluster.gcs_reply_timeout_ms = 3000', + 'cluster.gcs_block_retransmit_max_retries = 8', + 'cluster.cssd_heartbeat_interval_ms = 2000', + 'cluster.cssd_dead_deadband_factor = 10', + ]); + +$quad->start_quad; +usleep(3_000_000); + +for my $from (0 .. 3) +{ + is($quad->node($from)->safe_psql('postgres', 'SELECT 1'), '1', + "L1 node$from is alive"); + for my $to (0 .. 3) + { + next if $from == $to; + ok($quad->wait_for_peer_state($from, $to, 'connected', 45), + "L1 node$from sees node$to connected"); + } +} + +for my $node ($quad->nodes) +{ + is($node->safe_psql('postgres', 'SHOW cluster.xid_striping'), 'on', + 'L1 xid striping is active on the writer topology'); + is($node->safe_psql('postgres', 'SHOW cluster.crossnode_runtime_visibility'), 'on', + 'L1 runtime visibility is active on the writer topology'); + is($node->safe_psql('postgres', 'SHOW cluster.gcs_block_local_cache'), 'on', + 'L1 hold-until-revoked cache is active by default'); + cmp_ok(state_int($node, 'xid_stripe', 'xid_stripe_activated_floor'), '>', 0, + 'L1 xid stripe activation floor is published'); + is(exact_key_count($node, 'pcm', \@pcm_x_pcm_keys), scalar(@pcm_x_pcm_keys), + 'L1 PCM-X PCM observability key set is complete'); + is(exact_key_count($node, 'lmd', \@pcm_x_lmd_keys), scalar(@pcm_x_lmd_keys), + 'L1 PCM-X LMD observability key set is complete'); + state_int($node, 'pcm', $_) for @pcm_x_pcm_keys; + state_int($node, 'lmd', $_) for @pcm_x_lmd_keys; + is(state_int($node, 'pcm', 'pcm_x_runtime_state'), 1, + 'L1 PCM-X runtime is ACTIVE before the workload'); + cmp_ok(state_int($node, 'pcm', 'pcm_x_runtime_generation'), '>', 0, + 'L1 PCM-X runtime generation is published before the workload'); + $node->safe_psql('postgres', q{ + CREATE TABLE pcm_xq_hot ( + id integer, + v bigint NOT NULL + ) WITH (fillfactor = 100); + CREATE TABLE pcm_xq_self ( + id integer, + v bigint NOT NULL + ) WITH (fillfactor = 100) + }); +} + +my @paths = map { + $quad->node($_)->safe_psql('postgres', + q{SELECT pg_relation_filepath('pcm_xq_hot')}) +} (0 .. 3); +is(scalar(grep { $_ eq $paths[0] } @paths), 4, + 'L2 all nodes map the test table to the same relation file'); + +ok(write_retry($quad->node0, + q{INSERT INTO pcm_xq_hot(id, v) SELECT g, 0 FROM generate_series(1, 4) g}), + 'L2 seeded four distinct tuples'); +my ($self_seed_rc, $self_seed_out, $self_seed_err) = $quad->node0->psql('postgres', q{ + SET cluster.gcs_block_local_cache = off; + INSERT INTO pcm_xq_self(id, v) VALUES (1, 0) +}, timeout => 30); +is($self_seed_rc, 0, + 'L2 seeded the sole-S tuple with X released to N at unlock') + or diag("L2 sole-S seed stdout=[$self_seed_out] stderr=[$self_seed_err]"); +ok(write_retry($quad->node0, + q{VACUUM (FREEZE, DISABLE_PAGE_SKIPPING) pcm_xq_hot}), + 'L2 seed frozen so the queue test begins from a stable committed page image'); +ok(write_retry($quad->node0, + q{SELECT count(*) FROM pcm_xq_hot WHERE id BETWEEN 1 AND 4}), + 'L2 seed owner installed committed visibility hints'); +ok(write_retry($quad->node0, 'CHECKPOINT'), 'L2 seed checkpointed'); + +# Build a deterministic sole-requester S source before the four-writer leg. +# The cache-off INSERT above releases node0's X to N at content-lock unlock. +# No other node has touched this relation, so this cache-on node0 read is the +# unique N->S grant. The following node0 UPDATE must therefore take the +# sole-requester S->X handoff, not a remote-source transfer or cached-X path. +my ($self_read_rc, $self_read_out, $self_read_err) = $quad->node0->psql( + 'postgres', q{SELECT v FROM pcm_xq_self WHERE id = 1}, timeout => 30); +is($self_read_rc, 0, 'L2S sole requester acquired the only S copy'); +is($self_read_out, '0', 'L2S sole requester saw the seeded image') + or diag("L2S sole requester read stderr=[$self_read_err]"); + +my $self_handoff_before = state_int($quad->node0, 'gcs', + 'pcm_x_self_handoff_count'); +my $self_handoff_drain_before = state_int($quad->node0, 'gcs', + 'pcm_x_self_handoff_drain_count'); + +my ($self_write_rc, $self_write_out, $self_write_err) = $quad->node0->psql( + 'postgres', q{UPDATE pcm_xq_self SET v = v + 1 WHERE id = 1}, timeout => 30); +is($self_write_rc, 0, 'L2S sole-S requester completed S-to-X conversion') + or diag("L2S sole-S write stdout=[$self_write_out] stderr=[$self_write_err]"); + +my ($self_handoff_after, $self_handoff_drain_after); +my $self_drain_deadline = time() + 15; +do +{ + $self_handoff_after = state_int($quad->node0, 'gcs', + 'pcm_x_self_handoff_count'); + $self_handoff_drain_after = state_int($quad->node0, 'gcs', + 'pcm_x_self_handoff_drain_count'); + last if $self_handoff_after > $self_handoff_before + && $self_handoff_drain_after > $self_handoff_drain_before; + usleep(100_000); +} while (time() < $self_drain_deadline); + +cmp_ok($self_handoff_after - $self_handoff_before, '>', 0, + 'L2S sole-requester S source exercised the fused revoke-to-grant handoff'); +cmp_ok($self_handoff_drain_after - $self_handoff_drain_before, '>', 0, + 'L2S sole-requester S source released its immutable record at DRAIN'); +is($quad->node0->safe_psql('postgres', + q{SELECT v FROM pcm_xq_self WHERE id = 1}), '1', + 'L2S sole-requester conversion preserved exact page contents'); + +for my $i (0 .. 3) +{ + ok(write_retry($quad->node($i), + q{SELECT count(*) FROM pcm_xq_hot WHERE id BETWEEN 1 AND 4}), + "L2 node$i can read the seed"); +} +is($warmup_error_count, 0, + 'L2 warmup completed without any transient or terminal client error'); + +my $block_count = $quad->node0->safe_psql('postgres', q{ + SELECT count(DISTINCT split_part(trim(both '()' from ctid::text), ',', 1)) + FROM pcm_xq_hot WHERE id BETWEEN 1 AND 4 +}); +is($block_count, '1', + 'L2 four different tuple ids occupy one heap block/BufferTag'); + +my $tuple_map = $quad->node0->safe_psql('postgres', q{ + SELECT string_agg(id::text || ':' || ctid::text, ',' ORDER BY id) + FROM pcm_xq_hot WHERE id BETWEEN 1 AND 4 +}); +diag("L2 fixed hot-block tuple map: rel=$paths[0] tuples=$tuple_map"); + +my @pcm_before_by_node = map { + state_snapshot($quad->node($_), 'pcm', \@pcm_x_pcm_keys) +} (0 .. 3); +my @lmd_before_by_node = map { + state_snapshot($quad->node($_), 'lmd', \@pcm_x_lmd_keys) +} (0 .. 3); +my %pcm_before = %{aggregate_snapshots(\@pcm_before_by_node, \@pcm_x_pcm_keys)}; +my %lmd_before = %{aggregate_snapshots(\@lmd_before_by_node, \@pcm_x_lmd_keys)}; +my $queue_before = $pcm_before{pcm_x_queue_enqueue_count}; +my $denied_before = 0; +$denied_before += state_int($quad->node($_), 'gcs', + 'starvation_denied_pending_x_count') for (0 .. 3); +my $passive_s_before = 0; +$passive_s_before += state_int($quad->node($_), 'gcs', + 'invalidate_passive_s_release_count') for (0 .. 3); +my $start_at = $quad->node0->safe_psql('postgres', + q{SELECT (clock_timestamp() + interval '5 seconds')::text}); +my $script_dir = PostgreSQL::Test::Utils::tempdir(); +my @runs; + +for my $i (0 .. 3) +{ + my $id = $i + 1; + my $script = "$script_dir/node$i.sql"; + write_file($script, + "SELECT pg_sleep(GREATEST(0.0, EXTRACT(EPOCH FROM " + . "(TIMESTAMPTZ '$start_at' - clock_timestamp()))));\n" + . "UPDATE pcm_xq_hot SET v = v + 1 WHERE id = $id;\n"); + + my %run = (stdout => '', stderr => '', timed_out => 0); + my @cmd = ( + $quad->node($i)->installed_command('pgbench'), + '-n', '-c', '1', '-j', '1', '-T', '15', '--max-tries=1', + '-f', $script, '-h', $quad->node($i)->host, + '-p', $quad->node($i)->port, 'postgres'); + $run{handle} = start(\@cmd, '<', \undef, '>', \$run{stdout}, + '2>', \$run{stderr}, timeout(45)); + push @runs, \%run; +} + +for my $i (0 .. 3) +{ + my $run = $runs[$i]; + my $finished = eval { finish($run->{handle}); 1 }; + unless ($finished) + { + $run->{timed_out} = 1; + $run->{finish_error} = $@; + eval { $run->{handle}->kill_kill; }; + } + $run->{result} = eval { $run->{handle}->result(0) }; + $run->{result} = -1 unless defined($run->{result}); + ($run->{transactions}) = + $run->{stdout} =~ /number of transactions actually processed:\s+(\d+)/; + $run->{transactions} //= 0; + my @errors = $run->{stderr} =~ /^.*(?:ERROR|FATAL|PANIC):.*$/mg; + $run->{errors} = scalar(@errors); + diag("L3 node$i result=$run->{result} timed_out=$run->{timed_out} " + . "transactions=$run->{transactions} errors=$run->{errors} " + . 'finish_error=[' . ($run->{finish_error} // '') . '] ' + . "stderr=[$run->{stderr}]"); +} + +my ($gauges_drained, $pcm_after_by_node_ref) = wait_for_pcm_gauges_zero( + $quad, \@pcm_x_pcm_keys, \@pcm_x_final_gauge_keys, 30); +my @pcm_after_by_node = @{$pcm_after_by_node_ref}; +my @lmd_after_by_node = map { + state_snapshot($quad->node($_), 'lmd', \@pcm_x_lmd_keys) +} (0 .. 3); +my %pcm_after = %{aggregate_snapshots(\@pcm_after_by_node, \@pcm_x_pcm_keys)}; +my %lmd_after = %{aggregate_snapshots(\@lmd_after_by_node, \@pcm_x_lmd_keys)}; +my $queue_after = $pcm_after{pcm_x_queue_enqueue_count}; +my $denied_after = 0; +$denied_after += state_int($quad->node($_), 'gcs', + 'starvation_denied_pending_x_count') for (0 .. 3); +my $passive_s_after = 0; +$passive_s_after += state_int($quad->node($_), 'gcs', + 'invalidate_passive_s_release_count') for (0 .. 3); +diag('L3 path probes: pcm_x_queue_enqueue_delta=' + . ($queue_after - $queue_before) + . ' legacy_denied_pending_x_delta=' + . ($denied_after - $denied_before) + . ' passive_s_release_delta=' + . ($passive_s_after - $passive_s_before)); +for my $key (@pcm_x_pcm_keys) +{ + diag("L3 PCM-X state $key=$pcm_after{$key} delta=" + . ($pcm_after{$key} - $pcm_before{$key})); +} +for my $key (@pcm_x_lmd_keys) +{ + diag("L3 PCM-X LMD state $key=$lmd_after{$key} delta=" + . ($lmd_after{$key} - $lmd_before{$key})); +} +for my $i (0 .. 3) +{ + my $detail = $quad->node($i)->safe_psql('postgres', q{ + SELECT string_agg(key || '=' || value, ', ' ORDER BY key) + FROM pg_cluster_state + WHERE category = 'gcs' + AND key IN ( + 'block_master_not_holder_count', + 'block_x_self_ship_count', + 'drop_pinned_deny_count', + 'x_vs_s_no_carrier_denied_count', + 'block_invalidate_broadcast_count', + 'block_invalidate_ack_received_count', + 'invalidate_passive_s_release_count', + 'pcm_x_self_handoff_count', + 'pcm_x_self_handoff_drain_count', + 'invalidate_send_not_admitted_count', + 'forward_send_not_admitted_count', + 'reply_send_not_admitted_count') + }); + diag("L3 node$i GCS branch state: $detail"); + my $visibility = $quad->node($i)->safe_psql('postgres', q{ + SELECT string_agg(category || '.' || key || '=' || value, + ', ' ORDER BY category, key) + FROM pg_cluster_state + WHERE (category = 'cr' AND key IN ( + 'rtvis_verdict_wire_count', + 'rtvis_verdict_failclosed_count', + 'rtvis_verdict_exact_count', + 'rtvis_verdict_below_horizon_count', + 'rtvis_verdict_inadmissible_count')) + OR (category = 'xnode' AND key IN ( + 'c_resolve_count', 'c_tt_lookup_count', + 'c_memo_hit_count', 'c_memo_install_count')) + }); + diag("L3 node$i visibility state: $visibility"); + } + +for my $i (0 .. 3) +{ + is($runs[$i]->{timed_out}, 0, "L3 node$i writer met the hard deadline"); + is($runs[$i]->{result}, 0, "L3 node$i writer exited successfully"); + is($runs[$i]->{errors}, 0, "L3 node$i writer surfaced zero client errors"); + cmp_ok($runs[$i]->{transactions}, '>', 0, + "L3 node$i writer made progress"); + cmp_ok( + $pcm_after_by_node[$i]->{pcm_x_queue_enqueue_count} + - $pcm_before_by_node[$i]->{pcm_x_queue_enqueue_count}, + '>', 0, "L3 node$i PCM-X enqueue delta proves this requester used the queue"); +} + +cmp_ok($queue_after - $queue_before, '>=', 4, + 'L3 all four node writers entered the PCM-X queue protocol'); +is($denied_after - $denied_before, 0, + 'L3 no writer fell back to legacy reader pending-X denial'); +cmp_ok($passive_s_after - $passive_s_before, '>', 0, + 'L3 exact queue INVALIDATE exercised passive-pinned S release'); +for my $key (@positive_pcm_lifecycle_keys) +{ + cmp_ok($pcm_after{$key} - $pcm_before{$key}, '>', 0, + "L3 PCM-X lifecycle $key advanced"); +} +for my $key (@zero_pcm_failure_keys) +{ + is($pcm_after{$key} - $pcm_before{$key}, 0, + "L3 PCM-X failure counter $key stayed zero"); +} +for my $key (@positive_wfg_keys) +{ + cmp_ok($lmd_after{$key} - $lmd_before{$key}, '>', 0, + "L3 PCM-X WFG lifecycle $key advanced"); +} +is($lmd_after{pcm_convert_wfg_replace_fail_count} + - $lmd_before{pcm_convert_wfg_replace_fail_count}, + 0, 'L3 PCM-X WFG atomic replace failures stayed zero'); +is($lmd_after{pcm_convert_wfg_exact_remove_stale_count} + - $lmd_before{pcm_convert_wfg_exact_remove_stale_count}, + 0, 'L3 PCM-X WFG exact-remove stale identities stayed zero'); +ok($gauges_drained, 'L3 PCM-X terminal gauges drained within 30 seconds'); +for my $key (@pcm_x_final_gauge_keys) +{ + is($pcm_after{$key}, 0, "L3 final aggregate PCM-X gauge $key is zero"); +} + +my ($advanced_rc, $advanced, $advanced_err) = $quad->node0->psql('postgres', q{ + SELECT string_agg(id::text || ':' || row_count::text || ':' || total_v::text, + ',' ORDER BY id) + FROM ( + SELECT id, count(*) AS row_count, sum(v) AS total_v + FROM pcm_xq_hot + GROUP BY id + ) exact_rows +}, timeout => 30); +my $expected_exact = join(',', map { + my $id = $_ + 1; + $id . ':1:' . $runs[$_]->{transactions} +} (0 .. 3)); +is($advanced_rc, 0, 'L4 final exact-conservation query completed'); +is($advanced, $expected_exact, + 'L4 every logical id is unique and its value equals that node processed transactions') + or diag("L4 expected=[$expected_exact] stderr=[$advanced_err]"); + +my ($sum_rc, $sum_v, $sum_err) = $quad->node0->psql('postgres', + q{SELECT coalesce(sum(v), 0) FROM pcm_xq_hot}, timeout => 30); +my $expected_sum = 0; +$expected_sum += $runs[$_]->{transactions} for (0 .. 3); +is($sum_rc, 0, 'L4 aggregate conservation query completed'); +is($sum_v, "$expected_sum", + 'L4 aggregate value equals total committed pgbench transactions') + or diag("L4 expected_sum=$expected_sum stderr=[$sum_err]"); + +$quad->stop_quad; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index e6c0f05bb01..5ffd59af305 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -40,7 +40,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_ic_mock test_cluster_inject test_cluster_pgstat \ test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr \ test_cluster_scn test_cluster_scn_frontier test_cluster_block_format test_cluster_itl_slot \ - test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ + test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_pcm_own test_cluster_pcm_direct_init test_cluster_pcm_x_convert test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ @@ -96,10 +96,10 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_undo_authority \ test_cluster_undo_gcs \ test_cluster_undo_verdict \ - test_cluster_vis_undo_verdict_map \ - test_cluster_undo_horizon \ - test_cluster_lms_shard \ - test_cluster_gcs_block_dedup \ + test_cluster_vis_undo_verdict_map \ + test_cluster_undo_horizon \ + test_cluster_lms_shard test_cluster_pcm_x_image_fetch \ + test_cluster_gcs_block_dedup \ test_cluster_gcs_block_shard # Path to the cluster_version object (no PG deps, safe to link standalone). @@ -164,6 +164,9 @@ CLUSTER_LOCK_ACQUIRE_O = $(top_builddir)/src/backend/cluster/cluster_lock_acquir # the transition validator + counter accessors standalone (20 PG-runtime # deps stubbed in the test binary). CLUSTER_PCM_LOCK_O = $(top_builddir)/src/backend/cluster/cluster_pcm_lock.o +CLUSTER_PCM_OWN_O = $(top_builddir)/src/backend/cluster/cluster_pcm_own.o +CLUSTER_PCM_DIRECT_INIT_O = $(top_builddir)/src/backend/cluster/cluster_pcm_direct_init.o +CLUSTER_PCM_X_CONVERT_O = $(top_builddir)/src/backend/cluster/cluster_pcm_x_convert.o CLUSTER_IC_TIER1_O = $(top_builddir)/src/backend/cluster/cluster_ic_tier1.o CLUSTER_GCS_O = $(top_builddir)/src/backend/cluster/cluster_gcs.o @@ -201,6 +204,13 @@ test_cluster_gcs_block_dedup: test_cluster_gcs_block_dedup.c unit_test.h $(CLUST $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +CLUSTER_PCM_X_IMAGE_FETCH_O = $(top_builddir)/src/backend/cluster/cluster_pcm_x_image_fetch.o +test_cluster_pcm_x_image_fetch: test_cluster_pcm_x_image_fetch.c unit_test.h \ + $(CLUSTER_PCM_X_IMAGE_FETCH_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_PCM_X_IMAGE_FETCH_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + # spec-7.3 D9: test_cluster_gcs_block_shard links cluster_gcs_block_shard.o # (the REAL staging-path payload -> worker router, extracted as a pure file) # + cluster_lms_shard.o so the route == shard_for_tag agreement is asserted @@ -229,7 +239,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_ic_tier1_partial test_cluster_lms_outbound test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_scn_frontier test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd_outbound test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_lms_shard test_cluster_gcs_block_dedup test_cluster_gcs_block_shard test_cluster_undo_resid test_cluster_undo_authority test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map test_cluster_undo_horizon,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_ic_tier1_partial test_cluster_lms_outbound test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_scn_frontier test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd_outbound test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_writer_chain test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_mxid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_block_dedup_reclaim test_cluster_gcs_block_dedup_htab test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_multixact_served test_cluster_pcm_lock test_cluster_pcm_own test_cluster_pcm_direct_init test_cluster_pcm_x_convert test_cluster_pcm_x_image_fetch test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority test_cluster_lms_shard test_cluster_gcs_block_dedup test_cluster_gcs_block_shard test_cluster_undo_resid test_cluster_undo_authority test_cluster_undo_gcs test_cluster_undo_verdict test_cluster_vis_undo_verdict_map test_cluster_undo_horizon,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -1277,12 +1287,15 @@ test_cluster_lmd_graph: test_cluster_lmd_graph.c unit_test.h \ # standalone. The wait-state object depends only on pg_atomic (libpgport); # the test exercises publish / clear / read / reset + wait_seq ABA on a # ClusterLmdProcWaitState directly (no PGPROC / shmem needed). +$(CLUSTER_LMD_WAIT_STATE_O): $(top_srcdir)/src/include/cluster/cluster_lmd_wait_state.h + test_cluster_lmd_wait_state: test_cluster_lmd_wait_state.c unit_test.h \ + $(top_srcdir)/src/include/cluster/cluster_lmd_wait_state.h \ $(CLUSTER_VERSION_O) $(CLUSTER_LMD_WAIT_STATE_O) - $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CC) $(CFLAGS) $(PTHREAD_CFLAGS) $(CPPFLAGS) $< \ $(CLUSTER_VERSION_O) $(CLUSTER_LMD_WAIT_STATE_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ - $(top_builddir)/src/port/libpgport_srv.a -o $@ + $(top_builddir)/src/port/libpgport_srv.a $(PTHREAD_LIBS) -o $@ # spec-5.9 D3: test_cluster_cancel_token links cluster_cancel_token.o # standalone. The match logic (consume_against) only needs the spinlock + @@ -1345,6 +1358,37 @@ test_cluster_pcm_lock: test_cluster_pcm_lock.c unit_test.h \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-2.36a C1/D5a: the real cluster_pcm_own object is linked against a +# fake shmem region. The test also reads the production bufmgr source through +# an explicit build-time path so the D5a lock-order contract is paired with +# the shared decision helpers instead of being represented by a fake bufmgr. +test_cluster_pcm_own: test_cluster_pcm_own.c unit_test.h \ + $(CLUSTER_PCM_OWN_O) + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DBUFMGR_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/buffer/bufmgr.c"' \ + $< $(CLUSTER_PCM_OWN_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + +# S3-core direct-init proof is a dependency-light, process-local decision +# layer. The real bufmgr captures/revalidates the snapshots under its header +# spinlock; this test pins the byte-exact operation and rejection matrix. +test_cluster_pcm_direct_init: test_cluster_pcm_direct_init.c unit_test.h \ + $(CLUSTER_PCM_DIRECT_INIT_O) + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DBUFMGR_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/buffer/bufmgr.c"' \ + -DVM_SOURCE_PATH='"$(top_srcdir)/src/backend/access/heap/visibilitymap.c"' \ + -DFSM_SOURCE_PATH='"$(top_srcdir)/src/backend/storage/freespace/freespace.c"' \ + $< \ + $(CLUSTER_PCM_DIRECT_INIT_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + +# spec-2.36a S3-core: wire ABI + checked five-pool external shmem layout. +test_cluster_pcm_x_convert: test_cluster_pcm_x_convert.c unit_test.h \ + $(CLUSTER_PCM_X_CONVERT_O) + $(CC) $(CFLAGS) $(PTHREAD_CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_PCM_X_CONVERT_O) \ + $(top_builddir)/src/port/libpgport_srv.a $(PTHREAD_LIBS) -o $@ + # GCS-race round-4c tier1-partial-IO: test_cluster_ic_tier1_partial links # cluster_ic_tier1.o standalone and drives the PRODUCTION outbound-tail # queue/drain state machine over a real localhost TCP pair (initial-EAGAIN @@ -1403,7 +1447,10 @@ test_cluster_gcs_dispatch: test_cluster_gcs_dispatch.c unit_test.h \ # link cluster_gcs_block.o. Behavioral coverage (HC82 / HC83 / HC84 / # HC88 / HC89 paths) lives in cluster_tap t/111. test_cluster_gcs_block: test_cluster_gcs_block.c unit_test.h $(CLUSTER_VERSION_O) - $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CC) $(CFLAGS) $(CPPFLAGS) \ + -DGCS_BLOCK_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_gcs_block.c"' \ + -DLMS_SOURCE_PATH='"$(top_srcdir)/src/backend/cluster/cluster_lms.c"' \ + $< \ $(CLUSTER_VERSION_O) \ $(top_builddir)/src/common/libpgcommon_srv.a \ $(top_builddir)/src/port/libpgport_srv.a -o $@ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index e98905ddead..c7f08e9b005 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -34,10 +34,9 @@ * NOTES * This is a pgrac-original file. cluster_debug.c is a cross-module * aggregator -- linking it standalone requires stubs for the public - * symbols from seven other cluster_*.o files. The stubs below are - * the minimum set; the SRF body itself is never invoked from the - * unit test (we only take its address), so stub return values are - * inert. + * symbols from other cluster_*.o files. The stubs below are the + * minimum set. The SRF body is invoked to validate its category/key + * surface, while stub return values remain inert zero-state inputs. * *------------------------------------------------------------------------- */ @@ -48,7 +47,8 @@ #include "cluster/cluster_grd.h" /* ClusterGrdRecoveryCounters */ #include "cluster/cluster_hang.h" /* spec-5.11: ClusterHangDumpData for dump_hang stubs */ #include "cluster/cluster_hang_resolve.h" /* spec-5.12: ClusterHangResolveCounters for dump stubs */ -#include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ +#include "cluster/cluster_pcm_x_convert.h" +#include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ #include "cluster/cluster_touched_peers.h" /* spec-5.14 D6 self_hex stub */ #include "cluster/cluster_xnode_profile.h" /* spec-5.59 D1 profiling gate stubs */ #include "cluster/cluster_xnode_lever.h" /* spec-6.12 lever counter stub */ @@ -70,6 +70,12 @@ #include "unit_test.h" +#define CAPTURED_DUMP_ROWS_MAX 2048 +static const char *captured_dump_categories[CAPTURED_DUMP_ROWS_MAX]; +static const char *captured_dump_keys[CAPTURED_DUMP_ROWS_MAX]; +static int captured_dump_row_count; + + /* ---------- * Stubs needed to link cluster_debug.o standalone. cluster_debug.c * depends on (read-only): cluster_shmem (ClusterShmem global pointer), @@ -908,6 +914,25 @@ cluster_pcm_grd_get_summary(int *n_count, int *s_count, int *x_count, int *pi_ho *convert_queue_active = 0; } +bool +cluster_pcm_x_stats_snapshot(PcmXStatsSnapshot *snapshot_out) +{ + if (snapshot_out == NULL) + return false; + memset(snapshot_out, 0, sizeof(*snapshot_out)); + return false; +} + +PcmXRuntimeSnapshot +cluster_pcm_x_runtime_snapshot(void) +{ + PcmXRuntimeSnapshot snapshot = { 0 }; + + snapshot.state = PCM_X_RUNTIME_ACTIVE; + snapshot.gate_generation = 1; + return snapshot; +} + /* PGRAC spec-2.30 D9 R10 stub audit — 9 transition counter accessors. */ uint64 cluster_pcm_get_trans_n_to_s_count(void) @@ -1215,6 +1240,26 @@ cluster_gcs_get_block_dedup_legacy_pin_count(void) return 0; } uint64 +cluster_gcs_get_block_dedup_pcm_x_stage_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_block_dedup_pcm_x_replay_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_block_dedup_pcm_x_release_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_block_dedup_pcm_x_failclosed_count(void) +{ + return 0; +} +uint64 cluster_gcs_get_block_done_enqueue_drop_count(void) { return 0; @@ -1260,6 +1305,21 @@ cluster_gcs_get_invalidate_busy_received_count(void) { return 0; } +uint64 +cluster_gcs_get_invalidate_passive_s_release_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_pcm_x_self_handoff_count(void) +{ + return 0; +} +uint64 +cluster_gcs_get_pcm_x_self_handoff_drain_count(void) +{ + return 0; +} /* GCS serve-stall round-5 A2 stubs: 4 bounded-drop accessors. */ uint64 cluster_gcs_get_invalidate_parked_count(void) @@ -3051,14 +3111,21 @@ InitMaterializedSRF(FunctionCallInfo fcinfo pg_attribute_unused(), void tuplestore_putvalues(Tuplestorestate *state pg_attribute_unused(), - TupleDesc tdesc pg_attribute_unused(), Datum *values pg_attribute_unused(), + TupleDesc tdesc pg_attribute_unused(), Datum *values, bool *isnull pg_attribute_unused()) -{} +{ + UT_ASSERT(captured_dump_row_count < CAPTURED_DUMP_ROWS_MAX); + if (captured_dump_row_count >= CAPTURED_DUMP_ROWS_MAX) + return; + captured_dump_categories[captured_dump_row_count] = (const char *)DatumGetPointer(values[0]); + captured_dump_keys[captured_dump_row_count] = (const char *)DatumGetPointer(values[1]); + captured_dump_row_count++; +} text * -cstring_to_text(const char *s pg_attribute_unused()) +cstring_to_text(const char *s) { - return NULL; + return (text *)s; } char * @@ -4146,6 +4213,93 @@ UT_TEST(test_debug_dump_srf_linkable) UT_ASSERT_NOT_NULL((void *)cluster_dump_state); } +static int +captured_dump_count(const char *category, const char *key) +{ + int count = 0; + int i; + + for (i = 0; i < captured_dump_row_count; i++) { + if (strcmp(captured_dump_categories[i], category) != 0) + continue; + if (key == NULL || strcmp(captured_dump_keys[i], key) == 0) + count++; + } + return count; +} + +UT_TEST(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets) +{ + static const char *const pcm_keys[] = { + "pcm_x_runtime_state", + "pcm_x_runtime_generation", + "pcm_x_queue_enqueue_count", + "pcm_x_queue_admit_count", + "pcm_x_queue_confirm_count", + "pcm_x_queue_promotion_count", + "pcm_x_queue_transfer_count", + "pcm_x_queue_complete_count", + "pcm_x_queue_cancel_count", + "pcm_x_queue_revoke_count", + "pcm_x_queue_coalesced_count", + "pcm_x_queue_wait_count", + "pcm_x_queue_full_count", + "pcm_x_queue_stale_count", + "pcm_x_queue_miss_count", + "pcm_x_queue_recovery_blocked_count", + "pcm_x_queue_activating_reset_count", + "pcm_x_queue_depth", + "pcm_x_queue_depth_high_water", + "pcm_x_queue_active_tags", + "pcm_x_queue_live_tickets", + "pcm_x_queue_live_slots", + "pcm_x_local_retire_gate", + "pcm_x_local_retire_marker_count", + "pcm_x_local_retire_marker_ticket_id", + "pcm_x_own_begin_count", + "pcm_x_own_commit_count", + "pcm_x_own_abort_count", + "pcm_x_own_busy_count", + "pcm_x_own_corrupt_count", + }; + static const char *const lmd_keys[] = { + "pcm_convert_wfg_replace_count", + "pcm_convert_wfg_remove_count", + "pcm_convert_wfg_replace_fail_count", + "pcm_convert_wfg_exact_remove_stale_count", + }; + static const char *const gcs_keys[] = { + "dedup_pcm_x_stage_count", + "dedup_pcm_x_replay_count", + "dedup_pcm_x_release_count", + "dedup_pcm_x_failclosed_count", + "invalidate_passive_s_release_count", + "pcm_x_self_handoff_count", + "pcm_x_self_handoff_drain_count", + }; + LOCAL_FCINFO(fcinfo, 0); + ReturnSetInfo rsinfo; + int i; + + memset(fcinfo, 0, SizeForFunctionCallInfo(0)); + memset(&rsinfo, 0, sizeof(rsinfo)); + memset(captured_dump_categories, 0, sizeof(captured_dump_categories)); + memset(captured_dump_keys, 0, sizeof(captured_dump_keys)); + captured_dump_row_count = 0; + fcinfo->resultinfo = (fmNodePtr)&rsinfo; + (void)cluster_dump_state(fcinfo); + + UT_ASSERT_EQ(captured_dump_count("pcm", NULL), 58); + UT_ASSERT_EQ(captured_dump_count("lmd", NULL), 51); + UT_ASSERT_EQ(captured_dump_count("gcs", NULL), 119); + for (i = 0; i < (int)lengthof(pcm_keys); i++) + UT_ASSERT_EQ(captured_dump_count("pcm", pcm_keys[i]), 1); + for (i = 0; i < (int)lengthof(lmd_keys); i++) + UT_ASSERT_EQ(captured_dump_count("lmd", lmd_keys[i]), 1); + for (i = 0; i < (int)lengthof(gcs_keys); i++) + UT_ASSERT_EQ(captured_dump_count("gcs", gcs_keys[i]), 1); +} + /* ============================================================ * Iterator API on cluster_inject (added in spec-0.29 §1.4). @@ -4371,10 +4525,10 @@ cluster_lms_state_to_string(int s pg_attribute_unused()) * cluster_lmd_* accessors via cluster_debug.o; standalone test * harness must provide local zero-returning stubs. */ -int +ClusterLmdState cluster_lmd_get_state(void) { - return 0; + return CLUSTER_LMD_NOT_STARTED; } uint64 cluster_lmd_get_started_count(void) @@ -4407,7 +4561,7 @@ cluster_lmd_get_error_count(void) return 0; } const char * -cluster_lmd_state_to_string(int s pg_attribute_unused()) +cluster_lmd_state_to_string(ClusterLmdState s pg_attribute_unused()) { return "(stub)"; } @@ -4449,6 +4603,26 @@ cluster_lmd_revalidate_fail_count_get(void) return 0; } uint64 +cluster_lmd_pcm_convert_wfg_replace_count_get(void) +{ + return 0; +} +uint64 +cluster_lmd_pcm_convert_wfg_remove_count_get(void) +{ + return 0; +} +uint64 +cluster_lmd_pcm_convert_wfg_replace_fail_count_get(void) +{ + return 0; +} +uint64 +cluster_lmd_pcm_convert_wfg_exact_remove_stale_count_get(void) +{ + return 0; +} +uint64 cluster_lmd_cross_node_victim_pending_count_get(void) { return 0; @@ -4697,8 +4871,9 @@ UT_TEST(test_debug_phase_symbol_present) int main(void) { - UT_PLAN(11); + UT_PLAN(12); UT_RUN(test_debug_dump_srf_linkable); + UT_RUN(test_debug_dump_exposes_exact_pcm_x_lmd_and_gcs_key_sets); UT_RUN(test_debug_inject_get_count_callable); UT_RUN(test_debug_inject_get_state_at_out_of_range); UT_RUN(test_debug_inject_get_state_at_null_outs); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 5144f836ba1..6265d5e207e 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -70,6 +70,8 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_lmd_wait_state.h" +#include "cluster/cluster_pcm_x_convert.h" #include "cluster/cluster_thread_recovery.h" #include "common/hashfn.h" #include "storage/buf_internals.h" @@ -93,6 +95,55 @@ UT_DEFINE_GLOBALS(); +static char * +read_source_path(const char *path) +{ + FILE *file; + long length; + char *source; + + file = fopen(path, "rb"); + UT_ASSERT_NOT_NULL(file); + if (file == NULL) + return NULL; + UT_ASSERT_EQ(fseek(file, 0, SEEK_END), 0); + length = ftell(file); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(file, 0, SEEK_SET), 0); + source = malloc((size_t)length + 1); + UT_ASSERT_NOT_NULL(source); + if (source == NULL) { + fclose(file); + return NULL; + } + UT_ASSERT_EQ(fread(source, 1, (size_t)length, file), (size_t)length); + source[length] = '\0'; + fclose(file); + return source; +} + + +static char * +read_gcs_block_source(void) +{ + return read_source_path(GCS_BLOCK_SOURCE_PATH); +} + + +static int +count_occurrences(const char *source, const char *needle) +{ + int count = 0; + size_t needle_length = strlen(needle); + + while ((source = strstr(source, needle)) != NULL) { + count++; + source += needle_length; + } + return count; +} + + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -234,6 +285,38 @@ UT_TEST(test_gcs_block_reply_total_size_is_8240) } +UT_TEST(test_pcm_x_session_auth_sample_classifies_epoch_zero_and_torn_reads) +{ + ClusterGcsPcmXAuthSample sample; + + memset(&sample, 0, sizeof(sample)); + sample.connection_before_valid = true; + sample.connection_after_valid = true; + sample.slot_before_valid = true; + sample.slot_after_valid = true; + sample.fresh_before = true; + sample.fresh_after = true; + sample.session_before = 41; + sample.session_after = 41; + sample.slot_generation_before = 7; + sample.slot_generation_after = 7; + /* Both INITIAL epoch 0 and the registered RDMA connection generation 0 + * are live values, not empty sentinels. */ + UT_ASSERT_EQ(cluster_gcs_pcm_x_auth_sample_classify(&sample, 0), PCM_X_SESSION_AUTH_OK); + + sample.slot_generation_after++; + UT_ASSERT_EQ(cluster_gcs_pcm_x_auth_sample_classify(&sample, 0), PCM_X_SESSION_AUTH_SLOT_TORN); + sample.slot_generation_after = sample.slot_generation_before; + sample.fresh_after = false; + UT_ASSERT_EQ(cluster_gcs_pcm_x_auth_sample_classify(&sample, 0), + PCM_X_SESSION_AUTH_FRESH_NOT_READY); + sample.fresh_after = true; + sample.connection_generation_after = 1; + UT_ASSERT_EQ(cluster_gcs_pcm_x_auth_sample_classify(&sample, 0), + PCM_X_SESSION_AUTH_CONNECTION_TORN); +} + + UT_TEST(test_gcs_block_reply_key_is_compound) { GcsBlockReplyHeader hdr; @@ -643,10 +726,2529 @@ UT_TEST(test_undo_verdict_version_authority_distinct) } +UT_TEST(test_pcm_x_enqueue_ingress_binds_transport_epoch_and_master) +{ + PcmXEnqueuePayload request; + + memset(&request, 0, sizeof(request)); + request.identity.node_id = 1; + request.identity.procno = 7; + request.identity.cluster_epoch = 9; + request.identity.request_id = 11; + request.identity.wait_seq = 13; + request.prehandle.sender_session_incarnation = 17; + request.prehandle.prehandle_sequence = 19; + + UT_ASSERT(cluster_gcs_pcm_x_enqueue_ingress_valid(&request, sizeof(request), 1, 9, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_enqueue_ingress_valid(&request, sizeof(request) - 1, 1, 9, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_enqueue_ingress_valid(&request, sizeof(request), 0, 9, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_enqueue_ingress_valid(&request, sizeof(request), 1, 10, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_enqueue_ingress_valid(&request, sizeof(request), 1, 9, 3, 2)); + request.prehandle.prehandle_sequence = 0; + UT_ASSERT(!cluster_gcs_pcm_x_enqueue_ingress_valid(&request, sizeof(request), 1, 9, 2, 2)); +} + +UT_TEST(test_pcm_x_admit_confirm_ingress_binds_requester_and_master) +{ + PcmXPhasePayload phase; + + memset(&phase, 0, sizeof(phase)); + phase.ref.identity.node_id = 1; + phase.ref.identity.procno = 7; + phase.ref.identity.cluster_epoch = 9; + phase.ref.identity.request_id = 11; + phase.ref.identity.wait_seq = 13; + phase.ref.handle.ticket_id = 17; + phase.ref.handle.queue_generation = 19; + phase.phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + + UT_ASSERT(cluster_gcs_pcm_x_admit_confirm_ingress_valid(&phase, sizeof(phase), 1, 9, 2, 2)); + UT_ASSERT( + !cluster_gcs_pcm_x_admit_confirm_ingress_valid(&phase, sizeof(phase) - 1, 1, 9, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_admit_confirm_ingress_valid(&phase, sizeof(phase), 0, 9, 2, 2)); + phase.reason = 1; + UT_ASSERT(!cluster_gcs_pcm_x_admit_confirm_ingress_valid(&phase, sizeof(phase), 1, 9, 2, 2)); +} + +UT_TEST(test_pcm_x_admit_confirm_ack_binds_exact_master_source) +{ + PcmXPhasePayload phase; + + memset(&phase, 0, sizeof(phase)); + phase.ref.identity.node_id = 1; + phase.ref.identity.procno = 7; + phase.ref.identity.cluster_epoch = 9; + phase.ref.identity.request_id = 11; + phase.ref.identity.wait_seq = 13; + phase.ref.handle.ticket_id = 17; + phase.ref.handle.queue_generation = 19; + phase.phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + + UT_ASSERT(cluster_gcs_pcm_x_admit_confirm_ack_ingress_valid(&phase, sizeof(phase), 2, 9, 2, 1)); + UT_ASSERT( + !cluster_gcs_pcm_x_admit_confirm_ack_ingress_valid(&phase, sizeof(phase), 3, 9, 2, 1)); + phase.ref.grant_generation = 1; + UT_ASSERT( + !cluster_gcs_pcm_x_admit_confirm_ack_ingress_valid(&phase, sizeof(phase), 2, 9, 2, 1)); +} + +UT_TEST(test_pcm_x_cancel_requests_bind_exact_source_epoch_master_and_phase) +{ + PcmXPrehandleCancelPayload prehandle; + PcmXPhasePayload cancel; + + memset(&prehandle, 0, sizeof(prehandle)); + prehandle.identity.node_id = 1; + prehandle.identity.procno = 7; + prehandle.identity.cluster_epoch = 9; + prehandle.identity.request_id = 11; + prehandle.identity.wait_seq = 13; + prehandle.prehandle.sender_session_incarnation = 17; + prehandle.prehandle.prehandle_sequence = 19; + UT_ASSERT(cluster_gcs_pcm_x_prehandle_cancel_ingress_valid(&prehandle, sizeof(prehandle), 1, 9, + 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_prehandle_cancel_ingress_valid(&prehandle, sizeof(prehandle), 0, 9, + 2, 2)); + prehandle.prehandle.prehandle_sequence = 0; + UT_ASSERT(!cluster_gcs_pcm_x_prehandle_cancel_ingress_valid(&prehandle, sizeof(prehandle), 1, 9, + 2, 2)); + + memset(&cancel, 0, sizeof(cancel)); + cancel.ref.identity = prehandle.identity; + cancel.ref.identity.node_id = 1; + cancel.ref.handle.ticket_id = 23; + cancel.ref.handle.queue_generation = 29; + cancel.phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + UT_ASSERT(cluster_gcs_pcm_x_cancel_ingress_valid(&cancel, sizeof(cancel), 1, 9, 2, 2)); + cancel.phase = PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM; + UT_ASSERT(!cluster_gcs_pcm_x_cancel_ingress_valid(&cancel, sizeof(cancel), 1, 9, 2, 2)); +} + +UT_TEST(test_pcm_x_cancel_acks_bind_exact_master_and_canonical_payload) +{ + PcmXAdmitAckPayload prehandle_ack; + PcmXPhasePayload cancel_ack; + + memset(&prehandle_ack, 0, sizeof(prehandle_ack)); + prehandle_ack.ref.identity.node_id = 1; + prehandle_ack.ref.identity.procno = 7; + prehandle_ack.ref.identity.cluster_epoch = 9; + prehandle_ack.ref.identity.request_id = 11; + prehandle_ack.ref.identity.wait_seq = 13; + prehandle_ack.ref.handle.ticket_id = 17; + prehandle_ack.ref.handle.queue_generation = 19; + prehandle_ack.prehandle.sender_session_incarnation = 23; + prehandle_ack.prehandle.prehandle_sequence = 29; + prehandle_ack.result = PCM_X_QUEUE_OK; + prehandle_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL; + UT_ASSERT(cluster_gcs_pcm_x_prehandle_cancel_ack_ingress_valid( + &prehandle_ack, sizeof(prehandle_ack), 2, 9, 2, 1)); + prehandle_ack.result = PCM_X_QUEUE_DUPLICATE; + UT_ASSERT(!cluster_gcs_pcm_x_prehandle_cancel_ack_ingress_valid( + &prehandle_ack, sizeof(prehandle_ack), 2, 9, 2, 1)); + + memset(&cancel_ack, 0, sizeof(cancel_ack)); + cancel_ack.ref = prehandle_ack.ref; + cancel_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_CANCEL; + UT_ASSERT( + cluster_gcs_pcm_x_cancel_ack_ingress_valid(&cancel_ack, sizeof(cancel_ack), 2, 9, 2, 1)); + cancel_ack.ref.grant_generation = 1; + UT_ASSERT( + !cluster_gcs_pcm_x_cancel_ack_ingress_valid(&cancel_ack, sizeof(cancel_ack), 2, 9, 2, 1)); +} + +UT_TEST(test_pcm_x_wait_identity_maps_to_real_wfg_vertex) +{ + PcmXWaitIdentity identity; + ClusterLmdVertex vertex; + + memset(&identity, 0, sizeof(identity)); + identity.node_id = 2; + identity.procno = 7; + identity.xid = 11; + identity.cluster_epoch = 13; + identity.request_id = 17; + identity.wait_seq = 19; + memset(&vertex, 0xA5, sizeof(vertex)); + + cluster_gcs_pcm_x_vertex_from_identity(&identity, &vertex); + UT_ASSERT_EQ(vertex.node_id, 2); + UT_ASSERT_EQ(vertex.procno, (uint32)7); + UT_ASSERT_EQ(vertex.xid, (TransactionId)11); + UT_ASSERT_EQ(vertex.cluster_epoch, (uint64)13); + UT_ASSERT_EQ(vertex.request_id, (uint64)17); + UT_ASSERT_EQ(vertex.wait_seq, (uint64)19); + UT_ASSERT_EQ(vertex.local_start_ts_ms, (int64)0); +} + + +UT_TEST(test_pcm_x_initial_epoch_zero_is_exact_across_wire_classes) +{ + ClusterLmdWaitStateSnapshot wait_snapshot; + PcmXEnqueuePayload enqueue; + PcmXRetirePayload retire; + PcmXTicketRef ref; + + memset(&enqueue, 0, sizeof(enqueue)); + enqueue.identity.node_id = 1; + enqueue.identity.procno = 7; + enqueue.identity.cluster_epoch = 0; + enqueue.identity.request_id = 11; + enqueue.identity.wait_seq = 13; + enqueue.prehandle.sender_session_incarnation = 17; + enqueue.prehandle.prehandle_sequence = 19; + UT_ASSERT(cluster_gcs_pcm_x_enqueue_ingress_valid(&enqueue, sizeof(enqueue), 1, 0, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_enqueue_ingress_valid(&enqueue, sizeof(enqueue), 1, 1, 2, 2)); + + memset(&ref, 0, sizeof(ref)); + ref.identity = enqueue.identity; + ref.handle.ticket_id = 23; + ref.handle.queue_generation = 29; + UT_ASSERT(cluster_gcs_pcm_x_ticket_ref_wire_valid(&ref, 0)); + UT_ASSERT(!cluster_gcs_pcm_x_ticket_ref_wire_valid(&ref, 1)); + ref.grant_generation = 31; + UT_ASSERT(cluster_gcs_pcm_x_transfer_ref_wire_valid(&ref, 0)); + UT_ASSERT(cluster_gcs_pcm_x_terminal_ref_wire_valid(&ref, 0)); + UT_ASSERT(!cluster_gcs_pcm_x_transfer_ref_wire_valid(&ref, 1)); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = 0; + retire.master_session_incarnation = 37; + retire.retire_through_ticket_id = 41; + retire.sender_node = 3; + UT_ASSERT(cluster_gcs_pcm_x_retire_request_ingress_valid(&retire, sizeof(retire), 2, 37, 0, 3)); + UT_ASSERT( + !cluster_gcs_pcm_x_retire_request_ingress_valid(&retire, sizeof(retire), 2, 37, 1, 3)); + UT_ASSERT(cluster_gcs_pcm_x_retire_ack_ingress_valid(&retire, sizeof(retire), 3, 0, 37, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_retire_ack_ingress_valid(&retire, sizeof(retire), 3, 1, 37, 2)); + + memset(&wait_snapshot, 0, sizeof(wait_snapshot)); + wait_snapshot.active = true; + wait_snapshot.kind = CLUSTER_LMD_WAIT_PCM_CONVERT; + wait_snapshot.request_id = enqueue.identity.request_id; + wait_snapshot.cluster_epoch = 0; + wait_snapshot.wait_seq = enqueue.identity.wait_seq; + UT_ASSERT(cluster_gcs_pcm_x_wait_identity_matches( + &enqueue.identity, 1, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &wait_snapshot)); + UT_ASSERT(cluster_gcs_block_epoch_advance_stales_slot(true, 0, 1)); + UT_ASSERT(!cluster_gcs_block_epoch_advance_stales_slot(false, 0, 1)); + UT_ASSERT(!cluster_gcs_block_epoch_advance_stales_slot(true, 1, 1)); +} + +UT_TEST(test_pcm_x_blocker_header_ingress_binds_master_not_requester_source) +{ + PcmXBlockerSetHeaderPayload header; + + memset(&header, 0, sizeof(header)); + header.ref.identity.node_id = 1; + header.ref.identity.procno = 7; + header.ref.identity.cluster_epoch = 9; + header.ref.identity.request_id = 11; + header.ref.identity.wait_seq = 13; + header.ref.handle.ticket_id = 17; + header.ref.handle.queue_generation = 19; + header.set_generation = UINT64_C(0x100000001); + header.nblockers = 2; + header.set_crc32c = UINT32_C(0x12345678); + + /* Holder node 3 reports blockers for requester node 1 to tag master 2. */ + UT_ASSERT(cluster_gcs_pcm_x_blocker_header_ingress_valid(&header, sizeof(header), 3, 9, 2, 2)); + UT_ASSERT( + !cluster_gcs_pcm_x_blocker_header_ingress_valid(&header, sizeof(header) - 1, 3, 9, 2, 2)); + UT_ASSERT( + !cluster_gcs_pcm_x_blocker_header_ingress_valid(&header, sizeof(header), 3, 10, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_header_ingress_valid(&header, sizeof(header), 3, 9, 1, 2)); + header.set_generation = 0; + UT_ASSERT(!cluster_gcs_pcm_x_blocker_header_ingress_valid(&header, sizeof(header), 3, 9, 2, 2)); +} + +UT_TEST(test_pcm_x_blocker_edge_ingress_binds_blocker_to_holder_source) +{ + PcmXBlockerChunkPayload edge; + + memset(&edge, 0, sizeof(edge)); + edge.requester_node = 1; + edge.requester_procno = 7; + edge.cluster_epoch = 9; + edge.request_id = 11; + edge.handle.ticket_id = 17; + edge.handle.queue_generation = 19; + edge.set_generation = UINT64_C(0x100000001); + edge.blocker.node_id = 3; + edge.blocker.procno = 23; + edge.blocker.cluster_epoch = 9; + edge.blocker.request_id = 29; + edge.blocker.wait_seq = 31; + + UT_ASSERT(cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); + edge.blocker.node_id = 4; + UT_ASSERT(!cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); + edge.blocker.node_id = 3; + edge.blocker.cluster_epoch = 10; + UT_ASSERT(!cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); + edge.blocker.cluster_epoch = 9; + edge.grant_generation = 1; + UT_ASSERT(!cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); + edge.grant_generation = 0; + edge.blocker.request_id = 0; + edge.blocker.xid = (TransactionId)37; + UT_ASSERT(cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); + edge.blocker.xid = InvalidTransactionId; + UT_ASSERT(!cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); + edge.blocker.request_id = 29; + edge.blocker.wait_seq = 0; + UT_ASSERT(!cluster_gcs_pcm_x_blocker_edge_ingress_valid(&edge, sizeof(edge), 3, 9, 2, 2)); +} + +UT_TEST(test_pcm_x_blocker_ack_carries_full_generation_and_binds_master_source) +{ + PcmXPhasePayload ack; + PcmXPhasePayload probe; + PcmXTicketRef ref; + const uint64 generation = UINT64_C(0xFEDCBA9876543210); + + memset(&ack, 0, sizeof(ack)); + ack.ref.identity.node_id = 1; + ack.ref.identity.procno = 7; + ack.ref.identity.cluster_epoch = 9; + ack.ref.identity.request_id = 11; + ack.ref.identity.wait_seq = 13; + ack.ref.handle.ticket_id = 17; + ack.ref.handle.queue_generation = 19; + cluster_gcs_pcm_x_blocker_ack_set_generation(&ack, generation); + + UT_ASSERT_EQ(cluster_gcs_pcm_x_blocker_ack_generation(&ack), generation); + UT_ASSERT(cluster_gcs_pcm_x_blocker_ack_ingress_valid(&ack, sizeof(ack), 2, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_ack_ingress_valid(&ack, sizeof(ack), 4, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_ack_ingress_valid(&ack, sizeof(ack), 2, 10, 2, 3)); + cluster_gcs_pcm_x_blocker_ack_set_generation(&ack, 0); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_ack_ingress_valid(&ack, sizeof(ack), 2, 9, 2, 3)); + + /* Type 48 uses the same authenticated master->holder direction for both + * arms. An all-zero generation is the PROBE request; it is never a + * generation-exact ACK. */ + probe = ack; + UT_ASSERT_EQ(cluster_gcs_pcm_x_blocker_ack_generation(&probe), 0); + UT_ASSERT(cluster_gcs_pcm_x_blocker_probe_ingress_valid(&probe, sizeof(probe), 2, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_probe_ingress_valid(&probe, sizeof(probe), 4, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_probe_ingress_valid(&probe, sizeof(probe), 2, 10, 2, 3)); + cluster_gcs_pcm_x_blocker_ack_set_generation(&probe, 1); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_probe_ingress_valid(&probe, sizeof(probe), 2, 9, 2, 3)); + UT_ASSERT(cluster_gcs_pcm_x_blocker_ack_ingress_valid(&probe, sizeof(probe), 2, 9, 2, 3)); + + /* The ACK producer has no representation for the reserved PROBE value. */ + ref = ack.ref; + memset(&ack, 0xA5, sizeof(ack)); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_ack_build(&ref, 0, &ack)); + UT_ASSERT_EQ(cluster_gcs_pcm_x_blocker_ack_generation(&ack), 0); + UT_ASSERT(!cluster_gcs_pcm_x_blocker_ack_build(&ref, UINT64_MAX, &ack)); + UT_ASSERT_EQ(cluster_gcs_pcm_x_blocker_ack_generation(&ack), 0); + UT_ASSERT(cluster_gcs_pcm_x_blocker_ack_build(&ref, generation, &ack)); + UT_ASSERT(memcmp(&ack.ref, &ref, sizeof(ref)) == 0); + UT_ASSERT_EQ(cluster_gcs_pcm_x_blocker_ack_generation(&ack), generation); +} + +UT_TEST(test_pcm_x_drain_poll_binds_exact_master_and_generation) +{ + PcmXDrainPollPayload poll; + + memset(&poll, 0, sizeof(poll)); + poll.ref.identity.node_id = 1; + poll.ref.identity.procno = 7; + poll.ref.identity.cluster_epoch = 9; + poll.ref.identity.request_id = 11; + poll.ref.identity.wait_seq = 13; + poll.ref.handle.ticket_id = 17; + poll.ref.handle.queue_generation = 19; + poll.ref.grant_generation = 23; + poll.drain_generation = 29; + + /* The tag master is node 2; node 3 is one terminal participant. */ + UT_ASSERT(cluster_gcs_pcm_x_drain_poll_ingress_valid(&poll, sizeof(poll), 2, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_drain_poll_ingress_valid(&poll, sizeof(poll) - 1, 2, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_drain_poll_ingress_valid(&poll, sizeof(poll), 4, 9, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_drain_poll_ingress_valid(&poll, sizeof(poll), 2, 10, 2, 3)); + poll.drain_generation = 0; + UT_ASSERT(!cluster_gcs_pcm_x_drain_poll_ingress_valid(&poll, sizeof(poll), 2, 9, 2, 3)); + poll.drain_generation = 29; + poll.ref.grant_generation = UINT64_MAX; + UT_ASSERT(!cluster_gcs_pcm_x_drain_poll_ingress_valid(&poll, sizeof(poll), 2, 9, 2, 3)); +} + +UT_TEST(test_pcm_x_drain_ack_binds_participant_and_canonical_payload) +{ + PcmXPhasePayload ack; + + memset(&ack, 0, sizeof(ack)); + ack.ref.identity.node_id = 1; + ack.ref.identity.procno = 7; + ack.ref.identity.cluster_epoch = 9; + ack.ref.identity.request_id = 11; + ack.ref.identity.wait_seq = 13; + ack.ref.handle.ticket_id = 17; + ack.ref.handle.queue_generation = 19; + ack.ref.grant_generation = 23; + + /* Participant node 3 ACKs to the local tag master node 2. */ + UT_ASSERT(cluster_gcs_pcm_x_drain_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_drain_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 4, 2)); + ack.reason = 1; + UT_ASSERT(!cluster_gcs_pcm_x_drain_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 2, 2)); + ack.reason = 0; + ack.flags = 1; + UT_ASSERT(!cluster_gcs_pcm_x_drain_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 2, 2)); +} + +UT_TEST(test_pcm_x_retire_request_binds_master_session_and_target) +{ + PcmXRetirePayload retire; + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = 9; + retire.master_session_incarnation = 17; + retire.retire_through_ticket_id = 23; + retire.sender_node = 3; + + UT_ASSERT(cluster_gcs_pcm_x_retire_request_ingress_valid(&retire, sizeof(retire), 2, 17, 9, 3)); + UT_ASSERT( + !cluster_gcs_pcm_x_retire_request_ingress_valid(&retire, sizeof(retire), 2, 19, 9, 3)); + UT_ASSERT( + !cluster_gcs_pcm_x_retire_request_ingress_valid(&retire, sizeof(retire), 2, 17, 10, 3)); + retire.sender_node = 4; + UT_ASSERT( + !cluster_gcs_pcm_x_retire_request_ingress_valid(&retire, sizeof(retire), 2, 17, 9, 3)); +} + +UT_TEST(test_pcm_x_retire_ack_binds_responder_and_master_authority) +{ + PcmXRetirePayload ack; + + memset(&ack, 0, sizeof(ack)); + ack.cluster_epoch = 9; + ack.master_session_incarnation = 17; + ack.retire_through_ticket_id = 23; + ack.sender_node = 3; + + UT_ASSERT(cluster_gcs_pcm_x_retire_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 17, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_retire_ack_ingress_valid(&ack, sizeof(ack), 4, 9, 17, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_retire_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 19, 2)); + ack.flags = 1; + UT_ASSERT(!cluster_gcs_pcm_x_retire_ack_ingress_valid(&ack, sizeof(ack), 3, 9, 17, 2)); +} + + +UT_TEST(test_pcm_x_formation_identical_complete_samples_may_revalidate) +{ + PcmXPeerBinding after[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXPeerBinding before[PCM_X_PROTOCOL_NODE_LIMIT]; + + memset(before, 0, sizeof(before)); + before[0].cluster_epoch = 9; + before[0].peer_session_incarnation = 101; + before[1].cluster_epoch = 9; + before[1].peer_session_incarnation = 102; + memcpy(after, before, sizeof(after)); + UT_ASSERT(cluster_gcs_pcm_x_formation_samples_stable(true, before, true, after)); +} + + +UT_TEST(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop) +{ + PcmXPeerBinding after[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXPeerBinding before[PCM_X_PROTOCOL_NODE_LIMIT]; + + memset(before, 0, sizeof(before)); + before[0].cluster_epoch = 9; + before[0].peer_session_incarnation = 101; + memcpy(after, before, sizeof(after)); + UT_ASSERT(!cluster_gcs_pcm_x_formation_samples_stable(false, before, true, after)); + UT_ASSERT(!cluster_gcs_pcm_x_formation_samples_stable(true, before, false, after)); + after[0].peer_session_incarnation++; + UT_ASSERT(!cluster_gcs_pcm_x_formation_samples_stable(true, before, true, after)); + memcpy(after, before, sizeof(after)); + after[1].cluster_epoch = 10; + after[1].peer_session_incarnation = 102; + UT_ASSERT(!cluster_gcs_pcm_x_formation_samples_stable(true, before, true, after)); +} + + +UT_TEST(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close) +{ + UT_ASSERT(cluster_gcs_pcm_x_confirm_compensation_required(UINT64_C(7001), PCM_X_QUEUE_STALE)); + UT_ASSERT( + cluster_gcs_pcm_x_confirm_compensation_required(UINT64_C(7001), PCM_X_QUEUE_NOT_READY)); + UT_ASSERT(!cluster_gcs_pcm_x_confirm_compensation_required(0, PCM_X_QUEUE_STALE)); + UT_ASSERT(!cluster_gcs_pcm_x_confirm_compensation_required(UINT64_C(7001), PCM_X_QUEUE_OK)); + UT_ASSERT( + !cluster_gcs_pcm_x_confirm_compensation_required(UINT64_C(7001), PCM_X_QUEUE_DUPLICATE)); +} + + +static void +pcm_x_test_init_transfer_ref(PcmXTicketRef *ref) +{ + memset(ref, 0, sizeof(*ref)); + ref->identity.node_id = 1; + ref->identity.procno = 7; + ref->identity.xid = 9; + ref->identity.cluster_epoch = 11; + ref->identity.request_id = 13; + ref->identity.wait_seq = 17; + ref->identity.base_own_generation = 19; + ref->handle.ticket_id = 23; + ref->handle.queue_generation = 29; + ref->grant_generation = 31; +} + + +UT_TEST(test_pcm_x_revoke_ingress_binds_master_and_exact_transfer_key) +{ + PcmXRevokePayload revoke; + uint64 image_id; + + memset(&revoke, 0, sizeof(revoke)); + pcm_x_test_init_transfer_ref(&revoke.ref); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &image_id)); + revoke.image_id = image_id; + + /* Master node 2 revokes current holder node 3 for requester node 1. */ + UT_ASSERT(cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 2, 11, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke) - 1, 2, 11, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 3, 11, 2, 3)); + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 2, 12, 2, 3)); + revoke.ref.grant_generation = 0; + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 2, 11, 2, 3)); + revoke.ref.grant_generation = 31; + revoke.image_id = 0; + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 2, 11, 2, 3)); + revoke.image_id = UINT64CONST(0xe000000000000025); + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 2, 11, 2, 3)); + UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &revoke.image_id)); + UT_ASSERT(!cluster_gcs_pcm_x_revoke_ingress_valid(&revoke, sizeof(revoke), 2, 11, 2, 3)); +} + + +UT_TEST(test_pcm_x_image_ready_ingress_binds_holder_image_to_master) +{ + PcmXGrantPayload ready; + uint64 image_id; + + memset(&ready, 0, sizeof(ready)); + pcm_x_test_init_transfer_ref(&ready.ref); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &image_id)); + ready.image.image_id = image_id; + ready.image.source_node = 3; + ready.image.page_scn = 41; + ready.image.page_lsn = 43; + ready.image.page_checksum = UINT32_C(0x12345678); + + /* source_own_generation=0 is the legal first ownership generation. */ + UT_ASSERT(cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready) - 1, 3, 11, 2, 2)); + ready.image.source_node = 4; + UT_ASSERT(!cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + ready.image.source_node = 3; + UT_ASSERT(!cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 1, 2)); + ready.image.image_id = 0; + UT_ASSERT(!cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + ready.image.image_id = UINT64CONST(0xf000000000000025); + UT_ASSERT(!cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ready.image.image_id)); + UT_ASSERT(!cluster_gcs_pcm_x_image_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); +} + + +UT_TEST(test_pcm_x_prepare_grant_ingress_binds_master_to_requester) +{ + PcmXGrantPayload grant; + uint64 image_id; + + memset(&grant, 0, sizeof(grant)); + pcm_x_test_init_transfer_ref(&grant.ref); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &image_id)); + grant.image.image_id = image_id; + grant.image.source_node = 3; + + UT_ASSERT(cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 2, 11, 2, 1)); + UT_ASSERT(!cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 2, 11, 2, 4)); + UT_ASSERT(!cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 3, 11, 2, 1)); + grant.image.source_node = PCM_X_PROTOCOL_NODE_LIMIT; + UT_ASSERT(!cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 2, 11, 2, 1)); + grant.image.source_node = 3; + grant.ref.identity.request_id = 0; + UT_ASSERT(!cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 2, 11, 2, 1)); + grant.ref.identity.request_id = 13; + grant.image.image_id = UINT64CONST(0xe000000000000025); + UT_ASSERT(!cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 2, 11, 2, 1)); + UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &grant.image.image_id)); + UT_ASSERT(!cluster_gcs_pcm_x_prepare_grant_ingress_valid(&grant, sizeof(grant), 2, 11, 2, 1)); +} + + +UT_TEST(test_pcm_x_install_ready_ingress_is_canonical_requester_ack) +{ + PcmXInstallReadyPayload ready; + uint64 image_id; + + memset(&ready, 0, sizeof(ready)); + pcm_x_test_init_transfer_ref(&ready.ref); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &image_id)); + ready.image_id = image_id; + ready.result = PCM_X_QUEUE_OK; + ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + + UT_ASSERT(cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 3, 11, 2, 2)); + ready.result = PCM_X_QUEUE_DUPLICATE; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + ready.result = PCM_X_QUEUE_OK; + ready.phase = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + ready.flags = 1; + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + ready.flags = 0; + ready.image_id = UINT64CONST(0xf000000000000025); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); + UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ready.image_id)); + UT_ASSERT(!cluster_gcs_pcm_x_install_ready_ingress_valid(&ready, sizeof(ready), 1, 11, 2, 2)); +} + + +UT_TEST(test_pcm_x_commit_x_ingress_is_canonical_master_phase) +{ + PcmXPhasePayload commit; + + memset(&commit, 0, sizeof(commit)); + pcm_x_test_init_transfer_ref(&commit.ref); + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + + UT_ASSERT(cluster_gcs_pcm_x_commit_x_ingress_valid(&commit, sizeof(commit), 2, 11, 2, 1)); + UT_ASSERT(!cluster_gcs_pcm_x_commit_x_ingress_valid(&commit, sizeof(commit), 3, 11, 2, 1)); + UT_ASSERT(!cluster_gcs_pcm_x_commit_x_ingress_valid(&commit, sizeof(commit), 2, 11, 2, 4)); + commit.reason = 1; + UT_ASSERT(!cluster_gcs_pcm_x_commit_x_ingress_valid(&commit, sizeof(commit), 2, 11, 2, 1)); + commit.reason = 0; + commit.flags = 1; + UT_ASSERT(!cluster_gcs_pcm_x_commit_x_ingress_valid(&commit, sizeof(commit), 2, 11, 2, 1)); +} + + +UT_TEST(test_pcm_x_final_ack_ingress_binds_exact_committed_generation) +{ + PcmXFinalAckPayload ack; + uint64 image_id; + + memset(&ack, 0, sizeof(ack)); + pcm_x_test_init_transfer_ref(&ack.ref); + ack.ref.identity.base_own_generation = 0; + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 37, &image_id)); + ack.image_id = image_id; + ack.committed_own_generation = 1; + + UT_ASSERT(cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 3, 11, 2, 2)); + ack.committed_own_generation = 2; + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.ref.identity.base_own_generation = UINT64_MAX; + ack.committed_own_generation = 0; + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.ref.identity.base_own_generation = 0; + ack.committed_own_generation = 1; + ack.image_id = 0; + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + ack.image_id = UINT64CONST(0xe000000000000025); + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); + UT_ASSERT(cluster_pcm_x_image_id_encode(3, 37, &ack.image_id)); + UT_ASSERT(!cluster_gcs_pcm_x_final_ack_ingress_valid(&ack, sizeof(ack), 1, 11, 2, 2)); +} + + +UT_TEST(test_pcm_x_final_commit_ack_ingress_is_canonical_master_phase) +{ + PcmXPhasePayload ack; + + memset(&ack, 0, sizeof(ack)); + pcm_x_test_init_transfer_ref(&ack.ref); + ack.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + + UT_ASSERT(cluster_gcs_pcm_x_final_commit_ack_ingress_valid(&ack, sizeof(ack), 2, 11, 2, 1)); + UT_ASSERT(!cluster_gcs_pcm_x_final_commit_ack_ingress_valid(&ack, sizeof(ack), 3, 11, 2, 1)); + ack.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + UT_ASSERT(!cluster_gcs_pcm_x_final_commit_ack_ingress_valid(&ack, sizeof(ack), 2, 11, 2, 1)); + ack.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + ack.flags = 1; + UT_ASSERT(!cluster_gcs_pcm_x_final_commit_ack_ingress_valid(&ack, sizeof(ack), 2, 11, 2, 1)); +} + + +UT_TEST(test_pcm_x_final_confirm_ingress_is_canonical_requester_phase) +{ + PcmXPhasePayload confirm; + + memset(&confirm, 0, sizeof(confirm)); + pcm_x_test_init_transfer_ref(&confirm.ref); + confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + + UT_ASSERT( + cluster_gcs_pcm_x_final_confirm_ingress_valid(&confirm, sizeof(confirm), 1, 11, 2, 2)); + UT_ASSERT( + !cluster_gcs_pcm_x_final_confirm_ingress_valid(&confirm, sizeof(confirm), 3, 11, 2, 2)); + UT_ASSERT( + !cluster_gcs_pcm_x_final_confirm_ingress_valid(&confirm, sizeof(confirm), 1, 11, 3, 2)); + confirm.reason = 1; + UT_ASSERT( + !cluster_gcs_pcm_x_final_confirm_ingress_valid(&confirm, sizeof(confirm), 1, 11, 2, 2)); + confirm.reason = 0; + confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT( + !cluster_gcs_pcm_x_final_confirm_ingress_valid(&confirm, sizeof(confirm), 1, 11, 2, 2)); +} + +UT_TEST(test_pcm_x_master_drive_selects_exact_authority_and_next_holder) +{ + PcmAuthoritySnapshot authority; + const uint64 ticket_id = 73; + uint32 holders; + int32 holder; + int32 source; + + memset(&authority, 0, sizeof(authority)); + authority.state = PCM_STATE_X; + authority.x_holder_node = 2; + authority.master_holder.node_id = 2; + authority.pending_x_requester_node = 3; + authority.pending_x_since_lsn = UINT64_C(0x8000000000000000) | ticket_id; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 3, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holders, UINT32_C(1) << 2); + UT_ASSERT_EQ(source, 2); + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 3, ticket_id + 1, &holders, &source), + PCM_X_QUEUE_STALE); + authority.pending_x_requester_node = 2; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 2, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holders, UINT32_C(1) << 2); + UT_ASSERT_EQ(source, 2); + + authority.state = PCM_STATE_S; + authority.x_holder_node = -1; + authority.s_holders_bitmap = (UINT32_C(1) << 1) | (UINT32_C(1) << 3); + authority.master_holder.node_id = 1; + authority.pending_x_requester_node = 3; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 3, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holders, authority.s_holders_bitmap); + /* A requester S mirror must be the source even when another holder is + * available. Invalidating requester S first bumps its local ownership + * generation; the subsequent X commit would then be a second bump while + * FINAL_ACK is generation-exact at identity.base+1. */ + UT_ASSERT_EQ(source, 3); + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_STALE); + authority.pending_x_requester_node = 0; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(source, 1); + /* A canonical S master-holder that is also the requester retains that + * source role; the exact self handoff keeps the lifecycle finite. */ + authority.s_holders_bitmap = (UINT32_C(1) << 0) | (UINT32_C(1) << 1); + authority.master_holder.node_id = 0; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(source, 0); + /* A sole requester S copy reuses its exact REVOKING lifecycle as the + * requester grant reservation; selecting self is therefore finite and + * preserves the single ownership-generation bump. */ + authority.s_holders_bitmap = UINT32_C(1) << 0; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(source, 0); + authority.s_holders_bitmap = (UINT32_C(1) << 1) | (UINT32_C(1) << 3); + authority.master_holder.node_id = 2; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_CORRUPT); + + memset(&authority, 0, sizeof(authority)); + authority.state = PCM_STATE_N; + authority.x_holder_node = -1; + authority.master_holder.node_id = UINT32_MAX; + authority.pending_x_requester_node = 0; + authority.pending_x_since_lsn = UINT64_C(0x8000000000000000) | ticket_id; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holders, UINT32_C(1) << 0); + UT_ASSERT_EQ(source, 0); + authority.master_holder.node_id = 1; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_CORRUPT); + authority.master_holder.node_id = PCM_X_PROTOCOL_NODE_LIMIT; + UT_ASSERT_EQ( + cluster_gcs_pcm_x_authority_holders_exact(&authority, 0, ticket_id, &holders, &source), + PCM_X_QUEUE_CORRUPT); + + UT_ASSERT_EQ(cluster_gcs_pcm_x_next_unacked_holder((UINT32_C(1) << 1) | (UINT32_C(1) << 3), + UINT32_C(1) << 1, &holder), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder, 3); + UT_ASSERT_EQ(cluster_gcs_pcm_x_next_unacked_holder(UINT32_C(1) << 1, UINT32_C(1) << 1, &holder), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(holder, -1); + UT_ASSERT_EQ(cluster_gcs_pcm_x_next_unacked_holder(UINT32_C(1) << 1, UINT32_C(1) << 2, &holder), + PCM_X_QUEUE_CORRUPT); +} + +UT_TEST(test_pcm_x_master_drive_wiring_binds_grd_barrier_to_exact_ticket) +{ + char *source = read_gcs_block_source(); + char *claim; + char *cancel; + char *end; + char *finalize; + char *fail_closed; + char *graph; + char *handler; + char *note_stale; + char *prepare; + char *stale; + char *state; + char *reserve; + char *publish; + char *post_verify_state; + char *normalize; + char *revalidate; + char *clear; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + claim = strstr(source, "\ngcs_block_pcm_x_ensure_pending_x_claim("); + end = claim != NULL ? strstr(claim, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(claim); + UT_ASSERT_NOT_NULL(end); + if (claim != NULL && end != NULL) { + state = strstr(claim, "cluster_pcm_x_master_pending_x_claim_state_exact("); + reserve = strstr(claim, "cluster_pcm_lock_try_reserve_pending_x("); + publish = strstr(claim, "cluster_pcm_x_master_pending_x_claim_exact("); + revalidate + = publish != NULL ? strstr(publish, "cluster_pcm_lock_queue_pending_x_exact(") : NULL; + post_verify_state + = revalidate != NULL + ? strstr(revalidate, "cluster_pcm_x_master_pending_x_claim_state_exact(") + : NULL; + normalize + = post_verify_state != NULL ? strstr(post_verify_state, "PCM_X_QUEUE_NOT_READY") : NULL; + UT_ASSERT_NOT_NULL(state); + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(publish); + UT_ASSERT_NOT_NULL(revalidate); + UT_ASSERT_NOT_NULL(post_verify_state); + UT_ASSERT_NOT_NULL(normalize); + if (state != NULL && reserve != NULL && publish != NULL && revalidate != NULL + && post_verify_state != NULL && normalize != NULL) + UT_ASSERT(state < reserve && reserve < publish && publish < revalidate + && revalidate < post_verify_state && post_verify_state < normalize + && normalize < end); + clear = strstr(claim, "cluster_pcm_lock_clear_pending_x_if("); + UT_ASSERT(clear == NULL || clear > end); + } + cancel = strstr(source, "\ngcs_block_pcm_x_cancel_claimed_probe_exact("); + if (cancel != NULL) + cancel = strstr(cancel + 1, "\ngcs_block_pcm_x_cancel_claimed_probe_exact("); + end = cancel != NULL ? strstr(cancel, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(cancel); + UT_ASSERT_NOT_NULL(end); + if (cancel != NULL && end != NULL) { + graph = strstr(cancel, "cluster_lmd_graph_remove_edge_by_waiter_exact_result("); + stale = graph != NULL ? strstr(graph, "CLUSTER_LMD_GRAPH_REMOVE_STALE") : NULL; + note_stale = stale != NULL + ? strstr(stale, "cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(") + : NULL; + fail_closed + = note_stale != NULL ? strstr(note_stale, "cluster_pcm_x_runtime_fail_closed(") : NULL; + clear = strstr(cancel, "cluster_pcm_lock_clear_queue_pending_x_exact("); + finalize = strstr(cancel, "cluster_pcm_x_master_pending_x_cancel_finalize_exact("); + UT_ASSERT_NOT_NULL(graph); + UT_ASSERT_NOT_NULL(stale); + UT_ASSERT_NOT_NULL(note_stale); + UT_ASSERT_NOT_NULL(fail_closed); + UT_ASSERT_NOT_NULL(clear); + UT_ASSERT_NOT_NULL(finalize); + if (graph != NULL && stale != NULL && note_stale != NULL && fail_closed != NULL + && clear != NULL && finalize != NULL) + UT_ASSERT(graph < stale && stale < note_stale && note_stale < fail_closed + && fail_closed < clear && clear < finalize && finalize < end); + prepare = strstr(cancel, "cluster_pcm_x_master_pending_x_cancel_prepare_exact("); + UT_ASSERT(prepare == NULL || prepare > end); + } + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_cancel_envelope("); + end = handler != NULL ? strstr(handler, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(end); + if (handler != NULL && end != NULL) { + prepare = strstr(handler, "cluster_pcm_x_master_pending_x_cancel_prepare_exact("); + cancel = prepare != NULL ? strstr(prepare, "gcs_block_pcm_x_cancel_claimed_probe_exact(") + : NULL; + UT_ASSERT_NOT_NULL(prepare); + UT_ASSERT_NOT_NULL(cancel); + if (prepare != NULL && cancel != NULL) + UT_ASSERT(prepare < cancel && cancel < end); + UT_ASSERT(strstr(handler, "cluster_pcm_x_master_cancel_reversible_exact(") == NULL); + } + free(source); +} + +UT_TEST(test_pcm_x_cancel_cleanup_classifies_exact_wfg_and_post_clear_failure) +{ + char *source = read_gcs_block_source(); + char *ordinary; + char *claimed; + char *end; + char *exact; + char *removed; + char *stale; + char *note_stale; + char *fail_closed; + char *finalize; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + ordinary = strstr(source, "\ncluster_gcs_pcm_x_remove_cancelled_waiter("); + end = ordinary != NULL ? strstr(ordinary, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(ordinary); + UT_ASSERT_NOT_NULL(end); + if (ordinary != NULL && end != NULL) { + exact = strstr(ordinary, "cluster_lmd_graph_remove_edge_by_waiter_exact_result("); + removed = exact != NULL ? strstr(exact, "CLUSTER_LMD_GRAPH_REMOVE_REMOVED") : NULL; + stale = removed != NULL ? strstr(removed, "CLUSTER_LMD_GRAPH_REMOVE_STALE") : NULL; + note_stale = stale != NULL + ? strstr(stale, "cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(") + : NULL; + fail_closed + = note_stale != NULL ? strstr(note_stale, "cluster_pcm_x_runtime_fail_closed(") : NULL; + UT_ASSERT_NOT_NULL(exact); + UT_ASSERT_NOT_NULL(removed); + UT_ASSERT_NOT_NULL(stale); + UT_ASSERT_NOT_NULL(note_stale); + UT_ASSERT_NOT_NULL(fail_closed); + if (exact != NULL && removed != NULL && stale != NULL && note_stale != NULL + && fail_closed != NULL) + UT_ASSERT(exact < removed && removed < stale && stale < note_stale + && note_stale < fail_closed && fail_closed < end); + } + claimed = strstr(source, "\ngcs_block_pcm_x_cancel_claimed_probe_exact("); + if (claimed != NULL) + claimed = strstr(claimed + 1, "\ngcs_block_pcm_x_cancel_claimed_probe_exact("); + end = claimed != NULL ? strstr(claimed, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(claimed); + UT_ASSERT_NOT_NULL(end); + if (claimed != NULL && end != NULL) { + finalize = strstr(claimed, + "result = cluster_pcm_x_master_pending_x_cancel_finalize_exact(token)"); + fail_closed + = finalize != NULL ? strstr(finalize, "cluster_pcm_x_runtime_fail_closed(") : NULL; + UT_ASSERT_NOT_NULL(finalize); + UT_ASSERT_NOT_NULL(fail_closed); + if (finalize != NULL && fail_closed != NULL) + UT_ASSERT(finalize < fail_closed && fail_closed < end); + } + free(source); +} + +UT_TEST(test_pcm_x_terminal_retry_reclaims_cancel_cleanup_after_owner_death) +{ + char *source = read_gcs_block_source(); + char *cleanup; + char *kick; + char *end; + char *prepare; + char *claimed; + char *ordinary; + char *cancel_gate; + char *cancel_ack; + char *prehandle_ack; + char *snapshot; + char *stage; + char *stage_second; + char *detach; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + cleanup = strstr(source, "\ngcs_block_pcm_x_cancel_terminal_cleanup_exact("); + end = cleanup != NULL ? strstr(cleanup, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(cleanup); + UT_ASSERT_NOT_NULL(end); + if (cleanup != NULL && end != NULL) { + prepare = strstr(cleanup, "cluster_pcm_x_master_pending_x_cancel_prepare_exact("); + claimed = prepare != NULL ? strstr(prepare, "gcs_block_pcm_x_cancel_claimed_probe_exact(") + : NULL; + ordinary = prepare != NULL ? strstr(prepare, "cluster_gcs_pcm_x_remove_cancelled_waiter(") + : NULL; + UT_ASSERT_NOT_NULL(prepare); + UT_ASSERT_NOT_NULL(claimed); + UT_ASSERT_NOT_NULL(ordinary); + if (prepare != NULL && claimed != NULL && ordinary != NULL) + UT_ASSERT(prepare < claimed && prepare < ordinary && claimed < end && ordinary < end); + } + kick = strstr(source, "\ncluster_gcs_pcm_x_terminal_kick("); + end = kick != NULL ? strstr(kick, "\n}\n\n\nstatic") : NULL; + UT_ASSERT_NOT_NULL(kick); + UT_ASSERT_NOT_NULL(end); + if (kick != NULL && end != NULL) { + cancel_gate = strstr(kick, "ref->grant_generation == 0"); + cleanup = cancel_gate != NULL + ? strstr(cancel_gate, "gcs_block_pcm_x_cancel_terminal_cleanup_exact(") + : NULL; + snapshot = cleanup != NULL + ? strstr(cleanup, "cluster_pcm_x_master_cancel_ack_snapshot_exact(") + : NULL; + prehandle_ack + = snapshot != NULL ? strstr(snapshot, "PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK") : NULL; + cancel_ack + = prehandle_ack != NULL ? strstr(prehandle_ack, "PGRAC_IC_MSG_PCM_X_CANCEL_ACK") : NULL; + stage = snapshot != NULL ? strstr(snapshot, "cluster_gcs_pcm_x_stage_frame(") : NULL; + stage_second = stage != NULL ? strstr(stage + 1, "cluster_gcs_pcm_x_stage_frame(") : NULL; + detach = strstr(kick, "cluster_pcm_x_master_detach_terminal_exact("); + UT_ASSERT_NOT_NULL(cancel_gate); + UT_ASSERT_NOT_NULL(cleanup); + UT_ASSERT_NOT_NULL(snapshot); + UT_ASSERT_NOT_NULL(prehandle_ack); + UT_ASSERT_NOT_NULL(cancel_ack); + UT_ASSERT_NOT_NULL(stage); + UT_ASSERT_NOT_NULL(stage_second); + UT_ASSERT_NOT_NULL(detach); + if (cancel_gate != NULL && cleanup != NULL && snapshot != NULL && prehandle_ack != NULL + && cancel_ack != NULL && stage != NULL && stage_second != NULL && detach != NULL) + UT_ASSERT(cancel_gate < cleanup && cleanup < snapshot && snapshot < prehandle_ack + && prehandle_ack < cancel_ack && snapshot < stage && stage < stage_second + && stage_second < detach && detach < end); + } + free(source); +} + + +UT_TEST(test_pcm_x_invalidate_ack_matches_only_exact_unacked_holder) +{ + GcsBlockInvalidateAckPayload ack; + PcmXMasterDriveSnapshot snapshot; + + memset(&ack, 0, sizeof(ack)); + memset(&snapshot, 0, sizeof(snapshot)); + snapshot.ref.identity.tag.spcOid = 11; + snapshot.ref.identity.tag.dbOid = 12; + snapshot.ref.identity.tag.relNumber = 13; + snapshot.ref.identity.tag.blockNum = 14; + snapshot.ref.identity.cluster_epoch = 0; + snapshot.ref.identity.request_id = 19; + snapshot.ticket_state = PCM_XT_ACTIVE_TRANSFER; + snapshot.pending_s_holders_bitmap = (UINT32_C(1) << 1) | (UINT32_C(1) << 3); + snapshot.acked_s_holders_bitmap = UINT32_C(1) << 1; + ack.tag = snapshot.ref.identity.tag; + ack.epoch = snapshot.ref.identity.cluster_epoch; + ack.request_id = snapshot.ref.identity.request_id; + ack.sender_node = 3; + + UT_ASSERT_EQ(cluster_gcs_pcm_x_invalidate_ack_match_exact(&snapshot, &ack, 0, 3), + PCM_X_QUEUE_OK); + ack.ack_status = GCS_BLOCK_INVALIDATE_ACK_STATUS_RETRYABLE_BUSY; + UT_ASSERT_EQ(cluster_gcs_pcm_x_invalidate_ack_match_exact(&snapshot, &ack, 0, 3), + PCM_X_QUEUE_BUSY); + ack.ack_status = 0; + ack.sender_node = 1; + UT_ASSERT_EQ(cluster_gcs_pcm_x_invalidate_ack_match_exact(&snapshot, &ack, 0, 1), + PCM_X_QUEUE_DUPLICATE); + ack.sender_node = 2; + UT_ASSERT_EQ(cluster_gcs_pcm_x_invalidate_ack_match_exact(&snapshot, &ack, 0, 2), + PCM_X_QUEUE_STALE); + ack.sender_node = 3; + ack.ack_status = 1; + UT_ASSERT_EQ(cluster_gcs_pcm_x_invalidate_ack_match_exact(&snapshot, &ack, 0, 3), + PCM_X_QUEUE_BAD_STATE); + ack.ack_status = 0; + ack.request_id++; + UT_ASSERT_EQ(cluster_gcs_pcm_x_invalidate_ack_match_exact(&snapshot, &ack, 0, 3), + PCM_X_QUEUE_NOT_FOUND); +} + + +UT_TEST(test_pcm_x_final_ack_builds_exact_grd_handoff_token) +{ + PcmAuthoritySnapshot authority; + PcmXGrdHandoffToken handoff; + PcmXMasterFinalAckToken final; + + memset(&authority, 0, sizeof(authority)); + memset(&final, 0, sizeof(final)); + final.final_ack.ref.identity.tag.spcOid = 21; + final.final_ack.ref.identity.tag.dbOid = 22; + final.final_ack.ref.identity.tag.relNumber = 23; + final.final_ack.ref.identity.tag.blockNum = 24; + final.final_ack.ref.identity.node_id = 3; + final.final_ack.ref.identity.procno = 25; + final.final_ack.ref.identity.cluster_epoch = 0; + final.final_ack.ref.identity.request_id = 27; + final.final_ack.ref.handle.ticket_id = 35; + final.final_ack.ref.grant_generation = 28; + final.final_ack.image_id = 29; + final.final_ack.committed_own_generation = 30; + final.image.image_id = final.final_ack.image_id; + /* Generation zero is the legal first ownership generation. */ + final.image.source_own_generation = 0; + final.image.page_scn = 32; + final.image.page_lsn = 33; + final.image.source_node = 2; + final.image.page_checksum = 34; + + UT_ASSERT(cluster_gcs_pcm_x_grd_handoff_token_build(&final, &authority, &handoff)); + UT_ASSERT(BufferTagsEqual(&handoff.tag, &final.final_ack.ref.identity.tag)); + UT_ASSERT_EQ(handoff.cluster_epoch, 0); + UT_ASSERT_EQ(handoff.request_id, 27); + UT_ASSERT_EQ(handoff.ticket_id, 35); + UT_ASSERT_EQ(handoff.grant_generation, 28); + UT_ASSERT_EQ(handoff.image_id, 29); + UT_ASSERT_EQ(handoff.source_own_generation, 0); + UT_ASSERT_EQ(handoff.requester_node, 3); + UT_ASSERT_EQ(handoff.requester_procno, 25); + UT_ASSERT_EQ(handoff.source_node, 2); + UT_ASSERT_EQ(handoff.page_checksum, 34); + final.image.image_id++; + UT_ASSERT(!cluster_gcs_pcm_x_grd_handoff_token_build(&final, &authority, &handoff)); +} + + +UT_TEST(test_pcm_x_holder_image_evidence_never_uses_generation_as_presence) +{ + PcmXLocalHolderProgress progress; + PcmXTicketRef ref; + + memset(&progress, 0, sizeof(progress)); + memset(&ref, 0, sizeof(ref)); + ref.identity.node_id = 1; + ref.identity.cluster_epoch = 2; + ref.identity.request_id = 3; + ref.handle.ticket_id = 4; + ref.handle.queue_generation = 5; + ref.grant_generation = 6; + progress.ref = ref; + progress.image.image_id = 7; + progress.image.source_node = 2; + progress.image.source_own_generation = 0; + + /* REVOKE has already published image_id, but that alone is not READY. */ + UT_ASSERT(!cluster_gcs_pcm_x_holder_image_ready_exact(&progress, &ref, 7, 2)); + progress.last_response_opcode = PGRAC_IC_MSG_PCM_X_REVOKE; + progress.pending_opcode = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + progress.phase = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + UT_ASSERT(cluster_gcs_pcm_x_holder_image_ready_exact(&progress, &ref, 7, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_holder_image_ready_exact(&progress, &ref, 7, 3)); + + progress.pending_opcode = 0; + progress.phase = 0; + progress.last_response_opcode = PGRAC_IC_MSG_PCM_X_DRAIN_POLL; + progress.flags = PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK; + UT_ASSERT(cluster_gcs_pcm_x_holder_image_drained_exact(&progress, &ref, 7, 2)); + UT_ASSERT(!cluster_gcs_pcm_x_holder_image_drained_exact(&progress, &ref, 8, 2)); +} + + +UT_TEST(test_pcm_x_pending_x_marker_is_only_a_pre_handoff_gate) +{ + UT_ASSERT(cluster_gcs_pcm_x_transfer_pre_handoff_phase(0)); + UT_ASSERT(cluster_gcs_pcm_x_transfer_pre_handoff_phase(PGRAC_IC_MSG_PCM_X_REVOKE)); + UT_ASSERT(!cluster_gcs_pcm_x_transfer_pre_handoff_phase(PGRAC_IC_MSG_PCM_X_PREPARE_GRANT)); + UT_ASSERT(!cluster_gcs_pcm_x_transfer_pre_handoff_phase(PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK)); +} + + +UT_TEST(test_pcm_x_ready_publication_follows_exact_retained_commit) +{ + char *source = read_gcs_block_source(); + const char *begin; + const char *materialize; + const char *finish; + const char *publish; + const char *send; + const char *end; + + if (source == NULL) + return; + begin = strstr(source, "\ngcs_block_pcm_x_materialize_reserved_work("); + UT_ASSERT_NOT_NULL(begin); + if (begin == NULL) { + free(source); + return; + } + end = strstr(begin + 1, "\n}\n\n\n"); + materialize = strstr(begin, "cluster_gcs_block_dedup_pcm_x_materialize("); + finish = strstr(begin, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + publish = strstr(begin, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact("); + send = strstr(begin, "gcs_block_pcm_x_stage_ready_work("); + UT_ASSERT_NOT_NULL(end); + UT_ASSERT_NOT_NULL(materialize); + UT_ASSERT_NOT_NULL(finish); + UT_ASSERT_NOT_NULL(publish); + UT_ASSERT_NOT_NULL(send); + if (end != NULL && materialize != NULL && finish != NULL && publish != NULL && send != NULL) + UT_ASSERT(materialize < finish && finish < publish && publish < send && send < end); + free(source); +} + + +UT_TEST(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change) +{ + char *source = read_gcs_block_source(); + const char *abort; + const char *begin; + const char *copy; + const char *finish; + const char *materialize; + const char *publish; + const char *revoke_handler; + const char *drain; + const char *generic_install; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + begin = strstr(source, "\ngcs_block_pcm_x_materialize_reserved_work("); + abort = strstr(source, "\ngcs_block_pcm_x_abort_image_before_finish("); + revoke_handler = strstr(source, "\ncluster_gcs_handle_pcm_x_revoke_envelope("); + drain = strstr(source, "\ngcs_block_pcm_x_local_drain_apply_exact("); + generic_install = strstr(source, "\ngcs_block_install_block("); + UT_ASSERT_NOT_NULL(begin); + UT_ASSERT_NOT_NULL(abort); + UT_ASSERT_NOT_NULL(revoke_handler); + UT_ASSERT_NOT_NULL(drain); + UT_ASSERT_NOT_NULL(generic_install); + if (revoke_handler != NULL) { + UT_ASSERT_NOT_NULL(strstr(revoke_handler, "own_snapshot.pcm_state == (uint8)PCM_STATE_N")); + UT_ASSERT_NOT_NULL(strstr(revoke_handler, "own_snapshot.pcm_state == (uint8)PCM_STATE_S")); + UT_ASSERT_NOT_NULL(strstr(revoke_handler, "own_snapshot.pcm_state == (uint8)PCM_STATE_X")); + } + if (begin != NULL) { + UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_N")); + UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_S")); + UT_ASSERT_NOT_NULL(strstr(begin, "current.pcm_state == (uint8)PCM_STATE_X")); + UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_prepare_n_source_image(")); + UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_s_revoke(")); + UT_ASSERT_NOT_NULL(strstr(begin, "cluster_bufmgr_pcm_own_begin_x_revoke(")); + UT_ASSERT_NOT_NULL(strstr(begin, "cluster_pcm_x_revoke_finish_mode(")); + UT_ASSERT_NOT_NULL(strstr(begin, "CLUSTER_PCM_X_REVOKE_FINISH_DROP")); + copy = strstr(begin, "cluster_bufmgr_copy_block_for_gcs("); + materialize = strstr(begin, "cluster_gcs_block_dedup_pcm_x_materialize("); + finish = strstr(begin, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + publish = strstr(begin, "cluster_gcs_block_dedup_pcm_x_publish_ready_exact("); + UT_ASSERT_NOT_NULL(copy); + UT_ASSERT_NOT_NULL(materialize); + UT_ASSERT_NOT_NULL(finish); + UT_ASSERT_NOT_NULL(publish); + if (copy != NULL && materialize != NULL && finish != NULL && publish != NULL) + UT_ASSERT(copy < materialize && materialize < finish && finish < publish); + } + if (abort != NULL) { + UT_ASSERT_NOT_NULL(strstr(abort, "cluster_bufmgr_pcm_own_abort_n_revoke(")); + UT_ASSERT_NOT_NULL(strstr(abort, "cluster_bufmgr_pcm_own_abort_s_revoke(")); + UT_ASSERT_NOT_NULL(strstr(abort, "cluster_bufmgr_pcm_own_abort_x_revoke(")); + } + if (drain != NULL) { + const char *local_drain = strstr(drain, "cluster_pcm_x_local_drain_poll_exact("); + const char *duplicate_guard + = local_drain != NULL ? strstr(local_drain, "if (result != PCM_X_QUEUE_OK)") : NULL; + const char *release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); + const char *finish_mode_gate = strstr(drain, "cluster_pcm_x_revoke_finish_mode("); + const char *drop_arm = strstr(drain, "CLUSTER_PCM_X_REVOKE_FINISH_DROP"); + const char *release_retained + = strstr(drain, "cluster_bufmgr_pcm_own_release_retained_image("); + + UT_ASSERT_NOT_NULL(local_drain); + UT_ASSERT_NOT_NULL(duplicate_guard); + UT_ASSERT_NOT_NULL(release_record); + UT_ASSERT_NOT_NULL(finish_mode_gate); + UT_ASSERT_NOT_NULL(drop_arm); + UT_ASSERT_NOT_NULL(release_retained); + if (local_drain != NULL && duplicate_guard != NULL && release_record != NULL + && finish_mode_gate != NULL && drop_arm != NULL && release_retained != NULL) + UT_ASSERT(local_drain < duplicate_guard && duplicate_guard < release_record + && release_record < finish_mode_gate && finish_mode_gate < drop_arm + && drop_arm < release_retained); + } + if (generic_install != NULL) { + const char *content = strstr(generic_install, "LWLockAcquire(content_lock, LW_EXCLUSIVE)"); + const char *gate + = strstr(generic_install, "cluster_bufmgr_pcm_x_content_write_permitted(buf)"); + const char *copy = strstr(generic_install, "memcpy(page, block_data"); + + UT_ASSERT_NOT_NULL(content); + UT_ASSERT_NOT_NULL(gate); + UT_ASSERT_NOT_NULL(copy); + if (content != NULL && gate != NULL && copy != NULL) + UT_ASSERT(content < gate && gate < copy); + } + free(source); +} + + +UT_TEST(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper) +{ + char *source = read_gcs_block_source(); + const char *apply; + const char *handler; + const char *handler_end; + const char *raw_drain; + const char *stage; + const char *terminal; + const char *terminal_end; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_drain_poll_envelope("); + terminal = strstr(source, "\ncluster_gcs_pcm_x_terminal_kick("); + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(terminal); + if (handler != NULL) { + handler_end = strstr(handler + 1, "\n}\n\n\n"); + apply = strstr(handler, "gcs_block_pcm_x_local_drain_apply_exact("); + raw_drain = strstr(handler, "cluster_pcm_x_local_drain_poll_exact("); + UT_ASSERT_NOT_NULL(handler_end); + UT_ASSERT_NOT_NULL(apply); + if (handler_end != NULL && apply != NULL) + UT_ASSERT(apply < handler_end); + if (handler_end != NULL && raw_drain != NULL) + UT_ASSERT(raw_drain > handler_end); + } + if (terminal != NULL) { + terminal_end = strstr(terminal + 1, "\n}\n\n\n"); + stage = strstr(terminal, "cluster_gcs_pcm_x_stage_frame(PGRAC_IC_MSG_PCM_X_DRAIN_POLL"); + UT_ASSERT_NOT_NULL(terminal_end); + UT_ASSERT_NOT_NULL(stage); + if (terminal_end != NULL && stage != NULL) + UT_ASSERT(stage < terminal_end); + /* Self is not a special raw-drain arm: every participant, local or + * remote, enters the same authenticated envelope handler above. */ + if (terminal_end != NULL) { + raw_drain = strstr(terminal, "cluster_pcm_x_local_drain_poll_exact("); + apply = strstr(terminal, "gcs_block_pcm_x_local_drain_apply_exact("); + if (raw_drain != NULL) + UT_ASSERT(raw_drain > terminal_end); + if (apply != NULL) + UT_ASSERT(apply > terminal_end); + } + } + free(source); +} + + +UT_TEST(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal) +{ + char *source = read_gcs_block_source(); + const char *begin; + const char *end; + const char *mark; + const char *send; + const char *rollback; + + if (source == NULL) + return; + begin = strstr(source, "\ngcs_block_pcm_x_stage_ready_work("); + UT_ASSERT_NOT_NULL(begin); + if (begin == NULL) { + free(source); + return; + } + end = strstr(begin + 1, "\n}\n\n\n"); + mark = strstr(begin, "cluster_gcs_block_dedup_pcm_x_mark_staged_exact("); + send = strstr(begin, "cluster_gcs_pcm_x_stage_frame("); + rollback = strstr(begin, "cluster_gcs_block_dedup_pcm_x_unmark_staged_exact("); + UT_ASSERT_NOT_NULL(end); + UT_ASSERT_NOT_NULL(mark); + UT_ASSERT_NOT_NULL(send); + UT_ASSERT_NOT_NULL(rollback); + if (end != NULL && mark != NULL && send != NULL && rollback != NULL) + UT_ASSERT(mark < send && send < rollback && rollback < end); + free(source); +} + + +UT_TEST(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed) +{ + char *gcs_source = read_gcs_block_source(); + char *lms_source = read_source_path(LMS_SOURCE_PATH); + const char *owner_start; + const char *owner_exit; + const char *main_start; + const char *worker_start; + const char *main_call; + const char *worker_call; + + if (gcs_source == NULL || lms_source == NULL) { + free(gcs_source); + free(lms_source); + return; + } + owner_exit = strstr(gcs_source, "\ngcs_block_pcm_x_owner_exit("); + owner_start = strstr(gcs_source, "\ncluster_gcs_block_pcm_x_owner_start("); + UT_ASSERT_NOT_NULL(owner_exit); + UT_ASSERT_NOT_NULL(owner_start); + if (owner_exit != NULL) { + UT_ASSERT_NOT_NULL(strstr(owner_exit, "code != 0")); + UT_ASSERT_NOT_NULL(strstr(owner_exit, "cluster_pcm_x_runtime_fail_closed()")); + } + if (owner_start != NULL) { + UT_ASSERT_NOT_NULL(strstr(owner_start, "before_shmem_exit(")); + UT_ASSERT_NOT_NULL( + strstr(owner_start, "cluster_gcs_block_dedup_pcm_x_restart_audit(worker_id)")); + } + + main_start = strstr(lms_source, "\nLmsMain(void)"); + worker_start = strstr(lms_source, "\nLmsWorkerMain(int worker_id)"); + UT_ASSERT_NOT_NULL(main_start); + UT_ASSERT_NOT_NULL(worker_start); + main_call + = main_start != NULL ? strstr(main_start, "cluster_gcs_block_pcm_x_owner_start(0)") : NULL; + worker_call = worker_start != NULL + ? strstr(worker_start, "cluster_gcs_block_pcm_x_owner_start(worker_id)") + : NULL; + UT_ASSERT_NOT_NULL(main_call); + UT_ASSERT_NOT_NULL(worker_call); + if (main_start != NULL && main_call != NULL) + UT_ASSERT(main_call < strstr(main_start, "for (;;)")); + if (worker_start != NULL && worker_call != NULL) + UT_ASSERT(worker_call < strstr(worker_start, "for (;;)")); + UT_ASSERT_EQ(count_occurrences(lms_source, "cluster_gcs_block_pcm_x_owner_start("), 2); + free(gcs_source); + free(lms_source); +} + + +UT_TEST(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup) +{ + char *source = read_gcs_block_source(); + char *handler; + char *intercept; + char *generic_lookup; + char *serve; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + handler = strstr(source, "\ncluster_gcs_handle_block_request_envelope("); + serve = strstr(source, "\ngcs_block_pcm_x_serve_image_fetch("); + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(serve); + if (handler != NULL) { + intercept = strstr(handler, "gcs_block_pcm_x_serve_image_fetch(env, req, dedup_worker_id)"); + generic_lookup = strstr(handler, "cluster_gcs_block_dedup_lookup_or_register("); + UT_ASSERT_NOT_NULL(intercept); + UT_ASSERT_NOT_NULL(generic_lookup); + if (intercept != NULL && generic_lookup != NULL) + UT_ASSERT(intercept < generic_lookup); + } + if (serve != NULL) { + UT_ASSERT_NOT_NULL(strstr(serve, "cluster_pcm_x_image_fetch_request_exact(")); + UT_ASSERT(count_occurrences(serve, "cluster_pcm_x_local_holder_progress_exact(") >= 2); + UT_ASSERT_NOT_NULL(strstr(serve, "gcs_block_pcm_x_authenticated_session(")); + UT_ASSERT_NOT_NULL(strstr(serve, "gcs_block_pcm_x_revalidate_peer_binding(")); + UT_ASSERT_NOT_NULL(strstr(serve, "cluster_gcs_block_dedup_pcm_x_lookup(")); + UT_ASSERT_NOT_NULL(strstr(serve, "gcs_block_resend_cached_reply(")); + } + free(source); +} + + +UT_TEST(test_pcm_x_requester_fetch_revalidates_queue_and_reservation_before_install) +{ + char *source = read_gcs_block_source(); + const char *fetch; + const char *fetch_end; + const char *backoff_branch; + const char *backoff_wait; + const char *backoff_before; + const char *backoff_after; + const char *cv_wait; + const char *cv_timeout; + const char *cv_before; + const char *cv_after; + const char *install_call; + const char *install; + const char *install_end; + const char *install_runtime_before; + const char *install_publish; + const char *install_runtime_after; + char *reply_handler; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + fetch = strstr(source, "\ncluster_gcs_pcm_x_fetch_image_and_install("); + fetch_end = fetch != NULL ? strstr(fetch, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(fetch); + UT_ASSERT_NOT_NULL(fetch_end); + if (fetch != NULL && fetch_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(fetch, "gcs_block_try_reserve_exact_slot(")); + UT_ASSERT_NOT_NULL(strstr(fetch, "cluster_pcm_x_image_fetch_build_request(")); + UT_ASSERT_NOT_NULL(strstr(fetch, "const PcmXRuntimeSnapshot *request_runtime")); + UT_ASSERT_NOT_NULL(strstr(fetch, "cluster_pcm_x_image_fetch_reply_exact(")); + UT_ASSERT_NOT_NULL(strstr(fetch, "cluster_pcm_x_image_fetch_reservation_exact(")); + UT_ASSERT_NOT_NULL(strstr(fetch, "gcs_block_pcm_x_install_reserved_image_exact(")); + UT_ASSERT_NOT_NULL(strstr(fetch, "gcs_block_release_slot(slot)")); + backoff_branch = strstr(fetch, "if (retry_attempt > 0)"); + backoff_wait + = backoff_branch != NULL ? strstr(backoff_branch, "(void)WaitLatch(MyLatch") : NULL; + backoff_before + = backoff_wait != NULL + ? strstr(backoff_branch, "gcs_block_pcm_x_fetch_requester_authority_exact(") + : NULL; + backoff_after + = backoff_wait != NULL + ? strstr(backoff_wait, "gcs_block_pcm_x_fetch_requester_authority_exact(") + : NULL; + cv_wait = strstr(fetch, "ConditionVariableTimedSleep("); + cv_timeout = strstr(fetch, "timeout_ms = (long)"); + cv_before = cv_timeout != NULL && cv_wait != NULL + ? strstr(cv_timeout, "gcs_block_pcm_x_fetch_requester_authority_exact(") + : NULL; + cv_after = cv_wait != NULL + ? strstr(cv_wait, "gcs_block_pcm_x_fetch_requester_authority_exact(") + : NULL; + install_call = cv_after != NULL + ? strstr(cv_after, "gcs_block_pcm_x_install_reserved_image_exact(") + : NULL; + UT_ASSERT_NOT_NULL(backoff_branch); + UT_ASSERT_NOT_NULL(backoff_wait); + UT_ASSERT_NOT_NULL(backoff_before); + UT_ASSERT_NOT_NULL(backoff_after); + UT_ASSERT_NOT_NULL(cv_wait); + UT_ASSERT_NOT_NULL(cv_timeout); + UT_ASSERT_NOT_NULL(cv_before); + UT_ASSERT_NOT_NULL(cv_after); + UT_ASSERT_NOT_NULL(install_call); + if (backoff_branch != NULL && backoff_before != NULL && backoff_wait != NULL + && backoff_after != NULL && cv_timeout != NULL && cv_before != NULL && cv_wait != NULL + && cv_after != NULL && install_call != NULL) + UT_ASSERT(backoff_branch < backoff_before && backoff_before < backoff_wait + && backoff_wait < backoff_after && cv_timeout < cv_before + && cv_before < cv_wait && cv_wait < cv_after && cv_after < install_call + && install_call < fetch_end); + } + install = strstr(source, "\ngcs_block_pcm_x_install_reserved_image_exact("); + install_end = install != NULL ? strstr(install, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(install); + UT_ASSERT_NOT_NULL(install_end); + if (install != NULL && install_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(install, "const PcmXRuntimeSnapshot *request_runtime")); + install_runtime_before = strstr(install, "cluster_gcs_pcm_x_requester_runtime_exact("); + install_publish = strstr(install, "cluster_bufmgr_pcm_own_publish_installed_x_image("); + install_runtime_after + = install_publish != NULL + ? strstr(install_publish, "cluster_gcs_pcm_x_requester_runtime_exact(") + : NULL; + UT_ASSERT_NOT_NULL(install_runtime_before); + UT_ASSERT_NOT_NULL(install_publish); + UT_ASSERT_NOT_NULL(install_runtime_after); + if (install_runtime_before != NULL && install_publish != NULL + && install_runtime_after != NULL) + UT_ASSERT(install_runtime_before < install_publish + && install_publish < install_runtime_after + && install_runtime_after < install_end); + } + reply_handler = strstr(source, "\ncluster_gcs_handle_block_reply_envelope("); + UT_ASSERT_NOT_NULL(reply_handler); + if (reply_handler != NULL) { + UT_ASSERT_NOT_NULL( + strstr(reply_handler, "env->source_node_id != (uint32)hdr->sender_node")); + UT_ASSERT_NOT_NULL(strstr(reply_handler, "env->dest_node_id != (uint32)cluster_node_id")); + } + free(source); +} + + +UT_TEST(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x) +{ + char *source = read_gcs_block_source(); + const char *adopt; + const char *adopt_end; + const char *adopt_page_check; + const char *adopt_handoff; + const char *finish; + const char *finish_end; + const char *finish_preflight; + const char *finish_progress; + const char *finish_try; + const char *finish_fail_closed; + const char *finish_commit; + const char *finish_catch; + const char *finish_catch_guard; + const char *finish_catch_fail_closed; + const char *finish_catch_release; + const char *finish_catch_rethrow; + const char *finish_end_try; + const char *copy; + const char *materialize; + const char *materialize_end; + const char *self_arm; + const char *remote_finish; + const char *drain; + const char *drain_end; + const char *verify_x; + const char *release_record; + const char *self_return; + const char *release_retained; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + adopt = strstr(source, "\ncluster_gcs_pcm_x_adopt_self_image("); + finish = strstr(source, "\ncluster_gcs_pcm_x_finish_self_image_x("); + materialize = strstr(source, "\ngcs_block_pcm_x_materialize_reserved_work("); + drain = strstr(source, "\ngcs_block_pcm_x_local_drain_apply_exact("); + UT_ASSERT_NOT_NULL(adopt); + UT_ASSERT_NOT_NULL(finish); + UT_ASSERT_NOT_NULL(materialize); + UT_ASSERT_NOT_NULL(drain); + if (adopt == NULL || finish == NULL || materialize == NULL || drain == NULL) { + free(source); + return; + } + adopt_end = finish; + finish_end = strstr(finish + 1, "\n\n/*"); + materialize_end = strstr(materialize + 1, "\n\n\n"); + drain_end = strstr(drain + 1, "\n\n\n"); + UT_ASSERT_NOT_NULL(finish_end); + UT_ASSERT_NOT_NULL(materialize_end); + UT_ASSERT_NOT_NULL(drain_end); + copy = strstr(adopt, "memcpy(page"); + UT_ASSERT(copy == NULL || copy >= adopt_end); + adopt_page_check = strstr(adopt, "gcs_block_pcm_x_self_page_exact("); + adopt_handoff = strstr(adopt, "cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation("); + UT_ASSERT_NOT_NULL(adopt_page_check); + UT_ASSERT_NOT_NULL(adopt_handoff); + if (adopt_page_check != NULL && adopt_handoff != NULL) + UT_ASSERT(adopt_page_check < adopt_end && adopt_handoff < adopt_end); + if (finish_end != NULL) { + finish_preflight = strstr(finish, "cluster_bufmgr_pcm_own_snapshot("); + finish_progress = strstr(finish, "cluster_pcm_x_local_progress_exact("); + finish_try = strstr(finish, "PG_TRY();"); + finish_fail_closed = strstr(finish, "cluster_pcm_x_runtime_fail_closed();"); + finish_commit = strstr(finish, "cluster_bufmgr_pcm_own_finish_x_commit("); + UT_ASSERT_NOT_NULL(finish_preflight); + UT_ASSERT_NOT_NULL(finish_progress); + UT_ASSERT_NOT_NULL(finish_try); + UT_ASSERT_NOT_NULL(finish_fail_closed); + UT_ASSERT_NOT_NULL(finish_commit); + if (finish_preflight != NULL && finish_progress != NULL && finish_try != NULL + && finish_fail_closed != NULL && finish_commit != NULL) + UT_ASSERT(finish_preflight < finish_progress && finish_progress < finish_try + && finish_try < finish_commit && finish_commit < finish_end + && finish_fail_closed < finish_end); + finish_catch = finish_try != NULL ? strstr(finish_try, "PG_CATCH();") : NULL; + finish_catch_guard + = finish_catch != NULL ? strstr(finish_catch, "if (handoff_live || committed)") : NULL; + finish_catch_fail_closed + = finish_catch_guard != NULL + ? strstr(finish_catch_guard, "cluster_pcm_x_runtime_fail_closed();") + : NULL; + finish_catch_release = finish_catch_fail_closed != NULL + ? strstr(finish_catch_fail_closed, + "if (content_locked && LWLockHeldByMe(content_lock))") + : NULL; + finish_catch_rethrow + = finish_catch_release != NULL ? strstr(finish_catch_release, "PG_RE_THROW();") : NULL; + finish_end_try + = finish_catch_rethrow != NULL ? strstr(finish_catch_rethrow, "PG_END_TRY();") : NULL; + UT_ASSERT_NOT_NULL(finish_catch); + UT_ASSERT_NOT_NULL(finish_catch_guard); + UT_ASSERT_NOT_NULL(finish_catch_fail_closed); + UT_ASSERT_NOT_NULL(finish_catch_release); + UT_ASSERT_NOT_NULL(finish_catch_rethrow); + UT_ASSERT_NOT_NULL(finish_end_try); + if (finish_catch != NULL && finish_catch_guard != NULL && finish_catch_fail_closed != NULL + && finish_catch_release != NULL && finish_catch_rethrow != NULL + && finish_end_try != NULL) + UT_ASSERT(finish_catch < finish_catch_guard + && finish_catch_guard < finish_catch_fail_closed + && finish_catch_fail_closed < finish_catch_release + && finish_catch_release < finish_catch_rethrow + && finish_catch_rethrow < finish_end_try && finish_end_try < finish_end); + } + self_arm = strstr(materialize, "if (self_source_handoff)"); + remote_finish = strstr(materialize, "cluster_bufmgr_pcm_own_finish_revoke_retain("); + UT_ASSERT_NOT_NULL(self_arm); + UT_ASSERT_NOT_NULL(remote_finish); + if (self_arm != NULL && remote_finish != NULL && materialize_end != NULL) + UT_ASSERT(self_arm < remote_finish && remote_finish < materialize_end); + + verify_x = strstr(drain, "cluster_bufmgr_pcm_own_self_handoff_x_exact("); + release_record = strstr(drain, "cluster_gcs_block_dedup_pcm_x_release_exact("); + self_return + = release_record != NULL ? strstr(release_record, "if (self_source_handoff)") : NULL; + release_retained = strstr(drain, "cluster_bufmgr_pcm_own_release_retained_image("); + UT_ASSERT_NOT_NULL(verify_x); + UT_ASSERT_NOT_NULL(release_record); + UT_ASSERT_NOT_NULL(self_return); + UT_ASSERT_NOT_NULL(release_retained); + if (verify_x != NULL && release_record != NULL && self_return != NULL + && release_retained != NULL && drain_end != NULL) + UT_ASSERT(verify_x < release_record && release_record < self_return + && self_return < release_retained && release_retained < drain_end); + free(source); +} + + +UT_TEST(test_pcm_x_retire_wake_identity_is_wait_generation_exact) +{ + PcmXWaitIdentity identity; + ClusterLmdWaitStateSnapshot snapshot; + + memset(&identity, 0, sizeof(identity)); + identity.node_id = 2; + identity.procno = 7; + identity.xid = (TransactionId)42; + identity.cluster_epoch = UINT64_C(9); + identity.request_id = UINT64_C(23); + identity.wait_seq = UINT64_C(31); + memset(&snapshot, 0, sizeof(snapshot)); + snapshot.active = true; + snapshot.kind = CLUSTER_LMD_WAIT_PCM_CONVERT; + snapshot.request_id = identity.request_id; + snapshot.cluster_epoch = identity.cluster_epoch; + snapshot.xid = identity.xid; + snapshot.wait_seq = identity.wait_seq; + + UT_ASSERT(cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 1, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 7, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_BUSY, &snapshot)); + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_INACTIVE, &snapshot)); + snapshot.active = false; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + snapshot.active = true; + snapshot.kind = CLUSTER_LMD_WAIT_GES; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + snapshot.kind = CLUSTER_LMD_WAIT_PCM_CONVERT; + snapshot.request_id++; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + snapshot.request_id = identity.request_id; + snapshot.cluster_epoch++; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + snapshot.cluster_epoch = identity.cluster_epoch; + snapshot.wait_seq++; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + snapshot.wait_seq = identity.wait_seq; + snapshot.xid++; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); + snapshot.xid = identity.xid; + identity.request_id = 0; + UT_ASSERT(!cluster_gcs_pcm_x_wait_identity_matches( + &identity, 2, 8, CLUSTER_LMD_WAIT_STATE_READ_ACTIVE, &snapshot)); +} + + +UT_TEST(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles) +{ + char *source = read_gcs_block_source(); + const char *driver; + const char *driver_end; + const char *rekey_helper; + const char *rekey_helper_end; + const char *snapshot_before; + const char *rekey_exact; + const char *snapshot_after; + const char *wrapper; + const char *wrapper_end; + const char *cleanup; + const char *cleanup_end; + const char *handoff_pointer; + const char *handoff_active; + const char *handoff_publish; + const char *requester_clear; + const char *cleanup_handoff; + const char *cleanup_wfg; + const char *cleanup_release; + const char *cleanup_abort; + const char *cleanup_clear_helper; + const char *cleanup_clear_helper_end; + const char *cleanup_stale; + const char *cleanup_stale_clear; + const char *cleanup_stale_return; + const char *cleanup_owner_retry; + const char *cleanup_owner_retry_clear; + const char *cleanup_owner_retry_return; + const char *cleanup_fail_closed; + const char *cleanup_fail_closed_clear; + const char *cleanup_fail_closed_return; + const char *formation; + const char *formation_wait; + const char *authenticated_session; + const char *authority_preflight; + const char *request_id; + const char *wait_publish; + const char *nested_guard; + const char *ownership_snapshot; + const char *first_wait; + const char *join; + const char *leader_rekey; + const char *claim; + const char *enqueue; + const char *follower_snapshot; + const char *graph_replace; + const char *graph_clear; + const char *self_adopt; + const char *remote_fetch; + const char *install_ready; + const char *self_finish; + const char *remote_finish; + const char *final_ack; + const char *final_confirm; + const char *clear_wait; + const char *authority_helper; + const char *authority_helper_end; + const char *authority_runtime; + const char *authority_runtime_exact; + const char *authority_peer; + const char *wait_exact; + const char *wait_exact_end; + const char *wait_exact_first_authority; + const char *wait_exact_physical; + const char *wait_exact_second_authority; + const char *wait_scan; + int raw_wait_count = 0; + int exact_wait_count = 0; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + driver = strstr(source, "\ngcs_block_pcm_x_acquire_writer_impl("); + driver_end = driver != NULL ? strstr(driver, "\n}\n") : NULL; + rekey_helper = strstr(source, "\ngcs_block_pcm_x_rekey_leader_base_exact("); + rekey_helper_end = rekey_helper != NULL ? strstr(rekey_helper, "\n}\n") : NULL; + wrapper = strstr(source, "\ncluster_gcs_pcm_x_acquire_writer("); + wrapper_end = wrapper != NULL ? strstr(wrapper, "\n}\n") : NULL; + cleanup = strstr(source, "\ngcs_block_pcm_x_requester_cleanup_impl("); + cleanup_end = cleanup != NULL ? strstr(cleanup, "\n}\n") : NULL; + cleanup_clear_helper = strstr(source, "\ngcs_block_pcm_x_requester_clear_wait("); + cleanup_clear_helper_end + = cleanup_clear_helper != NULL ? strstr(cleanup_clear_helper, "\n}\n") : NULL; + authority_helper = strstr(source, "\ngcs_block_pcm_x_requester_authority_exact("); + authority_helper_end = authority_helper != NULL ? strstr(authority_helper, "\n}\n") : NULL; + wait_exact = strstr(source, "\ngcs_block_pcm_x_requester_wait_exact("); + wait_exact_end = wait_exact != NULL ? strstr(wait_exact, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(driver); + UT_ASSERT_NOT_NULL(driver_end); + UT_ASSERT_NOT_NULL(rekey_helper); + UT_ASSERT_NOT_NULL(rekey_helper_end); + UT_ASSERT_NOT_NULL(wrapper); + UT_ASSERT_NOT_NULL(wrapper_end); + UT_ASSERT_NOT_NULL(cleanup); + UT_ASSERT_NOT_NULL(cleanup_end); + UT_ASSERT_NOT_NULL(cleanup_clear_helper); + UT_ASSERT_NOT_NULL(cleanup_clear_helper_end); + UT_ASSERT_NOT_NULL(authority_helper); + UT_ASSERT_NOT_NULL(authority_helper_end); + UT_ASSERT_NOT_NULL(wait_exact); + UT_ASSERT_NOT_NULL(wait_exact_end); + if (driver == NULL || driver_end == NULL || wrapper == NULL || wrapper_end == NULL) { + free(source); + return; + } + UT_ASSERT_NOT_NULL(strstr(driver, "ClusterPcmXConvertShmem == NULL")); + UT_ASSERT_NOT_NULL(strstr(wrapper, "PG_TRY();")); + UT_ASSERT_NOT_NULL(strstr(wrapper, "gcs_block_pcm_x_requester_cleanup_noexcept(cleanup)")); + UT_ASSERT_NOT_NULL(strstr(wrapper, "ReThrowError(original_error)")); + handoff_pointer = strstr(wrapper, "cleanup->claim_handoff_out = claim_handed_off"); + handoff_active + = handoff_pointer != NULL ? strstr(handoff_pointer, "cleanup->active = true") : NULL; + handoff_publish = strstr(wrapper, "*claim_handed_off = true"); + requester_clear = handoff_publish != NULL ? strstr(handoff_publish, "memset(cleanup, 0") : NULL; + UT_ASSERT_NOT_NULL(handoff_pointer); + UT_ASSERT_NOT_NULL(handoff_active); + UT_ASSERT_NOT_NULL(handoff_publish); + UT_ASSERT_NOT_NULL(requester_clear); + if (handoff_pointer != NULL && handoff_active != NULL && handoff_publish != NULL + && requester_clear != NULL) + UT_ASSERT(handoff_pointer < handoff_active && handoff_publish < requester_clear + && requester_clear < wrapper_end); + cleanup_handoff = cleanup != NULL ? strstr(cleanup, "cleanup->claim_handoff_out") : NULL; + cleanup_wfg = cleanup != NULL ? strstr(cleanup, "if (cleanup->wfg_live)") : NULL; + cleanup_release = cleanup != NULL + ? strstr(cleanup, "cluster_gcs_pcm_x_writer_claim_release_and_wake_exact") + : NULL; + cleanup_abort + = cleanup != NULL ? strstr(cleanup, "cluster_pcm_x_local_writer_claim_abort_exact") : NULL; + UT_ASSERT_NOT_NULL(cleanup_handoff); + UT_ASSERT_NOT_NULL(cleanup_wfg); + UT_ASSERT_NOT_NULL(cleanup_release); + UT_ASSERT(cleanup_abort == NULL || cleanup_abort >= cleanup_end); + if (cleanup_handoff != NULL && cleanup_wfg != NULL && cleanup_end != NULL) + UT_ASSERT(cleanup_handoff < cleanup_wfg && cleanup_wfg < cleanup_end); + if (cleanup_clear_helper != NULL && cleanup_clear_helper_end != NULL) { + UT_ASSERT_NOT_NULL(strstr(cleanup_clear_helper, "cleanup->wait_published")); + UT_ASSERT_NOT_NULL(strstr(cleanup_clear_helper, "cluster_lmd_wait_state_clear(")); + UT_ASSERT_NOT_NULL(strstr(cleanup_clear_helper, "cleanup->wait_published = false")); + } + cleanup_stale = cleanup != NULL ? strstr(cleanup, "CLUSTER_LMD_GRAPH_REMOVE_STALE") : NULL; + cleanup_stale_clear + = cleanup_stale != NULL + ? strstr(cleanup_stale, "gcs_block_pcm_x_requester_clear_wait(cleanup)") + : NULL; + cleanup_stale_return = cleanup_stale_clear != NULL + ? strstr(cleanup_stale_clear, "return PCM_X_QUEUE_CORRUPT") + : NULL; + cleanup_owner_retry + = cleanup != NULL ? strstr(cleanup, "== CLUSTER_PCM_X_OWNER_EXIT_RETRY") : NULL; + cleanup_owner_retry_clear + = cleanup_owner_retry != NULL + ? strstr(cleanup_owner_retry, "gcs_block_pcm_x_requester_clear_wait(cleanup)") + : NULL; + cleanup_owner_retry_return = cleanup_owner_retry_clear != NULL + ? strstr(cleanup_owner_retry_clear, "return result;") + : NULL; + cleanup_fail_closed + = cleanup_owner_retry_return != NULL + ? strstr(cleanup_owner_retry_return, "cluster_pcm_x_runtime_fail_closed()") + : NULL; + cleanup_fail_closed_clear + = cleanup_fail_closed != NULL + ? strstr(cleanup_fail_closed, "gcs_block_pcm_x_requester_clear_wait(cleanup)") + : NULL; + cleanup_fail_closed_return = cleanup_fail_closed_clear != NULL + ? strstr(cleanup_fail_closed_clear, "return result;") + : NULL; + UT_ASSERT_NOT_NULL(cleanup_stale_clear); + UT_ASSERT_NOT_NULL(cleanup_stale_return); + UT_ASSERT_NOT_NULL(cleanup_owner_retry_clear); + UT_ASSERT_NOT_NULL(cleanup_owner_retry_return); + UT_ASSERT_NOT_NULL(cleanup_fail_closed_clear); + UT_ASSERT_NOT_NULL(cleanup_fail_closed_return); + if (cleanup_stale_clear != NULL && cleanup_stale_return != NULL) + UT_ASSERT(cleanup_stale_clear < cleanup_stale_return && cleanup_stale_return < cleanup_end); + if (cleanup_owner_retry_clear != NULL && cleanup_owner_retry_return != NULL) + UT_ASSERT(cleanup_owner_retry_clear < cleanup_owner_retry_return + && cleanup_owner_retry_return < cleanup_end); + if (cleanup_fail_closed_clear != NULL && cleanup_fail_closed_return != NULL) + UT_ASSERT(cleanup_fail_closed_clear < cleanup_fail_closed_return + && cleanup_fail_closed_return < cleanup_end); + authority_runtime = authority_helper != NULL + ? strstr(authority_helper, "cluster_pcm_x_runtime_snapshot()") + : NULL; + authority_runtime_exact + = authority_runtime != NULL + ? strstr(authority_runtime, "cluster_gcs_pcm_x_requester_runtime_exact(") + : NULL; + authority_peer + = authority_runtime_exact != NULL + ? strstr(authority_runtime_exact, "gcs_block_pcm_x_revalidate_peer_binding(") + : NULL; + wait_exact_first_authority + = wait_exact != NULL ? strstr(wait_exact, "gcs_block_pcm_x_requester_authority_exact(") + : NULL; + wait_exact_physical + = wait_exact_first_authority != NULL + ? strstr(wait_exact_first_authority, "gcs_block_pcm_x_requester_wait(wait_index)") + : NULL; + wait_exact_second_authority + = wait_exact_physical != NULL + ? strstr(wait_exact_physical + 1, "gcs_block_pcm_x_requester_authority_exact(") + : NULL; + UT_ASSERT_NOT_NULL(authority_runtime); + UT_ASSERT_NOT_NULL(authority_runtime_exact); + UT_ASSERT_NOT_NULL(authority_peer); + UT_ASSERT_NOT_NULL(wait_exact_first_authority); + UT_ASSERT_NOT_NULL(wait_exact_physical); + UT_ASSERT_NOT_NULL(wait_exact_second_authority); + if (authority_runtime != NULL && authority_runtime_exact != NULL && authority_peer != NULL + && authority_helper_end != NULL) + UT_ASSERT(authority_runtime < authority_runtime_exact + && authority_runtime_exact < authority_peer + && authority_peer < authority_helper_end); + if (wait_exact_first_authority != NULL && wait_exact_physical != NULL + && wait_exact_second_authority != NULL && wait_exact_end != NULL) + UT_ASSERT(wait_exact_first_authority < wait_exact_physical + && wait_exact_physical < wait_exact_second_authority + && wait_exact_second_authority < wait_exact_end); + formation = strstr(driver, "cluster_gcs_pcm_x_requester_formation_action("); + formation_wait = formation != NULL + ? strstr(formation, "gcs_block_pcm_x_requester_wait(&wait_index)") + : NULL; + authenticated_session = formation_wait != NULL + ? strstr(formation_wait, "gcs_block_pcm_x_authenticated_session(") + : NULL; + authority_preflight + = authenticated_session != NULL + ? strstr(authenticated_session, "gcs_block_pcm_x_requester_authority_exact(") + : NULL; + request_id = authority_preflight != NULL + ? strstr(authority_preflight, "gcs_block_pcm_x_next_request_id(") + : NULL; + wait_publish + = request_id != NULL ? strstr(request_id, "cluster_lmd_wait_state_publish(") : NULL; + nested_guard = wait_publish != NULL + ? strstr(wait_publish, "cluster_pcm_x_nested_wait_guard_before_block()") + : NULL; + ownership_snapshot + = nested_guard != NULL ? strstr(nested_guard, "cluster_bufmgr_pcm_own_snapshot(buf") : NULL; + first_wait = ownership_snapshot != NULL + ? strstr(ownership_snapshot, "gcs_block_pcm_x_requester_wait_exact(") + : NULL; + UT_ASSERT_NOT_NULL(formation); + UT_ASSERT_NOT_NULL(formation_wait); + UT_ASSERT_NOT_NULL(authenticated_session); + UT_ASSERT_NOT_NULL(authority_preflight); + UT_ASSERT_NOT_NULL(request_id); + UT_ASSERT_NOT_NULL(wait_publish); + UT_ASSERT_NOT_NULL(nested_guard); + UT_ASSERT_NOT_NULL(ownership_snapshot); + UT_ASSERT_NOT_NULL(first_wait); + if (formation != NULL && formation_wait != NULL && authenticated_session != NULL + && authority_preflight != NULL && request_id != NULL && wait_publish != NULL + && nested_guard != NULL && ownership_snapshot != NULL && first_wait != NULL + && driver_end != NULL) + UT_ASSERT(formation < formation_wait && formation_wait < authenticated_session + && authenticated_session < authority_preflight && authority_preflight < request_id + && request_id < wait_publish && wait_publish < nested_guard + && nested_guard < ownership_snapshot && ownership_snapshot < first_wait + && first_wait < driver_end); + wait_scan = driver; + while (wait_scan != NULL + && (wait_scan = strstr(wait_scan, "gcs_block_pcm_x_requester_wait(&wait_index)")) != NULL + && wait_scan < driver_end) { + raw_wait_count++; + wait_scan++; + } + wait_scan = driver; + while (wait_scan != NULL + && (wait_scan = strstr(wait_scan, "gcs_block_pcm_x_requester_wait_exact(")) != NULL + && wait_scan < driver_end) { + exact_wait_count++; + wait_scan++; + } + UT_ASSERT_EQ(raw_wait_count, 1); + UT_ASSERT_EQ(exact_wait_count, 9); + snapshot_before + = rekey_helper != NULL ? strstr(rekey_helper, "cluster_bufmgr_pcm_own_snapshot(") : NULL; + rekey_exact + = snapshot_before != NULL + ? strstr(snapshot_before, "cluster_pcm_x_local_leader_rekey_generation_exact(") + : NULL; + snapshot_after + = rekey_exact != NULL ? strstr(rekey_exact, "cluster_bufmgr_pcm_own_snapshot(") : NULL; + UT_ASSERT_NOT_NULL(snapshot_before); + UT_ASSERT_NOT_NULL(rekey_exact); + UT_ASSERT_NOT_NULL(snapshot_after); + if (snapshot_before != NULL && rekey_exact != NULL && snapshot_after != NULL + && rekey_helper_end != NULL) + UT_ASSERT(snapshot_before < rekey_exact && rekey_exact < snapshot_after + && snapshot_after < rekey_helper_end); + join = strstr(driver, "cluster_pcm_x_local_join_begin("); + leader_rekey = join != NULL ? strstr(join, "gcs_block_pcm_x_rekey_leader_base_exact(") : NULL; + claim = leader_rekey != NULL ? strstr(leader_rekey, "cluster_pcm_x_local_writer_claim_exact(") + : NULL; + enqueue = claim != NULL ? strstr(claim, "cluster_pcm_x_local_enqueue_arm_exact(") : NULL; + graph_clear = strstr(driver, "cluster_pcm_x_local_follower_wfg_clear_exact("); + follower_snapshot + = graph_clear != NULL + ? strstr(graph_clear, "cluster_pcm_x_local_follower_wfg_snapshot_exact(") + : NULL; + graph_replace = follower_snapshot != NULL + ? strstr(follower_snapshot, "cluster_lmd_graph_replace_waiter_edges_exact(") + : NULL; + self_adopt = strstr(driver, "cluster_gcs_pcm_x_adopt_self_image("); + remote_fetch = strstr(driver, "cluster_gcs_pcm_x_fetch_image_and_install("); + install_ready = strstr(driver, "cluster_pcm_x_local_install_ready_arm_exact("); + self_finish = strstr(driver, "cluster_gcs_pcm_x_finish_self_image_x("); + remote_finish = strstr(driver, "cluster_bufmgr_pcm_own_finish_x_commit("); + final_ack = strstr(driver, "cluster_pcm_x_local_final_ack_arm_exact("); + final_confirm = strstr(driver, "PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM"); + clear_wait = strstr(driver, "cluster_lmd_wait_state_clear("); + + UT_ASSERT_NOT_NULL(join); + UT_ASSERT_NOT_NULL(leader_rekey); + UT_ASSERT_NOT_NULL(claim); + UT_ASSERT_NOT_NULL(enqueue); + UT_ASSERT_NOT_NULL(follower_snapshot); + UT_ASSERT_NOT_NULL(graph_replace); + UT_ASSERT_NOT_NULL(graph_clear); + UT_ASSERT_NOT_NULL(self_adopt); + UT_ASSERT_NOT_NULL(remote_fetch); + UT_ASSERT_NOT_NULL(install_ready); + UT_ASSERT_NOT_NULL(self_finish); + UT_ASSERT_NOT_NULL(remote_finish); + UT_ASSERT_NOT_NULL(final_ack); + UT_ASSERT_NOT_NULL(final_confirm); + UT_ASSERT_NOT_NULL(clear_wait); + if (join != NULL && leader_rekey != NULL && claim != NULL && enqueue != NULL) + UT_ASSERT(join < leader_rekey && leader_rekey < claim && claim < enqueue + && enqueue < driver_end); + UT_ASSERT_NOT_NULL(strstr(driver, "cluster_pcm_x_local_lookup_exact(&handle.identity")); + UT_ASSERT_NOT_NULL( + strstr(driver, "cluster_gcs_pcm_x_role_refresh_exact(&handle, &fresh_handle)")); + UT_ASSERT_NOT_NULL(strstr(driver, "initial_own.pcm_state != (uint8)PCM_STATE_X")); + if (follower_snapshot != NULL && graph_replace != NULL && graph_clear != NULL) + UT_ASSERT(graph_clear < follower_snapshot && follower_snapshot < graph_replace + && graph_replace < driver_end); + if (self_adopt != NULL && remote_fetch != NULL && install_ready != NULL) + UT_ASSERT(self_adopt < install_ready && remote_fetch < install_ready); + if (self_finish != NULL && remote_finish != NULL && final_ack != NULL) + UT_ASSERT(self_finish < final_ack && remote_finish < final_ack); + if (final_confirm != NULL && clear_wait != NULL) + UT_ASSERT(final_confirm < clear_wait && clear_wait < driver_end); + free(source); +} + + +UT_TEST(test_pcm_x_requester_retry_policy_is_operation_exact) +{ + PcmXRuntimeSnapshot current; + PcmXRuntimeSnapshot start; + const struct { + GcsBlockPcmXRequesterSite site; + PcmXQueueResult result; + GcsBlockPcmXRetryAction action; + } cases[] = { + { GCS_BLOCK_PCM_X_RETRY_SITE_JOIN, PCM_X_QUEUE_NO_CAPACITY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_JOIN, PCM_X_QUEUE_STALE, GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, PCM_X_QUEUE_BUSY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_LEADER_REKEY, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, PCM_X_QUEUE_BUSY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, PCM_X_QUEUE_GATE_RETRY, + GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_ROLE_REFRESH, PCM_X_QUEUE_NOT_FOUND, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, PCM_X_QUEUE_BUSY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_CLAIM, PCM_X_QUEUE_NO_CAPACITY, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF, PCM_X_QUEUE_GATE_RETRY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_CUTOFF, PCM_X_QUEUE_BAD_STATE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_FOLLOWER_SNAPSHOT, PCM_X_QUEUE_BARRIER_CLOSED, + GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_FOLLOWER_SNAPSHOT, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_WFG_COMMIT, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_RESNAPSHOT_WFG }, + { GCS_BLOCK_PCM_X_RETRY_SITE_WFG_COMMIT, PCM_X_QUEUE_CORRUPT, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_WFG_CLEAR, PCM_X_QUEUE_NOT_READY, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, PCM_X_QUEUE_NOT_READY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, PCM_X_QUEUE_BUSY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_PROGRESS, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, PCM_X_QUEUE_BAD_STATE, + GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS }, + { GCS_BLOCK_PCM_X_RETRY_SITE_PRECOMMIT_ARM, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, PCM_X_QUEUE_BUSY, GCS_BLOCK_PCM_X_RETRY_WAIT }, + { GCS_BLOCK_PCM_X_RETRY_SITE_IMAGE_FETCH, PCM_X_QUEUE_BARRIER_CLOSED, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM, PCM_X_QUEUE_NOT_READY, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_ARM, PCM_X_QUEUE_BAD_STATE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + { GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, PCM_X_QUEUE_BAD_STATE, + GCS_BLOCK_PCM_X_RETRY_RELOAD_PROGRESS }, + { GCS_BLOCK_PCM_X_RETRY_SITE_POSTCOMMIT_REPLAY_ARM, PCM_X_QUEUE_STALE, + GCS_BLOCK_PCM_X_RETRY_FAIL_CLOSED }, + }; + size_t i; + + for (i = 0; i < lengthof(cases); i++) + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_retry_action(cases[i].site, cases[i].result), + cases[i].action); + + UT_ASSERT(cluster_gcs_pcm_x_nested_guard_retryable(PCM_X_QUEUE_BUSY)); + UT_ASSERT(cluster_gcs_pcm_x_nested_guard_retryable(PCM_X_QUEUE_GATE_RETRY)); + UT_ASSERT(cluster_gcs_pcm_x_nested_guard_retryable(PCM_X_QUEUE_STALE)); + UT_ASSERT(!cluster_gcs_pcm_x_nested_guard_retryable(PCM_X_QUEUE_BARRIER_CLOSED)); + UT_ASSERT(!cluster_gcs_pcm_x_nested_guard_retryable(PCM_X_QUEUE_NOT_READY)); + UT_ASSERT(!cluster_gcs_pcm_x_nested_guard_retryable(PCM_X_QUEUE_CORRUPT)); + + memset(&start, 0, sizeof(start)); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(&start), + GCS_BLOCK_PCM_X_FORMATION_WAIT); + start.state = PCM_X_RUNTIME_ACTIVE; + start.gate_generation = 7; + start.master_session_incarnation = 11; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(&start), + GCS_BLOCK_PCM_X_FORMATION_PROCEED); + current = start; + UT_ASSERT(cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current = start; + current.gate_generation = 0; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(¤t), + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED); + current = start; + current.master_session_incarnation = 0; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(¤t), + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED); + current = start; + current.gate_generation++; + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + current = start; + current.state = PCM_X_RUNTIME_RECOVERY_BLOCKED; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(¤t), + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED); + UT_ASSERT(!cluster_gcs_pcm_x_requester_runtime_exact(&start, ¤t)); + memset(¤t, 0, sizeof(current)); + current.gate_generation = 1; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(¤t), + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED); + memset(¤t, 0, sizeof(current)); + current.master_session_incarnation = 1; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(¤t), + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED); + current = start; + current.state = PCM_X_RUNTIME_SHUTTING_DOWN; + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_formation_action(¤t), + GCS_BLOCK_PCM_X_FORMATION_FAIL_CLOSED); +} + + +UT_TEST(test_pcm_x_remote_reservation_preflight_binds_identity_base) +{ + ClusterPcmOwnSnapshot live; + PcmXWaitIdentity identity; + + memset(&live, 0, sizeof(live)); + memset(&identity, 0, sizeof(identity)); + identity.tag.spcOid = 7; + identity.tag.dbOid = 8; + identity.tag.relNumber = 9; + identity.tag.forkNum = MAIN_FORKNUM; + identity.tag.blockNum = 10; + live.tag = identity.tag; + live.generation = 41; + live.reservation_token = 12; + live.pcm_state = (uint8)PCM_STATE_N; + identity.base_own_generation = live.generation; + + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), PCM_X_QUEUE_OK); + live.generation++; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_STALE); + live.generation = identity.base_own_generation; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_BUSY); + live.flags = PCM_OWN_FLAG_REVOKING; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_BUSY); + live.flags = PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_CORRUPT); + live.flags = UINT32_C(0x80); + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_CORRUPT); + live.flags = 0; + live.pcm_state = (uint8)PCM_STATE_S; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_STALE); + live.pcm_state = (uint8)PCM_STATE_N; + live.tag.blockNum++; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_STALE); + live.tag = identity.tag; + identity.base_own_generation = UINT64_MAX; + live.generation = UINT64_MAX; + UT_ASSERT_EQ(cluster_gcs_pcm_x_remote_reservation_preflight(&live, &identity), + PCM_X_QUEUE_COUNTER_EXHAUSTED); +} + + +UT_TEST(test_pcm_x_requester_wait_backoff_saturates) +{ + uint32 wait_index = 0; + uint32 i; + + for (i = 0; i < CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS + 4; i++) + wait_index = cluster_gcs_pcm_x_requester_wait_index_advance(wait_index); + UT_ASSERT_EQ(wait_index, CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS - 1); + UT_ASSERT_EQ(cluster_pcm_x_holder_retry_delay_ms(wait_index), + cluster_pcm_x_holder_retry_delay_ms(CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS + 20)); +} + + +UT_TEST(test_pcm_x_requester_cleanup_never_guesses_after_irreversible_boundary) +{ + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_cleanup_action(false, false, false, false), + GCS_BLOCK_PCM_X_CLEANUP_NONE); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_cleanup_action(true, false, false, false), + GCS_BLOCK_PCM_X_CLEANUP_CANCEL_LOCAL); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_cleanup_action(true, true, false, false), + GCS_BLOCK_PCM_X_CLEANUP_CANCEL_LOCAL); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_cleanup_action(false, true, false, false), + GCS_BLOCK_PCM_X_CLEANUP_PRESERVE_FAIL_CLOSED); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_cleanup_action(true, true, true, false), + GCS_BLOCK_PCM_X_CLEANUP_PRESERVE_FAIL_CLOSED); + UT_ASSERT_EQ(cluster_gcs_pcm_x_requester_cleanup_action(true, true, false, true), + GCS_BLOCK_PCM_X_CLEANUP_PRESERVE_FAIL_CLOSED); +} + + +UT_TEST(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve) +{ + char *source = read_gcs_block_source(); + const char *common; + const char *common_end; + const char *allocate; + const char *allocate_after; + const char *collect; + const char *exact_wake; + const char *wake; + const char *wake_end; + const char *read; + const char *match; + const char *set_latch; + const char *writer_release; + const char *writer_release_end; + const char *writer_collect; + const char *writer_wake; + const char *writer_allocate; + const char *writer_cleanup; + const char *writer_cleanup_end; + const char *writer_cleanup_try; + const char *writer_cleanup_exact; + const char *writer_cleanup_catch; + const char *writer_cleanup_flush; + const char *writer_cleanup_fail_closed; + const char *remote; + const char *remote_end; + const char *remote_apply; + const char *remote_ack; + const char *terminal; + const char *terminal_end; + const char *self_apply; + const char *self_resolve; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + wake = strstr(source, "\ngcs_block_pcm_x_wake_requester_exact("); + wake_end = wake != NULL ? strstr(wake, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(wake); + UT_ASSERT_NOT_NULL(wake_end); + if (wake != NULL && wake_end != NULL) { + read = strstr(wake, "cluster_lmd_wait_state_read_exact("); + match = read != NULL ? strstr(read, "cluster_gcs_pcm_x_wait_identity_matches(") : NULL; + set_latch = match != NULL ? strstr(match, "SetLatch(") : NULL; + UT_ASSERT_NOT_NULL(read); + UT_ASSERT_NOT_NULL(match); + UT_ASSERT_NOT_NULL(set_latch); + if (read != NULL && match != NULL && set_latch != NULL) + UT_ASSERT(read < match && match < set_latch && set_latch < wake_end); + } + writer_release = strstr(source, "\ncluster_gcs_pcm_x_writer_claim_release_and_wake_exact("); + writer_release_end = writer_release != NULL ? strstr(writer_release, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(writer_release); + UT_ASSERT_NOT_NULL(writer_release_end); + if (writer_release != NULL && writer_release_end != NULL) { + writer_collect + = strstr(writer_release, "cluster_pcm_x_local_writer_claim_release_collect_exact("); + writer_wake = writer_collect != NULL + ? strstr(writer_collect, "gcs_block_pcm_x_wake_requester_exact(") + : NULL; + writer_allocate = strstr(writer_release, "palloc"); + UT_ASSERT_NOT_NULL(writer_collect); + UT_ASSERT_NOT_NULL(writer_wake); + if (writer_collect != NULL && writer_wake != NULL) + UT_ASSERT(writer_collect < writer_wake && writer_wake < writer_release_end); + UT_ASSERT(writer_allocate == NULL || writer_allocate > writer_release_end); + } + writer_cleanup = strstr(source, "\ncluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept("); + writer_cleanup_end = writer_cleanup != NULL ? strstr(writer_cleanup, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(writer_cleanup); + UT_ASSERT_NOT_NULL(writer_cleanup_end); + if (writer_cleanup != NULL && writer_cleanup_end != NULL) { + writer_cleanup_try = strstr(writer_cleanup, "PG_TRY();"); + writer_cleanup_exact + = writer_cleanup_try != NULL + ? strstr(writer_cleanup_try, + "cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(claim)") + : NULL; + writer_cleanup_catch + = writer_cleanup_exact != NULL ? strstr(writer_cleanup_exact, "PG_CATCH();") : NULL; + writer_cleanup_flush = writer_cleanup_catch != NULL + ? strstr(writer_cleanup_catch, "FlushErrorState();") + : NULL; + writer_cleanup_fail_closed + = writer_cleanup_flush != NULL + ? strstr(writer_cleanup_flush, "cluster_pcm_x_runtime_fail_closed();") + : NULL; + UT_ASSERT_NOT_NULL(writer_cleanup_try); + UT_ASSERT_NOT_NULL(writer_cleanup_exact); + UT_ASSERT_NOT_NULL(writer_cleanup_catch); + UT_ASSERT_NOT_NULL(writer_cleanup_flush); + UT_ASSERT_NOT_NULL(writer_cleanup_fail_closed); + if (writer_cleanup_try != NULL && writer_cleanup_exact != NULL + && writer_cleanup_catch != NULL && writer_cleanup_flush != NULL + && writer_cleanup_fail_closed != NULL) + UT_ASSERT(writer_cleanup_try < writer_cleanup_exact + && writer_cleanup_exact < writer_cleanup_catch + && writer_cleanup_catch < writer_cleanup_flush + && writer_cleanup_flush < writer_cleanup_fail_closed + && writer_cleanup_fail_closed < writer_cleanup_end); + } + common = strstr(source, "\ngcs_block_pcm_x_local_retire_apply_and_wake("); + common_end = common != NULL ? strstr(common, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(common); + UT_ASSERT_NOT_NULL(common_end); + if (common != NULL && common_end != NULL) { + allocate = strstr(common, "palloc0("); + collect = allocate != NULL + ? strstr(allocate, "cluster_pcm_x_local_retire_up_to_collect_exact(") + : NULL; + exact_wake + = collect != NULL ? strstr(collect, "gcs_block_pcm_x_wake_requester_exact(") : NULL; + allocate_after = allocate != NULL ? strstr(allocate + 1, "palloc0(") : NULL; + UT_ASSERT_NOT_NULL(allocate); + UT_ASSERT_NOT_NULL(collect); + UT_ASSERT_NOT_NULL(exact_wake); + if (allocate != NULL && collect != NULL && exact_wake != NULL) + UT_ASSERT(allocate < collect && collect < exact_wake && exact_wake < common_end); + UT_ASSERT(allocate_after == NULL || allocate_after > common_end); + } + remote = strstr(source, "\ncluster_gcs_handle_pcm_x_retire_up_to_envelope("); + remote_end = remote != NULL ? strstr(remote, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(remote); + UT_ASSERT_NOT_NULL(remote_end); + if (remote != NULL && remote_end != NULL) { + remote_apply = strstr(remote, "gcs_block_pcm_x_local_retire_apply_and_wake("); + remote_ack = remote_apply != NULL ? strstr(remote_apply, "gcs_block_pcm_x_send_retire_ack(") + : NULL; + UT_ASSERT_NOT_NULL(remote_apply); + UT_ASSERT_NOT_NULL(remote_ack); + if (remote_apply != NULL && remote_ack != NULL) + UT_ASSERT(remote_apply < remote_ack && remote_ack < remote_end); + } + terminal = strstr(source, "\ncluster_gcs_pcm_x_terminal_kick("); + terminal_end = terminal != NULL ? strstr(terminal, "\n}\n\n\nstatic") : NULL; + UT_ASSERT_NOT_NULL(terminal); + UT_ASSERT_NOT_NULL(terminal_end); + if (terminal != NULL && terminal_end != NULL) { + self_apply = strstr(terminal, "gcs_block_pcm_x_local_retire_apply_and_wake("); + self_resolve = self_apply != NULL + ? strstr(self_apply, "cluster_pcm_x_master_retire_ack_resolve_exact(") + : NULL; + UT_ASSERT_NOT_NULL(self_apply); + UT_ASSERT_NOT_NULL(self_resolve); + if (self_apply != NULL && self_resolve != NULL) + UT_ASSERT(self_apply < self_resolve && self_resolve < terminal_end); + } + UT_ASSERT_EQ(count_occurrences(source, "gcs_block_pcm_x_local_retire_apply_and_wake("), 3); + UT_ASSERT_EQ(count_occurrences(source, "cluster_pcm_x_local_retire_up_to_collect_exact("), 1); + free(source); +} + + +UT_TEST(test_pcm_x_tagless_retire_uses_explicit_data_plane_handoff) +{ + char *source = read_gcs_block_source(); + const char *handler; + const char *handler_end; + const char *kick; + const char *kick_end; + const char *ack_send; + const char *request_stage; + + UT_ASSERT_NOT_NULL(source); + if (source == NULL) + return; + handler = strstr(source, "\ncluster_gcs_handle_pcm_x_retire_up_to_envelope("); + handler_end = handler != NULL ? strstr(handler + 1, "\n}\n\n\n") : NULL; + kick = strstr(source, "\ncluster_gcs_pcm_x_terminal_kick("); + kick_end = kick != NULL ? strstr(kick + 1, "\n}\n\n\nstatic") : NULL; + UT_ASSERT_NOT_NULL(handler); + UT_ASSERT_NOT_NULL(handler_end); + UT_ASSERT_NOT_NULL(kick); + UT_ASSERT_NOT_NULL(kick_end); + ack_send = handler != NULL ? strstr(handler, "gcs_block_pcm_x_send_retire_ack(") : NULL; + request_stage = kick != NULL ? strstr(kick, "gcs_block_pcm_x_stage_retire_up_to(") : NULL; + UT_ASSERT_NOT_NULL(ack_send); + UT_ASSERT_NOT_NULL(request_stage); + if (handler_end != NULL && ack_send != NULL) + UT_ASSERT(ack_send < handler_end); + if (kick_end != NULL && request_stage != NULL) + UT_ASSERT(request_stage < kick_end); + free(source); +} + + +UT_TEST(test_pcm_x_role_refresh_accepts_only_same_member_promotion) +{ + PcmXLocalHandle follower; + PcmXLocalHandle promoted; + + memset(&follower, 0, sizeof(follower)); + follower.identity.node_id = 1; + follower.identity.procno = 7; + follower.identity.xid = 11; + follower.identity.cluster_epoch = 13; + follower.identity.request_id = 17; + follower.identity.wait_seq = 19; + follower.identity.base_own_generation = 23; + follower.tag_slot.slot_index = 29; + follower.tag_slot.slot_generation = 31; + follower.membership_slot.slot_index = 37; + follower.membership_slot.slot_generation = 41; + follower.local_sequence = 43; + follower.local_round = 47; + follower.role = PCM_X_LOCAL_ROLE_FOLLOWER; + promoted = follower; + promoted.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + + UT_ASSERT(cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); + promoted.role = PCM_X_LOCAL_ROLE_FOLLOWER; + UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); + promoted = follower; + promoted.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + promoted.membership_slot.slot_generation++; + UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); + promoted = follower; + promoted.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + promoted.local_sequence++; + UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); + promoted = follower; + promoted.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + promoted.local_round++; + UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); + promoted = follower; + promoted.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + promoted.identity.base_own_generation++; + UT_ASSERT(!cluster_gcs_pcm_x_role_refresh_exact(&follower, &promoted)); +} + + int main(void) { - UT_PLAN(25); + UT_PLAN(76); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -657,6 +3259,7 @@ main(void) UT_RUN(test_gcs_block_lwlock_tranche_distinct); UT_RUN(test_gcs_block_wait_events_distinct); UT_RUN(test_gcs_block_reply_total_size_is_8240); + UT_RUN(test_pcm_x_session_auth_sample_classifies_epoch_zero_and_torn_reads); UT_RUN(test_gcs_block_reply_key_is_compound); UT_RUN(test_gcs_block_reserved_padding_present); UT_RUN(test_gcs_block_data_size_equals_blcksz); @@ -672,6 +3275,56 @@ main(void) UT_RUN(test_forward_payload_undo_verdict_kinds_no_collision); UT_RUN(test_undo_authority_fetch_tag_owner_roundtrip); UT_RUN(test_undo_verdict_version_authority_distinct); + UT_RUN(test_pcm_x_enqueue_ingress_binds_transport_epoch_and_master); + UT_RUN(test_pcm_x_admit_confirm_ingress_binds_requester_and_master); + UT_RUN(test_pcm_x_admit_confirm_ack_binds_exact_master_source); + UT_RUN(test_pcm_x_cancel_requests_bind_exact_source_epoch_master_and_phase); + UT_RUN(test_pcm_x_cancel_acks_bind_exact_master_and_canonical_payload); + UT_RUN(test_pcm_x_wait_identity_maps_to_real_wfg_vertex); + UT_RUN(test_pcm_x_initial_epoch_zero_is_exact_across_wire_classes); + UT_RUN(test_pcm_x_blocker_header_ingress_binds_master_not_requester_source); + UT_RUN(test_pcm_x_blocker_edge_ingress_binds_blocker_to_holder_source); + UT_RUN(test_pcm_x_blocker_ack_carries_full_generation_and_binds_master_source); + UT_RUN(test_pcm_x_formation_identical_complete_samples_may_revalidate); + UT_RUN(test_pcm_x_formation_transient_or_inconsistent_sample_is_tick_noop); + UT_RUN(test_pcm_x_confirm_publish_then_stale_requires_exact_graph_close); + UT_RUN(test_pcm_x_drain_poll_binds_exact_master_and_generation); + UT_RUN(test_pcm_x_drain_ack_binds_participant_and_canonical_payload); + UT_RUN(test_pcm_x_retire_request_binds_master_session_and_target); + UT_RUN(test_pcm_x_retire_ack_binds_responder_and_master_authority); + UT_RUN(test_pcm_x_revoke_ingress_binds_master_and_exact_transfer_key); + UT_RUN(test_pcm_x_image_ready_ingress_binds_holder_image_to_master); + UT_RUN(test_pcm_x_prepare_grant_ingress_binds_master_to_requester); + UT_RUN(test_pcm_x_install_ready_ingress_is_canonical_requester_ack); + UT_RUN(test_pcm_x_commit_x_ingress_is_canonical_master_phase); + UT_RUN(test_pcm_x_final_ack_ingress_binds_exact_committed_generation); + UT_RUN(test_pcm_x_final_commit_ack_ingress_is_canonical_master_phase); + UT_RUN(test_pcm_x_final_confirm_ingress_is_canonical_requester_phase); + UT_RUN(test_pcm_x_master_drive_selects_exact_authority_and_next_holder); + UT_RUN(test_pcm_x_master_drive_wiring_binds_grd_barrier_to_exact_ticket); + UT_RUN(test_pcm_x_cancel_cleanup_classifies_exact_wfg_and_post_clear_failure); + UT_RUN(test_pcm_x_terminal_retry_reclaims_cancel_cleanup_after_owner_death); + UT_RUN(test_pcm_x_invalidate_ack_matches_only_exact_unacked_holder); + UT_RUN(test_pcm_x_final_ack_builds_exact_grd_handoff_token); + UT_RUN(test_pcm_x_holder_image_evidence_never_uses_generation_as_presence); + UT_RUN(test_pcm_x_pending_x_marker_is_only_a_pre_handoff_gate); + UT_RUN(test_pcm_x_ready_publication_follows_exact_retained_commit); + UT_RUN(test_pcm_x_ready_materializes_exact_n_s_or_x_source_without_wire_change); + UT_RUN(test_pcm_x_self_and_remote_drain_share_full_image_release_wrapper); + UT_RUN(test_pcm_x_ready_admission_marks_before_send_and_rolls_back_refusal); + UT_RUN(test_pcm_x_lms_owner_death_and_restart_audit_fail_closed); + UT_RUN(test_pcm_x_image_fetch_intercepts_canonical_id_before_generic_dedup); + UT_RUN(test_pcm_x_requester_fetch_revalidates_queue_and_reservation_before_install); + UT_RUN(test_pcm_x_self_source_handoff_is_no_copy_and_drain_preserves_x); + UT_RUN(test_pcm_x_retire_wake_identity_is_wait_generation_exact); + UT_RUN(test_pcm_x_requester_driver_owns_fifo_and_transfer_lifecycles); + UT_RUN(test_pcm_x_requester_retry_policy_is_operation_exact); + UT_RUN(test_pcm_x_remote_reservation_preflight_binds_identity_base); + UT_RUN(test_pcm_x_requester_wait_backoff_saturates); + UT_RUN(test_pcm_x_requester_cleanup_never_guesses_after_irreversible_boundary); + UT_RUN(test_pcm_x_retire_commit_wakes_exact_waiters_before_ack_or_resolve); + UT_RUN(test_pcm_x_tagless_retire_uses_explicit_data_plane_handoff); + UT_RUN(test_pcm_x_role_refresh_accepts_only_same_member_promotion); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_3way.c b/src/test/cluster_unit/test_cluster_gcs_block_3way.c index 6bc03e1fe28..b929b77267f 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_3way.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_3way.c @@ -227,7 +227,7 @@ UT_TEST(test_s_barrier_set_pending_x_prototype_linkable) /* Compile-time guard that the prototype matches the header decl. * NULL call is unreachable at runtime but ensures the symbol * compiles into the test binary's external reference table. */ - void (*fp)(BufferTag, int32, uint64) = &cluster_pcm_lock_set_pending_x; + PcmPendingXReserveResult (*fp)(BufferTag, int32, uint64) = &cluster_pcm_lock_set_pending_x; UT_ASSERT(fp != NULL); } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c index 730a3100ab0..ee48f05f3ed 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_dedup.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_dedup.c @@ -59,6 +59,7 @@ #include "cluster/cluster_gcs_block_dedup.h" #include "cluster/cluster_lms_shard.h" #include "cluster/cluster_shmem.h" +#include "port/pg_crc32c.h" #include "storage/buf_internals.h" #include "storage/ipc.h" #include "storage/lwlock.h" @@ -124,6 +125,8 @@ static FakeDedupShardHtab fake_htab[CLUSTER_LMS_MAX_WORKERS]; static int fake_htab_init_seq; static Size fake_keysize; static Size fake_entrysize; +static int fake_hash_seq_init_count; +static int fake_hash_seq_term_count; /* ShmemInitStruct blob for the dedup ctl header + per-shard structs. */ static union { @@ -142,6 +145,8 @@ reset_fake_dedup(int n_workers, int max_entries) fake_htab_init_seq = 0; fake_keysize = 0; fake_entrysize = 0; + fake_hash_seq_init_count = 0; + fake_hash_seq_term_count = 0; memset(&fake_dedup_struct, 0, sizeof(fake_dedup_struct)); fake_dedup_struct_found = false; fake_before_shmem_exit_registered = 0; @@ -231,6 +236,7 @@ hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr) void hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp) { + fake_hash_seq_init_count++; status->hashp = hashp; status->curBucket = 0; status->curEntry = NULL; @@ -241,14 +247,18 @@ hash_seq_search(HASH_SEQ_STATUS *status) { FakeDedupShardHtab *h = (FakeDedupShardHtab *)status->hashp; - if (status->curBucket >= (uint32)h->count) + if (status->curBucket >= (uint32)h->count) { + hash_seq_term(status); return NULL; + } return h->entries[status->curBucket++]; } void hash_seq_term(HASH_SEQ_STATUS *status pg_attribute_unused()) -{} +{ + fake_hash_seq_term_count++; +} Size hash_estimate_size(long num_entries, Size entrysize) @@ -376,6 +386,98 @@ install_granted(int worker_id, const GcsBlockDedupKey *key) cluster_gcs_block_dedup_install_reply(worker_id, key, GCS_BLOCK_REPLY_GRANTED, &hdr, block); } +static GcsBlockPcmXImageBinding +make_pcm_x_binding(BufferTag tag, uint32 requester_node, uint32 requester_procno, + uint64 requester_request_id, uint64 epoch, uint64 image_id, + uint64 master_session) +{ + GcsBlockPcmXImageBinding binding; + + memset(&binding, 0, sizeof(binding)); + binding.identity.ref.identity.tag = tag; + binding.identity.ref.identity.node_id = (int32)requester_node; + binding.identity.ref.identity.procno = requester_procno; + binding.identity.ref.identity.xid = (TransactionId)17; + binding.identity.ref.identity.cluster_epoch = epoch; + binding.identity.ref.identity.request_id = requester_request_id; + binding.identity.ref.identity.wait_seq = 19; + binding.identity.ref.identity.base_own_generation = 23; + binding.identity.ref.handle.ticket_id = 29; + binding.identity.ref.handle.queue_generation = 31; + binding.identity.ref.grant_generation = 37; + binding.identity.image.image_id = image_id; + binding.identity.image.source_own_generation = 41; + binding.identity.image.page_scn = 43; + binding.identity.image.page_lsn = 47; + binding.identity.image.source_node = 0; + binding.identity.image.page_checksum = 53; + binding.master_session = master_session; + return binding; +} + +static GcsBlockReplyHeader +make_pcm_x_reply_header(const GcsBlockDedupKey *key, const GcsBlockPcmXImageBinding *binding) +{ + GcsBlockReplyHeader hdr; + + memset(&hdr, 0, sizeof(hdr)); + hdr.request_id = key->request_id; + hdr.page_lsn = binding->identity.image.page_lsn; + hdr.epoch = key->cluster_epoch; + hdr.checksum = binding->identity.image.page_checksum; + hdr.sender_node = (int32)binding->identity.image.source_node; + hdr.requester_backend_id = key->requester_backend_id; + hdr.transition_id = (uint8)PCM_TRANS_N_TO_S; + hdr.status = (uint8)GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; + GcsBlockReplyHeaderSetForwardingMasterNode(&hdr, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + return hdr; +} + +static uint32 +pcm_x_test_block_checksum(const char *page) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, page, GCS_BLOCK_DATA_SIZE); + FIN_CRC32C(crc); + return (uint32)crc; +} + +static void +prepare_pcm_x_page(char *page, GcsBlockPcmXImageBinding *binding, GcsBlockReplyHeader *hdr) +{ + PageHeaderData page_header; + + memcpy(&page_header, page, sizeof(page_header)); + PageXLogRecPtrSet(page_header.pd_lsn, (XLogRecPtr)binding->identity.image.page_lsn); + page_header.pd_block_scn = (SCN)binding->identity.image.page_scn; + memcpy(page, &page_header, sizeof(page_header)); + binding->identity.image.page_checksum = pcm_x_test_block_checksum(page); + hdr->page_lsn = binding->identity.image.page_lsn; + hdr->checksum = binding->identity.image.page_checksum; +} + +static GcsBlockPcmXImageResult +stage_pcm_x_ready(int worker_id, const GcsBlockDedupKey *key, const BufferTag *tag, + const GcsBlockPcmXImageBinding *binding, const GcsBlockReplyHeader *hdr, + const char *page) +{ + GcsBlockPcmXImageBinding reserved = *binding; + GcsBlockPcmXImageResult result; + + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + result = cluster_gcs_block_dedup_pcm_x_reserve(worker_id, key, tag, &reserved); + if (result != GCS_BLOCK_PCM_X_IMAGE_RESERVED && result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + return result; + result = cluster_gcs_block_dedup_pcm_x_materialize(worker_id, key, tag, binding, hdr, page); + if (result != GCS_BLOCK_PCM_X_IMAGE_STORED && result != GCS_BLOCK_PCM_X_IMAGE_DUPLICATE) + return result; + return cluster_gcs_block_dedup_pcm_x_publish_ready_exact(worker_id, key, tag, binding); +} + /* ============================================================ * U1 — per-worker isolation: an install on shard 0 is invisible to @@ -947,10 +1049,713 @@ UT_TEST(u16_capability_routing_truth_table) UT_ASSERT_EQ((int)cluster_gcs_block_dedup_get_in_flight_count(), 0); } + +/* PCM-X uses the existing 8KB entry payload without moving any established + * field or increasing the entry size. */ +UT_TEST(u17_pcm_x_binding_layout_is_zero_entry_growth) +{ + UT_ASSERT_EQ((int)sizeof(GcsBlockPcmXImageBinding), 136); + UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, entry_kind), 46); + UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, pcm_x_master_session), 48); + UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, reply_header), 56); + UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, payload_meta), 112); + UT_ASSERT_EQ((int)sizeof(((GcsBlockDedupEntry *)0)->payload_meta), 128); + UT_ASSERT_EQ((int)sizeof(GcsBlockDedupEntry), 8472); +} + + +/* An exact duplicate reuses immutable bytes. A generic install with the + * same key must not overwrite a staged PCM-X image. */ +UT_TEST(u18_pcm_x_stage_duplicate_and_generic_overwrite_refused) +{ + BufferTag tag = make_tag(110); + uint64 requester_id = gcs_reqid_requester(1, 2, 77); + uint64 image_id; + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding conflicting_binding; + GcsBlockReplyHeader hdr; + GcsBlockReplyHeader conflicting_hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + char overwrite[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 7, &image_id)); + key = make_key(1, 3, image_id, 0); + binding = make_pcm_x_binding(tag, 1, 5, requester_id, 0, image_id, 101); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x6a, sizeof(page)); + memset(overwrite, 0x7b, sizeof(overwrite)); + prepare_pcm_x_page(page, &binding, &hdr); + conflicting_binding = binding; + conflicting_binding.identity.image.page_scn++; + conflicting_binding.identity.image.page_lsn++; + conflicting_hdr = make_pcm_x_reply_header(&key, &conflicting_binding); + prepare_pcm_x_page(overwrite, &conflicting_binding, &conflicting_hdr); + + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &conflicting_binding, + &conflicting_hdr, overwrite), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + + /* The generic completion path cannot mutate a dedicated entry. */ + cluster_gcs_block_dedup_install_reply(0, &key, GCS_BLOCK_REPLY_GRANTED, &hdr, overwrite); + memset(&cached, 0, sizeof(cached)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT(memcmp(cached.block_data, page, sizeof(page)) == 0); + UT_ASSERT( + memcmp(&cached.payload_meta.pcm_x_identity, &binding.identity, sizeof(binding.identity)) + == 0); + UT_ASSERT_EQ((uint64)cached.pcm_x_master_session, (uint64)binding.master_session); + UT_ASSERT_EQ((int)cached.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_PCM_X_IMAGE); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_replay_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 2); +} + + +/* Generic lookup, remove and DONE must all reject a dedicated image. */ +UT_TEST(u19_pcm_x_entry_isolated_from_generic_lifecycle) +{ + BufferTag tag = make_tag(111); + uint64 image_id; + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockReplyHeader hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 8, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 78), 13, image_id, 102); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x5c, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, 1, + true, &cached), + (int)GCS_BLOCK_DEDUP_VALIDATION_FAIL); + cluster_gcs_block_dedup_remove(0, &key); + UT_ASSERT(!cluster_gcs_block_dedup_mark_done(0, &key, &tag, PCM_TRANS_N_TO_S)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 3); +} + + +/* Wall-clock, backend exit and node death are not application ACKs. Only + * the exact terminal binding may retire the image. */ +UT_TEST(u20_pcm_x_entry_survives_generic_gc_and_retires_exactly) +{ + BufferTag tag = make_tag(112); + uint64 image_id; + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding wrong; + GcsBlockReplyHeader hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 9, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 79), 13, image_id, 103); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x4d, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + cluster_gcs_block_dedup_sweep_expired((TimestampTz)INT64_MAX); + cluster_gcs_block_dedup_cleanup_on_backend_exit(1, 3); + cluster_gcs_block_dedup_cleanup_on_node_dead(1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + + wrong = binding; + wrong.identity.ref.grant_generation++; + memset(&cached, 0x5a, sizeof(cached)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &wrong, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cached.entry_kind, (int)GCS_BLOCK_DEDUP_ENTRY_GENERIC); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &wrong), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_release_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 3); +} + + +/* A full shared shard refuses staging without reclaiming live generic or + * dedicated entries. */ +UT_TEST(u21_pcm_x_stage_full_is_fail_closed) +{ + BufferTag tag = make_tag(113); + GcsBlockDedupEntry cached; + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockReplyHeader hdr; + uint64 image_id; + char page[GCS_BLOCK_DATA_SIZE]; + int i; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + for (i = 0; i < FAKE_DEDUP_CAP; i++) { + GcsBlockDedupKey ordinary = make_key(0, 1, (uint64)(2000 + i), 13); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register( + 0, &ordinary, tag, PCM_TRANS_N_TO_S, 0, false, &cached), + (int)GCS_BLOCK_DEDUP_MISS_REGISTERED); + } + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 10, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 80), 13, image_id, 104); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x3e, sizeof(page)); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_FULL); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_get_in_flight_count(), FAKE_DEDUP_CAP); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_full_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 1); +} + + +/* Capacity reservation is durable protocol evidence: generic time and + * process-lifecycle cleanup cannot retire it, and only its exact binding can. */ +UT_TEST(u22_pcm_x_reserved_entry_waits_for_exact_release) +{ + BufferTag tag = make_tag(114); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockDedupEntry cached; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 11, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 81), 13, image_id, 105); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &reserved, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + + cluster_gcs_block_dedup_sweep_expired((TimestampTz)INT64_MAX); + cluster_gcs_block_dedup_cleanup_on_backend_exit(1, 3); + cluster_gcs_block_dedup_cleanup_on_node_dead(1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &reserved, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 0); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_replay_count(), 0); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_release_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 2); +} + + +/* READY publication validates every byte carrier before changing RESERVED: + * local source, reply binding, page CRC, page LSN, page SCN and master session. */ +UT_TEST(u23_pcm_x_materialize_validation_is_fail_closed_and_byte_stable) +{ + BufferTag tag = make_tag(115); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockPcmXImageBinding bad_binding; + GcsBlockReplyHeader hdr; + GcsBlockReplyHeader bad_hdr; + char page[GCS_BLOCK_DATA_SIZE]; + char bad_page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 12, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 82), 13, image_id, 106); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x2d, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + + bad_binding = binding; + bad_binding.identity.image.source_node = 1; + bad_hdr = hdr; + bad_hdr.sender_node = 1; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + + bad_hdr = hdr; + bad_hdr.sender_node = 1; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + + memcpy(bad_page, page, sizeof(bad_page)); + bad_page[sizeof(bad_page) - 1] ^= 0x1; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, bad_page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + + bad_binding = binding; + bad_binding.identity.image.page_lsn++; + bad_hdr = hdr; + bad_hdr.page_lsn++; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + + bad_binding = binding; + bad_binding.identity.image.page_scn++; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + + bad_binding = binding; + bad_binding.identity.image.page_checksum++; + bad_hdr = hdr; + bad_hdr.checksum++; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &bad_hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_INVALID); + + bad_binding = binding; + bad_binding.master_session++; + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &bad_binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 7); +} + + +/* A canonical image id is intercepted before generic registration, including + * on a cold miss; there is no legacy fallback entry to complete later. */ +UT_TEST(u24_pcm_x_namespace_cannot_register_as_generic) +{ + BufferTag tag = make_tag(116); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockDedupEntry cached; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 13, &image_id)); + key = make_key(1, 3, image_id, 13); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_lookup_or_register(0, &key, tag, PCM_TRANS_N_TO_S, 1, + true, &cached), + (int)GCS_BLOCK_DEDUP_VALIDATION_FAIL); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 0); + + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 83), 13, image_id, 107); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_failclosed_count(), 1); +} + + +/* The LMS image pump must never let a READY resend monopolize a shard while + * an unmaterialized reservation is waiting. Once a READY leg is admitted to + * the outbound ring it disappears from work scans until an exact type-49 + * retransmit positively re-arms it. */ +UT_TEST(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged) +{ + BufferTag ready_tag = make_tag(117); + BufferTag reserved_tag = make_tag(118); + GcsBlockDedupKey ready_key; + GcsBlockDedupKey reserved_key; + GcsBlockPcmXImageBinding ready_binding; + GcsBlockPcmXImageBinding reserved_binding; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + uint64 ready_image_id; + uint64 reserved_image_id; + char page[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 14, &ready_image_id)); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 15, &reserved_image_id)); + ready_key = make_key(1, 3, ready_image_id, 13); + reserved_key = make_key(1, 4, reserved_image_id, 13); + ready_binding = make_pcm_x_binding(ready_tag, 1, 5, gcs_reqid_requester(1, 2, 84), 13, + ready_image_id, 108); + hdr = make_pcm_x_reply_header(&ready_key, &ready_binding); + memset(page, 0x1d, sizeof(page)); + prepare_pcm_x_page(page, &ready_binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &ready_key, &ready_tag, &ready_binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + reserved_binding = make_pcm_x_binding(reserved_tag, 1, 6, gcs_reqid_requester(1, 3, 85), 13, + reserved_image_id, 109); + reserved_binding.identity.image.page_scn = 0; + reserved_binding.identity.image.page_lsn = 0; + reserved_binding.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, + &reserved_binding), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + + memset(&work, 0, sizeof(work)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ(memcmp(&work.key, &reserved_key, sizeof(reserved_key)), 0); + UT_ASSERT_EQ(memcmp(&work.binding, &reserved_binding, sizeof(reserved_binding)), 0); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_release_exact(0, &reserved_key, &reserved_tag, + &reserved_binding), + (int)GCS_BLOCK_PCM_X_IMAGE_RELEASED); + + memset(&work, 0, sizeof(work)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT_EQ(memcmp(&work.key, &ready_key, sizeof(ready_key)), 0); + UT_ASSERT_EQ(memcmp(&work.binding, &ready_binding, sizeof(ready_binding)), 0); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_mark_staged_exact(0, &ready_key, &ready_tag, + &ready_binding), + (int)GCS_BLOCK_PCM_X_IMAGE_STAGED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_mark_staged_exact(0, &ready_key, &ready_tag, + &ready_binding), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); +} + + +/* A retransmitted type 49 is the only positive evidence that an admitted + * type 50 needs replay. Rearm accepts the reservation identity (page fields + * still zero) but validates the complete ticket/generation/session tuple; + * an almost-equal ticket cannot make a READY image sendable again. */ +UT_TEST(u26_pcm_x_ready_rearm_is_exact) +{ + BufferTag tag = make_tag(119); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockPcmXImageBinding wrong; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + uint64 image_id; + char page[GCS_BLOCK_DATA_SIZE]; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 16, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 86), 13, image_id, 110); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x0d, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_mark_staged_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_STAGED); + + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + wrong = reserved; + wrong.identity.ref.grant_generation++; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_rearm_exact(0, &key, &tag, &wrong), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_rearm_exact(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_REARMED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT_EQ(memcmp(&work.binding, &binding, sizeof(binding)), 0); +} + + +/* One pinned holder must not make the hash table's first RESERVED entry + * monopolize every LMS tick. Work selection is process-local round robin; + * the second scan must advance to the other exact reservation. */ +UT_TEST(u27_pcm_x_reserved_work_scan_rotates) +{ + BufferTag tag_a = make_tag(120); + BufferTag tag_b = make_tag(121); + GcsBlockDedupKey key_a; + GcsBlockDedupKey key_b; + GcsBlockPcmXImageBinding binding_a; + GcsBlockPcmXImageBinding binding_b; + GcsBlockPcmXImageWork first; + GcsBlockPcmXImageWork second; + uint64 image_a; + uint64 image_b; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 17, &image_a)); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 18, &image_b)); + key_a = make_key(1, 3, image_a, 13); + key_b = make_key(1, 4, image_b, 13); + binding_a = make_pcm_x_binding(tag_a, 1, 5, gcs_reqid_requester(1, 2, 87), 13, image_a, 111); + binding_b = make_pcm_x_binding(tag_b, 1, 6, gcs_reqid_requester(1, 3, 88), 13, image_b, 112); + binding_a.identity.image.page_scn = binding_a.identity.image.page_lsn = 0; + binding_a.identity.image.page_checksum = 0; + binding_b.identity.image.page_scn = binding_b.identity.image.page_lsn = 0; + binding_b.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key_a, &tag_a, &binding_a), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key_b, &tag_b, &binding_b), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &first), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &second), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT(memcmp(&first.key, &second.key, sizeof(first.key)) != 0); +} + + +/* The LMS tick must not rescan a potentially multi-thousand-entry generic + * dedup shard forever when no PCM-X byte work exists. One initial scan may + * establish the empty hint; an exact reservation must wake it again. The + * fake scan also mirrors dynahash's natural auto-term, so one full scan has + * exactly one registration and one termination. */ +UT_TEST(u28_pcm_x_idle_hint_avoids_empty_rescan_and_reserve_rearms) +{ + BufferTag tag = make_tag(122); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageWork work; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + UT_ASSERT_EQ(fake_hash_seq_init_count, 1); + UT_ASSERT_EQ(fake_hash_seq_term_count, 1); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + UT_ASSERT_EQ(fake_hash_seq_init_count, 1); + UT_ASSERT_EQ(fake_hash_seq_term_count, 1); + + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 19, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 89), 13, image_id, 113); + binding.identity.image.page_scn = binding.identity.image.page_lsn = 0; + binding.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ(fake_hash_seq_init_count, 2); + UT_ASSERT_EQ(fake_hash_seq_term_count, 2); +} + + +/* Materialized bytes are retained evidence, not a sendable READY image. The + * ownership X->N commit must happen between materialize and the explicit + * publication call; a restarted LMS scanning the intermediate state must not + * synthesize type 50 from it. */ +UT_TEST(u29_pcm_x_materialized_bytes_require_explicit_ready_publication) +{ + BufferTag tag = make_tag(123); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 20, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 90), 13, image_id, 114); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x4e, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_publish_ready_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_pcm_x_stage_count(), 1); +} + + +/* A replacement LMS must never infer progress from retained holder evidence. + * Startup audit is read-only: it detects any dedicated entry and leaves the + * exact bytes/reservation available for the recovery layer. */ +UT_TEST(u30_pcm_x_owner_restart_audit_detects_and_retains_evidence) +{ + BufferTag tag = make_tag(124); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding reserved; + GcsBlockReplyHeader hdr; + GcsBlockDedupEntry cached; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(!cluster_gcs_block_dedup_pcm_x_restart_audit(0)); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 21, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 91), 13, image_id, 115); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x5f, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + reserved = binding; + reserved.identity.image.page_scn = 0; + reserved.identity.image.page_lsn = 0; + reserved.identity.image.page_checksum = 0; + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &key, &tag, &reserved), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ( + (int)cluster_gcs_block_dedup_pcm_x_materialize(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT(cluster_gcs_block_dedup_pcm_x_restart_audit(0)); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_lookup(0, &key, &tag, &binding, &cached), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_READY); + UT_ASSERT_EQ((uint64)cluster_gcs_block_dedup_get_in_flight_count(), 1); +} + + +/* A pinned reservation must not starve an already READY image forever. The + * two work classes share one LMS tick budget, so when both remain runnable a + * READY leg must be selected no later than the tick after RESERVED. */ +UT_TEST(u31_pcm_x_work_classes_alternate_when_both_remain_runnable) +{ + BufferTag ready_tag = make_tag(125); + BufferTag reserved_tag = make_tag(126); + GcsBlockDedupKey ready_key; + GcsBlockDedupKey reserved_key; + GcsBlockPcmXImageBinding ready_binding; + GcsBlockPcmXImageBinding reserved_binding; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 ready_image_id; + uint64 reserved_image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 22, &ready_image_id)); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 23, &reserved_image_id)); + ready_key = make_key(1, 3, ready_image_id, 13); + reserved_key = make_key(1, 4, reserved_image_id, 13); + ready_binding = make_pcm_x_binding(ready_tag, 1, 5, gcs_reqid_requester(1, 2, 92), 13, + ready_image_id, 116); + hdr = make_pcm_x_reply_header(&ready_key, &ready_binding); + memset(page, 0x60, sizeof(page)); + prepare_pcm_x_page(page, &ready_binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &ready_key, &ready_tag, &ready_binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + + reserved_binding = make_pcm_x_binding(reserved_tag, 1, 6, gcs_reqid_requester(1, 3, 93), 13, + reserved_image_id, 117); + reserved_binding.identity.image.page_scn = 0; + reserved_binding.identity.image.page_lsn = 0; + reserved_binding.identity.image.page_checksum = 0; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_reserve(0, &reserved_key, &reserved_tag, + &reserved_binding), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_RESERVED); + UT_ASSERT_EQ(memcmp(&work.key, &reserved_key, sizeof(reserved_key)), 0); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT_EQ(memcmp(&work.key, &ready_key, sizeof(ready_key)), 0); +} + + +/* Outbound admission is a small transaction: the exact READY entry is marked + * first, and a ring refusal must roll that marker back so the image remains + * runnable. Repeating the rollback is an idempotent no-op, never corruption. */ +UT_TEST(u32_pcm_x_staged_marker_rolls_back_exactly) +{ + BufferTag tag = make_tag(127); + GcsBlockDedupKey key; + GcsBlockPcmXImageBinding binding; + GcsBlockPcmXImageBinding wrong; + GcsBlockPcmXImageWork work; + GcsBlockReplyHeader hdr; + char page[GCS_BLOCK_DATA_SIZE]; + uint64 image_id; + + reset_fake_dedup(2, FAKE_DEDUP_CAP); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, 24, &image_id)); + key = make_key(1, 3, image_id, 13); + binding = make_pcm_x_binding(tag, 1, 5, gcs_reqid_requester(1, 2, 94), 13, image_id, 118); + hdr = make_pcm_x_reply_header(&key, &binding); + memset(page, 0x61, sizeof(page)); + prepare_pcm_x_page(page, &binding, &hdr); + UT_ASSERT_EQ((int)stage_pcm_x_ready(0, &key, &tag, &binding, &hdr, page), + (int)GCS_BLOCK_PCM_X_IMAGE_STORED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_mark_staged_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_STAGED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + + wrong = binding; + wrong.identity.ref.grant_generation++; + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(0, &key, &tag, &wrong), + (int)GCS_BLOCK_PCM_X_IMAGE_STALE); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_NOT_FOUND); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_REARMED); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_next_work(0, &work), + (int)GCS_BLOCK_PCM_X_IMAGE_REPLAY); + UT_ASSERT_EQ(memcmp(&work.key, &key, sizeof(key)), 0); + UT_ASSERT_EQ((int)cluster_gcs_block_dedup_pcm_x_unmark_staged_exact(0, &key, &tag, &binding), + (int)GCS_BLOCK_PCM_X_IMAGE_DUPLICATE); +} + int main(void) { - UT_PLAN(16); + UT_PLAN(32); UT_RUN(u1_per_worker_isolation); UT_RUN(u2_dedup_lifecycle_per_shard); UT_RUN(u3_counters_sum_across_shards); @@ -967,6 +1772,22 @@ main(void) UT_RUN(u14_pinned_ttl_wire_hint_and_no_guc_reread); UT_RUN(u15_done_linger_beats_full_lifetime); UT_RUN(u16_capability_routing_truth_table); + UT_RUN(u17_pcm_x_binding_layout_is_zero_entry_growth); + UT_RUN(u18_pcm_x_stage_duplicate_and_generic_overwrite_refused); + UT_RUN(u19_pcm_x_entry_isolated_from_generic_lifecycle); + UT_RUN(u20_pcm_x_entry_survives_generic_gc_and_retires_exactly); + UT_RUN(u21_pcm_x_stage_full_is_fail_closed); + UT_RUN(u22_pcm_x_reserved_entry_waits_for_exact_release); + UT_RUN(u23_pcm_x_materialize_validation_is_fail_closed_and_byte_stable); + UT_RUN(u24_pcm_x_namespace_cannot_register_as_generic); + UT_RUN(u25_pcm_x_work_prefers_reserved_and_marks_ready_staged); + UT_RUN(u26_pcm_x_ready_rearm_is_exact); + UT_RUN(u27_pcm_x_reserved_work_scan_rotates); + UT_RUN(u28_pcm_x_idle_hint_avoids_empty_rescan_and_reserve_rearms); + UT_RUN(u29_pcm_x_materialized_bytes_require_explicit_ready_publication); + UT_RUN(u30_pcm_x_owner_restart_audit_detects_and_retains_evidence); + UT_RUN(u31_pcm_x_work_classes_alternate_when_both_remain_runnable); + UT_RUN(u32_pcm_x_staged_marker_rolls_back_exactly); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c index 227562cfd48..fa7a89f4c12 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_retransmit.c @@ -18,7 +18,7 @@ * L1 GcsBlockReplyStatus enum extends to 8 values (DENIED_DEDUP_FULL=7) * L2 GcsBlockDedupKey == 24B + offset lock * L3 GcsBlockDedupEntry == 8472B fixed-size StaticAssertDecl (8448 + 24 RC-F DONE) - * L4 GcsBlockDedupEntry.sf_dep_vec offset 112, block_data offset 240 + * L4 GcsBlockDedupEntry.payload_meta offset 112, block_data offset 240 * L5 GcsBlockDedupEntry.reply_header offset 56 (8-aligned for uint64) * L6 GcsBlockDedupEntry.completed_at_ts offset 8432 + registered 8440 * L7 GcsBlockDedupResult enum 5 values @@ -117,7 +117,7 @@ UT_TEST(test_dedup_entry_size_locked_at_8472) UT_TEST(test_dedup_entry_smart_fusion_dep_and_block_offsets) { - UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, sf_dep_vec), 112); + UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, payload_meta), 112); UT_ASSERT_EQ((int)offsetof(GcsBlockDedupEntry, block_data), 240); UT_ASSERT_EQ((int)GCS_BLOCK_DATA_SIZE, BLCKSZ); } diff --git a/src/test/cluster_unit/test_cluster_gcs_block_shard.c b/src/test/cluster_unit/test_cluster_gcs_block_shard.c index 4b43bd2bf43..72bc481e2f9 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block_shard.c +++ b/src/test/cluster_unit/test_cluster_gcs_block_shard.c @@ -51,6 +51,7 @@ #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_lms_shard.h" +#include "cluster/cluster_pcm_x_convert.h" #include "storage/buf_internals.h" #undef printf @@ -197,10 +198,10 @@ UT_TEST(test_route_ack_request_interleave_affinity) } /* ====================================================================== - * U3 -- DATA-plane registry partition pin ("every DATA msg_type has a - * declared shard key or is explicitly direct-send"): of the five - * registered DATA types, exactly REQUEST / FORWARD / INVALIDATE - * are ring-routable; REPLY and INVALIDATE-ACK are the whitelist + * U3 -- legacy DATA-plane registry partition pin ("every DATA msg_type has a + * declared shard key or is explicitly direct-send"): REQUEST / FORWARD / + * INVALIDATE / DONE are ring-routable; REPLY and INVALIDATE-ACK are + * the whitelist * (no tag; direct-sent from the receiving worker's own dispatch, * so they already ride the correct channel). A NEW DATA type that * tries to stage without a key is refused at runtime by this same @@ -380,10 +381,71 @@ UT_TEST(test_route_ignores_non_tag_fields) } } +/* Every staged PCM-X frame is tag-affine. RETIRE/RETIRE_ACK are the only + * direct-send members because their compact payload intentionally has no tag. */ +UT_TEST(test_pcm_x_route_truth_table) +{ + BufferTag tag = make_tag(1663, 5, 24001, MAIN_FORKNUM, 73); + union { + PcmXGrantPayload largest; + uint8 bytes[sizeof(PcmXGrantPayload)]; + } payload; + struct { + uint8 msg_type; + uint16 payload_len; + } staged[] = { + { PGRAC_IC_MSG_PCM_X_ENQUEUE, sizeof(PcmXEnqueuePayload) }, + { PGRAC_IC_MSG_PCM_X_ADMIT_ACK, sizeof(PcmXAdmitAckPayload) }, + { PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_BEGIN, sizeof(PcmXBlockerSetHeaderPayload) }, + { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_EDGE, sizeof(PcmXBlockerChunkPayload) }, + { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, sizeof(PcmXBlockerSetHeaderPayload) }, + { PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_REVOKE, sizeof(PcmXRevokePayload) }, + { PGRAC_IC_MSG_PCM_X_IMAGE_READY, sizeof(PcmXGrantPayload) }, + { PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, sizeof(PcmXGrantPayload) }, + { PGRAC_IC_MSG_PCM_X_INSTALL_READY, sizeof(PcmXInstallReadyPayload) }, + { PGRAC_IC_MSG_PCM_X_COMMIT_X, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_FINAL_ACK, sizeof(PcmXFinalAckPayload) }, + { PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL, sizeof(PcmXPrehandleCancelPayload) }, + { PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK, sizeof(PcmXAdmitAckPayload) }, + { PGRAC_IC_MSG_PCM_X_CANCEL, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_CANCEL_ACK, sizeof(PcmXPhasePayload) }, + { PGRAC_IC_MSG_PCM_X_DRAIN_POLL, sizeof(PcmXDrainPollPayload) }, + { PGRAC_IC_MSG_PCM_X_DRAIN_ACK, sizeof(PcmXPhasePayload) }, + }; + Size i; + int expected = cluster_lms_shard_for_tag(&tag, CLUSTER_LMS_MAX_WORKERS); + + memset(&payload, 0, sizeof(payload)); + memcpy(payload.bytes, &tag, sizeof(tag)); + for (i = 0; i < lengthof(staged); i++) { + UT_ASSERT_EQ(cluster_gcs_block_payload_shard(staged[i].msg_type, payload.bytes, + staged[i].payload_len, + CLUSTER_LMS_MAX_WORKERS), + expected); + UT_ASSERT_EQ(cluster_gcs_block_payload_shard(staged[i].msg_type, payload.bytes, + staged[i].payload_len - 1, + CLUSTER_LMS_MAX_WORKERS), + -1); + } + UT_ASSERT_EQ(cluster_gcs_block_payload_shard(PGRAC_IC_MSG_PCM_X_RETIRE_UP_TO, payload.bytes, + sizeof(PcmXRetirePayload), + CLUSTER_LMS_MAX_WORKERS), + -1); + UT_ASSERT_EQ(cluster_gcs_block_payload_shard(PGRAC_IC_MSG_PCM_X_RETIRE_ACK, payload.bytes, + sizeof(PcmXRetirePayload), + CLUSTER_LMS_MAX_WORKERS), + -1); +} + int main(void) { - UT_PLAN(7); + UT_PLAN(8); UT_RUN(test_route_matches_shard_for_tag); UT_RUN(test_route_ack_request_interleave_affinity); UT_RUN(test_route_registry_partition); @@ -391,6 +453,7 @@ main(void) UT_RUN(test_route_length_mismatch_refused); UT_RUN(test_route_n1_degenerate_zero); UT_RUN(test_route_ignores_non_tag_fields); + UT_RUN(test_pcm_x_route_truth_table); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_ic.c b/src/test/cluster_unit/test_cluster_ic.c index d7efadaea68..71877a8c7fc 100644 --- a/src/test/cluster_unit/test_cluster_ic.c +++ b/src/test/cluster_unit/test_cluster_ic.c @@ -596,10 +596,10 @@ UT_TEST(test_hello_wire_reference_bytes) * CAPS_REPLY_V1 meta bit (0x8) + GCS-race round-2 F6 completion-proof * (0x10) + round-3 P0-1 xid wrap barrier (0x20) + round-4 P0-1 * authority flock (0x40) + ownership-gen ruling② invalidate BUSY - * (0x80) + TT-lane undo-horizon idle sentinel (0x100) - * (smart-fusion is off in this fixture) */ + * (0x80) + TT-lane undo-horizon idle sentinel (0x100) + PCM-X + * conversion (0x200) (smart-fusion is off in this fixture) */ UT_ASSERT_EQ(wire[36], 0xFE); - UT_ASSERT_EQ(wire[37], 0x01); + UT_ASSERT_EQ(wire[37], 0x03); UT_ASSERT_EQ(wire[38], 0x00); UT_ASSERT_EQ(wire[39], 0x00); /* remaining _pad must be all zero: a CONTROL-plane HELLO with @@ -697,7 +697,12 @@ UT_TEST(test_hello_smart_fusion_capability_gate) PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 - | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1); + | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); + /* Keep the aggregate word byte-exact as well as symbolically composed: + * parallel protocol lanes have collided while preserving the same symbolic + * expectation, so the literal catches accidental bit reuse. */ + UT_ASSERT_EQ(cluster_ic_hello_capabilities(&parsed), (uint32)0x000003FEU); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_2; @@ -710,7 +715,8 @@ UT_TEST(test_hello_smart_fusion_capability_gate) PGRAC_IC_HELLO_CAP_UNDO_AUTHORITY_SERVE_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 - | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1); + | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 + | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); cluster_smart_fusion = true; cluster_interconnect_tier = CLUSTER_IC_TIER_3; @@ -724,7 +730,7 @@ UT_TEST(test_hello_smart_fusion_capability_gate) | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_V1 | PGRAC_IC_HELLO_CAP_CAPS_REPLY_V1 | PGRAC_IC_HELLO_CAP_GCS_DONE_V1 | PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1 | PGRAC_IC_HELLO_CAP_XID_AUTHORITY_FLOCK_V2 | PGRAC_IC_HELLO_CAP_GCS_INVAL_BUSY_V1 - | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1); + | PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 | PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1); cluster_smart_fusion = false; cluster_interconnect_tier = CLUSTER_IC_TIER_STUB; @@ -783,6 +789,31 @@ UT_TEST(test_hello_gcs_done_and_wrap_barrier_gates) UT_ASSERT_EQ((int)PGRAC_IC_MSG_GCS_BLOCK_DONE, 38); UT_ASSERT_EQ((int)PGRAC_IC_MSG_XID_NATIVE_DISABLE, 39); UT_ASSERT_EQ((int)PGRAC_IC_MSG_XID_NATIVE_DISABLE_ACK, 40); + UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, (uint32)0x00000200U); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ENQUEUE, 41); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_ACK, 42); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM, 43); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, 44); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_BLOCKER_SET_BEGIN, 45); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_BLOCKER_SET_EDGE, 46); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT, 47); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK, 48); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_REVOKE, 49); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_IMAGE_READY, 50); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, 51); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_INSTALL_READY, 52); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_COMMIT_X, 53); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_FINAL_ACK, 54); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK, 55); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM, 56); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL, 57); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK, 58); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_CANCEL, 59); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_CANCEL_ACK, 60); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_DRAIN_POLL, 61); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_DRAIN_ACK, 62); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_RETIRE_UP_TO, 63); + UT_ASSERT_EQ((int)PGRAC_IC_MSG_PCM_X_RETIRE_ACK, 64); /* default: both bits advertised */ cluster_ic_suppress_gcs_done_cap = false; @@ -792,6 +823,7 @@ UT_TEST(test_hello_gcs_done_and_wrap_barrier_gates) UT_ASSERT((cluster_ic_hello_capabilities(&parsed) & PGRAC_IC_HELLO_CAP_GCS_DONE_V1) != 0); UT_ASSERT((cluster_ic_hello_capabilities(&parsed) & PGRAC_IC_HELLO_CAP_XID_NATIVE_DISABLE_V1) != 0); + UT_ASSERT((cluster_ic_hello_capabilities(&parsed) & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) != 0); /* suppressed (old-binary simulation): DONE bit absent, the other * protocol bits untouched */ diff --git a/src/test/cluster_unit/test_cluster_lmd_graph.c b/src/test/cluster_unit/test_cluster_lmd_graph.c index 1552671a08d..c6df86ea834 100644 --- a/src/test/cluster_unit/test_cluster_lmd_graph.c +++ b/src/test/cluster_unit/test_cluster_lmd_graph.c @@ -97,6 +97,57 @@ UT_DEFINE_GLOBALS(); bool IsUnderPostmaster = false; +void * +palloc(Size size) +{ + void *ptr = malloc(size); + + if (ptr == NULL) + abort(); + return ptr; +} + +void * +repalloc(void *pointer, Size size) +{ + void *ptr = realloc(pointer, size); + + if (ptr == NULL) + abort(); + return ptr; +} + +void +pfree(void *pointer) +{ + free(pointer); +} + +bool +errstart(int elevel pg_attribute_unused(), const char *domain pg_attribute_unused()) +{ + return true; +} + +bool +errstart_cold(int elevel pg_attribute_unused(), const char *domain pg_attribute_unused()) +{ + return true; +} + +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{ + abort(); +} + /* * cluster_lmd_graph.c reads cluster.lmd_max_wait_edges (cluster_guc.c GUC). * The standalone harness defines it directly to avoid pulling cluster_guc.o. @@ -160,6 +211,8 @@ LWLockRelease(LWLock *lock pg_attribute_unused()) static int fake_htab_token; static int fake_htab_count; static int fake_htab_seq_index; +static int fake_htab_seq_init_count; +static int fake_hash_enter_null_fail_after = -1; static Size fake_htab_entrysize; static Size fake_htab_keysize; static union { @@ -199,6 +252,8 @@ ShmemInitHash(const char *name pg_attribute_unused(), long init_size pg_attribut fake_htab_keysize = infoP->keysize; fake_htab_count = 0; fake_htab_seq_index = 0; + fake_htab_seq_init_count = 0; + fake_hash_enter_null_fail_after = -1; memset(&fake_htab, 0, sizeof(fake_htab)); return (HTAB *)&fake_htab_token; } @@ -238,9 +293,15 @@ hash_search(HTAB *hashp pg_attribute_unused(), const void *keyPtr, HASHACTION ac if (action == HASH_FIND || action == HASH_REMOVE) return NULL; - if (action == HASH_ENTER_NULL) { + if (action == HASH_ENTER_NULL || action == HASH_ENTER) { char *entry; + if (action == HASH_ENTER_NULL && fake_hash_enter_null_fail_after >= 0) { + if (fake_hash_enter_null_fail_after-- == 0) { + fake_hash_enter_null_fail_after = -1; + return NULL; + } + } if (fake_htab_count >= FAKE_LMD_HTAB_MAX_ENTRIES) return NULL; @@ -257,6 +318,7 @@ void hash_seq_init(HASH_SEQ_STATUS *status pg_attribute_unused(), HTAB *hashp pg_attribute_unused()) { fake_htab_seq_index = 0; + fake_htab_seq_init_count++; } void * @@ -402,6 +464,55 @@ UT_TEST(test_remove_by_waiter_removes_all_blocker_edges) UT_ASSERT_EQ(cluster_lmd_graph_snapshot_copy(snap, 32, &gen), 0); } +UT_TEST(test_exact_remove_requires_wait_seq_and_preserves_new_wait_instance) +{ + ClusterLmdVertex waiter = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex stale = waiter; + ClusterLmdVertex blocker = mkvertex(2, 200, 7, 6000); + + reset_graph(); + waiter.wait_seq = UINT64_C(81); + stale.wait_seq = UINT64_C(80); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&waiter, &blocker, waiter.request_id)); + UT_ASSERT(!cluster_lmd_graph_remove_edge_by_waiter_exact(&stale)); + UT_ASSERT(cluster_lmd_graph_has_waiter(&waiter)); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 1); + UT_ASSERT(cluster_lmd_graph_remove_edge_by_waiter_exact(&waiter)); + UT_ASSERT(!cluster_lmd_graph_has_waiter(&waiter)); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 0); +} + +UT_TEST(test_exact_remove_result_distinguishes_absent_stale_and_removed) +{ + ClusterLmdVertex waiter = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex stale = waiter; + ClusterLmdVertex blocker = mkvertex(2, 200, 7, 6000); + uint64 generation; + + reset_graph(); + waiter.wait_seq = UINT64_C(81); + stale.wait_seq = UINT64_C(80); + generation = cluster_lmd_graph_generation_get(); + + UT_ASSERT_EQ(cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter), + CLUSTER_LMD_GRAPH_REMOVE_ABSENT); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), generation); + + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&waiter, &blocker, waiter.request_id)); + generation = cluster_lmd_graph_generation_get(); + UT_ASSERT_EQ(cluster_lmd_graph_remove_edge_by_waiter_exact_result(&stale), + CLUSTER_LMD_GRAPH_REMOVE_STALE); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), generation); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 1); + + UT_ASSERT_EQ(cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter), + CLUSTER_LMD_GRAPH_REMOVE_REMOVED); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), generation + 1); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 0); + UT_ASSERT_EQ(cluster_lmd_graph_remove_edge_by_waiter_exact_result(&waiter), + CLUSTER_LMD_GRAPH_REMOVE_ABSENT); +} + /* ============================================================ * U1c — distinct waiters are isolated; removing one keeps the other. @@ -462,6 +573,369 @@ UT_TEST(test_duplicate_edge_is_idempotent) } +/* ============================================================ + * U1e-U1j -- spec-2.36a C2 atomic blocker-set replacement. + * + * A PCM convert waiter cannot publish blockers as cancel + N independent + * adds: a concurrent snapshot could observe an empty/partial set, and an add + * failure would strand only a prefix. The batch API is all-or-nothing. + * ============================================================ */ + +UT_TEST(test_replace_waiter_edges_publishes_exact_batch) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex blockers[3] + = { mkvertex(0, 300, 7, 7000), mkvertex(2, 301, 7, 7001), mkvertex(3, 302, 7, 7002) }; + ClusterLmdWaitEdge snap[8]; + uint64 gen; + int n; + + reset_graph(); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + UT_ASSERT(cluster_lmd_graph_replace_waiter_edges(&w, blockers, 3, w.request_id)); + + n = cluster_lmd_graph_snapshot_copy(snap, 8, &gen); + UT_ASSERT_EQ(n, 3); + UT_ASSERT_EQ(count_edges(snap, n, &w, &old), 0); + for (int i = 0; i < 3; i++) + UT_ASSERT_EQ(count_edges(snap, n, &w, &blockers[i]), 1); +} + +UT_TEST(test_replace_waiter_edges_full_preserves_previous_set) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex other = mkvertex(2, 101, 7, 5001); + ClusterLmdVertex old[2] = { mkvertex(0, 200, 7, 6000), mkvertex(3, 201, 7, 6001) }; + ClusterLmdVertex replacement[3] + = { mkvertex(0, 300, 7, 7000), mkvertex(2, 301, 7, 7001), mkvertex(3, 302, 7, 7002) }; + ClusterLmdWaitEdge snap[64]; + uint64 gen_before; + uint64 gen_after; + uint64 full_before; + int n; + + cluster_lmd_max_wait_edges = 64; + reset_graph(); + cluster_lmd_max_wait_edges = 1024; + for (int i = 0; i < 2; i++) + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old[i], w.request_id)); + /* graph minimum capacity is 64; fill the remaining 62 entries. */ + for (int i = 0; i < 62; i++) { + ClusterLmdVertex b = mkvertex(0, 1000 + (uint32)i, 7, 8000 + (uint64)i); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&other, &b, other.request_id)); + } + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 64); + gen_before = cluster_lmd_graph_generation_get(); + full_before = cluster_lmd_wait_edge_full_count_get(); + + UT_ASSERT(!cluster_lmd_graph_replace_waiter_edges(&w, replacement, 3, w.request_id)); + gen_after = cluster_lmd_graph_generation_get(); + UT_ASSERT_EQ(gen_after, gen_before); + UT_ASSERT_EQ(cluster_lmd_wait_edge_full_count_get(), full_before + 1); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 64); + + n = cluster_lmd_graph_snapshot_copy(snap, 64, NULL); + for (int i = 0; i < 2; i++) + UT_ASSERT_EQ(count_edges(snap, n, &w, &old[i]), 1); + for (int i = 0; i < 3; i++) + UT_ASSERT_EQ(count_edges(snap, n, &w, &replacement[i]), 0); +} + +UT_TEST(test_replace_waiter_edges_can_shrink_at_capacity) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex other = mkvertex(2, 101, 7, 5001); + ClusterLmdVertex old1 = mkvertex(0, 200, 7, 6000); + ClusterLmdVertex old2 = mkvertex(3, 201, 7, 6001); + ClusterLmdVertex replacement = mkvertex(2, 300, 7, 7000); + ClusterLmdVertex extra = mkvertex(3, 999, 7, 9999); + + cluster_lmd_max_wait_edges = 64; + reset_graph(); + cluster_lmd_max_wait_edges = 1024; + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old1, w.request_id)); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old2, w.request_id)); + for (int i = 0; i < 62; i++) { + ClusterLmdVertex b = mkvertex(0, 1000 + (uint32)i, 7, 8000 + (uint64)i); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&other, &b, other.request_id)); + } + + UT_ASSERT(cluster_lmd_graph_replace_waiter_edges(&w, &replacement, 1, w.request_id)); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 63); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&other, &extra, other.request_id)); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 64); +} + +UT_TEST(test_replace_waiter_edges_deduplicates_blocker_identities) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex b1 = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex b2 = mkvertex(3, 201, 7, 6001); + ClusterLmdVertex blockers[3] = { b1, b1, b2 }; + ClusterLmdWaitEdge snap[4]; + int n; + + reset_graph(); + UT_ASSERT(cluster_lmd_graph_replace_waiter_edges(&w, blockers, 3, w.request_id)); + n = cluster_lmd_graph_snapshot_copy(snap, 4, NULL); + UT_ASSERT_EQ(n, 2); + UT_ASSERT_EQ(count_edges(snap, n, &w, &b1), 1); + UT_ASSERT_EQ(count_edges(snap, n, &w, &b2), 1); +} + +UT_TEST(test_replace_waiter_edges_rejects_self_cycle_without_mutation) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex blockers[2] = { mkvertex(3, 300, 7, 7000), w }; + ClusterLmdWaitEdge snap[4]; + uint64 gen_before; + int n; + + reset_graph(); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + gen_before = cluster_lmd_graph_generation_get(); + UT_ASSERT(!cluster_lmd_graph_replace_waiter_edges(&w, blockers, 2, w.request_id)); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before); + n = cluster_lmd_graph_snapshot_copy(snap, 4, NULL); + UT_ASSERT_EQ(n, 1); + UT_ASSERT_EQ(count_edges(snap, n, &w, &old), 1); +} + +UT_TEST(test_replace_waiter_edges_empty_set_removes_only_that_waiter) +{ + ClusterLmdVertex w1 = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex w2 = mkvertex(2, 101, 7, 5001); + ClusterLmdVertex b1 = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex b2 = mkvertex(3, 201, 7, 6001); + + reset_graph(); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w1, &b1, w1.request_id)); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w2, &b2, w2.request_id)); + UT_ASSERT(cluster_lmd_graph_replace_waiter_edges(&w1, NULL, 0, w1.request_id)); + UT_ASSERT(!cluster_lmd_graph_has_waiter(&w1)); + UT_ASSERT(cluster_lmd_graph_has_waiter(&w2)); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 1); +} + +UT_TEST(test_replace_waiter_edges_more_than_batch_at_capacity) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex other = mkvertex(2, 101, 7, 5001); + ClusterLmdVertex old[40]; + ClusterLmdVertex replacement[40]; + ClusterLmdWaitEdge snap[64]; + uint64 gen_before; + int scans_before; + int n; + + cluster_lmd_max_wait_edges = 64; + reset_graph(); + cluster_lmd_max_wait_edges = 1024; + for (int i = 0; i < 40; i++) { + old[i] = mkvertex(0, 2000 + (uint32)i, 7, 6000 + (uint64)i); + replacement[i] = mkvertex(3, 3000 + (uint32)i, 7, 7000 + (uint64)i); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old[i], w.request_id)); + } + for (int i = 0; i < 24; i++) { + ClusterLmdVertex b = mkvertex(0, 4000 + (uint32)i, 7, 8000 + (uint64)i); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&other, &b, other.request_id)); + } + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 64); + gen_before = cluster_lmd_graph_generation_get(); + scans_before = fake_htab_seq_init_count; + + UT_ASSERT(cluster_lmd_graph_replace_waiter_edges(&w, replacement, 40, w.request_id)); + UT_ASSERT_EQ(fake_htab_seq_init_count - scans_before, 1); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 64); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before + 80); + + n = cluster_lmd_graph_snapshot_copy(snap, 64, NULL); + UT_ASSERT_EQ(n, 64); + for (int i = 0; i < 40; i++) { + UT_ASSERT_EQ(count_edges(snap, n, &w, &old[i]), 0); + UT_ASSERT_EQ(count_edges(snap, n, &w, &replacement[i]), 1); + } +} + +UT_TEST(test_replace_waiter_edges_rejects_conflicting_duplicate_metadata) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex duplicate = mkvertex(3, 300, 7, 7000); + ClusterLmdWaitEdge snap[4]; + uint64 gen_before; + int n; + + for (int variant = 0; variant < 3; variant++) { + ClusterLmdVertex blockers[2] = { duplicate, duplicate }; + + reset_graph(); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + gen_before = cluster_lmd_graph_generation_get(); + if (variant == 0) + blockers[1].wait_seq++; + else if (variant == 1) + blockers[1].xid++; + else + blockers[1].local_start_ts_ms++; + + UT_ASSERT(!cluster_lmd_graph_replace_waiter_edges(&w, blockers, 2, w.request_id)); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 1); + n = cluster_lmd_graph_snapshot_copy(snap, 4, NULL); + UT_ASSERT_EQ(n, 1); + UT_ASSERT_EQ(count_edges(snap, n, &w, &old), 1); + } +} + +UT_TEST(test_replace_waiter_edges_insert_failure_rolls_back_exactly) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old[2] = { mkvertex(0, 200, 7, 6000), mkvertex(2, 201, 7, 6001) }; + ClusterLmdVertex replacement[3] + = { mkvertex(0, 300, 7, 7000), mkvertex(2, 301, 7, 7001), mkvertex(3, 302, 7, 7002) }; + ClusterLmdWaitEdge snap[8]; + uint64 gen_before; + uint64 full_before; + int n; + + reset_graph(); + for (int i = 0; i < 2; i++) + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old[i], w.request_id)); + gen_before = cluster_lmd_graph_generation_get(); + full_before = cluster_lmd_wait_edge_full_count_get(); + /* Let one replacement insert succeed, then fail the second non-throwing enter. */ + fake_hash_enter_null_fail_after = 1; + + UT_ASSERT(!cluster_lmd_graph_replace_waiter_edges(&w, replacement, 3, w.request_id)); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 2); + UT_ASSERT_EQ(cluster_lmd_wait_edge_full_count_get(), full_before + 1); + n = cluster_lmd_graph_snapshot_copy(snap, 8, NULL); + UT_ASSERT_EQ(n, 2); + for (int i = 0; i < 2; i++) + UT_ASSERT_EQ(count_edges(snap, n, &w, &old[i]), 1); + for (int i = 0; i < 3; i++) + UT_ASSERT_EQ(count_edges(snap, n, &w, &replacement[i]), 0); +} + +UT_TEST(test_replace_waiter_edges_rejects_request_id_mismatch) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex replacement = mkvertex(3, 300, 7, 7000); + uint64 gen_before; + + reset_graph(); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + gen_before = cluster_lmd_graph_generation_get(); + UT_ASSERT(!cluster_lmd_graph_replace_waiter_edges(&w, &replacement, 1, w.request_id + 1)); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 1); +} + +UT_TEST(test_replace_waiter_edges_rejects_raw_count_above_graph_bound) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old = mkvertex(2, 200, 7, 6000); + ClusterLmdVertex replacement = mkvertex(3, 300, 7, 7000); + uint64 gen_before; + + cluster_lmd_max_wait_edges = 64; + reset_graph(); + cluster_lmd_max_wait_edges = 1024; + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + gen_before = cluster_lmd_graph_generation_get(); + /* The raw count is rejected before the one-element pointer can be read. */ + UT_ASSERT(!cluster_lmd_graph_replace_waiter_edges(&w, &replacement, 65, w.request_id)); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 1); +} + +UT_TEST(test_replace_waiter_edges_scan_growth_retry_has_no_generation_side_effect) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + uint64 gen_before; + uint64 full_before; + int scans_before; + + cluster_lmd_max_wait_edges = 64; + reset_graph(); + cluster_lmd_max_wait_edges = 1024; + for (int i = 0; i < 40; i++) { + ClusterLmdVertex old = mkvertex(0, 2000 + (uint32)i, 7, 6000 + (uint64)i); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + } + gen_before = cluster_lmd_graph_generation_get(); + full_before = cluster_lmd_wait_edge_full_count_get(); + scans_before = fake_htab_seq_init_count; + + /* Empty replacement starts with capacity 32, forcing one lock-free grow/retry. */ + UT_ASSERT(cluster_lmd_graph_replace_waiter_edges(&w, NULL, 0, w.request_id)); + UT_ASSERT_EQ(fake_htab_seq_init_count - scans_before, 2); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), gen_before + 40); + UT_ASSERT_EQ(cluster_lmd_wait_edge_full_count_get(), full_before); + UT_ASSERT_EQ((int)cluster_lmd_wait_edge_count_get(), 0); +} + +UT_TEST(test_replace_waiter_edges_exact_returns_committed_generation) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex old = mkvertex(0, 200, 7, 6000); + ClusterLmdVertex replacement[2] = { mkvertex(2, 300, 7, 7000), mkvertex(3, 301, 7, 7001) }; + uint64 gen_before; + uint64 committed_generation; + + reset_graph(); + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old, w.request_id)); + gen_before = cluster_lmd_graph_generation_get(); + + committed_generation + = cluster_lmd_graph_replace_waiter_edges_exact(&w, replacement, 2, w.request_id); + + UT_ASSERT_NE(committed_generation, 0); + UT_ASSERT_EQ(committed_generation, gen_before + 3); + UT_ASSERT_EQ(cluster_lmd_graph_generation_get(), committed_generation); +} + +UT_TEST(test_replace_waiter_edges_exact_capacity_failure_is_byte_stable) +{ + ClusterLmdVertex w = mkvertex(1, 100, 7, 5000); + ClusterLmdVertex other = mkvertex(2, 101, 7, 5001); + ClusterLmdVertex old[2] = { mkvertex(0, 200, 7, 6000), mkvertex(3, 201, 7, 6001) }; + ClusterLmdVertex replacement[3] + = { mkvertex(0, 300, 7, 7000), mkvertex(2, 301, 7, 7001), mkvertex(3, 302, 7, 7002) }; + ClusterLmdWaitEdge before[64]; + ClusterLmdWaitEdge after[64]; + uint64 gen_before; + uint64 gen_after; + int n_before; + int n_after; + + cluster_lmd_max_wait_edges = 64; + reset_graph(); + cluster_lmd_max_wait_edges = 1024; + for (int i = 0; i < 2; i++) + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&w, &old[i], w.request_id)); + for (int i = 0; i < 62; i++) { + ClusterLmdVertex b = mkvertex(0, 1000 + (uint32)i, 7, 8000 + (uint64)i); + + UT_ASSERT(cluster_lmd_submit_wait_edge_real(&other, &b, other.request_id)); + } + n_before = cluster_lmd_graph_snapshot_copy(before, 64, &gen_before); + UT_ASSERT_EQ(n_before, 64); + + UT_ASSERT_EQ(cluster_lmd_graph_replace_waiter_edges_exact(&w, replacement, 3, w.request_id), 0); + + n_after = cluster_lmd_graph_snapshot_copy(after, 64, &gen_after); + UT_ASSERT_EQ(n_after, n_before); + UT_ASSERT_EQ(gen_after, gen_before); + UT_ASSERT_EQ(memcmp(after, before, sizeof(before)), 0); +} + + /* ============================================================ * U2a — spec-5.8 D2 (T2): a cross-node TX cycle expressed with holder * placeholders resolves so each placeholder blocker becomes the real waiter @@ -686,14 +1160,52 @@ UT_TEST(test_probe_round_two_identical_partial_subsets_never_complete) } +UT_TEST(test_pcm_convert_wfg_note_counters_are_narrow_and_exact) +{ + reset_graph(); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_replace_count_get(), 0); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_remove_count_get(), 0); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_replace_fail_count_get(), 0); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_exact_remove_stale_count_get(), 0); + + cluster_lmd_pcm_convert_wfg_note_replace(); + cluster_lmd_pcm_convert_wfg_note_replace(); + cluster_lmd_pcm_convert_wfg_note_remove(); + cluster_lmd_pcm_convert_wfg_note_replace_fail(); + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + cluster_lmd_pcm_convert_wfg_note_exact_remove_stale(); + + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_replace_count_get(), 2); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_remove_count_get(), 1); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_replace_fail_count_get(), 1); + UT_ASSERT_EQ(cluster_lmd_pcm_convert_wfg_exact_remove_stale_count_get(), 2); +} + + int main(void) { - UT_PLAN(17); + UT_PLAN(34); UT_RUN(test_multi_blocker_edges_not_overwritten); UT_RUN(test_remove_by_waiter_removes_all_blocker_edges); + UT_RUN(test_exact_remove_requires_wait_seq_and_preserves_new_wait_instance); + UT_RUN(test_exact_remove_result_distinguishes_absent_stale_and_removed); UT_RUN(test_distinct_waiters_isolated); UT_RUN(test_duplicate_edge_is_idempotent); + UT_RUN(test_replace_waiter_edges_publishes_exact_batch); + UT_RUN(test_replace_waiter_edges_full_preserves_previous_set); + UT_RUN(test_replace_waiter_edges_can_shrink_at_capacity); + UT_RUN(test_replace_waiter_edges_deduplicates_blocker_identities); + UT_RUN(test_replace_waiter_edges_rejects_self_cycle_without_mutation); + UT_RUN(test_replace_waiter_edges_empty_set_removes_only_that_waiter); + UT_RUN(test_replace_waiter_edges_more_than_batch_at_capacity); + UT_RUN(test_replace_waiter_edges_rejects_conflicting_duplicate_metadata); + UT_RUN(test_replace_waiter_edges_insert_failure_rolls_back_exactly); + UT_RUN(test_replace_waiter_edges_rejects_request_id_mismatch); + UT_RUN(test_replace_waiter_edges_rejects_raw_count_above_graph_bound); + UT_RUN(test_replace_waiter_edges_scan_growth_retry_has_no_generation_side_effect); + UT_RUN(test_replace_waiter_edges_exact_returns_committed_generation); + UT_RUN(test_replace_waiter_edges_exact_capacity_failure_is_byte_stable); UT_RUN(test_resolve_tx_placeholder_closes_cross_node_cycle); UT_RUN(test_resolve_tx_placeholder_no_match_unchanged); UT_RUN(test_resolve_tx_placeholder_invalid_xid_never_matches); @@ -707,6 +1219,7 @@ main(void) UT_RUN(test_probe_round_complete_exact_match); UT_RUN(test_probe_round_incomplete_partial_subset); UT_RUN(test_probe_round_two_identical_partial_subsets_never_complete); + UT_RUN(test_pcm_convert_wfg_note_counters_are_narrow_and_exact); UT_DONE(); return 0; } diff --git a/src/test/cluster_unit/test_cluster_lmd_wait_state.c b/src/test/cluster_unit/test_cluster_lmd_wait_state.c index cf5f30212bb..623c4ea1eeb 100644 --- a/src/test/cluster_unit/test_cluster_lmd_wait_state.c +++ b/src/test/cluster_unit/test_cluster_lmd_wait_state.c @@ -23,6 +23,15 @@ * (the value the caller stamps on the WFG vertex, spec-5.8 D1e). * U7: a published-but-not-cleared record stays active — documents why * the wiring must clear on EVERY exit path (leak => false-kill). + * U8: clear + republish never returns a cross-generation snapshot. + * U9: an in-progress writer is bounded and fails closed, rather than + * returning a partially updated tuple. + * U10: PCM_CONVERT is a distinct wait kind and preserves the full tuple. + * U11: the embedded PGPROC record keeps its 48-byte layout footprint. + * U12: exact read distinguishes a stable active tuple. + * U13: exact read distinguishes a stable inactive tuple. + * U14: exact read reports BUSY after bounded odd-sequence exhaustion. + * U15: exact read recovers to ACTIVE once the sequence becomes stable. * * The real GES / TX wait sites and their per-exit clear wiring are * covered end to end by the spec-5.8 TAP suite (D8); this file pins the @@ -47,6 +56,7 @@ */ #include "postgres.h" +#include #include #include "access/transam.h" @@ -157,12 +167,15 @@ UT_TEST(test_reset_preserves_wait_seq) /* Predecessor backend publishes then dies mid-wait (no clear). */ seq_before = cluster_lmd_wait_state_publish(&ws, CLUSTER_LMD_WAIT_GES, 77, 5, 200); + /* It may also die after opening, but before closing, the seqlock. */ + pg_atomic_write_u32(&ws.change_seq, pg_atomic_read_u32(&ws.change_seq) | 1U); /* A new backend reuses the slot — InitProcess reset runs. */ cluster_lmd_wait_state_reset(&ws); - /* The stale active flag is gone... */ + /* The stale active flag is gone and the abandoned write is closed... */ UT_ASSERT(!cluster_lmd_wait_state_read(&ws, &snap)); + UT_ASSERT_EQ((int)(pg_atomic_read_u32(&ws.change_seq) & 1U), 0); /* ...and the next publish does NOT restart wait_seq, so a stale victim * tuple recorded for the predecessor (seq_before) can never re-match. */ @@ -224,10 +237,268 @@ UT_TEST(test_uncleared_publish_stays_active) } +/* ============================================================ + * U8 — clear + republish never exposes a cross-generation tuple. + * ============================================================ */ + +#define TEAR_RACE_PUBLISHES 2000000 + +typedef struct WaitStateTearRace { + ClusterLmdProcWaitState ws; + pg_atomic_uint32 start; + pg_atomic_uint32 stop; + pg_atomic_uint32 mixed; +} WaitStateTearRace; + +static void * +tear_race_writer(void *arg) +{ + WaitStateTearRace *race = (WaitStateTearRace *)arg; + int i; + + while (pg_atomic_read_u32(&race->start) == 0) + ; + + for (i = 0; i < TEAR_RACE_PUBLISHES; i++) { + cluster_lmd_wait_state_clear(&race->ws); + if ((i & 1) == 0) + (void)cluster_lmd_wait_state_publish( + &race->ws, CLUSTER_LMD_WAIT_TX, UINT64CONST(0x4444444444444444), + UINT64CONST(0x5555555555555555), (TransactionId)0x66666666); + else + (void)cluster_lmd_wait_state_publish( + &race->ws, CLUSTER_LMD_WAIT_GES, UINT64CONST(0x1111111111111111), + UINT64CONST(0x2222222222222222), (TransactionId)0x33333333); + } + + pg_atomic_write_u32(&race->stop, 1); + return NULL; +} + +UT_TEST(test_clear_republish_snapshot_is_one_generation) +{ + WaitStateTearRace race; + ClusterLmdWaitStateSnapshot snap; + pthread_t writer; + int rc; + + cluster_lmd_wait_state_init(&race.ws); + pg_atomic_init_u32(&race.start, 0); + pg_atomic_init_u32(&race.stop, 0); + pg_atomic_init_u32(&race.mixed, 0); + (void)cluster_lmd_wait_state_publish( + &race.ws, CLUSTER_LMD_WAIT_GES, UINT64CONST(0x1111111111111111), + UINT64CONST(0x2222222222222222), (TransactionId)0x33333333); + + rc = pthread_create(&writer, NULL, tear_race_writer, &race); + UT_ASSERT_EQ(rc, 0); + pg_atomic_write_u32(&race.start, 1); + + while (pg_atomic_read_u32(&race.stop) == 0 && pg_atomic_read_u32(&race.mixed) == 0) { + if (cluster_lmd_wait_state_read(&race.ws, &snap)) { + bool odd_tuple; + bool even_tuple; + + odd_tuple = (snap.wait_seq & 1) != 0 && snap.kind == CLUSTER_LMD_WAIT_GES + && snap.request_id == UINT64CONST(0x1111111111111111) + && snap.cluster_epoch == UINT64CONST(0x2222222222222222) + && snap.xid == (TransactionId)0x33333333; + even_tuple = (snap.wait_seq & 1) == 0 && snap.kind == CLUSTER_LMD_WAIT_TX + && snap.request_id == UINT64CONST(0x4444444444444444) + && snap.cluster_epoch == UINT64CONST(0x5555555555555555) + && snap.xid == (TransactionId)0x66666666; + if (!odd_tuple && !even_tuple) + pg_atomic_write_u32(&race.mixed, 1); + } + } + + rc = pthread_join(writer, NULL); + UT_ASSERT_EQ(rc, 0); + UT_ASSERT_EQ((int)pg_atomic_read_u32(&race.mixed), 0); +} + + +/* ============================================================ + * U9 — an in-progress writer is bounded and fails closed. + * ============================================================ */ + +UT_TEST(test_writer_in_progress_fails_closed) +{ + ClusterLmdProcWaitState ws; + ClusterLmdWaitStateSnapshot snap; + uint32 stable_seq; + + cluster_lmd_wait_state_init(&ws); + (void)cluster_lmd_wait_state_publish(&ws, CLUSTER_LMD_WAIT_GES, 9, 8, 7); + stable_seq = pg_atomic_read_u32(&ws.change_seq); + pg_atomic_write_u32(&ws.change_seq, stable_seq | 1U); + + memset(&snap, 0x7f, sizeof(snap)); + UT_ASSERT(!cluster_lmd_wait_state_read(&ws, &snap)); + UT_ASSERT(!snap.active); + UT_ASSERT_EQ((int)snap.kind, (int)CLUSTER_LMD_WAIT_NONE); + UT_ASSERT_EQ((int)snap.request_id, 0); + UT_ASSERT_EQ((int)snap.cluster_epoch, 0); + UT_ASSERT_EQ((int)snap.xid, (int)InvalidTransactionId); + UT_ASSERT_EQ((int)snap.wait_seq, 0); +} + + +/* ============================================================ + * U10 — PCM_CONVERT is distinct and carries the complete tuple. + * ============================================================ */ + +UT_TEST(test_pcm_convert_kind_records_complete_tuple) +{ + ClusterLmdProcWaitState ws; + ClusterLmdWaitStateSnapshot snap; + uint64 wait_seq; + + cluster_lmd_wait_state_init(&ws); + wait_seq = cluster_lmd_wait_state_publish( + &ws, CLUSTER_LMD_WAIT_PCM_CONVERT, UINT64CONST(0x123456789abcdef0), + UINT64CONST(0x0fedcba987654321), (TransactionId)0xa5a5a5a5); + + UT_ASSERT_EQ((int)CLUSTER_LMD_WAIT_PCM_CONVERT, 3); + UT_ASSERT(cluster_lmd_wait_state_read(&ws, &snap)); + UT_ASSERT_EQ((int)snap.kind, (int)CLUSTER_LMD_WAIT_PCM_CONVERT); + UT_ASSERT(snap.request_id == UINT64CONST(0x123456789abcdef0)); + UT_ASSERT(snap.cluster_epoch == UINT64CONST(0x0fedcba987654321)); + UT_ASSERT_EQ((uint32)snap.xid, (uint32)0xa5a5a5a5); + UT_ASSERT(snap.wait_seq == wait_seq); +} + + +/* ============================================================ + * U11 — seqlock metadata consumes existing padding (no PGPROC growth). + * ============================================================ */ + +UT_TEST(test_wait_state_layout_stays_48_bytes) +{ + UT_ASSERT_EQ((int)sizeof(ClusterLmdProcWaitState), 48); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, change_seq), 0); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, active), 4); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, wait_seq), 8); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, kind), 16); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, request_id), 24); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, cluster_epoch), 32); + UT_ASSERT_EQ((int)offsetof(ClusterLmdProcWaitState, xid), 40); +} + + +/* ============================================================ + * U12 — exact read returns ACTIVE with the complete stable tuple. + * ============================================================ */ + +UT_TEST(test_exact_read_reports_stable_active) +{ + ClusterLmdProcWaitState ws; + ClusterLmdWaitStateSnapshot snap; + ClusterLmdWaitStateReadResult result; + uint64 wait_seq; + + cluster_lmd_wait_state_init(&ws); + wait_seq = cluster_lmd_wait_state_publish( + &ws, CLUSTER_LMD_WAIT_PCM_CONVERT, UINT64CONST(0x1020304050607080), + UINT64CONST(0x8877665544332211), (TransactionId)0x12345678); + + result = cluster_lmd_wait_state_read_exact(&ws, &snap); + UT_ASSERT_EQ((int)result, (int)CLUSTER_LMD_WAIT_STATE_READ_ACTIVE); + UT_ASSERT(snap.active); + UT_ASSERT_EQ((int)snap.kind, (int)CLUSTER_LMD_WAIT_PCM_CONVERT); + UT_ASSERT(snap.request_id == UINT64CONST(0x1020304050607080)); + UT_ASSERT(snap.cluster_epoch == UINT64CONST(0x8877665544332211)); + UT_ASSERT_EQ((uint32)snap.xid, (uint32)0x12345678); + UT_ASSERT(snap.wait_seq == wait_seq); +} + + +/* ============================================================ + * U13 — exact read returns INACTIVE only from a stable even tuple. + * ============================================================ */ + +UT_TEST(test_exact_read_reports_stable_inactive) +{ + ClusterLmdProcWaitState ws; + ClusterLmdWaitStateSnapshot snap; + ClusterLmdWaitStateReadResult result; + + cluster_lmd_wait_state_init(&ws); + (void)cluster_lmd_wait_state_publish(&ws, CLUSTER_LMD_WAIT_GES, 39, 38, 37); + cluster_lmd_wait_state_clear(&ws); + memset(&snap, 0x7f, sizeof(snap)); + + result = cluster_lmd_wait_state_read_exact(&ws, &snap); + UT_ASSERT_EQ((int)result, (int)CLUSTER_LMD_WAIT_STATE_READ_INACTIVE); + UT_ASSERT(!snap.active); + UT_ASSERT_EQ((int)snap.kind, (int)CLUSTER_LMD_WAIT_NONE); + UT_ASSERT_EQ((int)snap.request_id, 0); + UT_ASSERT_EQ((int)snap.cluster_epoch, 0); + UT_ASSERT_EQ((int)snap.xid, (int)InvalidTransactionId); + UT_ASSERT_EQ((int)snap.wait_seq, 0); +} + + +/* ============================================================ + * U14 — an odd writer sequence exhausts the bounded reader as BUSY. + * ============================================================ */ + +UT_TEST(test_exact_read_reports_busy_on_odd_exhaustion) +{ + ClusterLmdProcWaitState ws; + ClusterLmdWaitStateSnapshot snap; + ClusterLmdWaitStateReadResult result; + + cluster_lmd_wait_state_init(&ws); + (void)cluster_lmd_wait_state_publish(&ws, CLUSTER_LMD_WAIT_GES, 19, 18, 17); + pg_atomic_write_u32(&ws.change_seq, pg_atomic_read_u32(&ws.change_seq) | 1U); + memset(&snap, 0x7f, sizeof(snap)); + + result = cluster_lmd_wait_state_read_exact(&ws, &snap); + UT_ASSERT_EQ((int)result, (int)CLUSTER_LMD_WAIT_STATE_READ_BUSY); + UT_ASSERT(!snap.active); + UT_ASSERT_EQ((int)snap.kind, (int)CLUSTER_LMD_WAIT_NONE); + UT_ASSERT_EQ((int)snap.request_id, 0); + UT_ASSERT_EQ((int)snap.cluster_epoch, 0); + UT_ASSERT_EQ((int)snap.xid, (int)InvalidTransactionId); + UT_ASSERT_EQ((int)snap.wait_seq, 0); +} + + +/* ============================================================ + * U15 — a BUSY read recovers to ACTIVE after the writer closes. + * ============================================================ */ + +UT_TEST(test_exact_read_recovers_after_writer_closes) +{ + ClusterLmdProcWaitState ws; + ClusterLmdWaitStateSnapshot snap; + ClusterLmdWaitStateReadResult result; + uint32 stable_seq; + + cluster_lmd_wait_state_init(&ws); + (void)cluster_lmd_wait_state_publish(&ws, CLUSTER_LMD_WAIT_TX, 29, 28, 27); + stable_seq = pg_atomic_read_u32(&ws.change_seq); + pg_atomic_write_u32(&ws.change_seq, stable_seq | 1U); + + result = cluster_lmd_wait_state_read_exact(&ws, &snap); + UT_ASSERT_EQ((int)result, (int)CLUSTER_LMD_WAIT_STATE_READ_BUSY); + + pg_atomic_write_u32(&ws.change_seq, stable_seq + 2U); + result = cluster_lmd_wait_state_read_exact(&ws, &snap); + UT_ASSERT_EQ((int)result, (int)CLUSTER_LMD_WAIT_STATE_READ_ACTIVE); + UT_ASSERT(snap.active); + UT_ASSERT_EQ((int)snap.kind, (int)CLUSTER_LMD_WAIT_TX); + UT_ASSERT_EQ((int)snap.request_id, 29); + UT_ASSERT_EQ((int)snap.cluster_epoch, 28); + UT_ASSERT_EQ((int)snap.xid, 27); +} + + int main(void) { - UT_PLAN(7); + UT_PLAN(15); UT_RUN(test_publish_records_tuple_and_active); UT_RUN(test_clear_marks_inactive); UT_RUN(test_wait_seq_monotonic); @@ -235,6 +506,14 @@ main(void) UT_RUN(test_read_fresh_is_inactive); UT_RUN(test_publish_return_matches_read); UT_RUN(test_uncleared_publish_stays_active); + UT_RUN(test_clear_republish_snapshot_is_one_generation); + UT_RUN(test_writer_in_progress_fails_closed); + UT_RUN(test_pcm_convert_kind_records_complete_tuple); + UT_RUN(test_wait_state_layout_stays_48_bytes); + UT_RUN(test_exact_read_reports_stable_active); + UT_RUN(test_exact_read_reports_stable_inactive); + UT_RUN(test_exact_read_reports_busy_on_odd_exhaustion); + UT_RUN(test_exact_read_recovers_after_writer_closes); UT_DONE(); - return 0; + return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index 490ad2be5ba..6d57f5efb22 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -709,6 +709,10 @@ void cluster_grd_recovery_lmon_tick(void) {} +void +cluster_gcs_block_pcm_x_formation_tick(void) +{} + /* spec-2.34 D6 L104 stub: cluster_lmon.c LMON tick body calls * cluster_gcs_block_dedup_sweep_expired(now). Standalone fixture has * no dedup HTAB linked; vacuous stub. */ diff --git a/src/test/cluster_unit/test_cluster_lms_outbound.c b/src/test/cluster_unit/test_cluster_lms_outbound.c index a42df6db0d3..a94623d5621 100644 --- a/src/test/cluster_unit/test_cluster_lms_outbound.c +++ b/src/test/cluster_unit/test_cluster_lms_outbound.c @@ -54,6 +54,7 @@ #include "cluster/cluster_ic_router.h" /* cluster_ic_send_envelope prototype */ #include "cluster/cluster_lms.h" #include "cluster/cluster_shmem.h" +#include "cluster/cluster_write_fence.h" #include "miscadmin.h" #include "storage/lwlock.h" #include "storage/shmem.h" @@ -70,6 +71,35 @@ UT_DEFINE_GLOBALS(); ProcessingMode Mode = NormalProcessing; BackendType MyBackendType = B_LMS; +int cluster_node_id = 0; +static PcmXRuntimeState ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; +static bool ut_write_fence_enforcing = false; +static bool ut_write_fence_allowed = true; + +PcmXRuntimeSnapshot +cluster_pcm_x_runtime_snapshot(void) +{ + PcmXRuntimeSnapshot snapshot = { 0 }; + + snapshot.state = ut_pcm_x_runtime_state; + if (snapshot.state == PCM_X_RUNTIME_ACTIVE) { + snapshot.master_session_incarnation = 1; + snapshot.gate_generation = 1; + } + return snapshot; +} + +bool +cluster_write_fence_enforcing(void) +{ + return ut_write_fence_enforcing; +} + +bool +cluster_write_fence_allowed(void) +{ + return ut_write_fence_allowed; +} void ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) @@ -190,6 +220,33 @@ typedef struct UtSentRec { static UtSentRec ut_sent_log[64]; static int ut_sent_n = 0; static ClusterICSendResult ut_peer_rc[CLUSTER_MAX_NODES]; +static int ut_local_dispatch_count = 0; +static uint8 ut_local_dispatch_marker = 0; + +bool +cluster_ic_envelope_build(ClusterICEnvelope *out_env, uint8 msg_type, uint32 source_node_id, + uint32 dest_node_id, const void *payload, uint32 payload_length) +{ + memset(out_env, 0, sizeof(*out_env)); + out_env->msg_type = msg_type; + out_env->source_node_id = source_node_id; + out_env->dest_node_id = dest_node_id; + out_env->payload_length = payload_length; + (void)payload; + return true; +} + +bool +cluster_ic_dispatch_envelope(const ClusterICEnvelope *env, const void *payload, int32 peer_id) +{ + UT_ASSERT(env != NULL); + UT_ASSERT_EQ((int32)env->source_node_id, cluster_node_id); + UT_ASSERT_EQ((int32)env->dest_node_id, cluster_node_id); + UT_ASSERT_EQ(peer_id, cluster_node_id); + ut_local_dispatch_count++; + ut_local_dispatch_marker = env->payload_length > 0 ? *(const uint8 *)payload : 0; + return true; +} ClusterICSendResult cluster_ic_send_envelope(uint8 msg_type, int32 dest_node_id, const void *payload, @@ -221,13 +278,24 @@ static void ut_reset_log(void) { ut_sent_n = 0; + ut_local_dispatch_count = 0; + ut_local_dispatch_marker = 0; + ut_pcm_x_runtime_state = PCM_X_RUNTIME_ACTIVE; + ut_write_fence_enforcing = false; + ut_write_fence_allowed = true; memset(ut_sent_log, 0, sizeof(ut_sent_log)); } +static bool +ut_enqueue_typed_marker(int worker_id, uint8 msg_type, int32 dest, uint8 marker) +{ + return cluster_lms_outbound_enqueue(worker_id, msg_type, (uint32)dest, &marker, 1); +} + static bool ut_enqueue_marker(int worker_id, int32 dest, uint8 marker) { - return cluster_lms_outbound_enqueue(worker_id, UT_MSG_TYPE, (uint32)dest, &marker, 1); + return ut_enqueue_typed_marker(worker_id, UT_MSG_TYPE, dest, marker); } /* ============================================================ @@ -364,16 +432,85 @@ UT_TEST(test_blocked_peer_batch_keeps_per_peer_order) UT_ASSERT(d1_idx < d2_idx); } +/* + * PCM-X can hash a tag to the local node's master. A DATA frame staged by a + * backend must still execute on that tag's LMS worker: the generic IC send + * self-shortcut reports DONE without dispatching, which would otherwise turn + * ENQUEUE/ACK into a silent no-op. The worker therefore loopback-dispatches + * self frames and never hands them to the transport. + */ +UT_TEST(test_self_frame_dispatches_on_owning_worker) +{ + ut_reset_log(); + + UT_ASSERT(ut_enqueue_marker(5, cluster_node_id, 0xE1)); + UT_ASSERT_EQ(cluster_lms_outbound_drain_send(5), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(5), 0); + UT_ASSERT_EQ(ut_sent_n, 0); + UT_ASSERT_EQ(ut_local_dispatch_count, 1); + UT_ASSERT_EQ(ut_local_dispatch_marker, 0xE1); +} + + +/* A fail-closed runtime must retain a grant leg before transport admission. + * Later frames for the same peer stay behind it, while unrelated peers keep + * flowing. Core has no recovery proof that could make these old-incarnation + * frames runnable, so repeated drains must keep them parked. */ +UT_TEST(test_pcm_x_grant_frame_waits_for_active_runtime) +{ + ut_reset_log(); + ut_pcm_x_runtime_state = PCM_X_RUNTIME_RECOVERY_BLOCKED; + + UT_ASSERT(ut_enqueue_typed_marker(6, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT, UT_PEER_X, 0xF1)); + UT_ASSERT(ut_enqueue_marker(6, UT_PEER_X, 0xF2)); + UT_ASSERT(ut_enqueue_marker(6, UT_PEER_Y, 0xF3)); + (void)cluster_lms_outbound_drain_send(6); + + UT_ASSERT_EQ(ut_count_marker(0xF1), 0); + UT_ASSERT_EQ(ut_count_marker(0xF2), 0); + UT_ASSERT_EQ(ut_count_marker(0xF3), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(6), 2); + + (void)cluster_lms_outbound_drain_send(6); + UT_ASSERT_EQ(ut_count_marker(0xF1), 0); + UT_ASSERT_EQ(ut_count_marker(0xF2), 0); + UT_ASSERT_EQ(cluster_lms_outbound_depth(6), 2); +} + + +UT_TEST(test_pcm_x_grant_frame_waits_behind_write_fence) +{ + ut_reset_log(); + ut_write_fence_enforcing = true; + ut_write_fence_allowed = false; + + UT_ASSERT(ut_enqueue_typed_marker(7, PGRAC_IC_MSG_PCM_X_COMMIT_X, UT_PEER_X, 0xF4)); + UT_ASSERT(ut_enqueue_marker(7, UT_PEER_Y, 0xF5)); + (void)cluster_lms_outbound_drain_send(7); + + UT_ASSERT_EQ(ut_count_marker(0xF4), 0); + UT_ASSERT_EQ(ut_count_marker(0xF5), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(7), 1); + + ut_write_fence_allowed = true; + (void)cluster_lms_outbound_drain_send(7); + UT_ASSERT_EQ(ut_count_marker(0xF4), 1); + UT_ASSERT_EQ(cluster_lms_outbound_depth(7), 0); +} + int main(void) { - UT_PLAN(5); + UT_PLAN(8); UT_RUN(test_ring_shmem_init); UT_RUN(test_admitted_frame_is_never_resubmitted); UT_RUN(test_blocked_peer_does_not_starve_other_peer); UT_RUN(test_refused_frame_retained_and_delivered); UT_RUN(test_blocked_peer_batch_keeps_per_peer_order); + UT_RUN(test_self_frame_dispatches_on_owning_worker); + UT_RUN(test_pcm_x_grant_frame_waits_for_active_runtime); + UT_RUN(test_pcm_x_grant_frame_waits_behind_write_fence); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_lock_acquire.c b/src/test/cluster_unit/test_cluster_lock_acquire.c index d0b9b3904fd..a2b4b283256 100644 --- a/src/test/cluster_unit/test_cluster_lock_acquire.c +++ b/src/test/cluster_unit/test_cluster_lock_acquire.c @@ -51,6 +51,7 @@ #include "cluster/cluster_lmd_wait_state.h" #include "cluster/cluster_lock_acquire.h" #include "cluster/cluster_native_lock_probe.h" /* spec-5.3 same-lock-group helper */ +#include "cluster/cluster_pcm_x_convert.h" #include "miscadmin.h" #include "port/atomics.h" #include "storage/lock.h" @@ -487,6 +488,16 @@ cluster_lms_native_probe_wait_clear(const ClusterResId *resid pg_attribute_unuse /* spec-4.6 P0 regression harness — test-controlled GES reject reason * (GesRejectReason values; 0 = GRANT). */ static uint32 stub_ges_reject_reason = 0; +static PcmXQueueResult stub_pcm_x_nested_guard_result = PCM_X_QUEUE_OK; +static int stub_ges_request_wait_calls; +static int stub_ges_request_nowait_wait_calls; +static int stub_ges_convert_wait_calls; + +PcmXQueueResult +cluster_pcm_x_nested_wait_guard_before_block(void) +{ + return stub_pcm_x_nested_guard_result; +} uint32 cluster_ges_send_request_and_wait(const struct ClusterResId *resid pg_attribute_unused(), @@ -496,6 +507,7 @@ cluster_ges_send_request_and_wait(const struct ClusterResId *resid pg_attribute_ int timeout_ms pg_attribute_unused(), uint32 wait_event pg_attribute_unused()) { + stub_ges_request_wait_calls++; return stub_ges_reject_reason; } @@ -508,6 +520,7 @@ cluster_ges_send_request_nowait_and_wait( uint64 request_id pg_attribute_unused(), int timeout_ms pg_attribute_unused(), uint32 wait_event pg_attribute_unused()) { + stub_ges_request_nowait_wait_calls++; return stub_ges_reject_reason; } @@ -537,6 +550,7 @@ cluster_ges_send_convert_and_wait(const struct ClusterResId *resid pg_attribute_ uint64 convert_request_id pg_attribute_unused(), int timeout_ms pg_attribute_unused()) { + stub_ges_convert_wait_calls++; return stub_ges_reject_reason; } @@ -940,6 +954,7 @@ UT_TEST(test_ul_try_lock_nowait_s4_reject_mapping) { ClusterLockAcquireRequest req; int saved_node = cluster_node_id; + int nowait_calls = stub_ges_request_nowait_wait_calls; uint32 saved_reject = stub_ges_reject_reason; ClusterLockAcquireResult r; @@ -967,6 +982,7 @@ UT_TEST(test_ul_try_lock_nowait_s4_reject_mapping) stub_ges_reject_reason = GES_REJECT_REASON_TIMEOUT; r = cluster_lock_acquire_s4_remote_request_wait(&req); UT_ASSERT_EQ((int)r, (int)CLUSTER_LOCK_ACQUIRE_FAIL_TIMEOUT); + UT_ASSERT_EQ(stub_ges_request_nowait_wait_calls, nowait_calls + 3); /* BLOCKING + LOCK_CONFLICT is a protocol violation (blocking conflicts * enqueue a waiter, never REJECT) → FAIL_INTERNAL, never NOT_AVAIL. */ @@ -979,6 +995,49 @@ UT_TEST(test_ul_try_lock_nowait_s4_reject_mapping) cluster_node_id = saved_node; } +UT_TEST(test_pcm_x_nested_guard_fails_before_ges_request_and_convert_waits) +{ + ClusterLockAcquireRequest req; + ClusterLockAcquireResult result; + int request_calls; + int convert_calls; + + memset(&req, 0, sizeof(req)); + req.lockmode = AccessExclusiveLock; + req.current_mode = AccessShareLock; + req.request_id = UINT64_C(88001); + request_calls = stub_ges_request_wait_calls; + convert_calls = stub_ges_convert_wait_calls; + + stub_pcm_x_nested_guard_result = PCM_X_QUEUE_BARRIER_CLOSED; + result = cluster_lock_acquire_s4_remote_request_wait(&req); + UT_ASSERT_EQ(result, CLUSTER_LOCK_ACQUIRE_FAIL_SHARD_REMASTERING); + UT_ASSERT_EQ(stub_ges_request_wait_calls, request_calls); + + req.op = CLUSTER_LOCK_OP_CONVERT; + result = cluster_lock_acquire_s5_promote(&req); + UT_ASSERT_EQ(result, CLUSTER_LOCK_ACQUIRE_FAIL_SHARD_REMASTERING); + UT_ASSERT_EQ(stub_ges_convert_wait_calls, convert_calls); + + stub_pcm_x_nested_guard_result = PCM_X_QUEUE_BUSY; + result = cluster_lock_acquire_s4_remote_request_wait(&req); + UT_ASSERT_EQ(result, CLUSTER_LOCK_ACQUIRE_FAIL_SHARD_REMASTERING); + UT_ASSERT_EQ(stub_ges_request_wait_calls, request_calls); + + stub_pcm_x_nested_guard_result = PCM_X_QUEUE_CORRUPT; + result = cluster_lock_acquire_s5_promote(&req); + UT_ASSERT_EQ(result, CLUSTER_LOCK_ACQUIRE_FAIL_INTERNAL); + UT_ASSERT_EQ(stub_ges_convert_wait_calls, convert_calls); + + /* A positive leg proves the convert stub counter is wired to the actual + * CONVERT sender rather than the unrelated REQUEST_NOWAIT sender. */ + stub_pcm_x_nested_guard_result = PCM_X_QUEUE_OK; + stub_ges_reject_reason = GES_REJECT_REASON_NONE; + result = cluster_lock_acquire_s5_promote(&req); + UT_ASSERT_EQ(result, CLUSTER_LOCK_ACQUIRE_OK_CONVERTED); + UT_ASSERT_EQ(stub_ges_convert_wait_calls, convert_calls + 1); +} + /* * spec-5.3 — native-probe PG parallel lock-group exemption helper. Pure @@ -1055,7 +1114,7 @@ UT_DEFINE_GLOBALS(); int main(int argc pg_attribute_unused(), char **const argv pg_attribute_unused()) { - UT_PLAN(14); + UT_PLAN(15); UT_RUN(test_7step_api_surface_linkable_and_initial_counters_zero); UT_RUN(test_7step_s1_hc1_fail_closed); @@ -1069,6 +1128,7 @@ main(int argc pg_attribute_unused(), char **const argv pg_attribute_unused()) UT_RUN(test_7step_transaction_locktag_release_path_safe); UT_RUN(test_ul_session_advisory_globalize_gate); UT_RUN(test_ul_try_lock_nowait_s4_reject_mapping); + UT_RUN(test_pcm_x_nested_guard_fails_before_ges_request_and_convert_waits); UT_RUN(test_native_probe_same_lock_group_exempt); UT_RUN(test_s5_not_found_benign_narrow); diff --git a/src/test/cluster_unit/test_cluster_pcm_direct_init.c b/src/test/cluster_unit/test_cluster_pcm_direct_init.c new file mode 100644 index 00000000000..2eeb07bec7e --- /dev/null +++ b/src/test/cluster_unit/test_cluster_pcm_direct_init.c @@ -0,0 +1,435 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_pcm_direct_init.c + * Operation-scoped exact proof for the S3 PCM direct-init exception. + * + * Author: SqlRush + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_pcm_direct_init.h" +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +#ifndef BUFMGR_SOURCE_PATH +#error "BUFMGR_SOURCE_PATH must identify production bufmgr.c" +#endif +#ifndef VM_SOURCE_PATH +#error "VM_SOURCE_PATH must identify production visibilitymap.c" +#endif +#ifndef FSM_SOURCE_PATH +#error "FSM_SOURCE_PATH must identify production freespace.c" +#endif + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +static BufferTag +make_tag(ForkNumber forknum, BlockNumber blocknum) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.spcOid = 1663; + tag.dbOid = 1; + tag.relNumber = 20000; + tag.forkNum = forknum; + tag.blockNum = blocknum; + return tag; +} + +static ClusterPcmDirectInitSnapshot +make_snapshot(ClusterPcmDirectInitKind kind) +{ + ClusterPcmDirectInitSnapshot snapshot; + + memset(&snapshot, 0, sizeof(snapshot)); + snapshot.buf_id = 7; + snapshot.tag = make_tag(MAIN_FORKNUM, 42); + snapshot.generation = 11; + snapshot.reservation_token = 4; + snapshot.private_refcount = 1; + snapshot.buffer_type = (uint8)BUF_TYPE_CURRENT; + snapshot.pcm_state = (uint8)PCM_STATE_N; + snapshot.page_is_new = true; + + if (kind == CLUSTER_PCM_DIRECT_INIT_VM) + snapshot.tag.forkNum = VISIBILITYMAP_FORKNUM; + else if (kind == CLUSTER_PCM_DIRECT_INIT_FSM) + snapshot.tag.forkNum = FSM_FORKNUM; + + if (kind == CLUSTER_PCM_DIRECT_INIT_VM || kind == CLUSTER_PCM_DIRECT_INIT_FSM) + snapshot.buf_state = BM_TAG_VALID | BM_VALID | 1; + else + snapshot.buf_state = BM_TAG_VALID | BM_IO_IN_PROGRESS | 1; + + return snapshot; +} + +static char * +read_source(const char *path) +{ + FILE *fp; + char *source; + long length; + + fp = fopen(path, "rb"); + UT_ASSERT(fp != NULL); + if (fp == NULL) + return NULL; + UT_ASSERT_EQ(fseek(fp, 0, SEEK_END), 0); + length = ftell(fp); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(fp, 0, SEEK_SET), 0); + source = malloc((size_t)length + 1); + UT_ASSERT(source != NULL); + if (source == NULL) { + fclose(fp); + return NULL; + } + UT_ASSERT_EQ((long)fread(source, 1, (size_t)length, fp), length); + source[length] = '\0'; + fclose(fp); + return source; +} + +static void +assert_ordered(const char *source, const char *const *needles, int count) +{ + const char *cursor = source; + + for (int i = 0; i < count; i++) { + cursor = strstr(cursor, needles[i]); + UT_ASSERT(cursor != NULL); + if (cursor == NULL) + return; + cursor += strlen(needles[i]); + } +} + +static void +expect_valid_round_trip(ClusterPcmDirectInitKind kind) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot snapshot = make_snapshot(kind); + + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(kind, &snapshot, &proof), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_consume(kind, &snapshot, &proof), + CLUSTER_PCM_OWN_OK); +} + +UT_TEST(test_valid_read_miss_proof) +{ + expect_valid_round_trip(CLUSTER_PCM_DIRECT_INIT_READ_MISS); +} + +UT_TEST(test_valid_extend_proof) +{ + expect_valid_round_trip(CLUSTER_PCM_DIRECT_INIT_EXTEND); +} + +UT_TEST(test_valid_vm_and_fsm_proofs) +{ + expect_valid_round_trip(CLUSTER_PCM_DIRECT_INIT_VM); + expect_valid_round_trip(CLUSTER_PCM_DIRECT_INIT_FSM); +} + +UT_TEST(test_proof_is_single_use_and_kind_exact) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_READ_MISS); + + UT_ASSERT_EQ( + cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_READ_MISS, &snapshot, &proof), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ( + cluster_pcm_direct_init_proof_consume(CLUSTER_PCM_DIRECT_INIT_EXTEND, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ( + cluster_pcm_direct_init_proof_consume(CLUSTER_PCM_DIRECT_INIT_READ_MISS, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); +} + +UT_TEST(test_identity_mismatch_rejects_buf_tag_generation_and_token) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot base = make_snapshot(CLUSTER_PCM_DIRECT_INIT_EXTEND); + ClusterPcmDirectInitSnapshot changed; + +#define EXPECT_IDENTITY_REJECT(field, value) \ + do { \ + changed = base; \ + UT_ASSERT_EQ( \ + cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_EXTEND, &base, &proof), \ + CLUSTER_PCM_OWN_OK); \ + changed.field = (value); \ + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_consume(CLUSTER_PCM_DIRECT_INIT_EXTEND, \ + &changed, &proof), \ + CLUSTER_PCM_OWN_STALE); \ + } while (0) + + EXPECT_IDENTITY_REJECT(buf_id, 8); + EXPECT_IDENTITY_REJECT(generation, 12); + EXPECT_IDENTITY_REJECT(reservation_token, 5); + EXPECT_IDENTITY_REJECT(private_refcount, 2); + + changed = base; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_EXTEND, &base, &proof), + CLUSTER_PCM_OWN_OK); + changed.tag.blockNum++; + UT_ASSERT_EQ( + cluster_pcm_direct_init_proof_consume(CLUSTER_PCM_DIRECT_INIT_EXTEND, &changed, &proof), + CLUSTER_PCM_OWN_STALE); +#undef EXPECT_IDENTITY_REJECT +} + +UT_TEST(test_reuse_dirty_and_shape_are_rejected) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot base = make_snapshot(CLUSTER_PCM_DIRECT_INIT_READ_MISS); + ClusterPcmDirectInitSnapshot changed; + +#define EXPECT_SHAPE_REJECT(mutator) \ + do { \ + changed = base; \ + UT_ASSERT_EQ( \ + cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_READ_MISS, &base, &proof), \ + CLUSTER_PCM_OWN_OK); \ + mutator; \ + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_consume(CLUSTER_PCM_DIRECT_INIT_READ_MISS, \ + &changed, &proof), \ + CLUSTER_PCM_OWN_STALE); \ + } while (0) + + EXPECT_SHAPE_REJECT(changed.buf_state |= BM_DIRTY); + EXPECT_SHAPE_REJECT(changed.buf_state |= BM_JUST_DIRTIED); + EXPECT_SHAPE_REJECT(changed.buf_state |= BM_VALID); + EXPECT_SHAPE_REJECT(changed.buf_state &= ~BM_IO_IN_PROGRESS); + EXPECT_SHAPE_REJECT(changed.buffer_type = (uint8)BUF_TYPE_PI); + EXPECT_SHAPE_REJECT(changed.page_is_new = false); + EXPECT_SHAPE_REJECT(changed.tag.blockNum++); +#undef EXPECT_SHAPE_REJECT +} + +UT_TEST(test_state_s_and_live_reservations_are_rejected_by_class) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_VM); + + snapshot.pcm_state = (uint8)PCM_STATE_S; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_VM, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); + + snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_VM); + snapshot.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_VM, &snapshot, &proof), + CLUSTER_PCM_OWN_BUSY); + + snapshot.flags = PCM_OWN_FLAG_REVOKING; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_VM, &snapshot, &proof), + CLUSTER_PCM_OWN_BUSY); + + snapshot.flags = PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_VM, &snapshot, &proof), + CLUSTER_PCM_OWN_CORRUPT); +} + +UT_TEST(test_revalidate_rejects_state_pin_and_reservation_changes) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot base = make_snapshot(CLUSTER_PCM_DIRECT_INIT_VM); + ClusterPcmDirectInitSnapshot changed; + +#define EXPECT_REVALIDATE_REJECT(mutator, expected_result) \ + do { \ + changed = base; \ + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_VM, &base, &proof), \ + CLUSTER_PCM_OWN_OK); \ + mutator; \ + UT_ASSERT_EQ( \ + cluster_pcm_direct_init_proof_consume(CLUSTER_PCM_DIRECT_INIT_VM, &changed, &proof), \ + expected_result); \ + } while (0) + + EXPECT_REVALIDATE_REJECT(changed.pcm_state = (uint8)PCM_STATE_S, CLUSTER_PCM_OWN_STALE); + EXPECT_REVALIDATE_REJECT(changed.private_refcount = 0, CLUSTER_PCM_OWN_STALE); + EXPECT_REVALIDATE_REJECT(changed.flags = PCM_OWN_FLAG_GRANT_PENDING, CLUSTER_PCM_OWN_BUSY); + EXPECT_REVALIDATE_REJECT(changed.flags = PCM_OWN_FLAG_REVOKING, CLUSTER_PCM_OWN_BUSY); + EXPECT_REVALIDATE_REJECT(changed.flags = PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING, + CLUSTER_PCM_OWN_CORRUPT); +#undef EXPECT_REVALIDATE_REJECT +} + +UT_TEST(test_missing_backend_pin_is_rejected) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_FSM); + + snapshot.private_refcount = 0; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_FSM, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); + + snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_FSM); + snapshot.buf_state &= ~BUF_REFCOUNT_MASK; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_FSM, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); +} + +UT_TEST(test_vm_fsm_fork_and_valid_shape_are_exact) +{ + ClusterPcmDirectInitProof proof; + ClusterPcmDirectInitSnapshot snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_VM); + + snapshot.tag.forkNum = FSM_FORKNUM; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_VM, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); + + snapshot = make_snapshot(CLUSTER_PCM_DIRECT_INIT_FSM); + snapshot.buf_state &= ~BM_VALID; + snapshot.buf_state |= BM_IO_IN_PROGRESS; + UT_ASSERT_EQ(cluster_pcm_direct_init_proof_arm(CLUSTER_PCM_DIRECT_INIT_FSM, &snapshot, &proof), + CLUSTER_PCM_OWN_STALE); +} + +UT_TEST(test_bufmgr_consumes_proof_before_reservation_and_wire) +{ + char *source = read_source(BUFMGR_SOURCE_PATH); + static const char *const order[] + = { "cluster_bufmgr_pcm_gate_direct_init(", "cluster_pcm_direct_init_proof_consume", + "cluster_pcm_own_reservation_begin_exact", "cluster_pcm_lock_acquire_buffer" }; + + UT_ASSERT(source != NULL); + if (source != NULL) { + assert_ordered(source, order, lengthof(order)); + free(source); + } +} + +UT_TEST(test_read_miss_and_found_hit_have_no_raw_unproven_lock) +{ + char *source = read_source(BUFMGR_SOURCE_PATH); + static const char *const miss_order[] + = { "MemSet((char *) bufBlock, 0, BLCKSZ)", "CLUSTER_PCM_DIRECT_INIT_READ_MISS", + "cluster_bufmgr_pcm_gate_direct_init", + "LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE)" }; + static const char *const found_order[] + = { "if (found)", "if (mode == RBM_ZERO_AND_LOCK)", + "LockBuffer(BufferDescriptorGetBuffer(bufHdr), BUFFER_LOCK_EXCLUSIVE)" }; + + UT_ASSERT(source != NULL); + if (source != NULL) { + assert_ordered(source, miss_order, lengthof(miss_order)); + assert_ordered(source, found_order, lengthof(found_order)); + free(source); + } +} + +UT_TEST(test_extend_proof_is_after_zeroextend_and_before_lock) +{ + char *source = read_source(BUFMGR_SOURCE_PATH); + static const char *const order[] + = { "smgrzeroextend(bmr.smgr, fork, first_block, extend_by, false)", + "CLUSTER_PCM_DIRECT_INIT_EXTEND", "cluster_bufmgr_pcm_gate_direct_init", + "LWLockAcquire(BufferDescriptorGetContentLock(buf_hdr), LW_EXCLUSIVE)", + "TerminateBufferIO(buf_hdr, false, BM_VALID)" }; + + UT_ASSERT(source != NULL); + if (source != NULL) { + assert_ordered(source, order, lengthof(order)); + free(source); + } +} + +UT_TEST(test_vm_fsm_use_dedicated_init_wrappers) +{ + char *vm_source = read_source(VM_SOURCE_PATH); + char *fsm_source = read_source(FSM_SOURCE_PATH); + + UT_ASSERT(vm_source != NULL); + UT_ASSERT(fsm_source != NULL); + if (vm_source != NULL) { + UT_ASSERT(strstr(vm_source, "LockBufferForVisibilityMapPageInit(buf)") != NULL); + free(vm_source); + } + if (fsm_source != NULL) { + UT_ASSERT(strstr(fsm_source, "LockBufferForFreeSpaceMapPageInit(buf)") != NULL); + free(fsm_source); + } +} + +UT_TEST(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire) +{ + char *source = read_source(BUFMGR_SOURCE_PATH); + static const char *const order[] + = { "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode)", + "cluster_pcm_own_begin_grant_reservation(buf, &pcm_pending_base", + "cluster_pcm_lock_acquire_buffer(buf, pcm_mode)" }; + + UT_ASSERT(source != NULL); + if (source != NULL) { + assert_ordered(source, order, lengthof(order)); + free(source); + } +} + +UT_TEST(test_direct_init_one_shot_image_cannot_return_without_x) +{ + char *source = read_source(BUFMGR_SOURCE_PATH); + static const char *const order[] + = { "cluster_pcm_own_abort_grant_or_error(buf, &pending_base, pending_token", + "cluster_bufmgr_pcm_direct_init_no_grant_failclosed" }; + + UT_ASSERT(source != NULL); + if (source != NULL) { + assert_ordered(source, order, lengthof(order)); + free(source); + } +} + +UT_TEST(test_wire_throw_exact_aborts_reservation_before_rethrow) +{ + char *source = read_source(BUFMGR_SOURCE_PATH); + static const char *const order[] + = { "cluster_bufmgr_pcm_gate_direct_init(", "PG_CATCH();", + "cluster_pcm_own_abort_grant_after_error(buf, &pending_base, pending_token", + "\"direct-init acquire\"", "PG_RE_THROW();" }; + + UT_ASSERT(source != NULL); + if (source != NULL) { + assert_ordered(source, order, lengthof(order)); + free(source); + } +} + +int +main(void) +{ + UT_PLAN(17); + UT_RUN(test_valid_read_miss_proof); + UT_RUN(test_valid_extend_proof); + UT_RUN(test_valid_vm_and_fsm_proofs); + UT_RUN(test_proof_is_single_use_and_kind_exact); + UT_RUN(test_identity_mismatch_rejects_buf_tag_generation_and_token); + UT_RUN(test_reuse_dirty_and_shape_are_rejected); + UT_RUN(test_state_s_and_live_reservations_are_rejected_by_class); + UT_RUN(test_revalidate_rejects_state_pin_and_reservation_changes); + UT_RUN(test_missing_backend_pin_is_rejected); + UT_RUN(test_vm_fsm_fork_and_valid_shape_are_exact); + UT_RUN(test_bufmgr_consumes_proof_before_reservation_and_wire); + UT_RUN(test_read_miss_and_found_hit_have_no_raw_unproven_lock); + UT_RUN(test_extend_proof_is_after_zeroextend_and_before_lock); + UT_RUN(test_vm_fsm_use_dedicated_init_wrappers); + UT_RUN(test_valid_n_s_x_without_proof_enters_queue_before_legacy_wire); + UT_RUN(test_direct_init_one_shot_image_cannot_return_without_x); + UT_RUN(test_wire_throw_exact_aborts_reservation_before_rethrow); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index 527e8a4f339..1e509023051 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -953,6 +953,23 @@ UT_TEST(test_pcm_real_summary_counts_live_entries) UT_ASSERT_EQ(pi_total, 1); } +UT_TEST(test_pcm_grd_entry_abi_remains_264_bytes) +{ + reset_fake_pcm_runtime(4); + UT_ASSERT_EQ(fake_pcm_entrysize, 264); +} + +UT_TEST(test_pcm_grd_convert_queue_placeholder_remains_null) +{ + BufferTag tag = make_tag(64); + int n_count, s_count, x_count, pi_total, convert_q; + + reset_fake_pcm_runtime(4); + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + cluster_pcm_grd_get_summary(&n_count, &s_count, &x_count, &pi_total, &convert_q); + UT_ASSERT_EQ(convert_q, 0); +} + UT_TEST(test_pcm_real_wait_event_call_sites_are_exercised) { @@ -1495,6 +1512,341 @@ UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) UT_ASSERT_EQ(cluster_pcm_lock_clear_pending_x_if(stag, 2), false); } +static PcmXGrdHandoffToken +make_pcm_x_grd_handoff_token(BufferTag tag, const PcmAuthoritySnapshot *authority, + int32 source_node, int32 requester_node, uint32 requester_procno, + uint64 request_id) +{ + PcmXGrdHandoffToken token; + + memset(&token, 0, sizeof(token)); + token.tag = tag; + token.authority = *authority; + token.cluster_epoch = 17; + token.request_id = request_id; + token.ticket_id = request_id; + token.grant_generation = 23; + UT_ASSERT(cluster_pcm_x_image_id_encode(0, 29, &token.image_id)); + token.source_own_generation = 31; + token.page_scn = 0x4000; + token.page_lsn = 0x5000; + token.requester_node = requester_node; + token.source_node = source_node; + token.requester_procno = requester_procno; + token.page_checksum = 37; + return token; +} + +UT_TEST(test_pcm_authority_snapshot_is_one_entry_lock_view) +{ + BufferTag tag = make_tag(97); + PcmAuthoritySnapshot snapshot; + + reset_fake_pcm_runtime(4); + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + cluster_pcm_lock_set_pending_x(tag, 3, 0x1234); + + memset(&snapshot, 0x7f, sizeof(snapshot)); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ((int)snapshot.state, (int)PCM_STATE_S); + UT_ASSERT_EQ(snapshot.x_holder_node, -1); + UT_ASSERT_EQ(snapshot.s_holders_bitmap, (uint32)((1u << 1) | (1u << 2))); + UT_ASSERT_EQ(snapshot.master_holder.node_id, (uint32)1); + UT_ASSERT_EQ(snapshot.pending_x_requester_node, 3); + UT_ASSERT_EQ(snapshot.pending_x_since_lsn, (uint64)0x1234); + UT_ASSERT(snapshot.transition_count > 0); +} + +UT_TEST(test_pcm_queue_pending_x_reservation_never_overwrites_another_node) +{ + BufferTag tag = make_tag(101); + PcmAuthoritySnapshot snapshot; + + reset_fake_pcm_runtime(4); + cluster_node_id = 1; + /* Both producers lazily create the canonical N authority for a cold tag. */ + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 2, 0x1111), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ((int)snapshot.state, (int)PCM_STATE_N); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 2, 0x1111)); + /* A delayed same-node legacy clear cannot erase a queue-kind claim. */ + UT_ASSERT(!cluster_pcm_lock_clear_pending_x_if(tag, 2)); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 2, 0x1111)); + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 2, 0x1111)); + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 2, 0x1111), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ(snapshot.pending_x_requester_node, 2); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 2, 0x1111)); + + /* Legacy producers obey the same idle-only rule, including same-node. */ + UT_ASSERT_EQ(cluster_pcm_lock_set_pending_x(tag, 3, 0x1212), PCM_PENDING_X_RESERVE_OCCUPIED); + UT_ASSERT_EQ(cluster_pcm_lock_set_pending_x(tag, 2, 0x1313), PCM_PENDING_X_RESERVE_OCCUPIED); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ(snapshot.pending_x_requester_node, 2); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 2, 0x1111)); + + /* A different queue head cannot overwrite the live barrier. */ + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 0x2222), + PCM_PENDING_X_RESERVE_OCCUPIED); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ(snapshot.pending_x_requester_node, 2); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 2, 0x1111)); + + /* A same-node legacy round is not ticket-exact replay proof. */ + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 2, 0x3333), + PCM_PENDING_X_RESERVE_OCCUPIED); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ(snapshot.pending_x_requester_node, 2); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 2, 0x1111)); + + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 2, 0x1111)); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 0x4444), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ(snapshot.pending_x_requester_node, 3); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 3, 0x4444)); + /* A replay of the old release cannot erase the successor cookie. */ + UT_ASSERT(!cluster_pcm_lock_clear_queue_pending_x_exact(tag, 2, 0x1111)); + UT_ASSERT(cluster_pcm_lock_queue_pending_x_exact(tag, 3, 0x4444)); + UT_ASSERT(cluster_pcm_lock_clear_queue_pending_x_exact(tag, 3, 0x4444)); + + tag = make_tag(102); + UT_ASSERT_EQ(cluster_pcm_lock_set_pending_x(tag, 2, 0x5555), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &snapshot)); + UT_ASSERT_EQ((int)snapshot.state, (int)PCM_STATE_N); + UT_ASSERT(cluster_pcm_lock_clear_pending_x_if(tag, 2)); + + reset_fake_pcm_runtime(1); + UT_ASSERT_EQ(cluster_pcm_lock_set_pending_x(make_tag(103), 2, 0x6666), + PCM_PENDING_X_RESERVE_OK); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(make_tag(104), 3, 0x7777), + PCM_PENDING_X_RESERVE_NO_CAPACITY); +} + +UT_TEST(test_pcm_queue_handoff_x_exact_rejects_authority_drift) +{ + BufferTag tag = make_tag(98); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken token; + + reset_fake_pcm_runtime(4); + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9001), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + token = make_pcm_x_grd_handoff_token(tag, &before, 1, 3, 41, 9001); + + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_STALE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_S); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)((1u << 1) | (1u << 2))); +} + +UT_TEST(test_pcm_queue_handoff_x_exact_rejects_residual_s_holder) +{ + BufferTag tag = make_tag(99); + PcmAuthoritySnapshot before; + PcmXGrdHandoffToken token; + + reset_fake_pcm_runtime(4); + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9002), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + token = make_pcm_x_grd_handoff_token(tag, &before, 1, 3, 42, 9002); + + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_BAD_STATE); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_STATE_S); +} + +UT_TEST(test_pcm_queue_handoff_x_exact_commits_full_identity_and_replays) +{ + BufferTag tag = make_tag(100); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken token; + + reset_fake_pcm_runtime(4); + cluster_node_id = 1; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9003), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + token = make_pcm_x_grd_handoff_token(tag, &before, 1, 3, 43, 9003); + token.cluster_epoch = 0; + token.source_own_generation = 0; + { + PcmXGrdHandoffToken wrong_ticket = token; + + wrong_ticket.ticket_id++; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&wrong_ticket), + PCM_X_GRD_HANDOFF_BAD_STATE); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_STATE_S); + } + + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 3); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)0); + UT_ASSERT_EQ(after.pending_x_requester_node, -1); + UT_ASSERT_EQ(after.master_holder.node_id, (uint32)3); + UT_ASSERT_EQ(after.master_holder.procno, (uint32)43); + UT_ASSERT_EQ(after.master_holder.cluster_epoch, (uint64)0); + UT_ASSERT_EQ(after.master_holder.request_id, (uint64)9003); + UT_ASSERT_EQ(after.transition_count, before.transition_count + 1); + /* FINAL preserves page_lsn in the immutable A-record only. It must not + * promote one node's WAL position into the cross-node GRD version floor. */ + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_lsn_query(tag), (uint64)InvalidXLogRecPtr); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_scn_query(tag), (uint64)0x4000); + + /* SCN-only authority is a legal shape. Exact replay must not depend on a + * same-stream LSN floor that the GRD cannot establish across nodes. */ + cluster_pcm_lock_pi_watermark_retire_for_tag(tag); + cluster_pcm_lock_pi_watermark_scn_advance(tag, token.page_scn, CLUSTER_PCM_WM_SRC_REDECLARE, 1, + 7000, 17); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_lsn_query(tag), (uint64)InvalidXLogRecPtr); + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_DUPLICATE); +} + +UT_TEST(test_pcm_queue_handoff_x_exact_accepts_global_n_with_real_image) +{ + BufferTag tag = make_tag(105); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken token; + + reset_fake_pcm_runtime(4); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9004), PCM_PENDING_X_RESERVE_OK); + cluster_pcm_lock_pi_watermark_lsn_advance(tag, (XLogRecPtr)0x4000); + cluster_pcm_lock_pi_watermark_scn_advance(tag, (SCN)0x3000, CLUSTER_PCM_WM_SRC_REDECLARE, 1, + 7001, 17); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + UT_ASSERT_EQ((int)before.state, (int)PCM_STATE_N); + UT_ASSERT_EQ(before.master_holder.node_id, UINT32_MAX); + token = make_pcm_x_grd_handoff_token(tag, &before, 3, 3, 44, 9004); + + UT_ASSERT(token.image_id != 0); + { + PcmXGrdHandoffToken malformed = token; + + malformed.source_node = 2; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&malformed), + PCM_X_GRD_HANDOFF_BAD_STATE); + malformed = token; + malformed.ticket_id++; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&malformed), + PCM_X_GRD_HANDOFF_BAD_STATE); + malformed = token; + malformed.image_id = 29; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&malformed), PCM_X_GRD_HANDOFF_INVALID); + } + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 3); + UT_ASSERT_EQ(after.s_holders_bitmap, (uint32)0); + UT_ASSERT_EQ(after.pending_x_requester_node, -1); + UT_ASSERT_EQ(after.master_holder.node_id, (uint32)3); + UT_ASSERT_EQ(after.master_holder.procno, (uint32)44); + UT_ASSERT_EQ(after.transition_count, before.transition_count + 1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_lsn_query(tag), (uint64)0x4000); +} + +UT_TEST(test_pcm_queue_handoff_x_exact_accepts_ordered_self_x) +{ + BufferTag tag = make_tag(106); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken token; + + reset_fake_pcm_runtime(4); + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 2, 9005), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + UT_ASSERT_EQ((int)before.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(before.x_holder_node, 2); + token = make_pcm_x_grd_handoff_token(tag, &before, 2, 2, 45, 9005); + + UT_ASSERT(token.image_id != 0); + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.x_holder_node, 2); + UT_ASSERT_EQ(after.pending_x_requester_node, -1); + UT_ASSERT_EQ(after.master_holder.procno, (uint32)45); + UT_ASSERT_EQ(after.transition_count, before.transition_count + 1); +} + +UT_TEST(test_pcm_queue_handoff_x_exact_uses_scn_not_cross_stream_lsn) +{ + BufferTag tag = make_tag(107); + PcmAuthoritySnapshot before; + PcmAuthoritySnapshot after; + PcmXGrdHandoffToken token; + + reset_fake_pcm_runtime(4); + cluster_node_id = 3; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9006), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + token = make_pcm_x_grd_handoff_token(tag, &before, 3, 3, 46, 9006); + + /* Per-node WAL streams make page_lsn incomparable across holders. A + * numerically larger historical LSN must not reject an image whose Lamport + * page SCN is newer; page_lsn remains A-record evidence, not GRD version + * authority. */ + cluster_pcm_lock_pi_watermark_lsn_advance(tag, (XLogRecPtr)(token.page_lsn + 1)); + cluster_pcm_lock_pi_watermark_scn_advance(tag, (SCN)(token.page_scn - 1), + CLUSTER_PCM_WM_SRC_REDECLARE, 1, 7002, 17); + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.pending_x_requester_node, -1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_lsn_query(tag), + (uint64)(token.page_lsn + 1)); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_pi_watermark_scn_query(tag), (uint64)token.page_scn); + + tag = make_tag(108); + reset_fake_pcm_runtime(4); + cluster_node_id = 3; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9007), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + token = make_pcm_x_grd_handoff_token(tag, &before, 3, 3, 47, 9007); + cluster_pcm_lock_pi_watermark_scn_advance(tag, (SCN)(token.page_scn + 1), + CLUSTER_PCM_WM_SRC_REDECLARE, 1, 7003, 17); + token.page_scn = (SCN)0; + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_BAD_STATE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.pending_x_requester_node, 3); + + tag = make_tag(109); + reset_fake_pcm_runtime(4); + cluster_node_id = 3; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ(cluster_pcm_lock_try_reserve_pending_x(tag, 3, 9008), PCM_PENDING_X_RESERVE_OK); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &before)); + token = make_pcm_x_grd_handoff_token(tag, &before, 3, 3, 48, 9008); + cluster_pcm_lock_pi_watermark_scn_advance(tag, (SCN)(token.page_scn + 1), + CLUSTER_PCM_WM_SRC_REDECLARE, 1, 7004, 17); + UT_ASSERT_EQ(cluster_pcm_lock_queue_handoff_x_exact(&token), PCM_X_GRD_HANDOFF_BAD_STATE); + UT_ASSERT(cluster_pcm_lock_authority_snapshot(tag, &after)); + UT_ASSERT_EQ((int)after.state, (int)PCM_STATE_X); + UT_ASSERT_EQ(after.pending_x_requester_node, 3); +} + /* spec-5.2a D2 (U1): clean-page X-transfer arm is one-shot. arm(true) sets * the backend-local flag; consume() reads-and-clears it (the acquire path * calls consume() once so the eligibility can never leak into a SUBSEQUENT @@ -1528,7 +1880,7 @@ UT_TEST(test_clean_page_xfer_arm_is_one_shot) int main(void) { - UT_PLAN(42); + UT_PLAN(52); UT_RUN(test_pcm_lock_mode_constant_aliases_match_pcm_state); UT_RUN(test_pcm_lock_transition_count_is_9); UT_RUN(test_pcm_lock_transition_enum_values_are_1_to_9); @@ -1554,6 +1906,8 @@ main(void) UT_RUN(test_pcm_real_x_release_and_downgrade_require_owner); UT_RUN(test_pcm_real_upgrade_requires_single_s_holder); UT_RUN(test_pcm_real_summary_counts_live_entries); + UT_RUN(test_pcm_grd_entry_abi_remains_264_bytes); + UT_RUN(test_pcm_grd_convert_queue_placeholder_remains_null); UT_RUN(test_pcm_real_wait_event_call_sites_are_exercised); UT_RUN(test_pcm_H1_same_node_s_refcount_increments); UT_RUN(test_pcm_H2_last_s_release_transitions_to_n); @@ -1570,6 +1924,14 @@ main(void) UT_RUN(test_pcm_wm_prov_table_keeps_last_advance); UT_RUN(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades); UT_RUN(test_pcm_dead_node_cleanup_drops_holder_records); + UT_RUN(test_pcm_authority_snapshot_is_one_entry_lock_view); + UT_RUN(test_pcm_queue_pending_x_reservation_never_overwrites_another_node); + UT_RUN(test_pcm_queue_handoff_x_exact_rejects_authority_drift); + UT_RUN(test_pcm_queue_handoff_x_exact_rejects_residual_s_holder); + UT_RUN(test_pcm_queue_handoff_x_exact_commits_full_identity_and_replays); + UT_RUN(test_pcm_queue_handoff_x_exact_accepts_global_n_with_real_image); + UT_RUN(test_pcm_queue_handoff_x_exact_accepts_ordered_self_x); + UT_RUN(test_pcm_queue_handoff_x_exact_uses_scn_not_cross_stream_lsn); UT_RUN(test_clean_page_xfer_arm_is_one_shot); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_pcm_own.c b/src/test/cluster_unit/test_cluster_pcm_own.c new file mode 100644 index 00000000000..fe4ac590081 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_pcm_own.c @@ -0,0 +1,2023 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_pcm_own.c + * C1 ownership-reservation and D5a buffer-reuse contract tests. + * + * This binary links the production cluster_pcm_own object. Buffer-manager + * behavior that cannot be linked standalone is covered in two paired ways: + * the real reusable decision helpers are exercised here, and the production + * bufmgr source is checked to prove both eviction paths call those helpers + * before dropping header authority and use the saved-tag release API. + * + * Author: SqlRush + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_pcm_own.h" +#include "cluster/cluster_pcm_x_bufmgr.h" +#include "cluster/cluster_shmem.h" + +#include "unit_test.h" + +#include + +UT_DEFINE_GLOBALS(); + +int NBuffers = 4; + +static union { + uint64 align; + char bytes[4096]; +} fake_shmem; +static bool fake_found; + +static char *read_bufmgr_source(void); +static void assert_ordered_in_function(const char *source, const char *function_start, + const char *function_end, const char *const *needles, + int needle_count); +static void assert_source_range_contains(const char *start, const char *end, const char *needle); + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +void * +ShmemInitStruct(const char *name pg_attribute_unused(), Size size, bool *foundPtr) +{ + UT_ASSERT(size <= sizeof(fake_shmem.bytes)); + *foundPtr = fake_found; + fake_found = true; + return fake_shmem.bytes; +} + +Size +mul_size(Size s1, Size s2) +{ + return s1 * s2; +} + +void +cluster_shmem_register_region(const ClusterShmemRegion *region pg_attribute_unused()) +{} + +static void +reset_fixture(void) +{ + memset(&fake_shmem, 0xA5, sizeof(fake_shmem)); + fake_found = false; + ClusterPcmOwnArray = NULL; + cluster_pcm_own_shmem_init(); +} + +static void +assert_entry(uint64 generation, uint64 token, uint32 flags) +{ + UT_ASSERT_EQ(pg_atomic_read_u64(&ClusterPcmOwnArray[0].generation), generation); + UT_ASSERT_EQ(pg_atomic_read_u64(&ClusterPcmOwnArray[0].reservation_token), token); + UT_ASSERT_EQ(pg_atomic_read_u32(&ClusterPcmOwnArray[0].flags), flags); +} + +UT_TEST(test_shmem_initializes_complete_entry) +{ + int i; + + reset_fixture(); + UT_ASSERT_EQ(cluster_pcm_own_shmem_size(), (Size)NBuffers * sizeof(ClusterPcmOwnEntry)); + UT_ASSERT_EQ(sizeof(ClusterPcmOwnEntry), 24); + for (i = 0; i < NBuffers; i++) { + UT_ASSERT_EQ(pg_atomic_read_u64(&ClusterPcmOwnArray[i].generation), 0); + UT_ASSERT_EQ(pg_atomic_read_u64(&ClusterPcmOwnArray[i].reservation_token), 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&ClusterPcmOwnArray[i].flags), 0); + } +} + +UT_TEST(test_begin_abort_is_exact_and_monotonic) +{ + uint64 token = UINT64_MAX; + + reset_fixture(); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(token, 1); + assert_entry(0, 1, PCM_OWN_FLAG_GRANT_PENDING); + + /* A second begin cannot overwrite or advance the live token. */ + token = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_BUSY); + UT_ASSERT_EQ(token, 0); + assert_entry(0, 1, PCM_OWN_FLAG_GRANT_PENDING); + + /* Old/wrong cleanup is a strict no-op. */ + UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 0, 2, PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 1, 1, PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_STALE); + assert_entry(0, 1, PCM_OWN_FLAG_GRANT_PENDING); + + UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 0, 1, PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_OK); + assert_entry(0, 1, 0); + + /* A delayed duplicate abort cannot clear the next reservation. */ + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(token, 2); + UT_ASSERT_EQ(cluster_pcm_own_reservation_abort_exact(0, 0, 1, PCM_OWN_FLAG_GRANT_PENDING), + CLUSTER_PCM_OWN_STALE); + assert_entry(0, 2, PCM_OWN_FLAG_GRANT_PENDING); +} + +UT_TEST(test_invalid_live_flag_shapes_are_corrupt_not_busy) +{ + static const char *const classifier_contract[] + = { "cluster_pcm_own_reservation_token_get", "cluster_pcm_own_flags_get", + "cluster_pcm_own_classify_live_flags", "live_result != CLUSTER_PCM_OWN_OK", + "return live_result" }; + char *source; + uint64 token = UINT64_MAX; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 7); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, + PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_CORRUPT); + UT_ASSERT_EQ(token, 0); + assert_entry(0, 7, PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 7); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, (uint32)0x4); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_CORRUPT); + UT_ASSERT_EQ(token, 0); + assert_entry(0, 7, (uint32)0x4); + + /* Even a recognized singleton flag is corrupt without a published token. */ + reset_fixture(); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_CORRUPT); + UT_ASSERT_EQ(token, 0); + assert_entry(0, 0, PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_classify_live_flags(0, 0), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(cluster_pcm_own_classify_live_flags(0, 7), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(cluster_pcm_own_classify_live_flags(PCM_OWN_FLAG_GRANT_PENDING, 7), + CLUSTER_PCM_OWN_BUSY); + UT_ASSERT_EQ(cluster_pcm_own_classify_live_flags(PCM_OWN_FLAG_REVOKING, 7), + CLUSTER_PCM_OWN_BUSY); + UT_ASSERT_EQ( + cluster_pcm_own_classify_live_flags(PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING, 7), + CLUSTER_PCM_OWN_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_own_classify_live_flags(PCM_OWN_FLAG_GRANT_PENDING, 0), + CLUSTER_PCM_OWN_CORRUPT); + + source = read_bufmgr_source(); + assert_ordered_in_function(source, "\ncluster_pcm_own_bump_failure(", "\nstatic ", + classifier_contract, lengthof(classifier_contract)); + free(source); +} + +UT_TEST(test_grant_commit_is_exact_and_bumps_once) +{ + uint64 token; + uint64 committed = UINT64_MAX; + + reset_fixture(); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 0, token + 1, &committed), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(committed, 0); + assert_entry(0, token, PCM_OWN_FLAG_GRANT_PENDING); + + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 0, token, &committed), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(committed, 1); + assert_entry(1, token, 0); + + committed = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 0, token, &committed), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(committed, 0); + assert_entry(1, token, 0); + + /* A competing well-formed lifecycle is BUSY; a malformed tuple is + * corruption. Neither may be flattened into a retryable stale result. */ + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 9); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, PCM_OWN_FLAG_REVOKING); + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 0, 9, &committed), CLUSTER_PCM_OWN_BUSY); + assert_entry(0, 9, PCM_OWN_FLAG_REVOKING); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, + PCM_OWN_FLAG_REVOKING | PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 0, 9, &committed), CLUSTER_PCM_OWN_CORRUPT); + assert_entry(0, 9, PCM_OWN_FLAG_REVOKING | PCM_OWN_FLAG_GRANT_PENDING); +} + +UT_TEST(test_s_revoke_handoff_reuses_exact_token_and_bumps_once) +{ + uint64 committed = UINT64_MAX; + uint64 token; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_REVOKING, &token), + CLUSTER_PCM_OWN_OK); + assert_entry(7, token, PCM_OWN_FLAG_REVOKING); + + /* Stale identities cannot steal or rewrite the source revoke lifecycle. */ + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 8, token), CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token + 1), + CLUSTER_PCM_OWN_STALE); + assert_entry(7, token, PCM_OWN_FLAG_REVOKING); + + /* Handoff changes only the role of the same source lifecycle. */ + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token), CLUSTER_PCM_OWN_OK); + assert_entry(7, token, PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token), CLUSTER_PCM_OWN_OK); + assert_entry(7, token, PCM_OWN_FLAG_GRANT_PENDING); + + /* Malformed live tuples are corruption, never a stale/duplicate handoff. */ + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, token); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, + PCM_OWN_FLAG_REVOKING | PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token), + CLUSTER_PCM_OWN_CORRUPT); + assert_entry(7, token, PCM_OWN_FLAG_REVOKING | PCM_OWN_FLAG_GRANT_PENDING); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 0); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, PCM_OWN_FLAG_REVOKING); + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token), + CLUSTER_PCM_OWN_CORRUPT); + assert_entry(7, 0, PCM_OWN_FLAG_REVOKING); + + /* Restore the exact handed-off tuple before its sole generation bump. */ + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, token); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 7, token, &committed), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(committed, 8); + assert_entry(8, token, 0); + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token), CLUSTER_PCM_OWN_STALE); +} + +UT_TEST(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle) +{ + ClusterPcmOwnSnapshot base; + ClusterPcmOwnSnapshot live; + uint64 committed = UINT64_MAX; + uint64 token; + uint8 states[] = { (uint8)PCM_STATE_N, (uint8)PCM_STATE_S, (uint8)PCM_STATE_X }; + ClusterPcmXGrantReservationKind expected_kinds[] + = { CLUSTER_PCM_X_GRANT_RESERVATION_N_REVOKE_HANDOFF, + CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF, + CLUSTER_PCM_X_GRANT_RESERVATION_X_REVOKE_HANDOFF }; + int i; + + /* The added handoff arms must not broaden or shadow the ordinary new-token + * N reservation. */ + memset(&base, 0, sizeof(base)); + base.generation = 7; + base.reservation_token = 4; + base.pcm_state = (uint8)PCM_STATE_N; + live = base; + live.reservation_token = 5; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 5), + CLUSTER_PCM_X_GRANT_RESERVATION_N_NEW); + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, 4), + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); + + for (i = 0; i < lengthof(states); i++) { + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_REVOKING, &token), + CLUSTER_PCM_OWN_OK); + + memset(&base, 0, sizeof(base)); + base.generation = 7; + base.reservation_token = token; + base.flags = PCM_OWN_FLAG_REVOKING; + base.pcm_state = states[i]; + live = base; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, token), expected_kinds[i]); + UT_ASSERT_EQ(cluster_pcm_own_revoke_to_grant_handoff_exact(0, 7, token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, 7, token, &committed), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(committed, 8); + assert_entry(8, token, 0); + } +} + +UT_TEST(test_revoke_commit_is_exact_and_classifies_live_races) +{ + uint64 committed = UINT64_MAX; + uint64 token; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_REVOKING, &token), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(token, 1); + assert_entry(7, token, PCM_OWN_FLAG_REVOKING); + + /* A delayed/wrong lifecycle must not clear the current revoke. */ + UT_ASSERT_EQ(cluster_pcm_own_revoke_commit_exact(0, 7, token + 1, &committed), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(committed, 0); + assert_entry(7, token, PCM_OWN_FLAG_REVOKING); + + UT_ASSERT_EQ(cluster_pcm_own_revoke_commit_exact(0, 7, token, &committed), CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(committed, 8); + assert_entry(8, token, 0); + + /* A duplicate cannot bump the ownership generation twice. */ + committed = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_own_revoke_commit_exact(0, 7, token, &committed), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(committed, 0); + assert_entry(8, token, 0); + + /* A different well-formed lifecycle is contention, not corruption. */ + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 9); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, PCM_OWN_FLAG_GRANT_PENDING); + UT_ASSERT_EQ(cluster_pcm_own_revoke_commit_exact(0, 0, 9, &committed), CLUSTER_PCM_OWN_BUSY); + assert_entry(0, 9, PCM_OWN_FLAG_GRANT_PENDING); + + /* Malformed live metadata is corruption, not ordinary contention. */ + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 9); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, + PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); + UT_ASSERT_EQ(cluster_pcm_own_revoke_commit_exact(0, 0, 9, &committed), CLUSTER_PCM_OWN_CORRUPT); + assert_entry(0, 9, PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING); +} + +UT_TEST(test_revoke_retain_commit_keeps_exact_token_until_release) +{ + ClusterPcmOwnEvictionCapture capture; + uint64 committed = UINT64_MAX; + uint64 token; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_REVOKING, &token), + CLUSTER_PCM_OWN_OK); + + /* The retained commit bumps ownership exactly once but deliberately keeps + * the same live token: descriptor reuse remains fail-closed until the + * matching DRAIN/RELEASE_IMAGE arrives. */ + UT_ASSERT_EQ(cluster_pcm_own_revoke_retain_commit_exact(0, 7, token + 1, &committed), + CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(committed, 0); + assert_entry(7, token, PCM_OWN_FLAG_REVOKING); + + UT_ASSERT_EQ(cluster_pcm_own_revoke_retain_commit_exact(0, 7, token, &committed), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(committed, 8); + assert_entry(8, token, PCM_OWN_FLAG_REVOKING); + + memset(&capture, 0, sizeof(capture)); + capture.generation = committed; + capture.reservation_token = token; + capture.flags = PCM_OWN_FLAG_REVOKING; + UT_ASSERT(!cluster_pcm_own_eviction_reuse_allowed(&capture)); + + /* A stale DRAIN from either the pre-commit generation or a prior token is + * a strict no-op and cannot unpin a newer retained round. */ + UT_ASSERT_EQ(cluster_pcm_own_revoke_retain_release_exact(0, 7, token), CLUSTER_PCM_OWN_STALE); + UT_ASSERT_EQ(cluster_pcm_own_revoke_retain_release_exact(0, 8, token + 1), + CLUSTER_PCM_OWN_STALE); + assert_entry(8, token, PCM_OWN_FLAG_REVOKING); + + UT_ASSERT_EQ(cluster_pcm_own_revoke_retain_release_exact(0, 8, token), CLUSTER_PCM_OWN_OK); + assert_entry(8, token, 0); + UT_ASSERT_EQ(cluster_pcm_own_revoke_retain_release_exact(0, 8, token), CLUSTER_PCM_OWN_STALE); +} + +UT_TEST(test_revoke_commit_exhaustion_is_side_effect_free) +{ + uint64 committed = UINT64_MAX; + uint64 token; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, UINT64_MAX); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, 17); + pg_atomic_write_u32(&ClusterPcmOwnArray[0].flags, PCM_OWN_FLAG_REVOKING); + token = 17; + + UT_ASSERT_EQ(cluster_pcm_own_revoke_commit_exact(0, UINT64_MAX, token, &committed), + CLUSTER_PCM_OWN_EXHAUSTED); + UT_ASSERT_EQ(committed, 0); + assert_entry(UINT64_MAX, token, PCM_OWN_FLAG_REVOKING); +} + +UT_TEST(test_token_and_generation_never_wrap) +{ + uint64 token = UINT64_MAX; + uint64 last_token; + uint64 generation = 0; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].reservation_token, UINT64_MAX); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 0, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_EXHAUSTED); + UT_ASSERT_EQ(token, 0); + assert_entry(0, UINT64_MAX, 0); + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, UINT64_MAX - 1); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, UINT64_MAX - 1, + PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + last_token = token; + UT_ASSERT_EQ(cluster_pcm_own_grant_commit_exact(0, UINT64_MAX - 1, token, &generation), + CLUSTER_PCM_OWN_OK); + UT_ASSERT_EQ(generation, UINT64_MAX); + assert_entry(UINT64_MAX, token, 0); + + /* MAX is terminal: begin must fail before token/flag side effects. */ + token = UINT64_MAX; + UT_ASSERT_EQ( + cluster_pcm_own_reservation_begin_exact(0, UINT64_MAX, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_EXHAUSTED); + UT_ASSERT_EQ(token, 0); + assert_entry(UINT64_MAX, last_token, 0); + UT_ASSERT(!cluster_pcm_own_gen_bump_checked(0, &generation)); + UT_ASSERT_EQ(generation, UINT64_MAX); + assert_entry(UINT64_MAX, last_token, 0); +} + +UT_TEST(test_ordinary_generation_bump_rejects_live_reservation) +{ + uint64 generation = UINT64_MAX; + uint64 token; + + reset_fixture(); + pg_atomic_write_u64(&ClusterPcmOwnArray[0].generation, 7); + UT_ASSERT_EQ(cluster_pcm_own_reservation_begin_exact(0, 7, PCM_OWN_FLAG_GRANT_PENDING, &token), + CLUSTER_PCM_OWN_OK); + + /* Only the token-exact finish/revoke lifecycle may advance generation + * while a transient ownership flag is live. The ordinary transition + * helper must be a no-op so it cannot bypass the reservation token. */ + UT_ASSERT(!cluster_pcm_own_gen_bump_checked(0, &generation)); + UT_ASSERT_EQ(generation, 7); + assert_entry(7, token, PCM_OWN_FLAG_GRANT_PENDING); + + /* The same helper remains valid after exact cleanup makes the entry idle. */ + reset_fixture(); + UT_ASSERT(cluster_pcm_own_gen_bump_checked(0, &generation)); + UT_ASSERT_EQ(generation, 1); + assert_entry(1, 0, 0); +} + +UT_TEST(test_eviction_rejects_live_reservation_and_exhaustion) +{ + ClusterPcmOwnEvictionCapture capture; + + memset(&capture, 0, sizeof(capture)); + capture.generation = 9; + UT_ASSERT(cluster_pcm_own_eviction_reuse_allowed(&capture)); + + capture.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT(!cluster_pcm_own_eviction_reuse_allowed(&capture)); + capture.flags = PCM_OWN_FLAG_REVOKING; + UT_ASSERT(!cluster_pcm_own_eviction_reuse_allowed(&capture)); + capture.flags = 0; + capture.reservation_token = 3; + /* The single token is monotonic and remains nonzero while idle; flags are + * the only active-lifecycle marker. */ + UT_ASSERT(cluster_pcm_own_eviction_reuse_allowed(&capture)); + capture.generation = UINT64_MAX; + UT_ASSERT(!cluster_pcm_own_eviction_reuse_allowed(&capture)); +} + +static char * +read_bufmgr_source(void) +{ + FILE *file; + long length; + char *source; + + file = fopen(BUFMGR_SOURCE_PATH, "rb"); + UT_ASSERT_NOT_NULL(file); + UT_ASSERT_EQ(fseek(file, 0, SEEK_END), 0); + length = ftell(file); + UT_ASSERT(length > 0); + UT_ASSERT_EQ(fseek(file, 0, SEEK_SET), 0); + source = malloc((size_t)length + 1); + UT_ASSERT_NOT_NULL(source); + UT_ASSERT_EQ(fread(source, 1, (size_t)length, file), (size_t)length); + source[length] = '\0'; + fclose(file); + return source; +} + +static void +assert_commit_before_tail(const char *source, const char *function_start, const char *function_end) +{ + const char *begin = strstr(source, function_start); + const char *end; + const char *commit; + const char *tail; + + UT_ASSERT_NOT_NULL(begin); + if (begin == NULL) + return; + end = strstr(begin + strlen(function_start), function_end); + UT_ASSERT_NOT_NULL(end); + if (end == NULL) + return; + commit = strstr(begin, "cluster_pcm_own_eviction_commit_locked"); + tail = strstr(begin, "InvalidateBufferCommitTailLocked"); + UT_ASSERT_NOT_NULL(commit); + UT_ASSERT_NOT_NULL(tail); + if (commit == NULL || tail == NULL) + return; + UT_ASSERT(commit < tail); + UT_ASSERT(commit < end); + UT_ASSERT(tail < end); +} + +static int +count_occurrences(const char *source, const char *needle) +{ + int count = 0; + size_t needle_length = strlen(needle); + + while ((source = strstr(source, needle)) != NULL) { + count++; + source += needle_length; + } + return count; +} + +static void +assert_ordered_in_function(const char *source, const char *function_start, const char *function_end, + const char *const *needles, int needle_count) +{ + const char *cursor = strstr(source, function_start); + const char *end; + int i; + + UT_ASSERT_NOT_NULL(cursor); + if (cursor == NULL) + return; + end = strstr(cursor + strlen(function_start), function_end); + UT_ASSERT_NOT_NULL(end); + if (end == NULL) + return; + + for (i = 0; i < needle_count; i++) { + cursor = strstr(cursor, needles[i]); + UT_ASSERT_NOT_NULL(cursor); + if (cursor == NULL) + return; + UT_ASSERT(cursor < end); + if (cursor >= end) + return; + cursor += strlen(needles[i]); + } +} + +static void +assert_source_range_contains(const char *start, const char *end, const char *needle) +{ + const char *found; + + UT_ASSERT_NOT_NULL(start); + UT_ASSERT_NOT_NULL(end); + if (start == NULL || end == NULL) + return; + found = strstr(start, needle); + UT_ASSERT_NOT_NULL(found); + if (found != NULL) + UT_ASSERT(found < end); +} + +UT_TEST(test_bufmgr_d5a_commitlocked_uses_locked_commit_and_saved_tag_release) +{ + static const char *const commit_contract[] + = { "ClusterPcmOwnEvictionCapture eviction_capture", + "cluster_pcm_own_eviction_capture_locked", "cluster_pcm_own_eviction_commit_locked", + "eviction_result != CLUSTER_PCM_OWN_OK", "InvalidateBufferCommitTailLocked" }; + static const char *const tail_contract[] + = { "ClearBufferTag", "UnlockBufHdr", "cluster_pcm_lock_release_saved_tag_for_eviction" }; + char *source = read_bufmgr_source(); + + /* Descriptor reuse is a single header-authority commit. A live token, + * exhausted generation, or tuple mismatch must leave the old tag resident + * and return fail-closed; only an exact successful commit may clear the tag + * and later release the master holder by the saved immutable tag. */ + assert_commit_before_tail(source, "\nInvalidateBufferCommitLocked(", + "\n/*\n * InvalidateBufferCommitTailLocked"); + assert_ordered_in_function(source, "\nInvalidateBufferCommitLocked(", + "\n/*\n * InvalidateBufferCommitTailLocked", commit_contract, + lengthof(commit_contract)); + assert_ordered_in_function(source, "\nInvalidateBufferCommitTailLocked(", + "\n/*\n * InvalidateBufferTry", tail_contract, + lengthof(tail_contract)); + UT_ASSERT_NOT_NULL(strstr(source, "static bool\nInvalidateBufferCommitLocked")); + UT_ASSERT_NOT_NULL(strstr(source, "cluster_pcm_lock_release_saved_tag_for_eviction")); + UT_ASSERT_NULL(strstr(source, "buf->tag = *oldTag")); + free(source); +} + +UT_TEST(test_bufmgr_abort_cleanup_is_never_silent) +{ + static const char *const normal_cleanup[] + = { "cluster_pcm_own_abort_grant_reservation", "CLUSTER_PCM_OWN_OK", "ereport(ERROR" }; + static const char *const error_cleanup[] + = { "cluster_pcm_own_abort_grant_reservation", "CLUSTER_PCM_OWN_OK", "elog(LOG" }; + char *source = read_bufmgr_source(); + + /* Every normal false/READ_IMAGE exit must prove exact cleanup or ERROR. + * During PG_CATCH, preserve the original error but emit LOG evidence when + * exact cleanup did not converge. No call may discard the result. */ + UT_ASSERT_NULL(strstr(source, "(void) cluster_pcm_own_abort_grant_reservation")); + assert_ordered_in_function(source, "\ncluster_pcm_own_abort_grant_or_error(", + "\nstatic void\ncluster_pcm_own_abort_grant_after_error(", + normal_cleanup, lengthof(normal_cleanup)); + assert_ordered_in_function( + source, "\ncluster_pcm_own_abort_grant_after_error(", + "\nstatic ClusterPcmOwnResult\ncluster_pcm_own_abort_grant_after_master_rollback(", + error_cleanup, lengthof(error_cleanup)); + free(source); +} + +UT_TEST(test_bufmgr_finish_failure_rolls_back_acquired_master_grant) +{ + static const char *const rollback_contract[] + = { "cluster_pcm_own_finish_grant_reservation", + "PG_TRY", + "cluster_pcm_lock_release_buffer_for_eviction", + "PG_CATCH", + "elog(LOG", + "PG_RE_THROW", + "cluster_pcm_own_abort_grant_after_master_rollback", + "ereport(ERROR" }; + char *source = read_bufmgr_source(); + + /* Definition + direct-lock caller + LockBuffer caller. Keeping the real + * acquire and exact finish behind one helper prevents either entrance from + * leaking a master holder when local finish rejects the token/tuple. */ + UT_ASSERT(count_occurrences(source, "cluster_pcm_own_finish_grant_or_rollback(") >= 3); + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_or_rollback(", "\nstatic ", + rollback_contract, lengthof(rollback_contract)); + free(source); +} + +UT_TEST(test_bufmgr_s_base_rollback_normalizes_to_n_under_header_authority) +{ + static const char *const s_to_n_contract[] = { "LockBufHdr", + "base->pcm_state != (uint8)PCM_STATE_N", + "base->pcm_state != (uint8)PCM_STATE_S", + "base->pcm_state == (uint8)PCM_STATE_S", + "base->generation == UINT64_MAX", + "cluster_pcm_own_reservation_abort_exact", + "base->pcm_state == (uint8)PCM_STATE_S", + "cluster_pcm_own_gen_bump_checked", + "buf->pcm_state = (uint8)PCM_STATE_N", + "UnlockBufHdr" }; + char *source = read_bufmgr_source(); + + /* A legacy S-base acquire that reached master X cannot restore S after a + * failed local finish: the master rollback is X->N. The local half must + * therefore exact-abort the live token and commit S->N plus one checked + * generation bump under a single header-lock hold. N-base skips the bump + * and remains N. All prechecks precede abort, so rejection leaves the live + * flag as fail-closed evidence rather than advertising successful cleanup. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_abort_grant_after_master_rollback(", + "\nstatic void\ncluster_pcm_own_finish_grant_or_rollback(", + s_to_n_contract, lengthof(s_to_n_contract)); + free(source); +} + +UT_TEST(test_lockbuffer_content_error_uses_post_master_rollback_contract) +{ + static const char *const rethrow_contract[] + = { "Assert(original_error != NULL)", + "PG_TRY", + "cluster_pcm_lock_release_buffer_for_eviction", + "PG_CATCH", + "CopyErrorData", + "FlushErrorState", + "elog(LOG", + "if (master_released)", + "cluster_pcm_own_abort_grant_after_master_rollback", + "ReThrowError" }; + static const char *const content_error_contract[] + = { "MemoryContextSwitchTo(pcm_error_context)", + "original_error = CopyErrorData()", + "FlushErrorState()", + "cluster_bufmgr_pcm_x_holder_abort_acquiring", + "if (pcm_acquired)", + "cluster_pcm_own_rollback_grant_after_error_and_rethrow", + "else if (pcm_pending_set)", + "cluster_pcm_own_abort_grant_after_error", + "ReThrowError(original_error)" }; + char *source = read_bufmgr_source(); + + /* A content-lock error can occur after a durable master grant but before + * local finish. Both holder detach and remote release may themselves + * throw, so LockBuffer must copy and flush the original ErrorData before + * either cleanup begins. Cleanup failure is LOGged, leaves exact evidence + * fail-closed, and still rethrows the original error. Exact local + * convergence is legal only after release succeeded. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_rollback_grant_after_error_and_rethrow(", + "\nstatic ", rethrow_contract, lengthof(rethrow_contract)); + assert_ordered_in_function( + source, "\nLockBuffer(Buffer buffer, int mode)", + "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", + content_error_contract, lengthof(content_error_contract)); + free(source); +} + +UT_TEST(test_bufmgr_generation_bump_failure_is_classified_under_header_lock) +{ + static const char *const diagnostic_contract[] = { "cluster_pcm_own_reservation_token_get", + "cluster_pcm_own_flags_get", + "cluster_pcm_own_classify_live_flags", + "live_result != CLUSTER_PCM_OWN_OK", + "generation == UINT64_MAX", + "CLUSTER_PCM_OWN_EXHAUSTED" }; + static const char *const transition_contract[] + = { "LockBufHdr", "cluster_pcm_own_bump_locked", "UnlockBufHdr", + "cluster_pcm_own_report_bump_failure" }; + char *source = read_bufmgr_source(); + + /* A checked bump can reject either a live exact lifecycle or terminal MAX. + * Both observations must be made while header authority is still held and + * must not be collapsed into the misleading "exhausted" diagnosis. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_bump_failure(", "\nstatic ", + diagnostic_contract, lengthof(diagnostic_contract)); + assert_ordered_in_function(source, "\ncluster_pcm_own_transition(", "\n/*", transition_contract, + lengthof(transition_contract)); + UT_ASSERT(count_occurrences(source, "cluster_pcm_own_bump_failure(") >= 2); + UT_ASSERT_NOT_NULL(strstr(source, "active reservation")); + UT_ASSERT_NOT_NULL(strstr(source, "generation exhausted")); + free(source); +} + +UT_TEST(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier) +{ + static const char *const initial_reservation_contract[] + = { "cluster_pcm_own_begin_grant_reservation", "pcm_pending_result != CLUSTER_PCM_OWN_OK", + "cluster_pcm_own_report_bump_failure", "pcm_pending_set = true" }; + static const char *const revalidate_reservation_contract[] + = { "cluster_pcm_own_begin_grant_reservation", "pcm_pending_result != CLUSTER_PCM_OWN_OK", + "cluster_pcm_own_report_bump_failure", "pcm_pending_set = true" }; + char *source = read_bufmgr_source(); + + /* A live queue reservation is BUSY, not damaged metadata. Both legacy + * LockBuffer begin sites must preserve that distinction while malformed + * flag/token shapes continue through the same helper as DATA_CORRUPTED. */ + assert_ordered_in_function(source, "Legacy acquire path:", "PG_END_TRY();", + initial_reservation_contract, + lengthof(initial_reservation_contract)); + assert_ordered_in_function(source, "cluster_pcm_note_writer_cover_stale_detected();", + "cluster_pcm_note_writer_reverify_reacquire();", + revalidate_reservation_contract, + lengthof(revalidate_reservation_contract)); + free(source); +} + +UT_TEST(test_bufmgr_finish_rejects_invalid_state_and_initializes_acquire_result) +{ + static const char *const finish_gate[] + = { "new_pcm_state != (uint8)PCM_STATE_S", "new_pcm_state != (uint8)PCM_STATE_X", + "return CLUSTER_PCM_OWN_INVALID", "LockBufHdr" }; + char *source = read_bufmgr_source(); + + /* The only durable grant mirrors are S and X. Validate that before any + * header/sidecar mutation, and never let PG_TRY leave an indeterminate + * acquire result for its catch/finalize paths. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_x_commit(", + finish_gate, lengthof(finish_gate)); + UT_ASSERT_NOT_NULL(strstr(source, "grant_acquired = false")); + free(source); +} + +UT_TEST(test_bufmgr_finish_and_abort_gate_on_exact_base_state) +{ + static const char *const finish_contract[] + = { "LockBufHdr", "cluster_pcm_own_snapshot_locked", "cluster_pcm_x_grant_reservation_kind", + "cluster_pcm_own_grant_commit_exact" }; + static const char *const abort_contract[] + = { "LockBufHdr", "BufferTagsEqual", "buf->pcm_state != base->pcm_state", + "cluster_pcm_own_reservation_abort_exact" }; + char *source = read_bufmgr_source(); + + /* Tag/gen/token/flag identity is insufficient: a concurrent ownership + * transition that changed only the descriptor mirror must make both exact + * finish and abort return STALE before touching generation or flags. */ + assert_ordered_in_function(source, "\ncluster_pcm_own_finish_grant_reservation(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_x_commit(", + finish_contract, lengthof(finish_contract)); + assert_ordered_in_function(source, "\ncluster_pcm_own_abort_grant_reservation(", + "\nstatic void\ncluster_pcm_own_abort_grant_or_error(", + abort_contract, lengthof(abort_contract)); + free(source); +} + +UT_TEST(test_d5a_release_error_keeps_descriptor_out_of_freelist) +{ + static const char *const fail_closed_contract[] + = { "BufTableDelete", "LWLockRelease(oldPartitionLock)", + "PG_TRY", "cluster_pcm_lock_release_saved_tag_for_eviction", + "PG_CATCH", "elog(LOG", + "PG_RE_THROW", "StrategyFreeBuffer" }; + char *source = read_bufmgr_source(); + + /* A remote release may throw only after the old mapping is gone. Emit + * module evidence and rethrow before StrategyFreeBuffer, leaving the + * descriptor unmapped and non-reusable rather than losing a master holder + * through descriptor reuse. */ + assert_ordered_in_function(source, "\nInvalidateBufferCommitLocked(", + "\n/*\n * InvalidateBufferTry", fail_closed_contract, + lengthof(fail_closed_contract)); + free(source); +} + +UT_TEST(test_queue_begin_requires_normalized_n_snapshot) +{ + static const char *const normalized_n_gate[] + = { "expected->pcm_state != (uint8)PCM_STATE_N", "return CLUSTER_PCM_OWN_STALE" }; + char *source = read_bufmgr_source(); + + /* Ordinary queued acquisition must use a fresh normalized N snapshot. + * Sole-requester S conversion has a separate exact handoff API that reuses + * REVOKING and therefore still cannot enter this new-token path. */ + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_begin_x_reservation(", + "\nstatic ClusterPcmOwnResult\ncluster_pcm_own_finish_grant_reservation(", + normalized_n_gate, lengthof(normalized_n_gate)); + free(source); +} + +UT_TEST(test_queue_contract_exposes_prepare_only_begin_api) +{ + typedef ClusterPcmOwnResult (*BeginFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, uint64 *); + typedef ClusterPcmOwnResult (*HandoffFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, uint64 *); + typedef ClusterPcmOwnResult (*ReleaseSFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, + ClusterPcmOwnSnapshot *); + typedef ClusterPcmOwnResult (*FinishFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, uint64, + uint64 *); + typedef ClusterPcmOwnResult (*AbortFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, uint64); + + /* The queue owns timing, but not the reservation lifecycle: JOIN/WAIT must + * never call this begin API. ACTIVE_TRANSFER/PREPARE stores the returned + * token and all later finish/abort operations are exact. */ + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_begin_x_reservation), + BeginFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_handoff_s_revoke_to_x_reservation), HandoffFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation), HandoffFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_finish_s_release_to_n), ReleaseSFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_finish_x_commit), + FinishFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_abort_x_reservation), + AbortFn)); +} + +UT_TEST(test_queue_contract_exposes_opaque_retained_revoke_api) +{ + typedef ClusterPcmOwnResult (*BeginRevokeFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, + ClusterPcmOwnSnapshot *); + typedef ClusterPcmOwnResult (*PrepareNSourceFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, + ClusterPcmOwnSnapshot *, char *, XLogRecPtr *, + uint64 *); + typedef ClusterPcmOwnResult (*AbortRevokeFn)(BufferDesc *, const ClusterPcmOwnSnapshot *); + typedef ClusterPcmOwnResult (*FinishRetainFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, + XLogRecPtr, ClusterPcmOwnSnapshot *); + typedef ClusterPcmOwnResult (*ReleaseRetainedFn)(const BufferTag *, uint64); + typedef bool (*ContentWriteFn)(BufferDesc *); + + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_begin_x_revoke), + BeginRevokeFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_abort_x_revoke), + AbortRevokeFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_begin_s_revoke), + BeginRevokeFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_prepare_n_source_image), PrepareNSourceFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_abort_n_revoke), + AbortRevokeFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_abort_s_revoke), + AbortRevokeFn)); + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_finish_revoke_retain), + FinishRetainFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_release_retained_image), ReleaseRetainedFn)); + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_x_content_write_permitted), ContentWriteFn)); +} + +UT_TEST(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image) +{ + static const char *const prepare_contract[] + = { "expected_n->pcm_state != (uint8)PCM_STATE_N", + "cluster_pcm_own_snapshot_matches_locked", + "BM_VALID", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "BM_IO_ERROR", + "BM_IO_IN_PROGRESS", + "cluster_pcm_own_reservation_begin_exact", + "PCM_OWN_FLAG_REVOKING", + "smgrread", + "PageIsVerifiedExtended", + "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "cluster_pcm_own_snapshot_matches_locked", + "PCM_OWN_FLAG_REVOKING", + "BM_VALID", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "BM_IO_ERROR", + "BM_IO_IN_PROGRESS", + "memcpy((char *)BufHdrGetBlock(buf), scratch.data, BLCKSZ)", + "buf->buffer_type = (uint8)BUF_TYPE_CURRENT", + "memcpy(block_data, scratch.data, BLCKSZ)", + "PageGetLSN((Page)scratch.data)", + "pd_block_scn", + "*out_revoking = live" }; + char *source = read_bufmgr_source(); + + /* READY may be built only after one verified storage scratch has replaced + * the exact fenced N descriptor and supplied all image evidence. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_prepare_n_source_image(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_begin_s_revoke(", + prepare_contract, lengthof(prepare_contract)); + free(source); +} + +UT_TEST(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.forkNum = MAIN_FORKNUM; + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 0), CLUSTER_PCM_X_REVOKE_FINISH_RETAIN); + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 7), CLUSTER_PCM_X_REVOKE_FINISH_RETAIN); + tag.forkNum = INIT_FORKNUM; + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 3), CLUSTER_PCM_X_REVOKE_FINISH_RETAIN); + tag.forkNum = FSM_FORKNUM; + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 0), CLUSTER_PCM_X_REVOKE_FINISH_DROP); + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 1), CLUSTER_PCM_X_REVOKE_FINISH_BUSY); + tag.forkNum = VISIBILITYMAP_FORKNUM; + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 0), CLUSTER_PCM_X_REVOKE_FINISH_DROP); + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, UINT32_MAX), + CLUSTER_PCM_X_REVOKE_FINISH_BUSY); + tag.forkNum = InvalidForkNumber; + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(&tag, 0), CLUSTER_PCM_X_REVOKE_FINISH_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_revoke_finish_mode(NULL, 0), CLUSTER_PCM_X_REVOKE_FINISH_INVALID); +} + +UT_TEST(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm) +{ + static const char *const begin_contract[] = { "expected_s->pcm_state != (uint8) PCM_STATE_S", + "LockBufHdr", + "cluster_pcm_own_gen_get", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_pcm_own_classify_live_flags", + "cluster_pcm_own_reservation_begin_exact", + "PCM_OWN_FLAG_REVOKING", + "cluster_pcm_own_snapshot_locked", + "UnlockBufHdr" }; + static const char *const abort_contract[] + = { "expected_revoking->pcm_state != (uint8) PCM_STATE_S", + "LockBufHdr", + "cluster_pcm_own_classify_live_flags", + "cluster_pcm_own_reservation_abort_exact", + "PCM_OWN_FLAG_REVOKING", + "UnlockBufHdr" }; + static const char *const finish_contract[] = { "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LockBufHdr", + "cluster_pcm_own_gen_get", + "cluster_bufmgr_pcm_current_image_locked", + "BM_IO_IN_PROGRESS", + "PageGetLSN", + "cluster_pcm_own_revoke_retain_commit_exact", + "buf->pcm_state = (uint8) PCM_STATE_N", + "buf->buffer_type = (uint8) BUF_TYPE_PI", + "BM_DIRTY | BM_JUST_DIRTIED", + "BM_CHECKPOINT_NEEDED | BM_IO_ERROR", + "cluster_pcm_own_snapshot_locked" }; + static const char *const drop_contract[] = { "BufMappingPartitionLock", + "LWLockAcquire(partition_lock, LW_EXCLUSIVE)", + "BufTableLookup", + "LockBufHdr", + "BUF_STATE_GET_REFCOUNT", + "CLUSTER_PCM_X_REVOKE_FINISH_BUSY", + "PageGetLSN", + "cluster_pcm_own_revoke_commit_exact", + "buf->pcm_state = (uint8)PCM_STATE_N", + "cluster_pcm_own_snapshot_locked", + "InvalidateBufferCommitTailLocked" }; + char *source = read_bufmgr_source(); + const char *begin_s; + const char *abort_s; + const char *begin_x; + const char *abort_x; + const char *drop_helper; + const char *finish; + const char *finish_end; + + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_begin_s_revoke(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_abort_s_revoke(", + begin_contract, lengthof(begin_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_abort_s_revoke(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_begin_x_revoke(", + abort_contract, lengthof(abort_contract)); + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_release_retained_image(", finish_contract, + lengthof(finish_contract)); + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_finish_revoke_drop_unpinned(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_revoke_retain(", drop_contract, + lengthof(drop_contract)); + + /* Main/init passive PG pins are not PCM holders, so their retained commit + * must not recreate the S3 pin ring. VM/FSM take the separate exact-drop + * arm above, where a foreign pin returns BUSY before any ownership commit. */ + begin_s = strstr(source, "\ncluster_bufmgr_pcm_own_begin_s_revoke("); + abort_s = strstr(source, "\ncluster_bufmgr_pcm_own_abort_s_revoke("); + begin_x = strstr(source, "\ncluster_bufmgr_pcm_own_begin_x_revoke("); + abort_x = strstr(source, "\ncluster_bufmgr_pcm_own_abort_x_revoke("); + drop_helper = strstr(source, "\ncluster_bufmgr_pcm_own_finish_revoke_drop_unpinned("); + finish = strstr(source, "\ncluster_bufmgr_pcm_own_finish_revoke_retain("); + finish_end = strstr(source, "\ncluster_bufmgr_pcm_own_release_retained_image("); + UT_ASSERT_NOT_NULL(begin_s); + UT_ASSERT_NOT_NULL(abort_s); + UT_ASSERT_NOT_NULL(begin_x); + UT_ASSERT_NOT_NULL(abort_x); + UT_ASSERT_NOT_NULL(drop_helper); + UT_ASSERT_NOT_NULL(finish); + UT_ASSERT_NOT_NULL(finish_end); + /* buffer_type is a monotone hint: every exact source lifecycle must + * accept a yielded S+XCUR through the centralized current-image gate. */ + assert_source_range_contains(begin_s, abort_s, "cluster_bufmgr_pcm_current_image_locked"); + assert_source_range_contains(abort_s, begin_x, "cluster_bufmgr_pcm_current_image_locked"); + assert_source_range_contains(begin_x, abort_x, "cluster_bufmgr_pcm_current_image_locked"); + assert_source_range_contains(abort_x, drop_helper, "cluster_bufmgr_pcm_current_image_locked"); + assert_source_range_contains(drop_helper, finish, "cluster_bufmgr_pcm_current_image_locked"); + assert_source_range_contains(finish, finish_end, "cluster_bufmgr_pcm_current_image_locked"); + if (begin_s != NULL && begin_x != NULL) + UT_ASSERT(strstr(begin_s, "BUF_STATE_GET_REFCOUNT") == NULL + || strstr(begin_s, "BUF_STATE_GET_REFCOUNT") >= begin_x); + if (begin_x != NULL && drop_helper != NULL) + UT_ASSERT(strstr(begin_x, "BUF_STATE_GET_REFCOUNT") == NULL + || strstr(begin_x, "BUF_STATE_GET_REFCOUNT") >= drop_helper); + if (finish != NULL && finish_end != NULL) { + const char *refcount = strstr(finish, "BUF_STATE_GET_REFCOUNT"); + const char *drop = strstr(finish, "InvalidateBuffer"); + const char *legacy_pi = strstr(finish, "cluster_bufmgr_convert_to_pi_locked"); + const char *mapping = strstr(finish, "partition_lock"); + + UT_ASSERT(refcount == NULL || refcount >= finish_end); + UT_ASSERT(drop == NULL || drop >= finish_end); + UT_ASSERT(legacy_pi == NULL || legacy_pi >= finish_end); + UT_ASSERT(mapping == NULL || mapping >= finish_end); + } + free(source); +} + +UT_TEST(test_retained_image_release_and_writeback_gates_are_exact) +{ + static const char *const content_write_contract[] + = { "LockBufHdr", + "cluster_pcm_own_flags_get", + "PCM_OWN_FLAG_REVOKING", + "cluster_bufmgr_pcm_x_retained_image_locked", + "PCM_OWN_FLAG_GRANT_PENDING", + "UnlockBufHdr" }; + static const char *const release_contract[] + = { "source_generation + 1", + "BufTableHashCode", + "LWLockAcquire(partition_lock, LW_SHARED)", + "BufTableLookup", + "LockBufHdr", + "PCM_OWN_FLAG_REVOKING", + "LWLockRelease(partition_lock)", + "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "LockBufHdr", + "BufferTagsEqual", + "BM_VALID", + "BUF_TYPE_PI", + "PCM_STATE_N", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED | BM_IO_ERROR", + "PCM_OWN_FLAG_REVOKING", + "cluster_pcm_own_revoke_retain_release_exact" }; + char *source = read_bufmgr_source(); + const char *victim; + const char *sync; + const char *flush; + const char *dirty; + const char *hint; + const char *lockbuffer; + const char *conditional; + const char *resident_stamp; + const char *storage_refresh; + + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_release_retained_image(", + "\n/* ========================================================================\n * PGRAC " + "MODIFICATIONS by SqlRush — spec-6.12h D-h2", + release_contract, lengthof(release_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_content_write_permitted(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_snapshot_by_tag(", + content_write_contract, lengthof(content_write_contract)); + + /* The content-lock ordering is the race proof: an already-started flush + * owns SHARE and finishes before retain; after retain owns EXCLUSIVE, all + * later Sync/Flush/dirty paths see the immutable retained shape. */ + victim = strstr(source, "\nInvalidateVictimBuffer("); + sync = strstr(source, "\nSyncOneBuffer("); + flush = strstr(source, "\nFlushBuffer("); + dirty = strstr(source, "\nMarkBufferDirty(Buffer buffer)"); + hint = strstr(source, "\nMarkBufferDirtyHint(Buffer buffer, bool buffer_std)"); + lockbuffer = strstr(source, "\nLockBuffer(Buffer buffer, int mode)"); + conditional = strstr(source, "\nConditionalLockBuffer(Buffer buffer)"); + resident_stamp = strstr(source, "\ncluster_bufmgr_lock_resident_for_stamp("); + storage_refresh = strstr(source, "\ncluster_bufmgr_refresh_block_from_storage_for_gcs("); + UT_ASSERT_NOT_NULL(victim); + UT_ASSERT_NOT_NULL(sync); + UT_ASSERT_NOT_NULL(flush); + UT_ASSERT_NOT_NULL(dirty); + UT_ASSERT_NOT_NULL(hint); + UT_ASSERT_NOT_NULL(lockbuffer); + UT_ASSERT_NOT_NULL(conditional); + UT_ASSERT_NOT_NULL(resident_stamp); + UT_ASSERT_NOT_NULL(storage_refresh); + if (victim != NULL) + UT_ASSERT(strstr(victim, "cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked") + < strstr(victim, "ClearBufferTag")); + if (sync != NULL) { + const char *first_gate + = strstr(sync, "cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked"); + const char *content_share + = strstr(sync, "LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED)"); + const char *second_gate + = first_gate != NULL + ? strstr(first_gate + 1, + "cluster_bufmgr_pcm_x_retained_image_reuse_blocked_locked") + : NULL; + + UT_ASSERT_NOT_NULL(first_gate); + UT_ASSERT_NOT_NULL(content_share); + UT_ASSERT_NOT_NULL(second_gate); + if (first_gate != NULL) + UT_ASSERT(first_gate < strstr(sync, "result |= BUF_REUSABLE")); + if (content_share != NULL && second_gate != NULL) + UT_ASSERT(content_share < second_gate); + } + if (flush != NULL) + UT_ASSERT(strstr(flush, "cluster_bufmgr_pcm_x_retained_image_locked") + < strstr(flush, "StartBufferIO(buf, false)")); + if (dirty != NULL) + UT_ASSERT(strstr(dirty, "cluster_bufmgr_pcm_x_retained_image_locked") + < strstr(dirty, "buf_state |= BM_DIRTY")); + if (hint != NULL) + UT_ASSERT(strstr(hint, "cluster_bufmgr_pcm_x_retained_image_locked") + < strstr(hint, "BM_DIRTY | BM_JUST_DIRTIED")); + if (lockbuffer != NULL) { + const char *reserve = strstr(lockbuffer, "cluster_pcm_own_begin_grant_reservation("); + const char *content + = strstr(lockbuffer, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED)"); + const char *w1_revoke + = strstr(lockbuffer, "PCM_OWN_FLAG_GRANT_PENDING | PCM_OWN_FLAG_REVOKING"); + + UT_ASSERT_NOT_NULL(reserve); + UT_ASSERT_NOT_NULL(content); + UT_ASSERT_NOT_NULL(w1_revoke); + if (reserve != NULL && content != NULL) + UT_ASSERT(reserve < content); + } + if (conditional != NULL) { + const char *content + = strstr(conditional, "LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf)"); + const char *flags = strstr(conditional, "cluster_pcm_own_flags_get(buf->buf_id) != 0"); + const char *release + = strstr(conditional, "LWLockRelease(BufferDescriptorGetContentLock(buf))"); + + UT_ASSERT_NOT_NULL(content); + UT_ASSERT_NOT_NULL(flags); + UT_ASSERT_NOT_NULL(release); + if (content != NULL && flags != NULL && release != NULL) + UT_ASSERT(content < flags && flags < release); + } + if (resident_stamp != NULL) { + const char *content = strstr( + resident_stamp, "LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE)"); + const char *gate = strstr(resident_stamp, "cluster_bufmgr_pcm_x_retained_image_locked"); + + UT_ASSERT_NOT_NULL(content); + UT_ASSERT_NOT_NULL(gate); + if (content != NULL && gate != NULL) + UT_ASSERT(content < gate); + } + if (storage_refresh != NULL) { + const char *content = strstr(storage_refresh, "LWLockAcquire(content_lock, LW_EXCLUSIVE)"); + const char *gate = strstr(storage_refresh, "cluster_bufmgr_pcm_x_content_write_permitted"); + const char *copy = strstr(storage_refresh, "memcpy((char *) BufHdrGetBlock(buf)"); + + UT_ASSERT_NOT_NULL(content); + UT_ASSERT_NOT_NULL(gate); + UT_ASSERT_NOT_NULL(copy); + if (content != NULL && gate != NULL && copy != NULL) + UT_ASSERT(content < gate && gate < copy); + } + free(source); +} + +UT_TEST(test_retained_drain_retags_invalid_only_after_exact_token_release) +{ + static const char *const drain_contract[] = { "cluster_pcm_own_revoke_retain_release_exact", + "result == CLUSTER_PCM_OWN_OK", + "buf_state &= ~BM_VALID", + "buf->buffer_type", + "BUF_TYPE_CURRENT", + "UnlockBufHdr" }; + char *source = read_bufmgr_source(); + const char *release; + const char *release_end; + + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_release_retained_image(", + "\n/* ========================================================================\n * PGRAC " + "MODIFICATIONS by SqlRush — spec-6.12h D-h2", + drain_contract, lengthof(drain_contract)); + + release = strstr(source, "\ncluster_bufmgr_pcm_own_release_retained_image("); + release_end = release != NULL ? strstr(release + 1, "\n/* " + "==========================================" + "==============================\n * PGRAC ") + : NULL; + UT_ASSERT_NOT_NULL(release); + UT_ASSERT_NOT_NULL(release_end); + if (release != NULL && release_end != NULL) + UT_ASSERT(strstr(release, "InvalidateBufferCommitTailLocked") == NULL + || strstr(release, "InvalidateBufferCommitTailLocked") >= release_end); + + free(source); +} + +UT_TEST(test_queue_s_release_finish_is_header_exact_and_returns_fresh_n) +{ + static const char *const release_contract[] = { "expected_s->pcm_state != (uint8) PCM_STATE_S", + "expected_s->flags != 0", + "LockBufHdr", + "cluster_pcm_own_snapshot_matches_locked", + "cluster_pcm_own_bump_locked", + "buf->pcm_state = (uint8) PCM_STATE_N", + "cluster_pcm_own_snapshot_locked", + "UnlockBufHdr" }; + char *source = read_bufmgr_source(); + + /* The caller proves the exact remote RELEASE application ACK before this + * adapter. The adapter then normalizes only the matching S tuple and + * returns the fresh N generation that PREPARE must reserve. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_finish_s_release_to_n(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_begin_x_reservation(", + release_contract, lengthof(release_contract)); + free(source); +} + +UT_TEST(test_lockbuffer_pcm_x_holder_ledger_brackets_both_content_acquires) +{ + static const char *const unlock_contract[] + = { "cluster_bufmgr_pcm_x_holder_mark_releasing", + "LWLockRelease(BufferDescriptorGetContentLock(buf))", + "cluster_bufmgr_pcm_x_holder_unregister" }; + static const char *const acquire_contract[] + = { "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "LWLockAcquire(BufferDescriptorGetContentLock(buf)", + "LWLockRelease(BufferDescriptorGetContentLock(buf))", + "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "LWLockAcquire(BufferDescriptorGetContentLock(buf)", + "cluster_bufmgr_pcm_x_holder_activate(pcm_x_holder)" }; + char *source = read_bufmgr_source(); + + /* One ACQUIRING entry brackets the first content-lock acquire and remains + * published across the W1 release/reacquire fallback. Normal release + * publishes RELEASING before unlocking and unlinks only afterwards. */ + assert_ordered_in_function( + source, "\nLockBuffer(Buffer buffer, int mode)", + "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", + unlock_contract, lengthof(unlock_contract)); + assert_ordered_in_function( + source, "\nLockBuffer(Buffer buffer, int mode)", + "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", + acquire_contract, lengthof(acquire_contract)); + free(source); +} + +UT_TEST(test_bufmgr_pcm_x_holder_ledger_is_bounded_and_uses_private_identity) +{ + static const char *const identity_contract[] + = { "cluster_bufmgr_pcm_x_holder_find", + "entry != NULL", + "return entry", + "cluster_pcm_own_read", + "cluster_epoch_get_current", + "cluster_node_id", + "MyProc->pgprocno", + "cluster_bufmgr_pcm_x_holder_identity == UINT64_MAX", + "ERRCODE_PROGRAM_LIMIT_EXCEEDED", + "++cluster_bufmgr_pcm_x_holder_identity", + "key.identity.request_id = identity", + "key.identity.wait_seq = identity", + "key.identity.base_own_generation = own_generation", + "cluster_pcm_x_local_holder_register" }; + char *source = read_bufmgr_source(); + + /* The ledger is backend-local and no larger than PG's own maximum held + * LWLock set. Its checked identity populates only the protocol key; it + * must never borrow or mutate PGPROC's deadlock wait sequence. */ + UT_ASSERT_NOT_NULL( + strstr(source, "cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]")); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_prepare(", "\nstatic ", + identity_contract, lengthof(identity_contract)); + UT_ASSERT_NULL(strstr(source, "cluster_lmd_wait.wait_seq")); + free(source); +} + +UT_TEST(test_unlockbuffers_exceptionally_detaches_released_pcm_x_holders) +{ + static const char *const cleanup_contract[] + = { "LWLockHeldByMe(entry->content_lock)", + "cluster_pcm_x_local_holder_exceptional_detach_exact(&entry->handle", + "entry->content_lock" }; + static const char *const catch_contract[] + = { "!LWLockHeldByMe(BufferDescriptorGetContentLock(buf))", + "cluster_bufmgr_pcm_x_holder_abort_acquiring" }; + static const char *const eoxact_contract[] + = { "AtEOXact_LocalBuffers(isCommit)", + "cluster_bufmgr_pcm_x_holder_drain_deferred_nowait()", + "Assert(PrivateRefCountOverflowed == 0)" }; + char *source = read_bufmgr_source(); + const char *cleanup_begin; + const char *cleanup_end; + const char *mirror_read; + + /* AbortTransaction releases all LWLocks before UnlockBuffers. The + * process-local ledger must prove each content lock is gone before using + * the multi-state exceptional detach; an acquire that threw before owning + * the lock keeps the narrower ACQUIRING-only rollback. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_exception_cleanup_all(", + "\nstatic ", cleanup_contract, lengthof(cleanup_contract)); + cleanup_begin = strstr(source, "\ncluster_bufmgr_pcm_x_holder_exception_cleanup_all("); + UT_ASSERT_NOT_NULL(cleanup_begin); + cleanup_end = cleanup_begin == NULL ? NULL : strstr(cleanup_begin + 1, "\nstatic "); + UT_ASSERT_NOT_NULL(cleanup_end); + mirror_read = cleanup_begin == NULL ? NULL : strstr(cleanup_begin, "pcm_state"); + UT_ASSERT(mirror_read == NULL || (cleanup_end != NULL && mirror_read >= cleanup_end)); + UT_ASSERT(cleanup_begin == NULL || cleanup_end == NULL + || strstr(cleanup_begin, "WaitLatch(") == NULL + || strstr(cleanup_begin, "WaitLatch(") >= cleanup_end); + UT_ASSERT(cleanup_begin == NULL || cleanup_end == NULL + || strstr(cleanup_begin, "CHECK_FOR_INTERRUPTS()") == NULL + || strstr(cleanup_begin, "CHECK_FOR_INTERRUPTS()") >= cleanup_end); + assert_ordered_in_function(source, + "CLUSTER_INJECTION_POINT(\"cluster-pcm-writer-cached-x-stall\")", + "PG_END_TRY();", catch_contract, lengthof(catch_contract)); + UT_ASSERT_NOT_NULL(strstr(source, "cluster_bufmgr_pcm_x_holder_exception_cleanup_all();")); + assert_ordered_in_function(source, "\nAtEOXact_Buffers(bool isCommit)", + "\n/*\n * Initialize access to shared buffer pool", eoxact_contract, + lengthof(eoxact_contract)); + free(source); +} + +UT_TEST(test_bufmgr_pcm_x_holder_gate_retry_is_bounded_outside_content_lock) +{ + static const char *const wait_contract[] + = { "content_lock == NULL || LWLockHeldByMe(content_lock)", + "ereport(ERROR", + "cluster_pcm_x_holder_retry_delay_ms(wait_index)", + "CHECK_FOR_INTERRUPTS()", + "WaitLatch(MyLatch", + "WAIT_EVENT_PCM_BLOCK_CONVERT_WAIT", + "CHECK_FOR_INTERRUPTS()" }; + static const char *const register_contract[] + = { "cluster_bufmgr_pcm_x_holder_drain_deferred_nowait()", + "entry->phase == PCM_X_HOLDER_LEDGER_DEFERRED", + "cluster_bufmgr_pcm_x_holder_drain_deferred(entry)", + "cluster_pcm_x_local_holder_register(&key, &handle)", + "cluster_pcm_x_holder_register_retry_action(", + "cluster_bufmgr_pcm_x_holder_retry_wait(" }; + static const char *const unregister_contract[] + = { "entry->content_lock == NULL || LWLockHeldByMe(entry->content_lock)", + "cluster_pcm_x_local_holder_unregister_exact(&entry->handle)", + "cluster_pcm_x_holder_unregister_retry_action(result, waits_used)", + "CLUSTER_PCM_X_HOLDER_RETRY_DEFER", + "entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED", + "return" }; + static const PcmXQueueResult register_script[] + = { PCM_X_QUEUE_GATE_RETRY, PCM_X_QUEUE_GATE_RETRY, PCM_X_QUEUE_OK }; + char *source = read_bufmgr_source(); + const char *wait_begin; + const char *wait_end; + int i; + + for (i = 0; i < lengthof(register_script); i++) + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(register_script[i], true), + i + 1 == lengthof(register_script) ? CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE + : CLUSTER_PCM_X_HOLDER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_DUPLICATE, true), + CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_BARRIER_CLOSED, true), + CLUSTER_PCM_X_HOLDER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NOT_READY, true), + CLUSTER_PCM_X_HOLDER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NOT_READY, false), + CLUSTER_PCM_X_HOLDER_RETRY_FAIL); + UT_ASSERT_EQ(cluster_pcm_x_holder_register_retry_action(PCM_X_QUEUE_NO_CAPACITY, true), + CLUSTER_PCM_X_HOLDER_RETRY_FAIL); + for (i = 0; i < CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS; i++) { + UT_ASSERT_EQ(cluster_pcm_x_holder_unregister_retry_action(PCM_X_QUEUE_GATE_RETRY, i), + CLUSTER_PCM_X_HOLDER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_writer_release_retry_action(PCM_X_QUEUE_BUSY, i), + CLUSTER_PCM_X_WRITER_RETRY_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_holder_retry_delay_ms(i), 2L << i); + } + UT_ASSERT_EQ(cluster_pcm_x_holder_unregister_retry_action( + PCM_X_QUEUE_BUSY, CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS), + CLUSTER_PCM_X_HOLDER_RETRY_DEFER); + UT_ASSERT_EQ(cluster_pcm_x_holder_unregister_retry_action(PCM_X_QUEUE_NOT_FOUND, 0), + CLUSTER_PCM_X_HOLDER_RETRY_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_holder_unregister_retry_action(PCM_X_QUEUE_CORRUPT, 0), + CLUSTER_PCM_X_HOLDER_RETRY_FAIL); + UT_ASSERT_EQ(cluster_pcm_x_writer_release_retry_action(PCM_X_QUEUE_OK, 0), + CLUSTER_PCM_X_WRITER_RETRY_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_writer_release_retry_action(PCM_X_QUEUE_GATE_RETRY, + CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS), + CLUSTER_PCM_X_WRITER_RETRY_DEFER); + UT_ASSERT_EQ(cluster_pcm_x_writer_release_retry_action(PCM_X_QUEUE_NOT_READY, 0), + CLUSTER_PCM_X_WRITER_RETRY_FAIL); + UT_ASSERT_EQ(cluster_pcm_x_holder_retry_delay_ms(UINT32_C(99)), 32); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_OK, false, true), + CLUSTER_PCM_X_OWNER_EXIT_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_NOT_FOUND, true, true), + CLUSTER_PCM_X_OWNER_EXIT_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_BUSY, false, true), + CLUSTER_PCM_X_OWNER_EXIT_RETRY); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_GATE_RETRY, true, true), + CLUSTER_PCM_X_OWNER_EXIT_RETRY); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_BUSY, false, false), + CLUSTER_PCM_X_OWNER_EXIT_PRESERVE); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_NOT_READY, true, true), + CLUSTER_PCM_X_OWNER_EXIT_PRESERVE); + UT_ASSERT_EQ(cluster_pcm_x_owner_exit_action(PCM_X_QUEUE_CORRUPT, false, true), + CLUSTER_PCM_X_OWNER_EXIT_PRESERVE); + + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_retry_wait(", "\nstatic ", + wait_contract, lengthof(wait_contract)); + wait_begin = strstr(source, "\ncluster_bufmgr_pcm_x_holder_retry_wait("); + wait_end = wait_begin == NULL ? NULL : strstr(wait_begin + 1, "\nstatic "); + UT_ASSERT_NOT_NULL(wait_begin); + UT_ASSERT_NOT_NULL(wait_end); + UT_ASSERT(wait_begin == NULL || wait_end == NULL + || strstr(wait_begin, "ResetLatch(MyLatch)") == NULL + || strstr(wait_begin, "ResetLatch(MyLatch)") >= wait_end); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_prepare(", "\nstatic ", + register_contract, lengthof(register_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_unregister(", "\nstatic ", + unregister_contract, lengthof(unregister_contract)); + UT_ASSERT_NOT_NULL(strstr(source, "cluster_bufmgr_pcm_x_holder_drain_deferred_nowait();")); + free(source); +} + +UT_TEST(test_bufmgr_pcm_x_holder_reuse_and_deferred_failure_are_fail_closed) +{ + static const char *const exact_contract[] + = { "entry == NULL || buf == NULL", + "entry->buffer_id != buf->buf_id", + "entry->content_lock != BufferDescriptorGetContentLock(buf)", + "entry->handle.key.buffer_id != buf->buf_id", + "!BufferTagsEqual(&entry->handle.key.identity.tag, &buf->tag)", + "entry->handle.key.identity.node_id != cluster_node_id", + "MyProc == NULL", + "entry->handle.key.identity.procno != (uint32) MyProc->pgprocno", + "entry->handle.key.identity.request_id == 0", + "entry->handle.key.identity.wait_seq != entry->handle.key.identity.request_id", + "cluster_epoch = cluster_epoch_get_current()", + "runtime = cluster_pcm_x_runtime_snapshot()", + "entry->handle.key.identity.cluster_epoch != cluster_epoch", + "runtime.state != PCM_X_RUNTIME_ACTIVE", + "runtime.master_session_incarnation == 0", + "return true" }; + static const char *const prepare_contract[] + = { "entry = cluster_bufmgr_pcm_x_holder_find(buf)", + "if (entry != NULL)", + "entry->phase != PCM_X_HOLDER_LEDGER_ACQUIRING", + "!cluster_bufmgr_pcm_x_holder_entry_exact(entry, buf)", + "cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry)", + "cluster_bufmgr_pcm_x_holder_report_failure", + "LWLockHeldByMe(entry->content_lock)" }; + static const char *const deferred_contract[] + = { "cluster_bufmgr_pcm_x_holder_retry_wait(", + "wait_index++", + "wait_index % CLUSTER_PCM_X_HOLDER_RETRY_BATCH_WAITS == 0", + "runtime = cluster_pcm_x_runtime_snapshot()", + "runtime.state != PCM_X_RUNTIME_ACTIVE", + "runtime.master_session_incarnation == 0", + "cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry)", + "cluster_bufmgr_pcm_x_holder_report_failure" }; + static const char *const nowait_contract[] + = { "cluster_pcm_x_local_holder_exceptional_detach_exact", "PCM_X_QUEUE_GATE_RETRY", + "cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry)", "elog(LOG" }; + static const char *const abort_contract[] + = { "cluster_pcm_x_local_holder_abort_acquiring_exact", "PCM_X_QUEUE_GATE_RETRY", + "entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED", + "cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry)", "elog(LOG" }; + static const char *const exception_contract[] + = { "cluster_pcm_x_local_holder_exceptional_detach_exact", "PCM_X_QUEUE_GATE_RETRY", + "entry->phase = PCM_X_HOLDER_LEDGER_DEFERRED", + "cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry)", "elog(LOG" }; + static const char *const unregister_contract[] + = { "action != CLUSTER_PCM_X_HOLDER_RETRY_WAIT", + "cluster_bufmgr_pcm_x_holder_defer_fail_closed(entry)", + "cluster_bufmgr_pcm_x_holder_report_failure" }; + char *source = read_bufmgr_source(); + const char *entry_begin; + const char *entry_end; + + /* A W1 fallback may reuse one ACQUIRING ledger entry, but buffer-id alone + * is not identity: descriptor retag, backend/epoch drift, or a stale lock + * pointer must close the runtime and retain the exact old handle. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_entry_exact(", "\nstatic ", + exact_contract, lengthof(exact_contract)); + entry_begin = strstr(source, "\ncluster_bufmgr_pcm_x_holder_entry_exact("); + entry_end = entry_begin == NULL ? NULL : strstr(entry_begin + 1, "\nstatic "); + UT_ASSERT_NOT_NULL(entry_begin); + UT_ASSERT_NOT_NULL(entry_end); + UT_ASSERT(entry_begin == NULL || entry_end == NULL + || strstr(entry_begin, "cluster_epoch == 0") == NULL + || strstr(entry_begin, "cluster_epoch == 0") >= entry_end); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_prepare(", "\nstatic ", + prepare_contract, lengthof(prepare_contract)); + + /* A live RETIRE gate can legitimately span one retry batch. It cannot + * justify an infinite same-buffer wait after the formation runtime has + * left ACTIVE; all hard cleanup outcomes preserve exact evidence under a + * single explicit fail-closed transition. */ + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_drain_deferred(", "\nstatic ", + deferred_contract, lengthof(deferred_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_drain_deferred_nowait(", + "\nstatic ", nowait_contract, lengthof(nowait_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_abort_acquiring(", + "\nstatic ", abort_contract, lengthof(abort_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_exception_cleanup_all(", + "\nstatic ", exception_contract, lengthof(exception_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_unregister(", "\nstatic ", + unregister_contract, lengthof(unregister_contract)); + free(source); +} + +UT_TEST(test_queue_holder_snapshot_by_tag_is_mapping_and_header_exact) +{ + typedef ClusterPcmOwnResult (*SnapshotByTagFn)(const BufferTag *, int *, + ClusterPcmOwnSnapshot *); + static const char *const snapshot_contract[] + = { "BufTableHashCode", "LWLockAcquire(partition_lock, LW_SHARED)", + "BufTableLookup", "GetBufferDescriptor", + "LockBufHdr", "BufferTagsEqual", + "BM_VALID", "cluster_pcm_own_snapshot_locked", + "UnlockBufHdr", "LWLockRelease(partition_lock)" }; + char *source = read_bufmgr_source(); + + UT_ASSERT(__builtin_types_compatible_p(__typeof__(&cluster_bufmgr_pcm_own_snapshot_by_tag), + SnapshotByTagFn)); + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_snapshot_by_tag(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_finish_s_release_to_n(", snapshot_contract, + lengthof(snapshot_contract)); + free(source); +} + +UT_TEST(test_queue_passive_pinned_s_release_serializes_bytes_and_ownership) +{ + typedef ClusterPcmOwnResult (*PassiveReleaseFn)(const BufferTag *, XLogRecPtr *, uint64 *); + static const char *const release_contract[] + = { "BufTableHashCode", + "LWLockAcquire(partition_lock, LW_SHARED)", + "BufTableLookup", + "LockBufHdr", + "cluster_pcm_x_revoke_finish_mode(tag, shared_refcount)", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockRelease(partition_lock)", + "LWLockAcquire(content_lock, LW_EXCLUSIVE)", + "cluster_pcm_own_snapshot_matches_locked", + "PageGetLSN", + "XLogFlush", + "LockBufHdr", + "cluster_pcm_own_snapshot_matches_locked", + "cluster_pcm_own_bump_locked", + "buf->pcm_state = (uint8) PCM_STATE_N", + "buf->buffer_type = (uint8) BUF_TYPE_PI", + "BM_DIRTY | BM_JUST_DIRTIED | BM_CHECKPOINT_NEEDED", + "BM_IO_ERROR", + "cluster_bufmgr_unpin_for_gcs" }; + char *source = read_bufmgr_source(); + + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_release_pinned_s_for_gcs), PassiveReleaseFn)); + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_release_pinned_s_for_gcs(", + "\nClusterPcmOwnResult\ncluster_bufmgr_pcm_own_publish_installed_x_image(", + release_contract, lengthof(release_contract)); + free(source); +} + +UT_TEST(test_current_image_shape_accepts_monotone_xcur_after_x_to_s_yield) +{ + UT_ASSERT(cluster_pcm_x_current_image_shape((uint8)PCM_STATE_S, (uint8)BUF_TYPE_SCUR, true)); + UT_ASSERT(cluster_pcm_x_current_image_shape((uint8)PCM_STATE_S, (uint8)BUF_TYPE_XCUR, true)); + UT_ASSERT(!cluster_pcm_x_current_image_shape((uint8)PCM_STATE_X, (uint8)BUF_TYPE_SCUR, true)); + UT_ASSERT(cluster_pcm_x_current_image_shape((uint8)PCM_STATE_X, (uint8)BUF_TYPE_XCUR, true)); + UT_ASSERT(!cluster_pcm_x_current_image_shape((uint8)PCM_STATE_N, (uint8)BUF_TYPE_PI, true)); + UT_ASSERT(!cluster_pcm_x_current_image_shape((uint8)PCM_STATE_S, (uint8)BUF_TYPE_PI, true)); + UT_ASSERT(!cluster_pcm_x_current_image_shape((uint8)PCM_STATE_S, (uint8)BUF_TYPE_XCUR, false)); +} + +UT_TEST(test_queue_passive_n_mirror_is_never_gcs_ship_authority) +{ + static const char *const probe_contract[] + = { "LockBufHdr", "cluster_bufmgr_pcm_current_image_locked", "UnlockBufHdr" }; + static const char *const copy_contract[] = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "memcpy(dst, page, BLCKSZ)" }; + static const char *const live_sge_contract[] = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "*out_page_addr = page" }; + static const char *const smart_contract[] = { "LockBufHdr", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_bufmgr_pin_for_gcs_locked", + "LWLockAcquire(content_lock, LW_SHARED)", + "cluster_bufmgr_pcm_current_image_locked", + "memcpy(dst, page, BLCKSZ)" }; + char *source = read_bufmgr_source(); + + assert_ordered_in_function(source, "\ncluster_bufmgr_probe_block_for_gcs(", + "\n/*\n * Read the shared-storage version", probe_contract, + lengthof(probe_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_copy_block_for_gcs(", + "\n/*\n * Borrow a live shared_buffers page", copy_contract, + lengthof(copy_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_borrow_block_for_gcs_live_sge(", + "\nvoid\ncluster_bufmgr_release_block_for_gcs_live_sge(", + live_sge_contract, lengthof(live_sge_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_copy_block_for_gcs_smart_fusion(", + "\n/*\n * cluster_bufmgr_redeclare_scan_chunk", smart_contract, + lengthof(smart_contract)); + free(source); +} + +UT_TEST(test_queue_installed_image_publication_is_exact_and_content_locked) +{ + typedef ClusterPcmOwnResult (*PublishImageFn)(BufferDesc *, const ClusterPcmOwnSnapshot *, + uint64); + static const char *const publish_contract[] + = { "LWLockHeldByMe(BufferDescriptorGetContentLock(buf))", + "LockBufHdr", + "BufferTagsEqual", + "cluster_pcm_own_gen_get", + "cluster_pcm_own_reservation_token_get", + "PCM_OWN_FLAG_GRANT_PENDING", + "buf->pcm_state != (uint8) PCM_STATE_N", + "buf_state |= BM_VALID", + "UnlockBufHdr" }; + char *source = read_bufmgr_source(); + + UT_ASSERT(__builtin_types_compatible_p( + __typeof__(&cluster_bufmgr_pcm_own_publish_installed_x_image), PublishImageFn)); + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_publish_installed_x_image(", + "\nstatic ClusterPcmOwnResult\ncluster_pcm_own_begin_grant_reservation(", publish_contract, + lengthof(publish_contract)); + free(source); +} + +UT_TEST(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain) +{ + static const char *const handoff_contract[] = { "LWLockHeldByMe(content_lock)", + "LockBufHdr", + "cluster_pcm_own_classify_live_flags", + "cluster_bufmgr_pcm_current_image_locked", + "cluster_pcm_own_revoke_to_grant_handoff_exact", + "UnlockBufHdr" }; + static const char *const drain_proof_contract[] = { "BufMappingPartitionLock", + "LockBufHdr", + "source_generation + 1", + "flags != 0", + "PCM_STATE_X", + "BUF_TYPE_XCUR", + "UnlockBufHdr" }; + char *source = read_bufmgr_source(); + const char *handoff; + const char *handoff_end; + const char *forbidden; + + assert_ordered_in_function( + source, "\ncluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation(", + "\nstatic ClusterPcmOwnResult\ncluster_pcm_own_finish_grant_reservation(", handoff_contract, + lengthof(handoff_contract)); + handoff = strstr(source, "\ncluster_bufmgr_pcm_own_handoff_revoke_to_x_reservation("); + handoff_end + = handoff != NULL + ? strstr(handoff, + "\nstatic ClusterPcmOwnResult\ncluster_pcm_own_finish_grant_reservation(") + : NULL; + UT_ASSERT_NOT_NULL(handoff); + UT_ASSERT_NOT_NULL(handoff_end); + if (handoff != NULL && handoff_end != NULL) { + forbidden = strstr(handoff, "cluster_pcm_own_reservation_begin_exact("); + UT_ASSERT(forbidden == NULL || forbidden >= handoff_end); + forbidden = strstr(handoff, "cluster_pcm_own_reservation_abort_exact("); + UT_ASSERT(forbidden == NULL || forbidden >= handoff_end); + } + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_own_self_handoff_x_exact(", + "\n/* ==========", drain_proof_contract, + lengthof(drain_proof_contract)); + free(source); +} + +UT_TEST(test_queue_writer_grant_snapshot_is_claim_and_generation_exact) +{ + PcmXLocalWriterClaim claim; + ClusterPcmOwnSnapshot granted; + ClusterPcmOwnSnapshot live; + + memset(&claim, 0, sizeof(claim)); + claim.writer.identity.base_own_generation = 10; + claim.writer.membership_slot.slot_index = 3; + claim.writer.membership_slot.slot_generation = 5; + claim.writer.local_round = 7; + claim.writer.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + claim.active_slot = claim.writer.membership_slot; + claim.claim_generation = 11; + claim.local_round = claim.writer.local_round; + claim.role = claim.writer.role; + memset(&granted, 0, sizeof(granted)); + granted.generation = 11; + granted.reservation_token = 13; + granted.pcm_state = (uint8)PCM_STATE_X; + live = granted; + + UT_ASSERT(cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + live.generation++; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + live = granted; + live.reservation_token++; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + live = granted; + live.flags = PCM_OWN_FLAG_REVOKING; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + live = granted; + live.pcm_state = (uint8)PCM_STATE_S; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + live = granted; + claim.active_slot.slot_generation++; + UT_ASSERT(!cluster_pcm_x_writer_grant_snapshot_exact(&claim, &granted, &live)); + + UT_ASSERT(cluster_pcm_x_should_release_legacy_on_unlock(false, false)); + UT_ASSERT(!cluster_pcm_x_should_release_legacy_on_unlock(false, true)); + UT_ASSERT(!cluster_pcm_x_should_release_legacy_on_unlock(true, false)); + UT_ASSERT(!cluster_pcm_x_should_release_legacy_on_unlock(true, true)); + + UT_ASSERT(cluster_pcm_x_cached_cover_bypasses_queue(true, true, (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_bypasses_queue(false, true, (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_bypasses_queue(true, false, (uint8)PCM_STATE_X, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_bypasses_queue(true, true, (uint8)PCM_STATE_S, 0)); + UT_ASSERT(!cluster_pcm_x_cached_cover_bypasses_queue(true, true, (uint8)PCM_STATE_X, + PCM_OWN_FLAG_GRANT_PENDING)); + UT_ASSERT(!cluster_pcm_x_cached_cover_bypasses_queue(true, true, (uint8)PCM_STATE_X, + PCM_OWN_FLAG_REVOKING)); +} + +UT_TEST(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_authority) +{ + static const char *const prepare_contract[] + = { "entry->phase = PCM_X_WRITER_LEDGER_HANDOFF", + "cluster_gcs_pcm_x_acquire_writer(buf, &entry->claim", + "entry->claim_handed_off", + "cluster_bufmgr_pcm_own_snapshot(buf, &granted)", + "cluster_pcm_x_writer_grant_snapshot_exact(&entry->claim, &granted, &granted)", + "entry->phase = PCM_X_WRITER_LEDGER_ACQUIRING" }; + static const char *const unlock_contract[] + = { "pcm_x_writer_managed = pcm_x_writer != NULL", + "cluster_bufmgr_pcm_x_writer_mark_releasing(pcm_x_writer)", + "LWLockRelease(BufferDescriptorGetContentLock(buf))", + "cluster_bufmgr_pcm_x_writer_release(pcm_x_writer)", + "cluster_bufmgr_pcm_x_holder_unregister(pcm_x_holder)", + "cluster_pcm_x_should_release_legacy_on_unlock(", + "cluster_pcm_lock_unlock_content_buffer(buf, old_mode)" }; + static const char *const acquire_contract[] + = { "pcm_covered = cluster_pcm_x_cached_cover_bypasses_queue(", + "pcm_x_writer = cluster_bufmgr_pcm_x_writer_prepare(buf, pcm_mode)", + "pcm_x_holder = cluster_bufmgr_pcm_x_holder_prepare(buf)", + "LWLockAcquire(BufferDescriptorGetContentLock(buf)", + "cluster_bufmgr_pcm_x_writer_activate(pcm_x_writer)" }; + static const char *const cleanup_contract[] + = { "cluster_bufmgr_pcm_x_writer_exception_cleanup_all()", + "cluster_bufmgr_pcm_x_holder_exception_cleanup_all()" }; + static const char *const owner_exit_contract[] + = { "UnlockBuffers()", "cluster_bufmgr_pcm_x_owner_exit_drain()", "CheckForBufferLeaks()" }; + static const char *const owner_drain_contract[] + = { "cluster_bufmgr_pcm_x_writer_owner_exit_drain_once(runtime_active)", + "cluster_bufmgr_pcm_x_holder_owner_exit_drain_once(runtime_active)", + "if (!writer_retry && !holder_retry)", "pg_usleep(1000L)" }; + static const char *const snapshot_failure_contract[] = { + "release_result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", + "cluster_bufmgr_pcm_x_writer_clear(entry)", + "cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf" + }; + static const char *const writer_holder_publish_contract[] + = { "cluster_pcm_x_local_writer_holder_register_exact(", "entry->handle = handle", + "entry->phase = PCM_X_HOLDER_LEDGER_ACQUIRING", + "committed_own_generation != writer_entry->granted.generation", + "cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_CORRUPT, buf" }; + static const char *const writer_holder_runtime_contract[] + = { "runtime = cluster_pcm_x_runtime_snapshot()", "runtime.state != PCM_X_RUNTIME_ACTIVE", + "if (writer_authorized)", + "cluster_bufmgr_pcm_x_writer_report_failure(PCM_X_QUEUE_NOT_READY, buf", + "return NULL" }; + static const char *const deferred_release_contract[] + = { "cluster_bufmgr_pcm_x_writer_claim_entry_exact(entry, buf)", + "cluster_gcs_pcm_x_writer_claim_release_and_wake_exact(&entry->claim)" }; + static const char *const deferred_cleanup_contract[] + = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", + "cluster_bufmgr_pcm_x_writer_clear(entry)" }; + static const char *const abort_cleanup_contract[] + = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", + "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED" }; + static const char *const exception_cleanup_contract[] + = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", + "entry->phase = PCM_X_WRITER_LEDGER_DEFERRED" }; + static const char *const owner_exit_cleanup_contract[] + = { "result = cluster_gcs_pcm_x_writer_claim_cleanup_and_wake_noexcept(&entry->claim)", + "action = cluster_pcm_x_owner_exit_action(result, false, runtime_active)" }; + char *source = read_bufmgr_source(); + const char *writer_prepare; + const char *writer_prepare_end; + const char *protocol_external_wait; + const char *claim_exact; + const char *claim_exact_end; + const char *grant_exact; + const char *grant_exact_end; + const char *owner_drain; + const char *owner_drain_end; + + UT_ASSERT_NOT_NULL( + strstr(source, "cluster_bufmgr_pcm_x_writer_ledger[LWLOCK_MAX_HELD_BY_PROC]")); + UT_ASSERT_NOT_NULL( + strstr(source, "cluster_bufmgr_pcm_x_holder_ledger[LWLOCK_MAX_HELD_BY_PROC]")); + writer_prepare = strstr(source, "\ncluster_bufmgr_pcm_x_writer_prepare("); + writer_prepare_end = writer_prepare != NULL ? strstr(writer_prepare, "\nstatic ") : NULL; + protocol_external_wait = writer_prepare != NULL + ? strstr(writer_prepare, "cluster_bufmgr_pcm_x_holder_retry_wait(") + : NULL; + UT_ASSERT_NOT_NULL(writer_prepare); + UT_ASSERT_NOT_NULL(writer_prepare_end); + UT_ASSERT(protocol_external_wait == NULL || protocol_external_wait >= writer_prepare_end); + claim_exact = strstr(source, "\ncluster_bufmgr_pcm_x_writer_claim_entry_exact("); + claim_exact_end = claim_exact != NULL ? strstr(claim_exact, "\n}\n") : NULL; + grant_exact = strstr(source, "\ncluster_bufmgr_pcm_x_writer_entry_exact("); + grant_exact_end = grant_exact != NULL ? strstr(grant_exact, "\n}\n") : NULL; + UT_ASSERT_NOT_NULL(claim_exact); + UT_ASSERT_NOT_NULL(claim_exact_end); + UT_ASSERT_NOT_NULL(grant_exact); + UT_ASSERT_NOT_NULL(grant_exact_end); + if (claim_exact != NULL && claim_exact_end != NULL) + UT_ASSERT(strstr(claim_exact, "BufferTagsEqual(") == NULL + || strstr(claim_exact, "BufferTagsEqual(") >= claim_exact_end); + if (grant_exact != NULL && grant_exact_end != NULL) + UT_ASSERT(strstr(grant_exact, "BufferTagsEqual(") != NULL + && strstr(grant_exact, "BufferTagsEqual(") < grant_exact_end); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_prepare(", "\nstatic ", + prepare_contract, lengthof(prepare_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_prepare(", "\nstatic ", + snapshot_failure_contract, lengthof(snapshot_failure_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_release(", "\nstatic ", + deferred_release_contract, lengthof(deferred_release_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_drain_deferred_nowait(", + "\nstatic ", deferred_cleanup_contract, + lengthof(deferred_cleanup_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_abort_acquiring(", + "\nstatic ", abort_cleanup_contract, + lengthof(abort_cleanup_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_exception_cleanup_all(", + "\nstatic ", exception_cleanup_contract, + lengthof(exception_cleanup_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_writer_owner_exit_drain_once(", + "\nstatic ", owner_exit_cleanup_contract, + lengthof(owner_exit_cleanup_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_prepare(", "\nstatic ", + writer_holder_publish_contract, + lengthof(writer_holder_publish_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_holder_prepare(", "\nstatic ", + writer_holder_runtime_contract, + lengthof(writer_holder_runtime_contract)); + assert_ordered_in_function( + source, "\nLockBuffer(Buffer buffer, int mode)", + "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", + acquire_contract, lengthof(acquire_contract)); + assert_ordered_in_function( + source, "\nLockBuffer(Buffer buffer, int mode)", + "\n/*\n * Acquire the content_lock for the buffer, but only if we don't have to wait.", + unlock_contract, lengthof(unlock_contract)); + assert_ordered_in_function(source, "\nUnlockBuffers(void)", + "\n/*\n * Acquire or release the content_lock for the buffer.", + cleanup_contract, lengthof(cleanup_contract)); + assert_ordered_in_function(source, "\nAtProcExit_Buffers(int code, Datum arg)", + "\n/*\n *\t\tCheckForBufferLeaks", owner_exit_contract, + lengthof(owner_exit_contract)); + assert_ordered_in_function(source, "\ncluster_bufmgr_pcm_x_owner_exit_drain(", "\nstatic ", + owner_drain_contract, lengthof(owner_drain_contract)); + owner_drain = strstr(source, "\ncluster_bufmgr_pcm_x_owner_exit_drain("); + owner_drain_end = owner_drain != NULL ? strstr(owner_drain, "\nstatic ") : NULL; + UT_ASSERT_NOT_NULL(owner_drain); + UT_ASSERT_NOT_NULL(owner_drain_end); + if (owner_drain != NULL && owner_drain_end != NULL) { + UT_ASSERT(strstr(owner_drain, "CHECK_FOR_INTERRUPTS") == NULL + || strstr(owner_drain, "CHECK_FOR_INTERRUPTS") >= owner_drain_end); + UT_ASSERT(strstr(owner_drain, "WaitLatch(") == NULL + || strstr(owner_drain, "WaitLatch(") >= owner_drain_end); + UT_ASSERT(strstr(owner_drain, "ereport(") == NULL + || strstr(owner_drain, "ereport(") >= owner_drain_end); + } + free(source); +} + +int +main(void) +{ + UT_PLAN(44); + UT_RUN(test_shmem_initializes_complete_entry); + UT_RUN(test_begin_abort_is_exact_and_monotonic); + UT_RUN(test_invalid_live_flag_shapes_are_corrupt_not_busy); + UT_RUN(test_grant_commit_is_exact_and_bumps_once); + UT_RUN(test_s_revoke_handoff_reuses_exact_token_and_bumps_once); + UT_RUN(test_revoke_handoff_kinds_cover_n_s_x_with_one_lifecycle); + UT_RUN(test_revoke_commit_is_exact_and_classifies_live_races); + UT_RUN(test_revoke_retain_commit_keeps_exact_token_until_release); + UT_RUN(test_revoke_commit_exhaustion_is_side_effect_free); + UT_RUN(test_token_and_generation_never_wrap); + UT_RUN(test_ordinary_generation_bump_rejects_live_reservation); + UT_RUN(test_eviction_rejects_live_reservation_and_exhaustion); + UT_RUN(test_bufmgr_d5a_commitlocked_uses_locked_commit_and_saved_tag_release); + UT_RUN(test_bufmgr_abort_cleanup_is_never_silent); + UT_RUN(test_bufmgr_finish_failure_rolls_back_acquired_master_grant); + UT_RUN(test_bufmgr_s_base_rollback_normalizes_to_n_under_header_authority); + UT_RUN(test_lockbuffer_content_error_uses_post_master_rollback_contract); + UT_RUN(test_bufmgr_generation_bump_failure_is_classified_under_header_lock); + UT_RUN(test_lockbuffer_reservation_failures_use_busy_corrupt_classifier); + UT_RUN(test_bufmgr_finish_rejects_invalid_state_and_initializes_acquire_result); + UT_RUN(test_bufmgr_finish_and_abort_gate_on_exact_base_state); + UT_RUN(test_d5a_release_error_keeps_descriptor_out_of_freelist); + UT_RUN(test_queue_begin_requires_normalized_n_snapshot); + UT_RUN(test_queue_contract_exposes_prepare_only_begin_api); + UT_RUN(test_queue_contract_exposes_opaque_retained_revoke_api); + UT_RUN(test_queue_n_source_refresh_is_exact_and_publishes_only_complete_image); + UT_RUN(test_revoke_finish_mode_rejects_pinned_vm_fsm_and_retains_main); + UT_RUN(test_queue_revoke_retains_main_but_drops_unpinned_vm_fsm); + UT_RUN(test_retained_image_release_and_writeback_gates_are_exact); + UT_RUN(test_retained_drain_retags_invalid_only_after_exact_token_release); + UT_RUN(test_queue_s_release_finish_is_header_exact_and_returns_fresh_n); + UT_RUN(test_lockbuffer_pcm_x_holder_ledger_brackets_both_content_acquires); + UT_RUN(test_bufmgr_pcm_x_holder_ledger_is_bounded_and_uses_private_identity); + UT_RUN(test_unlockbuffers_exceptionally_detaches_released_pcm_x_holders); + UT_RUN(test_bufmgr_pcm_x_holder_gate_retry_is_bounded_outside_content_lock); + UT_RUN(test_bufmgr_pcm_x_holder_reuse_and_deferred_failure_are_fail_closed); + UT_RUN(test_queue_holder_snapshot_by_tag_is_mapping_and_header_exact); + UT_RUN(test_queue_passive_pinned_s_release_serializes_bytes_and_ownership); + UT_RUN(test_current_image_shape_accepts_monotone_xcur_after_x_to_s_yield); + UT_RUN(test_queue_installed_image_publication_is_exact_and_content_locked); + UT_RUN(test_queue_self_source_handoff_is_single_lifecycle_and_readonly_drain); + UT_RUN(test_queue_passive_n_mirror_is_never_gcs_ship_authority); + UT_RUN(test_queue_writer_grant_snapshot_is_claim_and_generation_exact); + UT_RUN(test_lockbuffer_pcm_x_writer_ledger_is_distinct_and_brackets_content_authority); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c new file mode 100644 index 00000000000..f9b62225ae3 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -0,0 +1,14270 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_pcm_x_convert.c + * PCM-X wire ABI and five-pool external shared-memory substrate tests. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_pcm_x_convert.c + * + * NOTES + * This is a pgrac-original file. It links cluster_pcm_x_convert.o + * standalone and supplies checked-arithmetic, shmem, and error stubs. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include + +#include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_pcm_x_convert.h" +#include "cluster/cluster_shmem.h" +#include "storage/buf_internals.h" +#include "storage/proc.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + + +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + if (PG_exception_stack != NULL) + siglongjmp(*PG_exception_stack, 1); + abort(); +} + +void +FlushErrorState(void) +{} + +int MaxBackends = 4; +int NBuffers = 8; +int cluster_lmd_max_wait_edges = 3; +int cluster_node_id = 0; + +static BufferDescPadded fake_buffer_descriptors[8]; +BufferDescPadded *BufferDescriptors = fake_buffer_descriptors; + +static PROC_HDR fake_proc_global; +PROC_HDR *ProcGlobal = &fake_proc_global; + +static union { + uint64 force_align; + char data[4194304]; +} fake_shmem; +static bool fake_shmem_found; +static Size fake_shmem_size; +static const ClusterShmemRegion *registered_region; +static int registered_region_count; + +static sigjmp_buf test_jump; +static bool arithmetic_jump_armed; +static bool ereport_jump_armed; +static int arithmetic_overflow_count; +static int ereport_count; +static Size arithmetic_overflow_left; +static Size arithmetic_overflow_right; +static int lwlock_init_count; +static int allocator_lock_init_count; +static int master_lock_init_count; +static int local_lock_init_count; +static LWLock *held_lwlocks[16]; +static LWLockMode held_lwlock_modes[16]; +static int held_lwlock_count; +static int max_held_lwlock_count; +static int allocator_lock_acquire_count; +static int allocator_lock_shared_count; +static int allocator_lock_exclusive_count; +static int domain_lock_acquire_count; +static int lwlock_release_count; +static int coalesce_interlock_mode; +static PcmXMasterTagSlot *coalesce_interlock_tag; +static PcmXMasterTicketSlot *coalesce_interlock_ticket; +static bool detach_interlock_armed; +static PcmXMasterTagSlot *detach_interlock_tag; +static PcmXMasterTicketSlot *detach_interlock_ticket; +static bool grant_interlock_armed; +static LWLock *grant_interlock_lock; +static PcmXMasterTicketSlot *grant_interlock_ticket; +static uint64 grant_interlock_generation; +static LWLock *local_generation_interlock_lock; +static PcmXSlotHeader *local_generation_interlock_slot; +static LWLock *holder_abort_interlock_lock; +static PcmXLocalMembershipSlot *holder_abort_interlock_target; +static int holder_abort_interlock_match; +static PcmXLocalTagSlot *local_retire_release_interlock_tag; +static PcmXPeerFrontier *local_retire_release_interlock_frontier; +static uint64 local_retire_release_interlock_ticket_id; +static uint32 local_retire_release_interlock_gate_owner; +static bool local_retire_release_interlock_observed_gate; +static bool local_retire_release_interlock_claimed; +static PcmXLocalTagSlot *local_abort_acquire_interlock_tag; +static PcmXPeerFrontier *local_abort_acquire_interlock_frontier; +static uint64 local_abort_acquire_interlock_ticket_id; +static uint32 local_abort_acquire_interlock_gate_owner; +static int local_abort_acquire_interlock_match; +static bool local_abort_acquire_interlock_observed_gate; +static bool local_abort_acquire_interlock_claimed; +static LWLock *local_transfer_peer_drift_lock; +static PcmXPeerFrontier *local_transfer_peer_drift_frontier; +static bool staged_prehandle_insert_exists_armed; +static PcmXPrehandleKey staged_prehandle_insert_exists_key; +static bool local_rekey_insert_failure_armed; +static int local_rekey_insert_failure_match; +static int local_rekey_insert_failure_phase; +static PcmXLocalMembershipSlot *local_rekey_insert_failure_member; +static PcmXWaitIdentity local_rekey_insert_failure_old_identity; +static PcmXWaitIdentity local_rekey_insert_failure_target_identity; +static PcmXDirectoryEntry *local_rekey_insert_failure_poisoned_entry; +static bool local_rekey_insert_failure_observed_old_identity; +static LWLock *lwlock_acquire_error_lock; +static LWLockMode lwlock_acquire_error_mode; +static int lwlock_acquire_error_match; +static bool iterating_held_lwlocks; +static int lock_acquire_during_iteration_count; + +static void maybe_publish_staged_prehandle_insert_exists(void); +static void maybe_inject_local_rekey_insert_failure(void); +static uint64 test_slot_generation(PcmXSlotHeader *slot); +static uint32 test_slot_state(PcmXSlotHeader *slot); +static void test_set_slot_state(PcmXSlotHeader *slot, uint32 state); +static uint32 test_slot_flags(PcmXSlotHeader *slot); +static bool ticket_refs_equal(const PcmXTicketRef *left, const PcmXTicketRef *right); +static void init_active_pcm_x(uint64 master_session_incarnation); +static void bind_local_master(int32 master_node, uint64 cluster_epoch, uint64 master_session); + +static void +test_image_id_domain_is_canonical_and_bounded(void) +{ + uint64 id0; + uint64 id31; + uint64 idmax; + uint64 sequence; + int32 master_node; + + UT_ASSERT(cluster_pcm_x_image_id_encode(0, 1, &id0)); + UT_ASSERT(cluster_pcm_x_image_id_encode(31, 9, &id31)); + UT_ASSERT(cluster_pcm_x_image_id_encode(31, PCM_X_IMAGE_ID_SEQ_MASK, &idmax)); + UT_ASSERT_EQ(id0, UINT64CONST(0xc000000000000001)); + UT_ASSERT_EQ(id31, UINT64CONST(0xdf00000000000009)); + UT_ASSERT_EQ(idmax, UINT64CONST(0xdfffffffffffffff)); + + UT_ASSERT(cluster_pcm_x_image_id_decode(id0, &master_node, &sequence)); + UT_ASSERT_EQ(master_node, 0); + UT_ASSERT_EQ(sequence, 1); + UT_ASSERT(cluster_pcm_x_image_id_decode(id31, &master_node, &sequence)); + UT_ASSERT_EQ(master_node, 31); + UT_ASSERT_EQ(sequence, 9); + UT_ASSERT(cluster_pcm_x_image_id_decode(idmax, &master_node, &sequence)); + UT_ASSERT_EQ(master_node, 31); + UT_ASSERT_EQ(sequence, PCM_X_IMAGE_ID_SEQ_MASK); + + UT_ASSERT(!cluster_pcm_x_image_id_encode(-1, 1, &id0)); + UT_ASSERT(!cluster_pcm_x_image_id_encode(PCM_X_PROTOCOL_NODE_LIMIT, 1, &id0)); + UT_ASSERT(!cluster_pcm_x_image_id_encode(0, 0, &id0)); + UT_ASSERT(!cluster_pcm_x_image_id_encode(0, PCM_X_IMAGE_ID_SEQ_MASK + 1, &id0)); + UT_ASSERT(!cluster_pcm_x_image_id_encode(0, 1, NULL)); + UT_ASSERT( + !cluster_pcm_x_image_id_decode(UINT64CONST(0xe000000000000001), &master_node, &sequence)); + UT_ASSERT( + !cluster_pcm_x_image_id_decode(UINT64CONST(0xf000000000000001), &master_node, &sequence)); + UT_ASSERT( + !cluster_pcm_x_image_id_decode(UINT64CONST(0x8000000000000001), &master_node, &sequence)); + UT_ASSERT(!cluster_pcm_x_image_id_decode(PCM_X_IMAGE_ID_DOMAIN, &master_node, &sequence)); +} + + +void +ExceptionalCondition(const char *condition_name pg_attribute_unused(), + const char *file_name pg_attribute_unused(), + int line_number pg_attribute_unused()) +{ + abort(); +} + +Size +add_size(Size left, Size right) +{ + if (left > SIZE_MAX - right) { + arithmetic_overflow_count++; + if (arithmetic_jump_armed) + siglongjmp(test_jump, 1); + abort(); + } + return left + right; +} + +Size +mul_size(Size left, Size right) +{ + if (left != 0 && right > SIZE_MAX / left) { + arithmetic_overflow_count++; + arithmetic_overflow_left = left; + arithmetic_overflow_right = right; + if (arithmetic_jump_armed) + siglongjmp(test_jump, 1); + abort(); + } + return left * right; +} + +void * +ShmemInitStruct(const char *name, Size size, bool *found_ptr) +{ + UT_ASSERT_STR_EQ(name, PCM_X_SHMEM_REGION_NAME); + UT_ASSERT(size <= sizeof(fake_shmem.data)); + fake_shmem_size = size; + *found_ptr = fake_shmem_found; + fake_shmem_found = true; + return fake_shmem.data; +} + +void +cluster_shmem_register_region(const ClusterShmemRegion *region) +{ + registered_region = region; + registered_region_count++; +} + +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + return elevel >= ERROR; +} + +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} + +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{ + ereport_count++; + if (ereport_jump_armed) + siglongjmp(test_jump, 1); + abort(); +} + +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} + +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +int +errhint(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +void +LWLockInitialize(LWLock *lock, int tranche_id) +{ + lock->tranche = (uint16)tranche_id; + lwlock_init_count++; + if (tranche_id == LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR) + allocator_lock_init_count++; + else if (tranche_id == LWTRANCHE_CLUSTER_PCM_X_MASTER) + master_lock_init_count++; + else if (tranche_id == LWTRANCHE_CLUSTER_PCM_X_LOCAL) + local_lock_init_count++; +} + +bool +LWLockAcquire(LWLock *lock, LWLockMode mode) +{ + UT_ASSERT_NOT_NULL(lock); + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR && mode == LW_EXCLUSIVE + && local_abort_acquire_interlock_tag != NULL + && --local_abort_acquire_interlock_match == 0) { + uint32 flags = test_slot_flags(&local_abort_acquire_interlock_tag->slot); + + UT_ASSERT_NOT_NULL(local_abort_acquire_interlock_frontier); + local_abort_acquire_interlock_observed_gate + = (flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0; + if (!local_abort_acquire_interlock_observed_gate) { + pg_atomic_write_u32(&ClusterPcmXConvertShmem->local_retire_gate, + local_abort_acquire_interlock_gate_owner); + local_abort_acquire_interlock_frontier->local_retire_in_progress_ticket_id + = local_abort_acquire_interlock_ticket_id; + local_abort_acquire_interlock_claimed = true; + } + local_abort_acquire_interlock_tag = NULL; + } + if (iterating_held_lwlocks) + lock_acquire_during_iteration_count++; + if (lock == lwlock_acquire_error_lock && mode == lwlock_acquire_error_mode + && --lwlock_acquire_error_match == 0) { + lwlock_acquire_error_lock = NULL; + UT_ASSERT_NOT_NULL(PG_exception_stack); + siglongjmp(*PG_exception_stack, 1); + } + UT_ASSERT(held_lwlock_count < (int)lengthof(held_lwlocks)); + held_lwlocks[held_lwlock_count] = lock; + held_lwlock_modes[held_lwlock_count] = mode; + held_lwlock_count++; + if (held_lwlock_count > max_held_lwlock_count) + max_held_lwlock_count = held_lwlock_count; + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR) { + allocator_lock_acquire_count++; + if (mode == LW_SHARED) + allocator_lock_shared_count++; + else if (mode == LW_EXCLUSIVE) + allocator_lock_exclusive_count++; + } else if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_MASTER + || lock->tranche == LWTRANCHE_CLUSTER_PCM_X_LOCAL) { + domain_lock_acquire_count++; + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_MASTER && coalesce_interlock_mode != 0) { + int mode = coalesce_interlock_mode; + + coalesce_interlock_mode = 0; + UT_ASSERT_NOT_NULL(coalesce_interlock_tag); + UT_ASSERT_NOT_NULL(coalesce_interlock_ticket); + coalesce_interlock_tag->head_index = PCM_X_INVALID_SLOT_INDEX; + coalesce_interlock_tag->tail_index = PCM_X_INVALID_SLOT_INDEX; + coalesce_interlock_tag->active_index = PCM_X_INVALID_SLOT_INDEX; + coalesce_interlock_ticket->next_index = PCM_X_INVALID_SLOT_INDEX; + coalesce_interlock_ticket->prev_index = PCM_X_INVALID_SLOT_INDEX; + test_set_slot_state(&coalesce_interlock_ticket->slot, PCM_XT_CANCELLED); + pg_atomic_write_u32(&coalesce_interlock_tag->queued_node_bitmap, + mode == 1 ? 0 : UINT32_C(1)); + pg_atomic_write_u32(&coalesce_interlock_tag->admission_gate, mode == 2 ? 1 : 0); + } + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_MASTER && detach_interlock_armed) { + detach_interlock_armed = false; + UT_ASSERT_NOT_NULL(detach_interlock_tag); + UT_ASSERT_NOT_NULL(detach_interlock_ticket); + UT_ASSERT_EQ(detach_interlock_tag->outstanding_ticket_count, 1); + detach_interlock_tag->outstanding_ticket_count = 0; + pg_atomic_write_u32(&detach_interlock_tag->admission_gate, 2); + test_set_slot_state(&detach_interlock_tag->slot, PCM_X_TAG_DETACHING); + test_set_slot_state(&detach_interlock_ticket->slot, PCM_XT_DETACHING); + } + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_MASTER && grant_interlock_armed + && lock == grant_interlock_lock && mode == LW_EXCLUSIVE) { + grant_interlock_armed = false; + UT_ASSERT_NOT_NULL(grant_interlock_ticket); + grant_interlock_ticket->ref.grant_generation = grant_interlock_generation; + } + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_LOCAL + && lock == local_generation_interlock_lock && mode == LW_SHARED + && local_generation_interlock_slot != NULL) { + PcmXSlotHeader *slot = local_generation_interlock_slot; + + local_generation_interlock_slot = NULL; + pg_atomic_write_u32(&slot->generation_change_seq, + pg_atomic_read_u32(&slot->generation_change_seq) | 1U); + } + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_LOCAL && lock == local_transfer_peer_drift_lock + && mode == LW_EXCLUSIVE && local_transfer_peer_drift_frontier != NULL) { + local_transfer_peer_drift_lock = NULL; + local_transfer_peer_drift_frontier->sender_session_incarnation++; + local_transfer_peer_drift_frontier = NULL; + } + } + return false; +} + +void +LWLockRelease(LWLock *lock) +{ + UT_ASSERT(held_lwlock_count > 0); + UT_ASSERT_EQ(held_lwlocks[held_lwlock_count - 1], lock); + held_lwlock_count--; + lwlock_release_count++; + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR + && local_retire_release_interlock_tag != NULL) { + uint32 flags = test_slot_flags(&local_retire_release_interlock_tag->slot); + + UT_ASSERT_NOT_NULL(local_retire_release_interlock_frontier); + local_retire_release_interlock_observed_gate + = (flags & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0; + if (!local_retire_release_interlock_observed_gate) { + pg_atomic_write_u32(&ClusterPcmXConvertShmem->local_retire_gate, + local_retire_release_interlock_gate_owner); + local_retire_release_interlock_frontier->local_retire_in_progress_ticket_id + = local_retire_release_interlock_ticket_id; + local_retire_release_interlock_claimed = true; + } + local_retire_release_interlock_tag = NULL; + } +} + +bool +LWLockHeldByMe(LWLock *lock) +{ + int i; + + if (lock == holder_abort_interlock_lock && holder_abort_interlock_target != NULL + && --holder_abort_interlock_match == 0) { + PcmXLocalMembershipSlot *target = holder_abort_interlock_target; + + holder_abort_interlock_lock = NULL; + holder_abort_interlock_target = NULL; + test_set_slot_state(&target->slot, PCM_XL_HOLDER_ACTIVE); + } + for (i = 0; i < held_lwlock_count; i++) + if (held_lwlocks[i] == lock) + return true; + return false; +} + +bool +LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode) +{ + int i; + + for (i = 0; i < held_lwlock_count; i++) { + if (held_lwlocks[i] == lock && held_lwlock_modes[i] == mode) { + if (lock->tranche == LWTRANCHE_CLUSTER_PCM_X_ALLOCATOR && mode == LW_EXCLUSIVE) { + maybe_publish_staged_prehandle_insert_exists(); + maybe_inject_local_rekey_insert_failure(); + } + return true; + } + } + return false; +} + +bool +LWLockAnyHeldByMe(LWLock *lock, int nlocks, size_t stride) +{ + char *begin = (char *)lock; + char *end = begin + nlocks * stride; + int i; + + for (i = 0; i < held_lwlock_count; i++) { + char *held = (char *)held_lwlocks[i]; + + if (held >= begin && held < end && (held - begin) % stride == 0) + return true; + } + return false; +} + +void +ForEachLWLockHeldByMe(void (*callback)(LWLock *, LWLockMode, void *), void *context) +{ + int i; + + UT_ASSERT(!iterating_held_lwlocks); + iterating_held_lwlocks = true; + for (i = 0; i < held_lwlock_count; i++) + callback(held_lwlocks[i], held_lwlock_modes[i], context); + iterating_held_lwlocks = false; +} + + +#define UT_EXPECT_ARITHMETIC_OVERFLOW(statement) \ + do { \ + if (sigsetjmp(test_jump, 1) == 0) { \ + arithmetic_jump_armed = true; \ + statement; \ + arithmetic_jump_armed = false; \ + UT_ASSERT(false); \ + } else { \ + arithmetic_jump_armed = false; \ + UT_ASSERT(arithmetic_overflow_count > 0); \ + } \ + } while (0) + +#define UT_EXPECT_EREPORT(statement) \ + do { \ + if (sigsetjmp(test_jump, 1) == 0) { \ + ereport_jump_armed = true; \ + statement; \ + ereport_jump_armed = false; \ + UT_ASSERT(false); \ + } else { \ + ereport_jump_armed = false; \ + UT_ASSERT(ereport_count > 0); \ + } \ + } while (0) + +#define UT_EXPECT_PG_EXCEPTION(statement) \ + do { \ + volatile bool caught_ = false; \ + PG_TRY(); \ + { \ + statement; \ + } \ + PG_CATCH(); \ + { \ + caught_ = true; \ + } \ + PG_END_TRY(); \ + UT_ASSERT(caught_); \ + } while (0) + + +static void +reset_fake_shmem(void) +{ + int i; + + memset(&fake_shmem, 0, sizeof(fake_shmem)); + memset(&fake_proc_global, 0, sizeof(fake_proc_global)); + memset(fake_buffer_descriptors, 0, sizeof(fake_buffer_descriptors)); + for (i = 0; i < NBuffers; i++) + fake_buffer_descriptors[i].bufferdesc.buf_id = i; + fake_proc_global.allProcCount = (uint32)(MaxBackends + NUM_AUXILIARY_PROCS); + fake_shmem_found = false; + fake_shmem_size = 0; + ClusterPcmXConvertShmem = NULL; + registered_region = NULL; + registered_region_count = 0; + arithmetic_jump_armed = false; + ereport_jump_armed = false; + arithmetic_overflow_count = 0; + ereport_count = 0; + arithmetic_overflow_left = 0; + arithmetic_overflow_right = 0; + lwlock_init_count = 0; + allocator_lock_init_count = 0; + master_lock_init_count = 0; + local_lock_init_count = 0; + memset(held_lwlocks, 0, sizeof(held_lwlocks)); + memset(held_lwlock_modes, 0, sizeof(held_lwlock_modes)); + held_lwlock_count = 0; + max_held_lwlock_count = 0; + allocator_lock_acquire_count = 0; + allocator_lock_shared_count = 0; + allocator_lock_exclusive_count = 0; + domain_lock_acquire_count = 0; + lwlock_release_count = 0; + coalesce_interlock_mode = 0; + coalesce_interlock_tag = NULL; + coalesce_interlock_ticket = NULL; + detach_interlock_armed = false; + detach_interlock_tag = NULL; + detach_interlock_ticket = NULL; + grant_interlock_armed = false; + grant_interlock_lock = NULL; + grant_interlock_ticket = NULL; + grant_interlock_generation = 0; + local_generation_interlock_lock = NULL; + local_generation_interlock_slot = NULL; + holder_abort_interlock_lock = NULL; + holder_abort_interlock_target = NULL; + holder_abort_interlock_match = 0; + local_retire_release_interlock_tag = NULL; + local_retire_release_interlock_frontier = NULL; + local_retire_release_interlock_ticket_id = 0; + local_retire_release_interlock_gate_owner = 0; + local_retire_release_interlock_observed_gate = false; + local_retire_release_interlock_claimed = false; + local_abort_acquire_interlock_tag = NULL; + local_abort_acquire_interlock_frontier = NULL; + local_abort_acquire_interlock_ticket_id = 0; + local_abort_acquire_interlock_gate_owner = 0; + local_abort_acquire_interlock_match = 0; + local_abort_acquire_interlock_observed_gate = false; + local_abort_acquire_interlock_claimed = false; + local_transfer_peer_drift_lock = NULL; + local_transfer_peer_drift_frontier = NULL; + cluster_node_id = 0; + staged_prehandle_insert_exists_armed = false; + memset(&staged_prehandle_insert_exists_key, 0, sizeof(staged_prehandle_insert_exists_key)); + local_rekey_insert_failure_armed = false; + local_rekey_insert_failure_match = 0; + local_rekey_insert_failure_phase = 0; + local_rekey_insert_failure_member = NULL; + memset(&local_rekey_insert_failure_old_identity, 0, + sizeof(local_rekey_insert_failure_old_identity)); + memset(&local_rekey_insert_failure_target_identity, 0, + sizeof(local_rekey_insert_failure_target_identity)); + local_rekey_insert_failure_poisoned_entry = NULL; + local_rekey_insert_failure_observed_old_identity = false; + lwlock_acquire_error_lock = NULL; + lwlock_acquire_error_mode = LW_SHARED; + lwlock_acquire_error_match = 0; + iterating_held_lwlocks = false; + lock_acquire_during_iteration_count = 0; +} + +static void +arm_lwlock_acquire_error(LWLock *lock, LWLockMode mode, int match) +{ + UT_ASSERT_NOT_NULL(lock); + UT_ASSERT(match > 0); + lwlock_acquire_error_lock = lock; + lwlock_acquire_error_mode = mode; + lwlock_acquire_error_match = match; +} + +static void +compute_runtime_layout(PcmXShmemLayout *layout) +{ + cluster_pcm_x_layout_compute((Size)MaxBackends, (Size)NUM_AUXILIARY_PROCS, (Size)NBuffers, + (Size)cluster_lmd_max_wait_edges, layout); +} + +static PcmXLocalMembershipSlot * +membership_slots(PcmXShmemHeader *header) +{ + return (PcmXLocalMembershipSlot + *)((char *)header + header->layout.pools[PCM_X_POOL_LOCAL_MEMBERSHIP].slots_offset); +} + +static PcmXMasterTagSlot * +master_tag_slots(PcmXShmemHeader *header) +{ + return (PcmXMasterTagSlot *)((char *)header + + header->layout.pools[PCM_X_POOL_MASTER_TAG].slots_offset); +} + +static PcmXMasterTicketSlot * +master_ticket_slots(PcmXShmemHeader *header) +{ + return (PcmXMasterTicketSlot *)((char *)header + + header->layout.pools[PCM_X_POOL_MASTER_TICKET].slots_offset); +} + +static PcmXBlockerSlot * +blocker_slots(PcmXShmemHeader *header) +{ + return (PcmXBlockerSlot *)((char *)header + + header->layout.pools[PCM_X_POOL_BLOCKER].slots_offset); +} + +static PcmXLocalTagSlot * +local_tag_slots(PcmXShmemHeader *header) +{ + return (PcmXLocalTagSlot *)((char *)header + + header->layout.pools[PCM_X_POOL_LOCAL_TAG].slots_offset); +} + +static PcmXSlotHeader * +pool_slot_header(PcmXShmemHeader *header, PcmXPoolKind kind, Size index) +{ + const PcmXPoolLayout *pool = &header->layout.pools[kind]; + + UT_ASSERT(index < pool->capacity); + return (PcmXSlotHeader *)((char *)header + pool->slots_offset + index * pool->slot_size); +} + +static PcmXDirectoryEntry * +directory_entries(PcmXShmemHeader *header, PcmXDirectoryKind kind, Size *capacity) +{ + Size offset; + + switch (kind) { + case PCM_X_DIR_MASTER_TAG: + offset = header->layout.pools[PCM_X_POOL_MASTER_TAG].directory_offset; + *capacity = header->layout.pools[PCM_X_POOL_MASTER_TAG].directory_capacity; + break; + case PCM_X_DIR_MASTER_TICKET_PREHANDLE: + offset = header->layout.master_ticket_directories.prehandle_offset; + *capacity = header->layout.master_ticket_directories.prehandle_capacity; + break; + case PCM_X_DIR_MASTER_TICKET_HANDLE: + offset = header->layout.master_ticket_directories.handle_offset; + *capacity = header->layout.master_ticket_directories.handle_capacity; + break; + case PCM_X_DIR_MASTER_TICKET_RETIRE: + offset = header->layout.master_ticket_directories.retire_offset; + *capacity = header->layout.master_ticket_directories.retire_capacity; + break; + case PCM_X_DIR_LOCAL_TAG: + offset = header->layout.pools[PCM_X_POOL_LOCAL_TAG].directory_offset; + *capacity = header->layout.pools[PCM_X_POOL_LOCAL_TAG].directory_capacity; + break; + case PCM_X_DIR_LOCAL_WAIT: + offset = header->layout.local_wait.directory_offset; + *capacity = header->layout.local_wait.directory_capacity; + break; + case PCM_X_DIR_LOCAL_HOLDER: + offset = header->layout.local_holder.directory_offset; + *capacity = header->layout.local_holder.directory_capacity; + break; + default: + UT_ASSERT(false); + offset = 0; + *capacity = 0; + break; + } + return (PcmXDirectoryEntry *)((char *)header + offset); +} + + +/* + * Inject the otherwise-impossible same-lock race: the admission preflight saw + * NOT_FOUND, but its staged publish later sees an exact resident entry. + */ +static void +maybe_publish_staged_prehandle_insert_exists(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXDirectoryEntry *entries; + PcmXMasterTicketSlot *tickets; + uint64 key_hash; + Size capacity; + Size i; + Size step; + + if (!staged_prehandle_insert_exists_armed || header == NULL) + return; + tickets = master_ticket_slots(header); + for (i = 0; i < header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; i++) { + PcmXMasterTicketSlot *ticket = &tickets[i]; + + if (test_slot_state(&ticket->slot) != PCM_X_SLOT_RESERVED_NONVISIBLE + || ticket->prehandle.sender_session_incarnation + != staged_prehandle_insert_exists_key.sender_session_incarnation + || ticket->prehandle.prehandle_sequence + != staged_prehandle_insert_exists_key.prehandle_sequence) + continue; + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &staged_prehandle_insert_exists_key, &key_hash)); + entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); + for (step = 0; step < capacity; step++) { + PcmXDirectoryEntry *entry = &entries[((key_hash % capacity) + step) % capacity]; + + if (entry->state != PCM_X_DIRECTORY_EMPTY) + continue; + entry->key_hash = key_hash; + entry->slot_index = i; + entry->slot_generation = test_slot_generation(&ticket->slot); + entry->reserved = 0; + pg_write_barrier(); + entry->state = PCM_X_DIRECTORY_OCCUPIED; + staged_prehandle_insert_exists_armed = false; + return; + } + return; + } +} + + +/* + * Fail the rekey target-key publication after the old key has been removed. + * The next allocator-X assertion belongs to rollback publication and clears + * the temporary invalid entry before the old key is scanned. This also + * records whether production kept the resident identity immutable throughout + * the allocator-only phase. + */ +static void +maybe_inject_local_rekey_insert_failure(void) +{ + PcmXShmemHeader *header = ClusterPcmXConvertShmem; + PcmXDirectoryEntry *entries; + uint64 key_hash; + Size capacity; + Size step; + + if (!local_rekey_insert_failure_armed || header == NULL) + return; + if (local_rekey_insert_failure_phase == 1) { + UT_ASSERT_NOT_NULL(local_rekey_insert_failure_poisoned_entry); + local_rekey_insert_failure_poisoned_entry->key_hash = 0; + local_rekey_insert_failure_poisoned_entry->slot_index = 0; + local_rekey_insert_failure_poisoned_entry->slot_generation = 0; + local_rekey_insert_failure_poisoned_entry->reserved = 0; + pg_write_barrier(); + local_rekey_insert_failure_poisoned_entry->state = PCM_X_DIRECTORY_EMPTY; + local_rekey_insert_failure_poisoned_entry = NULL; + local_rekey_insert_failure_phase = 2; + local_rekey_insert_failure_armed = false; + return; + } + UT_ASSERT(local_rekey_insert_failure_match > 0); + if (--local_rekey_insert_failure_match != 0) + return; + + UT_ASSERT_NOT_NULL(local_rekey_insert_failure_member); + local_rekey_insert_failure_observed_old_identity + = memcmp(&local_rekey_insert_failure_member->identity, + &local_rekey_insert_failure_old_identity, + sizeof(local_rekey_insert_failure_old_identity)) + == 0; + UT_ASSERT(cluster_pcm_x_directory_key_hash( + PCM_X_DIR_LOCAL_WAIT, &local_rekey_insert_failure_target_identity, &key_hash)); + entries = directory_entries(header, PCM_X_DIR_LOCAL_WAIT, &capacity); + for (step = 0; step < capacity; step++) { + PcmXDirectoryEntry *entry = &entries[((key_hash % capacity) + step) % capacity]; + + if (entry->state != PCM_X_DIRECTORY_EMPTY) + continue; + entry->key_hash = key_hash; + entry->slot_index = local_rekey_insert_failure_member - membership_slots(header); + entry->slot_generation = test_slot_generation(&local_rekey_insert_failure_member->slot); + entry->reserved = 0; + pg_write_barrier(); + entry->state = UINT32_MAX; + local_rekey_insert_failure_poisoned_entry = entry; + local_rekey_insert_failure_phase = 1; + return; + } + UT_ASSERT(false); +} + +static BufferTag +make_tag(BlockNumber block) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.spcOid = 1; + tag.dbOid = 2; + tag.relNumber = 3; + tag.forkNum = MAIN_FORKNUM; + tag.blockNum = block; + return tag; +} + +static PcmXWaitIdentity +make_wait_identity(BlockNumber block, int32 node_id, uint32 procno, uint64 request_id) +{ + PcmXWaitIdentity identity; + + memset(&identity, 0, sizeof(identity)); + identity.tag = make_tag(block); + identity.node_id = node_id; + identity.procno = procno; + identity.xid = (TransactionId)(100 + procno); + identity.cluster_epoch = UINT64_C(9); + identity.request_id = request_id; + identity.wait_seq = request_id + UINT64_C(1000); + /* Generation zero is deliberately valid for the first ownership tuple. */ + identity.base_own_generation = 0; + return identity; +} + + +static void +test_publish_raw_local_wait_mapping(PcmXShmemHeader *header, const PcmXWaitIdentity *identity, + PcmXSlotRef ref) +{ + PcmXDirectoryEntry *entries; + uint64 key_hash; + Size capacity; + Size step; + + UT_ASSERT_NOT_NULL(header); + UT_ASSERT_NOT_NULL(identity); + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_LOCAL_WAIT, identity, &key_hash)); + entries = directory_entries(header, PCM_X_DIR_LOCAL_WAIT, &capacity); + for (step = 0; step < capacity; step++) { + PcmXDirectoryEntry *entry = &entries[((key_hash % capacity) + step) % capacity]; + + UT_ASSERT(entry->state == PCM_X_DIRECTORY_EMPTY + || entry->state == PCM_X_DIRECTORY_OCCUPIED); + if (entry->state != PCM_X_DIRECTORY_EMPTY) + continue; + entry->key_hash = key_hash; + entry->slot_index = ref.slot_index; + entry->slot_generation = ref.slot_generation; + entry->reserved = 0; + pg_write_barrier(); + entry->state = PCM_X_DIRECTORY_OCCUPIED; + return; + } + UT_ASSERT(false); +} + + +static bool +test_raw_local_wait_mapping_exists(PcmXShmemHeader *header, const PcmXWaitIdentity *identity, + PcmXSlotRef expected) +{ + PcmXDirectoryEntry *entries; + uint64 key_hash; + Size capacity; + Size step; + + UT_ASSERT_NOT_NULL(header); + UT_ASSERT_NOT_NULL(identity); + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_LOCAL_WAIT, identity, &key_hash)); + entries = directory_entries(header, PCM_X_DIR_LOCAL_WAIT, &capacity); + for (step = 0; step < capacity; step++) { + PcmXDirectoryEntry *entry = &entries[((key_hash % capacity) + step) % capacity]; + + UT_ASSERT(entry->state == PCM_X_DIRECTORY_EMPTY + || entry->state == PCM_X_DIRECTORY_OCCUPIED); + if (entry->state == PCM_X_DIRECTORY_EMPTY) + return false; + if (entry->key_hash == key_hash && entry->slot_index == expected.slot_index + && entry->slot_generation == expected.slot_generation) + return true; + } + return false; +} + + +static void +prepare_promoted_rekey_fixture(BlockNumber block, uint64 master_session, uint64 base_own_generation, + PcmXLocalHandle *promoted_out) +{ + PcmXLocalHandle handles[2]; + int i; + + init_active_pcm_x(master_session); + bind_local_master(1, UINT64_C(9), master_session); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity = make_wait_identity( + block, 0, (uint32)(26 + i), UINT64_C(20000) + (uint64)block * 10 + (uint64)i); + + identity.base_own_generation = base_own_generation; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &handles[i]), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], promoted_out), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(promoted_out->role, PCM_X_LOCAL_ROLE_NODE_LEADER); +} + +static PcmXLocalHolderKey +make_local_holder_key(BlockNumber block, int32 node_id, uint32 procno, uint64 request_id, + int32 buffer_id) +{ + PcmXLocalHolderKey key; + + memset(&key, 0, sizeof(key)); + key.identity = make_wait_identity(block, node_id, procno, request_id); + key.buffer_id = buffer_id; + return key; +} + +static PcmXQueueResult +register_active_local_holder(const PcmXLocalHolderKey *key, PcmXLocalHolderHandle *handle) +{ + PcmXQueueResult activate_result; + PcmXQueueResult result; + + result = cluster_pcm_x_local_holder_register(key, handle); + if (result != PCM_X_QUEUE_OK && result != PCM_X_QUEUE_DUPLICATE) + return result; + activate_result = cluster_pcm_x_local_holder_activate_exact(handle); + UT_ASSERT_EQ(activate_result, + result == PCM_X_QUEUE_OK ? PCM_X_QUEUE_OK : PCM_X_QUEUE_DUPLICATE); + return result; +} + +static PcmXQueueResult +release_active_local_holder(const PcmXLocalHolderHandle *handle) +{ + PcmXQueueResult result; + + result = cluster_pcm_x_local_holder_mark_releasing_exact(handle); + if (result != PCM_X_QUEUE_OK) + return result; + return cluster_pcm_x_local_holder_unregister_exact(handle); +} + +static void +arm_local_retire_release_interlock(PcmXLocalTagSlot *tag_slot, int32 master_node, uint64 ticket_id) +{ + UT_ASSERT_NOT_NULL(ClusterPcmXConvertShmem); + UT_ASSERT_NOT_NULL(tag_slot); + UT_ASSERT(master_node >= 0 && master_node < PCM_X_PROTOCOL_NODE_LIMIT); + UT_ASSERT(ticket_id != 0); + UT_ASSERT_NULL(local_retire_release_interlock_tag); + local_retire_release_interlock_tag = tag_slot; + local_retire_release_interlock_frontier = &ClusterPcmXConvertShmem->peer_frontiers[master_node]; + local_retire_release_interlock_ticket_id = ticket_id; + local_retire_release_interlock_gate_owner = (uint32)master_node + 1; + local_retire_release_interlock_observed_gate = false; + local_retire_release_interlock_claimed = false; +} + +static void +arm_local_abort_acquire_interlock(PcmXLocalTagSlot *tag_slot, int32 master_node, uint64 ticket_id, + int allocator_acquire_match) +{ + UT_ASSERT_NOT_NULL(ClusterPcmXConvertShmem); + UT_ASSERT_NOT_NULL(tag_slot); + UT_ASSERT(master_node >= 0 && master_node < PCM_X_PROTOCOL_NODE_LIMIT); + UT_ASSERT(ticket_id != 0); + UT_ASSERT(allocator_acquire_match > 0); + UT_ASSERT_NULL(local_abort_acquire_interlock_tag); + local_abort_acquire_interlock_tag = tag_slot; + local_abort_acquire_interlock_frontier = &ClusterPcmXConvertShmem->peer_frontiers[master_node]; + local_abort_acquire_interlock_ticket_id = ticket_id; + local_abort_acquire_interlock_gate_owner = (uint32)master_node + 1; + local_abort_acquire_interlock_match = allocator_acquire_match; + local_abort_acquire_interlock_observed_gate = false; + local_abort_acquire_interlock_claimed = false; +} + +/* Model one independently live holder-transfer lane on an existing local + * tag. The tests using this helper deliberately isolate the final-writer + * detach interlock from the holder wire state machine: these exact persistent + * fields are the authority that detach must preserve until holder drain. */ +static void +seed_live_local_holder_transfer(PcmXLocalTagSlot *tag_slot, uint32 procno, uint64 request_id, + uint64 ticket_id) +{ + PcmXWaitIdentity identity; + + UT_ASSERT_NOT_NULL(tag_slot); + identity = make_wait_identity(tag_slot->tag.blockNum, 2, procno, request_id); + UT_ASSERT(memcmp(&identity.tag, &tag_slot->tag, sizeof(identity.tag)) == 0); + memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); + tag_slot->holder_ref.identity = identity; + tag_slot->holder_ref.handle.ticket_id = ticket_id; + tag_slot->holder_ref.handle.queue_generation = UINT64_C(1); + tag_slot->holder_ref.grant_generation = UINT64_C(1); + memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); + tag_slot->holder_reliable.state_sequence = UINT64_C(1); + tag_slot->holder_reliable.expected_responder_session = tag_slot->master_session_incarnation; + tag_slot->holder_reliable.expected_responder_node = tag_slot->master_node; + tag_slot->holder_reliable.pending_opcode = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + tag_slot->holder_reliable.phase = PGRAC_IC_MSG_PCM_X_IMAGE_READY; +} + +/* Build the exact steady-state boundary consumed by local round retirement: + * the closed writer cohort is gone, one next-round writer remains, and an + * independently published holder image has reached exact DRAIN. Individual + * tests corrupt one proof field at a time after this helper returns. */ +static PcmXLocalTagSlot * +prepare_retire_ready_round_with_holder(uint64 master_session, BlockNumber block, + PcmXLocalCutoff *cutoff_out, PcmXLocalHandle *late_out) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + identity = make_wait_identity(block, 0, 2, master_session + UINT64_C(100)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, cutoff_out), + PCM_X_QUEUE_OK); + identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(101)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, late_out), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + + tag_slot = &local_tag_slots(header)[late_out->tag_slot.slot_index]; + memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); + tag_slot->holder_ref.identity = leader.identity; + tag_slot->holder_ref.handle.ticket_id = master_session + UINT64_C(200); + tag_slot->holder_ref.handle.queue_generation = UINT64_C(1); + tag_slot->holder_ref.grant_generation = UINT64_C(1); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(300), + &tag_slot->holder_image.image_id)); + tag_slot->holder_image.source_own_generation = UINT64_C(1); + tag_slot->holder_image.page_scn = UINT64_C(2); + tag_slot->holder_image.page_lsn = UINT64_C(3); + tag_slot->holder_image.source_node = 0; + tag_slot->holder_image.page_checksum = UINT32_C(4); + memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); + tag_slot->holder_reliable.state_sequence = UINT64_C(1); + tag_slot->holder_reliable.last_responder_node = 1; + tag_slot->holder_reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_DRAIN_POLL; + tag_slot->holder_terminal_drain_generation = UINT64_C(1); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + pg_atomic_read_u32(&tag_slot->slot.state_flags) + | (PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK << PCM_X_SLOT_FLAGS_SHIFT)); + return tag_slot; +} + +static void +clear_local_holder_transfer(PcmXLocalTagSlot *tag_slot) +{ + UT_ASSERT_NOT_NULL(tag_slot); + memset(&tag_slot->holder_ref, 0, sizeof(tag_slot->holder_ref)); + memset(&tag_slot->holder_reliable, 0, sizeof(tag_slot->holder_reliable)); +} + +static PcmXEnqueuePayload +make_enqueue(PcmXWaitIdentity identity, uint64 sender_session, uint64 prehandle_sequence) +{ + PcmXEnqueuePayload request; + + memset(&request, 0, sizeof(request)); + request.identity = identity; + request.prehandle.sender_session_incarnation = sender_session; + request.prehandle.prehandle_sequence = prehandle_sequence; + return request; +} + +static void +bind_enqueue_peer(const PcmXEnqueuePayload *request) +{ + PcmXQueueResult result; + + result = cluster_pcm_x_peer_bind_ack_publish(request->identity.node_id, + request->identity.cluster_epoch, + request->prehandle.sender_session_incarnation); + UT_ASSERT(result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE); +} + +static void +bind_local_master(int32 master_node, uint64 cluster_epoch, uint64 master_session) +{ + PcmXQueueResult result; + + result = cluster_pcm_x_peer_bind_ack_publish(master_node, cluster_epoch, master_session); + UT_ASSERT(result == PCM_X_QUEUE_OK || result == PCM_X_QUEUE_DUPLICATE); +} + +static ClusterLmdVertex +make_blocker(int32 node_id, uint32 procno, uint64 request_id) +{ + ClusterLmdVertex blocker; + + memset(&blocker, 0, sizeof(blocker)); + blocker.node_id = node_id; + blocker.procno = procno; + blocker.cluster_epoch = UINT64_C(9); + blocker.request_id = request_id; + blocker.xid = (TransactionId)(200 + procno); + blocker.local_start_ts_ms = (int64)(request_id + UINT64_C(500)); + blocker.wait_seq = request_id + UINT64_C(1000); + return blocker; +} + +static uint32 +blocker_set_crc32c(const ClusterLmdVertex *blockers, Size nblockers) +{ + return cluster_pcm_x_blocker_set_crc32c(blockers, nblockers); +} + +typedef struct TestLocalParticipantTransfer { + PcmXLocalHandle leader; + PcmXLocalCutoff cutoff; + PcmXGrantPayload prepare; + PcmXDrainPollPayload poll; + uint64 master_session; +} TestLocalParticipantTransfer; + +/* Drive one requester that also held a local S pin to the exact pre-DRAIN + * boundary, while a different node remains the immutable image source. */ +static void +prepare_local_non_source_participant_transfer(BlockNumber block, uint64 master_session, + TestLocalParticipantTransfer *fixture) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalWriterClaim writer_claim; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload admit_confirm; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXLocalReliableToken token; + + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(block, 0, 2, master_session + UINT64_C(1), 3); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(block, 0, 3, master_session + UINT64_C(2)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &fixture->leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->leader, &writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->leader, &fixture->cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->leader, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = identity; + admit_ack.ref.handle.ticket_id = master_session + UINT64_C(100); + admit_ack.ref.handle.queue_generation = UINT64_C(1); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ( + cluster_pcm_x_local_apply_admit_ack_exact(&fixture->leader, &admit_ack, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->leader, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->leader, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&admit_ack.ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&admit_ack.ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &admit_ack.ref, blocker_snapshot.set_generation, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + + fixture->prepare.ref = admit_ack.ref; + fixture->prepare.ref.grant_generation = UINT64_C(1); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, master_session + UINT64_C(200), + &fixture->prepare.image.image_id)); + fixture->prepare.image.source_own_generation = UINT64_C(9); + fixture->prepare.image.page_scn = UINT64_C(10); + fixture->prepare.image.page_lsn = UINT64_C(11); + fixture->prepare.image.source_node = 2; + fixture->prepare.image.page_checksum = UINT32_C(12); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&fixture->leader, &fixture->prepare, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact( + &fixture->leader, &fixture->prepare.ref, &fixture->prepare.image, + &install_ready, &token), + PCM_X_QUEUE_OK); + memset(&commit, 0, sizeof(commit)); + commit.ref = fixture->prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&fixture->leader, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&fixture->leader, 1, &final_ack, &token), + PCM_X_QUEUE_OK); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = fixture->prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&fixture->leader, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + fixture->poll.ref = fixture->prepare.ref; + fixture->poll.drain_generation = UINT64_C(1); +} + +static PcmXBlockerSetHeaderPayload +make_blocker_header(const PcmXTicketRef *ref, uint64 set_generation, + const ClusterLmdVertex *blockers, Size nblockers) +{ + PcmXBlockerSetHeaderPayload header; + + memset(&header, 0, sizeof(header)); + header.ref = *ref; + header.set_generation = set_generation; + header.nblockers = (uint32)nblockers; + header.set_crc32c = blocker_set_crc32c(blockers, nblockers); + return header; +} + +static PcmXBlockerChunkPayload +make_blocker_chunk(const PcmXTicketRef *ref, uint64 set_generation, uint32 chunk_no, + ClusterLmdVertex blocker) +{ + PcmXBlockerChunkPayload chunk; + + memset(&chunk, 0, sizeof(chunk)); + chunk.tag = ref->identity.tag; + chunk.requester_node = ref->identity.node_id; + chunk.requester_procno = ref->identity.procno; + chunk.chunk_no = chunk_no; + chunk.cluster_epoch = ref->identity.cluster_epoch; + chunk.request_id = ref->identity.request_id; + chunk.handle = ref->handle; + chunk.grant_generation = ref->grant_generation; + chunk.set_generation = set_generation; + chunk.blocker = blocker; + return chunk; +} + +static void +admit_active_probe(BlockNumber block, int32 node_id, uint32 procno, uint64 request_id, + uint64 sender_session, uint64 prehandle_sequence, + PcmXMasterAdmission *admission_out) +{ + PcmXEnqueuePayload request = make_enqueue( + make_wait_identity(block, node_id, procno, request_id), sender_session, prehandle_sequence); + PcmXTicketRef active; + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, admission_out), PCM_X_QUEUE_OK); + UT_ASSERT_NOT_NULL(admission_out); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_exact(&admission_out->ref, UINT64_C(9000) + request_id), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&active, &admission_out->ref)); +} + +UT_TEST(test_master_pending_x_claim_is_active_ticket_exact) +{ + PcmXMasterAdmission admission; + PcmXTicketRef stale; + bool claimed = true; + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(805, 0, 7, UINT64_C(80006), UINT64_C(8106), 1, &admission); + + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_state_exact(&admission.ref, &claimed), + PCM_X_QUEUE_OK); + UT_ASSERT(!claimed); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_state_exact(&admission.ref, &claimed), + PCM_X_QUEUE_OK); + UT_ASSERT(claimed); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_DUPLICATE); + + stale = admission.ref; + stale.identity.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_state_exact(&stale, &claimed), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&stale), PCM_X_QUEUE_STALE); + + /* External GRD ownership must be released before an active claim can be + * cancelled; the engine must not silently discard that proof bit. */ + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); +} + +UT_TEST(test_master_pending_x_claim_state_classifies_cancel_progress_as_not_ready) +{ + PcmXMasterAdmission admission; + PcmXMasterPendingXReleaseToken release; + PcmXMasterProbeToken probe; + PcmXPhasePayload payload; + bool claimed = true; + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(812, 0, 13, UINT64_C(80012), UINT64_C(8112), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + bind_local_master(1, admission.ref.identity.cluster_epoch, UINT64_C(8212)); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 1, UINT64_C(8212), + &payload, &probe), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_state_exact(&admission.ref, &claimed), + PCM_X_QUEUE_NOT_READY); + + init_active_pcm_x(UINT64_C(78)); + admit_active_probe(813, 0, 14, UINT64_C(80013), UINT64_C(8113), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_state_exact(&admission.ref, &claimed), + PCM_X_QUEUE_NOT_READY); +} + +static void +arm_blocker_probe(const PcmXTicketRef *ref, int32 source_node, uint64 source_session) +{ + PcmXPhasePayload probe; + PcmXMasterProbeToken token; + + bind_local_master(source_node, ref->identity.cluster_epoch, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(ref, source_node, source_session, + &probe, &token), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&probe.ref, ref)); + UT_ASSERT_EQ(probe.reason, 0); + UT_ASSERT_EQ(probe.phase, 0); + UT_ASSERT_EQ(probe.flags, 0); + UT_ASSERT(ticket_refs_equal(&token.ref, ref)); + UT_ASSERT_EQ(token.expected_responder_node, source_node); + UT_ASSERT_EQ(token.expected_responder_session, source_session); +} + +static void +commit_empty_blocker_graph(const PcmXTicketRef *ref, uint64 graph_generation) +{ + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterTicketSlot *ticket; + PcmXSlotRef ticket_ref; + uint32 set_crc32c = (uint32)(graph_generation ^ (graph_generation >> 32)) | UINT32_C(1); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(ref, 1, NULL, 0, set_crc32c), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(ref, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + /* This legacy white-box fixture bypasses the 45-48 wire exchange. Mark + * the same durable ACK boundary that production reaches via + * blocker_probe_complete_exact before beginning transfer. */ + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &ref->handle, &ticket_ref), + PCM_X_DIRECTORY_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[ticket_ref.slot_index]; + ticket->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT; +} + +/* Production reserves the GRD cookie before this engine transition. Most + * state-machine tests intentionally mock only the engine, so centralize that + * prerequisite instead of weakening the production begin gate. */ +static PcmXQueueResult +test_master_begin_transfer_claimed_exact(const PcmXTicketRef *ref, uint64 graph_generation, + PcmXTicketRef *transfer_out) +{ + PcmXQueueResult claim_result = cluster_pcm_x_master_pending_x_claim_exact(ref); + + if (claim_result == PCM_X_QUEUE_CORRUPT || claim_result == PCM_X_QUEUE_COUNTER_EXHAUSTED + || claim_result == PCM_X_QUEUE_NO_CAPACITY) + return claim_result; + return cluster_pcm_x_master_begin_transfer_exact(ref, graph_generation, transfer_out); +} + +static PcmXQueueResult +test_master_begin_transfer_unclaimed_exact(const PcmXTicketRef *ref, uint64 graph_generation, + PcmXTicketRef *transfer_out) +{ + return cluster_pcm_x_master_begin_transfer_exact(ref, graph_generation, transfer_out); +} + +static bool +test_ticket_locator_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) +{ + return BufferTagsEqual(&left->identity.tag, &right->identity.tag) + && left->identity.node_id == right->identity.node_id + && left->identity.procno == right->identity.procno + && left->identity.xid == right->identity.xid + && left->identity.cluster_epoch == right->identity.cluster_epoch + && left->identity.request_id == right->identity.request_id + && left->identity.wait_seq == right->identity.wait_seq + && left->identity.base_own_generation == right->identity.base_own_generation + && left->handle.ticket_id == right->handle.ticket_id + && left->handle.queue_generation == right->handle.queue_generation; +} + +/* complete_exact is a legacy engine-only test primitive with no GRD handoff. + * Strip the mocked claim only for those legacy tests; production queue code + * has no caller and must complete through FINAL_CONFIRM. */ +static PcmXQueueResult +test_master_complete_legacy_exact(const PcmXTicketRef *ref) +{ + PcmXSlotRef ticket_ref; + + if (ref != NULL && ClusterPcmXConvertShmem != NULL + && cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &ref->handle, &ticket_ref) + == PCM_X_DIRECTORY_OK) { + PcmXMasterTicketSlot *ticket + = &master_ticket_slots(ClusterPcmXConvertShmem)[ticket_ref.slot_index]; + + if (cluster_pcm_x_runtime_snapshot().state == PCM_X_RUNTIME_ACTIVE + && (ticket_refs_equal(&ticket->ref, ref) + || (grant_interlock_armed && test_ticket_locator_equal(&ticket->ref, ref))) + && test_slot_state(&ticket->slot) == PCM_XT_ACTIVE_TRANSFER) { + uint32 packed = pg_atomic_read_u32(&ticket->slot.state_flags); + + packed &= ~(PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED << PCM_X_SLOT_FLAGS_SHIFT); + pg_atomic_write_u32(&ticket->slot.state_flags, packed); + } + } + return cluster_pcm_x_master_complete_exact(ref); +} + +#define cluster_pcm_x_master_begin_transfer_exact test_master_begin_transfer_claimed_exact +#define cluster_pcm_x_master_complete_exact test_master_complete_legacy_exact + +static Size +directory_occupied_count(PcmXShmemHeader *header, PcmXDirectoryKind kind) +{ + PcmXDirectoryEntry *entries; + Size capacity; + Size count = 0; + Size i; + + entries = directory_entries(header, kind, &capacity); + for (i = 0; i < capacity; i++) + if (entries[i].state == PCM_X_DIRECTORY_OCCUPIED) + count++; + return count; +} + +static void +init_active_pcm_x(uint64 master_session_incarnation) +{ + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_runtime_activate(master_session_incarnation)); +} + +static bool +ticket_refs_equal(const PcmXTicketRef *left, const PcmXTicketRef *right) +{ + return memcmp(left, right, sizeof(*left)) == 0; +} + +static bool +vertex_matches_ticket(const ClusterLmdVertex *vertex, const PcmXTicketRef *ticket) +{ + return vertex->node_id == ticket->identity.node_id && vertex->procno == ticket->identity.procno + && vertex->cluster_epoch == ticket->identity.cluster_epoch + && vertex->request_id == ticket->identity.request_id + && vertex->xid == ticket->identity.xid && vertex->local_start_ts_ms == 0 + && vertex->wait_seq == ticket->identity.wait_seq; +} + +static void +assert_master_queue_baseline(PcmXShmemHeader *header) +{ + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TAG), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TICKET_HANDLE), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TICKET_RETIRE), 0); +} + +static void +assert_local_queue_baseline(PcmXShmemHeader *header) +{ + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), 0); +} + +static void +arm_and_ack_master_terminal_leg(const PcmXTicketRef *ref, PcmXTerminalLegKind kind, + int32 responder_node) +{ + PcmXRetirePayload retire; + PcmXTerminalLegToken token; + uint64 responder_session; + + UT_ASSERT_NOT_NULL(ClusterPcmXConvertShmem); + UT_ASSERT(responder_node >= 0 && responder_node < PCM_X_PROTOCOL_NODE_LIMIT); + responder_session + = ClusterPcmXConvertShmem->peer_frontiers[responder_node].sender_session_incarnation; + UT_ASSERT(responder_session != 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(ref, kind, responder_node, + responder_session, &token), + PCM_X_QUEUE_OK); + if (kind == PCM_X_TERMINAL_LEG_DRAIN) + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(ref, kind, responder_node, + responder_session), + PCM_X_QUEUE_OK); + else { + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = ref->identity.cluster_epoch; + retire.master_session_incarnation = ClusterPcmXConvertShmem->master_session_incarnation; + retire.retire_through_ticket_id = ref->handle.ticket_id; + retire.sender_node = responder_node; + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_exact(&retire, responder_node, responder_session), + PCM_X_QUEUE_OK); + } +} + +static PcmXRetirePayload +make_retire_ack(const PcmXTicketRef *ref, int32 sender_node) +{ + PcmXRetirePayload retire; + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = ref->identity.cluster_epoch; + retire.master_session_incarnation = ClusterPcmXConvertShmem->master_session_incarnation; + retire.retire_through_ticket_id = ref->handle.ticket_id; + retire.sender_node = sender_node; + return retire; +} + +static void +drain_retire_and_detach_master(const PcmXMasterAdmission *admission) +{ + arm_and_ack_master_terminal_leg(&admission->ref, PCM_X_TERMINAL_LEG_DRAIN, + admission->ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission->ref, PCM_X_TERMINAL_LEG_RETIRE, + admission->ref.identity.node_id); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission->ref), PCM_X_QUEUE_OK); +} + +UT_TEST(test_master_pending_x_cancel_release_is_two_phase_and_replay_exact) +{ + PcmXMasterAdmission admission; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXMasterBlockerSnapshot blocker_snapshot; + PcmXMasterPendingXReleaseToken release; + PcmXMasterPendingXReleaseToken replay; + PcmXMasterPendingXReleaseToken stale; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXTerminalLegToken terminal; + PcmXTicketRef work; + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(806, 0, 8, UINT64_C(80007), UINT64_C(8107), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_blocker_probe(&admission.ref, 1, UINT64_C(8207)); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 1, UINT64_C(8207)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 1, UINT64_C(8207), + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&blocker_snapshot, NULL, 0, UINT64_C(9807)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 1, UINT64_C(8207)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING); + UT_ASSERT(ticket_refs_equal(&release.ref, &admission.ref)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&release, &replay, sizeof(release)) == 0); + /* The cold-path scanner must surface RELEASE_PENDING so cleanup survives + * the cancelling backend's death; DRAIN itself remains gated below. */ + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_work_next(&work), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission.ref)); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, admission.ref.identity.node_id, + UINT64_C(8107), &terminal), + PCM_X_QUEUE_NOT_READY); + + stale = release; + stale.master_session_incarnation++; + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&stale), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), 0); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0, UINT64_C(8107), &terminal), + PCM_X_QUEUE_OK); + before = *ticket; + /* A concurrent release worker may arrive after another worker finalized + * and already armed DRAIN. The old release token is now an exact replay; + * it must not compare its obsolete sequence against the terminal leg. */ + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0, UINT64_C(8107)), + PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + assert_master_queue_baseline(ClusterPcmXConvertShmem); +} + +UT_TEST(test_master_pending_x_cancel_before_first_probe_releases_immediately) +{ + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot initial; + PcmXMasterDriveSnapshot seeded; + PcmXMasterPendingXReleaseToken release; + PcmXMasterTicketSlot *ticket; + const uint32 unarmed_holders = (UINT32_C(1) << 1) | (UINT32_C(1) << 2); + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(809, 0, 11, UINT64_C(80010), UINT64_C(8110), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &initial), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_drive_bitmap_replace_exact(&initial, unarmed_holders, 0, &seeded), + PCM_X_QUEUE_OK); + /* A holder becomes a terminal participant only after its exact PROBE leg + * is durable, not merely because it appeared in the authority bitmap. */ + UT_ASSERT_EQ(seeded.involved_nodes_bitmap, UINT32_C(1) << 0); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING); + UT_ASSERT(release.state_sequence != 0); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + assert_master_queue_baseline(ClusterPcmXConvertShmem); +} + +UT_TEST(test_master_cancel_ack_snapshot_distinguishes_prehandle_and_confirmed_cancel) +{ + PcmXMasterAdmission admission; + PcmXMasterAdmission replay; + PcmXMasterPendingXReleaseToken release; + PcmXEnqueuePayload request; + bool prehandle = false; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(816, 0, 16, UINT64_C(80016)), UINT64_C(8116), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_cancel_ack_snapshot_exact(&admission.ref, &replay, &prehandle), + PCM_X_QUEUE_OK); + UT_ASSERT(prehandle); + UT_ASSERT(ticket_refs_equal(&replay.ref, &admission.ref)); + UT_ASSERT(memcmp(&replay.prehandle, &admission.prehandle, sizeof(replay.prehandle)) == 0); + + init_active_pcm_x(UINT64_C(78)); + admit_active_probe(817, 0, 17, UINT64_C(80017), UINT64_C(8117), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), PCM_X_QUEUE_OK); + prehandle = true; + UT_ASSERT_EQ( + cluster_pcm_x_master_cancel_ack_snapshot_exact(&admission.ref, &replay, &prehandle), + PCM_X_QUEUE_OK); + UT_ASSERT(!prehandle); + UT_ASSERT(ticket_refs_equal(&replay.ref, &admission.ref)); +} + +UT_TEST(test_master_pending_x_cancel_prepare_replay_validates_unlinked_tombstone) +{ + PcmXMasterAdmission admission; + PcmXMasterPendingXReleaseToken release; + PcmXMasterPendingXReleaseToken replay; + PcmXMasterTicketSlot *ticket; + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(810, 0, 12, UINT64_C(80011), UINT64_C(8111), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + ticket->next_index = admission.ticket_slot.slot_index; + memset(&replay, 0x7f, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &replay), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(replay.master_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(78)); + admit_active_probe(811, 0, 13, UINT64_C(80012), UINT64_C(8112), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&admission.ref, UINT64_C(9812)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT(ticket->blocker_set_generation != 0); + ticket->reliable.last_response_opcode = 0; + memset(&replay, 0x7f, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &replay), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(replay.master_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(81)); + admit_active_probe(818, 0, 18, UINT64_C(80018), UINT64_C(8118), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + ticket->graph_generation = 0; + memset(&replay, 0x7f, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &replay), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(replay.master_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(79)); + admit_active_probe(814, 0, 14, UINT64_C(80014), UINT64_C(8114), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + ticket->reliable.state_sequence = 0; + memset(&replay, 0x7f, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &replay), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(replay.master_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(80)); + admit_active_probe(815, 0, 15, UINT64_C(80015), UINT64_C(8115), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + ticket->reliable.expected_responder_session = UINT64_C(999); + memset(&replay, 0x7f, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &replay), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(replay.master_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_pending_x_cancel_intent_survives_partial_probe_and_nonempty_graph) +{ + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot blocker_snapshot; + PcmXMasterDriveSnapshot drive_snapshot; + PcmXMasterPendingXReleaseToken release; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXBlockerChunkPayload edge; + PcmXMasterProbeToken probe_replay; + PcmXPhasePayload probe_payload; + PcmXMasterTicketSlot *ticket; + ClusterLmdVertex blocker; + const uint64 source_session = UINT64_C(8217); + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(807, 0, 9, UINT64_C(80008), UINT64_C(8108), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_blocker_probe(&admission.ref, 1, source_session); + blocker = make_blocker(1, 10, UINT64_C(8308)); + blocker_commit = make_blocker_header(&admission.ref, 1, &blocker, 1); + edge = make_blocker_chunk(&admission.ref, 1, 0, blocker); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 1, source_session), + PCM_X_QUEUE_OK); + + /* Cancellation is durable while the holder's reliable PROBE leg finishes; + * it must neither enter transfer nor discard a partial staged set. */ + memset(&release, 0x7f, sizeof(release)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(release.master_session_incarnation, 0); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact(&admission.ref.identity.tag, + admission.ref.identity.cluster_epoch, + &drive_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(drive_snapshot.flags, PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_CANCEL_REQUESTED); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 1, source_session, + &probe_payload, &probe_replay), + PCM_X_QUEUE_DUPLICATE); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge, 1, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 1, source_session, + &entry, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&blocker_snapshot, &entry, 1, + UINT64_C(9808)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 1, source_session), + PCM_X_QUEUE_OK); + /* Once the in-flight PROBE has settled, durable cancellation freezes the + * holder fanout. Re-arming here would starve the release forever. */ + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 1, source_session, + &probe_payload, &probe_replay), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), + PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED + | PCM_X_MASTER_TICKET_F_PENDING_X_RELEASE_PENDING); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_finalize_exact(&release), PCM_X_QUEUE_OK); + /* Type-49 can be admitted to the outbound FIFO and then lost before the + * holder consumes it. Its BEGIN/EDGE/COMMIT retry must still reproduce + * the exact ACK from the cancellation tombstone. */ + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 1, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge, 1, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 1, source_session, + &entry, 1, &blocker_snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 1, source_session), + PCM_X_QUEUE_BAD_STATE); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 1, source_session, + &entry, 1, &blocker_snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + assert_master_queue_baseline(ClusterPcmXConvertShmem); +} + +UT_TEST(test_master_pending_x_cancel_loses_cleanly_to_active_transfer) +{ + PcmXMasterAdmission admission; + PcmXMasterPendingXReleaseToken release; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(808, 0, 10, UINT64_C(80009), UINT64_C(8109), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&admission.ref, UINT64_C(9809)); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9809), &transfer), + PCM_X_QUEUE_OK); + memset(&release, 0x7f, sizeof(release)); + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_cancel_prepare_exact(&admission.ref, &release), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(release.master_session_incarnation, 0); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), PCM_X_MASTER_TICKET_F_PENDING_X_CLAIMED); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +static bool +slot_refs_equal(PcmXSlotRef left, PcmXSlotRef right) +{ + return left.slot_index == right.slot_index && left.slot_generation == right.slot_generation; +} + +static uint64 +test_slot_generation(PcmXSlotHeader *slot) +{ + uint32 after; + uint32 before; + uint64 generation = 0; + int retry; + + for (retry = 0; retry < 16; retry++) { + before = pg_atomic_read_u32(&slot->generation_change_seq); + if ((before & 1U) != 0) + continue; + pg_read_barrier(); + generation = ((uint64)slot->slot_generation_hi << 32) | slot->slot_generation_lo; + pg_read_barrier(); + after = pg_atomic_read_u32(&slot->generation_change_seq); + if (before == after && (after & 1U) == 0) + return generation; + } + UT_ASSERT(false); + return 0; +} + +static void +test_set_slot_generation(PcmXSlotHeader *slot, uint64 generation) +{ + uint32 odd = pg_atomic_read_u32(&slot->generation_change_seq) | 1U; + + pg_atomic_write_u32(&slot->generation_change_seq, odd); + pg_write_barrier(); + slot->slot_generation_lo = (uint32)generation; + slot->slot_generation_hi = (uint32)(generation >> 32); + pg_write_barrier(); + pg_atomic_write_u32(&slot->generation_change_seq, odd + 1U); +} + +static uint32 +test_slot_state(PcmXSlotHeader *slot) +{ + return pg_atomic_read_u32(&slot->state_flags) & PCM_X_SLOT_STATE_MASK; +} + +static void +test_set_slot_state(PcmXSlotHeader *slot, uint32 state) +{ + uint32 current = pg_atomic_read_u32(&slot->state_flags); + uint32 desired; + + do { + desired = (current & PCM_X_SLOT_FLAGS_MASK) | state; + } while (!pg_atomic_compare_exchange_u32(&slot->state_flags, ¤t, desired)); +} + +static uint32 +test_slot_flags(PcmXSlotHeader *slot) +{ + return pg_atomic_read_u32(&slot->state_flags) >> PCM_X_SLOT_FLAGS_SHIFT; +} + +static PcmXSlotHeader * +reserve_slot(PcmXAllocatorKind kind, PcmXSlotRef *ref) +{ + PcmXSlotHeader *slot = NULL; + + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(kind, ref, &slot), PCM_X_ALLOC_OK); + UT_ASSERT_NOT_NULL(slot); + UT_ASSERT_EQ(test_slot_generation(slot), ref->slot_generation); + UT_ASSERT_EQ(test_slot_state(slot), PCM_X_SLOT_RESERVED_NONVISIBLE); + return slot; +} + +static void +assert_corrupt_reserve_has_no_ref(PcmXAllocatorKind kind) +{ + PcmXSlotRef ref = { 0, UINT64_C(99) }; + PcmXSlotHeader *slot = (PcmXSlotHeader *)(uintptr_t)1; + + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(kind, &ref, &slot), PCM_X_ALLOC_CORRUPT); + UT_ASSERT_EQ(ref.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ref.slot_generation, 0); + UT_ASSERT_NULL(slot); +} + +static void +find_prehandles_for_bucket(Size capacity, Size bucket, int needed, PcmXPrehandleKey *keys) +{ + uint64 sequence; + int found = 0; + + for (sequence = 1; sequence <= (uint64)capacity * 1024 && found < needed; sequence++) { + PcmXPrehandleKey key = { UINT64_C(71), sequence }; + uint64 hash; + + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, &hash)); + if ((Size)(hash % capacity) == bucket) + keys[found++] = key; + } + UT_ASSERT_EQ(found, needed); +} + + +UT_TEST(test_wire_abi_sizes_are_exact) +{ + UT_ASSERT_EQ(sizeof(PcmXWaitIdentity), 64); + UT_ASSERT_EQ(sizeof(PcmXLocalHolderKey), 72); + UT_ASSERT_EQ(sizeof(PcmXSlotRef), 16); + UT_ASSERT_EQ(sizeof(PcmXTicketHandle), 16); + UT_ASSERT_EQ(sizeof(PcmXPrehandleKey), 16); + UT_ASSERT_EQ(sizeof(PcmXTicketRef), 88); + UT_ASSERT_EQ(sizeof(PcmXImageToken), 40); + UT_ASSERT_EQ(sizeof(PcmXEnqueuePayload), 80); + UT_ASSERT_EQ(sizeof(PcmXPrehandleCancelPayload), 80); + UT_ASSERT_EQ(sizeof(PcmXAdmitAckPayload), 112); + UT_ASSERT_EQ(sizeof(PcmXPhasePayload), 96); + UT_ASSERT_EQ(sizeof(PcmXRevokePayload), 96); + UT_ASSERT_EQ(sizeof(PcmXGrantPayload), 128); + UT_ASSERT_EQ(sizeof(PcmXInstallReadyPayload), 104); + UT_ASSERT_EQ(sizeof(PcmXFinalAckPayload), 104); + UT_ASSERT_EQ(sizeof(PcmXBlockerSetHeaderPayload), 104); + UT_ASSERT_EQ(sizeof(PcmXBlockerChunkPayload), 128); + UT_ASSERT_EQ(sizeof(PcmXRetirePayload), 32); + UT_ASSERT_EQ(sizeof(PcmXDrainPollPayload), 96); + UT_ASSERT_EQ(sizeof(PcmXDirectoryEntry), 32); + UT_ASSERT_EQ(sizeof(PcmXMasterAdmission), 160); + UT_ASSERT_EQ(sizeof(PcmXLocalHandle), 112); + UT_ASSERT_EQ(sizeof(PcmXLocalCutoff), 48); + UT_ASSERT_EQ(sizeof(PcmXLocalHolderHandle), 104); + UT_ASSERT_EQ(sizeof(PcmXLocalHolderSnapshot), 32); + UT_ASSERT_EQ(sizeof(PcmXMasterBlockerEntry), 72); + UT_ASSERT_EQ(sizeof(PcmXMasterBlockerSnapshot), 112); +} + +UT_TEST(test_wire_abi_offsets_are_exact) +{ + UT_ASSERT_EQ(offsetof(PcmXWaitIdentity, base_own_generation), 56); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderKey, identity), 0); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderKey, buffer_id), 64); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderKey, reserved), 68); + UT_ASSERT_EQ(offsetof(PcmXTicketRef, handle), 64); + UT_ASSERT_EQ(offsetof(PcmXTicketRef, grant_generation), 80); + UT_ASSERT_EQ(offsetof(PcmXEnqueuePayload, prehandle), 64); + UT_ASSERT_EQ(offsetof(PcmXAdmitAckPayload, prehandle), 88); + UT_ASSERT_EQ(offsetof(PcmXAdmitAckPayload, result), 104); + UT_ASSERT_EQ(offsetof(PcmXGrantPayload, image), 88); + UT_ASSERT_EQ(offsetof(PcmXBlockerSetHeaderPayload, set_generation), 88); + UT_ASSERT_EQ(offsetof(PcmXBlockerChunkPayload, blocker), 80); + UT_ASSERT_EQ(offsetof(PcmXRetirePayload, sender_node), 24); + UT_ASSERT_EQ(offsetof(PcmXMasterAdmission, ticket_slot), 120); + UT_ASSERT_EQ(offsetof(PcmXLocalHandle, tag_slot), 64); + UT_ASSERT_EQ(offsetof(PcmXLocalHandle, local_sequence), 96); + UT_ASSERT_EQ(offsetof(PcmXLocalCutoff, master_session_incarnation), 24); + UT_ASSERT_EQ(offsetof(PcmXLocalCutoff, master_node), 40); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderHandle, tag_slot), 72); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderHandle, holder_slot), 88); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderSnapshot, holder_set_generation), 16); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderSnapshot, holder_count), 24); + UT_ASSERT_EQ(offsetof(PcmXMasterBlockerEntry, blocker), 16); + UT_ASSERT_EQ(offsetof(PcmXMasterBlockerEntry, chunk_no), 64); + UT_ASSERT_EQ(offsetof(PcmXMasterBlockerSnapshot, set_generation), 88); + UT_ASSERT_EQ(offsetof(PcmXMasterBlockerSnapshot, graph_generation), 96); + UT_ASSERT_EQ(offsetof(PcmXMasterBlockerSnapshot, blocker_count), 104); + UT_ASSERT_EQ(sizeof(PcmXMasterProbeToken), 112); + UT_ASSERT_EQ(offsetof(PcmXMasterProbeToken, state_sequence), 88); + UT_ASSERT_EQ(offsetof(PcmXMasterProbeToken, expected_responder_node), 104); +} + +UT_TEST(test_runtime_layout_abi_and_offsets_are_exact) +{ + UT_ASSERT_EQ(PCM_X_SHMEM_LAYOUT_VERSION, 12); + UT_ASSERT_EQ(PCM_X_LOCK_PARTITIONS, NUM_BUFFER_PARTITIONS); + UT_ASSERT_EQ(PCM_X_LWLOCK_COUNT, 257); + UT_ASSERT_EQ(sizeof(PcmXShmemLayout), 440); + UT_ASSERT_EQ(sizeof(PcmXAllocatorState), 32); + UT_ASSERT_EQ(sizeof(PcmXStats), 176); + UT_ASSERT_EQ(sizeof(PcmXStatsSnapshot), 224); + UT_ASSERT_EQ(sizeof(PcmXSlotHeader), 24); + UT_ASSERT_EQ(offsetof(PcmXSlotHeader, next_free), 0); + UT_ASSERT_EQ(offsetof(PcmXSlotHeader, generation_change_seq), 8); + UT_ASSERT_EQ(offsetof(PcmXSlotHeader, slot_generation_lo), 12); + UT_ASSERT_EQ(offsetof(PcmXSlotHeader, slot_generation_hi), 16); + UT_ASSERT_EQ(offsetof(PcmXSlotHeader, state_flags), 20); + UT_ASSERT_EQ(sizeof(PcmXReliableLegState), 56); + UT_ASSERT_EQ(sizeof(PcmXLocalProgress), 240); + UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_session_incarnation), 224); + UT_ASSERT_EQ(offsetof(PcmXLocalProgress, master_node), 232); + UT_ASSERT_EQ(sizeof(PcmXLocalHolderProgress), 160); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_session_incarnation), 144); + UT_ASSERT_EQ(offsetof(PcmXLocalHolderProgress, master_node), 152); + UT_ASSERT_EQ(sizeof(PcmXLocalBlockerSnapshot), 112); + UT_ASSERT_EQ(sizeof(PcmXMasterTagSlot), 120); + UT_ASSERT_EQ(sizeof(PcmXMasterTicketSlot), 384); + UT_ASSERT_EQ(sizeof(PcmXBlockerSlot), 128); + UT_ASSERT_EQ(sizeof(PcmXLocalTagSlot), 752); + UT_ASSERT_EQ(sizeof(PcmXLocalMembershipSlot), 168); + UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); + UT_ASSERT_EQ(sizeof(PcmXPeerBinding), 16); + UT_ASSERT_EQ(sizeof(PcmXOutboundTargetFrontier), 32); + UT_ASSERT_EQ(offsetof(PcmXMasterTagSlot, tag), 24); + UT_ASSERT_EQ(offsetof(PcmXMasterTagSlot, active_index), 96); + UT_ASSERT_EQ(offsetof(PcmXMasterTagSlot, admission_gate), 104); + UT_ASSERT_EQ(offsetof(PcmXMasterTagSlot, outstanding_ticket_count), 112); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, ref), 24); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, reliable), 168); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_head_index), 296); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_stage_head_index), 304); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_stage_set_generation), 312); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_stage_source_session), 320); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_set_source_session), 328); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_stage_next_chunk), 352); + UT_ASSERT_EQ(offsetof(PcmXMasterTicketSlot, blocker_set_source_node), 360); + UT_ASSERT_EQ(offsetof(PcmXBlockerSlot, blocker), 40); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, ref), 112); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, reliable), 240); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_head_index), 360); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, membership_count), 384); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, closed_round_member_count), 392); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, terminal_drain_generation), 400); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, committed_own_generation), 408); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_ref), 416); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_image), 504); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_reliable), 544); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, holder_terminal_drain_generation), 600); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_ref), 608); + UT_ASSERT_EQ(offsetof(PcmXLocalTagSlot, blocker_snapshot_reliable), 696); + UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, identity), 24); + UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, tag_slot_index), 128); + UT_ASSERT_EQ(offsetof(PcmXLocalMembershipSlot, admitted_round), 160); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, activation_retry_generation), 668); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, local_retire_gate), 672); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, allocator_lock), 768); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, master_locks), 896); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, local_locks), 17280); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); + UT_ASSERT_EQ(sizeof(PcmXShmemHeader), 36400); +} + +UT_TEST(test_lwlock_held_limit_is_shared_200) +{ + UT_ASSERT_EQ(LWLOCK_MAX_HELD_BY_PROC, 200); +} + +UT_TEST(test_default_capacity_formulas_are_exact) +{ + PcmXShmemLayout layout; + + cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); + UT_ASSERT_EQ(layout.process_capacity, 147); + UT_ASSERT_EQ(layout.locks_per_process, 200); + UT_ASSERT_EQ(layout.active_holder_capacity, 29400); + UT_ASSERT_EQ(layout.holder_tag_capacity, 16384); + UT_ASSERT_EQ(layout.node_ticket_capacity, 4704); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_MASTER_TAG].capacity, 4704); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_MASTER_TICKET].capacity, 4704); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_BLOCKER].capacity, 2048); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_LOCAL_TAG].capacity, 21088); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_LOCAL_MEMBERSHIP].capacity, 29547); +} + +UT_TEST(test_exactly_five_pools_and_bounded_directories) +{ + PcmXShmemLayout layout; + + cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); + UT_ASSERT_EQ(layout.pool_count, 5); + UT_ASSERT_EQ(PCM_X_POOL_COUNT, 5); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_MASTER_TAG].directory_capacity, + layout.pools[PCM_X_POOL_MASTER_TAG].capacity * 2); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_MASTER_TICKET].directory_capacity, 0); + UT_ASSERT_EQ(layout.master_ticket_directories.prehandle_capacity, + layout.pools[PCM_X_POOL_MASTER_TICKET].capacity * 2); + UT_ASSERT_EQ(layout.master_ticket_directories.handle_capacity, + layout.pools[PCM_X_POOL_MASTER_TICKET].capacity * 2); + UT_ASSERT_EQ(layout.master_ticket_directories.retire_capacity, + layout.pools[PCM_X_POOL_MASTER_TICKET].capacity * 2); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_BLOCKER].directory_capacity, 0); + UT_ASSERT_EQ(layout.pools[PCM_X_POOL_LOCAL_TAG].directory_capacity, + layout.pools[PCM_X_POOL_LOCAL_TAG].capacity * 2); + UT_ASSERT_EQ(layout.local_wait.directory_capacity, layout.local_wait.capacity * 2); + UT_ASSERT_EQ(layout.local_holder.directory_capacity, layout.local_holder.capacity * 2); +} + +UT_TEST(test_layout_v12_records_transfer_and_terminal_frontiers) +{ + PcmXShmemLayout layout; + + cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); + UT_ASSERT_EQ(layout.version, 12); + UT_ASSERT_EQ(layout.lock_partition_count, PCM_X_LOCK_PARTITIONS); + UT_ASSERT_EQ(layout.lwlock_count, PCM_X_LWLOCK_COUNT); + UT_ASSERT_EQ(sizeof(PcmXPeerFrontier), 48); + UT_ASSERT_EQ(sizeof(PcmXOutboundTargetFrontier), 32); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, peer_frontiers), 33664); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, stats), 35200); + UT_ASSERT_EQ(offsetof(PcmXShmemHeader, outbound_targets), 35376); +} + +UT_TEST(test_offsets_are_aligned_ordered_and_bounded) +{ + PcmXShmemLayout layout; + Size cursor; + int pool; + + cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); + cursor = layout.header_size; + for (pool = 0; pool < PCM_X_POOL_COUNT; pool++) { + const PcmXPoolLayout *p = &layout.pools[pool]; + + UT_ASSERT_EQ(p->slots_offset % MAXIMUM_ALIGNOF, 0); + UT_ASSERT(p->slots_offset >= cursor); + cursor = p->slots_offset + p->capacity * p->slot_size; + if (pool == PCM_X_POOL_MASTER_TICKET) { + UT_ASSERT_EQ(layout.master_ticket_directories.prehandle_offset % MAXIMUM_ALIGNOF, 0); + UT_ASSERT(layout.master_ticket_directories.prehandle_offset >= cursor); + cursor = layout.master_ticket_directories.prehandle_offset + + layout.master_ticket_directories.prehandle_capacity + * sizeof(PcmXDirectoryEntry); + UT_ASSERT_EQ(layout.master_ticket_directories.handle_offset % MAXIMUM_ALIGNOF, 0); + UT_ASSERT(layout.master_ticket_directories.handle_offset >= cursor); + cursor + = layout.master_ticket_directories.handle_offset + + layout.master_ticket_directories.handle_capacity * sizeof(PcmXDirectoryEntry); + UT_ASSERT_EQ(layout.master_ticket_directories.retire_offset % MAXIMUM_ALIGNOF, 0); + UT_ASSERT(layout.master_ticket_directories.retire_offset >= cursor); + cursor + = layout.master_ticket_directories.retire_offset + + layout.master_ticket_directories.retire_capacity * sizeof(PcmXDirectoryEntry); + } + if (p->directory_capacity > 0) { + UT_ASSERT_EQ(p->directory_offset % MAXIMUM_ALIGNOF, 0); + UT_ASSERT(p->directory_offset >= cursor); + cursor = p->directory_offset + p->directory_capacity * sizeof(PcmXDirectoryEntry); + } + } + UT_ASSERT(layout.local_wait.directory_offset >= cursor); + cursor = layout.local_wait.directory_offset + + layout.local_wait.directory_capacity * sizeof(PcmXDirectoryEntry); + UT_ASSERT(layout.local_holder.directory_offset >= cursor); + cursor = layout.local_holder.directory_offset + + layout.local_holder.directory_capacity * sizeof(PcmXDirectoryEntry); + UT_ASSERT(layout.total_size >= cursor); + UT_ASSERT_EQ(layout.total_size % MAXIMUM_ALIGNOF, 0); +} + +UT_TEST(test_membership_wait_and_holder_partitions_do_not_overlap) +{ + PcmXShmemLayout layout; + const PcmXPoolLayout *membership; + + cluster_pcm_x_layout_compute(122, 25, 16384, 1024, &layout); + membership = &layout.pools[PCM_X_POOL_LOCAL_MEMBERSHIP]; + UT_ASSERT_EQ(layout.local_wait.first_slot_index, 0); + UT_ASSERT_EQ(layout.local_wait.capacity, layout.process_capacity); + UT_ASSERT_EQ(layout.local_holder.first_slot_index, layout.process_capacity); + UT_ASSERT_EQ(layout.local_holder.capacity, layout.active_holder_capacity); + UT_ASSERT_EQ(layout.local_wait.slots_offset, membership->slots_offset); + UT_ASSERT_EQ(layout.local_holder.slots_offset, + membership->slots_offset + + layout.local_holder.first_slot_index * membership->slot_size); + UT_ASSERT_EQ(layout.local_wait.capacity + layout.local_holder.capacity, membership->capacity); +} + +UT_TEST(test_generation_zero_advances_without_being_a_sentinel) +{ + uint64 next = UINT64_MAX; + + UT_ASSERT(cluster_pcm_x_generation_next(0, &next)); + UT_ASSERT_EQ(next, 1); +} + +UT_TEST(test_generation_max_never_wraps) +{ + uint64 next = 77; + + UT_ASSERT(!cluster_pcm_x_generation_next(UINT64_MAX, &next)); + UT_ASSERT_EQ(next, 77); +} + +UT_TEST(test_slot_generation_seqlock_crosses_u32_rollover) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *free_slot; + PcmXSlotHeader *reserved_slot; + PcmXSlotRef ref; + Size free_index; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + free_index = header->allocator[PCM_X_ALLOC_MASTER_TAG].free_head; + free_slot = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, free_index); + test_set_slot_generation(free_slot, UINT64_C(0xffffffff)); + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_MASTER_TAG, &ref, &reserved_slot), + PCM_X_ALLOC_OK); + UT_ASSERT(reserved_slot == free_slot); + UT_ASSERT_EQ(ref.slot_generation, UINT64_C(0x100000000)); + UT_ASSERT_EQ(test_slot_generation(reserved_slot), UINT64_C(0x100000000)); + UT_ASSERT_EQ(pg_atomic_read_u32(&reserved_slot->generation_change_seq) & 1U, 0); +} + +typedef struct PcmXGenerationTearRace { + PcmXSlotHeader *slot; + pg_atomic_uint32 start; + pg_atomic_uint32 half_written; + pg_atomic_uint32 reader_checked; + pg_atomic_uint32 reader_attempts; + pg_atomic_uint32 half_write_overlaps; + pg_atomic_uint32 stop; +} PcmXGenerationTearRace; + +static void * +pcm_x_generation_tear_writer(void *arg) +{ + PcmXGenerationTearRace *race = (PcmXGenerationTearRace *)arg; + uint32 odd; + + while (pg_atomic_read_u32(&race->start) == 0) + ; + odd = pg_atomic_read_u32(&race->slot->generation_change_seq) | 1U; + pg_atomic_write_u32(&race->slot->generation_change_seq, odd); + pg_write_barrier(); + race->slot->slot_generation_lo = UINT32_C(0x44444444); + pg_write_barrier(); + pg_atomic_write_u32(&race->half_written, 1); + while (pg_atomic_read_u32(&race->reader_checked) == 0) + ; + pg_memory_barrier(); + race->slot->slot_generation_hi = UINT32_C(0x33333333); + pg_write_barrier(); + pg_atomic_write_u32(&race->slot->generation_change_seq, odd + 1U); + pg_atomic_write_u32(&race->stop, 1); + return NULL; +} + +UT_TEST(test_slot_generation_revalidate_never_accepts_torn_pair) +{ + PcmXGenerationTearRace race; + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload request; + PcmXSlotRef mixed; + PcmXSlotHeader *resolved; + uint64 raw_generation; + uint32 partition; + pthread_t writer; + int rc; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(611, 0, 2, UINT64_C(7611)), UINT64_C(81), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + test_set_slot_generation(&tag_slot->slot, UINT64CONST(0x1111111122222222)); + mixed = admission.tag_slot; + mixed.slot_generation = UINT64CONST(0x1111111144444444); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&request.identity.tag)); + + race.slot = &tag_slot->slot; + pg_atomic_init_u32(&race.start, 0); + pg_atomic_init_u32(&race.half_written, 0); + pg_atomic_init_u32(&race.reader_checked, 0); + pg_atomic_init_u32(&race.reader_attempts, 0); + pg_atomic_init_u32(&race.half_write_overlaps, 0); + pg_atomic_init_u32(&race.stop, 0); + rc = pthread_create(&writer, NULL, pcm_x_generation_tear_writer, &race); + UT_ASSERT_EQ(rc, 0); + if (rc != 0) + return; + pg_atomic_write_u32(&race.start, 1); + while (pg_atomic_read_u32(&race.half_written) == 0) + ; + pg_read_barrier(); + UT_ASSERT((pg_atomic_read_u32(&tag_slot->slot.generation_change_seq) & 1U) != 0); + raw_generation + = ((uint64)tag_slot->slot.slot_generation_hi << 32) | tag_slot->slot.slot_generation_lo; + UT_ASSERT_EQ(raw_generation, mixed.slot_generation); + pg_atomic_write_u32(&race.half_write_overlaps, 1); + pg_atomic_write_u32(&race.reader_attempts, 1); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + resolved = cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, mixed, PCM_X_TAG_LIVE, + &request.identity.tag, + cluster_pcm_x_tag_hash(&request.identity.tag)); + LWLockRelease(&header->master_locks[partition].lock); + UT_ASSERT_NULL(resolved); + UT_ASSERT((pg_atomic_read_u32(&tag_slot->slot.generation_change_seq) & 1U) != 0); + pg_write_barrier(); + pg_atomic_write_u32(&race.reader_checked, 1); + rc = pthread_join(writer, NULL); + UT_ASSERT_EQ(rc, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&race.reader_attempts), 1); + UT_ASSERT_EQ(pg_atomic_read_u32(&race.half_write_overlaps), 1); + UT_ASSERT_EQ(pg_atomic_read_u32(&race.stop), 1); + UT_ASSERT_EQ(test_slot_generation(&tag_slot->slot), UINT64CONST(0x3333333344444444)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_slot_generation_writer_in_progress_is_retryable) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload request; + PcmXSlotHeader *resolved; + uint32 partition; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(612, 0, 2, UINT64_C(7612)), UINT64_C(82), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + pg_atomic_write_u32(&tag_slot->slot.generation_change_seq, + pg_atomic_read_u32(&tag_slot->slot.generation_change_seq) | 1U); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&request.identity.tag)); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + resolved = cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, admission.tag_slot, + PCM_X_TAG_LIVE, &request.identity.tag, + cluster_pcm_x_tag_hash(&request.identity.tag)); + LWLockRelease(&header->master_locks[partition].lock); + UT_ASSERT_NULL(resolved); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_allocator_exact_release_reuses_with_next_generation) +{ + PcmXShmemHeader *header; + PcmXSlotRef first; + PcmXSlotRef second; + PcmXSlotHeader *slot; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + slot = reserve_slot(PCM_X_ALLOC_MASTER_TAG, &first); + UT_ASSERT_EQ(first.slot_index, 0); + UT_ASSERT_EQ(first.slot_generation, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].high_water, 1); + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TAG, first, + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].high_water, 1); + UT_ASSERT_EQ(test_slot_state(slot), PCM_X_SLOT_FREE); + (void)reserve_slot(PCM_X_ALLOC_MASTER_TAG, &second); + UT_ASSERT_EQ(second.slot_index, first.slot_index); + UT_ASSERT_EQ(second.slot_generation, first.slot_generation + 1); + first.slot_generation--; + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TAG, first, + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_STALE_REF); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); +} + +UT_TEST(test_allocator_generation_max_head_middle_tail_preserve_chain) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *slot; + PcmXSlotRef ref; + Size capacity; + Size i; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + slot = pool_slot_header(header, PCM_X_POOL_LOCAL_MEMBERSHIP, 0); + test_set_slot_generation(slot, UINT64_MAX); + (void)reserve_slot(PCM_X_ALLOC_LOCAL_WAIT, &ref); + UT_ASSERT_EQ(ref.slot_index, 1); + UT_ASSERT_EQ(test_slot_state(slot), PCM_X_SLOT_GENERATION_EXHAUSTED); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].generation_exhausted, 1); + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + test_set_slot_generation(pool_slot_header(header, PCM_X_POOL_LOCAL_MEMBERSHIP, 1), UINT64_MAX); + (void)reserve_slot(PCM_X_ALLOC_LOCAL_WAIT, &ref); + UT_ASSERT_EQ(ref.slot_index, 0); + (void)reserve_slot(PCM_X_ALLOC_LOCAL_WAIT, &ref); + UT_ASSERT_EQ(ref.slot_index, 2); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].generation_exhausted, 1); + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + capacity = header->layout.local_wait.capacity; + test_set_slot_generation(pool_slot_header(header, PCM_X_POOL_LOCAL_MEMBERSHIP, capacity - 1), + UINT64_MAX); + for (i = 0; i < capacity - 1; i++) { + (void)reserve_slot(PCM_X_ALLOC_LOCAL_WAIT, &ref); + UT_ASSERT_EQ(ref.slot_index, i); + } + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_LOCAL_WAIT, &ref, &slot), + PCM_X_ALLOC_NO_CAPACITY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].free_head, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].generation_exhausted, 1); +} + +UT_TEST(test_allocator_lost_free_head_is_corrupt) +{ + PcmXShmemHeader *header; + PcmXAllocatorState *allocator; + Size capacity; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + allocator = &header->allocator[PCM_X_ALLOC_MASTER_TAG]; + capacity = header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity; + UT_ASSERT(allocator->used + allocator->generation_exhausted < capacity); + allocator->free_head = PCM_X_INVALID_SLOT_INDEX; + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); +} + +UT_TEST(test_allocator_generation_exhausted_overflow_is_corrupt) +{ + PcmXShmemHeader *header; + PcmXAllocatorState *allocator; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + allocator = &header->allocator[PCM_X_ALLOC_MASTER_TAG]; + allocator->generation_exhausted = header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity + 1; + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); +} + +UT_TEST(test_allocator_live_free_head_is_corrupt) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *head; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 0); + test_set_slot_state(head, PCM_X_SLOT_RESERVED_NONVISIBLE); + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); +} + +UT_TEST(test_allocator_free_slot_with_packed_flags_is_corrupt) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *head; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 0); + pg_atomic_write_u32(&head->state_flags, + PCM_X_SLOT_FREE + | (PCM_X_LOCAL_TAG_F_REVOKE_BARRIER << PCM_X_SLOT_FLAGS_SHIFT)); + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); + UT_ASSERT_EQ(test_slot_state(head), PCM_X_SLOT_FREE); + UT_ASSERT_EQ(test_slot_flags(head), PCM_X_LOCAL_TAG_F_REVOKE_BARRIER); +} + +UT_TEST(test_allocator_self_loop_is_corrupt) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *head; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 0); + head->next_free = 0; + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); +} + +UT_TEST(test_allocator_invalid_next_is_corrupt) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *head; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 0); + head->next_free = header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity; + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); +} + +UT_TEST(test_allocator_truncated_free_chain_is_corrupt_before_mutation) +{ + PcmXShmemHeader *header; + PcmXAllocatorState *allocator; + PcmXSlotHeader *head; + PcmXSlotHeader *slot = (PcmXSlotHeader *)(uintptr_t)1; + PcmXSlotRef ref = { 0, UINT64_C(99) }; + Size free_head_before; + Size used_before; + uint64 generation_before; + uint32 state_before; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + allocator = &header->allocator[PCM_X_ALLOC_MASTER_TAG]; + head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, allocator->free_head); + UT_ASSERT(header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity - allocator->used + - allocator->generation_exhausted + > 1); + head->next_free = PCM_X_INVALID_SLOT_INDEX; + free_head_before = allocator->free_head; + used_before = allocator->used; + generation_before = test_slot_generation(head); + state_before = test_slot_state(head); + + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_MASTER_TAG, &ref, &slot), + PCM_X_ALLOC_CORRUPT); + UT_ASSERT_EQ(ref.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ref.slot_generation, 0); + UT_ASSERT_NULL(slot); + UT_ASSERT_EQ(allocator->free_head, free_head_before); + UT_ASSERT_EQ(allocator->used, used_before); + UT_ASSERT_EQ(test_slot_generation(head), generation_before); + UT_ASSERT_EQ(test_slot_state(head), state_before); +} + +UT_TEST(test_allocator_nonfree_successor_is_corrupt_before_reserve) +{ + PcmXShmemHeader *header; + PcmXAllocatorState *allocator; + PcmXSlotHeader *head; + PcmXSlotHeader *successor; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + allocator = &header->allocator[PCM_X_ALLOC_MASTER_TAG]; + head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 0); + successor = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 1); + UT_ASSERT_EQ(head->next_free, 1); + test_set_slot_state(successor, PCM_X_SLOT_RESERVED_NONVISIBLE); + assert_corrupt_reserve_has_no_ref(PCM_X_ALLOC_MASTER_TAG); + UT_ASSERT_EQ(allocator->free_head, 0); + UT_ASSERT_EQ(allocator->used, 0); + UT_ASSERT_EQ(test_slot_generation(head), 0); + UT_ASSERT_EQ(test_slot_state(head), PCM_X_SLOT_FREE); +} + +UT_TEST(test_allocator_release_rejects_nonfree_old_head) +{ + PcmXShmemHeader *header; + PcmXAllocatorState *allocator; + PcmXSlotHeader *released; + PcmXSlotHeader *old_head; + PcmXSlotRef ref; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + allocator = &header->allocator[PCM_X_ALLOC_MASTER_TAG]; + released = reserve_slot(PCM_X_ALLOC_MASTER_TAG, &ref); + UT_ASSERT_EQ(allocator->free_head, 1); + old_head = pool_slot_header(header, PCM_X_POOL_MASTER_TAG, 1); + test_set_slot_state(old_head, PCM_X_SLOT_RESERVED_NONVISIBLE); + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TAG, ref, + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_CORRUPT); + UT_ASSERT_EQ(allocator->free_head, 1); + UT_ASSERT_EQ(allocator->used, 1); + UT_ASSERT_EQ(test_slot_state(released), PCM_X_SLOT_RESERVED_NONVISIBLE); +} + +UT_TEST(test_membership_wait_and_holder_allocators_never_borrow) +{ + PcmXShmemHeader *header; + PcmXSlotHeader *slot; + PcmXSlotRef ref; + Size i; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + for (i = 0; i < header->layout.local_wait.capacity; i++) + (void)reserve_slot(PCM_X_ALLOC_LOCAL_WAIT, &ref); + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_LOCAL_WAIT, &ref, &slot), + PCM_X_ALLOC_NO_CAPACITY); + (void)reserve_slot(PCM_X_ALLOC_LOCAL_HOLDER, &ref); + UT_ASSERT_EQ(ref.slot_index, header->layout.local_holder.first_slot_index); + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + for (i = 0; i < header->layout.local_holder.capacity; i++) + (void)reserve_slot(PCM_X_ALLOC_LOCAL_HOLDER, &ref); + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_LOCAL_HOLDER, &ref, &slot), + PCM_X_ALLOC_NO_CAPACITY); + (void)reserve_slot(PCM_X_ALLOC_LOCAL_WAIT, &ref); + UT_ASSERT_EQ(ref.slot_index, 0); +} + +UT_TEST(test_directory_collision_uses_exact_key_not_hash_bucket) +{ + PcmXShmemHeader *header; + PcmXPrehandleKey keys[2]; + PcmXSlotRef refs[2]; + PcmXSlotRef found; + PcmXMasterTicketSlot *tickets[2]; + Size capacity; + Size ignored; + int i; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + (void)directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); + find_prehandles_for_bucket(capacity, 0, 2, keys); + for (i = 0; i < 2; i++) { + tickets[i] = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &refs[i]); + tickets[i]->prehandle = keys[i]; + UT_ASSERT_EQ(cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &keys[i], + refs[i], &found), + PCM_X_DIRECTORY_OK); + } + UT_ASSERT(!slot_refs_equal(refs[0], refs[1])); + for (i = 0; i < 2; i++) { + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &keys[i], &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, refs[i])); + } + (void)directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &ignored); +} + +UT_TEST(test_directory_backshift_preserves_wrapped_probe_chain) +{ + PcmXShmemHeader *header; + PcmXDirectoryEntry *entries; + PcmXPrehandleKey keys[3]; + PcmXSlotRef refs[3]; + PcmXSlotRef found; + Size capacity; + int i; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); + find_prehandles_for_bucket(capacity, capacity - 1, 3, keys); + for (i = 0; i < 3; i++) { + PcmXMasterTicketSlot *ticket + = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &refs[i]); + + ticket->prehandle = keys[i]; + UT_ASSERT_EQ(cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &keys[i], + refs[i], &found), + PCM_X_DIRECTORY_OK); + } + UT_ASSERT_EQ(entries[capacity - 1].slot_index, refs[0].slot_index); + UT_ASSERT_EQ(entries[0].slot_index, refs[1].slot_index); + UT_ASSERT_EQ(entries[1].slot_index, refs[2].slot_index); + UT_ASSERT_EQ( + cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &keys[1], refs[1]), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(entries[0].slot_index, refs[2].slot_index); + UT_ASSERT_EQ(entries[1].state, PCM_X_DIRECTORY_EMPTY); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &keys[2], &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, refs[2])); +} + +UT_TEST(test_directory_stale_generation_and_delete_ref_mismatch_fail_closed) +{ + PcmXPrehandleKey key = { UINT64_C(71), UINT64_C(9) }; + PcmXSlotRef first; + PcmXSlotRef other; + PcmXSlotRef found; + PcmXMasterTicketSlot *ticket; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + ticket = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &first); + ticket->prehandle = key; + UT_ASSERT_EQ( + cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, first, &found), + PCM_X_DIRECTORY_OK); + (void)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &other); + UT_ASSERT_EQ( + cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, other), + PCM_X_DIRECTORY_STALE_REF); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, &found), + PCM_X_DIRECTORY_OK); + test_set_slot_generation(&ticket->slot, test_slot_generation(&ticket->slot) + 1); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, &found), + PCM_X_DIRECTORY_STALE_REF); +} + +UT_TEST(test_directory_full_never_overwrites_existing_entries) +{ + PcmXShmemHeader *header; + PcmXDirectoryEntry *entries; + PcmXPrehandleKey first_key = { UINT64_C(71), UINT64_C(1) }; + PcmXPrehandleKey new_key = { UINT64_C(71), UINT64_C(2) }; + PcmXSlotRef first; + PcmXSlotRef candidate; + PcmXSlotRef found; + PcmXMasterTicketSlot *ticket; + uint64 first_hash; + Size capacity; + Size i; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); + ticket = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &first); + ticket->prehandle = first_key; + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &first_key, + &first_hash)); + for (i = 0; i < capacity; i++) { + entries[i].key_hash = first_hash; + entries[i].slot_index = first.slot_index; + entries[i].slot_generation = first.slot_generation; + entries[i].state = PCM_X_DIRECTORY_OCCUPIED; + } + ticket = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &candidate); + ticket->prehandle = new_key; + UT_ASSERT_EQ(cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &new_key, + candidate, &found), + PCM_X_DIRECTORY_FULL); + for (i = 0; i < capacity; i++) + UT_ASSERT_EQ(entries[i].slot_index, first.slot_index); +} + +UT_TEST(test_master_ticket_prehandle_and_handle_directories_are_independent) +{ + PcmXPrehandleKey prehandle = { UINT64_C(81), UINT64_C(3) }; + PcmXTicketHandle handle = { UINT64_C(9001), UINT64_C(5) }; + PcmXSlotRef ref; + PcmXSlotRef found; + PcmXMasterTicketSlot *ticket; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + ticket = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &ref); + ticket->prehandle = prehandle; + ticket->ref.handle = handle; + UT_ASSERT_EQ( + cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &prehandle, ref, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ( + cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_HANDLE, &handle, ref, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ( + cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &prehandle, ref), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &prehandle, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &handle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, ref)); +} + +UT_TEST(test_local_holder_directory_keeps_backend_identities_independent) +{ + BufferTag tag = make_tag(57); + PcmXLocalHolderKey keys[2]; + PcmXLocalMembershipSlot *slots[2]; + PcmXSlotRef refs[2]; + PcmXSlotRef found; + int i; + + memset(keys, 0, sizeof(keys)); + for (i = 0; i < 2; i++) { + keys[i].identity.tag = tag; + keys[i].identity.node_id = 2; + keys[i].identity.procno = 10 + i; + keys[i].identity.xid = 100 + i; + keys[i].identity.cluster_epoch = 71; + keys[i].identity.request_id = 900 + i; + keys[i].identity.wait_seq = 11 + i; + keys[i].identity.base_own_generation = 21; + keys[i].buffer_id = 7; + } + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + for (i = 0; i < 2; i++) { + slots[i] = (PcmXLocalMembershipSlot *)reserve_slot(PCM_X_ALLOC_LOCAL_HOLDER, &refs[i]); + slots[i]->identity = keys[i].identity; + slots[i]->buffer_id = keys[i].buffer_id; + UT_ASSERT_EQ( + cluster_pcm_x_directory_insert(PCM_X_DIR_LOCAL_HOLDER, &keys[i], refs[i], &found), + PCM_X_DIRECTORY_OK); + } + UT_ASSERT(!slot_refs_equal(refs[0], refs[1])); + for (i = 0; i < 2; i++) { + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_HOLDER, &keys[i], &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, refs[i])); + } + UT_ASSERT_EQ(cluster_pcm_x_directory_delete_exact(PCM_X_DIR_LOCAL_HOLDER, &keys[0], refs[0]), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_HOLDER, &keys[0], &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_HOLDER, &keys[1], &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, refs[1])); +} + +UT_TEST(test_local_holder_registry_returns_canonical_exact_snapshot) +{ + PcmXLocalHolderKey keys[3]; + PcmXLocalHolderHandle handles[3]; + PcmXLocalHolderHandle duplicate; + PcmXLocalHolderHandle snapshot_holders[3]; + PcmXLocalHolderHandle short_output; + PcmXLocalHolderHandle short_before; + PcmXLocalHolderHandle churn_handle; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalHolderSnapshot short_snapshot; + PcmXLocalHolderKey churn_key; + int registration_order[3] = { 2, 0, 1 }; + int i; + + keys[0] = make_local_holder_key(758, 0, 11, UINT64_C(71001), 1); + keys[1] = make_local_holder_key(758, 0, 12, UINT64_C(71002), 2); + keys[2] = make_local_holder_key(758, 1, 3, UINT64_C(71003), 3); + init_active_pcm_x(UINT64_C(7201)); + + for (i = 0; i < 3; i++) + UT_ASSERT_EQ(register_active_local_holder(&keys[registration_order[i]], + &handles[registration_order[i]]), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(register_active_local_holder(&keys[0], &duplicate), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(slot_refs_equal(duplicate.tag_slot, handles[0].tag_slot)); + UT_ASSERT(slot_refs_equal(duplicate.holder_slot, handles[0].holder_slot)); + + memset(snapshot_holders, 0, sizeof(snapshot_holders)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&keys[0].identity.tag, snapshot_holders, + lengthof(snapshot_holders), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 3); + UT_ASSERT_EQ(snapshot.holder_set_generation, 3); + UT_ASSERT(slot_refs_equal(snapshot.tag_slot, handles[0].tag_slot)); + /* Canonical 4-tuple order is independent of the intrusive-list order. */ + for (i = 0; i < 3; i++) { + UT_ASSERT(memcmp(&snapshot_holders[i].key, &keys[i], sizeof(keys[i])) == 0); + UT_ASSERT(slot_refs_equal(snapshot_holders[i].holder_slot, handles[i].holder_slot)); + } + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot_revalidate(&keys[0].identity.tag, &snapshot), + PCM_X_QUEUE_OK); + churn_key = make_local_holder_key(758, 1, 4, UINT64_C(71004), 4); + UT_ASSERT_EQ(register_active_local_holder(&churn_key, &churn_handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&churn_handle), PCM_X_QUEUE_OK); + /* Equal cardinality is not equal authority: balanced churn advances the set token. */ + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot_revalidate(&keys[0].identity.tag, &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&keys[0].identity.tag, snapshot_holders, + lengthof(snapshot_holders), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 3); + UT_ASSERT_EQ(snapshot.holder_set_generation, 5); + + memset(&short_output, 0xa5, sizeof(short_output)); + short_before = short_output; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&keys[0].identity.tag, &short_output, 1, + &short_snapshot), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(short_snapshot.holder_count, 3); + UT_ASSERT_EQ(short_snapshot.holder_set_generation, 5); + UT_ASSERT(memcmp(&short_output, &short_before, sizeof(short_output)) == 0); + + /* The first exact removal is from the middle of the intrusive holder list. */ + UT_ASSERT_EQ(release_active_local_holder(&handles[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot_revalidate(&keys[0].identity.tag, &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handles[0]), PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(release_active_local_holder(&handles[1]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&handles[2]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&keys[0].identity.tag, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 0); + UT_ASSERT_EQ(snapshot.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); +} + +UT_TEST(test_local_holder_stale_handle_cannot_unregister_reused_slot) +{ + PcmXLocalHolderKey key = make_local_holder_key(759, 0, 13, UINT64_C(72001), 4); + PcmXLocalHolderHandle first; + PcmXLocalHolderHandle second; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + + init_active_pcm_x(UINT64_C(7202)); + UT_ASSERT_EQ(register_active_local_holder(&key, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(register_active_local_holder(&key, &second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(second.holder_slot.slot_index, first.holder_slot.slot_index); + UT_ASSERT(second.holder_slot.slot_generation > first.holder_slot.slot_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&first), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(slot_refs_equal(resident.holder_slot, second.holder_slot)); + UT_ASSERT_EQ(release_active_local_holder(&second), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_capacity_failure_rolls_back_holder_only_tag) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey key = make_local_holder_key(760, 0, 14, UINT64_C(73001), 5); + PcmXLocalHolderHandle handle; + + init_active_pcm_x(UINT64_C(7203)); + header = ClusterPcmXConvertShmem; + header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].free_head = PCM_X_INVALID_SLOT_INDEX; + header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used = header->layout.local_holder.capacity; + header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].high_water = header->layout.local_holder.capacity; + UT_ASSERT_EQ(register_active_local_holder(&key, &handle), PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_HOLDER), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG), 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); +} + +UT_TEST(test_local_holder_snapshot_corruption_fails_closed_before_copy) +{ + PcmXLocalHolderKey key = make_local_holder_key(761, 0, 15, UINT64_C(74001), 6); + PcmXLocalHolderHandle handle; + PcmXLocalHolderHandle output; + PcmXLocalHolderHandle before; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalTagSlot *tag_slot; + + init_active_pcm_x(UINT64_C(7204)); + UT_ASSERT_EQ(register_active_local_holder(&key, &handle), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[handle.tag_slot.slot_index]; + tag_slot->active_holder_head_index + = ClusterPcmXConvertShmem->layout.local_holder.first_slot_index + + ClusterPcmXConvertShmem->layout.local_holder.capacity; + memset(&output, 0x5a, sizeof(output)); + before = output; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &output, 1, &snapshot), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT(memcmp(&output, &before, sizeof(output)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_holder_unregister_validates_before_unlinking_corrupt_tag) +{ + PcmXLocalHolderKey key = make_local_holder_key(764, 0, 20, UINT64_C(74002), 6); + PcmXLocalHolderHandle handle; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *holder; + Size holder_head; + Size holder_next; + Size holder_previous; + uint64 set_generation; + + init_active_pcm_x(UINT64_C(7207)); + UT_ASSERT_EQ(register_active_local_holder(&key, &handle), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[handle.tag_slot.slot_index]; + holder = &membership_slots(ClusterPcmXConvertShmem)[handle.holder_slot.slot_index]; + /* A holder-only tag cannot claim a local waiter queue head. */ + tag_slot->head_index = 0; + holder_head = tag_slot->active_holder_head_index; + set_generation = tag_slot->holder_set_generation; + holder_next = holder->next_index; + holder_previous = holder->prev_index; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handle), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder_head); + UT_ASSERT_EQ(tag_slot->holder_set_generation, set_generation); + UT_ASSERT_EQ(holder->next_index, holder_next); + UT_ASSERT_EQ(holder->prev_index, holder_previous); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_RELEASING); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 1); + UT_ASSERT_EQ(directory_occupied_count(ClusterPcmXConvertShmem, PCM_X_DIR_LOCAL_HOLDER), 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_holder_only_tag_is_adopted_by_local_queue_without_pool_borrowing) +{ + PcmXLocalHolderKey holder_key = make_local_holder_key(762, 0, 16, UINT64_C(75001), 7); + PcmXLocalHolderHandle holder; + PcmXLocalHandle waiter; + PcmXLocalHandle no_successor; + PcmXWaitIdentity waiter_identity = make_wait_identity(762, 0, 17, UINT64_C(75002)); + PcmXLocalTagSlot *tag_slot; + + init_active_pcm_x(UINT64_C(7205)); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->master_node, -1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, 0); + bind_local_master(1, waiter_identity.cluster_epoch, UINT64_C(7601)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&waiter_identity, 1, UINT64_C(7601), &waiter), + PCM_X_QUEUE_OK); + UT_ASSERT(slot_refs_equal(waiter.tag_slot, holder.tag_slot)); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, UINT64_C(7601)); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 1); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 1); + + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&waiter, &no_successor), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&waiter), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); +} + +UT_TEST(test_holder_only_tag_survives_last_cancelled_waiter_detach) +{ + PcmXLocalHolderKey holder_key = make_local_holder_key(765, 0, 21, UINT64_C(77001), 6); + PcmXWaitIdentity first_identity = make_wait_identity(765, 0, 22, UINT64_C(77002)); + PcmXWaitIdentity second_identity = make_wait_identity(765, 0, 23, UINT64_C(77003)); + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle snapshot_holder; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalHandle first; + PcmXLocalHandle second; + PcmXLocalHandle no_successor; + PcmXLocalTagSlot *tag_slot; + + init_active_pcm_x(UINT64_C(7208)); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, first_identity.cluster_epoch, UINT64_C(7801)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&first_identity, 1, UINT64_C(7801), &first), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&first, &no_successor), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_LIVE); + UT_ASSERT_EQ(tag_slot->membership_count, 0); + UT_ASSERT_EQ(tag_slot->master_node, -1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, 0); + UT_ASSERT_EQ(tag_slot->next_sequence, 1); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 1); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &snapshot_holder, 1, + &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(memcmp(&snapshot_holder, &holder, sizeof(holder)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&second_identity, 1, UINT64_C(7801), &second), + PCM_X_QUEUE_OK); + UT_ASSERT(slot_refs_equal(second.tag_slot, holder.tag_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&second, &no_successor), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); +} + +UT_TEST(test_local_holder_register_refuses_closed_revoke_barrier) +{ + PcmXWaitIdentity leader_identity = make_wait_identity(763, 0, 18, UINT64_C(76001)); + PcmXLocalHolderKey late_holder = make_local_holder_key(763, 0, 19, UINT64_C(76002), 0); + PcmXLocalHandle leader; + PcmXLocalHolderHandle rejected; + PcmXLocalCutoff cutoff; + + init_active_pcm_x(UINT64_C(7206)); + bind_local_master(1, leader_identity.cluster_epoch, UINT64_C(7701)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 1, UINT64_C(7701), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&late_holder, &rejected), + PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(directory_occupied_count(ClusterPcmXConvertShmem, PCM_X_DIR_LOCAL_HOLDER), 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_holder_prepare_is_visible_before_content_lock) +{ + PcmXLocalHolderKey key = make_local_holder_key(766, 0, 24, UINT64_C(78001), 7); + PcmXLocalHolderHandle handle; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalMembershipSlot *holder; + + init_active_pcm_x(UINT64_C(7209)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &handle), PCM_X_QUEUE_OK); + holder = &membership_slots(ClusterPcmXConvertShmem)[handle.holder_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_ACQUIRING); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(slot_refs_equal(resident.holder_slot, handle.holder_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&handle), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_abort_acquiring_unlinks_exact) +{ + PcmXLocalHolderKey key = make_local_holder_key(776, 0, 26, UINT64_C(87001), 7); + PcmXLocalHolderHandle handle; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalMembershipSlot *holder; + + init_active_pcm_x(UINT64_C(7214)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &handle), PCM_X_QUEUE_OK); + holder = &membership_slots(ClusterPcmXConvertShmem)[handle.holder_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_ACQUIRING); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 0); + UT_ASSERT_EQ(snapshot.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); + UT_ASSERT_EQ(directory_occupied_count(ClusterPcmXConvertShmem, PCM_X_DIR_LOCAL_HOLDER), 0); + UT_ASSERT_EQ(directory_occupied_count(ClusterPcmXConvertShmem, PCM_X_DIR_LOCAL_TAG), 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&handle), PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_holder_abort_acquiring_rejects_live_content_states) +{ + PcmXLocalHolderKey key = make_local_holder_key(777, 0, 27, UINT64_C(87002), 6); + PcmXLocalHolderHandle handle; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *holder; + Size holder_head; + uint64 holder_set_generation; + + init_active_pcm_x(UINT64_C(7215)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &handle), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[handle.tag_slot.slot_index]; + holder = &membership_slots(ClusterPcmXConvertShmem)[handle.holder_slot.slot_index]; + holder_head = tag_slot->active_holder_head_index; + holder_set_generation = tag_slot->holder_set_generation; + + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&handle), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_ACTIVE); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder_head); + UT_ASSERT_EQ(tag_slot->holder_set_generation, holder_set_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(slot_refs_equal(resident.holder_slot, handle.holder_slot)); + + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&handle), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_RELEASING); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder_head); + UT_ASSERT_EQ(tag_slot->holder_set_generation, holder_set_generation); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 1); + UT_ASSERT_EQ(directory_occupied_count(ClusterPcmXConvertShmem, PCM_X_DIR_LOCAL_HOLDER), 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handle), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_abort_acquiring_loses_race_to_activate) +{ + PcmXLocalHolderKey existing_key = make_local_holder_key(779, 0, 1, UINT64_C(87004), 4); + PcmXLocalHolderKey target_key = make_local_holder_key(779, 0, 2, UINT64_C(87005), 3); + PcmXLocalHolderHandle existing; + PcmXLocalHolderHandle target; + PcmXLocalHolderHandle residents[2]; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalMembershipSlot *target_slot; + PcmXQueueResult result; + uint32 partition; + + init_active_pcm_x(UINT64_C(7217)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&existing_key, &existing), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&target_key, &target), PCM_X_QUEUE_OK); + target_slot = &membership_slots(ClusterPcmXConvertShmem)[target.holder_slot.slot_index]; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&target_key.identity.tag)); + /* The third local-domain ownership check resolves target's successor, after + * abort has sampled ACQUIRING but before it commits the unlink. */ + holder_abort_interlock_lock = &ClusterPcmXConvertShmem->local_locks[partition].lock; + holder_abort_interlock_target = target_slot; + holder_abort_interlock_match = 3; + result = cluster_pcm_x_local_holder_abort_acquiring_exact(&target); + UT_ASSERT_EQ(result, PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_NULL(holder_abort_interlock_target); + UT_ASSERT_EQ(test_slot_state(&target_slot->slot), PCM_XL_HOLDER_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&target_key.identity.tag, residents, + lengthof(residents), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 2); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + if (result == PCM_X_QUEUE_BAD_STATE) { + UT_ASSERT_EQ(release_active_local_holder(&target), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&existing), PCM_X_QUEUE_OK); + } +} + +UT_TEST(test_local_holder_abort_acquiring_rejects_stale_aba_handle) +{ + PcmXLocalHolderKey key = make_local_holder_key(778, 0, 28, UINT64_C(87003), 5); + PcmXLocalHolderHandle first; + PcmXLocalHolderHandle second; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + + init_active_pcm_x(UINT64_C(7216)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(second.holder_slot.slot_index, first.holder_slot.slot_index); + UT_ASSERT(second.holder_slot.slot_generation > first.holder_slot.slot_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&first), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(slot_refs_equal(resident.holder_slot, second.holder_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_holder_exceptional_detach_requires_released_content_lock) +{ + PcmXLocalHolderKey key = make_local_holder_key(780, 0, 1, UINT64_C(87006), 4); + PcmXLocalHolderHandle handle; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + LWLock *content_lock = BufferDescriptorGetContentLock(&fake_buffer_descriptors[4].bufferdesc); + + init_active_pcm_x(UINT64_C(7218)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &handle), PCM_X_QUEUE_OK); + LWLockAcquire(content_lock, LW_SHARED); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&handle, content_lock), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + LWLockRelease(content_lock); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&handle, content_lock), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 0); +} + +UT_TEST(test_local_holder_exceptional_detach_accepts_live_content_states) +{ + PcmXLocalHolderKey active_key = make_local_holder_key(781, 0, 2, UINT64_C(87007), 5); + PcmXLocalHolderKey releasing_key = make_local_holder_key(782, 0, 3, UINT64_C(87008), 6); + PcmXLocalHolderHandle active; + PcmXLocalHolderHandle releasing; + LWLock *active_lock = BufferDescriptorGetContentLock(&fake_buffer_descriptors[5].bufferdesc); + LWLock *releasing_lock = BufferDescriptorGetContentLock(&fake_buffer_descriptors[6].bufferdesc); + + init_active_pcm_x(UINT64_C(7219)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&active_key, &active), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&active), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&active, active_lock), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&releasing_key, &releasing), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&releasing), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&releasing), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&releasing, releasing_lock), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_holder_exceptional_detach_rejects_stale_aba_handle) +{ + PcmXLocalHolderKey key = make_local_holder_key(783, 0, 4, UINT64_C(87009), 7); + PcmXLocalHolderHandle first; + PcmXLocalHolderHandle second; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + LWLock *content_lock = BufferDescriptorGetContentLock(&fake_buffer_descriptors[7].bufferdesc); + + init_active_pcm_x(UINT64_C(7220)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&first, content_lock), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(second.holder_slot.slot_index, first.holder_slot.slot_index); + UT_ASSERT(second.holder_slot.slot_generation > first.holder_slot.slot_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&first, content_lock), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_abort_acquiring_exact(&second), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_cleanup_lock_errors_return_corrupt_without_rethrow) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey acquiring_key = make_local_holder_key(784, 0, 5, UINT64_C(87010), 1); + PcmXLocalHolderKey active_key = make_local_holder_key(785, 0, 6, UINT64_C(87011), 2); + PcmXLocalHolderHandle acquiring; + PcmXLocalHolderHandle active; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *holder; + LWLock *content_lock = BufferDescriptorGetContentLock(&fake_buffer_descriptors[2].bufferdesc); + volatile PcmXQueueResult result = PCM_X_QUEUE_INVALID; + volatile bool caught = false; + + /* Abort cleanup runs inside an outer PG_CATCH. A lock-acquire exception + * must be converted to CORRUPT rather than replacing the original ERROR; + * the exact ACQUIRING evidence remains for fail-closed recovery. */ + init_active_pcm_x(UINT64_C(7221)); + header = ClusterPcmXConvertShmem; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&acquiring_key, &acquiring), PCM_X_QUEUE_OK); + holder = &membership_slots(header)[acquiring.holder_slot.slot_index]; + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_SHARED, 1); + PG_TRY(); + { + result = cluster_pcm_x_local_holder_abort_acquiring_exact(&acquiring); + } + PG_CATCH(); + { + caught = true; + } + PG_END_TRY(); + UT_ASSERT(!caught); + UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_ACQUIRING); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Exceptional cleanup has the same no-second-error contract for ACTIVE + * evidence after the content lock has already been released. */ + init_active_pcm_x(UINT64_C(7222)); + header = ClusterPcmXConvertShmem; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&active_key, &active), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&active), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[active.tag_slot.slot_index]; + holder = &membership_slots(header)[active.holder_slot.slot_index]; + result = PCM_X_QUEUE_INVALID; + caught = false; + /* Reach the post-domain handoff: holder/tag are already DETACHING and the + * guarded final allocator acquire must release the admission gate before + * the cleanup wrapper swallows its secondary ERROR. */ + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 1); + PG_TRY(); + { + result = cluster_pcm_x_local_holder_exceptional_detach_exact(&active, content_lock); + } + PG_CATCH(); + { + caught = true; + } + PG_END_TRY(); + UT_ASSERT(!caught); + UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_DETACHING); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_HOLDER), 1); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG), 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_holder_content_window_transitions_are_lock_free) +{ + PcmXLocalHolderKey key = make_local_holder_key(767, 0, 25, UINT64_C(79001), 6); + PcmXLocalHolderHandle handle; + PcmXLocalMembershipSlot *holder; + int allocator_before; + int domain_before; + + init_active_pcm_x(UINT64_C(7210)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &handle), PCM_X_QUEUE_OK); + holder = &membership_slots(ClusterPcmXConvertShmem)[handle.holder_slot.slot_index]; + allocator_before = allocator_lock_acquire_count; + domain_before = domain_lock_acquire_count; + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&handle), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_RELEASING); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&handle), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(allocator_lock_acquire_count, allocator_before); + UT_ASSERT_EQ(domain_lock_acquire_count, domain_before); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handle), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_finish_requires_post_content_releasing_state) +{ + PcmXLocalHolderKey key = make_local_holder_key(768, 0, 26, UINT64_C(80001), 5); + PcmXLocalHolderHandle handle; + + init_active_pcm_x(UINT64_C(7211)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handle), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handle), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&handle), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&handle), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_gate_contention_is_retryable_without_fail_closed) +{ + PcmXLocalHolderKey first_key = make_local_holder_key(769, 0, 27, UINT64_C(81001), 4); + PcmXLocalHolderKey late_key = make_local_holder_key(769, 0, 28, UINT64_C(81002), 3); + PcmXLocalHolderHandle first; + PcmXLocalHolderHandle rejected; + PcmXLocalTagSlot *tag_slot; + uint32 state_flags; + + init_active_pcm_x(UINT64_C(7212)); + UT_ASSERT_EQ(register_active_local_holder(&first_key, &first), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[first.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&late_key, &rejected), PCM_X_QUEUE_GATE_RETRY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 1); + UT_ASSERT_EQ(directory_occupied_count(ClusterPcmXConvertShmem, PCM_X_DIR_LOCAL_HOLDER), 1); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + UT_ASSERT_EQ(release_active_local_holder(&first), PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_holder_barrier_counts_acquiring_active_and_releasing) +{ + PcmXWaitIdentity leader_identity = make_wait_identity(770, 0, 1, UINT64_C(82001)); + PcmXLocalHolderKey holder_key = make_local_holder_key(770, 0, 2, UINT64_C(82002), 2); + PcmXLocalHandle leader; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle resident; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalHolderSnapshot empty; + PcmXLocalCutoff cutoff; + PcmXLocalMembershipSlot *holder_slot; + + init_active_pcm_x(UINT64_C(7213)); + bind_local_master(1, leader_identity.cluster_epoch, UINT64_C(8201)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 1, UINT64_C(8201), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&holder_key, &holder), PCM_X_QUEUE_OK); + holder_slot = &membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_XL_HOLDER_ACQUIRING); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &resident, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_snapshot_revalidate(&holder_key.identity.tag, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_XL_HOLDER_ACTIVE); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_snapshot_revalidate(&holder_key.identity.tag, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_XL_HOLDER_RELEASING); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_snapshot_revalidate(&holder_key.identity.tag, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_snapshot_revalidate(&holder_key.identity.tag, &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, NULL, 0, &empty), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(empty.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_probe_freeze_is_count_first_exact_for_holder_only_tag) +{ + PcmXLocalHolderKey holder_key = make_local_holder_key(773, 0, 4, UINT64_C(84001), 1); + PcmXLocalHolderKey late_key = make_local_holder_key(773, 0, 5, UINT64_C(84002), 2); + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle late; + PcmXLocalHolderHandle copied; + PcmXLocalHandle writer; + PcmXLocalHolderSnapshot count_snapshot; + PcmXLocalHolderSnapshot copied_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalCutoff cutoff; + PcmXTicketRef probe_ref; + PcmXTicketRef other_ref; + PcmXRevokePayload revoke; + PcmXGrantPayload ready; + PcmXGrantPayload replay; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef found; + uint64 master_session = UINT64_C(8301); + + init_active_pcm_x(UINT64_C(7215)); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(773, 2, 7, UINT64_C(84003)); + probe_ref.handle.ticket_id = UINT64_C(91); + probe_ref.handle.queue_generation = UINT64_C(3); + + /* The first count-only call persists the exact freeze even though the + * caller has not allocated its bounded copy array yet. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &count_snapshot), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(count_snapshot.holder_count, 1); + UT_ASSERT(slot_refs_equal(count_snapshot.tag_slot, holder.tag_slot)); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->cutoff_sequence, 0); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 0); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, master_session); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &probe_ref)); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, 0); + + memset(&copied, 0, sizeof(copied)); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &copied, 1, &copied_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&copied_snapshot, &count_snapshot, sizeof(copied_snapshot)) == 0); + UT_ASSERT(memcmp(&copied, &holder, sizeof(copied)) == 0); + + other_ref = probe_ref; + other_ref.handle.ticket_id++; + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&other_ref, 1, master_session, + &copied, 1, &copied_snapshot), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&late_key, &late), PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &holder_key.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, holder.tag_slot)); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &probe_ref)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Re-snapshot after the last local pin leaves, then drive the retained + * holder-only lane through its exact wire lifecycle. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &count_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(count_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &count_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(1); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(201), &revoke.image_id)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); + memset(&ready, 0, sizeof(ready)); + ready.ref = revoke.ref; + ready.image.image_id = revoke.image_id; + ready.image.source_own_generation = UINT64_C(61); + ready.image.page_scn = UINT64_C(62); + ready.image.page_lsn = UINT64_C(63); + ready.image.source_node = 0; + ready.image.page_checksum = UINT32_C(64); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &holder_key.identity.tag, holder_key.identity.cluster_epoch, 1, master_session, + &cutoff), + PCM_X_QUEUE_OK); + memset(&poll, 0, sizeof(poll)); + poll.ref = revoke.ref; + poll.drain_generation = UINT64_C(65); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = holder_key.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = revoke.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + + /* A cancelled writer remains as unlinked terminal evidence until detach. + * A concurrent holder PROBE must wait without publishing a freeze ref or + * fail-closing the runtime. */ + init_active_pcm_x(UINT64_C(7217)); + bind_local_master(1, UINT64_C(9), UINT64_C(8302)); + probe_ref.identity = make_wait_identity(782, 2, 9, UINT64_C(84004)); + probe_ref.handle.ticket_id = UINT64_C(92); + probe_ref.handle.queue_generation = UINT64_C(4); + { + PcmXWaitIdentity writer_identity = make_wait_identity(782, 0, 8, UINT64_C(84005)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&writer_identity, 1, UINT64_C(8302), &writer), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&writer, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, UINT64_C(8302), + NULL, 0, &count_snapshot), + PCM_X_QUEUE_NOT_READY); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[writer.tag_slot.slot_index]; + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&writer), PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +UT_TEST(test_local_queue_invalidate_authority_is_exact_and_read_only) +{ + PcmXLocalHolderKey holder_key = make_local_holder_key(774, 0, 4, UINT64_C(84101), 1); + PcmXLocalHolderHandle holder; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXTicketRef probe_ref; + uint64 master_session = UINT64_C(8302); + + init_active_pcm_x(UINT64_C(7216)); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(774, 2, 7, UINT64_C(84102)); + probe_ref.handle.ticket_id = UINT64_C(92); + probe_ref.handle.queue_generation = UINT64_C(4); + + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &holder, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + /* Freeze alone is not a drop certificate. */ + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder, 1, NULL, + 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id + 1, probe_ref.identity.node_id, 1, + master_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id + 1, 1, + master_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_queue_invalidate_authorize_exact( + &probe_ref.identity.tag, probe_ref.identity.cluster_epoch, + probe_ref.identity.request_id, probe_ref.identity.node_id, 1, + master_session + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_probe_freeze_closes_fifo_round_and_arm_consumes_reservation) +{ + PcmXWaitIdentity leader_identity = make_wait_identity(774, 0, 6, UINT64_C(85001)); + PcmXWaitIdentity late_identity = make_wait_identity(774, 0, 7, UINT64_C(85002)); + PcmXLocalHolderKey holder_key = make_local_holder_key(774, 0, 8, UINT64_C(85003), 3); + PcmXLocalHandle leader; + PcmXLocalHandle late; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle copied; + PcmXLocalHolderSnapshot snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalBlockerSnapshot empty_snapshot; + PcmXTicketRef probe_ref; + PcmXLocalTagSlot *tag_slot; + ClusterLmdVertex blocker; + uint64 master_session = UINT64_C(8401); + + init_active_pcm_x(UINT64_C(7216)); + bind_local_master(1, leader_identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(774, 2, 9, UINT64_C(85004)); + probe_ref.handle.ticket_id = UINT64_C(92); + probe_ref.handle.queue_generation = UINT64_C(4); + + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &copied, 1, &snapshot), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->cutoff_sequence, 1); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(memcmp(&copied, &holder, sizeof(copied)) == 0); + /* The production type-48 collector probes for a replay before it samples + * PGPROC wait state. A generation-zero freeze reservation is therefore + * an exact "not armed yet" result, never corruption. */ + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_lookup_exact(&probe_ref, 1, master_session, + &blocker_snapshot), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Writer admission remains legal but belongs to the next round; it can + * never enter the frozen round sampled by this PROBE. */ + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&late_identity, 1, master_session, &late), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late.local_round, leader.local_round + 1); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + + /* Snapshot publication upgrades the existing exact freeze reservation; + * it must not reject it as a conflicting in-flight set. */ + blocker = make_blocker(holder_key.identity.node_id, holder_key.identity.procno, + holder_key.identity.request_id); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &snapshot, &copied, 1, &blocker, 1, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, 1); + UT_ASSERT(ticket_refs_equal(&blocker_snapshot.ref, &probe_ref)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + + /* A nonempty set may resolve before transfer. The exact ACK tombstone + * remains the generation authority; same-ticket re-PROBE reuses the + * barrier and advances N -> N+1 instead of resetting to one. */ + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &copied, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &snapshot, &copied, 1, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(empty_snapshot.set_generation, 2); + UT_ASSERT_EQ(empty_snapshot.blocker_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_nested_wait_guard_snapshots_content_locks_then_checks_barriers) +{ + PcmXLocalHolderKey holder_key = make_local_holder_key(775, 0, 10, UINT64_C(86001), 1); + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle copied; + PcmXLocalHolderSnapshot snapshot; + PcmXTicketRef probe_ref; + PcmXLocalTagSlot *tag_slot; + LWLock unrelated; + uint64 master_session = UINT64_C(8501); + + init_active_pcm_x(UINT64_C(7217)); + fake_buffer_descriptors[1].bufferdesc.tag = holder_key.identity.tag; + fake_buffer_descriptors[2].bufferdesc.tag = make_tag(776); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(775, 2, 11, UINT64_C(86002)); + probe_ref.handle.ticket_id = UINT64_C(93); + probe_ref.handle.queue_generation = UINT64_C(5); + + /* publish-first ordering: before freeze, a held content lock is legal. + * The later freeze still enumerates that exact registered holder. */ + held_lwlocks[0] = &fake_buffer_descriptors[1].bufferdesc.content_lock; + held_lwlock_modes[0] = LW_EXCLUSIVE; + held_lwlock_count = 1; + UT_ASSERT_EQ(cluster_pcm_x_nested_wait_guard_before_block(), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(lock_acquire_during_iteration_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &copied, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.holder_count, 1); + UT_ASSERT(memcmp(&copied, &holder, sizeof(copied)) == 0); + + /* pcm_lock shares the BufferDesc address range but is not content_lock; + * it must not be reverse-mapped or rejected by the frozen tag. */ + held_lwlocks[0] = &fake_buffer_descriptors[1].bufferdesc.pcm_lock; + held_lwlock_modes[0] = LW_EXCLUSIVE; + held_lwlock_count = 1; + UT_ASSERT_EQ(cluster_pcm_x_nested_wait_guard_before_block(), PCM_X_QUEUE_OK); + + /* freeze-first ordering and multiple held locks: collection is bounded on + * the stack, ignores unrelated locks/tags, and checks PCM only after the + * callback has completed. */ + memset(&unrelated, 0, sizeof(unrelated)); + held_lwlocks[0] = &unrelated; + held_lwlock_modes[0] = LW_SHARED; + held_lwlocks[1] = &fake_buffer_descriptors[2].bufferdesc.content_lock; + held_lwlock_modes[1] = LW_SHARED; + held_lwlocks[2] = &fake_buffer_descriptors[1].bufferdesc.pcm_lock; + held_lwlock_modes[2] = LW_EXCLUSIVE; + held_lwlocks[3] = &fake_buffer_descriptors[1].bufferdesc.content_lock; + held_lwlock_modes[3] = LW_EXCLUSIVE; + held_lwlock_count = 4; + UT_ASSERT_EQ(cluster_pcm_x_nested_wait_guard_before_block(), PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(lock_acquire_during_iteration_count, 0); + + /* A torn tag generation is a closed decision. It may never be treated as + * no resident tag and permit the backend to sleep. */ + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + pg_atomic_write_u32(&tag_slot->slot.generation_change_seq, + pg_atomic_read_u32(&tag_slot->slot.generation_change_seq) | 1U); + UT_ASSERT_EQ(cluster_pcm_x_nested_wait_guard_before_block(), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + pg_atomic_write_u32(&tag_slot->slot.generation_change_seq, + pg_atomic_read_u32(&tag_slot->slot.generation_change_seq) + 1U); + UT_ASSERT_EQ(cluster_pcm_x_nested_wait_guard_before_block(), PCM_X_QUEUE_BARRIER_CLOSED); + held_lwlock_count = 0; +} + +UT_TEST(test_nested_wait_guard_allows_untracked_tag_before_runtime_formation) +{ + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + fake_buffer_descriptors[3].bufferdesc.tag = make_tag(778); + held_lwlocks[0] = &fake_buffer_descriptors[3].bufferdesc.content_lock; + held_lwlock_modes[0] = LW_EXCLUSIVE; + held_lwlock_count = 1; + + UT_ASSERT(cluster_pcm_x_runtime_snapshot().state != PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_nested_wait_guard_before_block(), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(lock_acquire_during_iteration_count, 0); + held_lwlock_count = 0; +} + +UT_TEST(test_local_holder_lock_free_transitions_reject_stale_aba_handle) +{ + PcmXLocalHolderKey key = make_local_holder_key(771, 0, 3, UINT64_C(83001), 1); + PcmXLocalHolderHandle first; + PcmXLocalHolderHandle second; + PcmXLocalMembershipSlot *holder; + int allocator_before; + int domain_before; + + init_active_pcm_x(UINT64_C(7214)); + UT_ASSERT_EQ(register_active_local_holder(&key, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&key, &second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(second.holder_slot.slot_index, first.holder_slot.slot_index); + UT_ASSERT(second.holder_slot.slot_generation > first.holder_slot.slot_generation); + holder = &membership_slots(ClusterPcmXConvertShmem)[second.holder_slot.slot_index]; + allocator_before = allocator_lock_acquire_count; + domain_before = domain_lock_acquire_count; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&first), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&first), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(test_slot_state(&holder->slot), PCM_XL_HOLDER_ACQUIRING); + UT_ASSERT_EQ(allocator_lock_acquire_count, allocator_before); + UT_ASSERT_EQ(domain_lock_acquire_count, domain_before); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&second), PCM_X_QUEUE_OK); +} + +UT_TEST(test_directory_delete_then_allocator_reuse_changes_generation) +{ + BufferTag tag = make_tag(44); + PcmXSlotRef old_ref; + PcmXSlotRef new_ref; + PcmXSlotRef found; + PcmXMasterTagSlot *slot; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + slot = (PcmXMasterTagSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TAG, &old_ref); + slot->tag = tag; + UT_ASSERT_EQ(cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TAG, &tag, old_ref, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TAG, &tag, old_ref), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TAG, old_ref, + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_OK); + slot = (PcmXMasterTagSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TAG, &new_ref); + slot->tag = tag; + UT_ASSERT_EQ(new_ref.slot_index, old_ref.slot_index); + UT_ASSERT_EQ(new_ref.slot_generation, old_ref.slot_generation + 1); + UT_ASSERT_EQ(cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TAG, &tag, new_ref, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TAG, &tag, old_ref), + PCM_X_DIRECTORY_STALE_REF); +} + +UT_TEST(test_allocator_and_directory_operations_take_only_allocator_lock) +{ + PcmXPrehandleKey key = { UINT64_C(91), UINT64_C(7) }; + PcmXSlotRef ref; + PcmXSlotRef found; + PcmXMasterTicketSlot *ticket; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + ticket = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &ref); + ticket->prehandle = key; + UT_ASSERT_EQ( + cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, ref, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, ref), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TICKET, ref, + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_OK); + UT_ASSERT(allocator_lock_acquire_count > 0); + UT_ASSERT(allocator_lock_shared_count > 0); + UT_ASSERT(allocator_lock_exclusive_count > 0); + UT_ASSERT_EQ(domain_lock_acquire_count, 0); + UT_ASSERT_EQ(max_held_lwlock_count, 1); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(lwlock_release_count, allocator_lock_acquire_count); +} + +UT_TEST(test_allocator_and_directory_reject_domain_lock_nesting) +{ + PcmXShmemHeader *header; + PcmXPrehandleKey key = { UINT64_C(91), UINT64_C(17) }; + PcmXSlotRef candidate; + PcmXSlotRef output; + PcmXSlotHeader *output_slot; + PcmXMasterTicketSlot *ticket; + int allocator_acquires_before; + uint32 partition = 3; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + ticket = (PcmXMasterTicketSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TICKET, &candidate); + ticket->prehandle = key; + allocator_acquires_before = allocator_lock_acquire_count; + + LWLockAcquire(&header->master_locks[partition].lock, LW_EXCLUSIVE); + output.slot_index = 0; + output.slot_generation = 99; + output_slot = (PcmXSlotHeader *)(uintptr_t)1; + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_MASTER_TAG, &output, &output_slot), + PCM_X_ALLOC_INVALID); + UT_ASSERT_EQ(output.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(output.slot_generation, 0); + UT_ASSERT_NULL(output_slot); + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TICKET, candidate, + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, &output), + PCM_X_DIRECTORY_INVALID); + UT_ASSERT_EQ(output.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(output.slot_generation, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_insert(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, candidate, &output), + PCM_X_DIRECTORY_INVALID); + UT_ASSERT_EQ(output.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(output.slot_generation, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &key, candidate), + PCM_X_DIRECTORY_INVALID); + UT_ASSERT_EQ(allocator_lock_acquire_count, allocator_acquires_before); + UT_ASSERT_EQ(held_lwlock_count, 1); + LWLockRelease(&header->master_locks[partition].lock); + + LWLockAcquire(&header->local_locks[partition].lock, LW_EXCLUSIVE); + output.slot_index = 0; + output.slot_generation = 99; + output_slot = (PcmXSlotHeader *)(uintptr_t)1; + UT_ASSERT_EQ(cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_LOCAL_TAG, &output, &output_slot), + PCM_X_ALLOC_INVALID); + UT_ASSERT_EQ(output.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(output.slot_generation, 0); + UT_ASSERT_NULL(output_slot); + UT_ASSERT_EQ(allocator_lock_acquire_count, allocator_acquires_before); + UT_ASSERT_EQ(held_lwlock_count, 1); + LWLockRelease(&header->local_locks[partition].lock); + UT_ASSERT_EQ(held_lwlock_count, 0); +} + +UT_TEST(test_slot_ref_revalidate_requires_exact_domain_tag_state_and_generation) +{ + PcmXShmemHeader *header; + BufferTag tag = make_tag(88); + BufferTag candidate_tag; + BufferTag wrong_tag; + BufferTag wrong_partition_tag; + PcmXMasterTagSlot *slot; + PcmXSlotRef old_ref; + PcmXSlotRef new_ref; + PcmXSlotRef stale_ref; + PcmXSlotHeader *resolved; + BlockNumber block; + uint32 candidate_hash; + uint32 candidate_partition; + uint32 tag_hash; + uint32 wrong_tag_hash = 0; + uint32 wrong_partition_hash = 0; + uint32 partition; + uint32 wrong_partition = 0; + bool found_same_partition = false; + bool found_wrong_partition = false; + + wrong_tag = tag; + wrong_partition_tag = tag; + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + slot = (PcmXMasterTagSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TAG, &old_ref); + slot->tag = tag; + test_set_slot_state(&slot->slot, PCM_XT_QUEUED); + tag_hash = cluster_pcm_x_tag_hash(&tag); + UT_ASSERT_EQ(tag_hash, cluster_pcm_x_tag_hash(&tag)); + partition = cluster_pcm_x_lock_partition(tag_hash); + for (block = 89; block < 89 + PCM_X_LOCK_PARTITIONS * 32; block++) { + candidate_tag = make_tag(block); + candidate_hash = cluster_pcm_x_tag_hash(&candidate_tag); + candidate_partition = cluster_pcm_x_lock_partition(candidate_hash); + if (!found_same_partition && candidate_hash != tag_hash + && candidate_partition == partition) { + wrong_tag = candidate_tag; + wrong_tag_hash = candidate_hash; + found_same_partition = true; + } + if (!found_wrong_partition && candidate_partition != partition) { + wrong_partition_tag = candidate_tag; + wrong_partition_hash = candidate_hash; + wrong_partition = candidate_partition; + found_wrong_partition = true; + } + if (found_same_partition && found_wrong_partition) + break; + } + UT_ASSERT(found_same_partition); + UT_ASSERT(found_wrong_partition); + UT_ASSERT(!BufferTagsEqual(&tag, &wrong_tag)); + UT_ASSERT(!BufferTagsEqual(&tag, &wrong_partition_tag)); + UT_ASSERT_NE(tag_hash, wrong_tag_hash); + UT_ASSERT_NE(tag_hash, wrong_partition_hash); + UT_ASSERT_EQ(wrong_tag_hash, cluster_pcm_x_tag_hash(&wrong_tag)); + UT_ASSERT_EQ(wrong_partition_hash, cluster_pcm_x_tag_hash(&wrong_partition_tag)); + UT_ASSERT_EQ(cluster_pcm_x_lock_partition(wrong_tag_hash), partition); + UT_ASSERT_NE(wrong_partition, partition); + + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, tag_hash)); + LWLockAcquire(&header->master_locks[wrong_partition].lock, LW_SHARED); + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, tag_hash)); + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, wrong_partition_hash)); + UT_ASSERT_EQ(test_slot_generation(&slot->slot), old_ref.slot_generation); + UT_ASSERT_EQ(test_slot_state(&slot->slot), PCM_XT_QUEUED); + UT_ASSERT(BufferTagsEqual(&slot->tag, &tag)); + LWLockRelease(&header->master_locks[wrong_partition].lock); + + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + NULL, tag_hash)); + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &wrong_tag, wrong_tag_hash)); + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, + PCM_XT_ADMITTING, &tag, tag_hash)); + stale_ref = old_ref; + stale_ref.slot_generation--; + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, stale_ref, + PCM_XT_QUEUED, &tag, tag_hash)); + resolved = cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, tag_hash); + UT_ASSERT_EQ(resolved, &slot->slot); + slot->tag = wrong_tag; + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, tag_hash)); + slot->tag = tag; + resolved = cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, tag_hash); + UT_ASSERT_EQ(resolved, &slot->slot); + test_set_slot_state(&slot->slot, PCM_X_SLOT_DETACHING); + LWLockRelease(&header->master_locks[partition].lock); + + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_MASTER_TAG, old_ref, + PCM_X_SLOT_DETACHING), + PCM_X_ALLOC_OK); + slot = (PcmXMasterTagSlot *)reserve_slot(PCM_X_ALLOC_MASTER_TAG, &new_ref); + UT_ASSERT_EQ(new_ref.slot_index, old_ref.slot_index); + UT_ASSERT(new_ref.slot_generation != old_ref.slot_generation); + slot->tag = tag; + test_set_slot_state(&slot->slot, PCM_XT_QUEUED); + LWLockAcquire(&header->master_locks[partition].lock, LW_SHARED); + UT_ASSERT_NULL(cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, old_ref, PCM_XT_QUEUED, + &tag, tag_hash)); + resolved = cluster_pcm_x_slot_ref_revalidate(PCM_X_ALLOC_MASTER_TAG, new_ref, PCM_XT_QUEUED, + &tag, tag_hash); + UT_ASSERT_EQ(resolved, &slot->slot); + LWLockRelease(&header->master_locks[partition].lock); +} + +UT_TEST(test_internal_directory_stale_ref_is_corrupt_and_fails_closed) +{ + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXMasterTicketSlot *ticket; + + init_active_pcm_x(UINT64_C(77)); + request + = make_enqueue(make_wait_identity(740, 0, 9, UINT64_C(50001)), UINT64_C(5101), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + test_set_slot_generation(&ticket->slot, admission.ticket_slot.slot_generation + 1); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(5201)), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_resident_exact_wire_mismatch_is_stale_without_side_effect) +{ + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTicketRef active; + PcmXTicketRef stale; + PcmXTicketRef transfer; + + init_active_pcm_x(UINT64_C(77)); + request + = make_enqueue(make_wait_identity(741, 0, 9, UINT64_C(50002)), UINT64_C(5102), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + before = *ticket; + + stale = admission.ref; + stale.identity.request_id++; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(5202)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(5202)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(5202), &transfer), + PCM_X_QUEUE_OK); + before = *ticket; + stale = transfer; + stale.grant_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&stale), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_same_lock_staged_insert_exists_is_corrupt_and_fails_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(742, 0, 9, UINT64_C(50003)), UINT64_C(5103), UINT64_C(1)); + bind_enqueue_peer(&request); + staged_prehandle_insert_exists_key = request.prehandle; + staged_prehandle_insert_exists_armed = true; + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_CORRUPT); + UT_ASSERT(!staged_prehandle_insert_exists_armed); + UT_ASSERT_EQ(header->next_ticket_id, 1); + UT_ASSERT_EQ(header->peer_frontiers[request.identity.node_id].next_expected_prehandle_sequence, + 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_composite_admission_publishes_all_indexes_and_coalesces_node) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission replay; + PcmXMasterAdmission coalesced; + PcmXMasterAdmission gated; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXEnqueuePayload follower_request; + PcmXEnqueuePayload gated_request; + PcmXSlotRef found; + uint64 next_ticket_before; + uint64 next_admission_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(700, 2, 12, UINT64_C(1001)), UINT64_C(501), UINT64_C(1)); + bind_enqueue_peer(&request); + memset(&first, 0, sizeof(first)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &first), PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(first.ref.handle.ticket_id, 1); + UT_ASSERT_EQ(first.ref.handle.queue_generation, first.ticket_slot.slot_generation); + UT_ASSERT_EQ(first.ref.grant_generation, 0); + UT_ASSERT_EQ(first.master_session_incarnation, 77); + UT_ASSERT_EQ(first.admission_sequence, 1); + UT_ASSERT((first.flags & PCM_X_ADMIT_F_QUEUE_HEAD) != 0); + UT_ASSERT(memcmp(&first.ref.identity, &request.identity, sizeof(request.identity)) == 0); + UT_ASSERT(memcmp(&first.prehandle, &request.prehandle, sizeof(request.prehandle)) == 0); + + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[first.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_LIVE); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ADMITTING); + UT_ASSERT_EQ(tag_slot->head_index, first.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, first.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), UINT32_C(1) << 2); + UT_ASSERT_EQ(ticket->tag_slot_index, first.tag_slot.slot_index); + UT_ASSERT_EQ(ticket->tag_slot_generation, first.tag_slot.slot_generation); + UT_ASSERT_EQ(ticket->next_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ticket->prev_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ticket->involved_nodes_bitmap, UINT32_C(1) << 2); + UT_ASSERT_EQ(header->peer_frontiers[2].cluster_epoch, request.identity.cluster_epoch); + UT_ASSERT_EQ(header->peer_frontiers[2].sender_session_incarnation, + request.prehandle.sender_session_incarnation); + UT_ASSERT_EQ(header->peer_frontiers[2].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(header->peer_frontiers[2].retired_prehandle_sequence, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TAG, &request.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, first.tag_slot)); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &request.prehandle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, first.ticket_slot)); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &first.ref.handle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, first.ticket_slot)); + + memset(&replay, 0, sizeof(replay)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&replay.ref, &first.ref)); + UT_ASSERT(slot_refs_equal(replay.tag_slot, first.tag_slot)); + UT_ASSERT(slot_refs_equal(replay.ticket_slot, first.ticket_slot)); + UT_ASSERT((replay.flags & PCM_X_ADMIT_F_EXACT_REPLAY) != 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + + follower_request + = make_enqueue(make_wait_identity(700, 2, 13, UINT64_C(1002)), UINT64_C(501), UINT64_C(2)); + memset(&coalesced, 0, sizeof(coalesced)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&follower_request, &coalesced), PCM_X_QUEUE_BUSY); + UT_ASSERT(ticket_refs_equal(&coalesced.ref, &first.ref)); + UT_ASSERT(slot_refs_equal(coalesced.ticket_slot, first.ticket_slot)); + UT_ASSERT((coalesced.flags & PCM_X_ADMIT_F_NODE_COALESCED) != 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(header->peer_frontiers[2].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &follower_request.prehandle, &found), + PCM_X_DIRECTORY_NOT_FOUND); + + gated_request + = make_enqueue(make_wait_identity(700, 3, 14, UINT64_C(1003)), UINT64_C(502), UINT64_C(1)); + bind_enqueue_peer(&gated_request); + next_ticket_before = header->next_ticket_id; + next_admission_before = tag_slot->next_admission_sequence; + pg_atomic_write_u32(&tag_slot->admission_gate, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&gated_request, &gated), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 1); + UT_ASSERT_EQ(header->next_ticket_id, next_ticket_before); + UT_ASSERT_EQ(tag_slot->next_admission_sequence, next_admission_before); + UT_ASSERT_EQ(header->peer_frontiers[3].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + pg_atomic_write_u32(&tag_slot->admission_gate, 0); +} + +UT_TEST(test_master_coalesce_scan_distinguishes_legal_publish_races) +{ + int mode; + + for (mode = 1; mode <= 3; mode++) { + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload second_request; + PcmXRuntimeSnapshot snapshot; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request = make_enqueue(make_wait_identity(722, 0, 2, UINT64_C(26001)), UINT64_C(2801), + UINT64_C(1)); + second_request = make_enqueue(make_wait_identity(722, 0, 3, UINT64_C(26002)), + UINT64_C(2801), UINT64_C(2)); + bind_enqueue_peer(&first_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + coalesce_interlock_tag = &master_tag_slots(header)[first.tag_slot.slot_index]; + coalesce_interlock_ticket = &master_ticket_slots(header)[first.ticket_slot.slot_index]; + coalesce_interlock_mode = mode; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + mode < 3 ? PCM_X_QUEUE_BUSY : PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(coalesce_interlock_mode, 0); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, + mode < 3 ? PCM_X_RUNTIME_ACTIVE : PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_stale_peer_frontier_is_retryable_without_publication) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXLocalHandle local; + PcmXEnqueuePayload request; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(725, 0, 2, UINT64_C(29001)), UINT64_C(3001), UINT64_C(1)); + bind_enqueue_peer(&request); + request.prehandle.sender_session_incarnation++; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(header->next_ticket_id, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(726, 0, 2, UINT64_C(29002)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(3101)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(3102), &local), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_resident_tag_authority_mismatch_is_corrupt) +{ + int kind; + + for (kind = 0; kind < 2; kind++) { + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload second_request; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request = make_enqueue(make_wait_identity(727, 0, 2, UINT64_C(29003)), UINT64_C(3201), + UINT64_C(1)); + second_request = make_enqueue(make_wait_identity(727, 1, 3, UINT64_C(29004)), + UINT64_C(3202), UINT64_C(1)); + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + if (kind == 0) + tag_slot->tag.blockNum++; + else + tag_slot->cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } + + for (kind = 0; kind < 4; kind++) { + PcmXShmemHeader *header; + PcmXLocalHandle first; + PcmXLocalHandle second; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity first_identity; + PcmXWaitIdentity second_identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_identity = make_wait_identity(728, 0, 2, UINT64_C(29005)); + second_identity = make_wait_identity(728, 0, 3, UINT64_C(29006)); + bind_local_master(1, first_identity.cluster_epoch, UINT64_C(3301)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&first_identity, 1, UINT64_C(3301), &first), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[first.tag_slot.slot_index]; + switch (kind) { + case 0: + tag_slot->tag.blockNum++; + break; + case 1: + tag_slot->cluster_epoch++; + break; + case 2: + tag_slot->master_node++; + break; + default: + tag_slot->master_session_incarnation++; + break; + } + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&second_identity, 1, UINT64_C(3301), &second), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_resident_tag_transitional_states_are_retryable) +{ + static const uint32 states[] + = { PCM_X_TAG_RESERVED_NONVISIBLE, PCM_X_TAG_DETACHING, PCM_X_TAG_RECOVERY_BLOCKED }; + Size i; + + for (i = 0; i < lengthof(states); i++) { + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload second_request; + PcmXQueueResult expected + = states[i] == PCM_X_TAG_RECOVERY_BLOCKED ? PCM_X_QUEUE_NOT_READY : PCM_X_QUEUE_BUSY; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request = make_enqueue(make_wait_identity(729, 0, 2, UINT64_C(29007)), UINT64_C(3401), + UINT64_C(1)); + second_request = make_enqueue(make_wait_identity(729, 1, 3, UINT64_C(29008)), + UINT64_C(3402), UINT64_C(1)); + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + test_set_slot_state(&tag_slot->slot, states[i]); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), expected); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } + + for (i = 0; i < lengthof(states); i++) { + PcmXShmemHeader *header; + PcmXLocalHandle first; + PcmXLocalHandle second; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity first_identity; + PcmXWaitIdentity second_identity; + PcmXQueueResult expected + = states[i] == PCM_X_TAG_RECOVERY_BLOCKED ? PCM_X_QUEUE_NOT_READY : PCM_X_QUEUE_BUSY; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_identity = make_wait_identity(730, 0, 2, UINT64_C(29009)); + second_identity = make_wait_identity(730, 0, 3, UINT64_C(29010)); + bind_local_master(1, first_identity.cluster_epoch, UINT64_C(3501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&first_identity, 1, UINT64_C(3501), &first), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[first.tag_slot.slot_index]; + test_set_slot_state(&tag_slot->slot, states[i]); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&second_identity, 1, UINT64_C(3501), &second), + expected); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } +} + +UT_TEST(test_peer_bind_ack_is_exact_and_mismatch_fails_closed) +{ + PcmXShmemHeader *header; + PcmXRuntimeSnapshot snapshot; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(-1, 9, 601), PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 9, 0), PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 0, 601), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->peer_frontiers[0].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[0].sender_session_incarnation, 601); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(header->peer_frontiers[0].retired_prehandle_sequence, 0); + UT_ASSERT_EQ(header->outbound_targets[0].flags, + PCM_X_OUTBOUND_TARGET_INITIALIZED | PCM_X_OUTBOUND_TARGET_BOUND); + UT_ASSERT_EQ(header->outbound_targets[0].cluster_epoch, 0); + UT_ASSERT_EQ(header->outbound_targets[0].target_session_incarnation, 601); + UT_ASSERT_EQ(header->outbound_targets[0].next_prehandle_sequence, 1); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->outbound_targets[0].mint_gate), 0); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 0, 601), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 1, 601), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(header->peer_frontiers[0].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[0].sender_session_incarnation, 601); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_activation_publishes_bound_frontiers_and_counters_before_active) +{ + PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXShmemHeader *header; + PcmXRuntimeSnapshot snapshot; + int i; + + memset(bindings, 0, sizeof(bindings)); + bindings[1].cluster_epoch = 0; + bindings[1].peer_session_incarnation = UINT64_C(601); + bindings[3].cluster_epoch = 0; + bindings[3].peer_session_incarnation = UINT64_C(603); + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_runtime_activate_bound(UINT64_C(77), bindings)); + header = ClusterPcmXConvertShmem; + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 77); + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + PcmXOutboundTargetFrontier *outbound = &header->outbound_targets[i]; + + UT_ASSERT_EQ(pg_atomic_read_u32(&outbound->mint_gate), 0); + UT_ASSERT_EQ(outbound->next_prehandle_sequence, 1); + UT_ASSERT((outbound->flags & PCM_X_OUTBOUND_TARGET_INITIALIZED) != 0); + UT_ASSERT_EQ(header->peer_frontiers[i].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(header->peer_frontiers[i].retired_prehandle_sequence, 0); + if (i == 1 || i == 3) { + UT_ASSERT((outbound->flags & PCM_X_OUTBOUND_TARGET_BOUND) != 0); + UT_ASSERT_EQ(outbound->cluster_epoch, 0); + UT_ASSERT_EQ(outbound->target_session_incarnation, + i == 1 ? UINT64_C(601) : UINT64_C(603)); + UT_ASSERT_EQ(header->peer_frontiers[i].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[i].sender_session_incarnation, + i == 1 ? UINT64_C(601) : UINT64_C(603)); + } else { + UT_ASSERT_EQ(outbound->flags, PCM_X_OUTBOUND_TARGET_INITIALIZED); + UT_ASSERT_EQ(outbound->cluster_epoch, 0); + UT_ASSERT_EQ(outbound->target_session_incarnation, 0); + UT_ASSERT_EQ(header->peer_frontiers[i].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[i].sender_session_incarnation, 0); + } + } + UT_ASSERT_EQ(cluster_pcm_x_runtime_peer_binding_revalidate_exact(1, 0, UINT64_C(601)), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_activation_rejects_partial_binding_before_claiming_gate) +{ + PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXRuntimeSnapshot snapshot; + + memset(bindings, 0, sizeof(bindings)); + bindings[2].cluster_epoch = UINT64_C(19); + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(!cluster_pcm_x_runtime_activate_bound(UINT64_C(77), bindings)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(snapshot.gate_generation, 0); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->outbound_targets[2].flags, 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->outbound_targets[2].next_prehandle_sequence, 0); +} + +UT_TEST(test_runtime_peer_binding_exact_revalidation_is_read_only) +{ + PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXShmemHeader *header; + PcmXPeerFrontier frontier_before; + PcmXOutboundTargetFrontier outbound_before; + + memset(bindings, 0, sizeof(bindings)); + bindings[2].cluster_epoch = 0; + bindings[2].peer_session_incarnation = UINT64_C(602); + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_runtime_activate_bound(UINT64_C(77), bindings)); + header = ClusterPcmXConvertShmem; + frontier_before = header->peer_frontiers[2]; + outbound_before = header->outbound_targets[2]; + + UT_ASSERT_EQ(cluster_pcm_x_runtime_peer_binding_revalidate_exact(2, 0, UINT64_C(602)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(memcmp(&header->peer_frontiers[2], &frontier_before, sizeof(frontier_before)), 0); + UT_ASSERT_EQ(memcmp(&header->outbound_targets[2], &outbound_before, sizeof(outbound_before)), + 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(allocator_lock_acquire_count, 1); + UT_ASSERT_EQ(allocator_lock_shared_count, 1); + UT_ASSERT_EQ(allocator_lock_exclusive_count, 0); + UT_ASSERT_EQ(lwlock_release_count, 1); +} + +UT_TEST(test_runtime_peer_binding_epoch_drift_fails_closed) +{ + PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]; + PcmXPeerFrontier frontier_before; + PcmXOutboundTargetFrontier outbound_before; + + memset(bindings, 0, sizeof(bindings)); + bindings[1].cluster_epoch = UINT64_C(19); + bindings[1].peer_session_incarnation = UINT64_C(601); + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_runtime_activate_bound(UINT64_C(77), bindings)); + frontier_before = ClusterPcmXConvertShmem->peer_frontiers[1]; + outbound_before = ClusterPcmXConvertShmem->outbound_targets[1]; + + UT_ASSERT_EQ( + cluster_pcm_x_runtime_peer_binding_revalidate_exact(1, UINT64_C(20), UINT64_C(601)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(memcmp(&ClusterPcmXConvertShmem->peer_frontiers[1], &frontier_before, + sizeof(frontier_before)), + 0); + UT_ASSERT_EQ(memcmp(&ClusterPcmXConvertShmem->outbound_targets[1], &outbound_before, + sizeof(outbound_before)), + 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_runtime_peer_binding_session_drift_fails_closed) +{ + PcmXPeerBinding bindings[PCM_X_PROTOCOL_NODE_LIMIT]; + + memset(bindings, 0, sizeof(bindings)); + bindings[1].cluster_epoch = UINT64_C(19); + bindings[1].peer_session_incarnation = UINT64_C(601); + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_runtime_activate_bound(UINT64_C(77), bindings)); + + UT_ASSERT_EQ( + cluster_pcm_x_runtime_peer_binding_revalidate_exact(1, UINT64_C(19), UINT64_C(602)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_runtime_peer_binding_empty_frontier_is_never_late_bound) +{ + PcmXShmemHeader *header; + PcmXPeerFrontier frontier_before; + PcmXOutboundTargetFrontier outbound_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + frontier_before = header->peer_frontiers[3]; + outbound_before = header->outbound_targets[3]; + + UT_ASSERT_EQ( + cluster_pcm_x_runtime_peer_binding_revalidate_exact(3, UINT64_C(19), UINT64_C(603)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(memcmp(&header->peer_frontiers[3], &frontier_before, sizeof(frontier_before)), 0); + UT_ASSERT_EQ(memcmp(&header->outbound_targets[3], &outbound_before, sizeof(outbound_before)), + 0); + UT_ASSERT_EQ(header->peer_frontiers[3].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[3].sender_session_incarnation, 0); + UT_ASSERT_EQ(header->outbound_targets[3].flags, PCM_X_OUTBOUND_TARGET_INITIALIZED); + UT_ASSERT_EQ(header->outbound_targets[3].cluster_epoch, 0); + UT_ASSERT_EQ(header->outbound_targets[3].target_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_runtime_peer_binding_invalid_node_is_rejected_without_closing_runtime) +{ + init_active_pcm_x(UINT64_C(77)); + UT_ASSERT_EQ( + cluster_pcm_x_runtime_peer_binding_revalidate_exact(-1, UINT64_C(19), UINT64_C(601)), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_runtime_peer_binding_revalidate_exact(PCM_X_PROTOCOL_NODE_LIMIT, + UINT64_C(19), UINT64_C(601)), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(allocator_lock_acquire_count, 0); +} + +UT_TEST(test_composite_admission_rejects_zero_wrapping_and_out_of_range_identity) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXLocalHandle local; + PcmXEnqueuePayload request; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(701, 0, 10, UINT64_C(2001)), UINT64_C(601), UINT64_C(1)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_NOT_READY); + assert_master_queue_baseline(header); + UT_ASSERT_EQ(header->peer_frontiers[0].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 1); + bind_enqueue_peer(&request); + + request.identity.request_id = 0; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_INVALID); + request + = make_enqueue(make_wait_identity(701, 0, 10, UINT64_C(2001)), UINT64_C(601), UINT64_C(1)); + request.identity.wait_seq = 0; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_INVALID); + request = make_enqueue(make_wait_identity(701, PCM_X_PROTOCOL_NODE_LIMIT, 10, UINT64_C(2001)), + UINT64_C(601), UINT64_C(1)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_INVALID); + request = make_enqueue( + make_wait_identity(701, 0, (uint32)header->layout.process_capacity, UINT64_C(2001)), + UINT64_C(601), UINT64_C(1)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_INVALID); + request = make_enqueue(make_wait_identity(701, 0, 10, UINT64_C(2001)), 0, UINT64_C(1)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_INVALID); + request = make_enqueue(make_wait_identity(701, 0, 10, UINT64_C(2001)), UINT64_C(601), 0); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_INVALID); + request + = make_enqueue(make_wait_identity(701, 0, 10, UINT64_C(2001)), UINT64_C(601), UINT64_C(2)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_NOT_READY); + request + = make_enqueue(make_wait_identity(701, 0, 10, UINT64_C(2001)), UINT64_C(601), UINT64_MAX); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + assert_master_queue_baseline(header); + + identity = make_wait_identity(701, 0, 10, UINT64_C(2001)); + identity.request_id = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &local), + PCM_X_QUEUE_INVALID); + identity = make_wait_identity(701, 0, 10, UINT64_C(2001)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &local), + PCM_X_QUEUE_NOT_READY); + assert_local_queue_baseline(header); + bind_local_master(1, identity.cluster_epoch, UINT64_C(77)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &local), + PCM_X_QUEUE_STALE); + assert_local_queue_baseline(header); + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&identity, PCM_X_PROTOCOL_NODE_LIMIT, UINT64_C(77), &local), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, 0, &local), PCM_X_QUEUE_INVALID); + assert_local_queue_baseline(header); +} + +UT_TEST(test_master_composite_capacity_failure_rolls_back_tag_indexes_and_id) +{ + PcmXShmemHeader *header; + PcmXAllocatorState saved_ticket_allocator; + PcmXDirectoryEntry *handle_entries; + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXTicketHandle expected_handle = { UINT64_C(1), UINT64_C(1) }; + uint64 expected_hash; + Size capacity; + Size directory_capacity; + Size i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + saved_ticket_allocator = header->allocator[PCM_X_ALLOC_MASTER_TICKET]; + capacity = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].free_head = PCM_X_INVALID_SLOT_INDEX; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].used = capacity; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].high_water = capacity; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].generation_exhausted = 0; + request + = make_enqueue(make_wait_identity(702, 0, 10, UINT64_C(3001)), UINT64_C(701), UINT64_C(1)); + bind_enqueue_peer(&request); + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(header->next_ticket_id, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TAG), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TICKET_HANDLE), 0); + UT_ASSERT_EQ(header->peer_frontiers[0].cluster_epoch, request.identity.cluster_epoch); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 1); + + header->allocator[PCM_X_ALLOC_MASTER_TICKET] = saved_ticket_allocator; + handle_entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_HANDLE, &directory_capacity); + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_HANDLE, &expected_handle, + &expected_hash)); + for (i = 0; i < directory_capacity; i++) { + handle_entries[i].state = PCM_X_DIRECTORY_OCCUPIED; + handle_entries[i].key_hash = expected_hash ^ UINT64_C(1); + } + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(header->next_ticket_id, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TAG), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE), 0); + UT_ASSERT_EQ(header->peer_frontiers[0].cluster_epoch, request.identity.cluster_epoch); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 1); + memset(handle_entries, 0, directory_capacity * sizeof(*handle_entries)); + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_ticket_and_admission_sequence_exhaustion_leave_no_holes) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission rejected; + PcmXMasterAdmission replay; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload request; + PcmXEnqueuePayload second_request; + PcmXSlotRef found; + uint64 next_ticket_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + header->next_ticket_id = PCM_X_MASTER_TICKET_ID_MAX + 1; + request + = make_enqueue(make_wait_identity(703, 0, 10, UINT64_C(4001)), UINT64_C(801), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &rejected), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(header->next_ticket_id, PCM_X_MASTER_TICKET_ID_MAX + 1); + assert_master_queue_baseline(header); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + tag_slot->next_admission_sequence = UINT64_MAX; + next_ticket_before = header->next_ticket_id; + second_request + = make_enqueue(make_wait_identity(703, 1, 11, UINT64_C(4002)), UINT64_C(802), UINT64_C(1)); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &rejected), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(header->next_ticket_id, next_ticket_before); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &second_request.prehandle, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&first.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&replay.ref, &first.ref)); + arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_DRAIN, + first.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_RETIRE, + first.ref.identity.node_id); + pg_atomic_write_u32(&tag_slot->admission_gate, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&first.ref), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(test_slot_state(&master_ticket_slots(header)[first.ticket_slot.slot_index].slot), + PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, &first.prehandle, &found), + PCM_X_DIRECTORY_OK); + pg_atomic_write_u32(&tag_slot->admission_gate, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&first.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 1); + UT_ASSERT_EQ(header->peer_frontiers[0].retired_prehandle_sequence, 1); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 2); + assert_master_queue_baseline(header); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_RETIRED); + assert_master_queue_baseline(header); +} + +UT_TEST(test_missing_live_prehandle_inside_frontier_is_corrupt_not_phantom) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterAdmission replay; + PcmXEnqueuePayload request; + PcmXRuntimeSnapshot snapshot; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(710, 0, 14, UINT64_C(11001)), UINT64_C(1201), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(cluster_pcm_x_directory_delete_exact(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &request.prehandle, admission.ticket_slot), + PCM_X_DIRECTORY_OK); + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(header->peer_frontiers[0].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(header->peer_frontiers[0].retired_prehandle_sequence, 0); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_confirmed_queue_is_node_fifo_with_one_active) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *tickets; + PcmXTicketRef active; + int nodes[3] = { 2, 0, 3 }; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(704, nodes[i], (uint32)(20 + i), UINT64_C(5001) + i), + UINT64_C(900) + i, UINT64_C(1)); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(admission[i].admission_sequence, (uint64)i + 1); + if (i == 0) { + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, 0), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ( + test_slot_state( + &master_ticket_slots(header)[admission[i].ticket_slot.slot_index].slot), + PCM_XT_ADMITTING); + } + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(100) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + UT_ASSERT_EQ(tag_slot->head_index, admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, admission[2].ticket_slot.slot_index); + UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, + admission[1].ticket_slot.slot_index); + UT_ASSERT_EQ(tickets[admission[1].ticket_slot.slot_index].next_index, + admission[2].ticket_slot.slot_index); + + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, + admission[0].ref.identity.cluster_epoch, + &active), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&active, &admission[0].ref)); + UT_ASSERT_EQ(tag_slot->active_index, admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(test_slot_state(&tickets[tag_slot->active_index].slot), PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, + admission[0].ref.identity.cluster_epoch, + &active), + PCM_X_QUEUE_BUSY); + UT_ASSERT(ticket_refs_equal(&active, &admission[0].ref)); + + for (i = 0; i < 3; i++) { + if (i > 0) { + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact( + &admission[i].ref.identity.tag, + admission[i].ref.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&active, &admission[i].ref)); + } + commit_empty_blocker_graph(&admission[i].ref, UINT64_C(100) + i); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[i].ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&admission[i]); + } + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_wfg_snapshot_head_has_no_blockers_and_confirms_exactly) +{ + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXMasterWfgSnapshot snapshot; + PcmXMasterWfgToken token; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(740, 0, 20, UINT64_C(15001)), UINT64_C(1901), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission.ref, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.blocker_count, 0); + UT_ASSERT(ticket_refs_equal(&snapshot.token.ticket, &admission.ref)); + UT_ASSERT_EQ(snapshot.token.predecessor_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(snapshot.token.active_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + token = snapshot.token; + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_revalidate_exact(&token, UINT64_C(8101)), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_QUEUED); + UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(8101)); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission.ref, &snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(snapshot.blocker_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_revalidate_exact(&token, UINT64_C(8101)), + PCM_X_QUEUE_DUPLICATE); +} + +UT_TEST(test_master_wfg_snapshot_second_ticket_names_true_predecessor) +{ + PcmXMasterAdmission admission[2]; + PcmXMasterWfgSnapshot snapshot; + PcmXEnqueuePayload request; + int i; + + init_active_pcm_x(UINT64_C(77)); + for (i = 0; i < 2; i++) { + request = make_enqueue(make_wait_identity(741, i, (uint32)(21 + i), UINT64_C(15101) + i), + UINT64_C(1911) + i, UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + if (i == 0) { + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission[i].ref, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_revalidate_exact(&snapshot.token, + UINT64_C(8111)), + PCM_X_QUEUE_OK); + } + } + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission[1].ref, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.blocker_count, 1); + UT_ASSERT(vertex_matches_ticket(&snapshot.blockers[0], &admission[0].ref)); + UT_ASSERT_EQ(snapshot.token.predecessor_slot.slot_index, admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(snapshot.token.active_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); +} + +UT_TEST(test_master_wfg_snapshot_deduplicates_active_predecessor_and_keeps_distinct_pair) +{ + PcmXMasterAdmission admission[3]; + PcmXMasterWfgSnapshot snapshot; + PcmXEnqueuePayload request; + PcmXTicketRef active; + int i; + + init_active_pcm_x(UINT64_C(77)); + for (i = 0; i < 3; i++) { + request = make_enqueue(make_wait_identity(742, i, (uint32)(23 + i), UINT64_C(15201) + i), + UINT64_C(1921) + i, UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + if (i == 0) { + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission[i].ref, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_revalidate_exact(&snapshot.token, + UINT64_C(8121)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact( + &admission[i].ref.identity.tag, + admission[i].ref.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + } else if (i == 1) { + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission[i].ref, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.blocker_count, 1); + UT_ASSERT(vertex_matches_ticket(&snapshot.blockers[0], &admission[0].ref)); + UT_ASSERT_EQ(snapshot.token.active_slot.slot_index, + admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(snapshot.token.predecessor_slot.slot_index, + admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_revalidate_exact(&snapshot.token, + UINT64_C(8122)), + PCM_X_QUEUE_OK); + } + } + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission[2].ref, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.blocker_count, 2); + UT_ASSERT(vertex_matches_ticket(&snapshot.blockers[0], &admission[0].ref)); + UT_ASSERT(vertex_matches_ticket(&snapshot.blockers[1], &admission[1].ref)); +} + +UT_TEST(test_master_wfg_structural_mutation_invalidates_snapshot_before_confirm) +{ + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXMasterTicketSlot *ticket; + PcmXMasterWfgSnapshot first_snapshot; + PcmXMasterWfgSnapshot stale_snapshot; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(743, 0, 26, UINT64_C(15301)), UINT64_C(1931), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&first.ref, &first_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_revalidate_exact(&first_snapshot.token, UINT64_C(8131)), + PCM_X_QUEUE_OK); + + request = make_enqueue(make_wait_identity(743, 1, 27, UINT64_C(15302)), UINT64_C(1932), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &second), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&second.ref, &stale_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(stale_snapshot.blocker_count, 1); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&first.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_revalidate_exact(&stale_snapshot.token, UINT64_C(8132)), + PCM_X_QUEUE_STALE); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[second.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ADMITTING); + UT_ASSERT_EQ(ticket->graph_generation, 0); + + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&second.ref, &stale_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(stale_snapshot.blocker_count, 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_revalidate_exact(&stale_snapshot.token, UINT64_C(8132)), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_master_wfg_snapshot_and_confirm_reject_stale_exact_identity) +{ + PcmXMasterAdmission admission; + PcmXMasterWfgSnapshot snapshot; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(744, 0, 28, UINT64_C(15401)), UINT64_C(1941), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&admission.ref, &snapshot), + PCM_X_QUEUE_OK); + + snapshot.token.ticket.identity.request_id++; + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_revalidate_exact(&snapshot.token, UINT64_C(8141)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_wfg_snapshot_exact(&snapshot.token.ticket, &snapshot), + PCM_X_QUEUE_STALE); +} + +UT_TEST(test_master_success_transfer_completes_fifo_and_promotes_every_node) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[4]; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *tickets; + PcmXTicketRef active; + PcmXTicketRef transfer; + uint64 queue_state; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 4; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(711, i, (uint32)(10 + i), UINT64_C(12001) + i), + UINT64_C(1301) + i, UINT64_C(1)); + + request.identity.cluster_epoch = 0; + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(301) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), UINT32_C(0x0f)); + queue_state = tag_slot->queue_state_sequence; + UT_ASSERT_EQ(queue_state, 4); + for (i = 0; i < 4; i++) { + UT_ASSERT_EQ( + cluster_pcm_x_master_promote_head_exact( + &admission[i].ref.identity.tag, admission[i].ref.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&active, &admission[i].ref)); + commit_empty_blocker_graph(&active, UINT64_C(301) + i); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(999), &transfer), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(test_slot_state(&tickets[admission[i].ticket_slot.slot_index].slot), + PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(301) + i, &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT(transfer.grant_generation != 0); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&admission[i].ref, UINT64_C(301) + i, + &active), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&active, &transfer)); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_DUPLICATE); + queue_state++; + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); + UT_ASSERT_EQ(test_slot_state(&tickets[admission[i].ticket_slot.slot_index].slot), + PCM_XT_COMPLETE); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & (UINT32_C(1) << i), 0); + admission[i].ref = transfer; + drain_retire_and_detach_master(&admission[i]); + } + UT_ASSERT_EQ(header->fully_retired_ticket_id, 4); + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_blocker_replace_canonical_snapshot_and_revalidate) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entries[2]; + PcmXMasterBlockerEntry before_entries[2]; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterBlockerSnapshot small_snapshot; + ClusterLmdVertex b1 = make_blocker(1, 11, UINT64_C(8101)); + ClusterLmdVertex b2 = make_blocker(2, 12, UINT64_C(8102)); + ClusterLmdVertex blockers[3] = { b2, b1, b1 }; + Size used_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(770, 0, 7, UINT64_C(8001), UINT64_C(7001), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 1, blockers, lengthof(blockers), UINT32_C(0x11111111)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 2); + UT_ASSERT_EQ(max_held_lwlock_count, 1); + + memset(entries, 0xA5, sizeof(entries)); + memcpy(before_entries, entries, sizeof(entries)); + memset(&small_snapshot, 0xA5, sizeof(small_snapshot)); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, entries, 1, &small_snapshot), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT(memcmp(entries, before_entries, sizeof(entries)) == 0); + UT_ASSERT_EQ(small_snapshot.blocker_count, 0); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&snapshot.ref, &admission.ref)); + UT_ASSERT_EQ(snapshot.set_generation, 1); + UT_ASSERT_EQ(snapshot.blocker_count, 2); + UT_ASSERT_EQ(snapshot.set_crc32c, UINT32_C(0x11111111)); + UT_ASSERT(memcmp(&entries[0].blocker, &b1, sizeof(b1)) == 0); + UT_ASSERT(memcmp(&entries[1].blocker, &b2, sizeof(b2)) == 0); + for (Size i = 0; i < lengthof(entries); i++) { + UT_ASSERT_EQ(entries[i].chunk_no, i); + UT_ASSERT_EQ(entries[i].reserved, 0); + UT_ASSERT_EQ(test_slot_state(&blocker_slots(header)[entries[i].slot_ref.slot_index].slot), + 2); + UT_ASSERT_EQ( + test_slot_generation(&blocker_slots(header)[entries[i].slot_ref.slot_index].slot), + entries[i].slot_ref.slot_generation); + } + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&snapshot, entries, + lengthof(entries)), + PCM_X_QUEUE_OK); + entries[1].slot_ref.slot_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&snapshot, entries, + lengthof(entries)), + PCM_X_QUEUE_STALE); + entries[1].slot_ref.slot_generation--; + + used_before = header->allocator[PCM_X_ALLOC_BLOCKER].used; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, + UINT32_C(0x11111111)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, used_before); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 3, blockers, lengthof(blockers), UINT32_C(0x33333333)), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, used_before); +} + +UT_TEST(test_master_blocker_replace_grow_shrink_empty_reclaims) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot snapshot; + ClusterLmdVertex blockers[3] + = { make_blocker(0, 10, UINT64_C(8201)), make_blocker(1, 11, UINT64_C(8202)), + make_blocker(2, 12, UINT64_C(8203)) }; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(771, 0, 8, UINT64_C(8002), UINT64_C(7002), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, blockers, 2, + UINT32_C(0x101)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 2); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, blockers, 3, + UINT32_C(0x102)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 3); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, blockers, 3, + UINT32_C(0x101)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 3); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 3, &blockers[2], 1, + UINT32_C(0x103)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 4, NULL, 0, UINT32_C(0x104)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.set_generation, 4); + UT_ASSERT_EQ(snapshot.blocker_count, 0); + UT_ASSERT_EQ(snapshot.set_crc32c, UINT32_C(0x104)); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&snapshot, NULL, 0), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_master_blocker_capacity_failure_is_byte_stable) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot ticket_before; + PcmXAllocatorState allocator_before; + PcmXBlockerSlot pool_before[8]; + PcmXSlotRef fillers[8]; + PcmXSlotHeader *slot; + ClusterLmdVertex old_blocker = make_blocker(1, 21, UINT64_C(8301)); + ClusterLmdVertex replacement[2] + = { make_blocker(2, 22, UINT64_C(8302)), make_blocker(3, 23, UINT64_C(8303)) }; + Size filler_count; + Size pool_capacity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(772, 0, 9, UINT64_C(8003), UINT64_C(7003), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &old_blocker, 1, + UINT32_C(0x201)), + PCM_X_QUEUE_OK); + pool_capacity = header->layout.pools[PCM_X_POOL_BLOCKER].capacity; + UT_ASSERT(pool_capacity <= lengthof(fillers)); + for (filler_count = 0; filler_count < pool_capacity - 2; filler_count++) + UT_ASSERT_EQ( + cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_BLOCKER, &fillers[filler_count], &slot), + PCM_X_ALLOC_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, pool_capacity - 1); + ticket_before = master_ticket_slots(header)[admission.ticket_slot.slot_index]; + allocator_before = header->allocator[PCM_X_ALLOC_BLOCKER]; + memcpy(pool_before, blocker_slots(header), sizeof(PcmXBlockerSlot) * pool_capacity); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 2, replacement, lengthof(replacement), UINT32_C(0x202)), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT(memcmp(&master_ticket_slots(header)[admission.ticket_slot.slot_index], &ticket_before, + sizeof(ticket_before)) + == 0); + UT_ASSERT( + memcmp(&header->allocator[PCM_X_ALLOC_BLOCKER], &allocator_before, sizeof(allocator_before)) + == 0); + UT_ASSERT(memcmp(blocker_slots(header), pool_before, sizeof(PcmXBlockerSlot) * pool_capacity) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(max_held_lwlock_count, 1); + + for (Size i = 0; i < filler_count; i++) + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_BLOCKER, fillers[i], + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_OK); +} + +UT_TEST(test_master_blocker_generation_and_duplicate_conflicts_fail_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + ClusterLmdVertex blocker = make_blocker(1, 14, UINT64_C(8401)); + ClusterLmdVertex conflict[2] + = { make_blocker(2, 15, UINT64_C(8402)), make_blocker(2, 15, UINT64_C(8402)) }; + ClusterLmdVertex too_many[4] + = { make_blocker(0, 18, UINT64_C(8410)), make_blocker(1, 19, UINT64_C(8411)), + make_blocker(2, 20, UINT64_C(8412)), make_blocker(3, 21, UINT64_C(8413)) }; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(773, 0, 10, UINT64_C(8004), UINT64_C(7004), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x301)), + PCM_X_QUEUE_OK); + before = master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x301)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&master_ticket_slots(header)[admission.ticket_slot.slot_index], &before, + sizeof(before)) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x302)), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(&master_ticket_slots(header)[admission.ticket_slot.slot_index], &before, + sizeof(before)) + == 0); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + admit_active_probe(774, 0, 11, UINT64_C(8005), UINT64_C(7005), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 1, too_many, lengthof(too_many), UINT32_C(0x300)), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, &blocker, 1, + UINT32_C(0x303)), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, UINT64_MAX, + &blocker, 1, UINT32_C(0x304)), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + admit_active_probe(775, 0, 12, UINT64_C(8006), UINT64_C(7006), 1, &admission); + conflict[1].wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 1, conflict, lengthof(conflict), UINT32_C(0x305)), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); +} + +UT_TEST(test_master_blocker_snapshot_detects_slot_aba_and_corrupt_link) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry old_entry; + PcmXMasterBlockerEntry new_entry; + PcmXMasterBlockerSnapshot old_snapshot; + PcmXMasterBlockerSnapshot new_snapshot; + PcmXBlockerSlot *slot; + ClusterLmdVertex old_blocker = make_blocker(1, 16, UINT64_C(8501)); + ClusterLmdVertex new_blocker = make_blocker(2, 17, UINT64_C(8502)); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(776, 0, 13, UINT64_C(8007), UINT64_C(7007), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &old_blocker, 1, + UINT32_C(0x401)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x402)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 3, &new_blocker, 1, + UINT32_C(0x403)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &new_entry, 1, &new_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(new_entry.slot_ref.slot_index, old_entry.slot_ref.slot_index); + UT_ASSERT(new_entry.slot_ref.slot_generation > old_entry.slot_ref.slot_generation); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&new_snapshot, &new_entry, 1), + PCM_X_QUEUE_OK); + + slot = &blocker_slots(header)[new_entry.slot_ref.slot_index]; + slot->next_index = new_entry.slot_ref.slot_index; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &new_entry, 1, &new_snapshot), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_blocker_snapshot_validates_every_owner_field) +{ + for (int variant = 0; variant < 7; variant++) { + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterTicketSlot *ticket; + PcmXBlockerSlot *slot; + ClusterLmdVertex blocker + = make_blocker(1, (uint32)(5 + variant), UINT64_C(8601) + (uint64)variant); + + init_active_pcm_x(UINT64_C(80) + (uint64)variant); + header = ClusterPcmXConvertShmem; + admit_active_probe((BlockNumber)(780 + variant), 0, (uint32)(14 + variant), + UINT64_C(8100) + (uint64)variant, UINT64_C(7100) + (uint64)variant, 1, + &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 1, &blocker, 1, UINT32_C(0x501) + (uint32)variant), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + slot = &blocker_slots(header)[entry.slot_ref.slot_index]; + switch (variant) { + case 0: + slot->handle.ticket_id++; + break; + case 1: + slot->handle.queue_generation++; + break; + case 2: + slot->owner_slot_generation++; + break; + case 3: + slot->owner_slot_index++; + break; + case 4: + slot->set_generation++; + break; + case 5: + slot->chunk_no++; + break; + default: + slot->direction++; + break; + } + UT_ASSERT_EQ(ticket->blocker_head_index, entry.slot_ref.slot_index); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_master_blocker_sets_are_isolated_across_tickets) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXMasterBlockerEntry first_entry; + PcmXMasterBlockerEntry second_entry; + PcmXMasterBlockerSnapshot first_snapshot; + PcmXMasterBlockerSnapshot second_snapshot; + ClusterLmdVertex first_blocker = make_blocker(2, 9, UINT64_C(8701)); + ClusterLmdVertex second_blocker = make_blocker(3, 10, UINT64_C(8702)); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(790, 0, 7, UINT64_C(8201), UINT64_C(7201), 1, &first); + admit_active_probe(791, 1, 8, UINT64_C(8202), UINT64_C(7202), 1, &second); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&first.ref, 1, &first_blocker, 1, + UINT32_C(0x601)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&second.ref, 1, &second_blocker, 1, + UINT32_C(0x602)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&first.ref, &first_entry, 1, &first_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&second.ref, &second_entry, 1, + &second_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT(first_entry.slot_ref.slot_index != second_entry.slot_ref.slot_index); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 2); + + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_set_replace_exact(&first.ref, 2, NULL, 0, UINT32_C(0x603)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&first_snapshot, &first_entry, 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&second_snapshot, &second_entry, 1), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_master_blocker_wire_stage_commits_exact_canonical_set) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entries[2]; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload begin; + PcmXBlockerSetHeaderPayload next; + PcmXBlockerChunkPayload edge[2]; + ClusterLmdVertex blockers[2]; + const uint64 source_session = UINT64_C(82501); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(791, 0, 11, UINT64_C(8251), UINT64_C(7251), 1, &admission); + blockers[0] = make_blocker(2, 3, UINT64_C(8751)); + blockers[1] = make_blocker(2, 4, UINT64_C(8752)); + begin = make_blocker_header(&admission.ref, 1, blockers, lengthof(blockers)); + edge[0] = make_blocker_chunk(&admission.ref, 1, 0, blockers[0]); + edge[1] = make_blocker_chunk(&admission.ref, 1, 1, blockers[1]); + arm_blocker_probe(&admission.ref, 2, source_session); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 2); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.set_generation, 1); + UT_ASSERT_EQ(snapshot.blocker_count, 2); + UT_ASSERT_EQ(snapshot.set_crc32c, begin.set_crc32c); + UT_ASSERT(memcmp(&entries[0].blocker, &blockers[0], sizeof(blockers[0])) == 0); + UT_ASSERT(memcmp(&entries[1].blocker, &blockers[1], sizeof(blockers[1])) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 2); + next = make_blocker_header(&admission.ref, 2, blockers, lengthof(blockers)); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&next, 2, source_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, entries, + lengthof(entries), UINT64_C(9251)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&next, 2, source_session), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_master_blocker_crc_ignores_vertex_abi_padding) +{ + ClusterLmdVertex zero_padding; + ClusterLmdVertex poisoned_padding; + + memset(&zero_padding, 0, sizeof(zero_padding)); + memset(&poisoned_padding, 0xA5, sizeof(poisoned_padding)); + zero_padding = make_blocker(2, 3, UINT64_C(8750)); + poisoned_padding.node_id = zero_padding.node_id; + poisoned_padding.procno = zero_padding.procno; + poisoned_padding.cluster_epoch = zero_padding.cluster_epoch; + poisoned_padding.request_id = zero_padding.request_id; + poisoned_padding.xid = zero_padding.xid; + poisoned_padding.local_start_ts_ms = zero_padding.local_start_ts_ms; + poisoned_padding.wait_seq = zero_padding.wait_seq; + + UT_ASSERT_EQ(blocker_set_crc32c(&zero_padding, 1), blocker_set_crc32c(&poisoned_padding, 1)); +} + +UT_TEST(test_master_blocker_wire_stage_crc_rejects_content_mismatch) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload begin; + PcmXBlockerChunkPayload edge; + PcmXBlockerSlot *slot; + ClusterLmdVertex blocker = make_blocker(2, 3, UINT64_C(8759)); + const uint64 source_session = UINT64_C(82509); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(799, 0, 19, UINT64_C(8259), UINT64_C(7259), 1, &admission); + begin = make_blocker_header(&admission.ref, 1, &blocker, 1); + edge = make_blocker_chunk(&admission.ref, 1, 0, blocker); + arm_blocker_probe(&admission.ref, 2, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge, 2, source_session), + PCM_X_QUEUE_OK); + slot = &blocker_slots(header)[master_ticket_slots(header)[admission.ticket_slot.slot_index] + .blocker_stage_head_index]; + slot->blocker.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, &entry, + 1, &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + master_ticket_slots(header)[admission.ticket_slot.slot_index].blocker_stage_set_generation, + 1); +} + +UT_TEST(test_master_blocker_stage_abort_reclaims_mixed_chain_and_fences_aba) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterProbeToken probe; + PcmXPhasePayload probe_payload; + PcmXBlockerSetHeaderPayload begin; + PcmXBlockerChunkPayload edge; + ClusterLmdVertex blockers[2]; + PcmXMasterTicketSlot *ticket; + const uint64 source_session = UINT64_C(82519); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(798, 0, 18, UINT64_C(8258), UINT64_C(7258), 1, &admission); + bind_local_master(2, admission.ref.identity.cluster_epoch, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 2, source_session, + &probe_payload, &probe), + PCM_X_QUEUE_OK); + blockers[0] = make_blocker(2, 3, UINT64_C(8758)); + blockers[1] = make_blocker(2, 4, UINT64_C(8759)); + begin = make_blocker_header(&admission.ref, 1, blockers, 2); + edge = make_blocker_chunk(&admission.ref, 1, 0, blockers[0]); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 2); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(ticket->blocker_stage_next_chunk, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_abort_exact( + &admission.ref, 1, 2, source_session, probe.state_sequence), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ(ticket->blocker_stage_set_generation, 0); + UT_ASSERT_EQ(ticket->blocker_stage_head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); + UT_ASSERT(ticket->reliable.state_sequence != probe.state_sequence); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_abort_exact( + &admission.ref, 1, 2, source_session, probe.state_sequence), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_master_blocker_probe_arm_is_zero_generation_and_idempotent) +{ + PcmXMasterAdmission admission; + PcmXMasterProbeToken first; + PcmXMasterProbeToken replay; + PcmXPhasePayload first_payload; + PcmXPhasePayload replay_payload; + PcmXMasterTicketSlot *ticket; + const uint64 source_session = UINT64_C(82520); + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(799, 0, 19, UINT64_C(8259), UINT64_C(7259), 1, &admission); + bind_local_master(2, admission.ref.identity.cluster_epoch, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 2, source_session, + &first_payload, &first), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(first_payload.reason, 0); + UT_ASSERT_EQ(first_payload.phase, 0); + UT_ASSERT_EQ(first_payload.flags, 0); + UT_ASSERT(ticket_refs_equal(&first_payload.ref, &admission.ref)); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 2, source_session, + &replay_payload, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&first_payload, &replay_payload, sizeof(first_payload)) == 0); + UT_ASSERT_EQ(first.state_sequence, replay.state_sequence); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); + UT_ASSERT_EQ(ticket->reliable.phase, PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); +} + +UT_TEST(test_master_drive_work_cursor_is_bounded_and_exact) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXTicketRef active; + BufferTag tag; + Size cursor; + uint64 cluster_epoch; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(805, 0, 25, UINT64_C(8302)), UINT64_C(7302), 1); + request.identity.cluster_epoch = 0; + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(8302)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission.ref.identity.tag, 0, &active), + PCM_X_QUEUE_OK); + cursor = admission.tag_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_master_drive_work_next(&cursor, 1, &tag, &cluster_epoch), + PCM_X_QUEUE_OK); + UT_ASSERT(BufferTagsEqual(&tag, &admission.ref.identity.tag)); + UT_ASSERT_EQ(cluster_epoch, admission.ref.identity.cluster_epoch); + UT_ASSERT_EQ(cursor, (admission.tag_slot.slot_index + 1) + % header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity); + + UT_ASSERT_EQ(cluster_pcm_x_master_drive_work_next(&cursor, 1, &tag, &cluster_epoch), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_epoch, 0); + UT_ASSERT(memcmp(&tag, &(BufferTag){ 0 }, sizeof(tag)) == 0); +} + + +UT_TEST(test_master_driver_waits_for_terminal_active_ticket_without_fail_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot snapshot; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; + BufferTag tag; + Size cursor; + uint64 cluster_epoch; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(807, 0, 27, UINT64_C(8304), UINT64_C(7304), 1, &admission); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + test_set_slot_state(&ticket->slot, PCM_XT_COMPLETE); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &active), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &snapshot), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + cursor = admission.tag_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_master_drive_work_next(&cursor, 1, &tag, &cluster_epoch), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_master_drive_snapshot_and_holder_bitmaps_are_exact) +{ + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot initial; + PcmXMasterDriveSnapshot seeded; + PcmXMasterDriveSnapshot acknowledged; + PcmXMasterDriveSnapshot observed; + uint32 holders = (UINT32_C(1) << 1) | (UINT32_C(1) << 2) | (UINT32_C(1) << 3); + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(804, 3, 24, UINT64_C(8301), UINT64_C(7301), 1, &admission); + + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &initial), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&initial.ref, &admission.ref)); + UT_ASSERT_EQ(initial.ticket_state, PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(initial.pending_s_holders_bitmap, 0); + UT_ASSERT_EQ(initial.acked_s_holders_bitmap, 0); + UT_ASSERT_EQ(initial.involved_nodes_bitmap, UINT32_C(1) << 3); + + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact(&initial, holders, 0, &seeded), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(seeded.pending_s_holders_bitmap, holders); + UT_ASSERT_EQ(seeded.acked_s_holders_bitmap, 0); + UT_ASSERT_EQ(seeded.involved_nodes_bitmap, UINT32_C(1) << 3); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact(&initial, holders, 0, &observed), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(observed.pending_s_holders_bitmap, holders); + + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact(&seeded, holders, UINT32_C(1) << 2, + &acknowledged), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(acknowledged.acked_s_holders_bitmap, UINT32_C(1) << 2); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_bitmap_replace_exact(&acknowledged, holders, + UINT32_C(1) << 4, &observed), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &observed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(observed.acked_s_holders_bitmap, UINT32_C(1) << 2); +} + +UT_TEST(test_master_drive_bitmap_replace_rejects_reliable_leg_drift) +{ + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot before; + PcmXMasterDriveSnapshot after; + PcmXMasterProbeToken probe; + PcmXPhasePayload payload; + const uint64 source_session = UINT64_C(8302); + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(805, 0, 25, UINT64_C(8302), UINT64_C(7302), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &before), + PCM_X_QUEUE_OK); + bind_local_master(2, admission.ref.identity.cluster_epoch, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_probe_arm_exact(&admission.ref, 2, source_session, + &payload, &probe), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_drive_bitmap_replace_exact(&before, UINT32_C(1) << 2, 0, &after), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &after), + PCM_X_QUEUE_OK); + UT_ASSERT(after.state_sequence != before.state_sequence); + UT_ASSERT_EQ(after.pending_opcode, PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); + UT_ASSERT_EQ(after.involved_nodes_bitmap, + (UINT32_C(1) << admission.ref.identity.node_id) | (UINT32_C(1) << 2)); +} + +UT_TEST(test_master_drive_snapshot_fails_closed_on_bitmap_corruption) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterDriveSnapshot snapshot; + PcmXMasterTicketSlot *ticket; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(806, 0, 26, UINT64_C(8303), UINT64_C(7303), 1, &admission); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->pending_s_holders_bitmap = UINT32_C(1) << 1; + ticket->acked_s_holders_bitmap = UINT32_C(1) << 2; + + UT_ASSERT_EQ(cluster_pcm_x_master_drive_snapshot_exact( + &admission.ref.identity.tag, admission.ref.identity.cluster_epoch, &snapshot), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_transfer_wire_49_56_is_generation_exact) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXTicketRef transfer; + PcmXRevokePayload revoke; + PcmXGrantPayload image_ready; + PcmXGrantPayload prepare; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXMasterFinalAckToken final_ack_token; + PcmXMasterFinalAckToken replay_final_ack_token; + PcmXMasterFinalAckToken stale_final_ack_token; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef guessed; + uint64 expected_image_id; + const uint64 requester_session = UINT64_C(7269); + const uint64 source_session = UINT64_C(8269); + const uint64 graph_generation = UINT64_C(9269); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(800, 0, 20, UINT64_C(8269), requester_session, 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + test_master_begin_transfer_unclaimed_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_DUPLICATE); + /* A lost type-48 ACK makes the holder replay the whole zero-edge set. + * ACTIVE_PROBE must retain enough exact evidence to replay that ACK. */ + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_DUPLICATE); + /* Even with the blocker graph fully committed, irreversible transfer is + * forbidden until the ticket carries the external pending-X claim. */ + UT_ASSERT_EQ( + test_master_begin_transfer_unclaimed_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_BAD_STATE); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + before = *ticket; + guessed = admission.ref; + guessed.grant_generation = UINT64_C(9369); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&guessed, graph_generation, &transfer), + PCM_X_QUEUE_INVALID); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(transfer.grant_generation, ticket->reliable.state_sequence); + UT_ASSERT(transfer.grant_generation != UINT64_C(9369)); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_DUPLICATE); + /* A delayed duplicate type-47 COMMIT can arrive after the exact ticket + * has already advanced into transfer. The blocker stage proves this is + * the same published set and returns DUPLICATE; probe_complete then sees + * the later phase as BAD_STATE. Re-emitting type 48 already closes this + * replay, so the GCS handler must not turn that benign phase race into a + * cluster-wide RECOVERY_BLOCKED transition. */ + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(!cluster_pcm_x_blocker_commit_completion_requires_recovery(PCM_X_QUEUE_DUPLICATE, + PCM_X_QUEUE_BAD_STATE)); + UT_ASSERT(!cluster_pcm_x_blocker_commit_completion_requires_recovery(PCM_X_QUEUE_DUPLICATE, + PCM_X_QUEUE_STALE)); + UT_ASSERT(!cluster_pcm_x_blocker_commit_completion_requires_recovery(PCM_X_QUEUE_DUPLICATE, + PCM_X_QUEUE_NOT_READY)); + UT_ASSERT(!cluster_pcm_x_blocker_commit_completion_requires_recovery(PCM_X_QUEUE_DUPLICATE, + PCM_X_QUEUE_BUSY)); + UT_ASSERT(cluster_pcm_x_blocker_commit_completion_requires_recovery(PCM_X_QUEUE_DUPLICATE, + PCM_X_QUEUE_CORRUPT)); + UT_ASSERT(cluster_pcm_x_blocker_commit_completion_requires_recovery(PCM_X_QUEUE_OK, + PCM_X_QUEUE_BAD_STATE)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_OK); + UT_ASSERT(cluster_pcm_x_image_id_encode(0, 1, &expected_image_id)); + UT_ASSERT_EQ(revoke.image_id, expected_image_id); + UT_ASSERT(revoke.image_id != transfer.grant_generation); + UT_ASSERT_EQ(header->next_image_id, 2); + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_DUPLICATE); + + memset(&image_ready, 0, sizeof(image_ready)); + image_ready.ref = transfer; + image_ready.image.image_id = revoke.image_id; + image_ready.image.source_own_generation = UINT64_C(44); + image_ready.image.page_scn = UINT64_C(55); + image_ready.image.page_lsn = UINT64_C(66); + image_ready.image.source_node = 2; + image_ready.image.page_checksum = UINT32_C(77); + UT_ASSERT_EQ(cluster_pcm_x_master_image_ready_exact(&image_ready, 2, source_session, &prepare), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&prepare.ref, &transfer)); + UT_ASSERT(memcmp(&prepare.image, &image_ready.image, sizeof(prepare.image)) == 0); + image_ready.image.image_id++; + UT_ASSERT_EQ(cluster_pcm_x_master_image_ready_exact(&image_ready, 2, source_session, &prepare), + PCM_X_QUEUE_STALE); + image_ready.image.image_id--; + + memset(&install_ready, 0, sizeof(install_ready)); + install_ready.ref = transfer; + install_ready.image_id = revoke.image_id; + install_ready.result = PCM_X_QUEUE_OK; + install_ready.phase = PGRAC_IC_MSG_PCM_X_INSTALL_READY; + UT_ASSERT_EQ( + cluster_pcm_x_master_install_ready_exact(&install_ready, 0, requester_session, &commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(commit.phase, PGRAC_IC_MSG_PCM_X_COMMIT_X); + + memset(&final_ack, 0, sizeof(final_ack)); + final_ack.ref = transfer; + final_ack.image_id = revoke.image_id; + final_ack.committed_own_generation = 1; + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + replay_final_ack_token = final_ack_token; + stale_final_ack_token = final_ack_token; + stale_final_ack_token.state_sequence++; + UT_ASSERT_EQ( + cluster_pcm_x_master_final_ack_finalize_exact(&stale_final_ack_token, &final_commit), + PCM_X_QUEUE_STALE); + stale_final_ack_token = final_ack_token; + stale_final_ack_token.image.page_scn++; + UT_ASSERT_EQ( + cluster_pcm_x_master_final_ack_finalize_exact(&stale_final_ack_token, &final_commit), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_COMMIT_X); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_commit.phase, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); + UT_ASSERT_EQ( + cluster_pcm_x_master_final_ack_finalize_exact(&replay_final_ack_token, &final_commit), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_prepare_exact(&final_ack, 0, requester_session, + &final_ack_token), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_final_ack_finalize_exact(&final_ack_token, &final_commit), + PCM_X_QUEUE_DUPLICATE); + final_ack.committed_own_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_master_final_ack_exact(&final_ack, 0, requester_session, &final_commit), + PCM_X_QUEUE_STALE); + + memset(&final_confirm, 0, sizeof(final_confirm)); + final_confirm.ref = transfer; + final_confirm.phase = PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM; + UT_ASSERT_EQ(cluster_pcm_x_master_final_confirm_exact(&final_confirm, 0, requester_session), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_COMPLETE); + UT_ASSERT_EQ(test_slot_flags(&ticket->slot), 0); + UT_ASSERT_EQ(cluster_pcm_x_master_final_confirm_exact(&final_confirm, 0, requester_session), + PCM_X_QUEUE_DUPLICATE); +} + +UT_TEST(test_master_grant_generation_exhaustion_never_wraps) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + const uint64 source_session = UINT64_C(8270); + const uint64 graph_generation = UINT64_C(9270); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(801, 0, 21, UINT64_C(8270), UINT64_C(7270), 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->reliable.state_sequence = UINT64_MAX; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(ticket->ref.grant_generation, 0); + UT_ASSERT_EQ(ticket->reliable.state_sequence, UINT64_MAX); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_image_id_allocator_encodes_node31_and_never_wraps) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload blocker_commit; + PcmXRevokePayload revoke; + PcmXTicketRef transfer; + uint64 expected_image_id; + const uint64 source_session = UINT64_C(8271); + const uint64 graph_generation = UINT64_C(9271); + + init_active_pcm_x(UINT64_C(77)); + cluster_node_id = 31; + header = ClusterPcmXConvertShmem; + admit_active_probe(802, 0, 22, UINT64_C(8271), UINT64_C(7271), 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_OK); + header->next_image_id = PCM_X_IMAGE_ID_SEQ_MASK; + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_OK); + UT_ASSERT(cluster_pcm_x_image_id_encode(31, PCM_X_IMAGE_ID_SEQ_MASK, &expected_image_id)); + UT_ASSERT_EQ(revoke.image_id, expected_image_id); + UT_ASSERT_EQ(header->next_image_id, PCM_X_IMAGE_ID_SEQ_MASK + 1); + + /* The first raw value beyond the 56-bit domain is a durable exhausted + * marker. It is neither masked nor wrapped into another wire identity. */ + init_active_pcm_x(UINT64_C(78)); + cluster_node_id = 31; + header = ClusterPcmXConvertShmem; + admit_active_probe(803, 0, 23, UINT64_C(8272), UINT64_C(7272), 1, &admission); + arm_blocker_probe(&admission.ref, 2, source_session); + blocker_commit = make_blocker_header(&admission.ref, 1, NULL, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&blocker_commit, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&blocker_commit, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, graph_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 1, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, graph_generation, &transfer), + PCM_X_QUEUE_OK); + header->next_image_id = PCM_X_IMAGE_ID_SEQ_MASK + 1; + UT_ASSERT_EQ(cluster_pcm_x_master_revoke_arm_exact(&transfer, 2, source_session, &revoke), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(revoke.image_id, 0); + UT_ASSERT_EQ(header->next_image_id, PCM_X_IMAGE_ID_SEQ_MASK + 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_master_blocker_wire_stage_preserves_old_set_until_commit) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry old_entry; + PcmXMasterBlockerSnapshot empty_snapshot; + PcmXMasterBlockerSnapshot old_snapshot; + PcmXBlockerSetHeaderPayload empty; + PcmXTicketRef transfer; + ClusterLmdVertex old_blocker = make_blocker(1, 5, UINT64_C(8761)); + const uint64 source_session = UINT64_C(82601); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(792, 0, 12, UINT64_C(8261), UINT64_C(7261), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact( + &admission.ref, 1, &old_blocker, 1, blocker_set_crc32c(&old_blocker, 1)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&old_snapshot, &old_entry, 1, + UINT64_C(9161)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &old_entry, 1, &old_snapshot), + PCM_X_QUEUE_OK); + empty = make_blocker_header(&admission.ref, 2, NULL, 0); + arm_blocker_probe(&admission.ref, 3, source_session); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_revalidate_exact(&old_snapshot, &old_entry, 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9261)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_probe_complete_exact(&admission.ref, 2, 3, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9261), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session + 1, + NULL, 0, &empty_snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&empty, 3, source_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&empty, 3, source_session, NULL, 0, + &empty_snapshot), + PCM_X_QUEUE_DUPLICATE); +} + +UT_TEST(test_master_blocker_wire_stage_rejects_reorder_conflict_and_wrong_source) +{ + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entries[2]; + PcmXMasterBlockerSnapshot snapshot; + PcmXBlockerSetHeaderPayload begin; + PcmXBlockerSetHeaderPayload conflict_begin; + PcmXBlockerChunkPayload edge[2]; + PcmXBlockerChunkPayload conflict_edge; + ClusterLmdVertex blockers[2]; + ClusterLmdVertex noncanonical; + const uint64 source_session = UINT64_C(82701); + + init_active_pcm_x(UINT64_C(77)); + admit_active_probe(793, 0, 13, UINT64_C(8271), UINT64_C(7271), 1, &admission); + blockers[0] = make_blocker(2, 6, UINT64_C(8771)); + blockers[1] = make_blocker(2, 7, UINT64_C(8772)); + begin = make_blocker_header(&admission.ref, 1, blockers, lengthof(blockers)); + edge[0] = make_blocker_chunk(&admission.ref, 1, 0, blockers[0]); + edge[1] = make_blocker_chunk(&admission.ref, 1, 1, blockers[1]); + arm_blocker_probe(&admission.ref, 2, source_session); + + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 3, source_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), + PCM_X_QUEUE_NOT_READY); + conflict_begin = begin; + conflict_begin.set_crc32c++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&conflict_begin, 2, source_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), + PCM_X_QUEUE_OK); + conflict_begin = begin; + conflict_begin.nblockers = UINT32_MAX; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&conflict_begin, 2, source_session, + NULL, 0, &snapshot), + PCM_X_QUEUE_INVALID); + conflict_edge = edge[0]; + conflict_edge.blocker.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_NOT_READY); + noncanonical = make_blocker(1, 8, UINT64_C(8773)); + conflict_edge = make_blocker_chunk(&admission.ref, 1, 1, noncanonical); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[1], 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 3, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, + lengthof(entries), &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&edge[0], 2, source_session), + PCM_X_QUEUE_DUPLICATE); + conflict_edge = edge[0]; + conflict_edge.blocker.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_edge_exact(&conflict_edge, 2, source_session), + PCM_X_QUEUE_STALE); +} + +UT_TEST(test_master_blocker_wire_stage_fences_transfer_and_cancel_until_commit) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerSnapshot empty_snapshot; + PcmXBlockerSetHeaderPayload begin; + PcmXTicketRef transfer; + ClusterLmdVertex blocker = make_blocker(2, 9, UINT64_C(8781)); + const uint64 source_session = UINT64_C(82801); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(794, 0, 14, UINT64_C(8281), UINT64_C(7281), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, + blocker_set_crc32c(NULL, 0)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9281)), + PCM_X_QUEUE_OK); + begin = make_blocker_header(&admission.ref, 2, &blocker, 1); + arm_blocker_probe(&admission.ref, 2, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9281), &transfer), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_DUPLICATE); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(795, 0, 15, UINT64_C(8282), UINT64_C(7282), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, NULL, 0, + blocker_set_crc32c(NULL, 0)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &empty_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&empty_snapshot, NULL, 0, UINT64_C(9282)), + PCM_X_QUEUE_OK); + begin = make_blocker_header(&admission.ref, 2, &blocker, 1); + arm_blocker_probe(&admission.ref, 2, source_session); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_begin_exact(&begin, 2, source_session), + PCM_X_QUEUE_DUPLICATE); +} + +UT_TEST(test_master_blocker_graph_commit_revalidates_exact_snapshot_and_generation) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerEntry stale_entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterBlockerSnapshot stale_snapshot; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + ClusterLmdVertex blocker = make_blocker(1, 1, UINT64_C(8801)); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(792, 0, 8, UINT64_C(8203), UINT64_C(7203), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x701)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + + stale_snapshot = snapshot; + stale_snapshot.set_generation++; + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&stale_snapshot, &entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + stale_entry = entry; + stale_entry.slot_ref.slot_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &stale_entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, 0), + PCM_X_QUEUE_INVALID); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(9101)); + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9101)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9100)), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9102)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->graph_generation, UINT64_C(9102)); + UT_ASSERT_EQ(max_held_lwlock_count, 1); +} + +UT_TEST(test_master_transfer_requires_committed_exact_empty_blocker_set) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef transfer; + ClusterLmdVertex blocker = make_blocker(1, 2, UINT64_C(8802)); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(793, 0, 9, UINT64_C(8204), UINT64_C(7204), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x702)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9201)), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_master_pending_x_claim_exact(&admission.ref), PCM_X_QUEUE_OK); + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), + PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x703)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->graph_generation, 0); + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, UINT64_C(9202)), + PCM_X_QUEUE_OK); + /* The direct replace fixture bypasses the 45-48 wire ACK. */ + ticket->reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_COMMIT; + before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9201), &transfer), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission.ref, UINT64_C(9202), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(transfer.grant_generation, ticket->reliable.state_sequence); +} + +UT_TEST(test_master_active_probe_cancel_requires_committed_exact_empty_blocker_set) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterBlockerEntry entry; + PcmXMasterBlockerSnapshot snapshot; + PcmXMasterBlockerSnapshot stale_snapshot; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + ClusterLmdVertex blocker = make_blocker(1, 3, UINT64_C(8803)); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + admit_active_probe(794, 0, 10, UINT64_C(8205), UINT64_C(7205), 1, &admission); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 1, &blocker, 1, + UINT32_C(0x704)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, &entry, 1, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, &entry, 1, UINT64_C(9401)), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + + stale_snapshot = snapshot; + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_set_replace_exact(&admission.ref, 2, NULL, 0, UINT32_C(0x705)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&stale_snapshot, &entry, 1, UINT64_C(9402)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_blocker_snapshot_exact(&admission.ref, NULL, 0, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_blocker_graph_commit_exact(&snapshot, NULL, 0, UINT64_C(9402)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); +} + +UT_TEST(test_master_locator_defers_mutable_grant_exactness_to_domain_lock) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTicketRef active; + PcmXTicketRef transfer; + PcmXQueueResult result; + uint64 grant_generation; + uint32 partition; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(743, 0, 9, UINT64_C(52001)), UINT64_C(5301), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(401)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(401)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(401), &transfer), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + grant_generation = transfer.grant_generation; + + /* Model a torn allocator-lock observation while the domain writer finishes + * publishing grant_generation. Locator lookup must use immutable fields; + * the full ref is accepted only after the master-domain interlock restores + * the stable value. */ + ticket->ref.grant_generation = UINT64CONST(0x3333333322222222); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&transfer.identity.tag)); + grant_interlock_lock = &header->master_locks[partition].lock; + grant_interlock_ticket = ticket; + grant_interlock_generation = grant_generation; + grant_interlock_armed = true; + result = cluster_pcm_x_master_complete_exact(&transfer); + UT_ASSERT_EQ(result, PCM_X_QUEUE_OK); + UT_ASSERT(!grant_interlock_armed); + grant_interlock_armed = false; + grant_interlock_lock = NULL; + grant_interlock_ticket = NULL; +} + +UT_TEST(test_admission_replay_and_confirm_stay_frozen_after_transfer) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterAdmission replay; + PcmXEnqueuePayload request; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; + PcmXTicketRef transfer; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(713, 0, 22, UINT64_C(16001)), UINT64_C(1701), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(501)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(501)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(501), &transfer), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &replay), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(replay.ref.grant_generation, 0); + UT_ASSERT_EQ(replay.ref.handle.ticket_id, admission.ref.handle.ticket_id); + UT_ASSERT_EQ(replay.ref.handle.queue_generation, admission.ref.handle.queue_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(501)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); + admission.ref = transfer; + drain_retire_and_detach_master(&admission); + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_application_replays_survive_terminal_successors) +{ + static const uint32 confirm_states[] + = { PCM_XT_QUEUED, PCM_XT_ACTIVE_PROBE, PCM_XT_ACTIVE_TRANSFER, + PCM_XT_COMPLETE, PCM_XT_CANCELLED, PCM_XT_RETIRE_CREDIT }; + static const uint32 complete_states[] = { PCM_XT_COMPLETE, PCM_XT_RETIRE_CREDIT }; + static const uint32 cancel_states[] = { PCM_XT_CANCELLED, PCM_XT_RETIRE_CREDIT }; + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + Size i; + + for (i = 0; i < lengthof(confirm_states); i++) { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(719, 0, 26, UINT64_C(24001)), UINT64_C(2501), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->graph_generation = UINT64_C(701); + if (confirm_states[i] == PCM_XT_ACTIVE_TRANSFER || confirm_states[i] == PCM_XT_COMPLETE) + ticket->ref.grant_generation = UINT64_C(2701); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_ADMIT_CONFIRM; + test_set_slot_state(&ticket->slot, confirm_states[i]); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(701)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(702)) + != PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + } + + for (i = 0; i < lengthof(complete_states); i++) { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef completion_ref; + PcmXTicketRef stale_ref; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(720, 0, 27, UINT64_C(24002)), UINT64_C(2502), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->graph_generation = UINT64_C(703); + ticket->ref.grant_generation = UINT64_C(2702); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + completion_ref = ticket->ref; + test_set_slot_state(&ticket->slot, complete_states[i]); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&completion_ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + stale_ref = completion_ref; + stale_ref.grant_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&stale_ref), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + } + + for (i = 0; i < lengthof(cancel_states); i++) { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef stale_ref; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(721, 0, 28, UINT64_C(24003)), UINT64_C(2503), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->graph_generation = UINT64_C(704); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_CANCEL; + test_set_slot_state(&ticket->slot, cancel_states[i]); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + stale_ref = admission.ref; + stale_ref.handle.queue_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale_ref), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + } + + /* RETIRE_CREDIT alone is not outcome authority. */ + init_active_pcm_x(UINT64_C(77)); + request + = make_enqueue(make_wait_identity(723, 0, 4, UINT64_C(24004)), UINT64_C(2504), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef cross_complete; + + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->ref.grant_generation = UINT64_C(2703); + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_CANCEL; + test_set_slot_state(&ticket->slot, PCM_XT_RETIRE_CREDIT); + cross_complete = ticket->ref; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&cross_complete), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } + + init_active_pcm_x(UINT64_C(77)); + request + = make_enqueue(make_wait_identity(724, 0, 5, UINT64_C(24005)), UINT64_C(2505), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + { + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(ClusterPcmXConvertShmem)[admission.tag_slot.slot_index]; + ticket->reliable.response_tombstone_mask = PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + test_set_slot_state(&ticket->slot, PCM_XT_RETIRE_CREDIT); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + } +} + +UT_TEST(test_delayed_pretransfer_cancel_fences_active_transfer) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXMasterTicketSlot *ticket; + PcmXRuntimeSnapshot snapshot; + PcmXTicketRef active; + PcmXTicketRef transfer; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(714, 0, 23, UINT64_C(16002)), UINT64_C(1702), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(502)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(502)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(502), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_BAD_STATE); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RECOVERY_BLOCKED); + UT_ASSERT_EQ(ticket->ref.grant_generation, transfer.grant_generation); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterAdmission queued_admission; + PcmXEnqueuePayload request; + PcmXEnqueuePayload queued_request; + PcmXMasterTicketSlot *ticket; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTagSlot tag_before; + PcmXTicketRef active; + PcmXTicketRef transfer; + PcmXSlotRef found; + uint64 queue_state; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(716, 0, 25, UINT64_C(21001)), UINT64_C(2201), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(601)), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + queue_state = tag_slot->queue_state_sequence; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_QUEUED); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(717, 0, 25, UINT64_C(21002)), UINT64_C(2202), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(602)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(602)); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + queue_state = tag_slot->queue_state_sequence; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(602), &transfer), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_ACTIVE_PROBE); + UT_ASSERT_EQ(ticket->ref.grant_generation, 0); + UT_ASSERT_EQ(tag_slot->active_index, admission.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(718, 0, 25, UINT64_C(21003)), UINT64_C(2203), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &admission.ref.handle, &found), + PCM_X_DIRECTORY_OK); + + /* A blocked runtime has no FINAL_ACK proof and must retain transfer evidence. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(733, 0, 2, UINT64_C(40001)), UINT64_C(4101), UINT64_C(1)); + queued_request + = make_enqueue(make_wait_identity(733, 1, 3, UINT64_C(40002)), UINT64_C(4102), UINT64_C(1)); + bind_enqueue_peer(&request); + bind_enqueue_peer(&queued_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&queued_request, &queued_admission), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(801)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&queued_admission.ref, UINT64_C(802)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(801)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(801), &transfer), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish( + request.identity.node_id, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT_EQ( + test_slot_state(&master_ticket_slots(header)[admission.ticket_slot.slot_index].slot), + PCM_XT_ACTIVE_TRANSFER); + UT_ASSERT_EQ( + test_slot_state(&master_ticket_slots(header)[queued_admission.ticket_slot.slot_index].slot), + PCM_XT_QUEUED); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_NOT_READY); +} + +UT_TEST(test_recovery_blocked_runtime_refuses_ack_and_local_mutators) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXRetirePayload retire_ack; + PcmXTerminalLegToken retire_token; + PcmXLocalCutoff cutoff; + PcmXLocalHandle leader; + PcmXLocalHandle output; + PcmXLocalMembershipSlot member_before; + PcmXLocalMembershipSlot *member; + PcmXLocalTagSlot tag_before; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(734, 0, 2, UINT64_C(43001)), UINT64_C(4401), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket_before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(901)), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket_before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, admission.ref.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, admission.ref.identity.node_id, + request.prehandle.sender_session_incarnation, &retire_token), + PCM_X_QUEUE_OK); + retire_ack = make_retire_ack(&admission.ref, admission.ref.identity.node_id); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket_before = *ticket; + UT_ASSERT_EQ( + cluster_pcm_x_peer_bind_ack_publish(0, request.identity.cluster_epoch, + request.prehandle.sender_session_incarnation + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_exact(&retire_ack, admission.ref.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(735, 0, 2, UINT64_C(43002)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(4501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(4501), &leader), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + member = &membership_slots(header)[leader.membership_slot.slot_index]; + tag_before = *tag_slot; + member_before = *member; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, identity.cluster_epoch, UINT64_C(4502)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, &output), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, identity.cluster_epoch, UINT64_C(4501)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(4501), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + member = &membership_slots(header)[leader.membership_slot.slot_index]; + tag_before = *tag_slot; + member_before = *member; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, identity.cluster_epoch, UINT64_C(4502)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &output), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(member, &member_before, sizeof(*member)) == 0); +} + +UT_TEST(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *tickets; + PcmXTerminalLegToken drain_token; + PcmXTerminalLegToken retire_token; + PcmXRetirePayload retire_ack0; + PcmXRetirePayload retire_ack1; + PcmXTicketRef stale; + PcmXSlotRef found; + uint64 queue_state_before; + uint64 responder_session; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(705, i, (uint32)(14 + i), UINT64_C(6001) + i), + UINT64_C(1001) + i, UINT64_C(1)); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(200) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + queue_state_before = tag_slot->queue_state_sequence; + stale = admission[1].ref; + stale.handle.queue_generation++; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&stale), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); + UT_ASSERT_EQ(tag_slot->head_index, admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, + admission[1].ticket_slot.slot_index); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before + 1); + queue_state_before = tag_slot->queue_state_sequence; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); + UT_ASSERT_EQ(test_slot_state(&tickets[admission[1].ticket_slot.slot_index].slot), + PCM_XT_CANCELLED); + UT_ASSERT_EQ(tickets[admission[0].ticket_slot.slot_index].next_index, + admission[2].ticket_slot.slot_index); + UT_ASSERT_EQ(tickets[admission[2].ticket_slot.slot_index].prev_index, + admission[0].ticket_slot.slot_index); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap) & (UINT32_C(1) << 1), 0); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), + PCM_X_QUEUE_NOT_READY); + responder_session = header->peer_frontiers[1].sender_session_incarnation; + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(&admission[1].ref, + PCM_X_TERMINAL_LEG_DRAIN, 1, + responder_session, &drain_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission[1].ref, PCM_X_TERMINAL_LEG_DRAIN, 0, + header->peer_frontiers[0].sender_session_incarnation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission[1].ref, PCM_X_TERMINAL_LEG_DRAIN, 1, responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&tickets[admission[1].ticket_slot.slot_index].slot), + PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact(&admission[1].ref, + PCM_X_TERMINAL_LEG_RETIRE, 1, + responder_session, &retire_token), + PCM_X_QUEUE_OK); + retire_ack0 = make_retire_ack(&admission[1].ref, 0); + retire_ack1 = make_retire_ack(&admission[1].ref, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact( + &retire_ack0, 0, header->peer_frontiers[0].sender_session_incarnation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack1, 1, responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 3); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &admission[1].prehandle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, admission[1].ticket_slot)); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[0].ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&admission[0]); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 2); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[2].ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&admission[2]); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 3); + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission cancelled; + PcmXMasterAdmission replay; + PcmXMasterAdmission successor; + PcmXPrehandleCancelPayload cancel; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload successor_request; + PcmXMasterTicketSlot *successor_ticket; + PcmXMasterTicketSlot successor_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request + = make_enqueue(make_wait_identity(706, 0, 10, UINT64_C(61001)), UINT64_C(6101), 1); + bind_enqueue_peer(&first_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + memset(&cancel, 0, sizeof(cancel)); + cancel.identity = first_request.identity; + cancel.prehandle = first_request.prehandle; + + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &cancelled), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&cancelled.ref, &first.ref)); + UT_ASSERT(memcmp(&cancelled.prehandle, &first.prehandle, sizeof(first.prehandle)) == 0); + UT_ASSERT_EQ(test_slot_state(&master_ticket_slots(header)[first.ticket_slot.slot_index].slot), + PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&replay.ref, &cancelled.ref)); + UT_ASSERT(memcmp(&replay.prehandle, &cancelled.prehandle, sizeof(replay.prehandle)) == 0); + + drain_retire_and_detach_master(&cancelled); + + successor_request + = make_enqueue(make_wait_identity(706, 0, 11, UINT64_C(61002)), UINT64_C(6101), 2); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&successor_request, &successor), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(successor.ticket_slot.slot_index, first.ticket_slot.slot_index); + UT_ASSERT(successor.ticket_slot.slot_generation != first.ticket_slot.slot_generation); + successor_ticket = &master_ticket_slots(header)[successor.ticket_slot.slot_index]; + successor_before = *successor_ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), + PCM_X_QUEUE_RETIRED); + UT_ASSERT(memcmp(successor_ticket, &successor_before, sizeof(*successor_ticket)) == 0); +} + +UT_TEST(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel) +{ + PcmXMasterAdmission admission; + PcmXMasterAdmission output; + PcmXPrehandleCancelPayload alias; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(709, 0, 14, UINT64_C(64001)), UINT64_C(6401), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + memset(&alias, 0, sizeof(alias)); + alias.identity = request.identity; + alias.identity.wait_seq++; + alias.prehandle = request.prehandle; + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&alias, &output), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ( + test_slot_state( + &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index].slot), + PCM_XT_ADMITTING); +} + +UT_TEST(test_master_prehandle_cancel_first_publishes_terminal_tombstone_before_late_enqueue) +{ + PcmXMasterAdmission cancelled; + PcmXMasterAdmission late_admission; + PcmXMasterAdmission replay; + PcmXPrehandleCancelPayload cancel; + PcmXEnqueuePayload request; + PcmXMasterTicketSlot *ticket; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(708, 0, 13, UINT64_C(63001)), UINT64_C(6301), 1); + bind_enqueue_peer(&request); + memset(&cancel, 0, sizeof(cancel)); + cancel.identity = request.identity; + cancel.prehandle = request.prehandle; + + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &cancelled), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[cancelled.ticket_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[0].next_expected_prehandle_sequence, 2); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ( + master_tag_slots(ClusterPcmXConvertShmem)[cancelled.tag_slot.slot_index].head_index, + PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ( + pg_atomic_read_u32(&master_tag_slots(ClusterPcmXConvertShmem)[cancelled.tag_slot.slot_index] + .queued_node_bitmap), + 0); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &late_admission), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&late_admission.ref, &cancelled.ref)); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_prehandle_cancel_exact(&cancel, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&replay.ref, &cancelled.ref)); +} + +UT_TEST(test_master_cancel_rejects_active_probe_without_mutation) +{ + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(707, 0, 12, UINT64_C(62001)), UINT64_C(6201), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission.ref, UINT64_C(6202)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(ClusterPcmXConvertShmem)[admission.ticket_slot.slot_index]; + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_reversible_exact(&active), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_master_unlink_corruption_is_byte_stable_before_fail_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before[3]; + PcmXMasterTicketSlot *tickets; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(748, i, (uint32)(20 + i), UINT64_C(61001) + i), + UINT64_C(6201) + i, UINT64_C(1)); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(701) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + /* Corrupt only the successor backlink. Validation must not first splice + * the predecessor out of the queue and erase the original evidence. */ + tickets[admission[2].ticket_slot.slot_index].prev_index = PCM_X_INVALID_SLOT_INDEX; + tag_before = *tag_slot; + for (i = 0; i < 3; i++) + ticket_before[i] = tickets[admission[i].ticket_slot.slot_index]; + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + for (i = 0; i < 3; i++) + UT_ASSERT(memcmp(&tickets[admission[i].ticket_slot.slot_index], &ticket_before[i], + sizeof(ticket_before[i])) + == 0); +} + +UT_TEST(test_master_cancel_requires_state_exact_active_locator) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[2]; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before[2]; + PcmXMasterTicketSlot *tickets; + PcmXEnqueuePayload request; + PcmXTicketRef active; + int i; + + /* An ACTIVE_PROBE cannot be cancelled after its active locator is lost. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(750, 0, 2, UINT64_C(64001)), UINT64_C(6501), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[0].ref, UINT64_C(801)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&request.identity.tag, + request.identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + tag_slot->active_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_slot_generation = 0; + tag_before = *tag_slot; + ticket_before[0] = tickets[admission[0].ticket_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(&tickets[admission[0].ticket_slot.slot_index], &ticket_before[0], + sizeof(ticket_before[0])) + == 0); + + /* Conversely a QUEUED ticket cannot be named by the active locator. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 2; i++) { + request = make_enqueue(make_wait_identity(751, i, (uint32)(2 + i), UINT64_C(64002) + i), + UINT64_C(6502) + i, 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(802) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + tickets = master_ticket_slots(header); + tag_slot->active_index = admission[1].ticket_slot.slot_index; + tag_slot->active_slot_generation = admission[1].ticket_slot.slot_generation; + tag_before = *tag_slot; + for (i = 0; i < 2; i++) + ticket_before[i] = tickets[admission[i].ticket_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + for (i = 0; i < 2; i++) + UT_ASSERT(memcmp(&tickets[admission[i].ticket_slot.slot_index], &ticket_before[i], + sizeof(ticket_before[i])) + == 0); +} + +UT_TEST(test_terminal_ack_requires_prearmed_leg_and_is_byte_stable) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXRetirePayload retire_ack; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(740, 0, 2, UINT64_C(50001)), UINT64_C(5101), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + retire_ack = make_retire_ack(&admission.ref, request.identity.node_id); + before = *ticket; + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_exact(&retire_ack, request.identity.node_id, + request.prehandle.sender_session_incarnation), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); +} + +UT_TEST(test_terminal_ack_wire_fields_are_exact_and_zero_side_effect) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXRetirePayload retire_ack; + PcmXTicketRef resolved; + PcmXTicketRef work; + PcmXTerminalLegToken drain; + PcmXTerminalLegToken replay; + PcmXTerminalLegToken retire; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(741, 0, 2, UINT64_C(50002)), UINT64_C(5102), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_work_next(&work), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&work, &admission.ref)); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + retire_ack = make_retire_ack(&admission.ref, request.identity.node_id); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &drain), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(drain.kind, PCM_X_TERMINAL_LEG_DRAIN); + UT_ASSERT_EQ(drain.expected_responder_node, request.identity.node_id); + UT_ASSERT_EQ(drain.expected_responder_session, request.prehandle.sender_session_incarnation); + UT_ASSERT(drain.state_sequence != 0); + UT_ASSERT(drain.drain_generation != 0); + UT_ASSERT_EQ(ticket->drain_generation, drain.drain_generation); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &drain, sizeof(replay)) == 0); + before = *ticket; + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, 1, + drain.expected_responder_session), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(ticket->reliable.pending_opcode, 0); + UT_ASSERT_EQ(ticket->reliable.phase, 0); + UT_ASSERT_EQ(ticket->reliable.expected_responder_node, 0); + UT_ASSERT_EQ(ticket->reliable.expected_responder_session, 0); + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, request.identity.node_id, + request.prehandle.sender_session_incarnation, &retire), + PCM_X_QUEUE_OK); + before = *ticket; + /* A closed DRAIN key replays without clearing the newly armed RETIRE leg. */ + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + memset(&resolved, 0, sizeof(resolved)); + UT_ASSERT_EQ( + cluster_pcm_x_master_retire_ack_resolve_exact(&retire_ack, retire.expected_responder_node, + retire.expected_responder_session, &resolved), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&resolved, &admission.ref)); + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire_ack, retire.expected_responder_node, + retire.expected_responder_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); +} + +UT_TEST(test_retire_ack_uses_persistent_exact_key_index) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXRetirePayload retire0; + PcmXRetirePayload retire1; + PcmXRetirePayload stale; + PcmXSlotRef found; + PcmXTerminalLegToken token; + uint64 session0; + uint64 session1 = UINT64_C(6102); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(745, 0, 2, UINT64_C(50006)), UINT64_C(5106), UINT64_C(1)); + bind_enqueue_peer(&request); + session0 = request.prehandle.sender_session_incarnation; + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + retire0 = make_retire_ack(&admission.ref, 0); + retire1 = make_retire_ack(&admission.ref, 1); + before = *ticket; + + /* A byte-exact ACK has no authority until arm publishes its shmem key. */ + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), + PCM_X_DIRECTORY_NOT_FOUND); + + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, session0, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, admission.ticket_slot)); + before = *ticket; + + stale = retire0; + stale.cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); + stale = retire0; + stale.master_session_incarnation++; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); + stale = retire0; + stale.retire_through_ticket_id++; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_STALE); + stale = retire0; + stale.sender_node = 1; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 1, session1), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 1, session1), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0 + 1), + PCM_X_QUEUE_STALE); + stale = retire0; + stale.flags = 1; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&stale, 0, session0), PCM_X_QUEUE_INVALID); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1, session1, &token), + PCM_X_QUEUE_OK); + before = *ticket; + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire0, 0, session0), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_master_retire_ack_exact(&retire1, 1, session1), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ticket->retire_acked_nodes_bitmap, ticket->involved_nodes_bitmap); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire0, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire1, &found), + PCM_X_DIRECTORY_NOT_FOUND); +} + +UT_TEST(test_early_retire_arm_does_not_publish_locator) +{ + PcmXMasterAdmission admission; + PcmXEnqueuePayload request; + PcmXRetirePayload retire; + PcmXSlotRef found; + PcmXTerminalLegToken token; + + init_active_pcm_x(UINT64_C(77)); + request = make_enqueue(make_wait_identity(749, 0, 13, UINT64_C(61001)), UINT64_C(6201), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + retire = make_retire_ack(&admission.ref, 0); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0, + request.prehandle.sender_session_incarnation, &token), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_RETIRE, &retire, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_terminal_outcome_mask_corruption_is_fail_closed) +{ + int mode; + + for (mode = 0; mode < 3; mode++) { + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken token; + PcmXQueueResult result; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(742 + mode, 0, 2, UINT64_C(50003) + mode), + UINT64_C(5103) + mode, UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + if (mode == 0) + ticket->reliable.response_tombstone_mask &= ~PCM_X_RESPONSE_TOMBSTONE_CANCEL; + else if (mode == 1) + ticket->reliable.response_tombstone_mask |= PCM_X_RESPONSE_TOMBSTONE_COMPLETE; + else + ticket->reliable.response_tombstone_mask |= UINT64_C(1) << 63; + before = *ticket; + if (mode == 0) + result = cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &token); + else + result = cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation); + UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_terminal_bitmap_superset_is_fail_closed_before_arm_or_ack_mutation) +{ + int mode; + + for (mode = 0; mode < 2; mode++) { + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTicketSlot before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken token; + PcmXQueueResult result; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue( + make_wait_identity(752 + mode, 0, (uint32)(2 + mode), UINT64_C(66001) + mode), + UINT64_C(6701) + mode, 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + if (mode == 1) + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &token), + PCM_X_QUEUE_OK); + /* Node 1 never participated. Neither arming nor consuming a valid + * node-0 ACK may let this impossible superset wedge in ACTIVE. */ + ticket->drained_nodes_bitmap = UINT32_C(1) << 1; + before = *ticket; + if (mode == 0) + result = cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &token); + else + result = cluster_pcm_x_master_terminal_leg_ack_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, token.expected_responder_node, + token.expected_responder_session); + UT_ASSERT_EQ(result, PCM_X_QUEUE_CORRUPT); + UT_ASSERT(memcmp(ticket, &before, sizeof(*ticket)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + } +} + +UT_TEST(test_terminal_detach_missing_retire_ack_is_retryable_not_ready) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + uint64 session1 = UINT64_C(6802); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(754, 0, 2, UINT64_C(66003)), UINT64_C(6801), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(1, request.identity.cluster_epoch, session1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + ticket->involved_nodes_bitmap |= UINT32_C(1) << 1; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, 1); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + UT_ASSERT_EQ(ticket->retire_acked_nodes_bitmap, UINT32_C(1)); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket_before = *ticket; + tag_before = *tag_slot; + + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_OK); + assert_master_queue_baseline(header); +} + +UT_TEST(test_terminal_detach_rejects_pending_leg) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot tag_before; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot ticket_before; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXTerminalLegToken drain; + PcmXTerminalLegToken retire; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(744, 0, 2, UINT64_C(50005)), UINT64_C(5105), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_DRAIN, request.identity.node_id, + request.prehandle.sender_session_incarnation, &drain), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_ack_exact(&admission.ref, drain.kind, + drain.expected_responder_node, + drain.expected_responder_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_terminal_leg_arm_exact( + &admission.ref, PCM_X_TERMINAL_LEG_RETIRE, request.identity.node_id, + request.prehandle.sender_session_incarnation, &retire), + PCM_X_QUEUE_OK); + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + /* Even a corrupt early bitmap cannot bypass an armed application leg. */ + ticket->retire_acked_nodes_bitmap = ticket->involved_nodes_bitmap; + ticket_before = *ticket; + tag_before = *tag_slot; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(ticket, &ticket_before, sizeof(*ticket)) == 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); +} + +UT_TEST(test_master_terminal_detach_rejects_hot_link_and_drain_corruption) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[2]; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXTicketRef active; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 2; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(745, i, (uint32)(7 + i), UINT64_C(55001) + i), + UINT64_C(5601) + i, 1); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(501) + i), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, + admission[0].ref.identity.cluster_epoch, + &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(501)); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission[0].ticket_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 2); + UT_ASSERT_EQ(tag_slot->head_index, admission[1].ticket_slot.slot_index); + tag_slot->head_index = admission[0].ticket_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[0].ref), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 2); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(746, 0, 9, UINT64_C(57001)), UINT64_C(5801), 1); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[0].ref, UINT64_C(601)), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact(&admission[0].ref.identity.tag, + admission[0].ref.identity.cluster_epoch, + &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(601)); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&active), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_DRAIN, 0); + arm_and_ack_master_terminal_leg(&admission[0].ref, PCM_X_TERMINAL_LEG_RETIRE, 0); + ticket = &master_ticket_slots(header)[admission[0].ticket_slot.slot_index]; + ticket->drained_nodes_bitmap = 0; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[0].ref), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); +} + +UT_TEST(test_master_terminal_detach_preserves_same_node_successor) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission successor; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload successor_request; + uint32 node_bit = UINT32_C(1); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request = make_enqueue(make_wait_identity(747, 0, 9, UINT64_C(59001)), UINT64_C(6001), 1); + successor_request + = make_enqueue(make_wait_identity(747, 0, 10, UINT64_C(59002)), UINT64_C(6001), 2); + bind_enqueue_peer(&first_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&first.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_DRAIN, + first.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&first.ref, PCM_X_TERMINAL_LEG_RETIRE, + first.ref.identity.node_id); + + /* The node bitmap identifies the current hot ticket, not every resident + * terminal from that node. A successor may therefore reuse the node bit + * while the older ticket waits to detach in retirement order. */ + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&successor_request, &successor), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + UT_ASSERT(slot_refs_equal(first.tag_slot, successor.tag_slot)); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 2); + UT_ASSERT_EQ(tag_slot->head_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), node_bit); + + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&first.ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(tag_slot->head_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, successor.ticket_slot.slot_index); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), node_bit); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&successor.ref), PCM_X_QUEUE_OK); + drain_retire_and_detach_master(&successor); + assert_master_queue_baseline(header); +} + +UT_TEST(test_master_tag_survives_until_every_terminal_ticket_detaches) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission[3]; + PcmXMasterTagSlot *tag_slot; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + for (i = 0; i < 3; i++) { + PcmXEnqueuePayload request + = make_enqueue(make_wait_identity(712, i, (uint32)(18 + i), UINT64_C(14001) + i), + UINT64_C(1501) + i, UINT64_C(1)); + + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(401) + i), + PCM_X_QUEUE_OK); + } + tag_slot = &master_tag_slots(header)[admission[0].tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 3); + for (i = 0; i < 3; i++) { + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[i].ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission[i].ref, PCM_X_TERMINAL_LEG_DRAIN, + admission[i].ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission[i].ref, PCM_X_TERMINAL_LEG_RETIRE, + admission[i].ref.identity.node_id); + } + UT_ASSERT_EQ(tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->active_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->queued_node_bitmap), 0); + for (i = 0; i < 3; i++) { + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission[i].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->fully_retired_ticket_id, (uint64)i + 1); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, (Size)(2 - i)); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TAG].used, i == 2 ? 0 : 1); + } + assert_master_queue_baseline(header); +} + +UT_TEST(test_last_terminal_detach_waits_for_staged_admission) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXRuntimeSnapshot snapshot; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(715, 0, 24, UINT64_C(19001)), UINT64_C(2001), + UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + pg_atomic_write_u32(&tag_slot->admission_gate, 1); + pg_atomic_write_u32(&tag_slot->queued_node_bitmap, UINT32_C(1) << 1); + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RETIRE_CREDIT); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_master_outstanding_counter_overflow_and_rollback_are_exact) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission first; + PcmXMasterAdmission second; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload first_request; + PcmXEnqueuePayload second_request; + PcmXSlotRef found; + uint64 queue_state_before; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + first_request + = make_enqueue(make_wait_identity(731, 0, 2, UINT64_C(36001)), UINT64_C(3701), UINT64_C(1)); + second_request + = make_enqueue(make_wait_identity(731, 1, 3, UINT64_C(36002)), UINT64_C(3702), UINT64_C(1)); + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + queue_state_before = tag_slot->queue_state_sequence; + tag_slot->outstanding_ticket_count = SIZE_MAX; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, SIZE_MAX); + UT_ASSERT_EQ(tag_slot->queue_state_sequence, queue_state_before); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 2); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &second_request.prehandle, &found), + PCM_X_DIRECTORY_OK); + ticket = &master_ticket_slots(header)[found.slot_index]; + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_RESERVED_NONVISIBLE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Existing-tag allocator exhaustion is retryable and releases its gate. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].free_head = PCM_X_INVALID_SLOT_INDEX; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].used + = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; + header->allocator[PCM_X_ALLOC_MASTER_TICKET].high_water + = header->layout.pools[PCM_X_POOL_MASTER_TICKET].capacity; + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(tag_slot->next_admission_sequence, 2); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->peer_frontiers[1].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Existing-tag directory failure rolls its reserved ticket back exactly. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_enqueue_peer(&first_request); + bind_enqueue_peer(&second_request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&first_request, &first), PCM_X_QUEUE_OK); + tag_slot = &master_tag_slots(header)[first.tag_slot.slot_index]; + { + PcmXDirectoryEntry *entries; + Size capacity; + Size i; + uint64 second_hash; + + entries = directory_entries(header, PCM_X_DIR_MASTER_TICKET_PREHANDLE, &capacity); + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_MASTER_TICKET_PREHANDLE, + &second_request.prehandle, &second_hash)); + for (i = 0; i < capacity; i++) { + if (entries[i].state == PCM_X_DIRECTORY_OCCUPIED) + continue; + entries[i].state = PCM_X_DIRECTORY_OCCUPIED; + entries[i].key_hash = second_hash ^ UINT64_C(1); + entries[i].slot_index = first.ticket_slot.slot_index; + entries[i].slot_generation = first.ticket_slot.slot_generation; + } + } + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&second_request, &second), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 1); + UT_ASSERT_EQ(tag_slot->next_admission_sequence, 2); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 1); + UT_ASSERT_EQ(header->next_ticket_id, 2); + UT_ASSERT_EQ(header->peer_frontiers[1].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_master_detach_second_caller_is_stale_not_fail_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXMasterTicketSlot *ticket; + PcmXEnqueuePayload request; + PcmXSlotRef found; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request + = make_enqueue(make_wait_identity(732, 0, 2, UINT64_C(38001)), UINT64_C(3901), UINT64_C(1)); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + ticket = &master_ticket_slots(header)[admission.ticket_slot.slot_index]; + detach_interlock_tag = tag_slot; + detach_interlock_ticket = ticket; + detach_interlock_armed = true; + UT_ASSERT_EQ(cluster_pcm_x_master_detach_terminal_exact(&admission.ref), PCM_X_QUEUE_STALE); + UT_ASSERT(!detach_interlock_armed); + UT_ASSERT_EQ(test_slot_state(&ticket->slot), PCM_XT_DETACHING); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(tag_slot->outstanding_ticket_count, 0); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TICKET_HANDLE, &admission.ref.handle, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_composite_join_publishes_one_leader_and_ordered_followers) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[3]; + PcmXLocalHandle duplicate; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *members; + PcmXSlotRef found; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, 0, UINT64_C(177)); + for (i = 0; i < 3; i++) { + PcmXWaitIdentity identity + = make_wait_identity(706, 0, (uint32)(17 + i), UINT64_C(7001) + i); + + identity.cluster_epoch = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &handles[i]), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(handles[i].local_sequence, (uint64)i + 1); + UT_ASSERT_EQ(handles[i].local_round, 1); + UT_ASSERT_EQ(handles[i].role, + i == 0 ? PCM_X_LOCAL_ROLE_NODE_LEADER : PCM_X_LOCAL_ROLE_FOLLOWER); + } + UT_ASSERT(slot_refs_equal(handles[0].tag_slot, handles[1].tag_slot)); + UT_ASSERT(slot_refs_equal(handles[0].tag_slot, handles[2].tag_slot)); + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_LIVE); + UT_ASSERT_EQ(tag_slot->head_index, handles[0].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->tail_index, handles[2].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->leader_index, handles[0].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->membership_count, 3); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 0); + UT_ASSERT_EQ(test_slot_state(&members[handles[0].membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(test_slot_state(&members[handles[1].membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(members[handles[0].membership_slot.slot_index].next_index, + handles[1].membership_slot.slot_index); + UT_ASSERT_EQ(members[handles[1].membership_slot.slot_index].next_index, + handles[2].membership_slot.slot_index); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 3); + + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &handles[0].identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, handles[0].tag_slot)); + for (i = 0; i < 3; i++) { + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &handles[i].identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, handles[i].membership_slot)); + } + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&handles[1].identity, 1, UINT64_C(177), &duplicate), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(slot_refs_equal(duplicate.membership_slot, handles[1].membership_slot)); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 3); +} + +UT_TEST(test_local_follower_wfg_publish_is_nonwaitable_then_exact) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalFollowerWfgSnapshot snapshot; + PcmXLocalFollowerWfgSnapshot stale; + PcmXLocalMembershipSlot *members; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, 0, UINT64_C(177)); + identity = make_wait_identity(806, 0, 17, UINT64_C(80601)); + identity.cluster_epoch = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(806, 0, 18, UINT64_C(80602)); + identity.cluster_epoch = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + members = membership_slots(ClusterPcmXConvertShmem); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&snapshot.waiter, &follower, sizeof(follower)) == 0); + UT_ASSERT(memcmp(&snapshot.blocker, &leader.identity, sizeof(leader.identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(901)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_WAITABLE_FOLLOWER); + UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, UINT64_C(901)); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(901)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(902)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(901)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(901)), + PCM_X_QUEUE_DUPLICATE); + + stale = snapshot; + stale.leader_slot_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&stale, UINT64_C(902)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(members[follower.membership_slot.slot_index].graph_generation, 0); +} + +UT_TEST(test_local_writer_claim_runs_closed_cohort_and_blocks_next_round) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle late; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalWriterClaim claim; + PcmXLocalWriterClaim released; + PcmXLocalWriterClaim stale; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXWaitIdentity successor; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(807, 0, 19, UINT64_C(80701)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(807, 0, 20, UINT64_C(80702)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(903)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_writer_slot_generation, leader.membership_slot.slot_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &stale), PCM_X_QUEUE_BUSY); + + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(807, 0, 21, UINT64_C(80703)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late.local_round, cutoff.next_round); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(909)), + PCM_X_QUEUE_OK); + released = claim; + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &follower.identity, sizeof(successor)) == 0); + UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(903)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_OK); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&late, &claim), PCM_X_QUEUE_BARRIER_CLOSED); + stale = released; + stale.claim_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&stale), PCM_X_QUEUE_STALE); +} + +UT_TEST(test_local_writer_claim_completion_is_fifo_and_one_shot) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower1; + PcmXLocalHandle follower2; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalWriterClaim aborted; + PcmXLocalWriterClaim busy; + PcmXLocalWriterClaim claim; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXWaitIdentity successor; + uint32 state_flags; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(809, 0, 24, UINT64_C(80901)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(809, 0, 25, UINT64_C(80902)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower1), + PCM_X_QUEUE_OK); + identity = make_wait_identity(809, 0, 26, UINT64_C(80903)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower2), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower1, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(904)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(905)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &busy), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + aborted = claim; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT(claim.claim_generation > aborted.claim_generation); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&aborted), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &busy), PCM_X_QUEUE_BUSY); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_BUSY); + UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ( + test_slot_flags( + &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index].slot) + & PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE, + 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + memset(&successor, 0, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &follower1.identity, sizeof(successor)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(905)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &busy), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower1, UINT64_C(904)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&wfg.blocker, &follower1.identity, sizeof(follower1.identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(906)), + PCM_X_QUEUE_OK); + memset(&successor, 0, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &follower2.identity, sizeof(successor)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower1, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower1, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(906)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower1, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower1), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &claim), PCM_X_QUEUE_OK); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor, &(PcmXWaitIdentity){ 0 }, sizeof(successor)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower2, &claim), PCM_X_QUEUE_BAD_STATE); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower2, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower2), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); +} + +UT_TEST(test_writer_and_holder_owner_exit_retry_preserves_exact_evidence) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHandle leader; + PcmXLocalWriterClaim claim; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot tag_before; + PcmXLocalMembershipSlot holder_before; + PcmXLocalMembershipSlot writer_before; + PcmXWaitIdentity identity; + PcmXWaitIdentity successor; + LWLock content_lock; + uint32 state_flags; + Size holder_used_before; + const uint64 master_session = UINT64_C(8177); + + init_active_pcm_x(UINT64_C(177)); + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(811, 0, 27, UINT64_C(81101), 1); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(811, 0, 28, UINT64_C(81102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + holder_used_before = ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; + memset(&content_lock, 0, sizeof(content_lock)); + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + memset(&successor, 0x7f, sizeof(successor)); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + PCM_X_QUEUE_GATE_RETRY); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder.holder_slot.slot_index); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, + holder_used_before); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, + holder_used_before - 1); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + + /* A different runtime may never consume either exact ledger. Both APIs + * preserve byte-identical shared evidence for deferred recovery. */ + init_active_pcm_x(UINT64_C(178)); + bind_local_master(1, UINT64_C(9), master_session + 1); + holder_key = make_local_holder_key(812, 0, 27, UINT64_C(81201), 1); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + identity = make_wait_identity(812, 0, 28, UINT64_C(81202)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 1, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + tag_before = *tag_slot; + holder_before = membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index]; + writer_before = membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index]; + cluster_pcm_x_runtime_fail_closed(); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_collect_exact(&claim, &successor), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_exceptional_detach_exact(&holder, &content_lock), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(&tag_before, tag_slot, sizeof(tag_before)) == 0); + UT_ASSERT(memcmp(&holder_before, + &membership_slots(ClusterPcmXConvertShmem)[holder.holder_slot.slot_index], + sizeof(holder_before)) + == 0); + UT_ASSERT(memcmp(&writer_before, + &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index], + sizeof(writer_before)) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_wfg_rejects_completed_blocker_semantic_aba) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalFollowerWfgSnapshot snapshot; + PcmXLocalWriterClaim claim; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(810, 0, 27, UINT64_C(81001)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(810, 0, 28, UINT64_C(81002)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&snapshot, UINT64_C(906)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &snapshot), + PCM_X_QUEUE_BUSY); +} + +UT_TEST(test_local_cancel_never_unlinks_an_active_writer) +{ + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle follower2; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalWriterClaim claim; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(811, 0, 4, UINT64_C(81101)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(811, 0, 5, UINT64_C(81102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(907)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(907)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + + /* ABORT clears active_writer without setting WRITER_COMPLETE. A later + * follower can still hold an exact edge to that former active writer, so + * cancellation must retain the slot until that edge is removed. */ + init_active_pcm_x(UINT64_C(78)); + bind_local_master(1, UINT64_C(9), UINT64_C(178)); + identity = make_wait_identity(812, 0, 6, UINT64_C(81201)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(812, 0, 7, UINT64_C(81202)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &follower), + PCM_X_QUEUE_OK); + identity = make_wait_identity(812, 0, 8, UINT64_C(81203)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &follower2), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(917)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(918)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(917)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(918)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower2, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&wfg.blocker, &follower.identity, sizeof(follower.identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(919)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower2, UINT64_C(919)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower2, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower2), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +UT_TEST(test_local_closed_round_never_promotes_a_late_joiner_early) +{ + PcmXLocalHandle leader; + PcmXLocalHandle late1; + PcmXLocalHandle late2; + PcmXLocalCutoff cutoff; + PcmXLocalMembershipSlot *members; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(808, 0, 21, UINT64_C(80801)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(808, 0, 22, UINT64_C(80802)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late1.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + + identity = make_wait_identity(808, 0, 23, UINT64_C(80803)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &late2), + PCM_X_QUEUE_OK); + members = membership_slots(ClusterPcmXConvertShmem); + UT_ASSERT_EQ(late2.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(test_slot_state(&members[late2.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(local_tag_slots(ClusterPcmXConvertShmem)[late2.tag_slot.slot_index].leader_index, + PCM_X_INVALID_SLOT_INDEX); +} + +UT_TEST(test_local_lookup_is_read_only_and_identity_exact) +{ + PcmXShmemHeader *header; + PcmXLocalHandle joined; + PcmXLocalHandle found; + PcmXLocalHandle cleared; + PcmXWaitIdentity identity; + PcmXWaitIdentity wrong; + Size tag_used; + Size wait_used; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(177)); + identity = make_wait_identity(706, 0, 27, UINT64_C(7027)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &joined), + PCM_X_QUEUE_OK); + tag_used = header->allocator[PCM_X_ALLOC_LOCAL_TAG].used; + wait_used = header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used; + + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&identity, &found), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&found, &joined, sizeof(found)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); + + wrong = identity; + wrong.request_id++; + memset(&found, 0xa5, sizeof(found)); + memset(&cleared, 0, sizeof(cleared)); + cleared.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + cleared.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&wrong, &found), PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT(memcmp(&found, &cleared, sizeof(found)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); +} + +UT_TEST(test_local_progress_is_exact_and_exposes_remote_wait_ledger) +{ + PcmXLocalHandle leader; + PcmXLocalHandle stale; + PcmXLocalProgress progress; + PcmXLocalProgress cleared; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload ack; + PcmXPhasePayload confirm; + PcmXLocalReliableToken token; + + init_active_pcm_x(UINT64_C(77)); + identity = make_wait_identity(706, 0, 28, UINT64_C(7028)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(177)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(177), &leader), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&progress.identity, &identity, sizeof(identity)) == 0); + UT_ASSERT_EQ(progress.local_sequence, leader.local_sequence); + UT_ASSERT_EQ(progress.local_round, leader.local_round); + UT_ASSERT_EQ(progress.member_state, PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(progress.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, 0); + + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&ack, 0, sizeof(ack)); + ack.ref.identity = identity; + ack.ref.handle.ticket_id = UINT64_C(9008); + ack.ref.handle.queue_generation = UINT64_C(8); + ack.prehandle = enqueue.prehandle; + ack.result = PCM_X_QUEUE_OK; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(177)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(177)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&progress.ref, &ack.ref)); + UT_ASSERT_EQ(progress.local_sequence, leader.local_sequence); + UT_ASSERT_EQ(progress.local_round, leader.local_round); + UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(progress.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(progress.reliable_state_sequence, 2); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK); + UT_ASSERT_EQ(progress.phase, PCM_X_LOCAL_RELIABLE_PHASE_NONE); + + stale = leader; + stale.local_sequence++; + memset(&progress, 0xa5, sizeof(progress)); + memset(&cleared, 0, sizeof(cleared)); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&stale, &progress), PCM_X_QUEUE_STALE); + UT_ASSERT(memcmp(&progress, &cleared, sizeof(progress)) == 0); +} + +UT_TEST(test_local_transfer_prepare_commit_and_final_ack_are_exact) +{ + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalProgress progress; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload admit_confirm; + PcmXLocalCutoff cutoff; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXTicketRef probe_ref; + PcmXGrantPayload prepare; + PcmXGrantPayload bad_prepare; + PcmXGrantPayload holder_replay; + PcmXRevokePayload revoke; + PcmXInstallReadyPayload install_ready; + PcmXPhasePayload commit; + PcmXFinalAckPayload final_ack; + PcmXPhasePayload final_commit; + PcmXPhasePayload final_confirm; + PcmXPhasePayload duplicate_confirm; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXLocalReliableToken token; + PcmXLocalReliableToken retry_token; + PcmXLocalWriterClaim writer_claim; + PcmXLocalWriterClaim stale_writer_claim; + PcmXLocalHolderKey writer_holder_key; + PcmXLocalHolderKey stale_holder_key; + PcmXLocalHolderHandle writer_holder; + PcmXLocalHolderHandle duplicate_holder; + uint64 committed_own_generation; + uint64 holder_set_generation; + Size holder_used; + const uint64 master_session = UINT64_C(178); + + init_active_pcm_x(UINT64_C(77)); + identity = make_wait_identity(707, 0, 27, UINT64_C(7029)); + bind_local_master(1, identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + identity = make_wait_identity(707, 0, 28, UINT64_C(7030)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&follower, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(908)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + writer_holder_key = make_local_holder_key(707, 0, 27, UINT64_C(7031), 3); + committed_own_generation = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(committed_own_generation, 0); + UT_ASSERT_EQ(writer_holder.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = leader.identity; + admit_ack.ref.handle.ticket_id = UINT64_C(9009); + admit_ack.ref.handle.queue_generation = UINT64_C(9); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &admit_ack, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &admit_confirm, 1, master_session), + PCM_X_QUEUE_OK); + + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(99); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(199), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(41); + prepare.image.page_scn = UINT64_C(42); + prepare.image.page_lsn = UINT64_C(43); + prepare.image.source_node = 0; + prepare.image.page_checksum = UINT32_C(44); + probe_ref = prepare.ref; + probe_ref.grant_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = prepare.ref; + revoke.image_id = prepare.image.image_id; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_image_ready_arm_exact(&prepare, 1, master_session, + &holder_replay), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&holder_replay, &prepare, sizeof(prepare)) == 0); + bad_prepare = prepare; + bad_prepare.image.image_id = UINT64CONST(0xe0000000000000c7); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &bad_prepare, 1, master_session), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); + UT_ASSERT(ticket_refs_equal(&progress.ref, &prepare.ref)); + UT_ASSERT(memcmp(&progress.image, &prepare.image, sizeof(progress.image)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&leader, &prepare.ref, &prepare.image, + &install_ready, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(install_ready.image_id, prepare.image.image_id); + UT_ASSERT_EQ(install_ready.phase, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_CONTENT_ACTIVE); + UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_INSTALL_READY); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1001), &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1001)); + memset(&commit, 0, sizeof(commit)); + commit.ref = prepare.ref; + commit.phase = PGRAC_IC_MSG_PCM_X_COMMIT_X; + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&leader, &commit, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_commit_x_exact(&leader, &commit, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + + UT_ASSERT_EQ(cluster_pcm_x_local_final_ack_arm_exact(&leader, 1, &final_ack, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_ack.image_id, prepare.image.image_id); + UT_ASSERT_EQ(final_ack.committed_own_generation, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_ACK); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1002), &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1002)); + memset(&final_commit, 0, sizeof(final_commit)); + final_commit.ref = prepare.ref; + final_commit.phase = PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK; + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact(&leader, &final_commit, 1, + master_session, &final_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(final_confirm.phase, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); + UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ(progress.phase, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_FINAL_COMMIT_ACK); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &token, UINT64_C(1003), &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, UINT64_C(1003)); + UT_ASSERT_EQ(local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index] + .committed_own_generation, + 1); + UT_ASSERT_EQ(cluster_pcm_x_local_final_commit_ack_exact( + &leader, &final_commit, 1, master_session, &duplicate_confirm, &token), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&duplicate_confirm, &final_confirm, sizeof(final_confirm)) == 0); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK) == 0); + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_FINAL_CONFIRM); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &prepare.ref)); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, master_session); + UT_ASSERT_EQ(tag_slot->cluster_epoch, identity.cluster_epoch); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].cluster_epoch, identity.cluster_epoch); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].sender_session_incarnation, + master_session); + UT_ASSERT_EQ(membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index] + .handle.ticket_id, + prepare.ref.handle.ticket_id); + UT_ASSERT_EQ( + test_slot_state( + &membership_slots(ClusterPcmXConvertShmem)[leader.membership_slot.slot_index].slot), + PCM_XL_GRANTED); + + writer_holder_key.identity.base_own_generation = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&writer_holder_key, &writer_holder), + PCM_X_QUEUE_BARRIER_CLOSED); + holder_used = ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; + holder_set_generation = tag_slot->holder_set_generation; + stale_writer_claim = writer_claim; + stale_writer_claim.claim_generation++; + committed_own_generation = UINT64_MAX; + UT_ASSERT_EQ( + cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &stale_writer_claim, + &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(committed_own_generation, 0); + stale_writer_claim = writer_claim; + stale_writer_claim.local_round++; + UT_ASSERT_EQ( + cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &stale_writer_claim, + &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key = writer_holder_key; + stale_holder_key.identity.base_own_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key.identity.base_own_generation = 2; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key = writer_holder_key; + stale_holder_key.identity.procno++; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + stale_holder_key = writer_holder_key; + stale_holder_key.identity.xid++; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &stale_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, holder_used); + UT_ASSERT_EQ(tag_slot->holder_set_generation, holder_set_generation); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(committed_own_generation, 1); + stale_writer_claim = writer_claim; + stale_writer_claim.claim_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &stale_writer_claim, &duplicate_holder, &committed_own_generation), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact(&writer_holder_key, &writer_claim, + &duplicate_holder, + &committed_own_generation), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&duplicate_holder, &writer_holder, sizeof(writer_holder)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&writer_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&writer_holder), PCM_X_QUEUE_OK); + + memset(&poll, 0, sizeof(poll)); + poll.ref = prepare.ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->active_writer_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&follower, UINT64_C(908)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&follower, &writer_claim), PCM_X_QUEUE_OK); + writer_holder_key = make_local_holder_key(707, 0, 28, UINT64_C(7032), 4); + writer_holder_key.identity.base_own_generation = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_writer_holder_register_exact( + &writer_holder_key, &writer_claim, &writer_holder, &committed_own_generation), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(committed_own_generation, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_activate_exact(&writer_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&writer_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_GRANTED); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_DRAIN_POLL); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = prepare.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&leader, &progress), PCM_X_QUEUE_NOT_FOUND); + assert_local_queue_baseline(ClusterPcmXConvertShmem); +} + +UT_TEST(test_local_tag_only_holder_transfer_persists_until_exact_drain) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey new_holder_key; + PcmXLocalHolderHandle new_holder; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXBlockerSetHeaderPayload blocker_header; + PcmXEnqueuePayload enqueue; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalCutoff cutoff; + PcmXLocalCutoff promoted_cutoff; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalReliableToken token; + PcmXLocalWriterClaim writer_claim; + PcmXTicketRef probe_ref; + PcmXLocalHolderProgress progress; + PcmXRevokePayload revoke; + PcmXRevokePayload bad_revoke; + PcmXGrantPayload ready; + PcmXGrantPayload replay; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXWaitIdentity identity; + PcmXWaitIdentity late_identity; + PcmXWaitIdentity wake_items[1]; + PcmXLocalWakeBatch wake_batch; + const uint64 master_session = UINT64_C(179); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(708, 2, 28, UINT64_C(7030)); + bind_local_master(1, identity.cluster_epoch, master_session); + memset(&revoke, 0, sizeof(revoke)); + revoke.ref.identity = identity; + revoke.ref.handle.ticket_id = UINT64_C(9010); + revoke.ref.handle.queue_generation = UINT64_C(10); + revoke.ref.grant_generation = UINT64_C(100); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(200), &revoke.image_id)); + probe_ref = revoke.ref; + probe_ref.grant_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&identity.tag, NULL, 0, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.tag_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &blocker_header, + NULL, 0), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_header.nblockers, 0); + UT_ASSERT_EQ(blocker_header.set_crc32c, blocker_set_crc32c(NULL, 0)); + late_identity = make_wait_identity(708, 0, 6, UINT64_C(7031)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&late_identity, 1, master_session, &late), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(late.local_round, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &identity.tag, identity.cluster_epoch, 1, master_session, &cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cutoff.cutoff_sequence, 0); + UT_ASSERT_EQ(cutoff.closed_round, 1); + UT_ASSERT_EQ(cutoff.next_round, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), + PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + bad_revoke = revoke; + bad_revoke.image_id = UINT64CONST(0xf0000000000000c8); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&bad_revoke, 1, master_session), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + /* The exact blocker ACK tombstone is durable evidence, but it cannot + * reopen the writer round before REVOKE/IMAGE_READY/DRAIN completes. */ + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&progress.ref, &revoke.ref)); + UT_ASSERT_EQ(progress.image.image_id, revoke.image_id); + UT_ASSERT_EQ(progress.master_session_incarnation, master_session); + UT_ASSERT_EQ(progress.master_node, 1); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_REVOKE); + + memset(&ready, 0, sizeof(ready)); + ready.ref = revoke.ref; + ready.image.image_id = revoke.image_id; + ready.image.source_own_generation = 0; + ready.image.page_scn = UINT64_C(52); + ready.image.page_lsn = UINT64_C(53); + ready.image.source_node = 0; + ready.image.page_checksum = UINT32_C(54); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); + memset(&replay, 0, sizeof(replay)); + UT_ASSERT_EQ( + cluster_pcm_x_local_holder_image_ready_arm_exact(&ready, 1, master_session, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&ready, &replay, sizeof(ready)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.pending_opcode, PGRAC_IC_MSG_PCM_X_IMAGE_READY); + UT_ASSERT_EQ(progress.image.source_own_generation, 0); + UT_ASSERT_EQ(progress.flags, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + + memset(&poll, 0, sizeof(poll)); + poll.ref = revoke.ref; + poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&late.identity, &next_leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(next_leader.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT(slot_refs_equal(next_leader.membership_slot, late.membership_slot)); + /* Promotion is visible for exact lookup, but holder cleanup is not complete + * until RETIRE consumes the terminal image lane. No timed wake or unrelated + * latch may let this successor claim, refreeze, or mint ENQUEUE early. */ + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&next_leader, &writer_claim), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&next_leader, &promoted_cutoff), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&next_leader, &enqueue, &token), + PCM_X_QUEUE_BUSY); + new_holder_key = make_local_holder_key(708, 0, 3, UINT64_C(7032), 3); + UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, PGRAC_IC_MSG_PCM_X_DRAIN_POLL); + UT_ASSERT_EQ(progress.image.source_own_generation, 0); + UT_ASSERT_EQ(progress.flags, PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = revoke.ref.handle.ticket_id; + retire.sender_node = 0; + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = lengthof(wake_items); + wake_batch.count = 0; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &next_leader.identity, sizeof(wake_items[0])) == 0); + /* A visible wake batch proves the full watermark and global admission gate + * committed before the caller can make the successor runnable. */ + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retired_ticket_id, + retire.retire_through_ticket_id); + wake_batch.count = 1; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, master_session, &wake_batch), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(wake_batch.count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_progress_exact(&identity.tag, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&next_leader, &writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&writer_claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&next_leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&next_leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_non_source_blocker_participant_drains_and_retires_exactly) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_keys[2]; + PcmXLocalHolderKey new_holder_key; + PcmXLocalHolderHandle holders[2]; + PcmXLocalHolderHandle new_holder; + PcmXLocalHolderHandle holder_copies[2]; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXDrainPollPayload poll; + PcmXDrainPollPayload stale_poll; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1801); + int i; + + /* Model the local evidence left on a non-source participant: two independent + * S pins are captured by PROBE, both are later released, and no holder_ref + * image-source lane is installed. DRAIN must therefore close from the exact + * type-48 blocker ACK tombstone alone. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + for (i = 0; i < 2; i++) { + holder_keys[i] + = make_local_holder_key(7091, 0, (uint32)(21 + i), UINT64_C(71001) + i, 4 + i); + UT_ASSERT_EQ(register_active_local_holder(&holder_keys[i], &holders[i]), PCM_X_QUEUE_OK); + } + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7091, 3, 23, UINT64_C(71003)); + probe_ref.handle.ticket_id = UINT64_C(91001); + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, holder_copies, 2, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, holder_copies, 2, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, + probe_ref.identity.cluster_epoch, + 1, master_session, &cutoff), + PCM_X_QUEUE_OK); + for (i = 0; i < 2; i++) + UT_ASSERT_EQ(release_active_local_holder(&holders[i]), PCM_X_QUEUE_OK); + + tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &probe_ref)); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_reliable.last_response_opcode, + PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK); + + memset(&poll, 0, sizeof(poll)); + poll.ref = probe_ref; + poll.ref.grant_generation = UINT64_C(101); + poll.drain_generation = UINT64_C(42); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(ticket_refs_equal(&tag_slot->holder_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &poll.ref)); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, poll.drain_generation); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + + stale_poll = poll; + stale_poll.ref.grant_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + PCM_X_QUEUE_STALE); + stale_poll = poll; + stale_poll.drain_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + PCM_X_QUEUE_STALE); + + new_holder_key = make_local_holder_key(7091, 0, 24, UINT64_C(71004), 6); + UT_ASSERT_EQ(register_active_local_holder(&new_holder_key, &new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = probe_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(release_active_local_holder(&new_holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalCutoff cutoff; + PcmXTicketRef probe_ref; + PcmXDrainPollPayload poll; + PcmXDrainPollPayload stale_poll; + PcmXRetirePayload retire; + PcmXSlotRef found; + const uint64 master_session = UINT64_C(1807); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(7107, 0, 21, UINT64_C(71101), 4); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7107, 3, 23, UINT64_C(71103)); + probe_ref.handle.ticket_id = UINT64_C(91101); + probe_ref.handle.queue_generation = UINT64_C(12); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&probe_ref.identity.tag, + probe_ref.identity.cluster_epoch, + 1, master_session, &cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + + memset(&poll, 0, sizeof(poll)); + poll.ref = probe_ref; + poll.drain_generation = UINT64_C(43); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + stale_poll = poll; + stale_poll.ref.grant_generation = UINT64_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + PCM_X_QUEUE_STALE); + stale_poll = poll; + stale_poll.drain_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, master_session), + PCM_X_QUEUE_STALE); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = probe_ref.identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_cancelled_participant_gen0_drain_requires_frozen_round) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef blocker_ref; + PcmXDrainPollPayload poll; + PcmXReliableLegState reliable_before; + uint64 set_generation; + uint32 state_flags; + const uint64 master_session = UINT64_C(1812); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(7112, 0, 21, UINT64_C(71301), 4); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + memset(&blocker_ref, 0, sizeof(blocker_ref)); + blocker_ref.identity = make_wait_identity(7112, 3, 23, UINT64_C(71303)); + blocker_ref.handle.ticket_id = UINT64_C(91112); + blocker_ref.handle.queue_generation = UINT64_C(14); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&blocker_ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&blocker_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &blocker_ref, blocker_snapshot.set_generation, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[holder_snapshot.tag_slot.slot_index]; + set_generation = tag_slot->blocker_set_generation; + reliable_before = tag_slot->blocker_snapshot_reliable; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags + & ~(PCM_X_LOCAL_TAG_F_REVOKE_BARRIER << PCM_X_SLOT_FLAGS_SHIFT)); + memset(&poll, 0, sizeof(poll)); + poll.ref = blocker_ref; + poll.drain_generation = UINT64_C(45); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &blocker_ref)); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, set_generation); + UT_ASSERT( + memcmp(&tag_slot->blocker_snapshot_reliable, &reliable_before, sizeof(reliable_before)) + == 0); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_holder_drain_validates_frozen_round_before_terminal_publish) +{ + PcmXLocalHandle leader; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot tag_after; + PcmXLocalTagSlot tag_before; + PcmXDrainPollPayload poll; + PcmXWaitIdentity identity; + uint32 before_state_flags; + uint32 state_flags; + const uint64 master_session = UINT64_C(1813); + + /* A holder DRAIN can arrive while an unrelated local writer cohort is still + * closed. Its first terminal publication must validate the common revoke + * barrier even though closed_round_member_count is nonzero. */ + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), master_session); + identity = make_wait_identity(7113, 0, 21, UINT64_C(71401)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + seed_live_local_holder_transfer(tag_slot, 22, UINT64_C(71402), UINT64_C(91402)); + memset(&poll, 0, sizeof(poll)); + poll.ref = tag_slot->holder_ref; + poll.drain_generation = UINT64_C(46); + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags + & ~(PCM_X_LOCAL_TAG_F_REVOKE_BARRIER << PCM_X_SLOT_FLAGS_SHIFT)); + tag_before = *tag_slot; + before_state_flags = pg_atomic_read_u32(&tag_before.slot.state_flags); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->slot.state_flags), + before_state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + tag_after = *tag_slot; + pg_atomic_write_u32(&tag_after.slot.state_flags, before_state_flags); + UT_ASSERT(memcmp(&tag_after, &tag_before, sizeof(tag_after)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* A torn first-arrival drain generation is equally structural and must not + * be overwritten merely because a writer remains in the frozen cohort. */ + init_active_pcm_x(UINT64_C(78)); + bind_local_master(1, UINT64_C(9), master_session + 1); + identity = make_wait_identity(7114, 0, 21, UINT64_C(71403)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 1, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + seed_live_local_holder_transfer(tag_slot, 22, UINT64_C(71404), UINT64_C(91404)); + tag_slot->holder_terminal_drain_generation = UINT64_C(45); + memset(&poll, 0, sizeof(poll)); + poll.ref = tag_slot->holder_ref; + poll.drain_generation = UINT64_C(46); + tag_before = *tag_slot; + before_state_flags = pg_atomic_read_u32(&tag_before.slot.state_flags); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->slot.state_flags), + before_state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + tag_after = *tag_slot; + pg_atomic_write_u32(&tag_after.slot.state_flags, before_state_flags); + UT_ASSERT(memcmp(&tag_after, &tag_before, sizeof(tag_after)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_same_ref_non_source_participant_retires_both_legs) +{ + TestLocalParticipantTransfer fixture; + PcmXRetirePayload retire; + PcmXLocalProgress progress; + PcmXLocalTagSlot *tag_slot; + + prepare_local_non_source_participant_transfer(7092, UINT64_C(1802), &fixture); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&fixture.poll, 1, fixture.master_session), + PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, fixture.master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.leader, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + assert_local_queue_baseline(ClusterPcmXConvertShmem); + + /* A participant form cannot claim that this requester was also the image + * source while omitting the source holder_ref lane. */ + prepare_local_non_source_participant_transfer(7093, UINT64_C(1803), &fixture); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[fixture.leader.tag_slot.slot_index]; + tag_slot->image.source_node = (uint32)fixture.poll.ref.identity.node_id; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&fixture.poll, 1, fixture.master_session), + PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, fixture.master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +UT_TEST(test_local_successful_retire_clears_round_image_before_promoting_successor) +{ + TestLocalParticipantTransfer fixture; + PcmXLocalHandle late; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalProgress progress; + PcmXLocalTagSlot *tag_slot; + PcmXRetirePayload retire; + PcmXWaitIdentity late_identity; + PcmXWaitIdentity wake_items[1]; + PcmXLocalWakeBatch wake_batch; + PcmXImageToken zero_image; + + prepare_local_non_source_participant_transfer(7098, UINT64_C(1808), &fixture); + late_identity = make_wait_identity(7098, 0, 4, UINT64_C(71808)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&late_identity, 1, fixture.master_session, &late), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(late.local_round, fixture.cutoff.next_round); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[late.tag_slot.slot_index]; + UT_ASSERT(tag_slot->image.image_id != 0); + UT_ASSERT(tag_slot->committed_own_generation != 0); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&fixture.poll, 1, fixture.master_session), + PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = lengthof(wake_items); + wake_batch.count = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, fixture.master_session, + &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &late.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&late.identity, &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(promoted.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + memset(&zero_image, 0, sizeof(zero_image)); + UT_ASSERT(memcmp(&tag_slot->image, &zero_image, sizeof(zero_image)) == 0); + UT_ASSERT_EQ(tag_slot->committed_own_generation, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, promoted.identity.base_own_generation + 1, &rekeyed), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&rekeyed, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.pending_opcode, 0); + UT_ASSERT_EQ(progress.last_response_opcode, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +typedef struct TestLocalCancelledDualTransfer { + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle promoted; + PcmXLocalCutoff cutoff; + PcmXDrainPollPayload poll; + uint64 master_session; +} TestLocalCancelledDualTransfer; + +static void +prepare_local_same_ref_cancelled_participant(BlockNumber block, uint64 master_session, + bool with_follower, + TestLocalCancelledDualTransfer *fixture) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity follower_identity; + PcmXWaitIdentity leader_identity; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload admit_confirm; + PcmXPhasePayload cancel; + PcmXLocalReliableToken token; + PcmXLocalWriterClaim writer_claim; + PcmXImageToken zero_image = { 0 }; + + UT_ASSERT_NOT_NULL(fixture); + memset(fixture, 0, sizeof(*fixture)); + fixture->master_session = master_session; + init_active_pcm_x(UINT64_C(77)); + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(block, 0, 21, master_session + UINT64_C(1), 4); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + leader_identity = make_wait_identity(block, 0, 23, master_session + UINT64_C(2)); + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&leader_identity, 1, master_session, &fixture->leader), + PCM_X_QUEUE_OK); + if (with_follower) { + follower_identity = make_wait_identity(block, 0, 24, master_session + UINT64_C(3)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 1, master_session, + &fixture->follower), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&fixture->leader, &writer_claim), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&fixture->leader, &fixture->cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&fixture->leader, &enqueue, &token), + PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = leader_identity; + admit_ack.ref.handle.ticket_id = master_session + UINT64_C(100); + admit_ack.ref.handle.queue_generation = UINT64_C(13); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ( + cluster_pcm_x_local_apply_admit_ack_exact(&fixture->leader, &admit_ack, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&fixture->leader, &admit_confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&fixture->leader, &admit_confirm, 1, + master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&admit_ack.ref, 1, master_session, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&admit_ack.ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact( + &admit_ack.ref, blocker_snapshot.set_generation, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_release_exact(&writer_claim), PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_arm_exact(&fixture->leader, &cancel, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_ack_exact(&fixture->leader, &cancel, 1, master_session, + with_follower ? &fixture->promoted : NULL), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[fixture->leader.tag_slot.slot_index]; + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &cancel.ref)); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &cancel.ref)); + UT_ASSERT(memcmp(&tag_slot->image, &zero_image, sizeof(zero_image)) == 0); + + fixture->poll.ref = cancel.ref; + fixture->poll.drain_generation = UINT64_C(44); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&fixture->poll, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&fixture->poll, 1, master_session), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_TERMINAL_MASK); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); +} + +UT_TEST(test_local_same_ref_cancelled_participant_gen0_retires_both_legs) +{ + TestLocalCancelledDualTransfer fixture; + PcmXShmemHeader *header; + PcmXLocalProgress progress; + PcmXDrainPollPayload stale_poll; + PcmXRetirePayload retire; + + prepare_local_same_ref_cancelled_participant(7108, UINT64_C(1808), false, &fixture); + header = ClusterPcmXConvertShmem; + stale_poll = fixture.poll; + stale_poll.ref.grant_generation = UINT64_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&stale_poll, 1, fixture.master_session), + PCM_X_QUEUE_STALE); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, fixture.master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.leader, &progress), + PCM_X_QUEUE_NOT_FOUND); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_final_closed_member_retire_promotes_late_round) +{ + TestLocalCancelledDualTransfer fixture; + PcmXLocalHandle late; + PcmXLocalHandle promoted; + PcmXLocalProgress progress; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot tag_before; + PcmXLocalMembershipSlot *late_member; + PcmXLocalMembershipSlot *terminal_member; + PcmXLocalMembershipSlot late_before; + PcmXLocalMembershipSlot terminal_before; + PcmXWaitIdentity late_identity; + PcmXEnqueuePayload enqueue; + PcmXLocalReliableToken token; + PcmXRetirePayload retire; + PcmXWaitIdentity wake_items[1]; + PcmXLocalWakeBatch wake_batch; + + prepare_local_same_ref_cancelled_participant(7115, UINT64_C(1815), false, &fixture); + late_identity = make_wait_identity(7115, 0, 25, UINT64_C(71505)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&late_identity, 1, fixture.master_session, &late), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(late.local_round, fixture.cutoff.next_round); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[late.tag_slot.slot_index]; + late_member = &membership_slots(ClusterPcmXConvertShmem)[late.membership_slot.slot_index]; + terminal_member + = &membership_slots(ClusterPcmXConvertShmem)[fixture.leader.membership_slot.slot_index]; + late_member->graph_generation = UINT64_C(71506); + test_set_slot_state(&late_member->slot, PCM_XL_WAITABLE_FOLLOWER); + tag_before = *tag_slot; + late_before = *late_member; + terminal_before = *terminal_member; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, fixture.master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(late_member, &late_before, sizeof(*late_member)) == 0); + UT_ASSERT(memcmp(terminal_member, &terminal_before, sizeof(*terminal_member)) == 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&ClusterPcmXConvertShmem->local_retire_gate), 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + late_member->graph_generation = 0; + test_set_slot_state(&late_member->slot, PCM_XL_JOINED_NONWAITABLE); + /* The first pass must reject an undersized process-local wake batch before + * consuming either terminal leg. A caller can then retry the same exact + * watermark with sufficient capacity. */ + tag_before = *tag_slot; + late_before = *late_member; + terminal_before = *terminal_member; + wake_batch.items = NULL; + wake_batch.capacity = 0; + wake_batch.count = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, fixture.master_session, + &wake_batch), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(wake_batch.count, 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT(memcmp(late_member, &late_before, sizeof(*late_member)) == 0); + UT_ASSERT(memcmp(terminal_member, &terminal_before, sizeof(*terminal_member)) == 0); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&ClusterPcmXConvertShmem->local_retire_gate), 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = lengthof(wake_items); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, fixture.master_session, + &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &late.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&ClusterPcmXConvertShmem->local_retire_gate), 0); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.leader, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&late.identity, &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(promoted.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, late.membership_slot)); + UT_ASSERT_EQ(tag_slot->local_round, fixture.cutoff.next_round); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&promoted, &enqueue, &token), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_cancelled_dual_retires_one_writer_from_multiwriter_round) +{ + TestLocalCancelledDualTransfer fixture; + PcmXShmemHeader *header; + PcmXLocalProgress progress; + PcmXLocalTagSlot *tag_slot; + PcmXEnqueuePayload successor_enqueue; + PcmXLocalReliableToken token; + PcmXRetirePayload retire; + PcmXWaitIdentity wake_items[1]; + PcmXLocalWakeBatch wake_batch; + + prepare_local_same_ref_cancelled_participant(7111, UINT64_C(1811), true, &fixture); + header = ClusterPcmXConvertShmem; + tag_slot = &local_tag_slots(header)[fixture.leader.tag_slot.slot_index]; + UT_ASSERT(slot_refs_equal(fixture.promoted.membership_slot, fixture.follower.membership_slot)); + UT_ASSERT_EQ(tag_slot->membership_count, 2); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 2); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = lengthof(wake_items); + wake_batch.count = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, fixture.master_session, + &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &fixture.promoted.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.leader, &progress), + PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&fixture.promoted, &progress), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(progress.member_state, PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &(PcmXTicketRef){ 0 })); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ( + cluster_pcm_x_local_enqueue_arm_exact(&fixture.promoted, &successor_enqueue, &token), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&successor_enqueue.identity, &fixture.promoted.identity, + sizeof(successor_enqueue.identity)) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_cancelled_dual_retire_validates_before_consuming_holder_evidence) +{ + TestLocalCancelledDualTransfer fixture; + PcmXLocalMembershipSlot *member; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef blocker_ref; + PcmXRetirePayload retire; + uint64 holder_drain_generation; + + prepare_local_same_ref_cancelled_participant(7109, UINT64_C(1809), false, &fixture); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[fixture.leader.tag_slot.slot_index]; + member = &membership_slots(ClusterPcmXConvertShmem)[fixture.leader.membership_slot.slot_index]; + blocker_ref = tag_slot->blocker_snapshot_ref; + holder_drain_generation = tag_slot->holder_terminal_drain_generation; + test_set_slot_state(&member->slot, PCM_XL_GRANTED); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, fixture.master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &blocker_ref)); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, holder_drain_generation); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* A torn admitted-round locator is rejected before either terminal leg is + * consumed, even though every flag/ref/drain field remains otherwise exact. */ + prepare_local_same_ref_cancelled_participant(7110, UINT64_C(1810), false, &fixture); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[fixture.leader.tag_slot.slot_index]; + member = &membership_slots(ClusterPcmXConvertShmem)[fixture.leader.membership_slot.slot_index]; + blocker_ref = tag_slot->blocker_snapshot_ref; + holder_drain_generation = tag_slot->holder_terminal_drain_generation; + member->admitted_round++; + retire.cluster_epoch = fixture.poll.ref.identity.cluster_epoch; + retire.master_session_incarnation = fixture.master_session; + retire.retire_through_ticket_id = fixture.poll.ref.handle.ticket_id; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, fixture.master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT(ticket_refs_equal(&tag_slot->blocker_snapshot_ref, &blocker_ref)); + UT_ASSERT_EQ(tag_slot->holder_terminal_drain_generation, holder_drain_generation); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK, + PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_MASK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_source_holder_rejects_parallel_blocker_terminal_lane) +{ + PcmXLocalCutoff cutoff; + PcmXLocalHandle late; + PcmXLocalTagSlot *tag_slot; + PcmXDrainPollPayload poll; + PcmXTicketRef blocker_ref; + const uint64 master_session = UINT64_C(1804); + + tag_slot = prepare_retire_ready_round_with_holder(master_session, 7094, &cutoff, &late); + memset(&poll, 0, sizeof(poll)); + poll.ref = tag_slot->holder_ref; + poll.drain_generation = tag_slot->holder_terminal_drain_generation; + tag_slot->blocker_snapshot_ref = tag_slot->holder_ref; + tag_slot->blocker_snapshot_ref.grant_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* The XOR invariant precedes locator classification. If the source lane + * names A while a corrupt blocker lane names B, B's DRAIN must not hide the + * dual-lane shape behind STALE merely because holder_ref != poll.ref. */ + tag_slot = prepare_retire_ready_round_with_holder(master_session + 1, 7106, &cutoff, &late); + blocker_ref = tag_slot->holder_ref; + blocker_ref.identity.procno++; + blocker_ref.identity.request_id++; + blocker_ref.identity.wait_seq++; + blocker_ref.handle.ticket_id++; + blocker_ref.handle.queue_generation++; + blocker_ref.grant_generation = 0; + tag_slot->blocker_snapshot_ref = blocker_ref; + memset(&poll, 0, sizeof(poll)); + poll.ref = blocker_ref; + poll.ref.grant_generation = tag_slot->holder_ref.grant_generation + 1; + poll.drain_generation = tag_slot->holder_terminal_drain_generation; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, master_session + 1), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_retire_claim_serializes_admission_handoffs) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderKey blocked_holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle blocked_holder; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle blocked_writer; + PcmXRetirePayload retire; + PcmXWaitIdentity writer_identity; + uint32 state_flags; + const uint64 master_session = UINT64_C(1805); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(7095, 0, 2, UINT64_C(72001), 3); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[holder.tag_slot.slot_index]; + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = UINT64_C(9); + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = UINT64_C(999); + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + + pg_atomic_write_u32(&header->local_retire_gate, 2); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(998); + blocked_holder_key = make_local_holder_key(7096, 0, 3, UINT64_C(72002), 4); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&blocked_holder_key, &blocked_holder), + PCM_X_QUEUE_GATE_RETRY); + writer_identity = make_wait_identity(7097, 0, 3, UINT64_C(72003)); + UT_ASSERT_EQ( + cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session, &blocked_writer), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + + +UT_TEST(test_local_retire_global_gate_blocks_direct_mutators) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalMembershipSlot *member; + PcmXLocalMembershipSlot member_before; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot tag_before; + PcmXLocalWriterClaim claim; + PcmXWaitIdentity identity; + const uint64 master_session = UINT64_C(1816); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + identity = make_wait_identity(7116, 0, 26, UINT64_C(71601)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + member = &membership_slots(header)[leader.membership_slot.slot_index]; + tag_before = *tag_slot; + member_before = *member; + + /* RETIRE's global claim wins before this tag-only mutator reaches its CAS. + * It must return retryable BUSY without changing any queue/member byte. */ + pg_atomic_write_u32(&header->local_retire_gate, 2); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(93001); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_BUSY); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(tag_before)) == 0); + UT_ASSERT(memcmp(member, &member_before, sizeof(member_before)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + header->peer_frontiers[1].local_retire_in_progress_ticket_id = 0; + pg_atomic_write_u32(&header->local_retire_gate, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_exact(&leader, &claim), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_writer_claim_abort_exact(&claim), PCM_X_QUEUE_OK); +} + + +UT_TEST(test_local_retire_global_gate_rejects_orphan_and_mismatched_evidence) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handle; + PcmXWaitIdentity identity; + const uint64 master_session = UINT64_C(1817); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + identity = make_wait_identity(7117, 0, 27, UINT64_C(71701)); + pg_atomic_write_u32(&header->local_retire_gate, PCM_X_PROTOCOL_NODE_LIMIT + 1); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &handle), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 1); + identity = make_wait_identity(7118, 0, 20, UINT64_C(71801)); + pg_atomic_write_u32(&header->local_retire_gate, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 1, &handle), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 2); + identity = make_wait_identity(7119, 0, 21, UINT64_C(71901)); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(93002); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 2, &handle), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(80)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 3); + identity = make_wait_identity(7120, 0, 22, UINT64_C(72001)); + pg_atomic_write_u32(&header->local_retire_gate, 2); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(93003); + header->peer_frontiers[2].local_retire_in_progress_ticket_id = UINT64_C(93004); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 3, &handle), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(81)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 4); + identity = make_wait_identity(7121, 0, 23, UINT64_C(72101)); + pg_atomic_write_u32(&header->local_retire_gate, 2); + header->peer_frontiers[2].local_retire_in_progress_ticket_id = UINT64_C(93005); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 4, &handle), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(82)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 5); + identity = make_wait_identity(7122, 0, 24, UINT64_C(72201)); + header->peer_frontiers[1].local_retired_ticket_id = UINT64_C(93006); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(93006); + pg_atomic_write_u32(&header->local_retire_gate, 2); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 5, &handle), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_retire_marker_blocks_cross_lock_tag_handoffs) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalHandle writer; + PcmXLocalHandle next_leader; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXWaitIdentity writer_identity; + PcmXRetirePayload retire; + Size holder_used; + Size tag_used; + Size wait_used; + const uint64 master_session = UINT64_C(1806); + + /* Build every external allocator<->tag handoff before letting RETIRE win + * the episode marker. None may enter its domain mutation afterward. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(7098, 0, 2, UINT64_C(72101), 3); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&holder), PCM_X_QUEUE_OK); + writer_identity = make_wait_identity(7099, 0, 3, UINT64_C(72102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session, &writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&writer, NULL), PCM_X_QUEUE_OK); + holder_used = header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used; + wait_used = header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used; + tag_used = header->allocator[PCM_X_ALLOC_LOCAL_TAG].used; + pg_atomic_write_u32(&header->local_retire_gate, 2); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(998); + + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&holder), PCM_X_QUEUE_GATE_RETRY); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&writer), PCM_X_QUEUE_BUSY); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7100, 2, 4, UINT64_C(72103)); + probe_ref.handle.ticket_id = UINT64_C(91003); + probe_ref.handle.queue_generation = UINT64_C(12); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, holder_used); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, wait_used); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, tag_used); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + /* Bufmgr retains this exact RELEASING handle when its bounded wait batch + * expires. Once RETIRE clears the marker, retrying that same handle must + * detach normally (no reconstructed identity and no fail-closed state). */ + header->peer_frontiers[1].local_retire_in_progress_ticket_id = 0; + pg_atomic_write_u32(&header->local_retire_gate, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, holder_used - 1); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&writer), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* A locally closed round is another tag->allocator handoff when it becomes + * empty. RETIRE owns the marker, so even a ready round must remain closed. */ + tag_slot + = prepare_retire_ready_round_with_holder(master_session + 1, 7101, &cutoff, &next_leader); + header = ClusterPcmXConvertShmem; + pg_atomic_write_u32(&header->local_retire_gate, 2); + header->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(999); + memset(&retire, 0, sizeof(retire)); + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_round_exact(&tag_slot->tag, UINT64_C(9), &cutoff, &writer), + PCM_X_QUEUE_BUSY); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->local_round, cutoff.closed_round); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_cross_lock_gate_precedes_retire_claim) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalCutoff cutoff; + PcmXLocalHandle next_leader; + PcmXLocalHandle writer; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXWaitIdentity writer_identity; + const uint64 master_session = UINT64_C(1808); + + /* Simulate RETIRE acquiring allocator_lock immediately after the operation + * releases it. The operation must already own ADMISSION_GATE, otherwise the + * hook publishes the RETIRE marker and exposes a forbidden overlap. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + holder_key = make_local_holder_key(7102, 0, 2, UINT64_C(72201), 3); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[holder.tag_slot.slot_index]; + arm_local_retire_release_interlock(tag_slot, 1, UINT64_C(92001)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_unregister_exact(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_NULL(local_retire_release_interlock_tag); + UT_ASSERT(local_retire_release_interlock_observed_gate); + UT_ASSERT(!local_retire_release_interlock_claimed); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 1); + holder_key = make_local_holder_key(7103, 0, 3, UINT64_C(72202), 4); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[holder.tag_slot.slot_index]; + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7103, 2, 4, UINT64_C(72203)); + probe_ref.handle.ticket_id = UINT64_C(92002); + probe_ref.handle.queue_generation = UINT64_C(13); + arm_local_retire_release_interlock(tag_slot, 1, UINT64_C(92003)); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session + 1, + &holder_copy, 1, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_NULL(local_retire_release_interlock_tag); + UT_ASSERT(local_retire_release_interlock_observed_gate); + UT_ASSERT(!local_retire_release_interlock_claimed); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + + tag_slot + = prepare_retire_ready_round_with_holder(master_session + 2, 7104, &cutoff, &next_leader); + header = ClusterPcmXConvertShmem; + arm_local_retire_release_interlock(tag_slot, 1, UINT64_C(92004)); + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_round_exact(&tag_slot->tag, UINT64_C(9), &cutoff, &writer), + PCM_X_QUEUE_OK); + UT_ASSERT_NULL(local_retire_release_interlock_tag); + UT_ASSERT(local_retire_release_interlock_observed_gate); + UT_ASSERT(!local_retire_release_interlock_claimed); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session + 3); + writer_identity = make_wait_identity(7105, 0, 5, UINT64_C(72204)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&writer_identity, 1, master_session + 3, &writer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&writer, NULL), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[writer.tag_slot.slot_index]; + arm_local_retire_release_interlock(tag_slot, 1, UINT64_C(92005)); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&writer), PCM_X_QUEUE_OK); + UT_ASSERT_NULL(local_retire_release_interlock_tag); + UT_ASSERT(local_retire_release_interlock_observed_gate); + UT_ASSERT(!local_retire_release_interlock_claimed); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); +} + +UT_TEST(test_local_new_transfer_abort_keeps_gate_until_allocator_cleanup) +{ + PcmXShmemHeader *header; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalTagSlot *reserved_tag; + PcmXTicketRef probe_ref; + PcmXSlotRef found; + uint32 partition; + const uint64 master_session = UINT64_C(1812); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + UT_ASSERT(header->allocator[PCM_X_ALLOC_LOCAL_TAG].free_head != PCM_X_INVALID_SLOT_INDEX); + reserved_tag = &local_tag_slots(header)[header->allocator[PCM_X_ALLOC_LOCAL_TAG].free_head]; + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7106, 2, 6, UINT64_C(72301)); + probe_ref.handle.ticket_id = UINT64_C(92006); + probe_ref.handle.queue_generation = UINT64_C(14); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&probe_ref.identity.tag)); + local_transfer_peer_drift_lock = &header->local_locks[partition].lock; + local_transfer_peer_drift_frontier = &header->peer_frontiers[1]; + /* prepare is allocator acquire #1; abort_new is #2. If the local-domain + * peer drift makes publication fail, the reserved tag gate must still be + * present when allocator cleanup begins, otherwise RETIRE can claim the + * gap and observe a legal cleanup as post-preflight corruption. */ + arm_local_abort_acquire_interlock(reserved_tag, 1, UINT64_C(92007), 2); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, master_session, + NULL, 0, &holder_snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_NULL(local_transfer_peer_drift_lock); + UT_ASSERT_NULL(local_transfer_peer_drift_frontier); + UT_ASSERT_NULL(local_abort_acquire_interlock_tag); + UT_ASSERT(local_abort_acquire_interlock_observed_gate); + UT_ASSERT(!local_abort_acquire_interlock_claimed); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); +} + +UT_TEST(test_local_retire_requires_exact_holder_and_blocker_evidence) +{ + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalCutoff cutoff; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXWaitIdentity identity; + + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(190), 774, &cutoff, &late); + tag_slot->holder_reliable.state_sequence = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(191), 775, &cutoff, &late); + tag_slot->holder_reliable.last_responder_node = 2; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(192), 776, &cutoff, &late); + tag_slot->holder_reliable.response_tombstone_mask = UINT32_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(193), 777, &cutoff, &late); + tag_slot->holder_ref.identity.cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(194), 778, &cutoff, &late); + tag_slot->holder_image.image_id = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(195), 779, &cutoff, &late); + tag_slot->holder_terminal_drain_generation = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + /* A nonzero blocker ref is an admission identity, not merely a durable + * busy bit. A stale epoch must halt instead of becoming an endless retry. */ + init_active_pcm_x(UINT64_C(196)); + identity = make_wait_identity(780, 2, 8, UINT64_C(9501)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(196)); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = identity; + probe_ref.handle.ticket_id = UINT64_C(9601); + probe_ref.handle.queue_generation = UINT64_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, UINT64_C(196), NULL, + 0, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &identity.tag, identity.cluster_epoch, 1, UINT64_C(196), &cutoff), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder_snapshot.tag_slot.slot_index]; + tag_slot->blocker_snapshot_ref.identity.cluster_epoch++; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + init_active_pcm_x(UINT64_C(198)); + identity = make_wait_identity(783, 2, 8, UINT64_C(9502)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(198)); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = identity; + probe_ref.handle.ticket_id = UINT64_C(9602); + probe_ref.handle.queue_generation = UINT64_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_probe_freeze_snapshot_exact(&probe_ref, 1, UINT64_C(198), NULL, + 0, &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &identity.tag, identity.cluster_epoch, 1, UINT64_C(198), &cutoff), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder_snapshot.tag_slot.slot_index]; + tag_slot->blocker_set_generation = UINT64_C(1); + tag_slot->blocker_snapshot_reliable.state_sequence = UINT64_C(1); + tag_slot->blocker_snapshot_reliable.last_responder_node = 1; + tag_slot->blocker_snapshot_reliable.last_response_opcode = PGRAC_IC_MSG_PCM_X_BLOCKER_SET_ACK; + tag_slot->blocker_snapshot_reliable.response_tombstone_mask = UINT32_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&identity.tag, identity.cluster_epoch, + &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + + /* Keep one positive leg beside the field-by-field negative matrix. */ + tag_slot = prepare_retire_ready_round_with_holder(UINT64_C(197), 781, &cutoff, &late); + UT_ASSERT_NOT_NULL(tag_slot); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_blocker_allocator_corruption_retains_new_tag_gate_evidence) +{ + PcmXShmemHeader *header; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalTagSlot *tag_slot; + PcmXTicketRef probe_ref; + PcmXSlotHeader *free_head; + PcmXSlotRef tag_ref; + Size free_head_index; + const uint64 master_session = UINT64_C(1809); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(7105, 2, 5, UINT64_C(72301)); + probe_ref.handle.ticket_id = UINT64_C(91301); + probe_ref.handle.queue_generation = UINT64_C(1); + memset(&holder_snapshot, 0, sizeof(holder_snapshot)); + holder_snapshot.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + + /* Even an empty blocker set validates the complete free chain before it + * publishes a generation. Force the canonical self-loop corruption after + * transfer-tag preparation has enough capacity to reserve its new tag. */ + free_head_index = header->allocator[PCM_X_ALLOC_BLOCKER].free_head; + UT_ASSERT(free_head_index != PCM_X_INVALID_SLOT_INDEX); + free_head = pool_slot_header(header, PCM_X_POOL_BLOCKER, free_head_index); + free_head->next_free = free_head_index; + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, NULL, 0, NULL, 0, + &blocker_snapshot), + PCM_X_QUEUE_CORRUPT); + + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &tag_ref), + PCM_X_DIRECTORY_OK); + tag_slot = &local_tag_slots(header)[tag_ref.slot_index]; + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_RESERVED_NONVISIBLE); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 0); +} + +UT_TEST(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke) +{ + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalBlockerSnapshot duplicate_snapshot; + PcmXBlockerSetHeaderPayload header; + PcmXBlockerChunkPayload edge; + PcmXRevokePayload revoke; + PcmXTicketRef probe_ref; + const uint64 master_session = UINT64_C(180); + + init_active_pcm_x(UINT64_C(77)); + holder_key = make_local_holder_key(709, 0, 1, UINT64_C(7031), 2); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(holder_snapshot.holder_count, 1); + UT_ASSERT(memcmp(&holder, &holder_copy, sizeof(holder)) == 0); + + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(709, 2, 2, UINT64_C(7032)); + probe_ref.handle.ticket_id = UINT64_C(9011); + probe_ref.handle.queue_generation = UINT64_C(11); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &duplicate_snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&blocker_snapshot, &duplicate_snapshot, sizeof(blocker_snapshot)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &header, NULL, 0), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header.nblockers, 0); + UT_ASSERT_EQ(header.set_crc32c, blocker_snapshot.set_crc32c); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + + memset(&revoke, 0, sizeof(revoke)); + revoke.ref = probe_ref; + revoke.ref.grant_generation = UINT64_C(101); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(201), &revoke.image_id)); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + UT_ASSERT_EQ( + cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &header, &edge, 1), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_revoke_apply_exact(&revoke, 1, master_session), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_blocker_snapshot_exact_lookup_replays_when_pool_is_full) +{ + PcmXShmemHeader *header; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalBlockerSnapshot replay_snapshot; + PcmXBlockerSetHeaderPayload block_header; + PcmXBlockerChunkPayload replay_edge; + PcmXTicketRef probe_ref; + PcmXSlotRef fillers[8]; + PcmXSlotHeader *slot; + ClusterLmdVertex blocker; + Size filler_count; + Size pool_capacity; + const uint64 master_session = UINT64_C(181); + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + holder_key = make_local_holder_key(710, 0, 1, UINT64_C(7041), 2); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_register(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(710, 2, 2, UINT64_C(7042)); + probe_ref.handle.ticket_id = UINT64_C(9012); + probe_ref.handle.queue_generation = UINT64_C(12); + blocker = make_blocker(0, holder_key.identity.procno, UINT64_C(7043)); + blocker.cluster_epoch = holder_key.identity.cluster_epoch; + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + &blocker, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + + pool_capacity = header->layout.pools[PCM_X_POOL_BLOCKER].capacity; + UT_ASSERT(pool_capacity <= lengthof(fillers)); + for (filler_count = 0; filler_count < pool_capacity - 1; filler_count++) + UT_ASSERT_EQ( + cluster_pcm_x_allocator_reserve(PCM_X_ALLOC_BLOCKER, &fillers[filler_count], &slot), + PCM_X_ALLOC_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, pool_capacity); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_lookup_exact(&probe_ref, 1, master_session, + &replay_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&replay_snapshot, &blocker_snapshot, sizeof(blocker_snapshot)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_copy_exact(&replay_snapshot, &block_header, + &replay_edge, 1), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&replay_edge.blocker, &blocker, sizeof(blocker)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_lookup_exact( + &probe_ref, 1, master_session + 1, &replay_snapshot), + PCM_X_QUEUE_STALE); + + for (Size i = 0; i < filler_count; i++) + UT_ASSERT_EQ(cluster_pcm_x_allocator_release_exact(PCM_X_ALLOC_BLOCKER, fillers[i], + PCM_X_SLOT_RESERVED_NONVISIBLE), + PCM_X_ALLOC_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_blocker_snapshot_accepts_only_holder_bound_nested_waits) +{ + PcmXLocalTagSlot *tag_slot; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalHolderSnapshot stale_holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXLocalBlockerSnapshot duplicate_snapshot; + PcmXBlockerSetHeaderPayload block_header; + PcmXBlockerChunkPayload edge; + PcmXTicketRef probe_ref; + ClusterLmdVertex nested_wait; + ClusterLmdVertex tx_wait; + ClusterLmdVertex unrelated_wait; + const uint64 master_session = UINT64_C(181); + + init_active_pcm_x(UINT64_C(77)); + holder_key = make_local_holder_key(710, 0, 1, UINT64_C(7033), 2); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[holder.tag_slot.slot_index]; + tag_slot->blocker_set_generation = UINT64_C(41); + + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(710, 2, 2, UINT64_C(7034)); + probe_ref.handle.ticket_id = UINT64_C(9012); + probe_ref.handle.queue_generation = UINT64_C(12); + unrelated_wait = make_blocker(0, 3, UINT64_C(7035)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &holder_snapshot, &holder_copy, 1, + &unrelated_wait, 1, &blocker_snapshot), + PCM_X_QUEUE_INVALID); + UT_ASSERT_EQ(tag_slot->blocker_set_generation, UINT64_C(41)); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); + + nested_wait = make_blocker(0, 1, UINT64_C(7036)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + &nested_wait, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, UINT64_C(42)); + UT_ASSERT_EQ(holder_snapshot.holder_set_generation, UINT64_C(1)); + UT_ASSERT_EQ(blocker_snapshot.blocker_count, 1); + stale_holder_snapshot = holder_snapshot; + stale_holder_snapshot.holder_set_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &stale_holder_snapshot, &holder_copy, 1, + &nested_wait, 1, &duplicate_snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session, &holder_snapshot, &holder_copy, 1, &nested_wait, + 1, &duplicate_snapshot), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&blocker_snapshot, &duplicate_snapshot, sizeof(blocker_snapshot)) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_local_blocker_snapshot_copy_exact(&blocker_snapshot, &block_header, &edge, 1), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(block_header.nblockers, 1); + UT_ASSERT(memcmp(&edge.blocker, &nested_wait, sizeof(nested_wait)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + + /* A later PROBE for the same ticket may replace the ACKed immutable set + * with the next generation. TX wait vertices legitimately carry a zero + * request_id and are exact through xid + wait_seq. */ + tx_wait = nested_wait; + tx_wait.request_id = 0; + tx_wait.xid = (TransactionId)UINT32_C(9042); + tx_wait.wait_seq = UINT64_C(8042); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + &tx_wait, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, UINT64_C(43)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session, + &holder_snapshot, &holder_copy, 1, + NULL, 0, &blocker_snapshot), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(blocker_snapshot.set_generation, UINT64_C(44)); + UT_ASSERT_EQ(blocker_snapshot.blocker_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_ack_exact(&probe_ref, blocker_snapshot.set_generation, + 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(ClusterPcmXConvertShmem->allocator[PCM_X_ALLOC_BLOCKER].used, 0); +} + +UT_TEST(test_local_duplicate_generation_overlap_is_stale_not_fail_closed) +{ + PcmXShmemHeader *header; + PcmXLocalHandle first; + PcmXLocalHandle duplicate; + PcmXLocalHandle cleared; + PcmXLocalTagSlot tag_before; + PcmXLocalMembershipSlot member_before; + PcmXLocalMembershipSlot member_after; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity identity; + Size local_tag_used_before; + Size local_wait_used_before; + Size tag_directory_before; + Size wait_directory_before; + uint32 odd; + uint32 partition; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + identity = make_wait_identity(744, 0, 6, UINT64_C(54001)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &first), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[first.tag_slot.slot_index]; + member = &membership_slots(header)[first.membership_slot.slot_index]; + tag_before = *tag_slot; + member_before = *member; + local_tag_used_before = header->allocator[PCM_X_ALLOC_LOCAL_TAG].used; + local_wait_used_before = header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used; + tag_directory_before = directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG); + wait_directory_before = directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT); + memset(&duplicate, 0xa5, sizeof(duplicate)); + memset(&cleared, 0, sizeof(cleared)); + cleared.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + cleared.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&identity.tag)); + local_generation_interlock_lock = &header->local_locks[partition].lock; + local_generation_interlock_slot = &member->slot; + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &duplicate), + PCM_X_QUEUE_STALE); + UT_ASSERT_NULL(local_generation_interlock_slot); + local_generation_interlock_lock = NULL; + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&duplicate, &cleared, sizeof(duplicate)) == 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, local_tag_used_before); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, local_wait_used_before); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG), tag_directory_before); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), wait_directory_before); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + member_after = *member; + pg_atomic_write_u32(&member_after.slot.generation_change_seq, + pg_atomic_read_u32(&member_before.slot.generation_change_seq)); + UT_ASSERT(memcmp(&member_after, &member_before, sizeof(member_after)) == 0); + odd = pg_atomic_read_u32(&member->slot.generation_change_seq); + UT_ASSERT((odd & 1U) != 0); + pg_atomic_write_u32(&member->slot.generation_change_seq, odd + 1U); +} + +UT_TEST(test_local_join_capacity_and_sequence_failure_roll_back_all_indexes) +{ + PcmXShmemHeader *header; + PcmXAllocatorState saved_wait_allocator; + PcmXDirectoryEntry *wait_entries; + PcmXLocalHandle first; + PcmXLocalHandle rejected; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXWaitIdentity second_identity; + PcmXSlotRef found; + uint64 identity_hash; + Size capacity; + Size directory_capacity; + Size i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + saved_wait_allocator = header->allocator[PCM_X_ALLOC_LOCAL_WAIT]; + capacity = header->layout.local_wait.capacity; + header->allocator[PCM_X_ALLOC_LOCAL_WAIT].free_head = PCM_X_INVALID_SLOT_INDEX; + header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used = capacity; + header->allocator[PCM_X_ALLOC_LOCAL_WAIT].high_water = capacity; + header->allocator[PCM_X_ALLOC_LOCAL_WAIT].generation_exhausted = 0; + identity = make_wait_identity(707, 0, 20, UINT64_C(8001)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &rejected), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG), 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), 0); + header->allocator[PCM_X_ALLOC_LOCAL_WAIT] = saved_wait_allocator; + assert_local_queue_baseline(header); + wait_entries = directory_entries(header, PCM_X_DIR_LOCAL_WAIT, &directory_capacity); + UT_ASSERT(cluster_pcm_x_directory_key_hash(PCM_X_DIR_LOCAL_WAIT, &identity, &identity_hash)); + for (i = 0; i < directory_capacity; i++) { + wait_entries[i].state = PCM_X_DIRECTORY_OCCUPIED; + wait_entries[i].key_hash = identity_hash ^ UINT64_C(1); + } + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &rejected), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_TAG), 0); + memset(wait_entries, 0, directory_capacity * sizeof(*wait_entries)); + assert_local_queue_baseline(header); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &first), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[first.tag_slot.slot_index]; + tag_slot->next_sequence = UINT64_MAX; + second_identity = make_wait_identity(707, 0, 21, UINT64_C(8002)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&second_identity, 1, UINT64_C(77), &rejected), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_TAG].used, 1); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 1); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &second_identity, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&first, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&first), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_unlink_corruption_preserves_fifo_evidence_before_fail_closed) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[3]; + PcmXLocalTagSlot tag_before; + PcmXLocalTagSlot tag_after; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot member_before[3]; + PcmXLocalMembershipSlot *members; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 3; i++) { + PcmXWaitIdentity identity + = make_wait_identity(749, 0, (uint32)(24 + i), UINT64_C(63001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + /* Corrupt only the successor backlink. The fail-closed admission gate is + * expected evidence; the FIFO tag fields and all three members are not. */ + members[handles[2].membership_slot.slot_index].prev_index = PCM_X_INVALID_SLOT_INDEX; + tag_before = *tag_slot; + for (i = 0; i < 3; i++) + member_before[i] = members[handles[i].membership_slot.slot_index]; + + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[1], NULL), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + tag_after = *tag_slot; + UT_ASSERT_EQ(test_slot_flags(&tag_after.slot), + test_slot_flags(&tag_before.slot) | PCM_X_LOCAL_TAG_F_ADMISSION_GATE); + pg_atomic_write_u32(&tag_after.slot.state_flags, + pg_atomic_read_u32(&tag_before.slot.state_flags)); + UT_ASSERT(memcmp(&tag_after, &tag_before, sizeof(tag_after)) == 0); + for (i = 0; i < 3; i++) + UT_ASSERT(memcmp(&members[handles[i].membership_slot.slot_index], &member_before[i], + sizeof(member_before[i])) + == 0); + + /* A link-consistent chain with non-monotonic admission sequences is still + * FIFO corruption. It must be rejected before leader rotation can strand + * a closed-round member behind a next-round candidate. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + for (i = 0; i < 3; i++) { + PcmXWaitIdentity identity + = make_wait_identity(750, 0, (uint32)(24 + i), UINT64_C(63011) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + members[handles[1].membership_slot.slot_index].local_sequence = 3; + members[handles[2].membership_slot.slot_index].local_sequence = 2; + tag_before = *tag_slot; + for (i = 0; i < 3; i++) + member_before[i] = members[handles[i].membership_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], NULL), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + tag_after = *tag_slot; + UT_ASSERT_EQ(test_slot_flags(&tag_after.slot), + test_slot_flags(&tag_before.slot) | PCM_X_LOCAL_TAG_F_ADMISSION_GATE); + pg_atomic_write_u32(&tag_after.slot.state_flags, + pg_atomic_read_u32(&tag_before.slot.state_flags)); + UT_ASSERT(memcmp(&tag_after, &tag_before, sizeof(tag_after)) == 0); + for (i = 0; i < 3; i++) + UT_ASSERT(memcmp(&members[handles[i].membership_slot.slot_index], &member_before[i], + sizeof(member_before[i])) + == 0); +} + +UT_TEST(test_local_cancel_requires_state_exact_leader_locator) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[2]; + PcmXLocalTagSlot tag_before; + PcmXLocalTagSlot tag_after; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot member_before[2]; + PcmXLocalMembershipSlot *members; + int i; + + /* A resident NODE_LEADER must own the exact tag leader locator. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(755, 0, (uint32)(2 + i), UINT64_C(69001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + tag_slot->leader_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->leader_slot_generation = 0; + tag_before = *tag_slot; + for (i = 0; i < 2; i++) + member_before[i] = members[handles[i].membership_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], NULL), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + tag_after = *tag_slot; + UT_ASSERT_EQ(test_slot_flags(&tag_after.slot), + test_slot_flags(&tag_before.slot) | PCM_X_LOCAL_TAG_F_ADMISSION_GATE); + pg_atomic_write_u32(&tag_after.slot.state_flags, + pg_atomic_read_u32(&tag_before.slot.state_flags)); + UT_ASSERT(memcmp(&tag_after, &tag_before, sizeof(tag_after)) == 0); + for (i = 0; i < 2; i++) + UT_ASSERT(memcmp(&members[handles[i].membership_slot.slot_index], &member_before[i], + sizeof(member_before[i])) + == 0); + + /* A resident follower must never be named by the tag leader locator. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(756, 0, (uint32)(2 + i), UINT64_C(69003) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + tag_slot->leader_index = handles[1].membership_slot.slot_index; + tag_slot->leader_slot_generation = handles[1].membership_slot.slot_generation; + tag_before = *tag_slot; + for (i = 0; i < 2; i++) + member_before[i] = members[handles[i].membership_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[1], NULL), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + tag_after = *tag_slot; + UT_ASSERT_EQ(test_slot_flags(&tag_after.slot), + test_slot_flags(&tag_before.slot) | PCM_X_LOCAL_TAG_F_ADMISSION_GATE); + pg_atomic_write_u32(&tag_after.slot.state_flags, + pg_atomic_read_u32(&tag_before.slot.state_flags)); + UT_ASSERT(memcmp(&tag_after, &tag_before, sizeof(tag_after)) == 0); + for (i = 0; i < 2; i++) + UT_ASSERT(memcmp(&members[handles[i].membership_slot.slot_index], &member_before[i], + sizeof(member_before[i])) + == 0); +} + +UT_TEST(test_local_cutoff_preserves_cancelled_tail_sequence) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[3]; + PcmXLocalHandle next_leader; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 3; i++) { + PcmXWaitIdentity identity + = make_wait_identity(740, 0, (uint32)(2 + i), UINT64_C(41001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->next_sequence, 4); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[2], NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->tail_index, handles[1].membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->next_sequence, 4); + + /* The cutoff is the admission high-water, not the latest still-linked + * member. It must include the cancelled terminal identity in this round. */ + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&handles[0], &cutoff), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cutoff.cutoff_sequence, 3); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 3); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[1], NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], NULL), PCM_X_QUEUE_OK); + for (i = 2; i >= 0; i--) + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact(&handles[0].identity.tag, + handles[0].identity.cluster_epoch, &cutoff, + &next_leader), + PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_revoke_cutoff_sends_late_writer_to_next_round) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalHandle duplicate_leader; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalCutoff cutoff; + PcmXLocalCutoff duplicate; + PcmXLocalCutoff wrapped; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload admit_confirm; + PcmXLocalReliableToken reliable_token; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *members; + PcmXSlotRef found; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 22, UINT64_C(9001)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &leader), + PCM_X_QUEUE_OK); + } + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 23, UINT64_C(9002)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &follower), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cutoff.cutoff_sequence, 2); + UT_ASSERT_EQ(cutoff.closed_round, 1); + UT_ASSERT_EQ(cutoff.next_round, 2); + UT_ASSERT_EQ(cutoff.master_node, 1); + UT_ASSERT_EQ(cutoff.master_session_incarnation, UINT64_C(77)); + UT_ASSERT_EQ(cutoff.reserved, 0); + UT_ASSERT(slot_refs_equal(cutoff.tag_slot, leader.tag_slot)); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(tag_slot->cutoff_sequence, 2); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 2); + + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &duplicate), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&duplicate, &cutoff, sizeof(cutoff)) == 0); + memset(&duplicate, 0, sizeof(duplicate)); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&leader.identity.tag, + leader.identity.cluster_epoch, 1, + UINT64_C(77), &duplicate), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&duplicate, &cutoff, sizeof(cutoff)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&leader.identity.tag, + leader.identity.cluster_epoch, 1, + UINT64_C(78), &duplicate), + PCM_X_QUEUE_STALE); + header->peer_frontiers[1].sender_session_incarnation = UINT64_C(78); + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact(&leader.identity.tag, + leader.identity.cluster_epoch, 1, + UINT64_C(77), &duplicate), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &leader.identity.tag, leader.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_STALE); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER) != 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + header->peer_frontiers[1].sender_session_incarnation = UINT64_C(77); + wrapped = cutoff; + wrapped.closed_round = UINT32_MAX; + wrapped.next_round = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &leader.identity.tag, leader.identity.cluster_epoch, &wrapped, &next_leader), + PCM_X_QUEUE_INVALID); + + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 24, UINT64_C(9003)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &late), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(late.local_sequence, 3); + UT_ASSERT_EQ(late.local_round, 2); + UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(910)), + PCM_X_QUEUE_OK); + members = membership_slots(header); + UT_ASSERT_EQ(test_slot_state(&members[late.membership_slot.slot_index].slot), + PCM_XL_WAITABLE_FOLLOWER); + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->membership_count, 3); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 2); + + /* Conservative cancellation cannot prove whether this follower was a + * former active blocker, so neither linked identity disappears while the + * late writer retains its exact WFG edge. */ + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&late, UINT64_C(910)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&follower, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), + PCM_X_QUEUE_BARRIER_CLOSED); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&follower), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 0); + UT_ASSERT_EQ(tag_slot->leader_index, PCM_X_INVALID_SLOT_INDEX); + /* Model a DRAIN/graph-removal race: retire must wait for exact removal and + * never classify an otherwise valid WAITABLE follower as corruption. */ + members[late.membership_slot.slot_index].graph_generation = UINT64_C(910); + test_set_slot_state(&members[late.membership_slot.slot_index].slot, PCM_XL_WAITABLE_FOLLOWER); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_clear_exact(&late, UINT64_C(910)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(next_leader.local_round, 2); + UT_ASSERT_EQ(next_leader.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT(slot_refs_equal(next_leader.membership_slot, late.membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &duplicate_leader), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(slot_refs_equal(duplicate_leader.membership_slot, next_leader.membership_slot)); + + /* A duplicate old-round RETIRE may arrive after the promoted leader has + * already entered the next ticket's remote protocol. That is idempotent + * progress, not corruption; no second wake handle is needed. */ + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&next_leader, &enqueue, &reliable_token), + PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = next_leader.identity; + admit_ack.ref.handle.ticket_id = UINT64_C(9201); + admit_ack.ref.handle.queue_generation = UINT64_C(21); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ( + cluster_pcm_x_local_apply_admit_ack_exact(&next_leader, &admit_ack, 1, UINT64_C(77)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_arm_exact(&next_leader, &admit_confirm, &reliable_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_ack_exact(&next_leader, &admit_confirm, 1, UINT64_C(77)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[next_leader.membership_slot.slot_index].slot), + PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &duplicate_leader), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(duplicate_leader.membership_slot.slot_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + + /* Cancel unlinks the closed-round member before terminal detach decrements + * membership_count. That evidence-retention window is legitimate and must + * remain a retry, not a structural failure. */ + init_active_pcm_x(UINT64_C(82)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(82)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 25, UINT64_C(9004)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(82), &leader), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->tail_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->closed_round_member_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &leader.identity.tag, leader.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &leader.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, leader.tag_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &leader.identity.tag, leader.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); + + /* A writer joining after the last closed member detached must still see the + * retained barrier and enter the next round before exact retirement. */ + init_active_pcm_x(UINT64_C(83)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(83)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 25, UINT64_C(9008)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(83), &leader), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 26, UINT64_C(9009)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(83), &late), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(late.local_round, cutoff.next_round); + UT_ASSERT_EQ(late.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); + UT_ASSERT(slot_refs_equal(next_leader.membership_slot, late.membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&next_leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&next_leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 25, UINT64_C(9004)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &leader), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + tag_slot->local_round = UINT32_MAX; + members = membership_slots(header); + members[leader.membership_slot.slot_index].admitted_round = UINT32_MAX; + leader.local_round = UINT32_MAX; + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_REVOKE_BARRIER, 0); + UT_ASSERT_EQ(tag_slot->cutoff_sequence, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); + + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(79)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 6, UINT64_C(9005)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(79), &leader), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + members = membership_slots(header); + tag_slot->local_round = UINT32_MAX; + members[leader.membership_slot.slot_index].admitted_round = UINT32_MAX; + leader.local_round = UINT32_MAX; + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &duplicate), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + + init_active_pcm_x(UINT64_C(80)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(80)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 7, UINT64_C(9006)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(80), &leader), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + tag_slot->cutoff_sequence = 0; + UT_ASSERT(tag_slot->closed_round_member_count > 0); + members = membership_slots(header); + members[leader.membership_slot.slot_index].admitted_round = tag_slot->local_round + 1; + leader.local_round = tag_slot->local_round + 1; + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &duplicate), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + + init_active_pcm_x(UINT64_C(81)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(81)); + { + PcmXWaitIdentity identity = make_wait_identity(708, 0, 8, UINT64_C(9007)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(81), &leader), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + tag_slot->closed_round_member_count = tag_slot->membership_count + 1; + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &duplicate), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); +} + +UT_TEST(test_local_retire_rejects_candidate_from_wrong_round) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalCutoff cutoff; + PcmXLocalMembershipSlot *candidate; + PcmXRuntimeSnapshot snapshot; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + identity = make_wait_identity(709, 0, 26, UINT64_C(9101)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(709, 0, 27, UINT64_C(9102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + candidate = &membership_slots(header)[late.membership_slot.slot_index]; + candidate->admitted_round = cutoff.closed_round; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&local_tag_slots(header)[late.tag_slot.slot_index].slot) + & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) + != 0); +} + +UT_TEST(test_local_cancel_rejects_waitable_successor_from_wrong_round) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalHandle late; + PcmXLocalFollowerWfgSnapshot wfg; + PcmXLocalCutoff cutoff; + PcmXLocalMembershipSlot *candidate; + PcmXRuntimeSnapshot snapshot; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + identity = make_wait_identity(709, 0, 6, UINT64_C(9111)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(709, 0, 7, UINT64_C(9112)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_snapshot_exact(&late, &wfg), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_follower_wfg_commit_exact(&wfg, UINT64_C(911)), + PCM_X_QUEUE_OK); + candidate = &membership_slots(header)[late.membership_slot.slot_index]; + candidate->admitted_round = cutoff.closed_round; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_CORRUPT); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&local_tag_slots(header)[late.tag_slot.slot_index].slot) + & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) + != 0); +} + +UT_TEST(test_local_retire_rejects_closed_head_and_active_generation_corruption) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalCutoff cutoff; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *candidate; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + identity = make_wait_identity(709, 0, 6, UINT64_C(9121)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(709, 0, 7, UINT64_C(9122)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + candidate = &membership_slots(header)[late.membership_slot.slot_index]; + candidate->local_sequence = cutoff.cutoff_sequence; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + identity = make_wait_identity(709, 0, 6, UINT64_C(9123)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(709, 0, 7, UINT64_C(9124)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[late.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->active_writer_index, PCM_X_INVALID_SLOT_INDEX); + tag_slot->active_writer_slot_generation = UINT64_C(99); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(79)); + identity = make_wait_identity(709, 0, 6, UINT64_C(9125)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(79), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(709, 0, 7, UINT64_C(9126)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(79), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[late.tag_slot.slot_index]; + UT_ASSERT(tag_slot->membership_count > 0); + UT_ASSERT(tag_slot->tail_index != PCM_X_INVALID_SLOT_INDEX); + tag_slot->head_index = PCM_X_INVALID_SLOT_INDEX; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(80)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(80)); + identity = make_wait_identity(709, 0, 6, UINT64_C(9127)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(80), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(709, 0, 7, UINT64_C(9128)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(80), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + candidate = &membership_slots(header)[late.membership_slot.slot_index]; + UT_ASSERT_EQ(candidate->prev_index, PCM_X_INVALID_SLOT_INDEX); + candidate->prev_index = late.membership_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + init_active_pcm_x(UINT64_C(81)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(81)); + identity = make_wait_identity(709, 0, 6, UINT64_C(9129)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(81), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT(tag_slot->leader_index != PCM_X_INVALID_SLOT_INDEX); + tag_slot->leader_slot_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_current_cutoff_snapshot_exact( + &leader.identity.tag, leader.identity.cluster_epoch, 1, UINT64_C(81), &cutoff), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_retire_duplicate_revalidates_leader_tag_linkage) +{ + PcmXShmemHeader *header; + PcmXLocalHandle leader; + PcmXLocalHandle late; + PcmXLocalHandle next_leader; + PcmXLocalHandle duplicate; + PcmXLocalCutoff cutoff; + PcmXLocalProgress progress; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *candidate; + PcmXRuntimeSnapshot snapshot; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + identity = make_wait_identity(710, 0, 26, UINT64_C(9201)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(710, 0, 27, UINT64_C(9202)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&next_leader, &progress), PCM_X_QUEUE_OK); + candidate = &membership_slots(header)[late.membership_slot.slot_index]; + candidate->tag_slot_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &duplicate), + PCM_X_QUEUE_CORRUPT); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&local_tag_slots(header)[late.tag_slot.slot_index].slot) + & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) + != 0); + + /* Barrier-absent replay is valid only after the old cohort count reached + * zero. A stale count must not be hidden behind DUPLICATE. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + identity = make_wait_identity(710, 0, 26, UINT64_C(9203)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(710, 0, 27, UINT64_C(9204)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[late.tag_slot.slot_index]; + tag_slot->closed_round_member_count = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &duplicate), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* A linked open-round FIFO must name its head as the unique leader. */ + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(79)); + identity = make_wait_identity(710, 0, 26, UINT64_C(9205)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(79), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + identity = make_wait_identity(710, 0, 27, UINT64_C(9206)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(79), &late), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&leader), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &next_leader), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[late.tag_slot.slot_index]; + tag_slot->leader_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->leader_slot_generation = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_round_exact( + &late.identity.tag, late.identity.cluster_epoch, &cutoff, &duplicate), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_cancel_is_exact_handoffs_leader_and_terminal_detach_restores_baseline) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[3]; + PcmXLocalHandle promoted; + PcmXLocalHandle stale; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *members; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 3; i++) { + PcmXWaitIdentity identity + = make_wait_identity(709, 0, (uint32)(26 + i), UINT64_C(10001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + members = membership_slots(header); + stale = handles[1]; + stale.membership_slot.slot_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&stale, NULL), PCM_X_QUEUE_STALE); + stale = handles[1]; + stale.local_round++; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&stale, NULL), PCM_X_QUEUE_STALE); + stale = handles[1]; + stale.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&stale, NULL), PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(members[handles[0].membership_slot.slot_index].next_index, + handles[1].membership_slot.slot_index); + + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[1], NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[handles[1].membership_slot.slot_index].slot), + PCM_XL_CANCELLED); + UT_ASSERT_EQ(members[handles[0].membership_slot.slot_index].next_index, + handles[2].membership_slot.slot_index); + UT_ASSERT_EQ(members[handles[2].membership_slot.slot_index].prev_index, + handles[0].membership_slot.slot_index); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[1]), PCM_X_QUEUE_OK); + + memset(&promoted, 0, sizeof(promoted)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], &promoted), PCM_X_QUEUE_OK); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, handles[2].membership_slot)); + UT_ASSERT_EQ(promoted.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT_EQ(tag_slot->leader_index, handles[2].membership_slot.slot_index); + UT_ASSERT_EQ(test_slot_state(&members[handles[2].membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[0]), PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&promoted, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&promoted), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_promoted_leader_identity_generation_rekey_is_exact) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[2]; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalHandle replay; + PcmXLocalHandle lookup; + PcmXLocalProgress progress; + PcmXLocalMembershipSlot *member; + PcmXEnqueuePayload enqueue; + PcmXLocalReliableToken reliable; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity new_identity; + PcmXSlotRef found; + Size directory_count; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(812, 0, (uint32)(26 + i), UINT64_C(12011) + i); + + identity.base_own_generation = UINT64_C(4); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &handles[i]), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[0]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(promoted.role, PCM_X_LOCAL_ROLE_NODE_LEADER); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, handles[1].membership_slot)); + old_identity = promoted.identity; + new_identity = old_identity; + new_identity.base_own_generation = UINT64_C(9); + directory_count = directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT); + + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, new_identity.base_own_generation, &rekeyed), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&rekeyed.identity, &new_identity, sizeof(new_identity)) == 0); + UT_ASSERT(slot_refs_equal(rekeyed.tag_slot, promoted.tag_slot)); + UT_ASSERT(slot_refs_equal(rekeyed.membership_slot, promoted.membership_slot)); + UT_ASSERT_EQ(rekeyed.local_sequence, promoted.local_sequence); + UT_ASSERT_EQ(rekeyed.local_round, promoted.local_round); + UT_ASSERT_EQ(rekeyed.role, promoted.role); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), directory_count); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &old_identity, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&old_identity, &lookup), PCM_X_QUEUE_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &new_identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&new_identity, &lookup), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&lookup, &rekeyed, sizeof(lookup)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&rekeyed, &progress), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&progress.identity, &new_identity, sizeof(new_identity)) == 0); + member = &membership_slots(header)[rekeyed.membership_slot.slot_index]; + UT_ASSERT(memcmp(&member->identity, &new_identity, sizeof(new_identity)) == 0); + + /* A caller that lost the first return value can replay the immutable old + * handle and recover the canonical new identity without a second move. */ + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, new_identity.base_own_generation, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &rekeyed, sizeof(replay)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &rekeyed, new_identity.base_own_generation, &replay), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &rekeyed, sizeof(replay)) == 0); + + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&rekeyed, &enqueue, &reliable), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &rekeyed, new_identity.base_own_generation + 1, &replay), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &new_identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); + UT_ASSERT_EQ(test_slot_flags(&local_tag_slots(header)[rekeyed.tag_slot.slot_index].slot) + & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, + 0); + UT_ASSERT_EQ(max_held_lwlock_count, 1); +} + +UT_TEST(test_local_promoted_leader_rekey_rejects_live_evidence_and_key_alias) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[3]; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity alias_identity; + PcmXSlotRef found; + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + old_identity = make_wait_identity(813, 0, 21, UINT64_C(13012)); + old_identity.base_own_generation = UINT64_C(5); + alias_identity = old_identity; + alias_identity.base_own_generation = UINT64_C(6); + { + PcmXWaitIdentity first = make_wait_identity(813, 0, 20, UINT64_C(13011)); + + first.base_own_generation = UINT64_C(5); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&first, 1, UINT64_C(78), &handles[0]), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&old_identity, 1, UINT64_C(78), &handles[1]), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&alias_identity, 1, UINT64_C(78), &handles[2]), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[0]), PCM_X_QUEUE_OK); + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + UT_ASSERT(memcmp(&promoted.identity, &old_identity, sizeof(old_identity)) == 0); + + member->graph_generation = UINT64_C(7); + UT_ASSERT_EQ( + cluster_pcm_x_local_leader_rekey_generation_exact(&promoted, UINT64_C(7), &rekeyed), + PCM_X_QUEUE_BUSY); + member->graph_generation = 0; + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &old_identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); + + tag_slot->active_writer_index = promoted.membership_slot.slot_index; + tag_slot->active_writer_slot_generation = promoted.membership_slot.slot_generation; + UT_ASSERT_EQ( + cluster_pcm_x_local_leader_rekey_generation_exact(&promoted, UINT64_C(7), &rekeyed), + PCM_X_QUEUE_BUSY); + tag_slot->active_writer_index = PCM_X_INVALID_SLOT_INDEX; + tag_slot->active_writer_slot_generation = 0; + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + + /* A target identity owned by another immutable membership is structural + * corruption, not transient contention. The old mapping must survive. */ + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &alias_identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, handles[2].membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, alias_identity.base_own_generation, &rekeyed), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &old_identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); +} + +UT_TEST(test_local_promoted_leader_rekey_allocator_handoff_error_releases_gate) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[2]; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity target_identity; + PcmXSlotRef found; + int i; + + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(79)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(814, 0, (uint32)(22 + i), UINT64_C(14011) + i); + + identity.base_own_generation = UINT64_C(8); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(79), &handles[i]), + PCM_X_QUEUE_OK); + } + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[0]), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + old_identity = promoted.identity; + target_identity = old_identity; + target_identity.base_own_generation = UINT64_C(10); + + /* The first allocator acquisition resolves and gates the old key. Throw + * on the second acquisition, before any directory mutation. */ + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 2); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, target_identity.base_own_generation, &rekeyed)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &old_identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &target_identity, &found), + PCM_X_DIRECTORY_NOT_FOUND); +} + + +UT_TEST(test_local_rekey_staged_target_directory_is_retryable_busy) +{ + PcmXShmemHeader *header; + PcmXLocalHandle promoted; + PcmXLocalHandle target_handle; + PcmXLocalHandle lookup; + PcmXLocalHandle zero_handle; + PcmXLocalProgress progress; + PcmXLocalProgress zero_progress; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity target_identity; + Size directory_count; + uint32 state_flags; + + prepare_promoted_rekey_fixture(815, UINT64_C(80), UINT64_C(8), &promoted); + header = ClusterPcmXConvertShmem; + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + old_identity = promoted.identity; + target_identity = old_identity; + target_identity.base_own_generation = UINT64_C(11); + target_handle = promoted; + target_handle.identity = target_identity; + directory_count = directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT); + + UT_ASSERT_EQ(cluster_pcm_x_directory_delete_exact(PCM_X_DIR_LOCAL_WAIT, &old_identity, + promoted.membership_slot), + PCM_X_DIRECTORY_OK); + test_publish_raw_local_wait_mapping(header, &target_identity, promoted.membership_slot); + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags | (PCM_X_LOCAL_TAG_F_ADMISSION_GATE << PCM_X_SLOT_FLAGS_SHIFT)); + + memset(&lookup, 0xa5, sizeof(lookup)); + memset(&progress, 0xa5, sizeof(progress)); + memset(&zero_handle, 0, sizeof(zero_handle)); + zero_handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + zero_handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + memset(&zero_progress, 0, sizeof(zero_progress)); + UT_ASSERT_EQ(cluster_pcm_x_local_lookup_exact(&target_identity, &lookup), PCM_X_QUEUE_BUSY); + UT_ASSERT(memcmp(&lookup, &zero_handle, sizeof(lookup)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_progress_exact(&target_handle, &progress), PCM_X_QUEUE_BUSY); + UT_ASSERT(memcmp(&progress, &zero_progress, sizeof(progress)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT( + test_raw_local_wait_mapping_exists(header, &target_identity, promoted.membership_slot)); + UT_ASSERT(!test_raw_local_wait_mapping_exists(header, &old_identity, promoted.membership_slot)); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), directory_count); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); +} + + +UT_TEST(test_local_rekey_third_local_acquire_error_retains_gate) +{ + PcmXShmemHeader *header; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalHandle zero_handle; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity target_identity; + Size directory_count; + uint32 partition; + + prepare_promoted_rekey_fixture(816, UINT64_C(81), UINT64_C(8), &promoted); + header = ClusterPcmXConvertShmem; + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + old_identity = promoted.identity; + target_identity = old_identity; + target_identity.base_own_generation = UINT64_C(12); + directory_count = directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&old_identity.tag)); + memset(&rekeyed, 0xa5, sizeof(rekeyed)); + memset(&zero_handle, 0, sizeof(zero_handle)); + zero_handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + zero_handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + + arm_lwlock_acquire_error(&header->local_locks[partition].lock, LW_EXCLUSIVE, 2); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, target_identity.base_own_generation, &rekeyed)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT(memcmp(&rekeyed, &zero_handle, sizeof(rekeyed)) == 0); + UT_ASSERT(!test_raw_local_wait_mapping_exists(header, &old_identity, promoted.membership_slot)); + UT_ASSERT( + test_raw_local_wait_mapping_exists(header, &target_identity, promoted.membership_slot)); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), directory_count); +} + + +UT_TEST(test_local_rekey_requires_canonical_fifo_leader) +{ + int scenario; + + for (scenario = 0; scenario < 3; scenario++) { + PcmXShmemHeader *header; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalHandle zero_handle; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity target_identity; + Size directory_count; + uint32 state_flags; + + prepare_promoted_rekey_fixture((BlockNumber)(817 + scenario), + UINT64_C(82) + (uint64)scenario, UINT64_C(8), &promoted); + header = ClusterPcmXConvertShmem; + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + old_identity = promoted.identity; + target_identity = old_identity; + target_identity.base_own_generation = UINT64_C(13) + (uint64)scenario; + directory_count = directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT); + memset(&rekeyed, 0xa5, sizeof(rekeyed)); + memset(&zero_handle, 0, sizeof(zero_handle)); + zero_handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + zero_handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + + switch (scenario) { + case 0: + state_flags = pg_atomic_read_u32(&member->slot.state_flags); + pg_atomic_write_u32( + &member->slot.state_flags, + state_flags | (PCM_X_LOCAL_MEMBER_F_WRITER_COMPLETE << PCM_X_SLOT_FLAGS_SHIFT)); + break; + case 1: + tag_slot->head_index = PCM_X_INVALID_SLOT_INDEX; + break; + case 2: + member->prev_index = member - membership_slots(header); + break; + default: + UT_ASSERT(false); + } + + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, target_identity.base_own_generation, &rekeyed), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT(memcmp(&rekeyed, &zero_handle, sizeof(rekeyed)) == 0); + UT_ASSERT( + test_raw_local_wait_mapping_exists(header, &old_identity, promoted.membership_slot)); + UT_ASSERT(!test_raw_local_wait_mapping_exists(header, &target_identity, + promoted.membership_slot)); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), directory_count); + } +} + + +UT_TEST(test_local_rekey_target_insert_failure_restores_old_key) +{ + PcmXShmemHeader *header; + PcmXLocalHandle promoted; + PcmXLocalHandle rekeyed; + PcmXLocalHandle zero_handle; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXWaitIdentity old_identity; + PcmXWaitIdentity target_identity; + Size directory_count; + + prepare_promoted_rekey_fixture(820, UINT64_C(85), UINT64_C(8), &promoted); + header = ClusterPcmXConvertShmem; + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + old_identity = promoted.identity; + target_identity = old_identity; + target_identity.base_own_generation = UINT64_C(16); + directory_count = directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT); + memset(&rekeyed, 0xa5, sizeof(rekeyed)); + memset(&zero_handle, 0, sizeof(zero_handle)); + zero_handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + zero_handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + + local_rekey_insert_failure_armed = true; + local_rekey_insert_failure_match = 2; + local_rekey_insert_failure_member = member; + local_rekey_insert_failure_old_identity = old_identity; + local_rekey_insert_failure_target_identity = target_identity; + UT_ASSERT_EQ(cluster_pcm_x_local_leader_rekey_generation_exact( + &promoted, target_identity.base_own_generation, &rekeyed), + PCM_X_QUEUE_CORRUPT); + UT_ASSERT(!local_rekey_insert_failure_armed); + UT_ASSERT_EQ(local_rekey_insert_failure_match, 0); + UT_ASSERT_EQ(local_rekey_insert_failure_phase, 2); + UT_ASSERT_NULL(local_rekey_insert_failure_poisoned_entry); + UT_ASSERT(local_rekey_insert_failure_observed_old_identity); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + UT_ASSERT(memcmp(&member->identity, &old_identity, sizeof(old_identity)) == 0); + UT_ASSERT(memcmp(&rekeyed, &zero_handle, sizeof(rekeyed)) == 0); + UT_ASSERT(test_raw_local_wait_mapping_exists(header, &old_identity, promoted.membership_slot)); + UT_ASSERT( + !test_raw_local_wait_mapping_exists(header, &target_identity, promoted.membership_slot)); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_WAIT), directory_count); +} + +UT_TEST(test_last_writer_detach_waits_for_independent_holder_transfer_before_tag_release) +{ + PcmXShmemHeader *header; + PcmXLocalHandle writers[2]; + PcmXLocalHandle promoted; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXTicketRef holder_ref_before; + PcmXReliableLegState holder_reliable_before; + PcmXSlotRef found; + uint32 state_flags; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(771, 0, (uint32)(26 + i), UINT64_C(11001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &writers[i]), + PCM_X_QUEUE_OK); + } + memset(&promoted, 0, sizeof(promoted)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&writers[0], &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&writers[0]), PCM_X_QUEUE_OK); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, writers[1].membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&promoted, NULL), PCM_X_QUEUE_OK); + + tag_slot = &local_tag_slots(header)[promoted.tag_slot.slot_index]; + member = &membership_slots(header)[promoted.membership_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, PCM_X_INVALID_SLOT_INDEX); + seed_live_local_holder_transfer(tag_slot, 28, UINT64_C(11003), UINT64_C(12001)); + holder_ref_before = tag_slot->holder_ref; + holder_reliable_before = tag_slot->holder_reliable; + + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&promoted), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_LIVE); + UT_ASSERT_EQ(test_slot_state(&member->slot), PCM_XL_CANCELLED); + UT_ASSERT_EQ(tag_slot->membership_count, 1); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, UINT64_C(77)); + UT_ASSERT(memcmp(&tag_slot->holder_ref, &holder_ref_before, sizeof(holder_ref_before)) == 0); + UT_ASSERT( + memcmp(&tag_slot->holder_reliable, &holder_reliable_before, sizeof(holder_reliable_before)) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &promoted.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.tag_slot)); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &promoted.identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); + + clear_local_holder_transfer(tag_slot); + state_flags = pg_atomic_read_u32(&tag_slot->slot.state_flags); + pg_atomic_write_u32(&tag_slot->slot.state_flags, + state_flags + | (PCM_X_LOCAL_TAG_F_HOLDER_TERMINAL_READY << PCM_X_SLOT_FLAGS_SHIFT)); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&promoted), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &promoted.identity.tag, &found), + PCM_X_DIRECTORY_OK); + pg_atomic_write_u32(&tag_slot->slot.state_flags, state_flags); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&promoted), PCM_X_QUEUE_OK); + assert_local_queue_baseline(header); +} + +UT_TEST(test_fifo_writer_drain_preserves_concurrent_holder_lane_until_transfer_drain) +{ + PcmXShmemHeader *header; + PcmXLocalHandle writers[2]; + PcmXLocalHandle promoted; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *holder_slot; + PcmXTicketRef holder_ref_before; + PcmXReliableLegState holder_reliable_before; + PcmXSlotRef found; + Size holder_head_before; + int i; + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(178)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(772, 0, (uint32)(26 + i), UINT64_C(12001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(178), &writers[i]), + PCM_X_QUEUE_OK); + } + holder_key = make_local_holder_key(772, 0, 28, UINT64_C(12003), 2); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[writers[0].tag_slot.slot_index]; + holder_slot = &membership_slots(header)[holder.holder_slot.slot_index]; + holder_head_before = tag_slot->active_holder_head_index; + UT_ASSERT_EQ(tag_slot->membership_count, 2); + UT_ASSERT(holder_head_before != PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_XL_HOLDER_ACTIVE); + seed_live_local_holder_transfer(tag_slot, 29, UINT64_C(12004), UINT64_C(13001)); + holder_ref_before = tag_slot->holder_ref; + holder_reliable_before = tag_slot->holder_reliable; + + /* Interleave the independent lanes: holder evidence is live before the + * two-writer FIFO becomes terminal and drains to its last membership. */ + memset(&promoted, 0, sizeof(promoted)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&writers[0], &promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&writers[0]), PCM_X_QUEUE_OK); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, writers[1].membership_slot)); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&promoted, NULL), PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&promoted), PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(tag_slot->active_holder_head_index, holder_head_before); + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_XL_HOLDER_ACTIVE); + UT_ASSERT_EQ(tag_slot->master_node, 1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, UINT64_C(178)); + UT_ASSERT(memcmp(&tag_slot->holder_ref, &holder_ref_before, sizeof(holder_ref_before)) == 0); + UT_ASSERT( + memcmp(&tag_slot->holder_reliable, &holder_reliable_before, sizeof(holder_reliable_before)) + == 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &promoted.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.tag_slot)); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &promoted.identity, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, promoted.membership_slot)); + + clear_local_holder_transfer(tag_slot); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&promoted), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->master_node, -1); + UT_ASSERT_EQ(tag_slot->master_session_incarnation, 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_WAIT, &promoted.identity, &found), + PCM_X_DIRECTORY_NOT_FOUND); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &promoted.identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(release_active_local_holder(&holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(directory_occupied_count(header, PCM_X_DIR_LOCAL_HOLDER), 0); + assert_local_queue_baseline(header); +} + +UT_TEST(test_local_terminal_detach_rejects_hot_link_and_closed_count_corruption) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handles[2]; + PcmXLocalMembershipSlot *member; + PcmXLocalTagSlot *tag_slot; + int i; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(77)); + for (i = 0; i < 2; i++) { + PcmXWaitIdentity identity + = make_wait_identity(747, 0, (uint32)(10 + i), UINT64_C(59001) + i); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(77), &handles[i]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + member = &membership_slots(header)[handles[1].membership_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[1], NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->tail_index, handles[0].membership_slot.slot_index); + tag_slot->tail_index = handles[1].membership_slot.slot_index; + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[1]), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(test_slot_state(&member->slot), PCM_XL_CANCELLED); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 2); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), UINT64_C(78)); + { + PcmXWaitIdentity identity = make_wait_identity(748, 0, 12, UINT64_C(60001)); + + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(78), &handles[0]), + PCM_X_QUEUE_OK); + } + tag_slot = &local_tag_slots(header)[handles[0].tag_slot.slot_index]; + member = &membership_slots(header)[handles[0].membership_slot.slot_index]; + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handles[0], NULL), PCM_X_QUEUE_OK); + tag_slot->closed_round_member_count = 1; + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handles[0]), PCM_X_QUEUE_CORRUPT); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(test_slot_state(&member->slot), PCM_XL_CANCELLED); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 1); +} + +UT_TEST(test_local_enqueue_arm_persists_ledger_and_mints_without_holes) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle leader; + PcmXWaitIdentity identity; + PcmXEnqueuePayload first; + PcmXEnqueuePayload replay; + PcmXLocalReliableToken token; + PcmXLocalReliableToken snapshot; + PcmXLocalReliableToken replay_token; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(760, 0, 2, UINT64_C(70001)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(7101)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(7101), &leader), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &first, &token), PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&first.identity, &identity, sizeof(identity)) == 0); + UT_ASSERT_EQ(first.prehandle.sender_session_incarnation, 77); + UT_ASSERT_EQ(first.prehandle.prehandle_sequence, 1); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 2); + UT_ASSERT_EQ(header->outbound_targets[0].next_prehandle_sequence, 1); + UT_ASSERT_EQ(header->outbound_targets[2].next_prehandle_sequence, 1); + UT_ASSERT_EQ(tag_slot->prehandle.sender_session_incarnation, 77); + UT_ASSERT_EQ(tag_slot->prehandle.prehandle_sequence, 1); + UT_ASSERT_EQ(tag_slot->reliable.state_sequence, 1); + UT_ASSERT_EQ(tag_slot->reliable.expected_responder_node, 1); + UT_ASSERT_EQ(tag_slot->reliable.expected_responder_session, 7101); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ENQUEUE); + UT_ASSERT_EQ(tag_slot->reliable.phase, PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &snapshot), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(snapshot.state_sequence, token.state_sequence); + UT_ASSERT(memcmp(&snapshot.identity, &identity, sizeof(identity)) == 0); + UT_ASSERT(memcmp(&snapshot.prehandle, &first.prehandle, sizeof(first.prehandle)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &replay, &replay_token), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(memcmp(&replay, &first, sizeof(first)) == 0); + UT_ASSERT_EQ(replay_token.state_sequence, token.state_sequence); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 2); +} + +UT_TEST(test_local_enqueue_busy_and_counter_exhaustion_leave_no_hole) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle leader; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXLocalReliableToken token; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(761, 0, 2, UINT64_C(70002)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(7102)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(7102), &leader), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + pg_atomic_write_u32(&header->outbound_targets[1].mint_gate, 1); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), + PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 1); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->prehandle.prehandle_sequence, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + pg_atomic_write_u32(&header->outbound_targets[1].mint_gate, 0); + header->outbound_targets[1].next_prehandle_sequence = UINT64_MAX; + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), + PCM_X_QUEUE_COUNTER_EXHAUSTED); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, UINT64_MAX); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->outbound_targets[1].mint_gate), 0); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->prehandle.prehandle_sequence, 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE) != 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + +UT_TEST(test_local_admit_ack_retry_and_confirm_are_exact) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *member; + PcmXLocalHandle leader; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload ack; + PcmXAdmitAckPayload stale_ack; + PcmXPhasePayload confirm; + PcmXPhasePayload stale_confirm; + PcmXLocalReliableToken enqueue_token; + PcmXLocalReliableToken retry_token; + PcmXLocalReliableToken stale_token; + PcmXLocalReliableToken confirm_token; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(762, 0, 2, UINT64_C(70003)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(7103)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(7103), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &enqueue_token), + PCM_X_QUEUE_OK); + stale_token = enqueue_token; + stale_token.state_sequence++; + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_retry_exact(&leader, &stale_token, 41, &retry_token), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ( + cluster_pcm_x_local_reliable_retry_exact(&leader, &enqueue_token, 42, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.state_sequence, enqueue_token.state_sequence); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.retry_deadline_ms, 42); + UT_ASSERT( + memcmp(&retry_token.prehandle, &enqueue_token.prehandle, sizeof(retry_token.prehandle)) + == 0); + + memset(&ack, 0, sizeof(ack)); + ack.ref.identity = identity; + ack.ref.handle.ticket_id = UINT64_C(9001); + ack.ref.handle.queue_generation = UINT64_C(3); + ack.prehandle = enqueue.prehandle; + ack.result = PCM_X_QUEUE_OK; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + ack.flags = PCM_X_ADMIT_F_QUEUE_HEAD; + stale_ack = ack; + stale_ack.prehandle.prehandle_sequence++; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &stale_ack, 1, UINT64_C(7103)), + PCM_X_QUEUE_STALE); + stale_ack = ack; + stale_ack.ref.identity.request_id++; + stale_ack.ref.identity.wait_seq++; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &stale_ack, 1, UINT64_C(7103)), + PCM_X_QUEUE_STALE); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ENQUEUE); + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(7104)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ENQUEUE); + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(7103)), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &ack.ref)); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->reliable.last_response_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_ACK); + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(7103)), + PCM_X_QUEUE_DUPLICATE); + + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &confirm_token), + PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&confirm.ref, &ack.ref)); + UT_ASSERT_EQ(confirm.reason, 0); + UT_ASSERT_EQ(confirm.phase, PCM_X_LOCAL_RELIABLE_PHASE_ADMIT_CONFIRM); + UT_ASSERT_EQ(confirm.flags, 0); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 2); + UT_ASSERT_EQ(tag_slot->reliable.state_sequence, 2); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM); + stale_confirm = confirm; + stale_confirm.ref.handle.ticket_id++; + UT_ASSERT_EQ( + cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &stale_confirm, 1, UINT64_C(7103)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(7103)), + PCM_X_QUEUE_OK); + member = &membership_slots(header)[leader.membership_slot.slot_index]; + UT_ASSERT_EQ(test_slot_state(&member->slot), PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->reliable.last_response_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(7103)), + PCM_X_QUEUE_DUPLICATE); +} + +UT_TEST(test_local_generic_reliable_ack_requires_exact_token_and_keys) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle leader; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload ack; + PcmXPhasePayload confirm; + PcmXLocalReliableToken token; + PcmXLocalReliableToken stale_token; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(763, 0, 2, UINT64_C(70004)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(7104)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(7104), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&ack, 0, sizeof(ack)); + ack.ref.identity = identity; + ack.ref.handle.ticket_id = UINT64_C(9002); + ack.ref.handle.queue_generation = UINT64_C(4); + ack.prehandle = enqueue.prehandle; + ack.result = PCM_X_QUEUE_OK; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &ack, 1, UINT64_C(7104)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), + PCM_X_QUEUE_OK); + stale_token = token; + stale_token.prehandle.prehandle_sequence++; + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_ack_exact(&leader, &stale_token, &confirm.ref, NULL, + PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, 1, + UINT64_C(7104)), + PCM_X_QUEUE_STALE); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_ack_exact(&leader, &token, &confirm.ref, NULL, + PGRAC_IC_MSG_PCM_X_ADMIT_CONFIRM_ACK, 1, + UINT64_C(7104)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(test_slot_state(&membership_slots(header)[leader.membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); +} + +UT_TEST(test_local_pending_enqueue_cancel_keeps_ledger_until_exact_ack_then_rotates_leader) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalTagSlot tag_before; + PcmXLocalMembershipSlot *members; + PcmXLocalMembershipSlot leader_before; + PcmXLocalMembershipSlot follower_before; + PcmXWaitIdentity leader_identity; + PcmXWaitIdentity follower_identity; + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle promoted; + PcmXLocalHandle replay_promoted; + PcmXLocalHandle no_successor; + PcmXEnqueuePayload enqueue; + PcmXEnqueuePayload successor_enqueue; + PcmXPrehandleCancelPayload cancel; + PcmXAdmitAckPayload ack; + PcmXAdmitAckPayload stale_ack; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXLocalReliableToken enqueue_token; + PcmXLocalReliableToken cancel_token; + PcmXLocalReliableToken retry_token; + PcmXLocalReliableToken successor_token; + PcmXWaitIdentity wake_items[1]; + PcmXLocalWakeBatch wake_batch; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + leader_identity = make_wait_identity(766, 0, 12, UINT64_C(71001)); + follower_identity = make_wait_identity(766, 0, 13, UINT64_C(71002)); + bind_local_master(1, leader_identity.cluster_epoch, UINT64_C(7201)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 1, UINT64_C(7201), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 1, UINT64_C(7201), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &enqueue_token), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_prehandle_cancel_arm_exact(&leader, &cancel, &cancel_token), + PCM_X_QUEUE_OK); + UT_ASSERT(memcmp(&cancel.identity, &leader.identity, sizeof(cancel.identity)) == 0); + UT_ASSERT(memcmp(&cancel.prehandle, &enqueue.prehandle, sizeof(cancel.prehandle)) == 0); + UT_ASSERT(cancel_token.state_sequence > enqueue_token.state_sequence); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + members = membership_slots(header); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL); + UT_ASSERT_EQ(tag_slot->reliable.phase, PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL); + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(test_slot_state(&members[leader.membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&leader, &no_successor), PCM_X_QUEUE_BUSY); + UT_ASSERT_EQ(tag_slot->leader_index, leader.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_retry_exact(&leader, &cancel_token, 91, &retry_token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry_token.retry_count, 1); + UT_ASSERT_EQ(retry_token.pending_opcode, PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL); + + memset(&ack, 0, sizeof(ack)); + ack.ref.identity = leader_identity; + ack.ref.handle.ticket_id = UINT64_C(9301); + ack.ref.handle.queue_generation = UINT64_C(7); + ack.prehandle = cancel.prehandle; + ack.result = PCM_X_QUEUE_OK; + ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_PREHANDLE_CANCEL; + stale_ack = ack; + stale_ack.prehandle.prehandle_sequence++; + UT_ASSERT_EQ(cluster_pcm_x_local_prehandle_cancel_ack_exact(&leader, &stale_ack, 1, + UINT64_C(7201), &promoted), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL); + + UT_ASSERT_EQ( + cluster_pcm_x_local_prehandle_cancel_ack_exact(&leader, &ack, 1, UINT64_C(7201), &promoted), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[leader.membership_slot.slot_index].slot), + PCM_XL_CANCELLED); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, follower.membership_slot)); + UT_ASSERT_EQ(tag_slot->leader_index, follower.membership_slot.slot_index); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->reliable.last_response_opcode, PGRAC_IC_MSG_PCM_X_PREHANDLE_CANCEL_ACK); + UT_ASSERT(memcmp(&tag_slot->prehandle, &ack.prehandle, sizeof(ack.prehandle)) == 0); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &ack.ref)); + UT_ASSERT_EQ(members[leader.membership_slot.slot_index].handle.ticket_id, + ack.ref.handle.ticket_id); + UT_ASSERT_EQ(cluster_pcm_x_local_prehandle_cancel_ack_exact(&leader, &ack, 1, UINT64_C(7201), + &replay_promoted), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(slot_refs_equal(replay_promoted.membership_slot, promoted.membership_slot)); + UT_ASSERT_EQ( + cluster_pcm_x_local_enqueue_arm_exact(&promoted, &successor_enqueue, &successor_token), + PCM_X_QUEUE_BUSY); + memset(&poll, 0, sizeof(poll)); + poll.ref = ack.ref; + poll.drain_generation = UINT64_C(1); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, UINT64_C(7201)), PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = ack.ref.identity.cluster_epoch; + retire.master_session_incarnation = UINT64_C(7201); + retire.retire_through_ticket_id = ack.ref.handle.ticket_id; + retire.sender_node = 0; + tag_before = *tag_slot; + leader_before = members[leader.membership_slot.slot_index]; + follower_before = members[follower.membership_slot.slot_index]; + wake_batch.items = NULL; + wake_batch.capacity = 0; + wake_batch.count = 1; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, UINT64_C(7201), &wake_batch), + PCM_X_QUEUE_NO_CAPACITY); + UT_ASSERT_EQ(wake_batch.count, 0); + UT_ASSERT(memcmp(tag_slot, &tag_before, sizeof(*tag_slot)) == 0); + UT_ASSERT( + memcmp(&members[leader.membership_slot.slot_index], &leader_before, sizeof(leader_before)) + == 0); + UT_ASSERT(memcmp(&members[follower.membership_slot.slot_index], &follower_before, + sizeof(follower_before)) + == 0); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); + memset(wake_items, 0, sizeof(wake_items)); + wake_batch.items = wake_items; + wake_batch.capacity = lengthof(wake_items); + wake_batch.count = 0; + UT_ASSERT_EQ( + cluster_pcm_x_local_retire_up_to_collect_exact(&retire, 1, UINT64_C(7201), &wake_batch), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(wake_batch.count, 1); + UT_ASSERT(memcmp(&wake_items[0], &promoted.identity, sizeof(wake_items[0])) == 0); + UT_ASSERT_EQ( + cluster_pcm_x_local_enqueue_arm_exact(&promoted, &successor_enqueue, &successor_token), + PCM_X_QUEUE_OK); + UT_ASSERT( + memcmp(&successor_enqueue.identity, &follower_identity, sizeof(successor_enqueue.identity)) + == 0); + UT_ASSERT_EQ(successor_enqueue.prehandle.prehandle_sequence, 2); + UT_ASSERT_EQ(tag_slot->ref.handle.ticket_id, 0); +} + +UT_TEST(test_local_remote_wait_cancel_is_exact_retriable_and_rotates_only_on_ack) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *members; + PcmXWaitIdentity leader_identity; + PcmXWaitIdentity follower_identity; + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle promoted; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload confirm; + PcmXPhasePayload cancel; + PcmXPhasePayload stale_ack; + PcmXLocalReliableToken token; + PcmXLocalReliableToken retry; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + leader_identity = make_wait_identity(767, 0, 14, UINT64_C(72001)); + follower_identity = make_wait_identity(767, 0, 15, UINT64_C(72002)); + bind_local_master(1, leader_identity.cluster_epoch, UINT64_C(7301)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 1, UINT64_C(7301), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 1, UINT64_C(7301), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = leader_identity; + admit_ack.ref.handle.ticket_id = UINT64_C(9401); + admit_ack.ref.handle.queue_generation = UINT64_C(8); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &admit_ack, 1, UINT64_C(7301)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(7301)), + PCM_X_QUEUE_OK); + + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_arm_exact(&leader, &cancel, &token), PCM_X_QUEUE_OK); + UT_ASSERT(ticket_refs_equal(&cancel.ref, &admit_ack.ref)); + UT_ASSERT_EQ(cancel.phase, PCM_X_LOCAL_RELIABLE_PHASE_CANCEL); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + members = membership_slots(header); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_CANCEL); + UT_ASSERT_EQ(test_slot_state(&members[leader.membership_slot.slot_index].slot), + PCM_XL_REMOTE_WAIT); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_JOINED_NONWAITABLE); + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_retry_exact(&leader, &token, 101, &retry), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(retry.pending_opcode, PGRAC_IC_MSG_PCM_X_CANCEL); + + stale_ack = cancel; + stale_ack.ref.handle.queue_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_local_cancel_ack_exact(&leader, &stale_ack, 1, UINT64_C(7301), &promoted), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_CANCEL); + UT_ASSERT_EQ( + cluster_pcm_x_local_cancel_ack_exact(&leader, &cancel, 1, UINT64_C(7301), &promoted), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(test_slot_state(&members[leader.membership_slot.slot_index].slot), + PCM_XL_CANCELLED); + UT_ASSERT_EQ(test_slot_state(&members[follower.membership_slot.slot_index].slot), + PCM_XL_NODE_LEADER); + UT_ASSERT(slot_refs_equal(promoted.membership_slot, follower.membership_slot)); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->reliable.last_response_opcode, PGRAC_IC_MSG_PCM_X_CANCEL_ACK); +} + +UT_TEST(test_local_exact_prepare_wins_cancel_vs_transfer_race) +{ + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXLocalHandle leader; + PcmXEnqueuePayload enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload confirm; + PcmXPhasePayload cancel; + PcmXGrantPayload prepare; + PcmXGrantPayload stale_prepare; + PcmXInstallReadyPayload install_ready; + PcmXLocalReliableToken token; + const uint64 master_session = UINT64_C(7311); + + init_active_pcm_x(UINT64_C(77)); + identity = make_wait_identity(769, 0, 18, UINT64_C(73101)); + bind_local_master(1, identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = identity; + admit_ack.ref.handle.ticket_id = UINT64_C(9411); + admit_ack.ref.handle.queue_generation = UINT64_C(11); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &admit_ack, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_arm_exact(&leader, &cancel, &token), PCM_X_QUEUE_OK); + + memset(&prepare, 0, sizeof(prepare)); + prepare.ref = admit_ack.ref; + prepare.ref.grant_generation = UINT64_C(12); + UT_ASSERT(cluster_pcm_x_image_id_encode(1, UINT64_C(212), &prepare.image.image_id)); + prepare.image.source_own_generation = UINT64_C(13); + prepare.image.page_scn = UINT64_C(14); + prepare.image.page_lsn = UINT64_C(15); + prepare.image.source_node = 2; + prepare.image.page_checksum = UINT32_C(16); + tag_slot = &local_tag_slots(ClusterPcmXConvertShmem)[leader.tag_slot.slot_index]; + stale_prepare = prepare; + stale_prepare.ref.handle.queue_generation++; + UT_ASSERT_EQ( + cluster_pcm_x_local_prepare_grant_exact(&leader, &stale_prepare, 1, master_session), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_CANCEL); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &admit_ack.ref)); + + UT_ASSERT_EQ(cluster_pcm_x_local_prepare_grant_exact(&leader, &prepare, 1, master_session), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, 0); + UT_ASSERT_EQ(tag_slot->reliable.last_response_opcode, PGRAC_IC_MSG_PCM_X_PREPARE_GRANT); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &prepare.ref)); + UT_ASSERT(memcmp(&tag_slot->image, &prepare.image, sizeof(prepare.image)) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_install_ready_arm_exact(&leader, &prepare.ref, &prepare.image, + &install_ready, &token), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_terminal_drain_retire_fences_successor_and_replays_watermark) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity leader_identity; + PcmXWaitIdentity follower_identity; + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalHandle promoted; + PcmXEnqueuePayload enqueue; + PcmXEnqueuePayload successor_enqueue; + PcmXAdmitAckPayload admit_ack; + PcmXPhasePayload confirm; + PcmXPhasePayload cancel; + PcmXDrainPollPayload poll; + PcmXRetirePayload retire; + PcmXLocalReliableToken token; + PcmXLocalReliableToken successor_token; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + leader_identity = make_wait_identity(768, 0, 16, UINT64_C(73001)); + follower_identity = make_wait_identity(768, 0, 17, UINT64_C(73002)); + bind_local_master(1, leader_identity.cluster_epoch, UINT64_C(7401)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 1, UINT64_C(7401), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 1, UINT64_C(7401), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + memset(&admit_ack, 0, sizeof(admit_ack)); + admit_ack.ref.identity = leader_identity; + admit_ack.ref.handle.ticket_id = UINT64_C(9501); + admit_ack.ref.handle.queue_generation = UINT64_C(9); + admit_ack.prehandle = enqueue.prehandle; + admit_ack.result = PCM_X_QUEUE_OK; + admit_ack.phase = PCM_X_LOCAL_RELIABLE_PHASE_ENQUEUE; + UT_ASSERT_EQ(cluster_pcm_x_local_apply_admit_ack_exact(&leader, &admit_ack, 1, UINT64_C(7401)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_arm_exact(&leader, &confirm, &token), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_admit_confirm_ack_exact(&leader, &confirm, 1, UINT64_C(7401)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_arm_exact(&leader, &cancel, &token), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_local_cancel_ack_exact(&leader, &cancel, 1, UINT64_C(7401), &promoted), + PCM_X_QUEUE_OK); + + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_READY) != 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) == 0); + UT_ASSERT_EQ(cluster_pcm_x_local_terminal_publish_exact(&cancel.ref, 1, UINT64_C(7401)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ( + cluster_pcm_x_local_enqueue_arm_exact(&promoted, &successor_enqueue, &successor_token), + PCM_X_QUEUE_BUSY); + UT_ASSERT(ticket_refs_equal(&tag_slot->ref, &cancel.ref)); + + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = leader_identity.cluster_epoch; + retire.master_session_incarnation = UINT64_C(7401); + retire.retire_through_ticket_id = cancel.ref.handle.ticket_id; + retire.sender_node = 0; + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, UINT64_C(7401)), + PCM_X_QUEUE_NOT_READY); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retired_ticket_id, 0); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); + + memset(&poll, 0, sizeof(poll)); + poll.ref = cancel.ref; + poll.drain_generation = UINT64_C(41); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, UINT64_C(7402)), PCM_X_QUEUE_STALE); + poll.ref.handle.queue_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, UINT64_C(7401)), PCM_X_QUEUE_STALE); + poll.ref = cancel.ref; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, UINT64_C(7401)), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(tag_slot->terminal_drain_generation, UINT64_C(41)); + UT_ASSERT((test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED) != 0); + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, UINT64_C(7401)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_local_terminal_publish_exact(&cancel.ref, 1, UINT64_C(7401)), + PCM_X_QUEUE_DUPLICATE); + poll.drain_generation++; + UT_ASSERT_EQ(cluster_pcm_x_local_drain_poll_exact(&poll, 1, UINT64_C(7401)), PCM_X_QUEUE_STALE); + + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, UINT64_C(7402)), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, UINT64_C(7401)), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retired_ticket_id, + retire.retire_through_ticket_id); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 0); + UT_ASSERT((test_slot_flags(&tag_slot->slot) + & (PCM_X_LOCAL_TAG_F_TERMINAL_READY | PCM_X_LOCAL_TAG_F_TERMINAL_DRAINED)) + == 0); + UT_ASSERT_EQ(tag_slot->ref.handle.ticket_id, 0); + UT_ASSERT_EQ(cluster_pcm_x_local_retire_up_to_exact(&retire, 1, UINT64_C(7401)), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ( + cluster_pcm_x_local_enqueue_arm_exact(&promoted, &successor_enqueue, &successor_token), + PCM_X_QUEUE_OK); +} + +UT_TEST(test_local_reliable_authority_change_fails_closed_without_dropping_ledger) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalHandle leader; + PcmXWaitIdentity identity; + PcmXEnqueuePayload enqueue; + PcmXLocalReliableToken token; + PcmXLocalReliableToken snapshot; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(764, 0, 2, UINT64_C(70005)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(7105)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(7105), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + header->outbound_targets[1].target_session_incarnation++; + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ENQUEUE); + UT_ASSERT_EQ(tag_slot->prehandle.prehandle_sequence, 1); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 2); + + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(765, 0, 2, UINT64_C(70006)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(7106)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(7106), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_enqueue_arm_exact(&leader, &enqueue, &token), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[leader.tag_slot.slot_index]; + header->peer_frontiers[1].sender_session_incarnation++; + UT_ASSERT_EQ(cluster_pcm_x_local_reliable_snapshot_exact(&leader, &snapshot), + PCM_X_QUEUE_STALE); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(tag_slot->reliable.pending_opcode, PGRAC_IC_MSG_PCM_X_ENQUEUE); + UT_ASSERT_EQ(tag_slot->prehandle.prehandle_sequence, 1); + UT_ASSERT_EQ(header->outbound_targets[1].next_prehandle_sequence, 2); +} + +UT_TEST(test_capacity_addition_overflow_is_checked) +{ + PcmXShmemLayout layout; + + UT_EXPECT_ARITHMETIC_OVERFLOW(cluster_pcm_x_layout_compute(SIZE_MAX, 1, 1, 1, &layout)); +} + +UT_TEST(test_capacity_multiplication_overflow_is_checked) +{ + PcmXShmemLayout layout; + + UT_EXPECT_ARITHMETIC_OVERFLOW( + cluster_pcm_x_layout_compute(SIZE_MAX / PCM_X_PROTOCOL_NODE_LIMIT + 1, 0, 1, 1, &layout)); +} + +UT_TEST(test_ticket_directory_capacity_doubling_overflow_is_checked) +{ + PcmXShmemLayout layout; + Size p = SIZE_MAX / PCM_X_PROTOCOL_NODE_LIMIT; + Size c = p * PCM_X_PROTOCOL_NODE_LIMIT; + + UT_ASSERT(c > SIZE_MAX / 2); + UT_EXPECT_ARITHMETIC_OVERFLOW(cluster_pcm_x_layout_compute(p, 0, 0, 1, &layout)); + UT_ASSERT_EQ(arithmetic_overflow_left, c); + UT_ASSERT_EQ(arithmetic_overflow_right, 2); +} + +UT_TEST(test_fresh_init_builds_independent_free_lists) +{ + PcmXShmemHeader *header; + PcmXLocalMembershipSlot *slots; + PcmXLocalMembershipSlot *holder_slots; + Size p; + Size h; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + slots = membership_slots(header); + holder_slots + = (PcmXLocalMembershipSlot *)((char *)header + header->layout.local_holder.slots_offset); + p = header->layout.local_wait.capacity; + h = header->layout.local_holder.capacity; + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].free_head, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].free_head, p); + UT_ASSERT_EQ(&holder_slots[header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].free_head + - header->layout.local_holder.first_slot_index], + &slots[p]); + UT_ASSERT_EQ(slots[0].slot.next_free, 1); + UT_ASSERT_EQ(slots[p - 1].slot.next_free, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(slots[p].slot.next_free, p + 1); + UT_ASSERT_EQ(slots[p + h - 1].slot.next_free, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].used, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_WAIT].high_water, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_LOCAL_HOLDER].high_water, 0); +} + +UT_TEST(test_fresh_init_is_recovery_blocked_and_initializes_all_locks_once) +{ + PcmXShmemHeader *header; + int i; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + UT_ASSERT_EQ(header->master_session_incarnation, 0); + UT_ASSERT_EQ(header->next_ticket_id, 1); + UT_ASSERT_EQ(header->next_image_id, 1); + UT_ASSERT_EQ(header->fully_retired_ticket_id, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->runtime_gate), PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(lwlock_init_count, PCM_X_LWLOCK_COUNT); + UT_ASSERT_EQ(allocator_lock_init_count, 1); + UT_ASSERT_EQ(master_lock_init_count, PCM_X_LOCK_PARTITIONS); + UT_ASSERT_EQ(local_lock_init_count, PCM_X_LOCK_PARTITIONS); + for (i = 0; i < PCM_X_PROTOCOL_NODE_LIMIT; i++) { + UT_ASSERT_EQ(header->peer_frontiers[i].cluster_epoch, 0); + UT_ASSERT_EQ(header->peer_frontiers[i].sender_session_incarnation, 0); + UT_ASSERT_EQ(header->peer_frontiers[i].next_expected_prehandle_sequence, 1); + UT_ASSERT_EQ(header->peer_frontiers[i].retired_prehandle_sequence, 0); + } +} + +UT_TEST(test_fresh_init_sets_every_slot_free_at_generation_zero) +{ + PcmXShmemHeader *header; + int pool; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + for (pool = 0; pool < PCM_X_POOL_COUNT; pool++) { + Size i; + + for (i = 0; i < header->layout.pools[pool].capacity; i++) { + PcmXSlotHeader *slot = pool_slot_header(header, (PcmXPoolKind)pool, i); + + UT_ASSERT_EQ(test_slot_generation(slot), 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&slot->generation_change_seq), 0); + UT_ASSERT_EQ(test_slot_state(slot), PCM_X_SLOT_FREE); + UT_ASSERT_EQ(test_slot_flags(slot), 0); + } + } +} + +UT_TEST(test_exec_backend_attach_preserves_mutable_state) +{ + PcmXShmemHeader *header; + PcmXRuntimeSnapshot snapshot; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + UT_ASSERT(cluster_pcm_x_runtime_activate(71)); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(3, 9, 44), PCM_X_QUEUE_OK); + header->allocator[PCM_X_ALLOC_MASTER_TICKET].used = 7; + header->peer_frontiers[3].next_expected_prehandle_sequence = 8; + header->peer_frontiers[3].retired_prehandle_sequence = 6; + header->outbound_targets[3].next_prehandle_sequence = 8; + cluster_pcm_x_convert_shmem_init(); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(ClusterPcmXConvertShmem, header); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_MASTER_TICKET].used, 7); + UT_ASSERT_EQ(header->peer_frontiers[3].cluster_epoch, 9); + UT_ASSERT_EQ(header->peer_frontiers[3].sender_session_incarnation, 44); + UT_ASSERT_EQ(header->peer_frontiers[3].next_expected_prehandle_sequence, 8); + UT_ASSERT_EQ(header->peer_frontiers[3].retired_prehandle_sequence, 6); + UT_ASSERT_EQ(header->outbound_targets[3].cluster_epoch, 9); + UT_ASSERT_EQ(header->outbound_targets[3].target_session_incarnation, 44); + UT_ASSERT_EQ(header->outbound_targets[3].next_prehandle_sequence, 8); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 71); + UT_ASSERT_EQ(snapshot.gate_generation, 1); + UT_ASSERT_EQ(lwlock_init_count, PCM_X_LWLOCK_COUNT); +} + +UT_TEST(test_runtime_gate_is_atomic_and_session_fenced) +{ + PcmXRuntimeSnapshot snapshot; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 0); + UT_ASSERT_EQ(snapshot.gate_generation, 0); + UT_ASSERT(!cluster_pcm_x_runtime_activate(0)); + UT_ASSERT(cluster_pcm_x_runtime_activate(77)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 77); + UT_ASSERT_EQ(snapshot.gate_generation, 1); + UT_ASSERT(!cluster_pcm_x_runtime_activate(78)); + UT_ASSERT( + !cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_RECOVERY_BLOCKED, PCM_X_RUNTIME_ACTIVE)); + UT_ASSERT( + cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 0); + UT_ASSERT_EQ(snapshot.gate_generation, 2); + /* A normal fail-stop close is not an implicit recovery authority. */ + UT_ASSERT(!cluster_pcm_x_runtime_activate(88)); +} + +UT_TEST(test_runtime_activating_phase_has_exact_recovery_reset) +{ + PcmXShmemHeader *header; + PcmXRuntimeSnapshot snapshot; + uint32 activating_gate; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + header->master_session_incarnation = 77; + activating_gate = (UINT32_C(3) << PCM_X_RUNTIME_GATE_STATE_BITS) | UINT32_C(3); + pg_atomic_write_u32(&header->runtime_gate, activating_gate); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(snapshot.gate_generation, 3); + UT_ASSERT(!cluster_pcm_x_runtime_reset_activating(2)); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->runtime_gate), activating_gate); + UT_ASSERT(cluster_pcm_x_runtime_reset_activating(3)); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->activation_retry_generation), 3); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->runtime_gate), + (UINT32_C(3) << PCM_X_RUNTIME_GATE_STATE_BITS) | PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT(cluster_pcm_x_runtime_activate(78)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 78); + UT_ASSERT_EQ(snapshot.gate_generation, 4); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->activation_retry_generation), 0); +} + +UT_TEST(test_runtime_gate_generation_prevents_active_aba) +{ + PcmXShmemHeader *header; + PcmXRuntimeSnapshot first; + PcmXRuntimeSnapshot second; + uint32 activating_gate; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_runtime_activate(77)); + first = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT( + cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED)); + UT_ASSERT(!cluster_pcm_x_runtime_activate(77)); + UT_ASSERT(!cluster_pcm_x_runtime_activate(88)); + + /* Only an exact ACTIVATING reset authorizes a later-session retry. */ + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + header->master_session_incarnation = 77; + activating_gate = (UINT32_C(3) << PCM_X_RUNTIME_GATE_STATE_BITS) | UINT32_C(3); + pg_atomic_write_u32(&header->runtime_gate, activating_gate); + UT_ASSERT(cluster_pcm_x_runtime_reset_activating(3)); + UT_ASSERT(cluster_pcm_x_runtime_activate(88)); + second = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(first.state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(second.state, PCM_X_RUNTIME_ACTIVE); + UT_ASSERT_EQ(first.master_session_incarnation, 77); + UT_ASSERT_EQ(second.master_session_incarnation, 88); + UT_ASSERT(first.gate_generation != second.gate_generation); +} + +UT_TEST(test_runtime_gate_generation_exhaustion_never_wraps) +{ + PcmXShmemHeader *header; + PcmXRuntimeSnapshot snapshot; + uint32 gate; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + gate = ((PCM_X_RUNTIME_GATE_GENERATION_MAX - 2) << PCM_X_RUNTIME_GATE_STATE_BITS) + | PCM_X_RUNTIME_RECOVERY_BLOCKED; + pg_atomic_write_u32(&header->runtime_gate, gate); + pg_atomic_write_u32(&header->activation_retry_generation, + PCM_X_RUNTIME_GATE_GENERATION_MAX - 2); + UT_ASSERT(cluster_pcm_x_runtime_activate(99)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.gate_generation, PCM_X_RUNTIME_GATE_GENERATION_MAX - 1); + UT_ASSERT( + cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_RECOVERY_BLOCKED)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(snapshot.gate_generation, PCM_X_RUNTIME_GATE_GENERATION_MAX); + UT_ASSERT(!cluster_pcm_x_runtime_activate(100)); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->runtime_gate), + ((PCM_X_RUNTIME_GATE_GENERATION_MAX << PCM_X_RUNTIME_GATE_STATE_BITS) + | PCM_X_RUNTIME_RECOVERY_BLOCKED)); +} + +UT_TEST(test_runtime_gate_generation_exhaustion_fails_closed) +{ + PcmXShmemHeader *header; + PcmXRuntimeSnapshot snapshot; + uint32 gate; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + gate = ((PCM_X_RUNTIME_GATE_GENERATION_MAX - 1) << PCM_X_RUNTIME_GATE_STATE_BITS) + | PCM_X_RUNTIME_ACTIVE; + header->master_session_incarnation = 99; + pg_atomic_write_u32(&header->runtime_gate, gate); + UT_ASSERT(!cluster_pcm_x_runtime_transition(PCM_X_RUNTIME_ACTIVE, PCM_X_RUNTIME_SHUTTING_DOWN)); + snapshot = cluster_pcm_x_runtime_snapshot(); + UT_ASSERT_EQ(snapshot.state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + UT_ASSERT_EQ(snapshot.master_session_incarnation, 0); + UT_ASSERT_EQ(snapshot.gate_generation, PCM_X_RUNTIME_GATE_GENERATION_MAX); +} + +UT_TEST(test_master_state_machine_requires_exact_positive_gates) +{ + PcmXMasterTicketState next; + + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_ADMITTING, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_EXACT, &next), + PCM_X_STEP_BLOCKED); + UT_ASSERT_EQ(next, PCM_XT_ADMITTING); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_ADMITTING, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_EXACT | PCM_X_G_WFG_READY, &next), + PCM_X_STEP_APPLIED); + UT_ASSERT_EQ(next, PCM_XT_QUEUED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_QUEUED, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_EXACT | PCM_X_G_WFG_READY, &next), + PCM_X_STEP_BLOCKED); + UT_ASSERT_EQ(next, PCM_XT_QUEUED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_QUEUED, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_EXACT | PCM_X_G_CONFIRM_TOMBSTONE, &next), + PCM_X_STEP_DUPLICATE); + UT_ASSERT_EQ(next, PCM_XT_QUEUED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_CANCELLED, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_EXACT, &next), + PCM_X_STEP_BLOCKED); + UT_ASSERT_EQ(next, PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_CANCELLED, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_EXACT | PCM_X_G_CONFIRM_TOMBSTONE, &next), + PCM_X_STEP_DUPLICATE); + UT_ASSERT_EQ(next, PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_ADMITTING, PCM_X_EVENT_ADMIT_CONFIRM, + PCM_X_G_WFG_READY, &next), + PCM_X_STEP_STALE); + UT_ASSERT_EQ(next, PCM_XT_ADMITTING); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_RETIRE_CREDIT, PCM_X_EVENT_COMMIT_COMPLETE, + PCM_X_G_EXACT, &next), + PCM_X_STEP_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_RETIRE_CREDIT, PCM_X_EVENT_COMMIT_COMPLETE, + PCM_X_G_EXACT | PCM_X_G_COMPLETE_TOMBSTONE, &next), + PCM_X_STEP_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_RETIRE_CREDIT, PCM_X_EVENT_CANCEL_EXACT, + PCM_X_G_EXACT | PCM_X_G_COMPLETE_TOMBSTONE, &next), + PCM_X_STEP_BLOCKED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_RETIRE_CREDIT, PCM_X_EVENT_CANCEL_EXACT, + PCM_X_G_EXACT | PCM_X_G_CANCEL_TOMBSTONE, &next), + PCM_X_STEP_DUPLICATE); +} + +UT_TEST(test_cancel_and_ownership_reservation_obey_authorization_boundary) +{ + PcmXMasterTicketState next; + + UT_ASSERT(!cluster_pcm_x_ownership_reservation_allowed(PCM_XT_ADMITTING)); + UT_ASSERT(!cluster_pcm_x_ownership_reservation_allowed(PCM_XT_QUEUED)); + UT_ASSERT(!cluster_pcm_x_ownership_reservation_allowed(PCM_XT_ACTIVE_PROBE)); + UT_ASSERT(cluster_pcm_x_ownership_reservation_allowed(PCM_XT_ACTIVE_TRANSFER)); + UT_ASSERT(!cluster_pcm_x_ownership_reservation_allowed(PCM_XT_COMPLETE)); + + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_QUEUED, PCM_X_EVENT_CANCEL_EXACT, + PCM_X_G_EXACT | PCM_X_G_REVERSIBLE, &next), + PCM_X_STEP_APPLIED); + UT_ASSERT_EQ(next, PCM_XT_CANCELLED); + UT_ASSERT_EQ(cluster_pcm_x_master_step(PCM_XT_ACTIVE_TRANSFER, PCM_X_EVENT_CANCEL_EXACT, + PCM_X_G_EXACT | PCM_X_G_REVERSIBLE, &next), + PCM_X_STEP_APPLIED); + UT_ASSERT_EQ(next, PCM_XT_RECOVERY_BLOCKED); +} + +UT_TEST(test_retire_requires_exact_drain_and_peer_ack_without_time) +{ + PcmXMasterTicketSlot ticket; + + memset(&ticket, 0, sizeof(ticket)); + pg_atomic_init_u32(&ticket.slot.generation_change_seq, 0); + ticket.slot.slot_generation_lo = 0; + ticket.slot.slot_generation_hi = 0; + pg_atomic_init_u32(&ticket.slot.state_flags, PCM_XT_COMPLETE); + ticket.involved_nodes_bitmap = UINT32_C(0x5); + ticket.drained_nodes_bitmap = UINT32_C(0x1); + ticket.reliable.retry_deadline_ms = 0; + UT_ASSERT(!cluster_pcm_x_ticket_drain_ready(&ticket)); + ticket.reliable.retry_deadline_ms = UINT64_MAX; + UT_ASSERT(!cluster_pcm_x_ticket_drain_ready(&ticket)); + ticket.drained_nodes_bitmap = ticket.involved_nodes_bitmap; + UT_ASSERT(cluster_pcm_x_ticket_drain_ready(&ticket)); + + test_set_slot_state(&ticket.slot, PCM_XT_RETIRE_CREDIT); + ticket.retire_acked_nodes_bitmap = UINT32_C(0x1); + UT_ASSERT(!cluster_pcm_x_ticket_retire_ready(&ticket)); + ticket.retire_acked_nodes_bitmap = ticket.involved_nodes_bitmap; + UT_ASSERT(cluster_pcm_x_ticket_retire_ready(&ticket)); + ticket.drained_nodes_bitmap = UINT32_C(0x1); + UT_ASSERT(!cluster_pcm_x_ticket_retire_ready(&ticket)); +} + +UT_TEST(test_lock_partition_is_bounded_and_stable) +{ + UT_ASSERT_EQ(cluster_pcm_x_lock_partition(0), 0); + UT_ASSERT_EQ(cluster_pcm_x_lock_partition(PCM_X_LOCK_PARTITIONS), 0); + UT_ASSERT_EQ(cluster_pcm_x_lock_partition(PCM_X_LOCK_PARTITIONS + 7), 7); + UT_ASSERT(cluster_pcm_x_lock_partition(UINT32_MAX) < PCM_X_LOCK_PARTITIONS); +} + +UT_TEST(test_attach_validator_rejects_magic_capacity_and_offset_mismatch) +{ + PcmXShmemHeader *header; + PcmXShmemLayout expected; + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + compute_runtime_layout(&expected); + UT_ASSERT_EQ(cluster_pcm_x_validate_attach(header, &expected), PCM_X_ATTACH_OK); + header->layout.magic++; + UT_ASSERT_EQ(cluster_pcm_x_validate_attach(header, &expected), PCM_X_ATTACH_MAGIC_MISMATCH); + header->layout.magic = expected.magic; + header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity++; + UT_ASSERT_EQ(cluster_pcm_x_validate_attach(header, &expected), PCM_X_ATTACH_LAYOUT_MISMATCH); + header->layout.pools[PCM_X_POOL_MASTER_TAG].capacity + = expected.pools[PCM_X_POOL_MASTER_TAG].capacity; + header->layout.master_ticket_directories.prehandle_offset++; + UT_ASSERT_EQ(cluster_pcm_x_validate_attach(header, &expected), PCM_X_ATTACH_LAYOUT_MISMATCH); + header->layout.master_ticket_directories.prehandle_offset + = expected.master_ticket_directories.prehandle_offset; + header->layout.local_holder.slots_offset++; + UT_ASSERT_EQ(cluster_pcm_x_validate_attach(header, &expected), PCM_X_ATTACH_LAYOUT_MISMATCH); +} + +UT_TEST(test_exec_backend_init_fails_closed_on_layout_mismatch) +{ + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + ClusterPcmXConvertShmem->layout.version++; + UT_EXPECT_EREPORT(cluster_pcm_x_convert_shmem_init()); + UT_ASSERT_NULL(ClusterPcmXConvertShmem); +} + +UT_TEST(test_registration_exposes_one_exact_region) +{ + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_register(); + UT_ASSERT_EQ(registered_region_count, 1); + UT_ASSERT_NOT_NULL(registered_region); + UT_ASSERT_STR_EQ(registered_region->name, PCM_X_SHMEM_REGION_NAME); + UT_ASSERT_STR_EQ(registered_region->owner_subsys, "pcm_x_convert"); + UT_ASSERT_EQ(registered_region->size_fn, cluster_pcm_x_convert_shmem_size); + UT_ASSERT_EQ(registered_region->init_fn, cluster_pcm_x_convert_shmem_init); + UT_ASSERT_EQ(registered_region->lwlock_count, PCM_X_LWLOCK_COUNT); + UT_ASSERT_EQ(registered_region->reserved_flags, 0); +} + + +UT_TEST(test_stats_initialize_zero_and_narrow_note_apis_are_exact) +{ + PcmXStatsSnapshot stats; + PcmXStatsSnapshot zero = { 0 }; + + reset_fake_shmem(); + memset(&stats, 0xff, sizeof(stats)); + UT_ASSERT(!cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT(memcmp(&stats, &zero, sizeof(stats)) == 0); + UT_ASSERT(!cluster_pcm_x_stats_snapshot(NULL)); + + cluster_pcm_x_convert_shmem_init(); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT(memcmp(&stats, &zero, sizeof(stats)) == 0); + + cluster_pcm_x_stats_note_enqueue(); + cluster_pcm_x_stats_note_wait(); + cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_NO_CAPACITY); + cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_STALE); + cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_NOT_FOUND); + cluster_pcm_x_stats_note_queue_result(PCM_X_QUEUE_INVALID); + cluster_pcm_x_stats_note_own_begin(); + cluster_pcm_x_stats_note_own_commit(); + cluster_pcm_x_stats_note_own_abort(); + cluster_pcm_x_stats_note_own_busy(); + cluster_pcm_x_stats_note_own_corrupt(); + + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.enqueue_count, 1); + UT_ASSERT_EQ(stats.wait_count, 1); + UT_ASSERT_EQ(stats.full_count, 1); + UT_ASSERT_EQ(stats.stale_count, 1); + UT_ASSERT_EQ(stats.miss_count, 1); + UT_ASSERT_EQ(stats.own_begin_count, 1); + UT_ASSERT_EQ(stats.own_commit_count, 1); + UT_ASSERT_EQ(stats.own_abort_count, 1); + UT_ASSERT_EQ(stats.own_busy_count, 1); + UT_ASSERT_EQ(stats.own_corrupt_count, 1); + UT_ASSERT_EQ(stats.admit_count, 0); + UT_ASSERT_EQ(stats.depth, 0); + UT_ASSERT_EQ(stats.active_tags, 0); + UT_ASSERT_EQ(stats.live_tickets, 0); + UT_ASSERT_EQ(stats.live_slots, 0); + UT_ASSERT_EQ(stats.local_retire_gate, 0); + UT_ASSERT_EQ(stats.local_retire_marker_count, 0); + UT_ASSERT_EQ(stats.local_retire_marker_ticket_id, 0); + pg_atomic_write_u32(&ClusterPcmXConvertShmem->local_retire_gate, 2); + ClusterPcmXConvertShmem->peer_frontiers[1].local_retire_in_progress_ticket_id = UINT64_C(8711); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.local_retire_gate, 2); + UT_ASSERT_EQ(stats.local_retire_marker_count, 1); + UT_ASSERT_EQ(stats.local_retire_marker_ticket_id, UINT64_C(8711)); + ClusterPcmXConvertShmem->peer_frontiers[1].local_retire_in_progress_ticket_id = 0; + pg_atomic_write_u32(&ClusterPcmXConvertShmem->local_retire_gate, 0); +} + + +UT_TEST(test_stats_follow_exact_master_and_local_success_transitions) +{ + PcmXMasterAdmission admission[2]; + PcmXTicketRef active; + PcmXTicketRef transfer; + PcmXEnqueuePayload request[2]; + PcmXLocalHandle leader; + PcmXLocalHandle follower; + PcmXLocalCutoff cutoff; + PcmXWaitIdentity leader_identity; + PcmXWaitIdentity follower_identity; + PcmXStatsSnapshot stats; + int i; + + init_active_pcm_x(UINT64_C(77)); + for (i = 0; i < 2; i++) { + request[i] = make_enqueue(make_wait_identity(760, i, (uint32)(2 + i), UINT64_C(70001) + i), + UINT64_C(7101) + i, 1); + bind_enqueue_peer(&request[i]); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request[i], &admission[i]), PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(7201) + i), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_admit_confirm_exact(&admission[i].ref, UINT64_C(7201) + i), + PCM_X_QUEUE_DUPLICATE); + } + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.admit_count, 2); + UT_ASSERT_EQ(stats.confirm_count, 2); + UT_ASSERT_EQ(stats.depth, 2); + UT_ASSERT_EQ(stats.depth_high_water, 2); + UT_ASSERT_EQ(stats.active_tags, 1); + UT_ASSERT_EQ(stats.live_tickets, 2); + UT_ASSERT_EQ(stats.live_slots, 3); + + UT_ASSERT_EQ(cluster_pcm_x_master_promote_head_exact( + &request[0].identity.tag, request[0].identity.cluster_epoch, &active), + PCM_X_QUEUE_OK); + commit_empty_blocker_graph(&active, UINT64_C(7201)); + UT_ASSERT_EQ(cluster_pcm_x_master_begin_transfer_exact(&active, UINT64_C(7201), &transfer), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ( + cluster_pcm_x_master_begin_transfer_exact(&admission[0].ref, UINT64_C(7201), &active), + PCM_X_QUEUE_DUPLICATE); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_complete_exact(&transfer), PCM_X_QUEUE_DUPLICATE); + admission[0].ref = transfer; + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission[1].ref), PCM_X_QUEUE_DUPLICATE); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.promotion_count, 1); + UT_ASSERT_EQ(stats.transfer_count, 1); + UT_ASSERT_EQ(stats.complete_count, 1); + UT_ASSERT_EQ(stats.cancel_count, 1); + UT_ASSERT_EQ(stats.depth, 0); + UT_ASSERT_EQ(stats.depth_high_water, 2); + + for (i = 0; i < 2; i++) + drain_retire_and_detach_master(&admission[i]); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.active_tags, 0); + UT_ASSERT_EQ(stats.live_tickets, 0); + UT_ASSERT_EQ(stats.live_slots, 0); + + leader_identity = make_wait_identity(761, 0, 2, UINT64_C(70003)); + follower_identity = make_wait_identity(761, 0, 3, UINT64_C(70004)); + bind_local_master(2, leader_identity.cluster_epoch, UINT64_C(7401)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&leader_identity, 2, UINT64_C(7401), &leader), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&follower_identity, 2, UINT64_C(7401), &follower), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(follower.role, PCM_X_LOCAL_ROLE_FOLLOWER); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&leader, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.coalesced_count, 1); + UT_ASSERT_EQ(stats.revoke_count, 1); + UT_ASSERT_EQ(stats.live_slots, 3); +} + + +UT_TEST(test_stats_count_fail_closed_and_exact_activating_reset_once) +{ + PcmXShmemHeader *header; + PcmXStatsSnapshot stats; + uint32 activating_gate; + + init_active_pcm_x(UINT64_C(77)); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 9, 601), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 10, 601), PCM_X_QUEUE_STALE); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.recovery_blocked_count, 1); + UT_ASSERT_EQ(cluster_pcm_x_peer_bind_ack_publish(0, 10, 601), PCM_X_QUEUE_NOT_READY); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.recovery_blocked_count, 1); + + reset_fake_shmem(); + cluster_pcm_x_convert_shmem_init(); + header = ClusterPcmXConvertShmem; + header->master_session_incarnation = 77; + activating_gate = (UINT32_C(3) << PCM_X_RUNTIME_GATE_STATE_BITS) | UINT32_C(3); + pg_atomic_write_u32(&header->runtime_gate, activating_gate); + UT_ASSERT(!cluster_pcm_x_runtime_reset_activating(2)); + UT_ASSERT(cluster_pcm_x_runtime_reset_activating(3)); + UT_ASSERT(!cluster_pcm_x_runtime_reset_activating(3)); + UT_ASSERT(cluster_pcm_x_stats_snapshot(&stats)); + UT_ASSERT_EQ(stats.activating_reset_count, 1); +} + + +UT_TEST(test_master_admit_lock_window_error_releases_gate_and_fails_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload request; + PcmXSlotRef tag_ref; + uint32 partition; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(800, 0, 2, UINT64_C(80001)), UINT64_C(8101), 1); + bind_enqueue_peer(&request); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&request.identity.tag)); + arm_lwlock_acquire_error(&header->master_locks[partition].lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_master_admit_begin(&request, &admission)); + + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_MASTER_TAG, &request.identity.tag, &tag_ref), + PCM_X_DIRECTORY_OK); + tag_slot = &master_tag_slots(header)[tag_ref.slot_index]; + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +UT_TEST(test_local_join_lock_window_error_releases_gate_and_fails_closed) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handle; + PcmXLocalTagSlot *tag_slot; + PcmXWaitIdentity identity; + PcmXSlotRef tag_ref; + uint32 partition; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(801, 0, 3, UINT64_C(80002)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(8102)); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&identity.tag)); + arm_lwlock_acquire_error(&header->local_locks[partition].lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION( + (void)cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(8102), &handle)); + + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &identity.tag, &tag_ref), + PCM_X_DIRECTORY_OK); + tag_slot = &local_tag_slots(header)[tag_ref.slot_index]; + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +UT_TEST(test_master_detach_lock_window_error_releases_gate_and_fails_closed) +{ + PcmXShmemHeader *header; + PcmXMasterAdmission admission; + PcmXMasterTagSlot *tag_slot; + PcmXEnqueuePayload request; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + request = make_enqueue(make_wait_identity(802, 0, 4, UINT64_C(80003)), UINT64_C(8103), 1); + bind_enqueue_peer(&request); + UT_ASSERT_EQ(cluster_pcm_x_master_admit_begin(&request, &admission), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_master_cancel_exact(&admission.ref), PCM_X_QUEUE_OK); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_DRAIN, + admission.ref.identity.node_id); + arm_and_ack_master_terminal_leg(&admission.ref, PCM_X_TERMINAL_LEG_RETIRE, + admission.ref.identity.node_id); + tag_slot = &master_tag_slots(header)[admission.tag_slot.slot_index]; + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 2); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_master_detach_terminal_exact(&admission.ref)); + + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(pg_atomic_read_u32(&tag_slot->admission_gate), 0); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +UT_TEST(test_local_detach_lock_window_error_releases_gate_and_fails_closed) +{ + PcmXShmemHeader *header; + PcmXLocalCutoff cutoff; + PcmXLocalHandle handle; + PcmXLocalHandle next_leader; + PcmXLocalTagSlot *tag_slot; + PcmXSlotRef found; + PcmXWaitIdentity identity; + + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(803, 0, 5, UINT64_C(80004)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(8104)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(8104), &handle), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handle, NULL), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[handle.tag_slot.slot_index]; + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_detach_terminal_exact(&handle)); + + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* retire_round also crosses from the local partition to the allocator + * while retaining the admission gate. An exception in that handoff must + * leave the exact DETACHING tag and directory entry as recovery evidence. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(804, 0, 6, UINT64_C(80005)); + bind_local_master(1, identity.cluster_epoch, UINT64_C(8105)); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, UINT64_C(8105), &handle), + PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_begin_revoke_cutoff_exact(&handle, &cutoff), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_cancel_exact(&handle, NULL), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_detach_terminal_exact(&handle), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[handle.tag_slot.slot_index]; + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_retire_round_exact( + &identity.tag, identity.cluster_epoch, &cutoff, &next_leader)); + + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &identity.tag, &found), + PCM_X_DIRECTORY_OK); + UT_ASSERT(slot_refs_equal(found, handle.tag_slot)); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +UT_TEST(test_local_holder_gate_handoffs_fail_closed) +{ + PcmXShmemHeader *header; + PcmXLocalTagSlot *tag_slot; + PcmXLocalMembershipSlot *holder_slot; + PcmXLocalHolderKey holder_key; + PcmXLocalHolderHandle holder; + PcmXLocalHolderHandle holder_copy; + PcmXLocalHolderSnapshot holder_snapshot; + PcmXLocalBlockerSnapshot blocker_snapshot; + PcmXTicketRef probe_ref; + PcmXSlotRef tag_ref; + PcmXSlotRef holder_ref; + ClusterLmdVertex blocker; + Size blocker_head; + uint32 partition; + const uint64 master_session = UINT64_C(8201); + + /* Holder registration publishes RESERVED tag/holder identities before the + * local-domain handoff. An ERROR must release the gate but retain both + * exact identities under RECOVERY_BLOCKED. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + holder_key = make_local_holder_key(805, 0, 9, UINT64_C(80501), 3); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&holder_key.identity.tag)); + arm_lwlock_acquire_error(&header->local_locks[partition].lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_holder_register(&holder_key, &holder)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &holder_key.identity.tag, &tag_ref), + PCM_X_DIRECTORY_OK); + UT_ASSERT_EQ(cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_HOLDER, &holder_key, &holder_ref), + PCM_X_DIRECTORY_OK); + tag_slot = &local_tag_slots(header)[tag_ref.slot_index]; + holder_slot = &membership_slots(header)[holder_ref.slot_index]; + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_RESERVED_NONVISIBLE); + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_X_SLOT_RESERVED_NONVISIBLE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Ordinary holder detach has the inverse shape: DETACHING identities are + * already durable when it crosses back to the allocator. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + holder_key = make_local_holder_key(806, 0, 10, UINT64_C(80601), 4); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_mark_releasing_exact(&holder), PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[holder.tag_slot.slot_index]; + holder_slot = &membership_slots(header)[holder.holder_slot.slot_index]; + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_holder_unregister_exact(&holder)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_DETACHING); + UT_ASSERT_EQ(test_slot_state(&holder_slot->slot), PCM_XL_DETACHING); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* A tag-only PROBE reservation owns the same gate before taking the local + * lock; cover the transfer-tag prepare path independently of registration. */ + init_active_pcm_x(UINT64_C(79)); + header = ClusterPcmXConvertShmem; + bind_local_master(1, UINT64_C(9), master_session); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(807, 2, 11, UINT64_C(80701)); + probe_ref.handle.ticket_id = UINT64_C(8202); + probe_ref.handle.queue_generation = UINT64_C(1); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&probe_ref.identity.tag)); + arm_lwlock_acquire_error(&header->local_locks[partition].lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_probe_freeze_snapshot_exact( + &probe_ref, 1, master_session, NULL, 0, &holder_snapshot)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ( + cluster_pcm_x_directory_find(PCM_X_DIR_LOCAL_TAG, &probe_ref.identity.tag, &tag_ref), + PCM_X_DIRECTORY_OK); + tag_slot = &local_tag_slots(header)[tag_ref.slot_index]; + UT_ASSERT_EQ(test_slot_state(&tag_slot->slot), PCM_X_TAG_RESERVED_NONVISIBLE); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* Existing-tag blocker publication reserves its external chain before the + * local lock. Guard failure must strand that chain only under fail-closed. */ + init_active_pcm_x(UINT64_C(80)); + header = ClusterPcmXConvertShmem; + holder_key = make_local_holder_key(808, 0, 12, UINT64_C(80801), 5); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session + 1); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(808, 2, 13, UINT64_C(80802)); + probe_ref.handle.ticket_id = UINT64_C(8203); + probe_ref.handle.queue_generation = UINT64_C(1); + blocker = make_blocker(0, holder_key.identity.procno, UINT64_C(80803)); + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&probe_ref.identity.tag)); + arm_lwlock_acquire_error(&header->local_locks[partition].lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_blocker_snapshot_arm_exact( + &probe_ref, 1, master_session + 1, &holder_snapshot, &holder_copy, 1, &blocker, 1, + &blocker_snapshot)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + tag_slot = &local_tag_slots(header)[holder.tag_slot.slot_index]; + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* ACK first makes the LIVE chain unreachable and DETACHING. The allocator + * cleanup lock is therefore another mandatory fail-closed handoff. */ + init_active_pcm_x(UINT64_C(81)); + header = ClusterPcmXConvertShmem; + holder_key = make_local_holder_key(809, 0, 14, UINT64_C(80901), 6); + UT_ASSERT_EQ(register_active_local_holder(&holder_key, &holder), PCM_X_QUEUE_OK); + bind_local_master(1, holder_key.identity.cluster_epoch, master_session + 2); + UT_ASSERT_EQ(cluster_pcm_x_local_holder_snapshot(&holder_key.identity.tag, &holder_copy, 1, + &holder_snapshot), + PCM_X_QUEUE_OK); + memset(&probe_ref, 0, sizeof(probe_ref)); + probe_ref.identity = make_wait_identity(809, 2, 15, UINT64_C(80902)); + probe_ref.handle.ticket_id = UINT64_C(8204); + probe_ref.handle.queue_generation = UINT64_C(1); + blocker = make_blocker(0, holder_key.identity.procno, UINT64_C(80903)); + UT_ASSERT_EQ(cluster_pcm_x_local_blocker_snapshot_arm_exact(&probe_ref, 1, master_session + 2, + &holder_snapshot, &holder_copy, 1, + &blocker, 1, &blocker_snapshot), + PCM_X_QUEUE_OK); + tag_slot = &local_tag_slots(header)[holder.tag_slot.slot_index]; + blocker_head = tag_slot->blocker_snapshot_head_index; + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 1); + UT_EXPECT_PG_EXCEPTION((void)cluster_pcm_x_local_blocker_ack_exact( + &probe_ref, blocker_snapshot.set_generation, 1, master_session + 2)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(header->allocator[PCM_X_ALLOC_BLOCKER].used, 1); + UT_ASSERT_EQ(test_slot_state(&blocker_slots(header)[blocker_head].slot), PCM_XB_DETACHING); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_head_index, PCM_X_INVALID_SLOT_INDEX); + UT_ASSERT_EQ(tag_slot->blocker_snapshot_count, 0); + UT_ASSERT_EQ(test_slot_flags(&tag_slot->slot) & PCM_X_LOCAL_TAG_F_ADMISSION_GATE, 0); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +UT_TEST(test_local_retire_episode_lock_errors_fail_closed) +{ + PcmXShmemHeader *header; + PcmXLocalHandle handle; + PcmXRetirePayload retire; + PcmXWaitIdentity identity; + uint32 partition; + const uint64 master_session = UINT64_C(8301); + + /* Once the monotonic marker is published, even the read-only candidate + * pass is part of an ambiguous retirement episode. */ + init_active_pcm_x(UINT64_C(77)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(810, 0, 16, UINT64_C(81001)); + bind_local_master(1, identity.cluster_epoch, master_session); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session, &handle), + PCM_X_QUEUE_OK); + memset(&retire, 0, sizeof(retire)); + retire.cluster_epoch = identity.cluster_epoch; + retire.master_session_incarnation = master_session; + retire.retire_through_ticket_id = UINT64_C(8302); + retire.sender_node = 0; + partition = cluster_pcm_x_lock_partition(cluster_pcm_x_tag_hash(&identity.tag)); + arm_lwlock_acquire_error(&header->local_locks[partition].lock, LW_SHARED, 1); + UT_EXPECT_PG_EXCEPTION( + (void)cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, + retire.retire_through_ticket_id); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 2); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); + + /* A retryable preflight normally clears the marker. If that cleanup lock + * itself errors, the marker remains a durable ambiguity witness. */ + init_active_pcm_x(UINT64_C(78)); + header = ClusterPcmXConvertShmem; + identity = make_wait_identity(811, 0, 17, UINT64_C(81101)); + bind_local_master(1, identity.cluster_epoch, master_session + 1); + UT_ASSERT_EQ(cluster_pcm_x_local_join_begin(&identity, 1, master_session + 1, &handle), + PCM_X_QUEUE_OK); + retire.master_session_incarnation = master_session + 1; + retire.retire_through_ticket_id = UINT64_C(8303); + arm_lwlock_acquire_error(&header->allocator_lock.lock, LW_EXCLUSIVE, 2); + UT_EXPECT_PG_EXCEPTION( + (void)cluster_pcm_x_local_retire_up_to_exact(&retire, 1, master_session + 1)); + UT_ASSERT_NULL(lwlock_acquire_error_lock); + UT_ASSERT_EQ(held_lwlock_count, 0); + UT_ASSERT_EQ(header->peer_frontiers[1].local_retire_in_progress_ticket_id, + retire.retire_through_ticket_id); + UT_ASSERT_EQ(pg_atomic_read_u32(&header->local_retire_gate), 2); + UT_ASSERT_EQ(cluster_pcm_x_runtime_snapshot().state, PCM_X_RUNTIME_RECOVERY_BLOCKED); +} + + +int +main(void) +{ + UT_PLAN(244); + UT_RUN(test_image_id_domain_is_canonical_and_bounded); + UT_RUN(test_wire_abi_sizes_are_exact); + UT_RUN(test_wire_abi_offsets_are_exact); + UT_RUN(test_runtime_layout_abi_and_offsets_are_exact); + UT_RUN(test_lwlock_held_limit_is_shared_200); + UT_RUN(test_default_capacity_formulas_are_exact); + UT_RUN(test_exactly_five_pools_and_bounded_directories); + UT_RUN(test_layout_v12_records_transfer_and_terminal_frontiers); + UT_RUN(test_offsets_are_aligned_ordered_and_bounded); + UT_RUN(test_membership_wait_and_holder_partitions_do_not_overlap); + UT_RUN(test_generation_zero_advances_without_being_a_sentinel); + UT_RUN(test_generation_max_never_wraps); + UT_RUN(test_slot_generation_seqlock_crosses_u32_rollover); + UT_RUN(test_slot_generation_revalidate_never_accepts_torn_pair); + UT_RUN(test_slot_generation_writer_in_progress_is_retryable); + UT_RUN(test_allocator_exact_release_reuses_with_next_generation); + UT_RUN(test_allocator_generation_max_head_middle_tail_preserve_chain); + UT_RUN(test_allocator_lost_free_head_is_corrupt); + UT_RUN(test_allocator_generation_exhausted_overflow_is_corrupt); + UT_RUN(test_allocator_live_free_head_is_corrupt); + UT_RUN(test_allocator_free_slot_with_packed_flags_is_corrupt); + UT_RUN(test_allocator_self_loop_is_corrupt); + UT_RUN(test_allocator_invalid_next_is_corrupt); + UT_RUN(test_allocator_truncated_free_chain_is_corrupt_before_mutation); + UT_RUN(test_allocator_nonfree_successor_is_corrupt_before_reserve); + UT_RUN(test_allocator_release_rejects_nonfree_old_head); + UT_RUN(test_membership_wait_and_holder_allocators_never_borrow); + UT_RUN(test_directory_collision_uses_exact_key_not_hash_bucket); + UT_RUN(test_directory_backshift_preserves_wrapped_probe_chain); + UT_RUN(test_directory_stale_generation_and_delete_ref_mismatch_fail_closed); + UT_RUN(test_directory_full_never_overwrites_existing_entries); + UT_RUN(test_master_ticket_prehandle_and_handle_directories_are_independent); + UT_RUN(test_local_holder_directory_keeps_backend_identities_independent); + UT_RUN(test_local_holder_registry_returns_canonical_exact_snapshot); + UT_RUN(test_local_holder_stale_handle_cannot_unregister_reused_slot); + UT_RUN(test_local_holder_capacity_failure_rolls_back_holder_only_tag); + UT_RUN(test_local_holder_snapshot_corruption_fails_closed_before_copy); + UT_RUN(test_local_holder_unregister_validates_before_unlinking_corrupt_tag); + UT_RUN(test_holder_only_tag_is_adopted_by_local_queue_without_pool_borrowing); + UT_RUN(test_holder_only_tag_survives_last_cancelled_waiter_detach); + UT_RUN(test_local_holder_register_refuses_closed_revoke_barrier); + UT_RUN(test_local_holder_prepare_is_visible_before_content_lock); + UT_RUN(test_local_holder_abort_acquiring_unlinks_exact); + UT_RUN(test_local_holder_abort_acquiring_rejects_live_content_states); + UT_RUN(test_local_holder_abort_acquiring_loses_race_to_activate); + UT_RUN(test_local_holder_abort_acquiring_rejects_stale_aba_handle); + UT_RUN(test_local_holder_exceptional_detach_requires_released_content_lock); + UT_RUN(test_local_holder_exceptional_detach_accepts_live_content_states); + UT_RUN(test_local_holder_exceptional_detach_rejects_stale_aba_handle); + UT_RUN(test_local_holder_cleanup_lock_errors_return_corrupt_without_rethrow); + UT_RUN(test_local_holder_content_window_transitions_are_lock_free); + UT_RUN(test_local_holder_finish_requires_post_content_releasing_state); + UT_RUN(test_local_holder_gate_contention_is_retryable_without_fail_closed); + UT_RUN(test_local_holder_barrier_counts_acquiring_active_and_releasing); + UT_RUN(test_local_probe_freeze_is_count_first_exact_for_holder_only_tag); + UT_RUN(test_local_queue_invalidate_authority_is_exact_and_read_only); + UT_RUN(test_local_probe_freeze_closes_fifo_round_and_arm_consumes_reservation); + UT_RUN(test_nested_wait_guard_snapshots_content_locks_then_checks_barriers); + UT_RUN(test_nested_wait_guard_allows_untracked_tag_before_runtime_formation); + UT_RUN(test_local_holder_lock_free_transitions_reject_stale_aba_handle); + UT_RUN(test_directory_delete_then_allocator_reuse_changes_generation); + UT_RUN(test_allocator_and_directory_operations_take_only_allocator_lock); + UT_RUN(test_allocator_and_directory_reject_domain_lock_nesting); + UT_RUN(test_slot_ref_revalidate_requires_exact_domain_tag_state_and_generation); + UT_RUN(test_internal_directory_stale_ref_is_corrupt_and_fails_closed); + UT_RUN(test_resident_exact_wire_mismatch_is_stale_without_side_effect); + UT_RUN(test_same_lock_staged_insert_exists_is_corrupt_and_fails_closed); + UT_RUN(test_master_composite_admission_publishes_all_indexes_and_coalesces_node); + UT_RUN(test_master_coalesce_scan_distinguishes_legal_publish_races); + UT_RUN(test_stale_peer_frontier_is_retryable_without_publication); + UT_RUN(test_resident_tag_authority_mismatch_is_corrupt); + UT_RUN(test_resident_tag_transitional_states_are_retryable); + UT_RUN(test_peer_bind_ack_is_exact_and_mismatch_fails_closed); + UT_RUN(test_activation_publishes_bound_frontiers_and_counters_before_active); + UT_RUN(test_activation_rejects_partial_binding_before_claiming_gate); + UT_RUN(test_runtime_peer_binding_exact_revalidation_is_read_only); + UT_RUN(test_runtime_peer_binding_epoch_drift_fails_closed); + UT_RUN(test_runtime_peer_binding_session_drift_fails_closed); + UT_RUN(test_runtime_peer_binding_empty_frontier_is_never_late_bound); + UT_RUN(test_runtime_peer_binding_invalid_node_is_rejected_without_closing_runtime); + UT_RUN(test_composite_admission_rejects_zero_wrapping_and_out_of_range_identity); + UT_RUN(test_master_composite_capacity_failure_rolls_back_tag_indexes_and_id); + UT_RUN(test_master_ticket_and_admission_sequence_exhaustion_leave_no_holes); + UT_RUN(test_missing_live_prehandle_inside_frontier_is_corrupt_not_phantom); + UT_RUN(test_master_confirmed_queue_is_node_fifo_with_one_active); + UT_RUN(test_master_pending_x_claim_is_active_ticket_exact); + UT_RUN(test_master_pending_x_claim_state_classifies_cancel_progress_as_not_ready); + UT_RUN(test_master_pending_x_cancel_release_is_two_phase_and_replay_exact); + UT_RUN(test_master_pending_x_cancel_before_first_probe_releases_immediately); + UT_RUN(test_master_cancel_ack_snapshot_distinguishes_prehandle_and_confirmed_cancel); + UT_RUN(test_master_pending_x_cancel_prepare_replay_validates_unlinked_tombstone); + UT_RUN(test_master_pending_x_cancel_intent_survives_partial_probe_and_nonempty_graph); + UT_RUN(test_master_pending_x_cancel_loses_cleanly_to_active_transfer); + UT_RUN(test_master_wfg_snapshot_head_has_no_blockers_and_confirms_exactly); + UT_RUN(test_master_wfg_snapshot_second_ticket_names_true_predecessor); + UT_RUN(test_master_wfg_snapshot_deduplicates_active_predecessor_and_keeps_distinct_pair); + UT_RUN(test_master_wfg_structural_mutation_invalidates_snapshot_before_confirm); + UT_RUN(test_master_wfg_snapshot_and_confirm_reject_stale_exact_identity); + UT_RUN(test_master_success_transfer_completes_fifo_and_promotes_every_node); + UT_RUN(test_master_blocker_replace_canonical_snapshot_and_revalidate); + UT_RUN(test_master_blocker_replace_grow_shrink_empty_reclaims); + UT_RUN(test_master_blocker_capacity_failure_is_byte_stable); + UT_RUN(test_master_blocker_generation_and_duplicate_conflicts_fail_closed); + UT_RUN(test_master_blocker_snapshot_detects_slot_aba_and_corrupt_link); + UT_RUN(test_master_blocker_snapshot_validates_every_owner_field); + UT_RUN(test_master_blocker_sets_are_isolated_across_tickets); + UT_RUN(test_master_blocker_wire_stage_commits_exact_canonical_set); + UT_RUN(test_master_blocker_crc_ignores_vertex_abi_padding); + UT_RUN(test_master_blocker_wire_stage_crc_rejects_content_mismatch); + UT_RUN(test_master_blocker_stage_abort_reclaims_mixed_chain_and_fences_aba); + UT_RUN(test_master_blocker_probe_arm_is_zero_generation_and_idempotent); + UT_RUN(test_master_drive_work_cursor_is_bounded_and_exact); + UT_RUN(test_master_driver_waits_for_terminal_active_ticket_without_fail_closed); + UT_RUN(test_master_drive_snapshot_and_holder_bitmaps_are_exact); + UT_RUN(test_master_drive_bitmap_replace_rejects_reliable_leg_drift); + UT_RUN(test_master_drive_snapshot_fails_closed_on_bitmap_corruption); + UT_RUN(test_master_transfer_wire_49_56_is_generation_exact); + UT_RUN(test_master_grant_generation_exhaustion_never_wraps); + UT_RUN(test_master_image_id_allocator_encodes_node31_and_never_wraps); + UT_RUN(test_master_blocker_wire_stage_preserves_old_set_until_commit); + UT_RUN(test_master_blocker_wire_stage_rejects_reorder_conflict_and_wrong_source); + UT_RUN(test_master_blocker_wire_stage_fences_transfer_and_cancel_until_commit); + UT_RUN(test_master_blocker_graph_commit_revalidates_exact_snapshot_and_generation); + UT_RUN(test_master_transfer_requires_committed_exact_empty_blocker_set); + UT_RUN(test_master_active_probe_cancel_requires_committed_exact_empty_blocker_set); + UT_RUN(test_master_locator_defers_mutable_grant_exactness_to_domain_lock); + UT_RUN(test_admission_replay_and_confirm_stay_frozen_after_transfer); + UT_RUN(test_master_application_replays_survive_terminal_successors); + UT_RUN(test_delayed_pretransfer_cancel_fences_active_transfer); + UT_RUN(test_recovery_blocked_runtime_cannot_promote_or_begin_transfer); + UT_RUN(test_recovery_blocked_runtime_refuses_ack_and_local_mutators); + UT_RUN(test_master_cancel_is_exact_and_unlinks_middle_without_fifo_damage); + UT_RUN(test_master_prehandle_cancel_replays_exactly_and_never_hits_reused_slot); + UT_RUN(test_master_prehandle_identity_alias_is_corruption_not_stale_cancel); + UT_RUN(test_master_prehandle_cancel_first_publishes_terminal_tombstone_before_late_enqueue); + UT_RUN(test_master_cancel_rejects_active_probe_without_mutation); + UT_RUN(test_master_unlink_corruption_is_byte_stable_before_fail_closed); + UT_RUN(test_master_cancel_requires_state_exact_active_locator); + UT_RUN(test_terminal_ack_requires_prearmed_leg_and_is_byte_stable); + UT_RUN(test_terminal_ack_wire_fields_are_exact_and_zero_side_effect); + UT_RUN(test_retire_ack_uses_persistent_exact_key_index); + UT_RUN(test_early_retire_arm_does_not_publish_locator); + UT_RUN(test_terminal_outcome_mask_corruption_is_fail_closed); + UT_RUN(test_terminal_bitmap_superset_is_fail_closed_before_arm_or_ack_mutation); + UT_RUN(test_terminal_detach_missing_retire_ack_is_retryable_not_ready); + UT_RUN(test_terminal_detach_rejects_pending_leg); + UT_RUN(test_master_terminal_detach_rejects_hot_link_and_drain_corruption); + UT_RUN(test_master_terminal_detach_preserves_same_node_successor); + UT_RUN(test_master_tag_survives_until_every_terminal_ticket_detaches); + UT_RUN(test_last_terminal_detach_waits_for_staged_admission); + UT_RUN(test_master_outstanding_counter_overflow_and_rollback_are_exact); + UT_RUN(test_master_detach_second_caller_is_stale_not_fail_closed); + UT_RUN(test_local_composite_join_publishes_one_leader_and_ordered_followers); + UT_RUN(test_local_follower_wfg_publish_is_nonwaitable_then_exact); + UT_RUN(test_local_writer_claim_runs_closed_cohort_and_blocks_next_round); + UT_RUN(test_local_writer_claim_completion_is_fifo_and_one_shot); + UT_RUN(test_writer_and_holder_owner_exit_retry_preserves_exact_evidence); + UT_RUN(test_local_wfg_rejects_completed_blocker_semantic_aba); + UT_RUN(test_local_cancel_never_unlinks_an_active_writer); + UT_RUN(test_local_closed_round_never_promotes_a_late_joiner_early); + UT_RUN(test_local_lookup_is_read_only_and_identity_exact); + UT_RUN(test_local_progress_is_exact_and_exposes_remote_wait_ledger); + UT_RUN(test_local_transfer_prepare_commit_and_final_ack_are_exact); + UT_RUN(test_local_tag_only_holder_transfer_persists_until_exact_drain); + UT_RUN(test_local_non_source_blocker_participant_drains_and_retires_exactly); + UT_RUN(test_local_cancelled_non_source_participant_gen0_drains_and_retires_exactly); + UT_RUN(test_local_cancelled_participant_gen0_drain_requires_frozen_round); + UT_RUN(test_local_holder_drain_validates_frozen_round_before_terminal_publish); + UT_RUN(test_local_same_ref_non_source_participant_retires_both_legs); + UT_RUN(test_local_successful_retire_clears_round_image_before_promoting_successor); + UT_RUN(test_local_same_ref_cancelled_participant_gen0_retires_both_legs); + UT_RUN(test_local_final_closed_member_retire_promotes_late_round); + UT_RUN(test_local_cancelled_dual_retires_one_writer_from_multiwriter_round); + UT_RUN(test_local_cancelled_dual_retire_validates_before_consuming_holder_evidence); + UT_RUN(test_local_source_holder_rejects_parallel_blocker_terminal_lane); + UT_RUN(test_local_retire_claim_serializes_admission_handoffs); + UT_RUN(test_local_retire_global_gate_blocks_direct_mutators); + UT_RUN(test_local_retire_global_gate_rejects_orphan_and_mismatched_evidence); + UT_RUN(test_local_retire_marker_blocks_cross_lock_tag_handoffs); + UT_RUN(test_local_cross_lock_gate_precedes_retire_claim); + UT_RUN(test_local_new_transfer_abort_keeps_gate_until_allocator_cleanup); + UT_RUN(test_local_retire_requires_exact_holder_and_blocker_evidence); + UT_RUN(test_local_blocker_allocator_corruption_retains_new_tag_gate_evidence); + UT_RUN(test_local_blocker_snapshot_replays_exact_holder_set_and_gates_revoke); + UT_RUN(test_local_blocker_snapshot_exact_lookup_replays_when_pool_is_full); + UT_RUN(test_local_blocker_snapshot_accepts_only_holder_bound_nested_waits); + UT_RUN(test_local_duplicate_generation_overlap_is_stale_not_fail_closed); + UT_RUN(test_local_join_capacity_and_sequence_failure_roll_back_all_indexes); + UT_RUN(test_local_unlink_corruption_preserves_fifo_evidence_before_fail_closed); + UT_RUN(test_local_cancel_requires_state_exact_leader_locator); + UT_RUN(test_local_cutoff_preserves_cancelled_tail_sequence); + UT_RUN(test_local_revoke_cutoff_sends_late_writer_to_next_round); + UT_RUN(test_local_retire_rejects_candidate_from_wrong_round); + UT_RUN(test_local_cancel_rejects_waitable_successor_from_wrong_round); + UT_RUN(test_local_retire_rejects_closed_head_and_active_generation_corruption); + UT_RUN(test_local_retire_duplicate_revalidates_leader_tag_linkage); + UT_RUN(test_local_cancel_is_exact_handoffs_leader_and_terminal_detach_restores_baseline); + UT_RUN(test_local_promoted_leader_identity_generation_rekey_is_exact); + UT_RUN(test_local_promoted_leader_rekey_rejects_live_evidence_and_key_alias); + UT_RUN(test_local_promoted_leader_rekey_allocator_handoff_error_releases_gate); + UT_RUN(test_local_rekey_staged_target_directory_is_retryable_busy); + UT_RUN(test_local_rekey_third_local_acquire_error_retains_gate); + UT_RUN(test_local_rekey_requires_canonical_fifo_leader); + UT_RUN(test_local_rekey_target_insert_failure_restores_old_key); + UT_RUN(test_last_writer_detach_waits_for_independent_holder_transfer_before_tag_release); + UT_RUN(test_fifo_writer_drain_preserves_concurrent_holder_lane_until_transfer_drain); + UT_RUN(test_local_terminal_detach_rejects_hot_link_and_closed_count_corruption); + UT_RUN(test_local_enqueue_arm_persists_ledger_and_mints_without_holes); + UT_RUN(test_local_enqueue_busy_and_counter_exhaustion_leave_no_hole); + UT_RUN(test_local_admit_ack_retry_and_confirm_are_exact); + UT_RUN(test_local_generic_reliable_ack_requires_exact_token_and_keys); + UT_RUN(test_local_pending_enqueue_cancel_keeps_ledger_until_exact_ack_then_rotates_leader); + UT_RUN(test_local_remote_wait_cancel_is_exact_retriable_and_rotates_only_on_ack); + UT_RUN(test_local_exact_prepare_wins_cancel_vs_transfer_race); + UT_RUN(test_local_terminal_drain_retire_fences_successor_and_replays_watermark); + UT_RUN(test_local_reliable_authority_change_fails_closed_without_dropping_ledger); + UT_RUN(test_capacity_addition_overflow_is_checked); + UT_RUN(test_capacity_multiplication_overflow_is_checked); + UT_RUN(test_ticket_directory_capacity_doubling_overflow_is_checked); + UT_RUN(test_fresh_init_builds_independent_free_lists); + UT_RUN(test_fresh_init_is_recovery_blocked_and_initializes_all_locks_once); + UT_RUN(test_fresh_init_sets_every_slot_free_at_generation_zero); + UT_RUN(test_exec_backend_attach_preserves_mutable_state); + UT_RUN(test_runtime_gate_is_atomic_and_session_fenced); + UT_RUN(test_runtime_activating_phase_has_exact_recovery_reset); + UT_RUN(test_runtime_gate_generation_prevents_active_aba); + UT_RUN(test_runtime_gate_generation_exhaustion_never_wraps); + UT_RUN(test_runtime_gate_generation_exhaustion_fails_closed); + UT_RUN(test_master_state_machine_requires_exact_positive_gates); + UT_RUN(test_cancel_and_ownership_reservation_obey_authorization_boundary); + UT_RUN(test_retire_requires_exact_drain_and_peer_ack_without_time); + UT_RUN(test_lock_partition_is_bounded_and_stable); + UT_RUN(test_attach_validator_rejects_magic_capacity_and_offset_mismatch); + UT_RUN(test_exec_backend_init_fails_closed_on_layout_mismatch); + UT_RUN(test_registration_exposes_one_exact_region); + UT_RUN(test_stats_initialize_zero_and_narrow_note_apis_are_exact); + UT_RUN(test_stats_follow_exact_master_and_local_success_transitions); + UT_RUN(test_stats_count_fail_closed_and_exact_activating_reset_once); + UT_RUN(test_master_admit_lock_window_error_releases_gate_and_fails_closed); + UT_RUN(test_local_join_lock_window_error_releases_gate_and_fails_closed); + UT_RUN(test_master_detach_lock_window_error_releases_gate_and_fails_closed); + UT_RUN(test_local_detach_lock_window_error_releases_gate_and_fails_closed); + UT_RUN(test_local_holder_gate_handoffs_fail_closed); + UT_RUN(test_local_retire_episode_lock_errors_fail_closed); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c new file mode 100644 index 00000000000..db72c591b24 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c @@ -0,0 +1,256 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_pcm_x_image_fetch.c + * Pure identity and byte-validation tests for the PCM-X requester image + * adapter. The transport remains the established GCS BLOCK request/reply + * family; these tests prove that its canonical image-id subdomain cannot + * be confused with an ordinary block request or a different queue round. + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_pcm_x_image_fetch.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_pcm_x_image_fetch.h" +#include "port/pg_crc32c.h" +#include "storage/bufpage.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + + +uint32 +cluster_gcs_block_compute_checksum(const char *block_data) +{ + pg_crc32c crc; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, block_data, GCS_BLOCK_DATA_SIZE); + FIN_CRC32C(crc); + return (uint32)crc; +} + + +static BufferTag +make_tag(uint32 blockno) +{ + BufferTag tag; + + memset(&tag, 0, sizeof(tag)); + tag.spcOid = 1663; + tag.dbOid = 1; + tag.relNumber = 910; + tag.forkNum = MAIN_FORKNUM; + tag.blockNum = blockno; + return tag; +} + + +static PcmXLocalProgress +make_requester_progress(void) +{ + PcmXLocalProgress progress; + uint64 image_id; + + memset(&progress, 0, sizeof(progress)); + progress.identity.tag = make_tag(17); + progress.identity.node_id = 1; + progress.identity.procno = 9; + progress.identity.xid = (TransactionId)23; + progress.identity.cluster_epoch = 0; + progress.identity.request_id = gcs_reqid_requester(1, 2, 41); + progress.identity.wait_seq = UINT64_C(43); + progress.identity.base_own_generation = UINT64_C(47); + progress.ref.identity = progress.identity; + progress.ref.handle.ticket_id = UINT64_C(53); + progress.ref.handle.queue_generation = UINT64_C(59); + progress.ref.grant_generation = UINT64_C(61); + UT_ASSERT(cluster_pcm_x_image_id_encode(2, UINT64_C(67), &image_id)); + progress.image.image_id = image_id; + /* Generation zero is a legal first ownership generation. */ + progress.image.source_own_generation = 0; + progress.image.page_scn = UINT64_C(71); + progress.image.page_lsn = UINT64_C(73); + progress.image.source_node = 3; + progress.local_sequence = UINT64_C(79); + progress.reliable_state_sequence = UINT64_C(83); + progress.local_round = 1; + progress.member_state = PCM_XL_REMOTE_WAIT; + progress.role = PCM_X_LOCAL_ROLE_NODE_LEADER; + progress.pending_opcode = 0; + progress.last_response_opcode = PGRAC_IC_MSG_PCM_X_PREPARE_GRANT; + progress.phase = 0; + progress.master_session_incarnation = UINT64_C(89); + progress.master_node = 2; + return progress; +} + + +static void +prepare_page(char *page, PcmXLocalProgress *progress) +{ + PageHeader header; + + memset(page, 0x5a, GCS_BLOCK_DATA_SIZE); + header = (PageHeader)page; + PageXLogRecPtrSet(header->pd_lsn, (XLogRecPtr)progress->image.page_lsn); + header->pd_block_scn = (SCN)progress->image.page_scn; + progress->image.page_checksum = cluster_gcs_block_compute_checksum(page); +} + + +UT_TEST(test_fetch_request_uses_exact_image_handle_without_reserved_overlay) +{ + PcmXLocalProgress progress = make_requester_progress(); + GcsBlockRequestPayload request; + static const uint8 zero_reserved[sizeof(request.reserved_0)] = { 0 }; + + memset(&request, 0xa5, sizeof(request)); + UT_ASSERT(cluster_pcm_x_image_fetch_build_request(&progress, 1, 3, &request)); + UT_ASSERT_EQ(request.request_id, progress.image.image_id); + UT_ASSERT_EQ(request.epoch, progress.identity.cluster_epoch); + UT_ASSERT(memcmp(&request.tag, &progress.identity.tag, sizeof(request.tag)) == 0); + UT_ASSERT_EQ(request.sender_node, 1); + UT_ASSERT_EQ(request.requester_backend_id, 3); + UT_ASSERT_EQ(request.transition_id, PCM_TRANS_N_TO_S); + UT_ASSERT(memcmp(request.reserved_0, zero_reserved, sizeof(zero_reserved)) == 0); + + progress.ref.identity.wait_seq++; + UT_ASSERT(!cluster_pcm_x_image_fetch_build_request(&progress, 1, 3, &request)); +} + + +UT_TEST(test_fetch_holder_authenticates_requester_master_and_record_generation) +{ + PcmXLocalProgress requester = make_requester_progress(); + PcmXLocalHolderProgress holder; + GcsBlockRequestPayload request; + ClusterICEnvelope env; + + UT_ASSERT(cluster_pcm_x_image_fetch_build_request(&requester, 1, 3, &request)); + memset(&holder, 0, sizeof(holder)); + holder.ref = requester.ref; + holder.image = requester.image; + holder.reliable_state_sequence = UINT64_C(97); + holder.pending_opcode = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + holder.last_response_opcode = PGRAC_IC_MSG_PCM_X_REVOKE; + holder.phase = PGRAC_IC_MSG_PCM_X_IMAGE_READY; + holder.master_session_incarnation = requester.master_session_incarnation; + holder.master_node = requester.master_node; + memset(&env, 0, sizeof(env)); + env.source_node_id = 1; + env.dest_node_id = 3; + env.payload_length = sizeof(request); + + UT_ASSERT(cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + env.source_node_id = 0; + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + env.source_node_id = 1; + holder.image.image_id++; + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); + holder.image = requester.image; + holder.master_node = 0; + UT_ASSERT(!cluster_pcm_x_image_fetch_request_exact(&env, &request, &holder, 3, 2, 0)); +} + + +UT_TEST(test_fetch_reply_reuses_gcs_checksum_and_rejects_old_or_torn_bytes) +{ + PcmXLocalProgress progress = make_requester_progress(); + GcsBlockReplyHeader reply; + char page[GCS_BLOCK_DATA_SIZE]; + + prepare_page(page, &progress); + memset(&reply, 0, sizeof(reply)); + reply.request_id = progress.image.image_id; + reply.page_lsn = progress.image.page_lsn; + reply.epoch = progress.identity.cluster_epoch; + reply.checksum = progress.image.page_checksum; + reply.sender_node = (int32)progress.image.source_node; + reply.requester_backend_id = 3; + reply.transition_id = PCM_TRANS_N_TO_S; + reply.status = GCS_BLOCK_REPLY_READ_IMAGE_FROM_XHOLDER; + GcsBlockReplyHeaderSetForwardingMasterNode(&reply, GCS_BLOCK_REPLY_NO_FORWARDING_MASTER); + + UT_ASSERT(cluster_pcm_x_image_fetch_reply_exact(&reply, page, &progress, 1, 3)); + page[GCS_BLOCK_DATA_SIZE - 1] ^= 1; + UT_ASSERT(!cluster_pcm_x_image_fetch_reply_exact(&reply, page, &progress, 1, 3)); + page[GCS_BLOCK_DATA_SIZE - 1] ^= 1; + reply.request_id++; + UT_ASSERT(!cluster_pcm_x_image_fetch_reply_exact(&reply, page, &progress, 1, 3)); + reply.request_id = progress.image.image_id; + GcsBlockReplyHeaderSetForwardingMasterNode(&reply, 2); + UT_ASSERT(!cluster_pcm_x_image_fetch_reply_exact(&reply, page, &progress, 1, 3)); +} + + +UT_TEST(test_fetch_install_requires_the_exact_live_grant_reservation) +{ + ClusterPcmOwnSnapshot base; + ClusterPcmOwnSnapshot live; + + memset(&base, 0, sizeof(base)); + base.tag = make_tag(17); + base.generation = UINT64_C(47); + base.reservation_token = UINT64_C(5); + base.pcm_state = PCM_STATE_N; + live = base; + live.reservation_token = UINT64_C(6); + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT(cluster_pcm_x_image_fetch_reservation_exact(&live, &base, UINT64_C(6))); + live.generation++; + UT_ASSERT(!cluster_pcm_x_image_fetch_reservation_exact(&live, &base, UINT64_C(6))); + live = base; + live.reservation_token = UINT64_C(6); + live.flags = PCM_OWN_FLAG_REVOKING; + UT_ASSERT(!cluster_pcm_x_image_fetch_reservation_exact(&live, &base, UINT64_C(6))); + + base.flags = PCM_OWN_FLAG_REVOKING; + base.pcm_state = PCM_STATE_S; + live = base; + live.flags = PCM_OWN_FLAG_GRANT_PENDING; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, UINT64_C(5)), + CLUSTER_PCM_X_GRANT_RESERVATION_S_REVOKE_HANDOFF); + UT_ASSERT(!cluster_pcm_x_image_fetch_reservation_exact(&live, &base, UINT64_C(5))); + live.reservation_token++; + UT_ASSERT_EQ(cluster_pcm_x_grant_reservation_kind(&live, &base, UINT64_C(5)), + CLUSTER_PCM_X_GRANT_RESERVATION_INVALID); +} + + +int +main(void) +{ + UT_PLAN(4); + UT_RUN(test_fetch_request_uses_exact_image_handle_without_reserved_overlay); + UT_RUN(test_fetch_holder_authenticates_requester_master_and_record_generation); + UT_RUN(test_fetch_reply_reuses_gcs_checksum_and_rejects_old_or_torn_bytes); + UT_RUN(test_fetch_install_requires_the_exact_live_grant_reservation); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_qvotec.c b/src/test/cluster_unit/test_cluster_qvotec.c index 3400d689623..55d7a761b8b 100644 --- a/src/test/cluster_unit/test_cluster_qvotec.c +++ b/src/test/cluster_unit/test_cluster_qvotec.c @@ -775,6 +775,7 @@ UT_TEST(test_qvotec_accessors_null_safe_pre_init) UT_ASSERT_EQ(cluster_qvotec_get_disks_ok_count(), 0); UT_ASSERT_EQ(cluster_qvotec_get_disks_total_count(), 0); UT_ASSERT_EQ(cluster_qvotec_get_current_epoch_at_boot(), 0); + UT_ASSERT_EQ(cluster_qvotec_get_self_incarnation(), 0); UT_ASSERT_STR_EQ(cluster_qvotec_get_collision_state_name(), "(uninitialised)"); } @@ -788,6 +789,9 @@ UT_TEST(test_qvotec_accessors_post_init) UT_ASSERT_EQ(cluster_qvotec_get_disks_ok_count(), 0); UT_ASSERT_EQ(cluster_qvotec_get_disks_total_count(), 0); UT_ASSERT_EQ(cluster_qvotec_get_current_epoch_at_boot(), 0); + UT_ASSERT_EQ(cluster_qvotec_get_self_incarnation(), 0); + cluster_qvotec_publish_self_incarnation(919); + UT_ASSERT_EQ(cluster_qvotec_get_self_incarnation(), 919); UT_ASSERT_STR_EQ(cluster_qvotec_get_collision_state_name(), "none"); } diff --git a/src/test/cluster_unit/test_cluster_sf_dep.c b/src/test/cluster_unit/test_cluster_sf_dep.c index fefb3f60509..148ffe4bb0a 100644 --- a/src/test/cluster_unit/test_cluster_sf_dep.c +++ b/src/test/cluster_unit/test_cluster_sf_dep.c @@ -218,6 +218,48 @@ UT_TEST(test_peer_cap_gen_renote_after_reconnect) UT_ASSERT_EQ(cluster_sf_peer_cap_bits(&cap), (uint32)0); } +UT_TEST(test_pcm_x_capability_does_not_alias_idle_horizon) +{ + ClusterSfPeerCap cap; + + memset(&cap, 0, sizeof(cap)); + UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1, (uint32)0x00000100U); + UT_ASSERT_EQ(PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, (uint32)0x00000200U); + UT_ASSERT((PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1 & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) == 0); + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1, 7); + UT_ASSERT((cluster_sf_peer_cap_bits(&cap) & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) == 0); + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, 8); + UT_ASSERT((cluster_sf_peer_cap_bits(&cap) & PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1) != 0); +} + +UT_TEST(test_pcm_x_capability_generation_snapshot_is_exact) +{ + ClusterSfPeerCap cap; + uint32 generation = UINT32_MAX; + + memset(&cap, 0, sizeof(cap)); + UT_ASSERT(!cluster_sf_peer_cap_generation_for_bits(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + &generation)); + UT_ASSERT_EQ(generation, (uint32)0); + + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_UNDO_HORIZON_IDLE_V1, 7); + generation = UINT32_MAX; + UT_ASSERT(!cluster_sf_peer_cap_generation_for_bits(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + &generation)); + UT_ASSERT_EQ(generation, (uint32)0); + + cluster_sf_peer_cap_note(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, 8); + UT_ASSERT(cluster_sf_peer_cap_generation_for_bits(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + &generation)); + UT_ASSERT_EQ(generation, (uint32)8); + + UT_ASSERT(cluster_sf_peer_cap_invalidate_gen(&cap, 8)); + generation = UINT32_MAX; + UT_ASSERT(!cluster_sf_peer_cap_generation_for_bits(&cap, PGRAC_IC_HELLO_CAP_PCM_X_CONVERT_V1, + &generation)); + UT_ASSERT_EQ(generation, (uint32)0); +} + int main(void) { @@ -230,5 +272,7 @@ main(void) UT_RUN(test_gcs_block_reply_v2_dep_extract_accepts_empty_v2_no_dep); UT_RUN(test_peer_cap_gen_note_query_invalidate); UT_RUN(test_peer_cap_gen_renote_after_reconnect); + UT_RUN(test_pcm_x_capability_does_not_alias_idle_horizon); + UT_RUN(test_pcm_x_capability_generation_snapshot_is_exact); UT_DONE(); } diff --git a/src/test/cluster_unit/test_cluster_shmem.c b/src/test/cluster_unit/test_cluster_shmem.c index 01e4dae49a4..c4d39031473 100644 --- a/src/test/cluster_unit/test_cluster_shmem.c +++ b/src/test/cluster_unit/test_cluster_shmem.c @@ -52,6 +52,7 @@ #include "postgres.h" #include "cluster/cluster_conf.h" +#include "cluster/cluster_pcm_x_convert.h" #include "cluster/cluster_shmem.h" /* @@ -386,6 +387,11 @@ void cluster_pcm_lock_module_init(void) {} +/* spec-2.36a S3-core stub: the PCM-X substrate has its own unit binary. */ +void +cluster_pcm_x_convert_shmem_register(void) +{} + /* Spec-2.32 D2 stub: cluster_init_shmem_module also calls * cluster_gcs_module_init (cluster_gcs.c). */ void From eb84a6a74f8f1a02b6b2a03214e6e33f3f9fc620 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 10:28:44 +0800 Subject: [PATCH 2/3] test(cluster): compare PCM-X blocker fields --- .../cluster_unit/test_cluster_pcm_x_convert.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/test/cluster_unit/test_cluster_pcm_x_convert.c b/src/test/cluster_unit/test_cluster_pcm_x_convert.c index f9b62225ae3..15e0734095b 100644 --- a/src/test/cluster_unit/test_cluster_pcm_x_convert.c +++ b/src/test/cluster_unit/test_cluster_pcm_x_convert.c @@ -1152,6 +1152,15 @@ make_blocker(int32 node_id, uint32 procno, uint64 request_id) return blocker; } +static bool +blockers_equal(const ClusterLmdVertex *left, const ClusterLmdVertex *right) +{ + return left->node_id == right->node_id && left->procno == right->procno + && left->cluster_epoch == right->cluster_epoch && left->request_id == right->request_id + && left->xid == right->xid && left->local_start_ts_ms == right->local_start_ts_ms + && left->wait_seq == right->wait_seq; +} + static uint32 blocker_set_crc32c(const ClusterLmdVertex *blockers, Size nblockers) { @@ -5410,8 +5419,8 @@ UT_TEST(test_master_blocker_replace_canonical_snapshot_and_revalidate) UT_ASSERT_EQ(snapshot.set_generation, 1); UT_ASSERT_EQ(snapshot.blocker_count, 2); UT_ASSERT_EQ(snapshot.set_crc32c, UINT32_C(0x11111111)); - UT_ASSERT(memcmp(&entries[0].blocker, &b1, sizeof(b1)) == 0); - UT_ASSERT(memcmp(&entries[1].blocker, &b2, sizeof(b2)) == 0); + UT_ASSERT(blockers_equal(&entries[0].blocker, &b1)); + UT_ASSERT(blockers_equal(&entries[1].blocker, &b2)); for (Size i = 0; i < lengthof(entries); i++) { UT_ASSERT_EQ(entries[i].chunk_no, i); UT_ASSERT_EQ(entries[i].reserved, 0); @@ -5783,8 +5792,8 @@ UT_TEST(test_master_blocker_wire_stage_commits_exact_canonical_set) UT_ASSERT_EQ(snapshot.set_generation, 1); UT_ASSERT_EQ(snapshot.blocker_count, 2); UT_ASSERT_EQ(snapshot.set_crc32c, begin.set_crc32c); - UT_ASSERT(memcmp(&entries[0].blocker, &blockers[0], sizeof(blockers[0])) == 0); - UT_ASSERT(memcmp(&entries[1].blocker, &blockers[1], sizeof(blockers[1])) == 0); + UT_ASSERT(blockers_equal(&entries[0].blocker, &blockers[0])); + UT_ASSERT(blockers_equal(&entries[1].blocker, &blockers[1])); UT_ASSERT_EQ(cluster_pcm_x_master_blocker_stage_commit_exact(&begin, 2, source_session, entries, lengthof(entries), &snapshot), PCM_X_QUEUE_DUPLICATE); From b6ece5e3be5a2380b5f43e06fed7c21f4e6b8690 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Sat, 18 Jul 2026 10:46:32 +0800 Subject: [PATCH 3/3] fix(cluster): initialize PCM-X analyzer paths --- src/backend/cluster/cluster_gcs_block.c | 11 ++++++++--- src/backend/cluster/cluster_lmd_wait_state.c | 5 +++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index f8bbef6c0e7..da357dc8b2a 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -8378,6 +8378,10 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim || MyProc->pgprocno < 0 || MyBackendId <= 0 || cluster_node_id < 0 || cluster_node_id >= PCM_X_PROTOCOL_NODE_LIMIT) return PCM_X_QUEUE_INVALID; + memset(&handle, 0, sizeof(handle)); + handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; + memset(&progress, 0, sizeof(progress)); for (;;) { GcsBlockPcmXFormationAction formation_action; @@ -8476,9 +8480,6 @@ gcs_block_pcm_x_acquire_writer_impl(BufferDesc *buf, PcmXLocalWriterClaim *claim } break; } - memset(&handle, 0, sizeof(handle)); - handle.tag_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; - handle.membership_slot.slot_index = PCM_X_INVALID_SLOT_INDEX; for (;;) { PcmXWaitIdentity identity; @@ -10398,6 +10399,10 @@ gcs_block_pcm_x_probe_collect_and_stage(const PcmXPhasePayload *probe, int32 mas if (result != PCM_X_QUEUE_OK) goto probe_done; if (holder_snapshot.holder_count != 0) { + if (holders == NULL) { + result = PCM_X_QUEUE_CORRUPT; + goto probe_done; + } if (ProcGlobal == NULL) { result = PCM_X_QUEUE_NOT_READY; goto probe_done; diff --git a/src/backend/cluster/cluster_lmd_wait_state.c b/src/backend/cluster/cluster_lmd_wait_state.c index a7308d7afc2..1abe4b1d438 100644 --- a/src/backend/cluster/cluster_lmd_wait_state.c +++ b/src/backend/cluster/cluster_lmd_wait_state.c @@ -116,13 +116,14 @@ ClusterLmdWaitStateReadResult cluster_lmd_wait_state_read_exact(ClusterLmdProcWaitState *ws, ClusterLmdWaitStateSnapshot *out) { ClusterLmdWaitStateSnapshot snapshot; - uint32 before; - uint32 after; int retry; memset(out, 0, sizeof(*out)); for (retry = 0; retry < CLUSTER_LMD_WAIT_STATE_READ_RETRIES; retry++) { + uint32 before; + uint32 after; + before = pg_atomic_read_u32(&ws->change_seq); if ((before & 1U) != 0) continue;