diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 32c47d81..b755b0a3 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -8,11 +8,19 @@ reviews: review_status: false review_details: false poem: false - request_changes_workflow: false + request_changes_workflow: true + + auto_review: - enabled: true - drafts: false + enabled: true # Disables automatic reviews on push + drafts: true + base_branches: + - ".*" + auto_incremental_review: true + + # Never pause after repeated pushes. + auto_pause_after_reviewed_commits: 0 path_instructions: - path: "**/*.rs" @@ -26,11 +34,9 @@ reviews: For Rust workspace code: - Prioritize ownership, borrowing, lifetimes, unsafe usage, panic paths, concurrency, and API contract correctness. - - Do not comment on minor style preferences, formatting, naming bikeshedding, speculative refactors, or micro-optimizations. - Do not suggest changes unless the issue is concrete and actionable. - path: "{**/*.md,**/*.rs}" instructions: | Check docs and rustdoc for factual consistency with the code. Flag only real mismatches, broken examples, stale comments, or important omissions. - Ignore wording nits unless they are genuine typos or change meaning. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..8ec98ed8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.rs linguist-language=Rust +Cargo.toml linguist-language=Rust +Cargo.lock linguist-language=Rust diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 7044ec5f..003cb7f0 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -6,6 +6,12 @@ type: Bug labels: ["bug"] --- + + ## Problem diff --git a/.github/ISSUE_TEMPLATE/change_request.md b/.github/ISSUE_TEMPLATE/change_request.md index 50b40309..7b36f843 100644 --- a/.github/ISSUE_TEMPLATE/change_request.md +++ b/.github/ISSUE_TEMPLATE/change_request.md @@ -6,6 +6,12 @@ type: Feature labels: ["enhancement"] --- + + ## Goal diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index b0a2c6ed..803d4b3d 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -2,9 +2,15 @@ name: Spike / question about: Ask a design question or request an investigation title: "" -labels: question +labels: ["question"] --- + + ## Question diff --git a/.github/ISSUE_TEMPLATE/task.md b/.github/ISSUE_TEMPLATE/task.md index c7c3f712..f005f160 100644 --- a/.github/ISSUE_TEMPLATE/task.md +++ b/.github/ISSUE_TEMPLATE/task.md @@ -6,6 +6,12 @@ type: Task labels: [] --- + + ## Outcome diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index ffd1f751..3d946471 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,9 @@ + + ## What changed @@ -7,8 +13,5 @@ Closes #ISSUE ## Impact -## Validation - - ## Reviewer notes diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..da075d1b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,80 @@ +name: Run CI + +on: + push: + branches: [master, dev] + pull_request: + types: [opened, reopened, synchronize] + +permissions: + contents: read + +env: + AGAVE_VERSION: v4.2.0 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + check: + name: ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: Format + command: cargo fmt --check + cache: false + sbf: false + nextest: false + - name: Clippy + command: cargo clippy --quiet --all-features --all-targets -- -D warnings + cache: true + sbf: true + nextest: false + - name: Tests + command: cargo nextest run --cargo-quiet --nff + cache: true + sbf: true + nextest: true + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Install Rust toolchain + run: rustup toolchain install + + - name: Cache Rust build artifacts + if: matrix.cache + uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.name }} + cache-on-failure: true + + - name: Cache Solana SBF tools + if: matrix.sbf + id: cache-solana + uses: actions/cache@v4 + with: + path: | + ~/.local/share/solana + ~/.cache/solana + key: solana-sbf-${{ runner.os }}-${{ runner.arch }}-${{ env.AGAVE_VERSION }} + + - name: Install Solana CLI + if: matrix.sbf && steps.cache-solana.outputs.cache-hit != 'true' + run: sh -c "$(curl -sSfL https://release.anza.xyz/$AGAVE_VERSION/install)" + + - name: Add Solana CLI to PATH + if: matrix.sbf + run: echo "$HOME/.local/share/solana/install/active_release/bin" >> "$GITHUB_PATH" + + - name: Install cargo-nextest + if: matrix.nextest + uses: taiki-e/install-action@nextest + + - name: Run ${{ matrix.name }} + run: ${{ matrix.command }} diff --git a/.gitignore b/.gitignore index 1798e8ba..8b202a78 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ /target -Cargo.lock AGENTS.md +CLAUDE.md diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 00000000..cf3a784b --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,5455 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "agave-feature-set" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "402f8462d64e03c9b09a96b1898f7dbc8b9c5d6e9534fb777905df599ad5d787" +dependencies = [ + "ahash", + "solana-epoch-schedule", + "solana-hash", + "solana-keypair", + "solana-pubkey", + "solana-sha256-hasher", + "solana-svm-feature-set", +] + +[[package]] +name = "agave-precompiles" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a97e478437ac3d1cc1446600afb972e446736b301b02f0c1ba31d81bb0c91b6" +dependencies = [ + "agave-feature-set", + "bincode", + "ed25519-dalek 1.0.1", + "libsecp256k1", + "openssl", + "solana-ed25519-program", + "solana-keccak-hasher", + "solana-message", + "solana-precompile-error", + "solana-pubkey", + "solana-sdk-ids", + "solana-secp256k1-program", + "solana-secp256r1-program", +] + +[[package]] +name = "agave-transaction-view" +version = "4.1.1" +dependencies = [ + "agave-transaction-view", + "bincode", + "criterion", + "solana-hash", + "solana-instruction", + "solana-keypair", + "solana-message", + "solana-packet", + "solana-program-runtime", + "solana-pubkey", + "solana-sdk-ids", + "solana-short-vec", + "solana-signature", + "solana-signer", + "solana-svm-transaction", + "solana-system-interface", + "solana-transaction", + "solana-transaction-context", + "wincode", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ark-bn254" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a22f4561524cd949590d78d7d4c5df8f592430d221f7f3c9497bbafd8972120f" +dependencies = [ + "ark-ec 0.4.2", + "ark-ff 0.4.2", + "ark-std 0.4.0", +] + +[[package]] +name = "ark-bn254" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d69eab57e8d2663efa5c63135b2af4f396d66424f88954c21104125ab6b3e6bc" +dependencies = [ + "ark-ec 0.5.0", + "ark-ff 0.5.0", + "ark-std 0.5.0", +] + +[[package]] +name = "ark-ec" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "defd9a439d56ac24968cca0571f598a61bc8c55f71d50a89cda591cb750670ba" +dependencies = [ + "ark-ff 0.4.2", + "ark-poly 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "hashbrown 0.13.2", + "itertools 0.10.5", + "num-traits", + "zeroize", +] + +[[package]] +name = "ark-ec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d68f2d516162846c1238e755a7c4d131b892b70cc70c471a8e3ca3ed818fce" +dependencies = [ + "ahash", + "ark-ff 0.5.0", + "ark-poly 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "educe", + "fnv", + "hashbrown 0.15.5", + "itertools 0.13.0", + "num-bigint", + "num-integer", + "num-traits", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" +dependencies = [ + "ark-ff-asm 0.4.2", + "ark-ff-macros 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "digest 0.10.7", + "itertools 0.10.5", + "num-bigint", + "num-traits", + "paste", + "rustc_version", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a177aba0ed1e0fbb62aa9f6d0502e9b46dad8c2eab04c14258a1212d2557ea70" +dependencies = [ + "ark-ff-asm 0.5.0", + "ark-ff-macros 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "educe", + "itertools 0.13.0", + "num-bigint", + "num-traits", + "paste", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-asm" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-ff-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09be120733ee33f7693ceaa202ca41accd5653b779563608f1234f78ae07c4b3" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-poly" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d320bfc44ee185d899ccbadfa8bc31aab923ce1558716e1997a1e74057fe86bf" +dependencies = [ + "ark-ff 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "hashbrown 0.13.2", +] + +[[package]] +name = "ark-poly" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "579305839da207f02b89cd1679e50e67b4331e2f9294a57693e5051b7703fe27" +dependencies = [ + "ahash", + "ark-ff 0.5.0", + "ark-serialize 0.5.0", + "ark-std 0.5.0", + "educe", + "fnv", + "hashbrown 0.15.5", +] + +[[package]] +name = "ark-serialize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" +dependencies = [ + "ark-serialize-derive 0.4.2", + "ark-std 0.4.0", + "digest 0.10.7", + "num-bigint", +] + +[[package]] +name = "ark-serialize" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" +dependencies = [ + "ark-serialize-derive 0.5.0", + "ark-std 0.5.0", + "arrayvec", + "digest 0.10.7", + "num-bigint", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae3281bc6d0fd7e549af32b52511e1302185bd688fd3359fa36423346ff682ea" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-serialize-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ark-std" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand 0.8.7", +] + +[[package]] +name = "ark-std" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246a225cc6131e9ee4f24619af0f19d67761fff15d7ccc22e42b80846e69449a" +dependencies = [ + "num-traits", + "rand 0.8.7", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "ascii" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" + +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + +[[package]] +name = "assoc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitcode" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6ed1b54d8dc333e7be604d00fa9262f4635485ffea923647b6521a5fff045d" +dependencies = [ + "arrayvec", + "bitcode_derive", + "bytemuck", + "glam", + "serde", +] + +[[package]] +name = "bitcode_derive" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "238b90427dfad9da4a9abd60f3ec1cdee6b80454bde49ed37f1781dd8e9dc7f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] + +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake3" +version = "1.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "blst" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20659f9bbee16cbbd2f7393e40ab6309f5a98f76a2eb57a995ec508b72387fe" +dependencies = [ + "cc", + "glob", + "threadpool", + "zeroize", +] + +[[package]] +name = "blstrs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a8a8ed6fefbeef4a8c7b460e4110e12c5e22a5b7cf32621aae6ad650c4dcf29" +dependencies = [ + "blst", + "byte-slice-cast", + "ff", + "group", + "pairing", + "rand_core 0.6.4", + "serde", + "subtle", +] + +[[package]] +name = "borsh" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88b7ea17d208c4193f2c1e6de3c35fe71f98c96982d5ced308bdcc749ff6e1f" +dependencies = [ + "borsh-derive", + "bytes", + "cfg_aliases", +] + +[[package]] +name = "borsh-derive" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f347189c62a579b8cd5f80714efa178f52e461dc2e6d701d264f5ff22e566c" +dependencies = [ + "once_cell", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bv" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8834bb1d8ee5dc048ee3124f2c7c1afcc6bc9aed03f11e9dfd8c69470a5db340" +dependencies = [ + "feature-probe", + "serde", +] + +[[package]] +name = "byte-slice-cast" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7575182f7272186991736b70173b0ea045398f984bf5ebbb3804736ce1330c9d" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0e56a716f1e132ff6bf4bdac1c944a3fcdc1cae65f70a4a2a1ac3b401d2d1f" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "clonetree" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bfe46018b627d1c990f2ac173e876b1d82b18dc3b6b05691dae2fe4a07ba5b5" +dependencies = [ + "ignore", + "reflink-copy", + "thiserror 2.0.19", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" +dependencies = [ + "ascii", + "byteorder", + "either", + "memchr", + "unreachable", +] + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "curve25519-dalek" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "curve25519-dalek-derive", + "digest 0.10.7", + "fiat-crypto", + "rand_core 0.6.4", + "rustc_version", + "serde", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.19", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "doxygen-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" +dependencies = [ + "phf", +] + +[[package]] +name = "eager" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abe71d579d1812060163dff96056261deb5bf6729b100fa2e36a68b9649ba3d3" + +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest 0.10.7", + "elliptic-curve", + "rfc6979", + "signature 2.2.0", + "spki", +] + +[[package]] +name = "ed25519" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cff35c70bba8a626e3185d8cd48cc11b5437e1a5bcd15b9b5fa3c64b6dfee7" +dependencies = [ + "signature 1.6.4", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "pkcs8", + "signature 2.2.0", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +dependencies = [ + "curve25519-dalek 3.2.0", + "ed25519 1.5.3", + "rand 0.7.3", + "serde", + "sha2 0.9.9", + "zeroize", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek 4.1.3", + "ed25519 2.2.3", + "rand_core 0.6.4", + "serde", + "sha2 0.10.9", + "subtle", + "zeroize", +] + +[[package]] +name = "educe" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7bc049e1bd8cdeb31b68bbd586a9464ecf9f3944af3958a7a9d0f8b9799417" +dependencies = [ + "enum-ordinalize", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest 0.10.7", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + +[[package]] +name = "enum-iterator" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4549325971814bda7a44061bf3fe7e487d447cba01e4220a4b454d630d7a016" +dependencies = [ + "enum-iterator-derive", +] + +[[package]] +name = "enum-iterator-derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "enum-ordinalize" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89dd01549b09589510cf0647475075d12071456586d70f5c75c98ae2a5537677" +dependencies = [ + "enum-ordinalize-derive", +] + +[[package]] +name = "enum-ordinalize-derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65863d15a4ce2888bd2f0f543cc963d3879c3a022c8ee43f6141d479a3ac815" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +dependencies = [ + "getrandom 0.4.3", +] + +[[package]] +name = "feature-probe" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "835a3dc7d1ec9e75e2b5fb4ba75396837112d2060b03f7d43bc1897c7f7211da" + +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "bitvec", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "five8" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f76610e969fa1784327ded240f1e28a3fd9520c9cec93b636fcf62dd37f772" +dependencies = [ + "five8_core", +] + +[[package]] +name = "five8_const" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a0f1728185f277989ca573a402716ae0beaaea3f76a8ff87ef9dd8fb19436c5" +dependencies = [ + "five8_core", +] + +[[package]] +name = "five8_core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059c31d7d36c43fe39d89e55711858b4da8be7eb6dabac23c7289b1a19489406" + +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "fastrand", + "futures-core", + "futures-sink", + "spin", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gdbstub" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e02bf1b1a624d96925c608f1b268d82a76cbc587ce9e59f7c755e9ea11c75c" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "log", + "managed", + "num-traits", + "paste", +] + +[[package]] +name = "generator" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b854b0e584ead1a33f18b2fcad7cf7be18b3875c78816b753639aa501513ae" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "wasm-bindgen", +] + +[[package]] +name = "glam" +version = "0.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360bd2cd76e0cd9032d42cf2922155cecea2685b0cfa4630c3246df030bcfd6" + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand 0.8.7", + "rand_core 0.6.4", + "rand_xorshift", + "subtle", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heed" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad82d6598ccf1dac15c8b758a1bd282b755b6776be600429176757190a1b0202" +dependencies = [ + "bitflags 2.13.1", + "byteorder", + "heed-traits", + "heed-types", + "libc", + "lmdb-master-sys", + "once_cell", + "page_size", + "synchronoise", + "url", +] + +[[package]] +name = "heed-traits" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" + +[[package]] +name = "heed-types" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" +dependencies = [ + "byteorder", + "heed-traits", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac-drbg" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ea0a1394df5b6574da6e0c1ade9e78868c9fb0a4e5ef4428e32da4676b85b1" +dependencies = [ + "digest 0.9.0", + "generic-array", + "hmac 0.8.1", +] + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ignore" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b69833ed729dc5aa7d19541d96d6cf8e9137194207a04916d658e43168402f" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "k256" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" +dependencies = [ + "cfg-if", + "ecdsa", + "elliptic-curve", + "once_cell", + "sha2 0.10.9", + "signature 2.2.0", +] + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libsecp256k1" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79019718125edc905a079a70cfa5f3820bc76139fc91d6f9abc27ea2a887139" +dependencies = [ + "arrayref", + "base64", + "digest 0.9.0", + "hmac-drbg", + "libsecp256k1-core", + "libsecp256k1-gen-ecmult", + "libsecp256k1-gen-genmult", + "rand 0.8.7", + "serde", + "sha2 0.9.9", + "typenum", +] + +[[package]] +name = "libsecp256k1-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be9b9bb642d8522a44d533eab56c16c738301965504753b03ad1de3425d5451" +dependencies = [ + "crunchy", + "digest 0.9.0", + "subtle", +] + +[[package]] +name = "libsecp256k1-gen-ecmult" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3038c808c55c87e8a172643a7d87187fc6c4174468159cb3090659d55bcb4809" +dependencies = [ + "libsecp256k1-core", +] + +[[package]] +name = "libsecp256k1-gen-genmult" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db8d6ba2cec9eacc40e6e8ccc98931840301f1006e95647ceb2dd5c3aa06f7c" +dependencies = [ + "libsecp256k1-core", +] + +[[package]] +name = "light-poseidon" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c9a85a9752c549ceb7578064b4ed891179d20acd85f27318573b64d2d7ee7ee" +dependencies = [ + "ark-bn254 0.4.0", + "ark-ff 0.4.2", + "num-bigint", + "thiserror 1.0.69", +] + +[[package]] +name = "light-poseidon" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47a1ccadd0bb5a32c196da536fd72c59183de24a055f6bf0513bf845fefab862" +dependencies = [ + "ark-bn254 0.5.0", + "ark-ff 0.5.0", + "num-bigint", + "thiserror 1.0.69", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lmdb-master-sys" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaeb9bd22e73bd1babffff614994b341e9b2008de7bb73bf1f7e9154f1978f8b" +dependencies = [ + "cc", + "doxygen-rs", + "libc", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "magic-root-interface" +version = "0.1.0" +dependencies = [ + "solana-account", + "solana-instruction", + "solana-pubkey", + "wincode", +] + +[[package]] +name = "magic-root-program" +version = "0.1.0" +dependencies = [ + "magic-root-interface", + "magicblock-engine-nucleus", + "solana-account", + "solana-instruction", + "solana-instruction-error", + "solana-program-runtime", + "solana-pubkey", + "solana-sdk-ids", + "solana-svm-callback", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-transaction-context", + "wincode", +] + +[[package]] +name = "magicblock-accountsdb" +version = "0.1.0" +dependencies = [ + "ahash", + "assert_matches", + "bincode", + "bytemuck", + "clonetree", + "derive_more", + "heed", + "magicblock-accountsdb", + "magicblock-engine-nucleus", + "memmap2", + "parking_lot", + "scc", + "solana-account", + "solana-pubkey", + "thiserror 2.0.19", + "tracing", + "twox-hash", +] + +[[package]] +name = "magicblock-engine" +version = "0.1.0" +dependencies = [ + "agave-transaction-view", + "derive_more", + "magic-root-interface", + "magic-root-program", + "magicblock-engine", + "magicblock-engine-nucleus", + "magicblock-keeper", + "magicblock-ledger", + "magicblock-processor", + "num_cpus", + "oneshot", + "solana-account", + "solana-compute-budget-program", + "solana-instruction", + "solana-instruction-error", + "solana-keypair", + "solana-message", + "solana-packet", + "solana-program-runtime", + "solana-pubkey", + "solana-sdk-ids", + "solana-signer", + "solana-system-interface", + "solana-system-program", + "solana-sysvar", + "solana-transaction", + "thiserror 2.0.19", + "tokio", + "tracing", + "v42-calculator-interface", + "wincode", +] + +[[package]] +name = "magicblock-engine-nucleus" +version = "0.1.0" +dependencies = [ + "agave-transaction-view", + "derive_more", + "futures", + "heed", + "oneshot", + "prometheus", + "serde", + "serde_with", + "solana-hash", + "solana-instruction", + "solana-instruction-error", + "solana-keypair", + "solana-message", + "solana-pubkey", + "solana-signature", + "solana-signer", + "solana-svm", + "solana-transaction", + "solana-transaction-error", + "tempfile", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "v42-calculator-interface", + "wincode", +] + +[[package]] +name = "magicblock-keeper" +version = "0.1.0" +dependencies = [ + "agave-feature-set", + "ahash", + "arc-swap", + "derive_more", + "flume", + "magicblock-accountsdb", + "magicblock-engine-nucleus", + "magicblock-keeper", + "magicblock-ledger", + "oneshot", + "parking_lot", + "scc", + "serde", + "smallvec", + "solana-account", + "solana-feature-gate-interface", + "solana-hash", + "solana-instruction", + "solana-keypair", + "solana-message", + "solana-program-runtime", + "solana-pubkey", + "solana-sdk-ids", + "solana-signature", + "solana-signer", + "solana-svm", + "solana-sysvar", + "solana-transaction-error", + "tar", + "thiserror 2.0.19", + "tokio", + "tracing", + "zstd", +] + +[[package]] +name = "magicblock-ledger" +version = "0.1.0" +dependencies = [ + "agave-transaction-view", + "bitcode", + "bytemuck", + "derive_more", + "flume", + "heed", + "magicblock-engine-nucleus", + "magicblock-ledger", + "memmap2", + "num_cpus", + "oneshot", + "parking_lot", + "rustix", + "solana-pubkey", + "solana-signature", + "solana-transaction-error", + "thiserror 2.0.19", + "tokio", + "tracing", + "wincode", + "zstd", +] + +[[package]] +name = "magicblock-processor" +version = "0.1.0" +dependencies = [ + "agave-feature-set", + "agave-precompiles", + "agave-transaction-view", + "ahash", + "blake3", + "derive_more", + "magicblock-accountsdb", + "magicblock-engine-nucleus", + "magicblock-keeper", + "oneshot", + "solana-account", + "solana-compute-budget-instruction", + "solana-hash", + "solana-instruction", + "solana-keypair", + "solana-precompile-error", + "solana-program-runtime", + "solana-pubkey", + "solana-sdk-ids", + "solana-signature", + "solana-svm", + "solana-svm-transaction", + "solana-syscalls", + "solana-sysvar", + "solana-transaction-error", + "thiserror 2.0.19", + "tokio", + "tracing", + "v42-calculator-interface", +] + +[[package]] +name = "magicblock-replicator" +version = "0.1.0" +dependencies = [ + "derive_more", + "magicblock-engine", + "magicblock-engine-nucleus", + "magicblock-keeper", + "magicblock-ledger", + "scc", + "snedfile", + "solana-account", + "solana-keypair", + "solana-pubkey", + "solana-signature", + "solana-sysvar", + "thiserror 2.0.19", + "tokio", + "tracing", + "v42-calculator-interface", + "wincode", +] + +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oneshot" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe21416a02c693fb9f980befcb230ecc70b0b3d1cc4abf88b9675c4c1457f0c" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "openssl" +version = "0.10.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45" +dependencies = [ + "bitflags 2.13.1", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pairing" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fec4625e73cf41ef4bb6846cafa6d44736525f442ba45e407c4a000a13996f" +dependencies = [ + "group", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pbkdf2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.7", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "thiserror 2.0.19", +] + +[[package]] +name = "qualifier_attr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e2e25ee72f5b24d773cae88422baddefff7714f97aab68d96fe2b6fc4a28fb2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "reflink-copy" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9dd7ab4af0363d5ccfd2838d782a28196cf32a5cc2e4fe3c5dc83f2be588b8b" +dependencies = [ + "cfg-if", + "libc", + "rustix", + "windows", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac 0.12.1", + "subtle", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "saa" +version = "5.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f5acb362a0e75c2a963532fa7fabf13dff81626dc494df16488d30befcbea0" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scc" +version = "3.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8af0b99483d1c3e59471d4f0cb58b244169436a8979c889a91a3f697075ea01" +dependencies = [ + "saa", + "sdd", + "serde", +] + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sdd" +version = "4.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1836bad8bdc9c6d665b63202da3d9c6d60ed1e597cae63620e21ebf89a3595a9" +dependencies = [ + "saa", +] + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_with" +version = "3.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +dependencies = [ + "serde_core", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2-const-stable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f179d4e11094a893b82fff208f74d448a7512f99f5a0acbd5c679b705f83ed9" + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "shuttle" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9a8db61a44e2b663f169a08206a789bcbd22ba32011e14951562848e7b9c98" +dependencies = [ + "assoc", + "bitvec", + "generator", + "hex", + "owo-colors", + "rand 0.8.7", + "rand_core 0.6.4", + "rand_pcg", + "scoped-tls", + "smallvec", + "tracing", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "snedfile" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b49d9659526e7275bf3cd23e43cff8d14e4864ff1eb1e3836a07decbc3689b4" +dependencies = [ + "libc", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "solana-account" +version = "4.3.1" +dependencies = [ + "bincode", + "bitflags 2.13.1", + "serde", + "serde_bytes", + "solana-account", + "solana-account-info", + "solana-clock", + "solana-instruction-error", + "solana-pubkey", + "solana-sdk-ids", + "solana-sysvar", + "thiserror 2.0.19", + "wincode", +] + +[[package]] +name = "solana-account-info" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9cf16495d9eb53e3d04e72366a33bb1c20c24e78c171d8b8f5978357b63ae95" +dependencies = [ + "solana-address", + "solana-program-error", + "solana-program-memory", +] + +[[package]] +name = "solana-address" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c93e262f671bf402e1040e4a7e40b05d81da5956c7681948c975a0997517bb" +dependencies = [ + "borsh", + "bytemuck", + "bytemuck_derive", + "curve25519-dalek 4.1.3", + "five8", + "five8_const", + "rand 0.9.5", + "serde", + "serde_derive", + "sha2-const-stable", + "solana-atomic-u64", + "solana-define-syscall 5.2.0", + "solana-program-error", + "solana-sanitize", + "solana-sha256-hasher", + "wincode", +] + +[[package]] +name = "solana-atomic-u64" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "085db4906d89324cef2a30840d59eaecf3d4231c560ec7c9f6614a93c652f501" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "solana-big-mod-exp" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30c80fb6d791b3925d5ec4bf23a7c169ef5090c013059ec3ed7d0b2c04efa085" +dependencies = [ + "num-bigint", + "num-traits", + "solana-define-syscall 3.0.0", +] + +[[package]] +name = "solana-bincode" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278a1a5bad62cd9da89ac8d4b7ec444e83caa8ae96aa656dfc27684b28d49a5d" +dependencies = [ + "bincode", + "serde_core", + "solana-instruction-error", +] + +[[package]] +name = "solana-blake3-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7116e1d942a2432ca3f514625104757ab8a56233787e95144c93950029e31176" +dependencies = [ + "blake3", + "solana-define-syscall 4.0.1", + "solana-hash", +] + +[[package]] +name = "solana-bls12-381-syscall" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e8256fda0542709fd1320a96d81c3c66e10ac5f680cca9b991d7d3b0e3fe86" +dependencies = [ + "blst", + "blstrs", + "bytemuck", + "bytemuck_derive", + "pairing", +] + +[[package]] +name = "solana-bn254" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ff13a8867fcc7b0f1114764e1bf6191b4551dcaf93729ddc676cd4ec6abc9f" +dependencies = [ + "ark-bn254 0.5.0", + "ark-ec 0.5.0", + "ark-ff 0.5.0", + "ark-serialize 0.5.0", + "bytemuck", + "solana-define-syscall 5.2.0", + "thiserror 2.0.19", +] + +[[package]] +name = "solana-borsh" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c04abbae16f57178a163125805637b8a076175bb5c0002fb04f4792bea901cf7" +dependencies = [ + "borsh", +] + +[[package]] +name = "solana-builtins-default-costs" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40331a0b7a959114d0cf8ad6d1fdd74c1fb15acd9bdede706f63517e649c1" +dependencies = [ + "agave-feature-set", + "ahash", + "solana-pubkey", + "solana-sdk-ids", +] + +[[package]] +name = "solana-clock" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0acdace90d96e2c9e70d681465b4fe888b6bcf27c354ae9774e9f8a3b72923d" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", +] + +[[package]] +name = "solana-compute-budget" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5029b1e141344bba1261d5c28de8067cb89927bb06fbbfa71df6ef84d25eaf1b" +dependencies = [ + "solana-fee-structure", + "solana-program-runtime", +] + +[[package]] +name = "solana-compute-budget-instruction" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921873119f821519389b1909697a5b6286c7eb3cba72a5ba3ef172489dde531c" +dependencies = [ + "agave-feature-set", + "solana-borsh", + "solana-builtins-default-costs", + "solana-compute-budget", + "solana-compute-budget-interface", + "solana-instruction", + "solana-packet", + "solana-pubkey", + "solana-sdk-ids", + "solana-svm-transaction", + "solana-transaction-error", +] + +[[package]] +name = "solana-compute-budget-interface" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8292c436b269ad23cecc8b24f7da3ab07ca111661e25e00ce0e1d22771951ab9" +dependencies = [ + "borsh", + "solana-instruction", + "solana-sdk-ids", +] + +[[package]] +name = "solana-compute-budget-program" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c05c42c6d7a20d221ba188d0ab6914081910e8ff4b35c02e6536b94104f575c" +dependencies = [ + "solana-program-runtime", +] + +[[package]] +name = "solana-cpi" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dea26709d867aada85d0d3617db0944215c8bb28d3745b912de7db13a23280c" +dependencies = [ + "solana-account-info", + "solana-define-syscall 4.0.1", + "solana-instruction", + "solana-program-error", + "solana-pubkey", + "solana-stable-layout", +] + +[[package]] +name = "solana-curve25519" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b4d2a4bf0d0b0a86c22111917e86e8bd39a7b31420fb2c7d73eb83761fc7af" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "curve25519-dalek 4.1.3", + "solana-define-syscall 5.2.0", + "subtle", + "thiserror 2.0.19", +] + +[[package]] +name = "solana-define-syscall" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9697086a4e102d28a156b8d6b521730335d6951bd39a5e766512bbe09007cee" + +[[package]] +name = "solana-define-syscall" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5b1c0bc1d4a4d10c88a4100499d954c09d3fecfae4912c1a074dff68b1738" + +[[package]] +name = "solana-define-syscall" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8209ece2bd9f1450e672858ffc0e5c8c786ff6916d2a862b126dd0128f380f" + +[[package]] +name = "solana-ed25519-program" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1419197f1c06abf760043f6d64ba9d79a03ad5a43f18c7586471937122094da" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "solana-instruction", + "solana-sdk-ids", +] + +[[package]] +name = "solana-epoch-rewards" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf7eb4986b0b1d6f562b21f75a836f1a6df6e00c275efcef50aab5c144dc59e" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-hash", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", +] + +[[package]] +name = "solana-epoch-schedule" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8116e6ffa6002237d5ab5edcbda17f9ba66b6742c45a89c9fb40a94dbacd4c1d" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-program-error", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", +] + +[[package]] +name = "solana-feature-gate-interface" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7545e02f91da1d6996f32b18f7796aa01e0682f8f3a7434b82cd1a10448add" +dependencies = [ + "bincode", + "serde", + "serde_derive", + "solana-account", + "solana-account-info", + "solana-instruction", + "solana-program-error", + "solana-pubkey", + "solana-rent", + "solana-sdk-ids", + "solana-system-interface", +] + +[[package]] +name = "solana-fee-calculator" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef67f01cc6a0c72e99a08d0d484683f995de4c80e9568728fa77d1537f9b7e09" +dependencies = [ + "log", + "serde", + "serde_derive", + "wincode", +] + +[[package]] +name = "solana-fee-structure" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e2abdb1223eea8ec64136f39cb1ffcf257e00f915c957c35c0dd9e3f4e700b0" + +[[package]] +name = "solana-get-sysvar" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef3bc859fc036ed490146793557386cbfae614ebba4adc704c37d94350824ed4" +dependencies = [ + "solana-address", + "solana-define-syscall 5.2.0", + "solana-program-error", +] + +[[package]] +name = "solana-hash" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe51db00ac3aa9f950d1e6201a126acfa26e6d81bc4a183ba64ec02effcad883" +dependencies = [ + "bytemuck", + "bytemuck_derive", + "five8", + "serde", + "serde_derive", + "solana-atomic-u64", + "solana-sanitize", + "wincode", +] + +[[package]] +name = "solana-hash-512" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce934b02bab639341f2fd62c3e7e3c39dcceb47f0196b7630ed1f82ecb704bd" + +[[package]] +name = "solana-instruction" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ebb0ffd19263051bc3f683fcc086134b8ff23af894dcb63f7563c7137b42f1" +dependencies = [ + "bincode", + "serde", + "solana-define-syscall 5.2.0", + "solana-instruction-error", + "solana-pubkey", + "wincode", +] + +[[package]] +name = "solana-instruction-error" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b7d34343838343a3755b7dfb1e438d94c6db2263b519cfe3c2257af932b6e93" +dependencies = [ + "num-traits", + "serde", + "serde_derive", + "solana-program-error", + "wincode", +] + +[[package]] +name = "solana-instructions-sysvar" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e38363a181313d607f7a118df2b401bb27b477a0104b09283cbf504f5f0f00cc" +dependencies = [ + "bitflags 2.13.1", + "solana-account-info", + "solana-instruction", + "solana-instruction-error", + "solana-program-error", + "solana-sanitize", + "solana-sdk-ids", + "solana-serialize-utils", + "solana-sysvar-id", +] + +[[package]] +name = "solana-keccak-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed1c0d16d6fdeba12291a1f068cdf0d479d9bff1141bf44afd7aa9d485f65ef8" +dependencies = [ + "sha3", + "solana-define-syscall 4.0.1", + "solana-hash", +] + +[[package]] +name = "solana-keypair" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263d614c12aa267a3278703175fd6440552ca61bc960b5a02a4482720c53438b" +dependencies = [ + "ed25519-dalek 2.2.0", + "five8", + "five8_core", + "rand 0.9.5", + "solana-address", + "solana-seed-phrase", + "solana-signature", + "solana-signer", +] + +[[package]] +name = "solana-last-restart-slot" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c22474b83d3c7c318e1c3a725784fc2d1d03b728e36369e58ce48769a61ed85e" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", +] + +[[package]] +name = "solana-loader-v3-interface" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68029ab11d9c891d4ce23ada75745e40f983c5674af366f447b672569634231b" +dependencies = [ + "serde", + "serde_bytes", + "serde_derive", + "solana-instruction", + "solana-pubkey", + "solana-sdk-ids", + "solana-system-interface", +] + +[[package]] +name = "solana-message" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a634d1db65a393d4e87ec49ef97c8dfb12201e2ba9e5faf87ff34e632f29e509" +dependencies = [ + "blake3", + "lazy_static", + "serde", + "serde_derive", + "solana-address", + "solana-hash", + "solana-instruction", + "solana-sanitize", + "solana-sdk-ids", + "solana-short-vec", + "solana-transaction-error", + "wincode", +] + +[[package]] +name = "solana-msg" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726b7cbbc6be6f1c6f29146ac824343b9415133eee8cce156452ad1db93f8008" +dependencies = [ + "solana-define-syscall 5.2.0", +] + +[[package]] +name = "solana-native-token" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8dd4c280dca9d046139eb5b7a5ac9ad10403fbd64964c7d7571214950d758f" + +[[package]] +name = "solana-nonce" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4172d5b33a0a38fcdb2af8f84406570dd51567267da96c28ad0f31fc11621c0" +dependencies = [ + "serde", + "serde_derive", + "solana-fee-calculator", + "solana-hash", + "solana-pubkey", + "solana-sha256-hasher", + "wincode", +] + +[[package]] +name = "solana-nonce-account" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b81bf7e2aa8c443724051507c1007d0b6d9c34b70925d3232dc78f5ca485209e" +dependencies = [ + "solana-account", + "solana-hash", + "solana-nonce", + "solana-sdk-ids", + "wincode", +] + +[[package]] +name = "solana-packet" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a2a582d7863548f047f49aa3745c076cfa9e1364baa080ef0c1210f0e4ebda" +dependencies = [ + "bitflags 2.13.1", + "solana-pubkey", +] + +[[package]] +name = "solana-poseidon" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "737b8ab25bf4cc8e618f80f1fe40709b2ace708bc764a36b8a4c81eea8c07034" +dependencies = [ + "ark-bn254 0.4.0", + "ark-bn254 0.5.0", + "light-poseidon 0.2.0", + "light-poseidon 0.4.0", + "solana-define-syscall 4.0.1", + "thiserror 2.0.19", +] + +[[package]] +name = "solana-precompile-error" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cafcd950de74c6c39d55dc8ca108bbb007799842ab370ef26cf45a34453c31e1" +dependencies = [ + "num-traits", +] + +[[package]] +name = "solana-program-entrypoint" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c9b0a1ff494e05f503a08b3d51150b73aa639544631e510279d6375f290997" +dependencies = [ + "solana-account-info", + "solana-define-syscall 4.0.1", + "solana-program-error", + "solana-pubkey", +] + +[[package]] +name = "solana-program-error" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f04fa578707b3612b095f0c8e19b66a1233f7c42ca8082fcb3b745afcc0add6" + +[[package]] +name = "solana-program-memory" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4068648649653c2c50546e9a7fb761791b5ab0cda054c771bb5808d3a4b9eb52" +dependencies = [ + "solana-define-syscall 4.0.1", +] + +[[package]] +name = "solana-program-runtime" +version = "4.1.1" +dependencies = [ + "assert_matches", + "base64", + "bincode", + "cfg-if", + "itertools 0.13.0", + "qualifier_attr", + "scc", + "serde", + "solana-account", + "solana-account-info", + "solana-clock", + "solana-epoch-rewards", + "solana-epoch-schedule", + "solana-fee-structure", + "solana-hash", + "solana-instruction", + "solana-keypair", + "solana-last-restart-slot", + "solana-loader-v3-interface", + "solana-program-entrypoint", + "solana-program-runtime", + "solana-pubkey", + "solana-rent", + "solana-sbpf", + "solana-sdk-ids", + "solana-signer", + "solana-slot-hashes", + "solana-stable-layout", + "solana-svm-callback", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-measure", + "solana-svm-timings", + "solana-svm-transaction", + "solana-svm-type-overrides", + "solana-system-interface", + "solana-sysvar", + "solana-sysvar-id", + "solana-transaction", + "solana-transaction-context", + "test-case", + "thiserror 2.0.19", +] + +[[package]] +name = "solana-pubkey" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7db719574990de7e8b0f55a8593ac92a5ccb42c8ce67b3e4bf05b139d5d9ee71" +dependencies = [ + "rand 0.9.5", + "solana-address", +] + +[[package]] +name = "solana-rent" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f0d780bf8e8a1fe8b5b5fce1acad6b209485b86dec246e7523d5e4a8b7c7fc" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-sysvar-id", +] + +[[package]] +name = "solana-sanitize" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf09694a0fc14e5ffb18f9b7b7c0f15ecb6eac5b5610bf76a1853459d19daf9" + +[[package]] +name = "solana-sbpf" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f84c593fa3d4131045b606dec5acf9d8eac73791bc786ca9911057aec8f43ec" +dependencies = [ + "byteorder", + "combine", + "gdbstub", + "hash32", + "libc", + "log", + "rand 0.8.7", + "rustc-demangle", + "shuttle", + "thiserror 2.0.19", + "winapi", +] + +[[package]] +name = "solana-sdk-ids" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "def234c1956ff616d46c9dd953f251fa7096ddbaa6d52b165218de97882b7280" +dependencies = [ + "solana-address", +] + +[[package]] +name = "solana-sdk-macro" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8765316242300c48242d84a41614cb3388229ec353ba464f6fe62a733e41806f" +dependencies = [ + "bs58", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "solana-secp256k1-program" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad4cf8232f7aef9ff2dd95d701f63e3c11909dec2400def5c361be29d24291e7" +dependencies = [ + "digest 0.10.7", + "k256", + "serde", + "serde_derive", + "sha3", + "solana-signature", +] + +[[package]] +name = "solana-secp256k1-recover" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce0d2e8d53946f04e789476d6c407e4997b7fb412483df5fc10c075c65cb2edf" +dependencies = [ + "k256", + "solana-define-syscall 5.2.0", + "thiserror 2.0.19", +] + +[[package]] +name = "solana-secp256r1-program" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445d8e12592631d76fc4dc57858bae66c9fd7cc838c306c62a472547fc9d0ce6" +dependencies = [ + "bytemuck", + "openssl", + "solana-instruction", + "solana-sdk-ids", +] + +[[package]] +name = "solana-seed-phrase" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc905b200a95f2ea9146e43f2a7181e3aeb55de6bc12afb36462d00a3c7310de" +dependencies = [ + "hmac 0.12.1", + "pbkdf2", + "sha2 0.10.9", +] + +[[package]] +name = "solana-serialize-utils" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "761357b0853c9623bf12c1d2314b3d6160a85b087b84c45224fb85766d22616b" +dependencies = [ + "solana-instruction-error", + "solana-pubkey", + "solana-sanitize", +] + +[[package]] +name = "solana-sha256-hasher" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db7dc3011ea4c0334aaaa7e7128cb390ecf546b28d412e9bf2064680f57f588f" +dependencies = [ + "sha2 0.10.9", + "solana-define-syscall 4.0.1", + "solana-hash", +] + +[[package]] +name = "solana-sha512-hasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11e10e103ab5bd52af341e7bc2f4123a2f4e66bb7ba4e6ca40f1e00cd6314e02" +dependencies = [ + "sha2 0.10.9", + "solana-define-syscall 5.2.0", + "solana-hash-512", +] + +[[package]] +name = "solana-short-vec" +version = "3.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8250a4495aad49ad20556a607da53bdcb20de78da10b65afbf918b7f1de647" +dependencies = [ + "serde_core", + "wincode", +] + +[[package]] +name = "solana-signature" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0364c7577c3c82a693ce28a1febc8d1b5d1b0a175fdc2114ae6186b69effe1e" +dependencies = [ + "ed25519-dalek 2.2.0", + "five8", + "rand 0.9.5", + "serde", + "serde-big-array", + "serde_derive", + "solana-sanitize", + "wincode", +] + +[[package]] +name = "solana-signer" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520bd6021163ee517f4bdc7ae03ded904f97e11320001ba0b3355f45eb14f558" +dependencies = [ + "solana-pubkey", + "solana-signature", + "solana-transaction-error", +] + +[[package]] +name = "solana-slot-hashes" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7ce2b4b8911bf2db3de7b6266e67bfc21a6a9f8c566fb096d9782ca2ad16ee" +dependencies = [ + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-hash", + "solana-sdk-ids", + "solana-sysvar-id", +] + +[[package]] +name = "solana-slot-history" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40427c04d3e808493cb5e3d1a97cef84d7c15cb6f89b15c5684d0d4027105600" +dependencies = [ + "bv", + "serde", + "serde_derive", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sysvar-id", +] + +[[package]] +name = "solana-stable-layout" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9f6a291ba063a37780af29e7db14bdd3dc447584d8ba5b3fc4b88e2bbc982fa" +dependencies = [ + "solana-instruction", + "solana-pubkey", +] + +[[package]] +name = "solana-stake-history" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c736a6aa0e53b9d264d90b06589fc0996f49f882f3e71842ed754fc57ffc1a43" +dependencies = [ + "serde", + "serde_derive", + "solana-clock", + "solana-get-sysvar", + "solana-sdk-ids", + "solana-sysvar-id", +] + +[[package]] +name = "solana-stake-interface" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49eb5c77484214c3484921e2cdda79d185373118b5458c1b2df0f1a04c3bc30" +dependencies = [ + "num-traits", + "solana-clock", + "solana-instruction", + "solana-program-error", + "solana-pubkey", + "solana-system-interface", +] + +[[package]] +name = "solana-svm" +version = "4.1.1" +dependencies = [ + "ahash", + "bincode", + "env_logger", + "magic-root-interface", + "qualifier_attr", + "rand 0.9.5", + "serde", + "solana-account", + "solana-clock", + "solana-ed25519-program", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-fee-structure", + "solana-hash", + "solana-instruction", + "solana-instructions-sysvar", + "solana-keypair", + "solana-loader-v3-interface", + "solana-message", + "solana-native-token", + "solana-precompile-error", + "solana-program-runtime", + "solana-pubkey", + "solana-rent", + "solana-sbpf", + "solana-sdk-ids", + "solana-signature", + "solana-signer", + "solana-svm", + "solana-svm-callback", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-transaction", + "solana-svm-type-overrides", + "solana-system-interface", + "solana-sysvar", + "solana-transaction", + "solana-transaction-context", + "solana-transaction-error", +] + +[[package]] +name = "solana-svm-callback" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3170a3cfc032f3efca975154dee904ecefea5ae11fbd50787c062886196d32c1" +dependencies = [ + "solana-account", + "solana-clock", + "solana-precompile-error", + "solana-pubkey", +] + +[[package]] +name = "solana-svm-feature-set" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3d427bb7cd5182365e7e89d73fcfcb54565d15445ad2d228921811c3836099" + +[[package]] +name = "solana-svm-log-collector" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afdf3074910afe016266bf73606724543b6829b5656221c8060ebcfcc844b39a" +dependencies = [ + "log", +] + +[[package]] +name = "solana-svm-measure" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ecaaadf4ddaabdba865c7f9aade0c51ac7c393c786f8b4e9590127f5ee9d62" + +[[package]] +name = "solana-svm-timings" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b179027c8bf618df4a7f02f937ddef54770f8f5584195c94a03437ee6dc7c7f3" +dependencies = [ + "eager", + "enum-iterator", + "solana-pubkey", +] + +[[package]] +name = "solana-svm-transaction" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa27dd1eb7ce4d339cf9dfc1dfb70866488e4e1436fb05728aea7af0185191a" +dependencies = [ + "solana-hash", + "solana-message", + "solana-pubkey", + "solana-sdk-ids", + "solana-signature", + "solana-transaction", +] + +[[package]] +name = "solana-svm-type-overrides" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e0ca52c449480ab3a1ef614ae10576d6ccc33730a0f4d8b4a69f38e8decb622" +dependencies = [ + "rand 0.9.5", + "shuttle", +] + +[[package]] +name = "solana-syscalls" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d855514df361d211bd254929ab816321f3bd136f3b985605c68ba68a1b1d729" +dependencies = [ + "bincode", + "libsecp256k1", + "num-traits", + "solana-account", + "solana-account-info", + "solana-big-mod-exp", + "solana-blake3-hasher", + "solana-bls12-381-syscall", + "solana-bn254", + "solana-clock", + "solana-cpi", + "solana-curve25519", + "solana-hash", + "solana-hash-512", + "solana-instruction", + "solana-keccak-hasher", + "solana-poseidon", + "solana-program-entrypoint", + "solana-program-runtime", + "solana-pubkey", + "solana-sbpf", + "solana-sdk-ids", + "solana-secp256k1-recover", + "solana-sha256-hasher", + "solana-sha512-hasher", + "solana-stable-layout", + "solana-stake-interface", + "solana-svm-feature-set", + "solana-svm-log-collector", + "solana-svm-type-overrides", + "solana-sysvar", + "solana-sysvar-id", + "solana-transaction-context", + "thiserror 2.0.19", +] + +[[package]] +name = "solana-system-interface" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55b54965bf0b76fa8e2b35376583efddd4d916618cfe595bf48c7d7b55a9e628" +dependencies = [ + "num-traits", + "serde", + "serde_derive", + "solana-address", + "solana-instruction", + "solana-msg", + "solana-program-error", + "wincode", +] + +[[package]] +name = "solana-system-program" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84eaae98e1576c1b490760d102ca15cb0cfc7ed2b52012dc0d06698a2529e093" +dependencies = [ + "bincode", + "log", + "solana-account", + "solana-bincode", + "solana-fee-calculator", + "solana-instruction", + "solana-nonce", + "solana-nonce-account", + "solana-packet", + "solana-program-runtime", + "solana-pubkey", + "solana-sdk-ids", + "solana-svm-log-collector", + "solana-system-interface", + "solana-sysvar", + "solana-transaction-context", +] + +[[package]] +name = "solana-sysvar" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada7045bbfdd802af08fbc9c7e2b56ddabb20599d22e4c3840fcef5e7afb8324" +dependencies = [ + "base64", + "bincode", + "bytemuck", + "bytemuck_derive", + "lazy_static", + "serde", + "serde_derive", + "solana-account-info", + "solana-clock", + "solana-define-syscall 5.2.0", + "solana-epoch-rewards", + "solana-epoch-schedule", + "solana-fee-calculator", + "solana-get-sysvar", + "solana-hash", + "solana-instruction", + "solana-last-restart-slot", + "solana-program-entrypoint", + "solana-program-error", + "solana-program-memory", + "solana-pubkey", + "solana-rent", + "solana-sdk-ids", + "solana-sdk-macro", + "solana-slot-hashes", + "solana-slot-history", + "solana-stake-history", + "solana-sysvar-id", +] + +[[package]] +name = "solana-sysvar-id" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17358d1e9a13e5b9c2264d301102126cf11a47fd394cdf3dec174fe7bc96e1de" +dependencies = [ + "solana-address", + "solana-sdk-ids", +] + +[[package]] +name = "solana-transaction" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a98253cefd62dea714b67926d3d918c3b9f7d66993f1f2322ac7549ea991dc3e" +dependencies = [ + "serde", + "serde_derive", + "solana-address", + "solana-hash", + "solana-instruction", + "solana-instruction-error", + "solana-message", + "solana-sanitize", + "solana-sdk-ids", + "solana-short-vec", + "solana-signature", + "solana-signer", + "solana-transaction-error", + "wincode", +] + +[[package]] +name = "solana-transaction-context" +version = "4.1.1" +dependencies = [ + "bincode", + "serde", + "solana-account", + "solana-account-info", + "solana-instruction", + "solana-instructions-sysvar", + "solana-program-entrypoint", + "solana-pubkey", + "solana-rent", + "solana-sbpf", + "solana-sdk-ids", + "solana-system-interface", + "solana-transaction-context", + "static_assertions", +] + +[[package]] +name = "solana-transaction-error" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a949797bc1ac31836d0070e791083028a8c2e0d493fa4cf51c0bed7a04c65c22" +dependencies = [ + "serde", + "serde_derive", + "solana-instruction-error", + "solana-sanitize", + "wincode", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synchronoise" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" +dependencies = [ + "crossbeam-queue", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tar" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "test-case-core", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl 2.0.19", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.13+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "twox-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unreachable" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "382810877fe448991dfc7f0dd6e3ae5d58088fd0ea5e35189655f84e6814fa56" +dependencies = [ + "void", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "v42-calculator-interface" +version = "0.1.0" +dependencies = [ + "solana-instruction", + "solana-pubkey", +] + +[[package]] +name = "v42-calculator-program" +version = "0.1.0" +dependencies = [ + "solana-account-info", + "solana-cpi", + "solana-instruction", + "solana-msg", + "solana-program-entrypoint", + "solana-program-error", + "solana-pubkey", + "solana-sysvar", + "v42-calculator-interface", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "wincode" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d967db7705dc29120bb6e8ce5b5a2e27734ed5976d1c904e95bd238d1c3c5a" +dependencies = [ + "pastey", + "proc-macro2", + "quote", + "thiserror 2.0.19", + "wincode-derive", +] + +[[package]] +name = "wincode-derive" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15ab90b719560d0fda79c74550ad1c948d17b118765942838055ebaf34d67071" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 85ad1f11..c0d6e21a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,22 @@ [workspace] +members = [ + "accountsdb", + "engine", + "keeper", + "ledger", + "nucleus", + "processor", + "programs/magic-root-interface", + "programs/magic-root-program", + "programs/v42-calculator-interface", + "programs/v42-calculator-program", + "replicator", + "solana/account", + "solana/program-runtime", + "solana/svm", + "solana/transaction-context", + "solana/transaction-view" +] resolver = "3" [workspace.package] @@ -10,11 +28,127 @@ repository = "https://github.com/magicblock-labs/engine" rust-version = "1.94.1" version = "0.1.0" +[workspace.dependencies] +accountsdb = { path = "accountsdb", package = "magicblock-accountsdb" } +engine = { path = "engine", package = "magicblock-engine" } +keeper = { path = "keeper", package = "magicblock-keeper" } +ledger = { path = "ledger", package = "magicblock-ledger" } +magic-root-interface = { path = "programs/magic-root-interface" } +magic-root-program = { path = "programs/magic-root-program" } +nucleus = { path = "nucleus", package = "magicblock-engine-nucleus" } +processor = { path = "processor", package = "magicblock-processor" } +solana-account = { path = "solana/account" } +solana-program-runtime = { path = "solana/program-runtime" } +solana-svm = { path = "solana/svm" } +solana-transaction-context = { path = "solana/transaction-context" } +v42-calculator-interface = { path = "programs/v42-calculator-interface", default-features = false } + +ahash = "0.8.12" +arc-swap = "1.9.1" +assert_matches = "1.5.0" +base64 = "0.22.1" +bincode = "1.3.3" +bitcode = "0.6.9" +bitflags = "2.11.1" +blake3 = "1.8.5" +bytemuck = "1.25" +cfg-if = "1.0.4" +clonetree = "0.0.2" +criterion = "0.8.2" +derive_more = "2.1.1" +env_logger = "0.11.8" +flume = { version = "0.12" } +futures = { version = "0.3.32", default-features = false } +heed = { version = "0.22.1", default-features = false } +itertools = "0.13.0" +memmap2 = "0.9.10" +num_cpus = "1.17.0" +oneshot = "0.2.1" +parking_lot = "0.12.5" +prometheus = { version = "0.14.0", default-features = false } +qualifier_attr = "0.2.2" +rand = "0.9.2" +rustix = { version = "1.1.4" } +scc = "3.8.4" +serde = "1.0.228" +serde_bytes = "0.11.19" +serde_with = { version = "3.21.0", default-features = false } +smallvec = "1.15.2" +snedfile = "0.1" +tar = "0.4.45" +tempfile = "3" +thiserror = "2.0.17" +tokio = "1.52.1" +tokio-util = "0.7.18" +tracing = "0.1.44" +tracing-subscriber = "0.3.23" +twox-hash = { version = "2.1.2", default-features = false } +wincode = "0.5.1" +zstd = { version = "0.13.3", default-features = false } + +agave-feature-set = "4.1.1" +agave-precompiles = "4.1.1" +agave-syscalls = { package = "solana-syscalls", version = "=4.1.1", default-features = false } +agave-transaction-view = "4.1.1" +solana-account-info = "3.1.1" +solana-clock = "3.1.0" +solana-compute-budget-instruction = "=4.1.1" +solana-compute-budget-program = "=4.1.1" +solana-cpi = "3.1.0" +solana-ed25519-program = "3.0.0" +solana-epoch-rewards = "3.0.1" +solana-epoch-schedule = "3.1.0" +solana-feature-gate-interface = "4.0.0" +solana-fee-calculator = "=3.2.2" +solana-fee-structure = "3.0.0" +solana-hash = "4.3.0" +solana-instruction = "=3.4.0" +solana-instruction-error = "=2.4.0" +solana-instructions-sysvar = "4.0.0" +solana-keypair = "3.1.2" +solana-last-restart-slot = "3.0.0" +solana-loader-v3-interface = "7.0.0" +solana-message = "4.1.1" +solana-msg = "3.1.0" +solana-native-token = "3.0.0" +solana-packet = "=4.2.0" +solana-precompile-error = "3.0.0" +solana-program-entrypoint = "3.1.1" +solana-program-error = "3.0.1" +solana-pubkey = "=4.2.0" +solana-rent = "4.2.0" +solana-sbpf = "=0.21.0" +solana-sdk-ids = "3.1.0" +solana-short-vec = "=3.2.2" +solana-signature = "=3.4.1" +solana-signer = "3.0.1" +solana-slot-hashes = "3.0.1" +solana-stable-layout = "3.0.1" +solana-svm-callback = "4.1.1" +solana-svm-feature-set = "4.1.1" +solana-svm-log-collector = "4.1.1" +solana-svm-measure = "4.1.1" +solana-svm-timings = "4.1.1" +solana-svm-transaction = "4.1.1" +solana-svm-type-overrides = "4.1.1" +solana-system-interface = "=3.2.0" +solana-system-program = "=4.1.1" +solana-sysvar = "4.0.0" +solana-sysvar-id = "3.1.0" +solana-transaction = "4.1.1" +solana-transaction-error = "=3.3.1" + +[patch.crates-io] +agave-transaction-view = { path = "solana/transaction-view" } +solana-account = { path = "solana/account" } +solana-program-runtime = { path = "solana/program-runtime" } +solana-svm = { path = "solana/svm" } +solana-transaction-context = { path = "solana/transaction-context" } + [workspace.lints.rust] missing_docs = "deny" rust_2018_idioms = { level = "warn", priority = -1 } unreachable_pub = "warn" -unsafe_op_in_unsafe_fn = "allow" unused_lifetimes = "warn" unused_macro_rules = "warn" unused_qualifications = "warn" diff --git a/README.md b/README.md new file mode 100644 index 00000000..5f8eb4dd --- /dev/null +++ b/README.md @@ -0,0 +1,353 @@ +

MagicBlock Engine

+ +

+ Execution engine for ephemeral rollups — Solana transactions over durable, locally-owned state. +

+ +

+ License Apache-2.0 + Rust 1.96.1 + Edition 2024 + Solana SVM + Status experimental + Version 0.1.0 +

+ +--- + +MagicBlock Engine executes Solana transactions for ephemeral rollups. It owns +account state, records transaction and block history, and exposes asynchronous +APIs for execution, simulation, reads, and subscriptions. + +## ✨ Highlights + +| | | | +| :-- | :-- | :-- | +| ⚙️ **Runs Solana programs** — a real SVM, without the overhead of a validator | 🗃️ **Storage that fits the account** — engine-owned state on disk, chain-mirrored state in memory | 📚 **Retained history** — transactions and blocks kept in segments you can retain or drop wholesale | +| 🔁 **Replication** — mirror a live engine onto standby nodes over TCP | 🩹 **Recoverable startup** — restores snapshots and verifies replayed history after a crash | 📡 **Async APIs** — execute, simulate, read, and subscribe over live state | + +## 📖 Contents + +- [🚀 Starting the engine](#-starting-the-engine) +- [🛑 Shutdown](#-shutdown) +- [🔁 Replication](#-replication) +- [📦 Account state](#-account-state) +- [📨 Transactions](#-transactions) +- [📡 Subscriptions](#-subscriptions) +- [🩹 Startup and recovery](#-startup-and-recovery) +- [🧩 Workspace layout](#-workspace-layout) + +--- + +## 🚀 Starting the engine + +Bringing up an engine is mostly filling in one struct and awaiting one call — +everything underneath (storage, ledger, scheduler, background tasks) is wired up +for you. + +The embedding service must retain both the engine and its `ShutdownManager`. +The manager coordinates every background service started by `Engine::new`. + +```rust +use std::{num::NonZeroU64, path::PathBuf, time::Duration}; + +use engine::Engine; +use keeper::builder::KeeperBuilder; +use nucleus::{ + config::{AccountsDBParams, BlockstoreParams, LedgerParams}, + shutdown::ShutdownManager, +}; +use solana_keypair::Keypair; +use solana_sysvar::rent::Rent; + +async fn open_engine( + home: PathBuf, +) -> engine::Result<(Engine, ShutdownManager)> { + let mut shutdown = ShutdownManager::default(); + let builder = KeeperBuilder { + authority: Keypair::new().into(), + accountsdb: AccountsDBParams { + directory: home.join("accountsdb"), + lru_capacity: 10_000, + }, + ledger: LedgerParams { + directory: home.join("ledger"), + size_limit: 256 * 1024 * 1024 * 1024, + }, + blockstore: BlockstoreParams { + blocktime: Duration::from_millis(400), + superblock: NonZeroU64::new(16).unwrap(), + }, + builtins: Default::default(), + programs: Default::default(), + accounts: Default::default(), + rent: Rent::default(), + }; + + let engine = Engine::new(builder, None, &mut shutdown).await?; + Ok((engine, shutdown)) +} +``` + +The second argument chooses who advances blocks. `None` runs the built-in +pacer, which produces blocks on its own clock — the standalone case. Passing a +channel instead makes block boundaries caller-driven, as replication followers +do when they step in time with a leader. External producers supply the slot and +timestamp; the sequencer computes and overwrites the block hash and parent. + +The two modes also start differently: the built-in pacer wipes chain-mirrored +volatile accounts at startup (internal system accounts stay available), so a +standalone engine begins from clean external state. An external pacer keeps +whatever volatile state was restored, which replication depends on. + +--- + +## 🛑 Shutdown + +Shutdown isn't a hard stop — it unwinds in tiers, so in-flight work drains and +durable state lands on disk before the process goes away. + +The host waits for an OS signal or premature service termination with +`ShutdownManager::wait`. It should then stop external ingress and call +`ShutdownManager::terminate` while retaining the engine handle. + +```rust +let cause = shutdown.wait().await; + +// Stop accepting transactions and other external work here. +shutdown.terminate().await; +``` + +`wait` returns whether shutdown was requested by an OS signal or by a managed +service terminating early. Embedding processes can use the service reason to +distinguish recoverable lifecycle events, such as a replication snapshot that +requires reopening the engine, from fatal failures. + +Shutdown proceeds by service tier: + +1. A replication client stops consuming upstream state. +2. The pacemaker stops producing boundaries and calls `Engine::shutdown`. +3. The already-drained sequencer and terminally-synced ledger appender stop. +4. Ledger readers, simulation, subscriptions, and other backing services stop. + +Internal pacing publishes a final block and flushes durable state. External +pacing also writes volatile state to `CURRENT/volatile.db` after flushing the +corresponding ledger cursor. The final sync explicitly closes ledger workers, +so retained but inactive engine handles cannot hold shutdown open. Each tier +has a bounded termination window. + +--- + +## 🔁 Replication + +Point a follower at a leader and it keeps itself in sync — replaying the stream +when it can, and pulling a fresh snapshot when it has fallen too far behind. + +Replication keeps a standby engine in step with a live one: a **leader** serves +its history over TCP, and one or more **followers** replay that stream to stay +current. On the leader machine, bind a dispatcher to a reachable address and +serve the retained ledger: + +```rust +use std::sync::Arc; + +use replicator::ReplicationDispatcher; + +let allowed = Arc::from([follower_identity]); +ReplicationDispatcher::spawn(bind_addr, engine.clone(), allowed, &mut shutdown).await?; +``` + +On the follower machine, open its engine with an external pacer and connect the +client to the leader's address: + +```rust +use replicator::ReplicationClient; +use tokio::sync::mpsc; + +let (block_tx, block_rx) = mpsc::channel(16); +builder.authority.remote = Some(leader_identity); +let engine = Engine::new(builder, Some(block_rx), &mut shutdown).await?; +ReplicationClient::spawn(leader_addr, engine.clone(), block_tx, &mut shutdown)?; +``` + +Leader and follower local keypairs do not need to match. The server allowlist +contains follower local identities and denies all access when empty. The +follower's remote authority identifies its immediate upstream, whose signed +responses must arrive within 30 seconds of the follower's clock. + +The external pacer keeps replicated blocks ordered with transactions, resets, +and seals. If the leader's retained stream cannot satisfy the follower's cursor, +it sends the newest snapshot. The client stages it, reports `RestartRequired` +through the follower's shutdown manager, and the follower host reopens its +engine from the same directories. + +--- + +## 📦 Account state + +You never have to decide where an account lives — the engine watches what each +account *is* and keeps it in the right place on its own. + +The engine holds two kinds of accounts and stores each where it makes sense: + +- Accounts the engine controls — delegated, ephemeral, and transient — are + authoritative here and **persisted to disk**. +- Accounts that only mirror external chain or system state — read-only, + placeholders, and sysvars — are kept **in volatile memory**. + +An account's `AccountMode::authoritative()` classification decides which side it +belongs to. When that changes, accountsdb moves the account and drops the stale +copy from the other backend, so there is only ever one live copy. `Transient` +accounts remain authoritative and persisted even though runtime code cannot +mutate them. + +To replace accounts directly, use `Engine::account(pubkey)`. `create`, `update`, +and `delete` each run as one signed, committed transaction and require the local +signer to match the engine authority. + +```rust +use solana_account::{AccountBuilder, AccountMode}; +use solana_pubkey::Pubkey; + +let key = Pubkey::new_unique(); +let owner = Pubkey::new_unique(); +let account = AccountBuilder::default() + .lamports(2_000_000) + .owner(owner) + .mode(AccountMode::ReadOnly) + .slot(1) + .data(vec![1, 2, 3, 4]) + .build(); + +engine.account(key).create(account, None).await?; + +let replacement = AccountBuilder::default() + .lamports(2_000_000) + .owner(owner) + .mode(AccountMode::ReadOnly) + .slot(2) + .data(vec![5; 4]) + .build(); +engine.account(key).update(replacement).await?; +engine.account(key).delete().await?; +``` + +Each mutation is one committed transaction. `create` can also run optional +post-finalize instructions in that transaction; if an instruction fails, the +creation does not commit. Complete-account patches cover non-flag fields, and +finalization atomically installs the caller-supplied flags without changing +lamports. Callers are responsible for supplying current state; later +replacements remain subject to the account's slot and lifecycle rules. Internal +create composition places post-finalize instructions immediately after +finalization. + +Missing external accounts can be coordinated with `Engine::accounts().ensure`. +The first caller receives `MissingAccount::Load`; concurrent callers receive a +wait handle for the same pubkey. After storing the account, the loader calls +`AccountLoad::complete(mode)` to publish success and update recency tracking for +non-authoritative accounts. Dropping the load guard instead wakes waiters with a +failed outcome. + +--- + +## 📨 Transactions + +Hand it whatever you've already got — a few instructions, a `Message`, or raw +encoded bytes — and pick how much you want to wait around for. + +`Engine::transaction` accepts an instruction slice, `Message`, sanitized +`TransactionView`, or encoded transaction bytes. Instruction slices and messages +use the effective authority as payer and the local signer with the latest +blockhash, so local composition requires those identities to match. + +```rust +use engine::Engine; +use solana_instruction::Instruction; + +async fn submit( + engine: &Engine, + instructions: &[Instruction], +) -> engine::Result<()> { + engine + .transaction(instructions)? + .execute() + .await? + .map_err(Into::into) +} +``` + +- `execute` waits for the committed transaction result. +- `schedule` queues execution without waiting for its result. +- `simulate` executes against owned account copies without committing state. + +--- + +## 📡 Subscriptions + +No polling loops — subscribe to what you care about and the engine pushes +updates as they happen. + +Keeper accessors expose dedicated Tokio channels for live state: + +```rust +let mut account_updates = engine.accounts().subscribe(key).await; +let mut blocks = engine.blocks().subscribe(); + +let account = account_updates.recv().await.expect("account stream is open"); +let block = blocks.recv().await.expect("block stream is open"); +``` + +Related accessors subscribe to program-owned accounts, cache evictions, +snapshot completion, transaction status, logs, processed transactions, and +service messages. Signatures use terminal oneshot channels; other multicast +streams give each consumer a bounded queue and disconnect a consumer that falls +behind. Processed transactions, service messages, and cache evictions each have +one process-lifetime consumer and apply producer backpressure when its queue is +full. + +--- + +## 🩹 Startup and recovery + +After an interrupted write, the next start checks local state against retained +history and restores a retained snapshot when necessary. + +Every startup reconciles the account store with the transaction history. A +crash, corruption, and a staged replication snapshot enter the same recovery +path, but recovery requires a valid retained snapshot when the current store +cannot be used. + +Concretely: keeper validates the account store against the retained ledger. A +corrupt store, or a valid one whose latest checkpoint trails the ledger, is +replaced with the newest retained snapshot. If that restored state still trails +the ledger tip, the engine replays the missing history to catch up, checking the +rebuilt state against each recorded checkpoint and refusing to continue +(`ReplayError::StateMismatch`) if they diverge. When the store is already +current, nothing runs. + +--- + +## 🧩 Workspace layout + +| Crate | Role | +| :-- | :-- | +| `nucleus` | Shared ledger, runtime, metrics, TLS, and shutdown types. | +| `solana/*` | The runtime forks required by the engine account model. | +| `accountsdb` | Owns persisted and volatile account storage and snapshots. | +| `ledger` | Stores transactions, execution records, blocks, and superblocks. | +| `keeper` | Opens both stores and provides caches, reads, and subscriptions. | +| `processor` | Schedules transactions across SVM executors and commits results. | +| `programs/*` | MagicRoot and the v42 test program and interfaces. | +| `engine` | Wires the execution engine and exposes the public handle. | +| `replicator` | Streams durable engine state between nodes. | + +Transactions are appended before execution, then paired with execution metadata. +Successful dirty accounts are written through accountsdb and live notifications +are published. Superblock boundaries quiesce execution while keeper snapshots +accountsdb and archives it beside the next retained ledger segment. + +--- + +

+ Built with 🦀 Rust · licensed under Apache-2.0 · © MagicBlock contributors +

diff --git a/accountsdb/Cargo.toml b/accountsdb/Cargo.toml new file mode 100644 index 00000000..d6cc7aeb --- /dev/null +++ b/accountsdb/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "magicblock-accountsdb" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "accountsdb" + +[features] +testkit = [] + +[dependencies] +nucleus = { workspace = true, features = ["heed", "metrics"] } + +ahash = { workspace = true } +bincode = { workspace = true } +bytemuck = { workspace = true, features = ["derive", "extern_crate_std"] } +clonetree = { workspace = true } +derive_more = { workspace = true, features = ["from"] } +heed = { workspace = true } +memmap2 = { workspace = true } +parking_lot = { workspace = true } +scc = { workspace = true, features = ["serde"] } +thiserror = { workspace = true } +tracing = { workspace = true } +twox-hash = { workspace = true, features = ["alloc", "xxhash3_64"] } + +solana-account = { workspace = true, features = ["serde"] } +solana-pubkey = { workspace = true, features = ["bytemuck"] } + +[dev-dependencies] +accountsdb = { workspace = true, features = ["testkit"] } +assert_matches = { workspace = true } +nucleus = { workspace = true, features = ["testkit"] } + +[lints] +workspace = true diff --git a/accountsdb/README.md b/accountsdb/README.md new file mode 100644 index 00000000..2c25bfb5 --- /dev/null +++ b/accountsdb/README.md @@ -0,0 +1,66 @@ +# `magicblock-accountsdb` + +Accountsdb routes account state between two backends according to +`AccountMode::authoritative()`: + +- `PersistedStore` is an mmap-backed account file with LMDB indexes. It holds + delegated, ephemeral, and transient accounts controlled by the engine. +- `VolatileStore` is an in-memory map for externally owned state that can be + fetched again. + +Every store operation touches the backend required by both the account's current +representation and authoritative classification. This commits borrowed images +in persistent storage, inserts owned images there, updates owned volatile +images, and removes stale copies after mode changes or closure. `Transient` +remains authoritative and runtime-immutable until its lifecycle state resolves. + +`AccountsDB::commit` is the ledger-transaction boundary. It stores successful +account transitions and then advances a persistent transaction counter; empty +transitions from failed executions advance the counter as well. Direct `store` +operations used for initialization, sysvars, and administrative writes do not. + +## Persisted layout + +`CURRENT/storage.db` contains a metadata header followed by account images in the +borrowed `solana-account` layout. Each image includes its full pubkey so scans can +recover keys without the index. Offsets are measured in 8-byte `StorageUnit`s. +The transaction counter is metadata and is not part of the account checksum. + +The LMDB index under `CURRENT/index` contains: + +- `accounts`: account key tag to storage offset and owner tag. +- `programs`: owner tag to account offsets. +- `freelist`: image size to reusable offsets. + +`PersistedProgramIter` retains its read transaction for the persisted portion of +iteration. The optional `testkit` feature uses smaller maps and growth blocks +without changing the on-disk format. + +## Writes and compaction + +A persisted batch commits its LMDB transaction once. If applying or committing +the batch fails, already committed borrowed images are rolled back so indexed +state remains authoritative. Freed image spans enter the freelist. + +Defragmentation requires exclusive access. Snapshot export packs tail accounts +into exact holes or the smallest fitting holes that leave a minimum useful +remainder. It copies only between non-overlapping spans and publishes all +relocations in one index transaction. Vacated source spans are deferred to the +next pass, so some fragmented layouts may stall. + +After validation, keeper startup repeats committed packing passes to a fixed +point before exposing the database to readers. Snapshot export runs one pass. +Both paths synchronously flush successful changes. + +## Snapshots and volatile state + +`AccountsDB::snapshot` requires exclusive write access. It records the +superblock id, runs one packing pass and flushes persisted state, clones the +active tree, and serializes the current volatile map into the clone's +`volatile.db`. + +`dump(None)` writes `CURRENT/volatile.db` for a clean externally paced shutdown. +The next open restores that file into memory and removes it. `reset` instead +removes chain-mirrored volatile accounts while preserving internal system +accounts and rebuilding their owner indexes. Persisted engine-authoritative +state is never reset. diff --git a/accountsdb/src/lib.rs b/accountsdb/src/lib.rs new file mode 100644 index 00000000..d1dc7ff0 --- /dev/null +++ b/accountsdb/src/lib.rs @@ -0,0 +1,349 @@ +#![doc = include_str!("../README.md")] + +use std::{ + cell::RefCell, + collections::BTreeSet, + path::{Path, PathBuf}, + sync::atomic::Ordering::*, +}; + +use derive_more::From; +use nucleus::Slot; +use nucleus::heed::RoTxnTls; +use solana_account::{AccountSeqLock, AccountSharedData, CoWAccount}; +use solana_pubkey::Pubkey; +use tracing::{info, warn}; + +use crate::{ + store::{DatabaseVersion, PersistedProgramIter, PersistedStore}, + volatile::VolatileStore, +}; + +pub use snapshot::{BackupOp, SnapshotError, SnapshotResult}; +pub use store::mmap::STORAGE_FILE; + +mod metrics; +mod snapshot; +mod store; +mod volatile; + +#[cfg(test)] +mod tests; + +/// Active database subdirectory. +const ACTIVE_DIR: &str = "CURRENT"; + +/// Top-level account store backed by persisted and volatile backends. +pub struct AccountsDB { + /// On-disk store for engine-authoritative account modes. + persisted: PersistedStore, + /// Rebuildable in-memory store for non-authoritative account modes. + volatile: VolatileStore, + /// Database root directory. + root: PathBuf, +} + +impl AccountsDB { + /// Opens or creates the database at `root`. + pub fn new(root: impl AsRef) -> Result { + let root = root.as_ref().to_owned(); + let path = Self::directory(&root); + let persisted = PersistedStore::new(&path)?; + let volatile = VolatileStore::new(&path)?; + info!(?path, "opened accountsdb"); + let db = Self { persisted, volatile, root }; + metrics::init(&db); + Ok(db) + } + + /// Returns the active database directory under `root`. + pub fn directory(root: &Path) -> PathBuf { + root.join(ACTIVE_DIR) + } + + /// Stores accounts in the backend that matches their current form. + /// + /// Persistent modes are kept in persisted storage. Other modes are kept in + /// volatile storage. Each batch also touches the opposite backend so stale + /// copies are removed after mode changes. Persisted failures roll back + /// borrowed images before the caller sees the error. + pub fn store<'a, AC>(&self, accounts: AC) -> Result<()> + where + AC: IntoIterator + Clone, + ::IntoIter: Clone, + { + let iter = accounts.clone().into_iter().filter(persisted); + self.persisted.upsert(iter)?; + + let iter = accounts.into_iter().filter(volatile); + self.volatile.upsert(iter); + + Ok(()) + } + + /// Commits one ledger transaction's account transitions. + /// + /// The transaction count advances only after every supplied transition is + /// stored successfully. Empty transitions count, including failed SVM + /// executions that reached the commit path without account writes. + pub fn commit<'a, AC>(&self, accounts: AC) -> Result<()> + where + AC: IntoIterator + Clone, + ::IntoIter: Clone, + { + self.store(accounts)?; + self.persisted.meta().transactions.fetch_add(1, Release); + Ok(()) + } + + /// Creates a loader that reuses a read transaction for persisted lookups. + pub fn loader(&self) -> AccountLoader<'_> { + AccountLoader::new(self) + } + + /// Iterates program-owned accounts across both backends. + pub fn program(&self, owner: &Pubkey) -> Result> { + let persisted = self.persisted.program(*owner)?; + let volatile = self.volatile.program(owner); + Ok(ProgramIter { persisted, volatile, db: self }) + } + + /// Returns the latest slot persisted in the database metadata. + pub fn slot(&self) -> Slot { + self.persisted.meta().slot.load(Acquire) + } + + /// Sets the database slot and flushes dirty pages asynchronously. + pub fn set_slot(&self, slot: Slot) -> Result<()> { + self.persisted.meta().slot.store(slot, Release); + self.flush(false) + } + + /// Returns the id of the last sealed superblock recorded in the database metadata. + pub fn superblock(&self) -> Slot { + self.persisted.meta().superblock.load(Acquire) + } + + /// Returns the number of successfully committed ledger transactions. + pub fn transactions(&self) -> u64 { + self.persisted.meta().transactions.load(Acquire) + } + + /// Records the last sealed superblock id. Set on snapshot, and on replay + /// before recomputing the checksum to compare against a seal. + pub fn set_superblock(&self, superblock: u64) { + self.persisted.meta().superblock.store(superblock, Release); + } + + /// Flushes persisted account storage, forcing synchronous durability when requested. + pub fn flush(&self, force: bool) -> Result<()> { + self.persisted.flush(force).map_err(Into::into) + } + + /// Validates the persisted store checksum and on-disk format version. + pub fn validate(&self) -> Result<()> { + self.persisted.validate() + } + + /// Compacts persisted storage to a non-overlapping packing fixed point. + /// + /// This must run only after validation and before loaders or iterators are + /// created. Vacated sources become eligible on the following pass, and all + /// successful passes are flushed synchronously before returning. + pub fn compact(&mut self) -> Result { + let mut reclaimed = 0; + let mut changed = false; + loop { + // SAFETY: `&mut self` excludes readers and writers through this handle; + // the store owns its LMDB environment and mapped storage. + let pass = unsafe { self.persisted.defragment() }?; + reclaimed += pass.reclaimed; + changed |= pass.changed(); + if !pass.changed() { + break; + } + } + if changed { + self.flush(true)?; + } + Ok(reclaimed) + } + + /// Returns the last checksum published on superblock boundary. + pub fn checksum(&self) -> u64 { + self.persisted.meta().checksum.load(Acquire) + } + + /// Drops chain-mirrored volatile state while retaining system accounts; + /// persisted state is left untouched. + /// + /// Chain-owned accounts can be fetched again when synchronization resumes. + /// System accounts hold internal runtime state and survive the reset; their + /// volatile owner indexes are rebuilt. Persisted, engine-authoritative state + /// is never reset. + pub fn reset(&self) { + self.volatile.reset(); + } +} + +/// Loader that caches a read transaction for persisted account lookups. +pub struct AccountLoader<'a> { + /// Cached read transaction for the persisted index. + txn: RefCell>>, + /// Database handle used for volatile and persisted lookups. + db: &'a AccountsDB, +} + +impl<'a> AccountLoader<'a> { + /// Creates a new loader bound to `db`. + pub fn new(db: &'a AccountsDB) -> Self { + Self { txn: Default::default(), db } + } + + /// Loads one account, reusing the persisted read transaction across calls. + /// + /// Reuse the loader for batch lookups to keep them on the same persisted + /// index snapshot. Persisted accounts take precedence over volatile ones. + pub fn load(&self, pubkey: &Pubkey) -> Result> { + let txn = &mut self.txn.borrow_mut(); + if let Some(acc) = self.db.persisted.load(txn, pubkey)? { + metrics::load(StoreKind::Persisted); + return Ok(Some(acc.into())); + } + let account = self.db.volatile.load(pubkey).map(Into::into); + if account.is_some() { + metrics::load(StoreKind::Volatile); + } else { + metrics::load(StoreKind::Absent); + } + Ok(account) + } + + /// Applies `reader` to an account image stable across a concurrent publish. + /// + /// Prefer this over [`Self::load`] when reading fields from persisted + /// accounts that may be updated concurrently. The reader may be called more + /// than once when the borrowed image changes, so it should have no side + /// effects. + pub fn read(&self, pubkey: &Pubkey, reader: F) -> Result> + where + F: Fn(&AccountSharedData) -> R, + { + let Some(account) = self.load(pubkey)? else { + return Ok(None); + }; + Ok(Some(AccountSeqLock::new(account).read(reader))) + } + + /// Returns whether an account exists in either backend. + pub fn contains(&self, pubkey: &Pubkey) -> Result { + let txn = &mut self.txn.borrow_mut(); + if self.db.persisted.contains(txn, pubkey)? { + return Ok(true); + } + let contains = self.db.volatile.contains(pubkey); + Ok(contains) + } +} + +/// Iterates program-owned accounts across both backends. +pub struct ProgramIter<'a> { + /// Persisted program accounts. + persisted: Option>, + /// Volatile program pubkeys. + volatile: BTreeSet, + /// Database handle used to resolve volatile accounts. + db: &'a AccountsDB, +} + +impl<'a> Iterator for ProgramIter<'a> { + type Item = AccountEntry; + /// Yields authoritative accounts first, then volatile ones. + fn next(&mut self) -> Option { + if let Some(persisted) = &mut self.persisted { + // Yield authoritative entries first. + if let Some(item) = persisted.next() { + return Some(item); + } + } + // Release the persisted read txn before draining volatile entries. + let _ = self.persisted.take(); + // Then drain the in-memory set of non-authoritative accounts. + while let Some(pubkey) = self.volatile.pop_first() { + if let Some(account) = self.db.volatile.load(&pubkey) { + return Some((pubkey, account.into())); + } + warn!(%pubkey, "volatile program set references a missing account; skipping"); + } + None + } +} + +/// Errors returned by accountsdb. +#[derive(Debug, thiserror::Error, From)] +pub enum AccountsDBError { + /// LMDB key-value codec error. + #[error("LMDB key/value codec error: {0}")] + Codec(#[source] heed::BoxedError), + /// Filesystem error. + #[error("filesystem I/O error: {0}")] + IO(#[source] std::io::Error), + /// LMDB index access error. + #[error("LMDB index error: {0}")] + Index(#[source] heed::Error), + /// Storage allocation would exceed the maximum mapped size. + #[error("mapped storage exceeded the 32 GiB limit")] + Allocation, + /// Opened database version is not supported by current implementation. + #[error("unsupported database version: {0:?}")] + UnsupportedVersion(DatabaseVersion), + /// Database was corrupted during the shutdown/crash. + #[error("database integrity check failed")] + Corruption, + /// Volatile snapshot serialization error. + #[error("volatile snapshot serialization error: {0}")] + Serde(#[source] Box), +} + +/// Result type used by the accountsdb crate. +type Result = std::result::Result; +/// Account key plus shared account payload. +pub type AccountEntry = (Pubkey, AccountSharedData); + +/// Classification used by accountsdb metrics. +#[derive(Clone, Copy)] +pub(crate) enum StoreKind { + /// Mmap-backed persisted storage. + Persisted, + /// In-memory volatile storage. + Volatile, + /// Account was absent from both storage backends. + Absent, +} + +impl StoreKind { + /// Returns the Prometheus label value for this classification. + pub(crate) fn label(self) -> &'static str { + match self { + StoreKind::Persisted => "persisted", + StoreKind::Volatile => "volatile", + StoreKind::Absent => "absent", + } + } +} + +/// Returns `true` for entries that must touch persisted storage. +fn persisted(entry: &&AccountEntry) -> bool { + match entry.1.cow() { + CoWAccount::Borrowed(_) => true, + CoWAccount::Owned(_) => entry.1.mode().authoritative(), + } +} + +/// Returns `true` for entries that must touch volatile storage. +fn volatile(entry: &&AccountEntry) -> bool { + match entry.1.cow() { + CoWAccount::Borrowed(_) => !entry.1.mode().authoritative(), + CoWAccount::Owned(_) => true, + } +} diff --git a/accountsdb/src/metrics.rs b/accountsdb/src/metrics.rs new file mode 100644 index 00000000..a1263c96 --- /dev/null +++ b/accountsdb/src/metrics.rs @@ -0,0 +1,199 @@ +//! Prometheus metrics for accountsdb. + +use std::sync::{OnceLock, atomic::Ordering::*}; + +use nucleus::metrics as metric; +use nucleus::metrics::{IntCounter, IntGaugeVec, MetricOperation, MetricSpec, OperationCounters}; + +use crate::{AccountsDB, StoreKind, store::Stats}; + +/// Process-wide accountsdb metrics registered in the default Prometheus registry. +static METRICS: OnceLock = OnceLock::new(); + +/// Persisted account image load counter. +const READS: MetricSpec = MetricSpec { + name: "accountsdb_persisted_reads", + help: "Persisted account image loads.", +}; +/// Borrowed account commit counter. +const COMMITS: MetricSpec = MetricSpec { + name: "accountsdb_persisted_commits", + help: "Borrowed account commits into persisted storage.", +}; +/// Fresh mapped-storage allocation counter. +const ALLOCS: MetricSpec = MetricSpec { + name: "accountsdb_persisted_allocs", + help: "Fresh allocations from the mapped persisted storage file.", +}; +/// Persisted freelist reuse counter. +const REALLOCS: MetricSpec = MetricSpec { + name: "accountsdb_persisted_reallocs", + help: "Allocations reused from the persisted freelist.", +}; +/// Defragmentation relocation counter. +const COMPACTIONS: MetricSpec = MetricSpec { + name: "accountsdb_persisted_compactions", + help: "Persisted account relocations during defragmentation.", +}; +/// Persisted account removal counter. +const REMOVALS: MetricSpec = MetricSpec { + name: "accountsdb_persisted_removals", + help: "Persisted account removals.", +}; + +/// Persisted storage resize counter. +const RESIZES: MetricSpec = MetricSpec { + name: "accountsdb_persisted_resizes", + help: "Persisted storage file resizes.", +}; +/// Account load counter grouped by source or absence. +const LOADS: MetricSpec = MetricSpec { + name: "accountsdb_loads", + help: "Account loads by source or absence.", +}; +/// Operation latency histogram recorded in microseconds. +const OPERATION_TIME: MetricSpec = MetricSpec { + name: "accountsdb_operation_duration_micros", + help: "Accountsdb operation duration distribution in microseconds.", +}; +/// Account count gauge grouped by backend store. +const ACCOUNTS: MetricSpec = MetricSpec { + name: "accountsdb_accounts", + help: "Current accountsdb account count by backend store.", +}; + +/// Label used to separate persisted and volatile account counts. +const STORE_LABEL: &str = "store"; + +/// Accountsdb operation used as a low-cardinality operation label. +#[derive(Clone, Copy)] +pub(crate) enum Operation { + /// Persisted store flush path. + Flush, + /// Persisted checksum path. + Checksum, + /// Accountsdb snapshot path. + Snapshot, + /// Volatile-state dump path. + Dump, + /// Persisted store defragmentation path. + Defragmentation, +} + +impl MetricOperation for Operation { + /// Returns the Prometheus label value for this operation. + fn label(self) -> &'static str { + match self { + Operation::Flush => "flush", + Operation::Checksum => "checksum", + Operation::Snapshot => "snapshot", + Operation::Dump => "dump", + Operation::Defragmentation => "defragmentation", + } + } +} + +/// Registers accountsdb metrics once, seeding durable counters from persisted stats. +pub(crate) fn init(db: &AccountsDB) { + METRICS.get_or_init(|| Metrics::new(db.persisted.storage.stats())); +} + +/// Records one persisted account image load. +pub(crate) fn read() { + metric::with_metrics(&METRICS, |m| m.reads.inc()); +} + +/// Records one borrowed account commit into persisted storage. +pub(crate) fn commit() { + metric::with_metrics(&METRICS, |m| m.commits.inc()); +} + +/// Records one fresh allocation from the mapped persisted storage file. +pub(crate) fn alloc() { + metric::with_metrics(&METRICS, |m| m.allocs.inc()); +} + +/// Records one allocation reuse from the persisted freelist. +pub(crate) fn realloc() { + metric::with_metrics(&METRICS, |m| m.reallocs.inc()); +} + +/// Records persisted account relocations during defragmentation. +pub(crate) fn compaction(count: u64) { + metric::with_metrics(&METRICS, |m| m.compactions.inc_by(count)); +} + +/// Records one persisted account removal. +pub(crate) fn removal() { + metric::with_metrics(&METRICS, |m| m.removals.inc()); +} + +/// Records one persisted storage file resize. +pub(crate) fn resize() { + metric::with_metrics(&METRICS, |m| m.resizes.inc()); +} + +/// Refreshes the current account count for `store`. +pub(crate) fn accounts(store: StoreKind, count: u64) { + metric::with_metrics(&METRICS, |m| { + m.accounts.with_label_values(&[store.label()]).set(metric::gauge_value(count)); + }); +} + +/// Starts an operation timer that records latency when the returned guard drops. +pub(crate) fn time(op: Operation) -> metric::OperationTimer<'static> { + op.time(METRICS.get().map(|m| &m.operations)) +} + +/// Records one account load satisfied by `store`. +pub(crate) fn load(store: StoreKind) { + metric::with_metrics(&METRICS, |m| m.loads[store as usize].inc()); +} + +/// Owns all Prometheus collectors registered by accountsdb. +struct Metrics { + /// Durable persisted account image load counter. + reads: IntCounter, + /// Durable borrowed account commit counter. + commits: IntCounter, + /// Durable fresh allocation counter. + allocs: IntCounter, + /// Durable freelist reuse counter. + reallocs: IntCounter, + /// Durable defragmentation relocation counter. + compactions: IntCounter, + /// Durable persisted account removal counter. + removals: IntCounter, + /// Durable persisted storage resize counter. + resizes: IntCounter, + /// Per-`StoreKind` load counters pre-resolved from `loads_vec`. + loads: [IntCounter; 3], + /// Runtime operation duration and completion counters. + operations: OperationCounters, + /// Runtime account count gauge labeled by backend store. + accounts: IntGaugeVec, +} + +impl Metrics { + /// Builds collectors and seeds durable counters from persisted mmap stats. + fn new(stats: &Stats) -> Self { + let loads_vec = metric::counter_vec(LOADS, &[STORE_LABEL]); + let loads = [ + loads_vec.with_label_values(&[StoreKind::Persisted.label()]), + loads_vec.with_label_values(&[StoreKind::Volatile.label()]), + loads_vec.with_label_values(&[StoreKind::Absent.label()]), + ]; + Self { + reads: metric::counter(READS, stats.reads.load(Relaxed)), + commits: metric::counter(COMMITS, stats.commits.load(Relaxed)), + allocs: metric::counter(ALLOCS, stats.allocs.load(Relaxed)), + reallocs: metric::counter(REALLOCS, stats.reallocs.load(Relaxed)), + compactions: metric::counter(COMPACTIONS, stats.compactions.load(Relaxed)), + removals: metric::counter(REMOVALS, stats.removals.load(Relaxed)), + resizes: metric::counter(RESIZES, stats.resizes.load(Relaxed)), + loads, + operations: OperationCounters::new(OPERATION_TIME), + accounts: metric::gauge_vec(ACCOUNTS, &[STORE_LABEL]), + } + } +} diff --git a/accountsdb/src/snapshot.rs b/accountsdb/src/snapshot.rs new file mode 100644 index 00000000..d5190008 --- /dev/null +++ b/accountsdb/src/snapshot.rs @@ -0,0 +1,116 @@ +//! Snapshot export helpers. + +use std::{ + fs::{self, File}, + io::{self, BufWriter}, + path::PathBuf, +}; + +use nucleus::MB; +use tracing::info; + +use crate::{ + ACTIVE_DIR, AccountsDB, + metrics::{self, Operation}, +}; + +/// Snapshot directory prefix. +const PREFIX: &str = "snapshot-"; +/// Snapshot payload filename for the volatile store. +pub(crate) const VOLATILE_DB_FILE: &str = "volatile.db"; + +/// Errors while writing a snapshot directory. +#[derive(thiserror::Error, Debug)] +pub enum SnapshotError { + /// I/O while writing the snapshot. + #[error("snapshot export I/O error")] + IO(#[from] io::Error), + /// Failed to flush the persisted store before copying the tree. + #[error("failed to flush persisted store")] + Flush(#[from] heed::Error), + /// Failed to serialize the volatile store into the snapshot. + #[error("failed to serialize volatile store")] + Serde(#[from] Box), + /// Failed to clone the active database tree into the snapshot slot. + #[error("failed to clone snapshot tree")] + FsClone(#[from] Box), + /// No archived snapshot could be restored. + #[error("no valid archived accountsdb snapshot found")] + Missing, +} + +/// Result type used by snapshot export and restore helpers. +pub type SnapshotResult = Result; + +/// Active database backup operation. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BackupOp { + /// Move the active database tree to its backup path. + Save, + /// Move the saved backup tree back to the active database path. + Restore, +} + +impl AccountsDB { + /// Writes a superblock snapshot under `root`. + /// + /// # Safety + /// The caller must ensure exclusive access while the snapshot is in + /// progress. The persisted backend runs one non-overlapping packing pass + /// and is flushed before the active tree is cloned and the volatile store + /// is rewritten in the clone. That ordering keeps the exported state + /// coherent only when no concurrent access can race with the export. + pub unsafe fn snapshot(&self, superblock: u64) -> SnapshotResult { + let _timer = metrics::time(Operation::Snapshot); + let src = self.root.join(ACTIVE_DIR); + let dst = self.root.join(format!("{PREFIX}{superblock:0>9}")); + self.set_superblock(superblock); + // SAFETY: snapshot owns exclusive access, so defrag cannot race with + // readers or writers while compacting the persisted store. + unsafe { self.persisted.defragment() }?; + // Persisted state must reach disk before we copy the active tree. + self.persisted.flush(true)?; + // Clone the whole active tree, then replace the volatile payload below. + clonetree::clone_tree(src, &dst, &Default::default()).map_err(Box::new)?; + self.dump(Some(&dst))?; + + Ok(dst) + } + + /// Serializes volatile accounts into `volatile.db` under `dst`. + /// + /// When `dst` is omitted, writes into the active database tree so the next + /// open restores the volatile store and consumes the file. Callers must + /// prevent concurrent account writes to obtain a coherent image. + pub fn dump(&self, dst: Option<&PathBuf>) -> SnapshotResult<()> { + let _timer = metrics::time(Operation::Dump); + let path = match dst { + Some(dst) => dst.join(VOLATILE_DB_FILE), + None => Self::directory(&self.root).join(VOLATILE_DB_FILE), + }; + let db = File::options().create(true).truncate(true).write(true).open(path)?; + let mut buffered = BufWriter::with_capacity(4 * MB, db); + bincode::serialize_into(&mut buffered, &self.volatile.accounts)?; + let db = buffered.into_inner().map_err(|e| e.into_error())?; + db.sync_data().map_err(Into::into) + } + + /// Saves or restores the active database tree and returns its destination. + /// + /// After restoring, callers must drop this instance and reopen the database: + /// its open handles still refer to the removed active tree. + pub fn backup(&self, op: BackupOp) -> SnapshotResult { + let active = self.root.join(ACTIVE_DIR); + let backup = self.root.join(format!("{ACTIVE_DIR}.bkp")); + let (from, to) = match op { + BackupOp::Save => (&active, &backup), + BackupOp::Restore => (&backup, &active), + }; + if to.exists() { + fs::remove_dir_all(to)?; + } + info!(?op, "accountsdb backup"); + fs::rename(from, to)?; + Ok(to.clone()) + } +} diff --git a/accountsdb/src/store/defrag.rs b/accountsdb/src/store/defrag.rs new file mode 100644 index 00000000..a4fecd81 --- /dev/null +++ b/accountsdb/src/store/defrag.rs @@ -0,0 +1,343 @@ +#![allow(unsafe_op_in_unsafe_fn)] + +use std::{collections::BTreeSet, ops::Range}; + +use heed::Result; +use solana_account::BorrowedAccount; +use tracing::info; + +use crate::{ + metrics::{self, Operation}, + store::kv::{Offset, OwnerAndOffset}, +}; + +use super::PersistedStore; + +/// Smallest useful destination remainder, in 8-byte storage units. +pub(crate) const MIN_REMAINDER: u32 = 43; +type Fit = (u32, Offset, usize); + +/// Result of one committed packing pass. +pub(crate) struct Defragged { + pub(crate) moved: usize, + pub(crate) reclaimed: u32, +} + +impl Defragged { + pub(crate) fn changed(&self) -> bool { + self.moved > 0 || self.reclaimed > 0 + } +} + +/// Free span in the persisted image file, measured in storage units. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Hole { + offset: Offset, + units: u32, +} + +impl Hole { + fn new((units, offset): (u32, Offset)) -> Self { + Self { offset, units } + } + + fn end(self) -> Offset { + self.offset + self.units + } +} + +/// Adjacent entry-time holes treated as one packing destination. +struct Run { + parts: Range, + free: Hole, +} + +impl Run { + fn take(&mut self, units: u32) -> Offset { + debug_assert!(units <= self.free.units); + let dst = self.free.offset; + self.free.offset = self.free.offset + units; + self.free.units -= units; + dst + } +} + +/// One account relocation planned against entry-time free space. +#[derive(Clone, Copy)] +struct Move { + src: Offset, + dst: Offset, + units: u32, +} + +impl Move { + fn source(self) -> Hole { + Hole { + offset: self.src, + units: self.units, + } + } +} + +/// Temporary state for one non-overlapping packing pass. +struct Defrag<'a> { + store: &'a PersistedStore, + holes: Vec, + runs: Vec, + moves: Vec, + tail: Offset, +} + +impl PersistedStore { + /// Packs tail accounts into holes that existed at the start of this pass. + /// + /// Adjacent freelist entries form logical runs. An account uses an exact + /// fit when available, otherwise the smallest run that leaves at least + /// [`MIN_REMAINDER`] units. Destination remainders may accept more accounts + /// in this pass; vacated source spans are deferred until a later pass. Some + /// fragmented layouts therefore cannot progress. + /// + /// This operation is not crash-safe: interruption after publishing moved + /// offsets can leave the active tree inconsistent and require a backup. + /// + /// # Safety + /// + /// No concurrent access may touch the persisted index or mapped storage + /// while offsets are rewritten and bytes are moved. + pub(crate) unsafe fn defragment(&self) -> Result { + let _timer = metrics::time(Operation::Defragmentation); + Defrag::new(self)?.execute() + } +} + +impl<'a> Defrag<'a> { + /// Reads a consistent entry-time layout and plans tail-to-left moves. + /// + /// # Safety + /// + /// The store must be exclusively accessed, and indexed offsets must point + /// to valid serialized accounts in its mapped storage. + unsafe fn new(store: &'a PersistedStore) -> Result { + let (mut holes, mut accounts) = { + let txn = store.index.env.read_txn()?; + let holes = store + .index + .freelist + .iter(&txn)? + .map(|r| r.map(Hole::new)) + .collect::>>()?; + let accounts = if holes.is_empty() { + Vec::new() + } else { + store + .index + .accounts + .iter(&txn)? + .map(|r| r.map(|(_, data)| data.offset)) + .collect::>>()? + }; + (holes, accounts) + }; + holes.sort_unstable(); + accounts.sort_unstable(); + + let runs = Self::runs(&holes); + let mut defrag = Self { + store, + holes, + runs, + moves: Vec::new(), + tail: Offset(store.storage.cursor()), + }; + defrag.pack(accounts.into_iter().rev()); + Ok(defrag) + } + + /// Groups physically adjacent holes without changing their freelist shape. + fn runs(holes: &[Hole]) -> Vec { + let mut runs = Vec::new(); + let mut i = 0; + while i < holes.len() { + let first = i; + let offset = holes[i].offset; + let mut end = holes[i].end(); + i += 1; + while let Some(hole) = holes.get(i) + && hole.offset == end + { + end = hole.end(); + i += 1; + } + runs.push(Run { + parts: first..i, + free: Hole { offset, units: end - offset }, + }); + } + runs + } + + /// Selects the best exact fit or the best fit with a useful remainder. + fn fit(fit: &BTreeSet, units: u32) -> Option { + let &(largest, _, _) = fit.last()?; + if units > largest { + return None; + } + + let low = (units, Offset(0), 0); + let high = (units, Offset(u32::MAX), usize::MAX); + if let Some(exact) = fit.range(low..=high).next() { + return Some(*exact); + } + + let minimum = units.checked_add(MIN_REMAINDER)?; + if minimum > largest { + return None; + } + fit.range((minimum, Offset(0), 0)..).next().copied() + } + + /// Packs accounts in descending source order into eligible runs. + /// + /// # Safety + /// + /// Every supplied offset must point to a valid serialized account, and no + /// concurrent access may modify the index, freelist, or mapped storage. + unsafe fn pack(&mut self, accounts: impl Iterator) { + // Best fit by remaining units, then by the lowest current offset. + let mut fit: BTreeSet = self + .runs + .iter() + .enumerate() + .map(|(i, run)| (run.free.units, run.free.offset, i)) + .collect(); + let mut eligible = self.runs.len(); + + for src in accounts { + // Runs are already ordered by their physical end. + while eligible > 0 && self.runs[eligible - 1].free.end() > src { + let i = eligible - 1; + fit.remove(&(self.runs[i].free.units, self.runs[i].free.offset, i)); + eligible -= 1; + } + if fit.is_empty() { + break; + } + + let units = BorrowedAccount::span(self.store.storage.at(src)); + let Some((remaining, start, i)) = Self::fit(&fit, units) else { + continue; + }; + fit.remove(&(remaining, start, i)); + let dst = self.runs[i].take(units); + self.moves.push(Move { src, dst, units }); + let free = self.runs[i].free; + if free.units > 0 { + fit.insert((free.units, free.offset, i)); + } + } + } + + /// Returns the first unit in the final free suffix without re-sorting it. + fn compacted_tail(&self) -> Offset { + let mut run = self.runs.len(); + let mut movement = 0; + let mut tail = self.tail; + + loop { + while run > 0 && self.runs[run - 1].free.units == 0 { + run -= 1; + } + let free = (run > 0).then(|| self.runs[run - 1].free); + let source = self.moves.get(movement).copied().map(Move::source); + let (hole, from_run) = match (free, source) { + (Some(free), Some(source)) => (free.max(source), free.offset >= source.offset), + (Some(free), None) => (free, true), + (None, Some(source)) => (source, false), + (None, None) => break, + }; + if hole.end() != tail { + break; + } + tail = hole.offset; + if from_run { + run -= 1; + } else { + movement += 1; + } + } + tail + } + + /// Copies the plan and publishes all index and freelist changes. + /// + /// # Safety + /// + /// The entry-time layout must remain unchanged since planning, and no + /// concurrent access may observe or modify storage while moves publish. + unsafe fn execute(self) -> Result { + let tail = self.compacted_tail(); + let outcome = Defragged { + moved: self.moves.len(), + reclaimed: self.tail - tail, + }; + if !outcome.changed() { + info!("nothing to defragment"); + return Ok(outcome); + } + + // Entry-time destinations are disjoint, so every source remains intact + // until the complete plan has been copied. + for movement in &self.moves { + self.store.storage.at(movement.src).copy_to_nonoverlapping( + self.store.storage.at(movement.dst), + movement.units as usize, + ); + } + + let mut txn = self.store.index.env.write_txn()?; + for movement in &self.moves { + let ptr = self.store.storage.at(movement.src); + let pubkey = BorrowedAccount::pubkey(ptr); + let owner = BorrowedAccount::init(ptr).owner().into(); + let data = OwnerAndOffset { owner, offset: movement.dst }; + self.store.index.relocate(&pubkey, movement.src, data, &mut txn)?; + } + self.publish(tail, &mut txn)?; + txn.commit()?; + + self.store.storage.stats().compact(outcome.moved); + if outcome.reclaimed > 0 { + self.store.storage.shrink(tail.0)?; + } + info!( + moved = outcome.moved, + reclaimed = outcome.reclaimed, + "defragmented persisted storage" + ); + Ok(outcome) + } + + /// Publishes final free spans while retaining untouched component sizes. + fn publish(&self, tail: Offset, txn: &mut heed::RwTxn<'_>) -> Result<()> { + for run in &self.runs { + for &hole in &self.holes[run.parts.clone()] { + let offset = hole.offset.max(run.free.offset); + let end = hole.end().min(tail); + if offset == hole.offset && end == hole.end() { + continue; + } + self.store.index.freelist.delete_one_duplicate(txn, &hole.units, &hole.offset)?; + if offset < end { + self.store.index.freelist.put(txn, &(end - offset), &offset)?; + } + } + } + for movement in &self.moves { + if movement.src < tail { + let end = movement.source().end().min(tail); + self.store.index.freelist.put(txn, &(end - movement.src), &movement.src)?; + } + } + Ok(()) + } +} diff --git a/accountsdb/src/store/index.rs b/accountsdb/src/store/index.rs new file mode 100644 index 00000000..7b3c95c2 --- /dev/null +++ b/accountsdb/src/store/index.rs @@ -0,0 +1,207 @@ +//! LMDB index for persisted accounts. +//! +//! The index maps compact pubkey tags to storage offsets and owner tags, +//! plus a freelist keyed by image size. + +use std::{fs, mem, path::Path}; + +use heed::{ + Database, DatabaseFlags, Env, EnvFlags, EnvOpenOptions, IntegerComparator, Result, RoIter, + RoTxn, RwTxn, iteration_method::MoveOnCurrentKeyDuplicates, +}; +use nucleus::heed::{DatabaseIndex, RoTxnTls}; +use solana_pubkey::Pubkey; + +use crate::store::kv::{KeyTail, Offset, OwnerAndOffset, PubkeyBytes, U32LE}; + +/// LMDB map size for the index database. +#[cfg(feature = "testkit")] +const INDEX_MAP_SIZE: usize = nucleus::MB; +#[cfg(not(feature = "testkit"))] +const INDEX_MAP_SIZE: usize = nucleus::GB; +/// Subdirectory used for the LMDB index. +const INDEX_SUBDIR: &str = "index"; +/// Accounts table name. +const ACCOUNTS_INDEX: &str = "accounts"; +/// Program ownership table name. +const PROGRAMS_INDEX: &str = "programs"; +/// Freelist table name. +const FREELIST_INDEX: &str = "freelist"; + +/// Iterator over all persisted accounts in pubkey order. +type RoAccountIter<'a> = RoIter<'a, PubkeyBytes, OwnerAndOffset>; +/// Duplicate iterator over program-owned persisted accounts. +type RoProgramIter<'a> = RoIter<'a, KeyTail, Offset, MoveOnCurrentKeyDuplicates>; +/// Iterator over persisted accounts. +pub(crate) struct AccountIter<'a> { + /// Iterator over `pubkey -> account` entries. + pub(super) inner: RoAccountIter<'a>, + /// Keeps the read transaction alive for the iterator lifetime. + pub(super) _txn: RoTxnTls<'a>, +} +/// Duplicate iterator over persisted accounts for one owner. +pub(crate) struct OwnerIter<'a> { + /// Duplicates iterator over `owner -> account` entries. + pub(crate) inner: RoProgramIter<'a>, + /// Keeps the read transaction alive for the iterator lifetime. + pub(crate) _txn: RoTxnTls<'a>, +} + +/// LMDB index over persisted account offsets and owners. +pub(crate) struct Index { + /// LMDB environment for the on-disk index. + pub(super) env: Env, + /// Account pubkey -> offset + owner keytag. + pub(super) accounts: Database, + /// Owner keytag -> offset. + pub(super) programs: Database, + /// Image size -> offset. + pub(super) freelist: Database, +} + +impl Index { + /// Opens or creates the index directory and databases. + pub(crate) fn new(path: &Path) -> crate::Result { + let path = path.join(INDEX_SUBDIR); + fs::create_dir_all(&path)?; + // SAFETY: this process owns the index directory for the lifetime of + // the database, so the backing files are not mutated behind LMDB's back. + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(3) + .map_size(INDEX_MAP_SIZE) + .flags(EnvFlags::WRITE_MAP) + .flags(EnvFlags::NO_READ_AHEAD) + .flags(EnvFlags::NO_SYNC) + .open(path)? + }; + + let mut txn = env.write_txn()?; + let accounts = env.database_options().name(ACCOUNTS_INDEX).types().create(&mut txn)?; + let programs = env + .database_options() + .name(PROGRAMS_INDEX) + .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) + .types() + .create(&mut txn)?; + let freelist = env + .database_options() + .name(FREELIST_INDEX) + .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) + .key_comparator() + .types() + .create(&mut txn)?; + txn.commit()?; + Ok(Self { + env, + accounts, + programs, + freelist, + }) + } + + /// Returns the persisted offset for `pubkey`. + pub(crate) fn offset(&self, key: &Pubkey, txn: &RoTxn<'_>) -> Result> { + let entry = self.accounts.get(txn, key)?; + Ok(entry.map(|e| e.offset)) + } + + /// Takes a freed span from the freelist when one matches `units`. + pub(crate) fn allocate(&self, units: u32, txn: &mut RwTxn<'_>) -> Result> { + let offset = self.freelist.get(txn, &units)?; + if let Some(offset) = offset { + self.freelist.delete_one_duplicate(txn, &units, &offset)?; + Ok(Some(offset)) + } else { + Ok(None) + } + } + + /// Inserts an account and its owner mapping. + pub(crate) fn insert( + &self, + key: &Pubkey, + data: OwnerAndOffset, + txn: &mut RwTxn<'_>, + ) -> Result<()> { + self.accounts.put(txn, key, &data)?; + let OwnerAndOffset { owner, offset } = data; + self.programs.put(txn, &owner, &offset) + } + + /// Removes an account and returns its persisted offset. + pub(crate) fn delete(&self, key: &Pubkey, txn: &mut RwTxn<'_>) -> Result> { + let Some(entry) = self.accounts.get(txn, key)? else { + return Ok(None); + }; + + let OwnerAndOffset { owner, offset } = entry; + self.accounts.delete(txn, key)?; + + self.programs.delete_one_duplicate(txn, &owner, &offset)?; + Ok(Some(offset)) + } + + /// Returns the duplicate iterator for accounts owned by `owner`. + pub(crate) fn program<'a>(&'a self, owner: Pubkey) -> Result>> { + let owner = owner.into(); + let txn = self.env.read_txn()?; + let Some(iter) = self.programs.get_duplicates(&txn, &owner)? else { + return Ok(None); + }; + // The duplicate iterator borrows `txn`; storing it in the wrapper keeps + // the borrow alive for the iterator lifetime. + // SAFETY: the wrapper owns `txn`, so the duplicate iterator cannot outlive it. + let iter = unsafe { mem::transmute::, RoProgramIter<'a>>(iter) }; + Ok(Some(OwnerIter { _txn: txn, inner: iter })) + } + + /// Returns an iterator over all accounts in pubkey order. + pub(crate) fn accounts<'a>(&'a self) -> Result> { + let txn = self.env.read_txn()?; + let iter = self.accounts.iter(&txn)?; + // The iterator borrows `txn`; storing it in the wrapper keeps the + // transaction alive for the iterator lifetime. + // SAFETY: the wrapper owns `txn`, so the iterator cannot outlive it. + let iter = unsafe { mem::transmute::, RoAccountIter<'a>>(iter) }; + Ok(AccountIter { _txn: txn, inner: iter }) + } + + /// Moves an account entry to a new owner while preserving its offset. + pub(crate) fn update_owner( + &self, + acc: &Pubkey, + new: KeyTail, + txn: &mut RwTxn<'_>, + ) -> Result<()> { + let Some(val) = self.accounts.get(txn, acc)? else { + return Ok(()); + }; + let OwnerAndOffset { owner: old, offset } = val; + self.programs.delete_one_duplicate(txn, &old, &offset)?; + + let data = OwnerAndOffset { owner: new, offset }; + self.accounts.put(txn, acc, &data)?; + self.programs.put(txn, &new, &offset) + } + + /// Moves an account entry to a new offset while preserving its owner. + pub(crate) fn relocate( + &self, + key: &Pubkey, + old: Offset, + new: OwnerAndOffset, + txn: &mut RwTxn<'_>, + ) -> Result<()> { + self.accounts.put(txn, key, &new)?; + let OwnerAndOffset { owner, offset } = new; + self.programs.delete_one_duplicate(txn, &owner, &old)?; + self.programs.put(txn, &owner, &offset) + } +} + +impl DatabaseIndex for Index { + fn env(&self) -> &Env { + &self.env + } +} diff --git a/accountsdb/src/store/kv.rs b/accountsdb/src/store/kv.rs new file mode 100644 index 00000000..75b26a1e --- /dev/null +++ b/accountsdb/src/store/kv.rs @@ -0,0 +1,123 @@ +use std::{array, borrow::Cow, ops}; + +use bytemuck::{Pod, Zeroable}; +use heed::{BoxedError, BytesDecode, BytesEncode, byteorder::LittleEndian, types::U32}; +use solana_pubkey::Pubkey; + +/// Result type used by LMDB byte codecs. +pub(crate) type CodecResult = Result; +/// Little-endian `u32` value stored in the freelist. +pub(super) type U32LE = U32; +/// Offset into mapped storage, measured in storage units. +#[derive(Clone, Copy, Zeroable, Pod, PartialEq, Eq, PartialOrd, Ord)] +#[repr(C)] +pub(crate) struct Offset(pub(super) u32); + +/// Compact 16-byte LMDB tag derived from the tail half of a pubkey. +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub(crate) struct KeyTail([u8; 16]); + +impl From for KeyTail { + fn from(v: Pubkey) -> Self { + Self(array::from_fn(|i| v.as_array()[i + size_of::()])) + } +} + +/// Full 32-byte pubkey codec for the accounts table. +pub(super) struct PubkeyBytes; + +/// LMDB value for the accounts table. +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub(crate) struct OwnerAndOffset { + /// Owner key tag for the stored account image. + pub(crate) owner: KeyTail, + /// Offset into mapped storage. + pub(crate) offset: Offset, +} + +impl<'a> BytesEncode<'a> for KeyTail { + type EItem = Self; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + Ok(bytemuck::bytes_of(item).into()) + } +} + +impl<'a> BytesDecode<'a> for KeyTail { + type DItem = &'a Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + bytemuck::try_from_bytes(bytes).map_err(Into::into) + } +} + +impl<'a> BytesEncode<'a> for PubkeyBytes { + type EItem = Pubkey; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + Ok(item.as_array().into()) + } +} + +impl<'a> BytesDecode<'a> for PubkeyBytes { + type DItem = &'a Pubkey; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + bytemuck::try_from_bytes(bytes).map_err(Into::into) + } +} + +impl<'a> BytesEncode<'a> for OwnerAndOffset { + type EItem = Self; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + Ok(bytemuck::bytes_of(item).into()) + } +} + +impl<'a> BytesDecode<'a> for OwnerAndOffset { + type DItem = Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + bytemuck::try_pod_read_unaligned(bytes).map_err(Into::into) + } +} + +impl<'a> BytesEncode<'a> for Offset { + type EItem = Self; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + U32LE::bytes_encode(&item.0) + } +} + +impl<'a> BytesDecode<'a> for Offset { + type DItem = Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + U32LE::bytes_decode(bytes).map(Self) + } +} + +impl ops::Add for Offset { + type Output = Self; + fn add(self, rhs: u32) -> Self::Output { + Self(self.0 + rhs) + } +} + +impl ops::Sub for Offset { + type Output = Self; + fn sub(self, rhs: u32) -> Self::Output { + Self(self.0 - rhs) + } +} + +impl ops::Sub for Offset { + type Output = u32; + fn sub(self, rhs: Self) -> Self::Output { + self.0 - rhs.0 + } +} diff --git a/accountsdb/src/store/mmap.rs b/accountsdb/src/store/mmap.rs new file mode 100644 index 00000000..26ce556b --- /dev/null +++ b/accountsdb/src/store/mmap.rs @@ -0,0 +1,306 @@ +//! Mapped storage for persisted account images. +//! +//! The file reserves a small meta header at the front, followed by the raw +//! account images written in `solana-account`'s borrowed layout. + +use std::{ + fs::File, + io::{self, Write}, + ops::Range, + os::fd::AsRawFd, + path::Path, + ptr::NonNull, + sync::atomic::{AtomicU32, AtomicU64, Ordering::*}, +}; + +use memmap2::{MmapMut, MmapOptions}; +use nucleus::MB; +use parking_lot::Mutex; +use solana_account::{STORAGE_UNIT, StorageUnit}; +use tracing::{debug, error}; + +use crate::{ + AccountsDBError, Result, metrics, + store::{DatabaseVersion, VERSION, kv::Offset}, +}; + +/// Bytes reserved at the front of the mapped file for metadata. +const DATABASE_META_RESERVATION: usize = 256; +/// Filename used for the mapped storage file. +pub const STORAGE_FILE: &str = "storage.db"; +/// Growth block for the mapped storage file. +#[cfg(feature = "testkit")] +pub(crate) const STORAGE_BLOCK: u64 = 16 * MB as u64; +#[cfg(not(feature = "testkit"))] +pub(crate) const STORAGE_BLOCK: u64 = 256 * MB as u64; +/// Initial file size: one storage block plus the metadata reservation. +const INIT_STORAGE_SIZE: u64 = STORAGE_BLOCK + DATABASE_META_RESERVATION as u64; +/// Maximum mapped storage size. +#[cfg(feature = "testkit")] +const MMAP_SIZE: usize = 64 * MB; +#[cfg(not(feature = "testkit"))] +const MMAP_SIZE: usize = u32::MAX as usize * STORAGE_UNIT + DATABASE_META_RESERVATION; + +/// One allocation inside the mapped storage. +pub(crate) struct Allocation { + /// Offset from the start of the storage area, in storage units. + pub(crate) offset: Offset, + /// Pointer to the start of the allocated image. + pub(crate) ptr: NonNull, +} + +/// Mapped storage backing persisted account images. +pub(crate) struct MappedStorage { + /// Pointer to the reserved metadata header. + meta: NonNull, + /// Full file mapping. + mmap: MmapMut, + /// Start of the account image region. + head: NonNull, + /// File handle used for resizing. + file: Mutex, +} + +#[repr(C)] +#[derive(Default)] +/// Runtime counters for the persisted backend. +pub(crate) struct Stats { + /// Persisted image loads. + pub(crate) reads: AtomicU64, + /// `BorrowedAccount::commit` calls. + pub(crate) commits: AtomicU64, + /// Fresh allocations on backing storage. + pub(crate) allocs: AtomicU64, + /// Freelist allocation reuse. + pub(crate) reallocs: AtomicU64, + /// Account relocations during defrag. + pub(crate) compactions: AtomicU64, + /// Persisted deletes. + pub(crate) removals: AtomicU64, + /// File resizes. + pub(crate) resizes: AtomicU64, +} + +impl Stats { + /// Counts one persisted read. + pub(crate) fn read(&self) { + self.reads.fetch_add(1, Relaxed); + metrics::read(); + } + + /// Counts one borrowed account commit. + pub(crate) fn commit(&self) { + self.commits.fetch_add(1, Relaxed); + metrics::commit(); + } + + /// Counts one fresh allocation from the mapped file. + pub(crate) fn alloc(&self) { + self.allocs.fetch_add(1, Relaxed); + metrics::alloc(); + } + + /// Counts one freelist reuse. + pub(crate) fn realloc(&self) { + self.reallocs.fetch_add(1, Relaxed); + metrics::realloc(); + } + + /// Counts relocations during defragmentation. + pub(crate) fn compact(&self, count: usize) { + let count = count as u64; + self.compactions.fetch_add(count, Relaxed); + metrics::compaction(count); + } + + /// Counts one persisted removal. + pub(crate) fn remove(&self) { + self.removals.fetch_add(1, Relaxed); + metrics::removal(); + } + + /// Counts one file resize. + pub(crate) fn resize(&self) { + self.resizes.fetch_add(1, Relaxed); + metrics::resize(); + } +} + +#[repr(C)] +#[derive(Default)] +/// Metadata header stored at the front of the mapped file. +pub(crate) struct DatabaseMeta { + /// On-disk format version. + version: DatabaseVersion, + /// Last computed database checksum. + pub(crate) checksum: AtomicU64, + /// Current slot. + pub(crate) slot: AtomicU64, + /// Id of the last sealed superblock; folded into the checksum fingerprint. + pub(crate) superblock: AtomicU64, + /// Transactions whose account-state commit completed successfully. + pub(crate) transactions: AtomicU64, + /// Current backing file length in bytes. + len: AtomicU64, + /// Database statistics. + stats: Stats, + /// Next allocation cursor. + pub(super) cursor: AtomicU32, +} + +impl MappedStorage { + /// Opens or creates the mapped storage file. + pub(crate) fn new(path: &Path) -> Result { + let path = path.join(STORAGE_FILE); + let mut file = + File::options().create(true).truncate(false).read(true).write(true).open(path)?; + let fd = file.as_raw_fd(); + // SAFETY: the file is opened read/write and mapped for the full fixed size. + let mut mmap = unsafe { MmapOptions::new().len(MMAP_SIZE).map_mut(fd)? }; + if file.metadata()?.len() == 0 { + file.set_len(INIT_STORAGE_SIZE)?; + file.flush()?; + let meta = DatabaseMeta { + version: VERSION, + len: INIT_STORAGE_SIZE.into(), + slot: 1.into(), + ..Default::default() + }; + // SAFETY: the first bytes of the mapping are reserved for `DatabaseMeta`. + unsafe { mmap.as_mut_ptr().cast::().write(meta) }; + mmap.flush()?; + } + // SAFETY: the mapping is at least `DATABASE_META_RESERVATION` bytes long, + // so the meta header and account head pointers stay within the map. + let (meta, head) = unsafe { + let head = mmap.as_mut_ptr().add(DATABASE_META_RESERVATION); + let head = NonNull::new_unchecked(head.cast()); + let meta = NonNull::new_unchecked(mmap.as_mut_ptr().cast()); + (meta, head) + }; + let file = Mutex::new(file); + Ok(Self { meta, mmap, head, file }) + } + + /// Flushes dirty pages to durable storage. + pub(crate) fn flush(&self, sync: bool) -> io::Result<()> { + let range = self.active(); + if sync { + self.mmap.flush_range(range.start, range.len()) + } else { + self.mmap.flush_async_range(range.start, range.len()) + } + } + + /// Validates the opened storage format. + pub(crate) fn validate(&self) -> Result<()> { + let meta = self.meta(); + if meta.version != VERSION { + Err(AccountsDBError::UnsupportedVersion(meta.version)) + } else { + Ok(()) + } + } + + /// Returns a pointer inside the account image region. + pub(crate) fn at(&self, offset: Offset) -> NonNull { + // SAFETY: private call sites pass offsets from the index, allocator, or + // defrag cursor and uphold the mapped-region bounds. + unsafe { self.head.add(offset.0 as usize) } + } + + /// Returns the runtime counters. + pub(crate) fn stats(&self) -> &Stats { + &self.meta().stats + } + + /// Allocates a fresh span of `units` storage units. + pub(crate) fn allocate(&self, units: u32) -> Result { + let meta = self.meta(); + let mut offset = meta.cursor.load(Acquire); + loop { + let end = offset.checked_add(units).ok_or(AccountsDBError::Allocation)?; + let needed = Self::bytes(end as u64); + if needed > meta.len.load(Acquire) { + self.grow(needed)?; + } + if let Err(updated) = meta.cursor.compare_exchange(offset, end, AcqRel, Acquire) { + offset = updated; + } else { + break; + } + } + self.stats().alloc(); + let offset = Offset(offset); + let ptr = self.at(offset); + Ok(Allocation { offset, ptr }) + } + + /// Returns a shared reference to the metadata header. + pub(crate) fn meta(&self) -> &DatabaseMeta { + // SAFETY: `meta` points to the reserved header at the front of the map. + unsafe { &*self.meta.as_ptr() } + } + + /// Returns the current allocation cursor in storage units. + pub(crate) fn cursor(&self) -> u32 { + self.meta().cursor.load(Acquire) + } + + /// Shrinks the file to the current cursor. + pub(super) fn shrink(&self, units: u32) -> io::Result<()> { + self.resize(Self::bytes(units as u64), u64::le)?; + self.meta().cursor.store(units, Release); + Ok(()) + } + + /// Returns the active byte range, including the metadata reservation. + fn active(&self) -> Range { + 0..Self::bytes(self.cursor() as u64) as usize + } + + /// Converts storage units into file bytes, including the metadata reservation. + fn bytes(units: u64) -> u64 { + units * STORAGE_UNIT as u64 + DATABASE_META_RESERVATION as u64 + } + + /// Grows the file to at least `len` bytes. + /// Rounds up to a storage block before resizing. + fn grow(&self, mut len: u64) -> Result<()> { + len = len.div_ceil(STORAGE_BLOCK) * STORAGE_BLOCK; + if len > MMAP_SIZE as u64 { + error!( + requested = len, + limit = MMAP_SIZE, + "mapped storage limit exceeded" + ); + return Err(AccountsDBError::Allocation); + } + self.resize(len, u64::ge).map_err(Into::into) + } + + /// Resizes the file when the current size does not satisfy `cmp`. + /// + /// The file is updated before the new size is published into metadata so + /// readers never observe a larger size than the actual mapping. + fn resize(&self, len: u64, cmp: fn(&u64, &u64) -> bool) -> io::Result<()> { + let mut file = self.file.lock(); + if cmp(&file.metadata()?.len(), &len) { + return Ok(()); + } + // Resize the file first, then publish the new size into metadata. + file.set_len(len)?; + file.flush()?; + self.meta().len.store(len, Release); + self.stats().resize(); + self.mmap.flush()?; + debug!(len, "resized storage file"); + Ok(()) + } +} + +// SAFETY: the `NonNull` pointers point into the owned `mmap` and are never +// reseated; concurrent access is synchronized through atomics in the metadata +// header and the `Mutex`, so the storage is safe to send and share. +unsafe impl Send for MappedStorage {} +unsafe impl Sync for MappedStorage {} diff --git a/accountsdb/src/store/mod.rs b/accountsdb/src/store/mod.rs new file mode 100644 index 00000000..9214362c --- /dev/null +++ b/accountsdb/src/store/mod.rs @@ -0,0 +1,303 @@ +//! Persisted account load and write path. +//! +//! This module coordinates the mmap, LMDB index, and borrowed account layout. + +use core::{hash::Hasher, slice}; +use std::sync::atomic::Ordering::{Acquire, Release}; + +use solana_account::{ + AccountMode, AccountSharedData, BorrowedAccount, CoWAccount::*, DirtyMarkers, OwnedAccount, +}; +use solana_pubkey::Pubkey; +use tracing::{error, warn}; + +use nucleus::heed::{DatabaseIndex, OptRoTxn, OptRwTxn, read_txn, write_txn}; +use twox_hash::XxHash3_64; + +use crate::{ + AccountEntry, AccountsDBError, Result, StoreKind, + metrics::{self, Operation}, + store::{ + index::{Index, OwnerIter}, + kv::{Offset, OwnerAndOffset}, + mmap::{DatabaseMeta, MappedStorage}, + }, +}; + +mod defrag; +pub(crate) mod index; +mod kv; +pub(crate) mod mmap; + +#[cfg(test)] +pub(crate) use defrag::MIN_REMAINDER; +pub(crate) use mmap::Stats; + +/// Current on-disk storage format version. +pub(crate) const VERSION: DatabaseVersion = 1; +/// Version tag stored in the metadata header. +pub(crate) type DatabaseVersion = u64; + +/// Persisted store backed by the mmap and LMDB index. +pub(crate) struct PersistedStore { + /// Mapped account storage. + pub(crate) storage: MappedStorage, + /// LMDB index over persisted accounts. + pub(crate) index: Index, +} + +/// Iterator over persisted program-owned accounts. +pub(crate) struct PersistedProgramIter<'a> { + /// Keeps the read transaction alive while iterating. + iter: OwnerIter<'a>, + /// Mapped storage backing the returned borrowed accounts. + mmap: &'a MappedStorage, +} + +impl PersistedStore { + /// Opens or creates the persisted store at `path`. + pub(crate) fn new(path: &std::path::Path) -> Result { + let index = Index::new(path)?; + let storage = MappedStorage::new(path)?; + Ok(Self { storage, index }) + } + + /// Loads the persisted image for `pubkey` from the mapped file. + pub(crate) fn load<'e>( + &'e self, + txn: OptRoTxn<'_, 'e>, + pubkey: &Pubkey, + ) -> Result> { + let txn = read_txn(self.index.env(), txn)?; + let offset = self.index.offset(pubkey, txn)?; + offset.is_some().then(|| self.storage.stats().read()); + // SAFETY: offsets come from the persisted index and point into the map. + Ok(offset.map(|o| unsafe { BorrowedAccount::init(self.storage.at(o)) })) + } + + /// Returns whether a persisted account image exists for `pubkey`. + pub(crate) fn contains<'e>(&'e self, txn: OptRoTxn<'_, 'e>, pubkey: &Pubkey) -> Result { + let txn = read_txn(self.index.env(), txn)?; + self.index.offset(pubkey, txn).map(|o| o.is_some()).map_err(Into::into) + } + + /// Applies a batch of account updates to the persisted store. + /// + /// Borrowed accounts in authoritative modes are committed in place. Owned + /// accounts in those modes are serialized into the mmap. Other modes delete + /// stale persisted entries. If the LMDB commit fails or database runs out of + /// space, the borrowed images are rolled back so in-memory state stays + /// aligned with the durable index. + pub(crate) fn upsert<'a, AC>(&self, accounts: AC) -> Result<()> + where + AC: IntoIterator + Clone, + { + let mut applied = 0; + let mut result = Ok(()); + let mut txn = None; + for entry in accounts.clone() { + result = self.apply(entry, &mut txn); + if result.is_err() { + break; + } + applied += 1; + } + // Commit once after the batch so the index and mmap stay in sync. + if let Some(txn) = txn + && result.is_ok() + { + metrics::accounts(StoreKind::Persisted, self.index.accounts.len(&txn)?); + result = txn.commit().map_err(Into::into); + } + if let Err(error) = &result { + warn!(applied, ?error, "accounts persistence failed; rolling back"); + // Only borrowed accounts need rollback here: owned inserts never + // mutate an existing borrowed image in place. + let processed = accounts.into_iter().take(applied).map(|(_, a)| a); + Self::rollback(processed); + } + + result + } + + /// Returns the persisted program iterator for `owner`. + pub(crate) fn program(&self, owner: Pubkey) -> Result>> { + let i = self.index.program(owner)?; + Ok(i.map(|iter| PersistedProgramIter { iter, mmap: &self.storage })) + } + + /// Flushes the mapped storage and LMDB index to durable storage. + pub(crate) fn flush(&self, sync: bool) -> heed::Result<()> { + let _timer = metrics::time(Operation::Flush); + self.index.flush()?; + if sync { + let checksum = self.checksum()?; + self.meta().checksum.store(checksum, Release); + } + self.storage.flush(sync)?; + Ok(()) + } + + /// Validates the persisted store checksum and on-disk format version. + pub(crate) fn validate(&self) -> Result<()> { + self.storage.validate()?; + if self.storage.cursor() == 0 { + return Ok(()); + } + let expected = self.meta().checksum.load(Acquire); + let actual = self.checksum()?; + if expected != actual { + error!(expected, actual, "state checksum mismatch"); + return Err(AccountsDBError::Corruption); + } + + Ok(()) + } + + /// Applies one account state transition to the persisted backend. + fn apply<'e>(&'e self, acc: &AccountEntry, txn: OptRwTxn<'_, 'e>) -> Result<()> { + let (pubkey, account) = acc; + // An account that has moved to a non-authoritative mode, or has been + // closed, no longer belongs here, so drop any stale persisted entry. + if !account.mode().authoritative() || account.is(AccountMode::Closed) { + self.delete(pubkey, txn)?; + if let Borrowed(acc) = account.cow() { + acc.commit(); + } + return Ok(()); + } + + let markers = account.markers(); + match account.cow() { + Borrowed(acc) => self.update(pubkey, acc, markers, txn), + Owned(acc) => self.insert(pubkey, acc, txn), + } + } + + /// Rolls back borrowed accounts that were already touched in the batch. + fn rollback<'a, AC>(accounts: AC) + where + AC: Iterator, + { + for acc in accounts { + if !acc.dirty() { + continue; + } + let Borrowed(acc) = acc.cow() else { continue }; + // SAFETY: only borrowed accounts were updated before the failed commit. + unsafe { acc.rollback() }; + } + } + + /// Commits a borrowed image after updating its owner mapping if needed. + fn update<'e>( + &'e self, + pubkey: &Pubkey, + acc: &BorrowedAccount, + markers: &DirtyMarkers, + txn: OptRwTxn<'_, 'e>, + ) -> Result<()> { + if markers.contains(DirtyMarkers::OWNER) { + let txn = write_txn(self.index.env(), txn)?; + let owner = acc.owner().into(); + self.index.update_owner(pubkey, owner, txn)?; + } + if !markers.intersects(DirtyMarkers::all()) { + return Ok(()); + } + acc.commit(); + self.storage.stats().commit(); + Ok(()) + } + + /// Serializes an owned image into mapped storage and records its offset. + fn insert<'e>( + &'e self, + pubkey: &Pubkey, + acc: &OwnedAccount, + txn: OptRwTxn<'_, 'e>, + ) -> Result<()> { + let txn = write_txn(self.index.env(), txn)?; + let units = acc.units(); + let owner = acc.owner().into(); + + let (ptr, offset) = if let Some(offset) = self.index.allocate(units, txn)? { + let ptr = self.storage.at(offset); + self.storage.stats().realloc(); + (ptr, offset) + } else { + let alloc = self.storage.allocate(units)?; + (alloc.ptr, alloc.offset) + }; + let data = OwnerAndOffset { owner, offset }; + if let Some(offset) = self.index.delete(pubkey, txn)? { + self.free(offset, txn)?; + } + self.index.insert(pubkey, data, txn)?; + // SAFETY: `ptr` points at a fresh span inside the mapped storage and + // `units` is the exact serialized size of this owned account. + unsafe { + let buffer = slice::from_raw_parts_mut(ptr.as_ptr(), units as usize); + acc.serialize(buffer, pubkey); + }; + Ok(()) + } + + /// Returns one persisted span to the freelist. + fn free(&self, offset: Offset, txn: &mut heed::RwTxn<'_>) -> Result<()> { + // SAFETY: `offset` was returned by the index and still points at a valid image. + let space = unsafe { BorrowedAccount::span(self.storage.at(offset)) }; + self.index.freelist.put(txn, &space, &offset)?; + Ok(()) + } + + /// Removes a persisted image and returns its storage span to the freelist. + fn delete<'e>(&'e self, pubkey: &Pubkey, txn: OptRwTxn<'_, 'e>) -> Result<()> { + let txn = write_txn(self.index.env(), txn)?; + let Some(offset) = self.index.delete(pubkey, txn)? else { + return Ok(()); + }; + self.free(offset, txn)?; + self.storage.stats().remove(); + Ok(()) + } + + /// Returns the persisted storage metadata header. + pub(crate) fn meta(&self) -> &DatabaseMeta { + self.storage.meta() + } + + /// Computes a deterministic checksum over persisted accounts in pubkey order. + fn checksum(&self) -> heed::Result { + let _timer = metrics::time(Operation::Checksum); + let mut hasher = XxHash3_64::new(); + let mut iter = self.index.accounts()?; + hasher.write(&self.meta().slot.load(Acquire).to_le_bytes()); + hasher.write(&self.meta().superblock.load(Acquire).to_le_bytes()); + for entry in &mut iter.inner { + let (pubkey, data) = entry?; + hasher.write(pubkey.as_array()); + // SAFETY: offsets come from the persisted accounts index and point + // into the mapped storage for this store. + let account = unsafe { BorrowedAccount::init(self.storage.at(data.offset)) }; + hasher.write(account.storage()); + } + Ok(hasher.finish()) + } +} + +impl<'a> Iterator for PersistedProgramIter<'a> { + type Item = AccountEntry; + + fn next(&mut self) -> Option { + let (_, offset) = self.iter.inner.next()?.ok()?; + let ptr = self.mmap.at(offset); + // The image prefix stores the full pubkey, so iteration can recover it + // without consulting LMDB again. + // SAFETY: the iterator yields offsets stored in the same mapped database. + self.mmap.stats().read(); + let pubkey = unsafe { BorrowedAccount::pubkey(ptr) }; + let account = unsafe { BorrowedAccount::init(ptr).into() }; + Some((pubkey, account)) + } +} diff --git a/accountsdb/src/tests.rs b/accountsdb/src/tests.rs new file mode 100644 index 00000000..ac2ff604 --- /dev/null +++ b/accountsdb/src/tests.rs @@ -0,0 +1,764 @@ +//! Integration-style unit tests for the two-backend account store. +//! +//! Each test drives a realistic multi-step flow through the public `AccountsDB` +//! surface and reaches into `pub(crate)` internals only to assert *which* +//! backend a given account landed in — the crate's central persisted/volatile +//! invariant that no public method exposes directly. + +use std::sync::atomic::Ordering::{Relaxed, Release}; + +use assert_matches::assert_matches; +use nucleus::{ + heed::{DatabaseIndex, read_txn}, + testkit::{TempDir, init_tracing, tempdir}, +}; +use solana_account::{ + AccountBuilder, AccountMode, AccountSharedData, ReadableAccount, WritableAccount, +}; +use solana_pubkey::Pubkey; + +use super::*; +use crate::{snapshot::VOLATILE_DB_FILE, store::MIN_REMAINDER}; + +/// Fresh database on a throwaway directory; the `TempDir` must outlive the db. +fn db() -> (TempDir, AccountsDB) { + init_tracing(); + let dir = tempdir(); + let db = AccountsDB::new(dir.path()).unwrap(); + (dir, db) +} + +/// Owned mutable (persisted) account carrying `data`; its size follows the data. +fn mutable_data(lamports: u64, data: Vec, owner: &Pubkey) -> AccountSharedData { + let mut a = AccountSharedData::new(lamports, data.len(), owner); + a.set_data_from_slice(&data); + a.set_mode(AccountMode::Delegated).unwrap(); + a +} + +/// Empty mutable (persisted) account; `owner` defaults to the system program. +fn delegated(lamports: u64) -> AccountSharedData { + AccountBuilder::default() + .lamports(lamports) + .mode(AccountMode::Delegated) + .build() +} + +/// Stores one account, the shape every single-account write below takes. +fn store(db: &AccountsDB, pubkey: Pubkey, account: AccountSharedData) { + db.store(&[(pubkey, account)]).unwrap(); +} + +/// Whether a persisted image exists for `pubkey`. +fn in_persisted(db: &AccountsDB, pubkey: &Pubkey) -> bool { + let mut txn = None; + db.persisted.contains(&mut txn, pubkey).unwrap() +} + +/// Whether a volatile entry exists for `pubkey`. +fn in_volatile(db: &AccountsDB, pubkey: &Pubkey) -> bool { + db.volatile.contains(pubkey) +} + +/// Pubkeys `owner` owns, in iteration order (persisted first, then volatile). +fn program(db: &AccountsDB, owner: &Pubkey) -> Vec { + db.program(owner).unwrap().map(|(k, _)| k).collect() +} + +/// Balance of the account currently loaded for `pubkey`. +fn lamports(db: &AccountsDB, pubkey: &Pubkey) -> u64 { + db.loader().load(pubkey).unwrap().unwrap().lamports() +} + +/// Loads the account currently stored for `pubkey`. +/// +/// A persisted account comes back as a *borrowed* image and a volatile one as +/// *owned*; storing the loaded value back is how the engine drives mode changes +/// through the routing layer (a freshly built owned account with a +/// non-authoritative mode is filtered out of the persisted backend entirely). +fn reload(db: &AccountsDB, pubkey: &Pubkey) -> AccountSharedData { + db.loader().load(pubkey).unwrap().unwrap() +} + +/// Closes `pubkey`, deleting it from whichever backend currently holds it. +/// +/// Goes through the load→mutate→store path so the account is a *borrowed* image +/// the routing layer will actually evict (see [`reload`]). +fn close(db: &AccountsDB, pubkey: &Pubkey) { + let mut acc = reload(db, pubkey); + if acc.is(AccountMode::Delegated) { + acc.set_mode(AccountMode::Transient).unwrap(); + } + if acc.is(AccountMode::Transient) { + acc.set_mode(AccountMode::ReadOnly).unwrap(); + } + acc.set_mode(AccountMode::Closed).unwrap(); + store(db, *pubkey, acc); +} + +/// Allocation high-water mark of the persisted store, in storage units. +fn cursor(db: &AccountsDB) -> u32 { + db.persisted.storage.cursor() +} + +/// Current persisted offset for one live account. +fn offset(db: &AccountsDB, pubkey: &Pubkey) -> impl Copy + PartialEq + use<> { + let mut txn = None; + let txn = read_txn(db.persisted.index.env(), &mut txn).unwrap(); + db.persisted.index.offset(pubkey, txn).unwrap().unwrap() +} + +/// Defragments until a pass makes no change and returns the reclaimed total. +fn defrag_to_stable(db: &AccountsDB) -> u32 { + let mut total = 0; + loop { + // SAFETY: the test is the sole owner of the store during defrag. + let pass = unsafe { db.persisted.defragment() }.unwrap(); + total += pass.reclaimed; + if !pass.changed() { + return total; + } + } +} + +/// Builds a mutable account with an exact persisted span. +fn mutable_units(lamports: u64, units: u32, owner: &Pubkey) -> AccountSharedData { + let account = mutable_at_least(lamports, units, owner); + assert_eq!(account.owned().units(), units); + account +} + +/// Builds the smallest mutable account spanning at least `units` storage units. +fn mutable_at_least(lamports: u64, units: u32, owner: &Pubkey) -> AccountSharedData { + (0..=units as usize * solana_account::STORAGE_UNIT) + .map(|len| mutable_data(lamports, vec![0; len], owner)) + .find(|account| account.owned().units() >= units) + .unwrap() +} + +// Routing, both eviction directions, owner remap and Closed/reset handling in +// one flow — the persisted-vs-volatile invariant is what this whole crate +// exists to enforce. +#[test] +fn test_routing_and_persistence_flips() { + let (_dir, db) = db(); + let (p, q) = (Pubkey::new_unique(), Pubkey::new_unique()); + let (a, b) = (Pubkey::new_unique(), Pubkey::new_unique()); + + let aacc = AccountBuilder::default().lamports(10).owner(p).mode(AccountMode::Delegated); + let bacc = AccountBuilder::default().lamports(20).owner(p); + // `a` is authoritative, `b` is non-authoritative; both are owned by `p`. + db.store(&[(a, aacc.build()), (b, bacc.build())]).unwrap(); + assert!(in_persisted(&db, &a) && !in_volatile(&db, &a)); + assert!(in_volatile(&db, &b) && !in_persisted(&db, &b)); + + // Loader reads across both backends; contains agrees. + let loader = db.loader(); + assert_eq!(loader.load(&a).unwrap().unwrap().lamports(), 10); + assert_eq!(loader.load(&b).unwrap().unwrap().lamports(), 20); + assert!(loader.contains(&a).unwrap() && loader.contains(&b).unwrap()); + assert!(!loader.contains(&Pubkey::new_unique()).unwrap()); + drop(loader); + + // Persisted account is yielded before the volatile one. + assert_eq!(program(&db, &p), vec![a, b]); + + // Loading a persisted account returns a borrowed image; mutating its owner + // and re-storing must commit in place and remap the program index. + let mut borrowed = reload(&db, &a); + borrowed.set_owner(q); + store(&db, a, borrowed); + assert_eq!(program(&db, &p), vec![b]); // `a` left p's set + assert_eq!(program(&db, &q), vec![a]); // and joined q's + + // Transient is immutable to programs but remains persistent while its + // lifecycle state is unresolved. + let mut flip = reload(&db, &a); + flip.set_mode(AccountMode::Transient).unwrap(); + flip.set_lamports(30); + store(&db, a, flip); + let transient = reload(&db, &a); + assert!(!transient.mutable()); + assert!(in_persisted(&db, &a) && !in_volatile(&db, &a)); + assert_eq!(lamports(&db, &a), 30); + + // Resolving to ReadOnly evicts the persisted copy into volatile. + let mut flip = reload(&db, &a); + flip.set_mode(AccountMode::ReadOnly).unwrap(); + store(&db, a, flip); + assert!(!in_persisted(&db, &a) && in_volatile(&db, &a)); + assert_eq!(lamports(&db, &a), 30); + + // ReadOnly → Delegated evicts it back into persisted. + let mut flip = reload(&db, &a); + flip.set_mode(AccountMode::Delegated).unwrap(); + store(&db, a, flip); + assert!(in_persisted(&db, &a) && !in_volatile(&db, &a)); + + // Closing removes it from both backends. + close(&db, &a); + assert!(!in_persisted(&db, &a) && !in_volatile(&db, &a)); + + // reset() drops volatile mirror only; persisted state is authoritative. + let c = Pubkey::new_unique(); + store(&db, c, mutable_data(50, vec![], &p)); + db.reset(); + assert!(!in_volatile(&db, &b)); + assert!(in_persisted(&db, &c)); +} + +// Both migration directions remove the source image and owner mapping, retain +// the account contents across reopen, and recycle persisted storage. +#[test] +fn test_store_kind_migration_invariants() { + let dir = tempdir(); + let (persisted_owner, volatile_owner, reuse_owner) = ( + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ); + let (key, reuse) = (Pubkey::new_unique(), Pubkey::new_unique()); + let data = vec![1, 2, 3, 4]; + let assert_migrated = |db: &AccountsDB, persisted: bool, owner: Pubkey| { + assert_eq!(in_persisted(db, &key), persisted); + assert_eq!(in_volatile(db, &key), !persisted); + assert_eq!(program(db, &owner), vec![key]); + let account = reload(db, &key); + assert_eq!(account.owner(), &owner); + assert_eq!(account.lamports(), 20); + assert_eq!(account.data(), data); + }; + + { + let db = AccountsDB::new(dir.path()).unwrap(); + store(&db, key, mutable_data(10, data.clone(), &persisted_owner)); + let base = cursor(&db); + + let mut account = reload(&db, &key); + account.set_mode(AccountMode::Transient).unwrap(); + account.set_mode(AccountMode::ReadOnly).unwrap(); + account.set_owner(volatile_owner); + account.set_lamports(20); + store(&db, key, account); + + assert_migrated(&db, false, volatile_owner); + assert!(program(&db, &persisted_owner).is_empty()); + + // A same-sized persisted account must reuse the span released by the + // migration instead of extending the mmap. + store(&db, reuse, mutable_data(30, data.clone(), &reuse_owner)); + assert_eq!(cursor(&db), base); + + db.dump(None).unwrap(); + } + + { + let db = AccountsDB::new(dir.path()).unwrap(); + assert_migrated(&db, false, volatile_owner); + assert!(program(&db, &persisted_owner).is_empty()); + + let mut account = reload(&db, &key); + account.set_mode(AccountMode::Delegated).unwrap(); + account.set_owner(persisted_owner); + store(&db, key, account); + + assert_migrated(&db, true, persisted_owner); + assert!(program(&db, &volatile_owner).is_empty()); + + db.flush(true).unwrap(); + // Persist a stale volatile copy if cleanup regresses, so the final open + // can verify source-store cleanup rather than merely losing memory state. + db.dump(None).unwrap(); + } + + let db = AccountsDB::new(dir.path()).unwrap(); + assert_migrated(&db, true, persisted_owner); + assert!(program(&db, &volatile_owner).is_empty()); + assert_eq!(program(&db, &reuse_owner), vec![reuse]); +} + +// Persisted state and metadata survive a close/reopen, and validate() accepts +// the synced checksum. +#[test] +fn test_persistence_reopen_and_validate() { + let dir = tempdir(); + let keys: Vec = (0..8).map(|_| Pubkey::new_unique()).collect(); + + let (checksum, before) = { + let db = AccountsDB::new(dir.path()).unwrap(); + for (i, k) in keys.iter().enumerate() { + store(&db, *k, delegated(100 + i as u64)); + } + let discarded = Pubkey::new_unique(); + store(&db, discarded, delegated(0)); + close(&db, &discarded); + db.set_slot(42).unwrap(); + // Sync the checksum into the header so a reopen can validate against it. + db.persisted.flush(true).unwrap(); + assert!(db.validate().is_ok()); + (db.checksum(), cursor(&db)) + }; + + let mut db = AccountsDB::new(dir.path()).unwrap(); + assert!(db.validate().is_ok()); + let reclaimed = db.compact().unwrap(); + assert_eq!(reclaimed, before - cursor(&db)); + assert!(reclaimed > 0); + for (i, k) in keys.iter().enumerate() { + assert_eq!(lamports(&db, k), 100 + i as u64); + } + assert_eq!(db.slot(), 42); + assert_eq!(db.checksum(), checksum); + assert!(db.validate().is_ok()); +} + +// A clean-shutdown dump lives in the active tree, is restored on the next open, +// and is then removed so volatile state returns to its in-memory-only form. +#[test] +fn test_dump_restores_volatile_on_reopen() { + let dir = tempdir(); + let key = Pubkey::new_unique(); + let active = AccountsDB::directory(dir.path()); + let dump = active.join(VOLATILE_DB_FILE); + + { + let db = AccountsDB::new(dir.path()).unwrap(); + let account = AccountBuilder::default().lamports(42).mode(AccountMode::ReadOnly).build(); + store(&db, key, account); + db.dump(None).unwrap(); + assert!(dump.exists(), "dump is written into the active tree"); + } + + let db = AccountsDB::new(dir.path()).unwrap(); + assert_eq!(lamports(&db, &key), 42); + assert!(in_volatile(&db, &key)); + assert!(!dump.exists(), "restored dump is consumed on open"); +} + +// A freed span is reused for a same-sized insert instead of growing the file; +// genuinely new accounts still extend it. +#[test] +fn test_freelist_reuse_and_growth() { + let (_dir, db) = db(); + let stats = || { + let s = db.persisted.storage.stats(); + (s.allocs.load(Relaxed), s.reallocs.load(Relaxed)) + }; + + let k1 = Pubkey::new_unique(); + store(&db, k1, delegated(1)); + let base = cursor(&db); + let (_, reallocs) = stats(); + + // Close k1 (returns its span to the freelist), then insert a same-sized + // account: it should land in the freed span without advancing the cursor. + close(&db, &k1); + store(&db, Pubkey::new_unique(), delegated(2)); + assert_eq!(cursor(&db), base); + assert_eq!(stats().1, reallocs + 1); + + // Fresh accounts have no reusable span, so the file grows. + let (allocs, _) = stats(); + for _ in 0..8 { + store(&db, Pubkey::new_unique(), delegated(1)); + } + assert!(cursor(&db) > base); + assert!(stats().0 > allocs); +} + +// Defragmentation reclaims interior holes while preserving every live account's +// content, ownership index, and checksum. +#[test] +fn test_defragment_preserves_live_accounts() { + let (_dir, db) = db(); + let owner = Pubkey::new_unique(); + let keys: Vec = (0..16).map(|_| Pubkey::new_unique()).collect(); + for (i, k) in keys.iter().enumerate() { + store(&db, *k, mutable_data(100 + i as u64, vec![], &owner)); + } + + // Punch alternating holes; keep the survivors for later comparison. + let mut live = Vec::new(); + for (i, k) in keys.iter().enumerate() { + if i % 2 == 0 { + close(&db, k); + } else { + live.push((*k, 100 + i as u64)); + } + } + db.persisted.flush(true).unwrap(); + let checksum = db.checksum(); + let before = cursor(&db); + + let reclaimed = defrag_to_stable(&db); + assert_eq!(reclaimed, before - cursor(&db)); + assert!(cursor(&db) < before); + + // Every survivor still loads unchanged and remains program-indexed. + for (k, lam) in &live { + let acc = db.loader().load(k).unwrap().unwrap(); + assert_eq!(acc.lamports(), *lam); + assert_eq!(acc.owner(), &owner); + } + let mut owned = program(&db, &owner); + owned.sort(); + let mut expected: Vec = live.iter().map(|(k, _)| *k).collect(); + expected.sort(); + assert_eq!(owned, expected); + + // Relocating images must not change the content checksum. + db.persisted.flush(true).unwrap(); + assert_eq!(db.checksum(), checksum); +} + +/// Exact and thresholded best-fit packing updates component holes correctly, +/// while deferred source holes become usable only by a later committed pass. +#[test] +fn test_defragment_best_fit_and_deferred_holes() { + let owner = Pubkey::new_unique(); + let small = mutable_units(1, 21, &owner); + let small_units = small.owned().units(); + let medium = mutable_at_least(2, MIN_REMAINDER, &owner); + let medium_units = medium.owned().units(); + + // Two adjacent component holes form one run. The small tail account leaves + // a useful remainder, which the following account consumes exactly. + { + let (_dir, db) = db(); + let (small_hole, medium_hole, anchor, medium_key, small_key) = ( + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ); + store(&db, small_hole, small.clone()); + store(&db, medium_hole, medium.clone()); + store(&db, anchor, small.clone()); + store(&db, medium_key, medium.clone()); + store(&db, small_key, small.clone()); + let medium_dst = offset(&db, &medium_hole); + let small_dst = offset(&db, &small_hole); + close(&db, &small_hole); + close(&db, &medium_hole); + + let pass = unsafe { db.persisted.defragment() }.unwrap(); + assert_eq!(pass.moved, 2); + assert_eq!(pass.reclaimed, small_units + medium_units); + assert!(offset(&db, &medium_key) == medium_dst); + assert!(offset(&db, &small_key) == small_dst); + assert_eq!(lamports(&db, &medium_key), 2); + assert_eq!(lamports(&db, &small_key), 1); + assert_eq!(lamports(&db, &anchor), 1); + + let before = cursor(&db); + store(&db, Pubkey::new_unique(), small.clone()); + assert_eq!(cursor(&db), before + small_units); + } + + // Exact fit wins first. The next account skips a hole whose remainder is + // just below the threshold. Two accounts instead use a wider hole and + // publish a useful suffix. + { + let (_dir, db) = db(); + let short_remainder = (MIN_REMAINDER - 1) & !1; + let near_units = small_units + short_remainder; + let near = mutable_units(3, near_units, &owner); + let wide = mutable_units(4, 2 * small_units + medium_units, &owner); + let ( + near_hole, + anchor_a, + wide_hole, + anchor_b, + exact_hole, + anchor_c, + wide_key_a, + wide_key_b, + exact_key, + ) = ( + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ); + store(&db, near_hole, near); + store(&db, anchor_a, small.clone()); + store(&db, wide_hole, wide); + store(&db, anchor_b, small.clone()); + store(&db, exact_hole, small.clone()); + store(&db, anchor_c, small.clone()); + store(&db, wide_key_a, small.clone()); + store(&db, wide_key_b, small.clone()); + store(&db, exact_key, small.clone()); + let near_dst = offset(&db, &near_hole); + let wide_dst = offset(&db, &wide_hole); + let exact_dst = offset(&db, &exact_hole); + close(&db, &near_hole); + close(&db, &wide_hole); + close(&db, &exact_hole); + + let pass = unsafe { db.persisted.defragment() }.unwrap(); + assert_eq!(pass.moved, 3); + assert_eq!(pass.reclaimed, 63); + assert!(offset(&db, &wide_key_b) == wide_dst); + assert!(offset(&db, &exact_key) == exact_dst); + + let before = cursor(&db); + store(&db, Pubkey::new_unique(), medium.clone()); + assert_eq!(cursor(&db), before); + let near_key = Pubkey::new_unique(); + store(&db, near_key, mutable_units(5, near_units, &owner)); + assert_eq!(cursor(&db), before); + assert!(offset(&db, &near_key) == near_dst); + } + + // The first pass moves only the middle account. Its source joins the next + // hole after commit, and public startup compaction exhausts later passes. + { + let (_dir, mut db) = db(); + let second = mutable_units(3, 2 * medium_units - small_units, &owner); + let (first_hole, middle, second_hole, tail) = ( + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + Pubkey::new_unique(), + ); + store(&db, first_hole, small.clone()); + store(&db, middle, small.clone()); + store(&db, second_hole, second); + store(&db, tail, medium.clone()); + let middle_dst = offset(&db, &first_hole); + let tail_dst = offset(&db, &middle); + close(&db, &first_hole); + close(&db, &second_hole); + let before = cursor(&db); + + let pass = unsafe { db.persisted.defragment() }.unwrap(); + assert_eq!((pass.moved, pass.reclaimed), (1, 0)); + assert_eq!(cursor(&db), before); + assert!(offset(&db, &middle) == middle_dst); + + assert_eq!(db.compact().unwrap(), 2 * medium_units); + assert!(offset(&db, &tail) == tail_dst); + assert_eq!(lamports(&db, &middle), 1); + assert_eq!(lamports(&db, &tail), 2); + + let before = cursor(&db); + store(&db, Pubkey::new_unique(), small.clone()); + assert_eq!(cursor(&db), before + small_units); + } +} + +// A snapshot is a self-contained tree: reopening it restores persisted accounts +// and bootstraps the volatile store from volatile.db, which is then consumed. +#[test] +fn test_snapshot_export_and_volatile_restore() { + let src = tempdir(); + let (a, b) = (Pubkey::new_unique(), Pubkey::new_unique()); + + let snapshot = { + let db = AccountsDB::new(src.path()).unwrap(); + let aacc = AccountBuilder::default().lamports(10).mode(AccountMode::Delegated); + let bacc = AccountBuilder::default().lamports(20).mode(AccountMode::ReadOnly); + db.store(&[(a, aacc.build()), (b, bacc.build())]).unwrap(); + // SAFETY: the test holds exclusive access to the store. + unsafe { db.snapshot(1) }.unwrap() + }; + + // Adopt the snapshot as a new database's active tree. + let dst = tempdir(); + let active = AccountsDB::directory(dst.path()); + std::fs::rename(&snapshot, &active).unwrap(); + let db = AccountsDB::new(dst.path()).unwrap(); + + assert_eq!(lamports(&db, &a), 10); + assert_eq!(lamports(&db, &b), 20); + assert!(in_persisted(&db, &a)); + assert!(in_volatile(&db, &b)); + // The volatile payload is single-sourced back into memory on open. + assert!(!active.join(VOLATILE_DB_FILE).exists()); + + // Backup renames the active tree out and back. + let saved = db.backup(BackupOp::Save).unwrap(); + assert!(saved.exists() && !active.exists()); + db.backup(BackupOp::Restore).unwrap(); + assert!(active.exists()); +} + +// validate() flags a persisted checksum that no longer matches the images. +#[test] +fn test_corruption_detection() { + let (_dir, db) = db(); + for _ in 0..4 { + store(&db, Pubkey::new_unique(), delegated(1)); + } + db.persisted.flush(true).unwrap(); + assert!(db.validate().is_ok()); + + // Corrupt the recorded checksum; recomputation must no longer agree. + db.persisted.meta().checksum.store(0xDEAD_BEEF, Release); + assert_matches!(db.validate(), Err(AccountsDBError::Corruption)); +} + +// Several freed spans of one size accumulate as duplicates under a single +// freelist key and are all reissued before the file grows — the N>1 duplicate +// case a broken DUP config silently loses. +#[test] +fn test_freelist_multi_duplicate_reuse() { + let (_dir, db) = db(); + let reallocs = || db.persisted.storage.stats().reallocs.load(Relaxed); + + const N: usize = 6; + let keys: Vec = (0..N).map(|_| Pubkey::new_unique()).collect(); + for k in &keys { + store(&db, *k, delegated(1)); + } + let base = cursor(&db); + // Close them all: N same-size spans return to the freelist as N duplicates. + for k in &keys { + close(&db, k); + } + let reused = reallocs(); + + // Each of N fresh same-size inserts must land in a freed span, so the cursor + // never advances and every insert is a reuse. + for _ in 0..N { + store(&db, Pubkey::new_unique(), delegated(2)); + } + assert_eq!(cursor(&db), base); + assert_eq!(reallocs(), reused + N as u64); +} + +// An immutable account changing owner is re-homed in the volatile program index +// and the now-empty old owner set is pruned. +#[test] +fn test_volatile_owner_remap() { + let (_dir, db) = db(); + let (x, y) = (Pubkey::new_unique(), Pubkey::new_unique()); + let k = Pubkey::new_unique(); + + store( + &db, + k, + AccountBuilder::default().lamports(10).owner(x).build(), + ); + assert_eq!(program(&db, &x), vec![k]); + + // Re-store the volatile account under a new owner. + let mut moved = reload(&db, &k); + moved.set_owner(y); + store(&db, k, moved); + + assert_eq!(program(&db, &x), Vec::::new()); // old set pruned + assert_eq!(program(&db, &y), vec![k]); + assert!(in_volatile(&db, &k)); +} + +// The freelist reuses a span only on an exact size match, and accounts of mixed +// sizes survive defragmentation with their data intact. +#[test] +fn test_variable_sizes_and_exact_freelist() { + let (_dir, db) = db(); + let owner = Pubkey::new_unique(); + let reallocs = || db.persisted.storage.stats().reallocs.load(Relaxed); + + // A freed large span cannot satisfy a smaller allocation: sizes differ, so + // the small insert allocates fresh rather than reusing the hole. + let big = Pubkey::new_unique(); + store(&db, big, mutable_data(1, vec![0; 4096], &owner)); + close(&db, &big); + let before = reallocs(); + let small = Pubkey::new_unique(); + store(&db, small, mutable_data(2, vec![0; 64], &owner)); + assert_eq!(reallocs(), before); // size mismatch -> no reuse + + // Store a spread of sizes with distinct data, punch an interior hole, then + // defragment and confirm every survivor keeps its exact bytes. + let sizes = [8usize, 512, 100, 4096, 1]; + let mut live = Vec::new(); + for (i, &space) in sizes.iter().enumerate() { + let k = Pubkey::new_unique(); + let data: Vec = (0..space).map(|b| (b as u8).wrapping_add(i as u8)).collect(); + store(&db, k, mutable_data(i as u64, data.clone(), &owner)); + live.push((k, data)); + } + close(&db, &small); + + defrag_to_stable(&db); + for (k, data) in &live { + assert_eq!( + db.loader().load(k).unwrap().unwrap().data(), + data.as_slice() + ); + } +} + +// The checksum hashes accounts in pubkey order, so it depends only on content — +// not on insertion order or the resulting on-disk offsets. +#[test] +fn test_checksum_order_independent() { + let keys: Vec = (0..8).map(|_| Pubkey::new_unique()).collect(); + + let checksum = |order: &[usize]| { + let (_dir, db) = db(); + for &i in order { + store(&db, keys[i], delegated(100 + i as u64)); + } + db.persisted.flush(true).unwrap(); + db.checksum() + }; + + let forward: Vec = (0..keys.len()).collect(); + let reversed: Vec = (0..keys.len()).rev().collect(); + assert_eq!(checksum(&forward), checksum(&reversed)); +} + +// 2 MiB accounts overflow the initial storage block, forcing the file to grow; +// removing half then defragmenting reclaims the large holes and shrinks the +// cursor back — growth and compaction over multi-megabyte images. +#[test] +fn test_large_accounts_growth_and_defrag() { + const SIZE: usize = 2 << 20; // 2 MiB of data per account + const COUNT: usize = 12; // ~24 MiB total, past the 16 MiB test block + + let (_dir, db) = db(); + let owner = Pubkey::new_unique(); + let resizes = || db.persisted.storage.stats().resizes.load(Relaxed); + let baseline = resizes(); + + // Distinct fill byte per account so content is verifiable without retaining + // the expected bytes. + let keys: Vec = (0..COUNT).map(|_| Pubkey::new_unique()).collect(); + for (i, k) in keys.iter().enumerate() { + store(&db, *k, mutable_data(i as u64, vec![i as u8; SIZE], &owner)); + } + // Crossing the initial block must have grown the file. + assert!(resizes() > baseline); + + // Close every other account to punch large interior holes. + let mut live = Vec::new(); + for (i, k) in keys.iter().enumerate() { + if i % 2 == 0 { + close(&db, k); + } else { + live.push((*k, i as u8)); + } + } + let before = cursor(&db); + + let reclaimed = defrag_to_stable(&db); + assert_eq!(reclaimed, before - cursor(&db)); + assert!(cursor(&db) < before); + + // Every survivor keeps its full 2 MiB image byte-for-byte. + for (k, fill) in &live { + let acc = db.loader().load(k).unwrap().unwrap(); + assert_eq!(acc.data().len(), SIZE); + assert!(acc.data().iter().all(|&b| b == *fill)); + } +} diff --git a/accountsdb/src/volatile.rs b/accountsdb/src/volatile.rs new file mode 100644 index 00000000..8e296291 --- /dev/null +++ b/accountsdb/src/volatile.rs @@ -0,0 +1,129 @@ +//! In-memory account cache and program ownership sets. + +use std::{ + collections::BTreeSet, + fs::{self, File}, + io::BufReader, + path::Path, +}; + +use ahash::RandomState; +use scc::HashMap; +use solana_account::{AccountMode, AccountSharedData, OwnedAccount, ReadableAccount}; +use solana_pubkey::Pubkey; +use tracing::info; + +use crate::{Result, StoreKind, metrics, snapshot::VOLATILE_DB_FILE}; + +/// Owned accounts keyed by account pubkey. +type AccountsMap = HashMap; +/// Program ownership sets keyed by owner pubkey. +type ProgramsMap = HashMap, RandomState>; + +/// Volatile account store backed by concurrent hash maps. +pub(crate) struct VolatileStore { + /// Current owned accounts. + pub(crate) accounts: AccountsMap, + /// Program owner -> account pubkeys. + pub(crate) programs: ProgramsMap, +} + +impl VolatileStore { + /// Opens the volatile store, optionally bootstrapping from a snapshot file. + /// + /// If `volatile.db` exists, it is loaded into memory and then removed from + /// the snapshot directory so the active tree stays single-sourced. + pub(crate) fn new(path: &Path) -> Result { + const CAP: usize = 2048; + let snapshot = path.join(VOLATILE_DB_FILE); + let accounts: AccountsMap = if snapshot.exists() { + let mut r = BufReader::new(File::open(&snapshot)?); + let accs: AccountsMap = bincode::deserialize_from(&mut r)?; + fs::remove_file(snapshot)?; + info!( + count = accs.len(), + "restored volatile accounts from snapshot" + ); + accs + } else { + AccountsMap::with_capacity_and_hasher(CAP, Default::default()) + }; + + let programs = ProgramsMap::with_capacity_and_hasher(CAP, Default::default()); + accounts.iter_sync(|&pk, acc| { + BTreeSet::insert(&mut programs.entry_sync(acc.owner()).or_default(), pk) + }); + Ok(Self { accounts, programs }) + } + + /// Stores volatile accounts and keeps the program ownership sets in sync. + pub(crate) fn upsert<'a, AC>(&self, accounts: AC) + where + AC: IntoIterator, + { + for (pubkey, account) in accounts { + // An account that has moved to an authoritative mode, or has been + // closed, drops any stale volatile copy. + if account.mode().authoritative() || account.is(AccountMode::Closed) { + self.delete(pubkey); + continue; + } + // Non-authoritative accounts stay volatile and update the program + // mapping. + let owner = *account.owner(); + { + let mut set = self.programs.entry_sync(owner).or_default(); + BTreeSet::insert(&mut set, *pubkey); + } + let Some(prev) = self.accounts.upsert_sync(*pubkey, account.owned()) else { + continue; + }; + if prev.owner() == owner { + continue; + } + // Only the old owner set needs cleanup; the new owner was inserted above. + self.programs.remove_if_sync(&prev.owner(), |set| { + set.remove(pubkey); + set.is_empty() + }); + } + metrics::accounts(StoreKind::Volatile, self.accounts.len() as u64); + } + + /// Returns the owned account currently cached for `pubkey`. + pub(crate) fn load(&self, pubkey: &Pubkey) -> Option { + let entry = self.accounts.get_sync(pubkey)?; + Some(entry.get().clone()) + } + + /// Returns whether a volatile account exists for `pubkey`. + pub(crate) fn contains(&self, pubkey: &Pubkey) -> bool { + self.accounts.contains_sync(pubkey) + } + + /// Returns the owned accounts currently mapped to `owner`. + pub(crate) fn program(&self, owner: &Pubkey) -> BTreeSet { + self.programs.read_sync(owner, |_, s| s.clone()).unwrap_or_default() + } + + /// Drops chain-mirrored accounts while retaining internal system accounts. + pub(crate) fn reset(&self) { + self.programs.clear_sync(); + self.accounts.retain_sync(|k, a| { + if !a.is(AccountMode::System) { + return false; + } + let mut set = self.programs.entry_sync(a.owner()).or_default(); + BTreeSet::insert(&mut set, *k) + }); + } + + /// Removes the cached account and drops its owner mapping. + fn delete(&self, pubkey: &Pubkey) { + let Some(e) = self.accounts.remove_sync(pubkey) else { return }; + self.programs.remove_if_sync(&e.1.owner(), |set| { + set.remove(pubkey); + set.is_empty() + }); + } +} diff --git a/clippy.toml b/clippy.toml index db02f67e..19eebeb6 100644 --- a/clippy.toml +++ b/clippy.toml @@ -19,7 +19,6 @@ max-struct-bools = 2 check-private-items = true disallowed-methods = [ - { path = "std::option::Option::unwrap", reason = "Use expect with context or handle the None case explicitly" }, - { path = "std::result::Result::unwrap", reason = "Use expect with context or propagate the error explicitly" }, - { path = "std::thread::sleep", reason = "Avoid timing-based flakiness in workspace code; isolate retries/backoff behind abstractions" } + { path = "std::thread::sleep", reason = "Avoid timing-based flakiness in workspace code; use event driven logic" }, + { path = "tokio::time::sleep", reason = "Avoid timing-based flakiness in workspace code; use event driven logic" } ] diff --git a/engine/Cargo.toml b/engine/Cargo.toml new file mode 100644 index 00000000..00dc5f68 --- /dev/null +++ b/engine/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "magicblock-engine" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "engine" + +[features] +testkit = ["keeper/testkit", "nucleus/testkit", "tokio/time"] + +[dependencies] +keeper = { workspace = true } +ledger = { workspace = true } +magic-root-interface = { workspace = true } +magic-root-program = { workspace = true } +nucleus = { workspace = true, features = ["config", "shutdown"] } +processor = { workspace = true } + +derive_more = { workspace = true } +num_cpus = { workspace = true } +oneshot = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } +wincode = { workspace = true } + +agave-transaction-view = { workspace = true } +solana-account = { workspace = true } +solana-compute-budget-program = { workspace = true, features = ["agave-unstable-api"] } +solana-instruction = { workspace = true } +solana-keypair = { workspace = true } +solana-message = { workspace = true } +solana-program-runtime = { workspace = true } +solana-pubkey = { workspace = true } +solana-sdk-ids = { workspace = true } +solana-signer = { workspace = true } +solana-system-program = { workspace = true, features = ["agave-unstable-api"] } +solana-transaction = { workspace = true, features = ["wincode"] } + +[dev-dependencies] +keeper = { workspace = true, features = ["testkit"] } +magicblock-engine = { path = ".", features = ["testkit"] } +nucleus = { workspace = true, features = ["testkit"] } +v42-calculator-interface = { workspace = true, features = ["builder"] } + +solana-instruction-error = { workspace = true } +solana-packet = { workspace = true } +solana-signer = { workspace = true } +solana-system-interface = { workspace = true, features = ["bincode"] } +solana-sysvar = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync", "time"] } + +[lints] +workspace = true diff --git a/engine/README.md b/engine/README.md new file mode 100644 index 00000000..c68ac2e4 --- /dev/null +++ b/engine/README.md @@ -0,0 +1,62 @@ +# `magicblock-engine` + +This crate exposes `Engine`, the consumer-facing handle over keeper state, +transaction sequencing, simulation, block pacing, recovery, and MagicRoot +account operations. It registers MagicRoot and the System Program as native +builtins before keeper opens startup state. + +`Engine::signer` is always the local keypair. `Engine::authority` returns the +configured remote authority for a replica, or the local identity when no +override is configured. Replication uses that distinction to sign locally while +authenticating its immediate upstream. + +## Account replacement + +`AccountAccessor::{create, update}` composes complete-account MagicRoot patch +transactions. Replacement slots are monotonic: a newer slot is accepted, an +equal slot requires a genuine account-mode transition, and an older slot is +rejected even when the mode changes. Failed replacements are transactionally +rolled back. Complete-account patch sequences cover non-flag fields, while +finalization atomically installs the caller-supplied complete flag value without +changing lamports. Callers are responsible for supplying current state; later +replacements remain subject to the account's slot and lifecycle rules. `create` +appends any `PostFinalize` actions immediately after finalization in the same +transaction. Magicblock construction rejects instruction, address, account-meta, +and instruction-data lengths that cannot be represented by the V1 wire fields. + +## Startup and recovery + +Keeper restores an accountsdb snapshot when the active store is corrupt, its +sealed superblock trails the retained ledger, or its committed transaction count +trails the ledger's durable count. Accountsdb's count is a checkpoint high-water +mark, so a count ahead of the locally retained ledger is current, including for +snapshots staged by a replication follower. Superblock lag remains recoverable +independently of the counters. + +If accountsdb then trails the ledger tip, `Engine::new` replays retained entries +from the successor of its sealed snapshot through a temporary sequencer. Replay +quiesces at superblock seals and compares the reconstructed checksum with the +recorded seal. A mismatch returns `ReplayError::StateMismatch`. Current state +opens without replay when its slot and transaction count are each at least the +ledger values. After replay actually runs, the final transaction counts must be +equal or startup returns `ReplayError::StateMismatch`. + +Internal pacing appends one reset marker at the current slot and clears +chain-mirrored volatile accounts before the pacemaker task starts. Internal +system accounts remain available. Replicas use external pacing and retain +restored volatile state. External block producers supply the slot and timestamp; +the sequencer overwrites hash-chain metadata with its locally computed hash and +parent. + +## Shutdown + +Shutdown behavior follows the pacing source. Internal pacing publishes a final +block and flushes durable state. External pacing flushes the durable cursor +before writing `CURRENT/volatile.db`, allowing the next open and replication +handshake to resume from matching state. The pacemaker holds the sequencer +barrier while issuing a terminal ledger sync, which closes the appender and +reader workers without waiting for every engine handle to be dropped. + +The embedding service retains the `ShutdownManager` passed to `Engine::new` and +calls `terminate` after stopping external ingress. The manager stops the +replication client, pacemaker, sequencer, and backing services in order. diff --git a/engine/src/accessor.rs b/engine/src/accessor.rs new file mode 100644 index 00000000..87116c34 --- /dev/null +++ b/engine/src/accessor.rs @@ -0,0 +1,106 @@ +//! Account- and transaction-scoped operation facades. + +use std::{sync::atomic::Ordering, time::Duration}; + +use keeper::{ExecutionRecord, TransactionView}; +use magic_root_interface::MagicRootInstruction; +use processor::{SequencerMessage, Simulation, SimulatorMessage}; +use solana_account::OwnedAccount; +use solana_instruction::Instruction; +use solana_pubkey::Pubkey; +use solana_transaction::TransactionResult; +use tokio::time; + +use crate::{Engine, error::EngineError, error::Result, transaction}; + +/// Upper bound on awaiting a submitted transaction's committed result. +const EXECUTION_TIMEOUT: Duration = Duration::from_secs(8); + +/// Account-scoped operations bound to a single `pubkey`. +pub struct AccountAccessor<'a> { + pub(crate) pubkey: Pubkey, + pub(crate) engine: &'a Engine, +} + +/// Transaction-submission operations bound to an engine instance. +pub struct TransactionAccessor<'a> { + pub(crate) engine: &'a Engine, + pub(crate) transaction: TransactionView, +} + +impl AccountAccessor<'_> { + /// Creates the account by patching in every field and finalizing it, + /// optionally running follow-up `actions` once it is finalized. + pub async fn create( + &self, + acc: impl Into, + actions: Option>, + ) -> Result<()> { + let mut instructions = MagicRootInstruction::compose_account(self.pubkey, acc.into())?; + if let Some(actions) = actions { + instructions.push(MagicRootInstruction::PostFinalize(actions).compose(self.pubkey)?); + } + self.execute(instructions).await + } + + /// Updates the account by patching in every field of `account` + pub async fn update(&self, acc: impl Into) -> Result<()> { + let instructions = MagicRootInstruction::compose_account(self.pubkey, acc.into())?; + self.execute(instructions).await + } + + /// Closes the account. + pub async fn delete(&self) -> Result<()> { + let instructions = vec![MagicRootInstruction::Delete.compose(self.pubkey)?]; + self.execute(instructions).await + } + + /// Composes the instructions into a signed engine transaction, executes it, + /// and flattens the committed transaction result into the engine error type. + async fn execute(&self, instructions: Vec) -> Result<()> { + let txn = transaction::magicblock(&instructions, self.engine)?; + self.engine.transaction(txn)?.execute().await?.map_err(Into::into) + } +} + +impl TransactionAccessor<'_> { + /// Submits `transaction` for execution and awaits its committed result. + /// A timeout does not cancel the submitted transaction. + pub async fn execute(self) -> Result> { + if self.engine.terminating.load(Ordering::Acquire) { + return Err(EngineError::ShuttingDown); + } + let signature = self.transaction.signatures()[0]; + let msg = SequencerMessage::Transaction(self.transaction); + let rx = self.engine.transactions().subscribe_signature(signature).await; + self.engine.sequencer.send(msg).await?; + let status = time::timeout(EXECUTION_TIMEOUT, rx) + .await + .map_err(|_| EngineError::TransactionTimeout)? + .map_err(|e| e.to_string())?; + Ok(status.result) + } + + /// Submits `transaction` for execution without awaiting its result. + pub async fn schedule(self) -> Result<()> { + if self.engine.terminating.load(Ordering::Acquire) { + return Err(EngineError::ShuttingDown); + } + let msg = SequencerMessage::Transaction(self.transaction); + self.engine.sequencer.send(msg).await.map_err(Into::into) + } + + /// Simulates `transaction` against current state without committing it. + pub async fn simulate(self) -> Result> { + if self.engine.terminating.load(Ordering::Acquire) { + return Err(EngineError::ShuttingDown); + } + let (response, rx) = oneshot::channel(); + let msg = SimulatorMessage::Transaction(Simulation { + transaction: self.transaction, + response, + }); + self.engine.sequencer.simulation.send(msg).await?; + rx.await.map_err(Into::into) + } +} diff --git a/engine/src/error.rs b/engine/src/error.rs new file mode 100644 index 00000000..0e297bee --- /dev/null +++ b/engine/src/error.rs @@ -0,0 +1,91 @@ +//! Engine error types. + +use agave_transaction_view::result::TransactionViewError; +use derive_more::From; +use keeper::error::KeeperError; +use ledger::{LedgerError, LedgerRequestError}; +use nucleus::shutdown::Service; +use processor::ProcessorError; +use solana_message::CompileError; +use solana_transaction::{InstructionError, SignerError, TransactionError}; +use tokio::sync::mpsc::error::SendError; + +/// Result type used by engine APIs. +pub type Result = std::result::Result; + +/// Failures surfaced by the top-level engine. +#[derive(From, thiserror::Error, Debug)] +pub enum EngineError { + /// A durable-state (keeper) operation failed. + #[error("state error: {0}")] + State(#[source] KeeperError), + /// Scheduling or executing a transaction failed. + #[error("processor error: {0}")] + Processor(#[source] ProcessorError), + /// Replaying the ledger into volatile state on startup failed. + #[error("replay error: {0}")] + Replay(#[source] ReplayError), + /// A background service is no longer reachable. + #[error("service became unavailable: {0:?}")] + ServiceUnavailable(Service), + /// The engine has begun coordinated shutdown and rejects new work. + #[error("engine is shutting down")] + ShuttingDown, + /// Timed out waiting for a submitted transaction's committed result. + #[error("timed out waiting for transaction result")] + TransactionTimeout, + /// Signing a transaction with the engine authority failed. + #[error("signature error: {0}")] + Signature(#[source] SignerError), + /// Serializing or deserializing a transaction failed. + #[error("serialization error: {0}")] + Serde(#[source] wincode::Error), + /// Sanitizing a serialized transaction into a transaction view failed. + #[error("transaction sanitization: {0:?}")] + Sanitization(TransactionViewError), + /// Compiling instructions into a versioned transaction message failed. + #[error("transaction compilation failed: {0}")] + TransactionCompile(#[source] CompileError), + /// A submitted transaction carried an invalid signature. + #[error("transaction signature verification failed")] + SignatureVerification, + /// A submitted transaction was committed with an execution failure. + #[error("transaction execution failed: {0}")] + TransactionExecution(#[source] TransactionError), + /// An unexpected internal failure carrying a contextual message. + #[error("internal error: {0}")] + Internal(String), +} + +impl From> for EngineError { + fn from(_: SendError) -> Self { + Self::ServiceUnavailable(Service::Sequencer) + } +} +impl From for EngineError { + fn from(error: InstructionError) -> Self { + Self::TransactionExecution(TransactionError::InstructionError(0, error)) + } +} +impl From for EngineError { + fn from(_: oneshot::RecvError) -> Self { + Self::ServiceUnavailable(Service::Sequencer) + } +} + +/// Failures raised while replaying retained ledger entries on startup. +#[derive(From, thiserror::Error, Debug)] +pub enum ReplayError { + /// A retained transaction could not be sanitized into a transaction view. + #[error("transaction sanitization: {0:?}")] + Sanitization(TransactionViewError), + /// The replayed account state checksum diverged from the sealed superblock. + #[error("replayed state checksum mismatch")] + StateMismatch, + /// Waiting for the ledger reader's replay response failed. + #[error("ledger replay request failed: {0}")] + Request(#[source] LedgerRequestError), + /// Reading or decoding retained ledger entries failed. + #[error("ledger replay failed: {0}")] + Ledger(#[source] LedgerError), +} diff --git a/engine/src/lib.rs b/engine/src/lib.rs new file mode 100644 index 00000000..954737d0 --- /dev/null +++ b/engine/src/lib.rs @@ -0,0 +1,220 @@ +#![doc = include_str!("../README.md")] + +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Instant, +}; + +use derive_more::Deref; +use keeper::{Keeper, builder::KeeperBuilder, error::KeeperError}; +use ledger::schema::OwnedBlockstoreEntry; +use magic_root_program::entrypoint::MagicRootEntrypoint; +use nucleus::{ + runtime::{self, BarrierHandle, SequencerHandle}, + shutdown::{Service, ShutdownManager, ShutdownReason}, +}; +use processor::{SequencerMessage, sequencer::Sequencer}; +use solana_compute_budget_program::Entrypoint as ComputeBudgetEntrypoint; +use solana_program_runtime::{ + loaded_programs::{ProgramCache, ProgramCacheEntry}, + solana_sbpf::program::BuiltinFunctionDefinition, +}; +use solana_pubkey::Pubkey; +use solana_system_program::system_processor::Entrypoint as SystemProgramEntrypoint; +use tracing::{error, info}; + +mod accessor; +mod error; +pub mod pacemaker; +mod transaction; + +#[cfg(feature = "testkit")] +pub mod testkit; + +pub use accessor::{AccountAccessor, TransactionAccessor}; +pub use error::{EngineError, ReplayError, Result}; +pub use transaction::IntoTransactionView; + +use crate::pacemaker::{ExternalPacer, PaceMaker}; + +/// Top-level engine handle: owns the durable state and the sequencer submission +/// channels. +#[derive(Deref, Clone)] +pub struct Engine { + /// Durable engine state (accountsdb + ledger), shared across components. + #[deref] + state: Arc, + /// Submission handle into the sequencer's execution and simulation channels. + sequencer: SequencerHandle, + /// Rejects new transactions once coordinated shutdown begins. + terminating: Arc, +} + +impl Engine { + /// Builds and starts the engine. + /// + /// Opens durable state through the keeper builder (coming up on persisted + /// state), replays retained ledger entries to rebuild volatile state only when + /// recovering from a rewound accountsdb, starts the live sequencer, and spawns + /// the pacemaker using the builder's blockstore timing. + pub async fn new( + mut builder: KeeperBuilder, + pacer: Option, + shutdown: &mut ShutdownManager, + ) -> Result { + let cache = Arc::new(ProgramCache::default()); + let cpus = (num_cpus::get().saturating_sub(2)).max(2); + + builder.builtins.insert( + magic_root_interface::ID, + (MagicRootEntrypoint::vm, MagicRootEntrypoint::codegen), + ); + builder.builtins.insert( + solana_system_program::id(), + ( + SystemProgramEntrypoint::vm, + SystemProgramEntrypoint::codegen, + ), + ); + builder.builtins.insert( + solana_sdk_ids::compute_budget::id(), + ( + ComputeBudgetEntrypoint::vm, + ComputeBudgetEntrypoint::codegen, + ), + ); + + for (&id, builtin) in &builder.builtins { + let entry = ProgramCacheEntry::new_builtin(*builtin); + cache.assign_program(id, entry.into()); + } + let blockstore = builder.blockstore; + let state = Arc::new(builder.build(shutdown).await?); + Self::try_replay(&state, &cache, cpus).await?; + let (service, sequencer) = Sequencer::new(cpus / 2, state.clone(), cache, shutdown, false)?; + service.spawn()?; + let terminating = Arc::new(AtomicBool::new(false)); + let engine = Self { state, sequencer, terminating }; + PaceMaker::spawn(engine.clone(), pacer, blockstore, shutdown)?; + info!(authority = %engine.authority(), cpus, "engine started"); + Ok(engine) + } + + /// Quiesces execution and closes durable state. + /// + /// `dump` serializes chain-mirrored state for an externally paced replica to + /// restore on its next open. Internally paced leaders clear that state once + /// during startup instead and only flush durable state here. + pub async fn shutdown(&self, dump: bool) -> Result<()> { + info!(dump, "shutting down the engine"); + self.terminating.store(true, Ordering::Release); + let _guard = self.barrier().await?; + if dump { + self.accounts().dump(None).map_err(KeeperError::from)?; + } + self.sync(true).map_err(Into::into) + } + + /// Returns an accessor for mutating the account at `pubkey`. + pub fn account(&self, pubkey: Pubkey) -> AccountAccessor<'_> { + AccountAccessor { engine: self, pubkey } + } + + /// Returns an accessor for signing and submitting transactions. + pub fn transaction(&self, transaction: T) -> Result> + where + T: IntoTransactionView, + { + let transaction = transaction.compose(self)?; + Ok(TransactionAccessor { engine: self, transaction }) + } + + /// Drains in-flight execution and keeps the sequencer paused until the handle is dropped. + pub async fn barrier(&self) -> Result { + let (controller, guard) = runtime::barrier(); + self.sequencer.send(SequencerMessage::Barrier(guard)).await?; + controller.acknowledged.await?; + Ok(controller.released) + } + + /// Applies one retained ledger entry through the engine's ordered paths. + /// + /// Seal and reset entries quiesce execution before touching shared state; + /// a reconstructed seal whose checksum differs returns + /// [`ReplayError::StateMismatch`]. + pub async fn replay(&self, entry: OwnedBlockstoreEntry) -> Result<()> { + match entry { + OwnedBlockstoreEntry::Transaction(txn) => self.transaction(txn)?.schedule().await?, + OwnedBlockstoreEntry::Block(block) => { + self.sequencer.send(SequencerMessage::Block(block)).await?; + } + OwnedBlockstoreEntry::Superblock(expected) => { + let _guard = self.barrier().await?; + let previous = self.superblocks().sealed().id; + self.accounts().set_superblock(expected.id); + self.sync(false)?; + let observed = self.superblocks().sealed(); + if observed != expected { + error!(?observed, ?expected, "state mismatch; aborting replay"); + self.accounts().set_superblock(previous); + self.sync(false)?; + Err(ReplayError::StateMismatch)?; + } + } + OwnedBlockstoreEntry::Reset(slot) => { + let _guard = self.barrier().await?; + self.reset(slot)?; + } + }; + Ok(()) + } + + /// Rebuilds state through a temporary replay sequencer when accountsdb trails + /// the retained ledger, then stops every temporary service before returning. + async fn try_replay(state: &Arc, cache: &Arc, cpus: usize) -> Result<()> { + let timer = Instant::now(); + let Some(mut replayer) = state.replay().await? else { + return Ok(()); + }; + let mut shutdown = ShutdownManager::default(); + let mut sh = shutdown.handle(Service::LedgerReplayer); + let (service, sequencer) = + Sequencer::new(cpus, state.clone(), cache.clone(), &mut shutdown, true)?; + service.spawn()?; + let engine = Self { + state: state.clone(), + sequencer, + terminating: Default::default(), + }; + while let Some(entry) = replayer.rx.recv().await { + engine.replay(entry).await?; + } + replayer + .response + .recv_timeout() + .await + .map_err(ReplayError::from)? + .map_err(ReplayError::from)?; + + drop(engine.barrier().await?); + engine.sync(false)?; + let accountsdb = state.accounts().transactions(); + let ledger = state.ledger().transactions(); + if accountsdb != ledger { + error!( + accountsdb, + ledger, "transaction count mismatch; aborting replay" + ); + Err(ReplayError::StateMismatch)?; + } + + let slot = state.blocks().latest().slot; + info!(slot, duration = ?timer.elapsed(), "ledger replay complete"); + sh.terminate(ShutdownReason::Signalled); + shutdown.terminate().await; + Ok(()) + } +} diff --git a/engine/src/pacemaker.rs b/engine/src/pacemaker.rs new file mode 100644 index 00000000..dbea204c --- /dev/null +++ b/engine/src/pacemaker.rs @@ -0,0 +1,190 @@ +//! Block-boundary pacing. + +use std::{num::NonZeroU64, time::Duration}; + +use derive_more::Deref; +use ledger::schema::Block; +use nucleus::{ + Slot, + config::BlockstoreParams, + shutdown::{Service, ShutdownHandle, ShutdownManager, ShutdownReason}, + unix_time, +}; +use processor::{SequencerMessage, SimulatorMessage}; +use tokio::{ + sync::mpsc::Receiver, + time::{self, Interval, MissedTickBehavior}, +}; +use tracing::error; + +use crate::{Engine, Result}; + +/// Channel used by external block producers. +pub type ExternalPacer = Receiver; + +/// Emits block boundaries into engine execution paths. +#[derive(Deref)] +pub struct PaceMaker { + /// Engine handle used to submit each boundary. + #[deref] + engine: Engine, + /// Source for the next block boundary. + pacer: Pacer, + /// Number of slots sealed into each superblock. + superblock: NonZeroU64, +} + +/// Source of block boundaries. +pub enum Pacer { + /// Interval-driven slot production. + Internal(BlockTicker), + /// Externally supplied block boundaries. + External(ExternalPacer), +} + +/// State for interval-driven slot production. +pub struct BlockTicker { + /// Next slot to emit. + slot: Slot, + /// Block production interval. + ticker: Interval, +} + +impl BlockTicker { + /// Builds an interval ticker starting at the engine's current slot. + pub(crate) fn new(engine: &Engine, blocktime: Duration) -> Self { + let slot = engine.blocks().current_slot(); + let mut ticker = time::interval(blocktime); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + ticker.reset(); + BlockTicker { slot, ticker } + } + + /// Returns the next block boundary and advances the slot cursor. + pub(crate) fn block(&mut self) -> Block { + let time = unix_time().as_secs() as i64; + let block = Block::new(self.slot, time); + self.slot += 1; + block + } +} + +/// Block boundary submitted by an external producer. +/// +/// The caller supplies its slot and timestamp. The sequencer overwrites the +/// hash and parent with locally computed hash-chain metadata. +pub struct ExternalBlock { + /// Boundary to enqueue. + pub block: Block, + /// Notified after the pacemaker handles the boundary locally. + /// + /// On ordinary slots this means the boundary was queued and the keeper slot + /// was advanced. On superblock slots it also includes the synchronous seal. + pub submitted: oneshot::Sender<()>, +} + +impl ExternalBlock { + /// Pairs a boundary with the receiver signalled once the pacemaker has locally + /// handled it, letting the submitter await ordered application. + pub fn new(block: Block) -> (Self, oneshot::Receiver<()>) { + let (submitted, guard) = oneshot::channel(); + let block = Self { block, submitted }; + (block, guard) + } +} + +impl PaceMaker { + /// Registers and starts the pacemaker task. + /// + /// Uses an external block source when supplied. Otherwise it records one + /// reset at the keeper's current slot, clears chain-mirrored volatile state, + /// and starts emitting slots on the configured block interval. + pub fn spawn( + engine: Engine, + pacer: Option, + blockstore: BlockstoreParams, + shutdown: &mut ShutdownManager, + ) -> Result<()> { + let pacer = match pacer { + Some(rx) => Pacer::External(rx), + None => { + let ticker = BlockTicker::new(&engine, blockstore.blocktime); + engine.reset(ticker.slot)?; + Pacer::Internal(ticker) + } + }; + let shutdown = shutdown.handle(Service::PaceMaker); + let superblock = blockstore.superblock; + let pacemaker = Self { engine, pacer, superblock }; + tokio::spawn(pacemaker.run(shutdown)); + Ok(()) + } + + /// Paces block boundaries until shutdown or the block source is exhausted. + /// + /// Shutdown follows the pacing mode. Internal pacing publishes one last + /// block and flushes durable state. External pacing also checkpoints + /// volatile state alongside its durable cursor for the next upstream + /// handshake. + async fn run(mut self, mut shutdown: ShutdownHandle) { + let mut res = loop { + let next = tokio::select! { + biased; + _ = shutdown.signalled() => None, + next = self.next() => next, + }; + let Some((block, submission)) = next else { + break Ok(()); + }; + if let Err(error) = self.handle(block).await { + break Err(error); + } + if let Some(submission) = submission { + let _ = submission.send(()); + } + }; + res = if let Pacer::Internal(ref mut t) = self.pacer { + // Await every shutdown step even after an earlier failure. + let b = t.block(); + res.and(self.handle(b).await).and(self.shutdown(false).await) + } else { + res.and(self.shutdown(true).await) + }; + // Release engine storage before the manager can reopen it. + drop(self); + if let Err(error) = res { + error!(?error, "pace maker terminated with critical failure"); + shutdown.terminate(ShutdownReason::Error(error.into())); + } else { + shutdown.terminate(ShutdownReason::Signalled); + } + } + + /// Waits for the next block boundary without applying it. + async fn next(&mut self) -> Option<(Block, Option>)> { + match &mut self.pacer { + Pacer::Internal(t) => { + t.ticker.tick().await; + Some((t.block(), None)) + } + Pacer::External(rx) => rx.recv().await.map(|msg| (msg.block, Some(msg.submitted))), + } + } + + /// Advances the execution and simulation environments to `block`, sealing a + /// superblock when the slot lands on the configured interval. + /// + /// The seal is taken behind a barrier and runs synchronously: it exports an + /// accountsdb snapshot, which is only coherent while no store operation can + /// race it. Holding the boundary here is what buys that exclusivity, at the + /// cost of stalling block production until the seal completes. + async fn handle(&self, block: Block) -> Result<()> { + self.sequencer.send(SequencerMessage::Block(block)).await?; + self.sequencer.simulation.send(SimulatorMessage::Block(block)).await?; + if block.slot.is_multiple_of(self.superblock.get()) { + let _guard = self.barrier().await?; + self.finalize_superblock()?; + } + Ok(()) + } +} diff --git a/engine/src/testkit.rs b/engine/src/testkit.rs new file mode 100644 index 00000000..7939834c --- /dev/null +++ b/engine/src/testkit.rs @@ -0,0 +1,177 @@ +//! Shared black-box harness for engine-backed integration suites. +//! +//! Builds a real [`Engine`] over [`keeper::testkit`] directories with internal or +//! externally controlled pacing. Compiled only under the `testkit` feature. +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use derive_more::Deref; +use keeper::{ + ExecutionRecord, + builder::KeeperBuilder, + testkit::{Dirs, SUPERBLOCK, await_archive, block, keeper_builder}, +}; +use nucleus::{Slot, config::Authority, ledger::BlockstorePosition, shutdown::ShutdownManager}; +use solana_account::AccountSharedData; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_transaction::TransactionResult; +use tokio::{sync::mpsc, time}; + +use crate::{Engine, IntoTransactionView, pacemaker::ExternalBlock}; + +const TIMEOUT: Duration = Duration::from_secs(4); + +/// Block pacing for a [`TestEngine`]. +pub enum Pacing { + /// The test supplies blocks through [`TestEngine::pacer`]. + External, + /// The engine runs its own pacemaker. + Internal, +} + +/// A running engine plus its deterministic pacing and lifecycle handles. +#[derive(Deref)] +pub struct TestEngine { + #[deref] + engine: Engine, + shutdown: ShutdownManager, + authority: Authority, + dirs: Dirs, + pacer: Option>, + slot: Slot, +} + +impl TestEngine { + /// Starts the standard test engine on fresh directories. + pub async fn new() -> Self { + Self::with(Dirs::default(), Arc::new(Keypair::new())).await + } + + /// Starts the standard test engine over `dirs` with `authority`. + pub async fn with(dirs: Dirs, authority: impl Into) -> Self { + Self::try_with(dirs, authority).await.unwrap() + } + + /// Fallible [`Self::with`], used when startup failure is the assertion. + pub async fn try_with(dirs: Dirs, authority: impl Into) -> crate::Result { + let mut builder = keeper_builder(&dirs); + builder.authority = authority.into(); + Self::try_from_builder(dirs, builder, Pacing::External).await + } + + /// Starts an engine from a caller-configured keeper builder. + /// + /// `dirs` must own the directories referenced by `builder` and outlive the + /// resulting engine. + pub async fn from_builder(dirs: Dirs, builder: KeeperBuilder, pacing: Pacing) -> Self { + Self::try_from_builder(dirs, builder, pacing).await.unwrap() + } + + /// Fallible [`Self::from_builder`]. + pub async fn try_from_builder( + dirs: Dirs, + builder: KeeperBuilder, + pacing: Pacing, + ) -> crate::Result { + let authority = builder.authority.clone(); + let (pacer, rx) = match pacing { + Pacing::External => { + let (tx, rx) = mpsc::channel(64); + (Some(tx), Some(rx)) + } + Pacing::Internal => (None, None), + }; + let mut shutdown = ShutdownManager::default(); + let engine = Engine::new(builder, rx, &mut shutdown).await?; + let slot = engine.blocks().current_slot(); + Ok(Self { + engine, + shutdown, + authority, + dirs, + pacer, + slot, + }) + } + + /// Cloneable external pacemaker sender for services under test. + /// + /// # Panics + /// + /// Panics if the engine is internally paced. + pub fn pacer(&self) -> mpsc::Sender { + self.pacer.clone().expect("engine is externally paced") + } + + /// Mutable lifecycle manager used to register or await test services. + pub fn shutdown(&mut self) -> &mut ShutdownManager { + &mut self.shutdown + } + + /// Drains engine work, flushes queued ledger appends, and returns the durable cursor. + pub async fn sync(&self) -> BlockstorePosition { + drop(self.barrier().await.unwrap()); + self.superblocks().sync(false).unwrap(); + self.superblocks().position() + } + + /// Full committed account, or `None` when absent/closed. + pub fn get_account(&self, key: Pubkey) -> Option { + self.engine.accounts().loader().load(&key).unwrap() + } + + /// Executes instructions and returns the committed transaction result. + pub async fn execute(&self, txn: impl IntoTransactionView) -> TransactionResult<()> { + self.transaction(txn).unwrap().execute().await.unwrap() + } + + /// Simulates instructions without committing them. + pub async fn simulate( + &self, + txn: impl IntoTransactionView, + ) -> TransactionResult { + self.transaction(txn).unwrap().simulate().await.unwrap() + } + + /// Schedules instructions without awaiting commit. + pub async fn schedule(&self, txn: impl IntoTransactionView) { + self.transaction(txn).unwrap().schedule().await.unwrap(); + } + + /// Advances `n` block boundaries. + /// + /// # Panics + /// + /// Panics if the engine is internally paced. + pub async fn advance(&mut self, n: u64) { + for _ in 0..n { + let (block, submitted) = ExternalBlock::new(block(self.slot)); + self.pacer().send(block).await.unwrap(); + time::timeout(TIMEOUT, submitted) + .await + .expect("pacemaker accepts the block in time") + .expect("pacemaker reports block submission"); + self.slot += 1; + } + } + + /// Seals the next superblock and waits for its snapshot archive. + pub async fn seal_and_archive(&mut self) -> PathBuf { + let boundary = self.slot.next_multiple_of(SUPERBLOCK.into()); + while self.slot <= boundary { + self.advance(1).await; + } + await_archive(self).await + } + + /// Stops every service and returns the directories and authority for reopen. + pub async fn close(self) -> (Dirs, Authority) { + let Self { + mut shutdown, dirs, authority, .. + } = self; + shutdown.terminate().await; + (dirs, authority) + } +} diff --git a/engine/src/transaction.rs b/engine/src/transaction.rs new file mode 100644 index 00000000..d4be48c3 --- /dev/null +++ b/engine/src/transaction.rs @@ -0,0 +1,122 @@ +//! Composing values into sanitized transaction views. + +use agave_transaction_view::{ + MAGICBLOCK_INSTRUCTION_TRACE_LENGTH, MAX_MAGICBLOCK_ACCOUNT_LOCKS, + transaction_version::{MAGICBLOCK_PREFIX, TransactionVersion}, +}; +use keeper::TransactionView; +use solana_instruction::Instruction; +use solana_message::{ + VersionedMessage, + v1::{self, SIGNATURE_SIZE}, +}; +use solana_signer::Signer; +use solana_transaction::{Message, Transaction, TransactionError, versioned::VersionedTransaction}; + +use crate::{Engine, error::EngineError, error::Result}; + +/// Conversion of anything composable into an executable +/// transaction into a sanitized [`TransactionView`]. +pub trait IntoTransactionView { + /// Composes `self` into a sanitized [`TransactionView`], signing with + /// `engine`'s authority and latest blockhash where applicable. + fn compose(self, engine: &Engine) -> Result; +} + +impl IntoTransactionView for Message { + fn compose(self, engine: &Engine) -> Result { + let mut transaction = Transaction::new_unsigned(self); + transaction.try_sign(&[engine.signer()], engine.blockhash())?; + transaction.compose(engine) + } +} + +impl IntoTransactionView for Transaction { + fn compose(self, engine: &Engine) -> Result { + let data = wincode::serialize(&self).map_err(wincode::Error::from)?; + data.compose(engine) + } +} + +impl IntoTransactionView for &[Instruction] { + fn compose(self, engine: &Engine) -> Result { + let msg = Message::new(self, Some(&engine.authority())); + msg.compose(engine) + } +} + +impl IntoTransactionView for &[Instruction; N] { + fn compose(self, engine: &Engine) -> Result { + self.as_slice().compose(engine) + } +} + +impl IntoTransactionView for Vec { + fn compose(self, engine: &Engine) -> Result { + TransactionView::try_new_sanitized(self.into(), true)?.compose(engine) + } +} + +impl IntoTransactionView for TransactionView { + fn compose(self, engine: &Engine) -> Result { + if matches!(self.version(), TransactionVersion::Magicblock) + && self.static_account_keys()[0] != engine.authority() + { + return Err(EngineError::SignatureVerification); + } + sigverify(&self)?; + Ok(self) + } +} + +/// The engine's sole signature-verification point. +/// +/// Execution is trustless: every submission funnels through the +/// [`TransactionView`] `compose` and is verified here, including replay and +/// replication of already-committed transactions. No path reaches the +/// sequencer unverified, so downstream code may assume the fee payer and every +/// required signer actually signed. +fn sigverify(view: &TransactionView) -> Result<()> { + // Sanitization guarantees one static key for every required signature. + let message = view.message_data(); + for (signature, key) in view.signatures().iter().zip(view.static_account_keys()) { + if !signature.verify(key.as_ref(), message) { + return Err(EngineError::SignatureVerification); + } + } + Ok(()) +} + +/// Composes an Engine-private transaction and signs its final +/// Magicblock wire representation with the Engine authority. +pub(crate) fn magicblock(instructions: &[Instruction], engine: &Engine) -> Result> { + let message = v1::Message::try_compile(&engine.authority(), instructions, engine.blockhash())?; + let message = VersionedMessage::V1(message); + // These checks are merely future proof defenses, currently it should be + // impossible to construct a transaction which might violate any of them + if message.instructions().len() > MAGICBLOCK_INSTRUCTION_TRACE_LENGTH { + Err(TransactionError::SanitizeFailure)?; + } else if message.static_account_keys().len() > MAX_MAGICBLOCK_ACCOUNT_LOCKS { + Err(TransactionError::TooManyAccountLocks)?; + } + for ix in message.instructions() { + if ix.accounts.len() > MAX_MAGICBLOCK_ACCOUNT_LOCKS { + Err(TransactionError::TooManyAccountLocks)?; + } + } + + // Reserve the trailing signature slot without signing the V1 prefix, which + // is replaced below before the only signing operation. + let transaction = VersionedTransaction { + signatures: vec![Default::default()], + message, + }; + let mut data = wincode::serialize(&transaction).map_err(wincode::Error::from)?; + // Patch the transaction prefix to allow for larger tranaction limits + data[0] = MAGICBLOCK_PREFIX; + + let signature_offset = data.len() - SIGNATURE_SIZE; + let signature = engine.signer().sign_message(&data[..signature_offset]); + data[signature_offset..].copy_from_slice(signature.as_ref()); + Ok(data) +} diff --git a/engine/tests/accounts.rs b/engine/tests/accounts.rs new file mode 100644 index 00000000..28e0e739 --- /dev/null +++ b/engine/tests/accounts.rs @@ -0,0 +1,460 @@ +//! Account CRUD through the MagicRoot builtin — the privileged mutation path +//! exposed by `AccountAccessor`. This path is untested below the engine: it needs +//! the always-on MagicRoot builtin plus the executor's per-thread authority +//! (MagicRoot authorizes the transaction's fee payer against it). Asserts the +//! create/update/delete round-trip and the sponsor-balance invariant, and +//! that post-finalize actions actually run. +#![cfg(test)] + +use engine::{Engine, EngineError, testkit::TestEngine}; +use keeper::testkit::{ + V42_ID, load_v42_data, load_v42_lamports, patterned_bytes, store_v42, v42_builder, +}; +use magic_root_interface::MagicRootInstruction; +use solana_account::{AccountBuilder, AccountMode, OwnedAccount, ReadableAccount}; +use solana_instruction_error::InstructionError; +use solana_pubkey::Pubkey; +use solana_system_interface::MAX_PERMITTED_DATA_LENGTH; +use solana_sysvar::rent::Rent; +use solana_transaction::TransactionError; +use v42_calculator_interface::builder::{Expr as E, transfer}; + +/// Rent-exempt for the data sizes used below; the SVM rejects a created account +/// that falls under the rent floor. +const LAMPORTS: u64 = 2_000_000; +const SLOT: u64 = 42; + +/// Account with explicit lifecycle state, funded at the shared rent-exempt balance. +fn account(owner: Pubkey, data: Vec, mode: AccountMode, slot: u64) -> OwnedAccount { + AccountBuilder::default() + .lamports(LAMPORTS) + .owner(owner) + .mode(mode) + .slot(slot) + .data(data) + .build() +} + +/// Delegated account with `data` at `slot`. +fn delegated(owner: Pubkey, data: Vec, slot: u64) -> OwnedAccount { + account(owner, data, AccountMode::Delegated, slot) +} + +/// Materializes `mode`, entering transient through its required delegated state. +async fn create_with(engine: &Engine, key: Pubkey, owner: Pubkey, mode: AccountMode) { + let initial = if mode == AccountMode::Transient { + delegated(owner, vec![1], SLOT - 1) + } else { + account(owner, vec![1], mode, SLOT) + }; + engine + .account(key) + .create(initial, None) + .await + .expect("initial account is created"); + if mode == AccountMode::Transient { + engine + .account(key) + .update(account(owner, vec![1], mode, SLOT)) + .await + .expect("delegated account enters transient"); + } +} + +/// Asserts MagicRoot rejected the slot patch in a complete-account sequence. +fn assert_non_advancing_slot(error: EngineError) { + let errored = matches!( + error, + EngineError::TransactionExecution(TransactionError::InstructionError( + 2, + InstructionError::InvalidArgument + )) + ); + assert!(errored, "unexpected replacement error: {error:?}"); +} + +/// Asserts MagicRoot rejected the mode patch in a complete-account sequence. +fn assert_invalid_mode_transition(error: EngineError) { + let errored = matches!( + error, + EngineError::TransactionExecution(TransactionError::InstructionError( + 1, + InstructionError::InvalidArgument + )) + ); + assert!(errored, "unexpected replacement error: {error:?}"); +} + +// The full lifecycle. `create` materializes a fresh account by patching every +// non-flag field, balancing lamport patches against the authority, then +// finalizing its flags; `update` overwrites an existing account or materializes +// a fresh key; and +// `delete` closes it. Mutations here keep the balance constant after creation. +#[tokio::test(flavor = "multi_thread")] +async fn account_crud_lifecycle() { + let te = TestEngine::new().await; + let key = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + + let created = account( + owner, + vec![1, 2, 3, 4, 5, 6, 7, 8], + AccountMode::ReadOnly, + 10, + ); + let authority_before = te.get_account(te.authority()).expect("sponsor exists").lamports(); + + te.account(key).create(created, None).await.unwrap(); + + let acc = te.get_account(key).expect("created account exists"); + assert_eq!(acc.lamports(), LAMPORTS); + assert_eq!(acc.owner(), &owner); + assert_eq!(acc.data(), &[1, 2, 3, 4, 5, 6, 7, 8]); + assert!(acc.is(AccountMode::ReadOnly)); + + let authority_after = te.get_account(te.authority()).expect("sponsor exists").lamports(); + assert_eq!( + authority_before - authority_after, + LAMPORTS, + "the lamport patch sponsors the created balance from the authority" + ); + + // update overwrites the existing account in place: same-length data (the + // patch sequence replaces the exact data length) and identical lamports. + // Read-only accounts remain replaceable after finalization. + te.account(key) + .update(account(owner, vec![5; 16], AccountMode::ReadOnly, 11)) + .await + .unwrap(); + let acc = te.get_account(key).expect("still exists"); + assert_eq!(acc.data(), &[5; 16], "update replaced the data wholesale"); + assert_eq!(acc.owner(), &owner, "update replaced the patched owner"); + assert!(acc.is(AccountMode::ReadOnly)); + + // delete: the account is gone from storage. + te.account(key).delete().await.unwrap(); + assert!(te.get_account(key).is_none(), "deleted account is removed"); + + // update also materializes a fresh account the same way create does, minus + // the post-finalize actions. + let key2 = Pubkey::new_unique(); + te.account(key2).update(delegated(owner, vec![3; 8], 10)).await.unwrap(); + assert_eq!(te.get_account(key2).expect("materialized").data(), &[3; 8]); + + te.close().await; +} + +// Account cloning reconstructs every field and data chunk in one atomic private +// transaction. Growing the same clone through the 64 KiB boundary and beyond, +// then shrinking it below the boundary and to empty, proves replacement keeps +// the exact data length. The caller supplies each successive current state. +#[tokio::test(flavor = "multi_thread")] +async fn account_clone_create_and_update_accept_large_data() { + const MAX_DATA_LEN: usize = 128 * 1024 + 1; + + let te = TestEngine::new().await; + let key = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let lamports = Rent::default().minimum_balance(MAX_DATA_LEN); + + for (index, (len, seed)) in [ + (u16::MAX as usize, 1), + (64 * 1024, 2), + (MAX_DATA_LEN, 3), + (32 * 1024, 4), + (0, 5), + ] + .into_iter() + .enumerate() + { + let data = patterned_bytes(len, seed); + let account = AccountBuilder::default() + .lamports(lamports) + .owner(owner) + .mode(AccountMode::ReadOnly) + .slot(SLOT + index as u64) + .data(data.clone()); + + if index == 0 { + te.account(key).create(account, None).await.unwrap(); + } else { + te.account(key).update(account).await.unwrap(); + } + + let stored = te.get_account(key).expect("large account exists"); + assert_eq!(stored.lamports(), lamports); + assert_eq!(stored.owner(), &owner); + assert!(stored.is(AccountMode::ReadOnly)); + assert_eq!(stored.slot(), SLOT + index as u64); + assert_eq!(stored.data(), data); + } + + te.close().await; +} + +/// Proves an exact maximum-sized Solana account can run a PostFinalize SBPF +/// action above trace index 64, while 257 subsequent V42 self-CPIs hit the CPI +/// trace limit and roll back both the account creation and an earlier action. +#[tokio::test(flavor = "multi_thread")] +async fn account_create_accepts_max_data_with_post_finalize() { + const CPI_CALLS: usize = 257; + + let te = TestEngine::new().await; + let key = Pubkey::new_unique(); + let source = store_v42(&te, 7, AccountMode::Delegated); + let output = store_v42(&te, 0, AccountMode::Ephemeral); + let data = patterned_bytes(MAX_PERMITTED_DATA_LENGTH as usize, 42); + let account = AccountBuilder::default() + .lamports(Rent::default().minimum_balance(data.len())) + .owner(Pubkey::new_unique()) + .mode(AccountMode::Delegated) + .slot(SLOT) + .data(data.clone()); + let action = transfer(source, output, 1); + + te.account(key) + .create(account, Some(vec![action])) + .await + .expect("maximum-sized account and post-finalize action execute atomically"); + + let stored = te.get_account(key).expect("maximum-sized account exists"); + assert_eq!(stored.data().len(), MAX_PERMITTED_DATA_LENGTH as usize); + assert!(stored.data() == data, "maximum-sized account data differs"); + assert_eq!(load_v42_data(&te, source), Some(6)); + assert_eq!(load_v42_data(&te, output), Some(1)); + + let failed_key = Pubkey::new_unique(); + let failed_account = AccountBuilder::default() + .lamports(Rent::default().minimum_balance(data.len())) + .owner(Pubkey::new_unique()) + .mode(AccountMode::Delegated) + .slot(SLOT) + .data(data); + let excessive_cpis = (1..CPI_CALLS) + .fold(E::lit(1).cpi(), |expr, _| expr + E::lit(1).cpi()) + .compose(output, &[]); + let error = te + .account(failed_key) + .create( + failed_account, + Some(vec![transfer(source, output, 1), excessive_cpis]), + ) + .await + .expect_err("257 V42 self-CPIs exceed the trace limit"); + + assert!( + matches!( + error, + EngineError::TransactionExecution(TransactionError::InstructionError( + _, + InstructionError::MaxInstructionTraceLengthExceeded + )) + ), + "unexpected CPI trace error: {error:?}" + ); + assert!( + te.get_account(failed_key).is_none(), + "failed creation was rolled back" + ); + assert_eq!(load_v42_data(&te, source), Some(6)); + assert_eq!(load_v42_data(&te, output), Some(1)); + + te.close().await; +} + +/// Proves program-cache entries follow the complete v42 account lifecycle: +/// transaction-local deletion hides a loaded program immediately but rolls back +/// on a later instruction failure, while committed deletion evicts the shared +/// entry so invalid executable data restored at the same key cannot use stale code. +#[tokio::test(flavor = "multi_thread")] +async fn account_program_cache_tracks_v42_lifecycle() { + let te = TestEngine::new().await; + let seeded = te.get_account(V42_ID).expect("v42 program is seeded"); + let program = Pubkey::new_unique(); + let closeable = AccountBuilder::from(seeded.clone()) + .mode(AccountMode::Ephemeral) + .slot(seeded.slot() + 1); + te.account(program).create(closeable, None).await.unwrap(); + + let output = Pubkey::new_unique(); + te.accounts() + .store(&[( + output, + v42_builder(0, AccountMode::Ephemeral).owner(program).build(), + )]) + .unwrap(); + let invoke = |value| { + let mut instruction = E::lit(value).compose(output, &[]); + instruction.program_id = program; + instruction.accounts.last_mut().unwrap().pubkey = program; + instruction + }; + + te.execute(&[invoke(42)]) + .await + .expect("fresh v42 program executes and primes the shared cache"); + assert_eq!(load_v42_data(&te, output), Some(42)); + + let delete = MagicRootInstruction::Delete.compose(program).unwrap(); + assert_eq!( + te.execute(&[delete, invoke(7)]).await, + Err(TransactionError::InstructionError( + 1, + InstructionError::UnsupportedProgramId + )), + "deletion hides the program from later instructions in the transaction" + ); + assert!( + te.get_account(program).is_some(), + "failed transaction rolls back account deletion" + ); + + te.execute(&[invoke(7)]) + .await + .expect("rolled-back deletion preserves the shared cache entry"); + assert_eq!(load_v42_data(&te, output), Some(7)); + + te.account(program).delete().await.unwrap(); + assert!( + te.get_account(program).is_none(), + "committed deletion removes the account" + ); + + let invalid = AccountBuilder::from(seeded).mode(AccountMode::Ephemeral).data(vec![0]); + te.accounts().store(&[(program, invalid.build())]).unwrap(); + assert_eq!( + te.execute(&[invoke(9)]).await, + Err(TransactionError::InstructionError( + 0, + InstructionError::UnsupportedProgramId + )), + "invalid restored executable cannot run through a stale compiled entry" + ); + + te.close().await; +} + +// Complete replacements are monotonic by slot. An equal-slot replacement is +// meaningful only when it performs a real lifecycle transition; mode is patched +// before slot, and no-op mode writes deliberately leave the mode marker clean. +#[tokio::test(flavor = "multi_thread")] +async fn account_replacement_slot_ordering() { + let te = TestEngine::new().await; + let owner = Pubkey::new_unique(); + + for (from, to) in [ + (AccountMode::ReadOnly, AccountMode::Delegated), + (AccountMode::Placeholder, AccountMode::Ephemeral), + ] { + let key = Pubkey::new_unique(); + create_with(&te, key, owner, from).await; + te.account(key) + .update(account(owner, vec![2], to, SLOT)) + .await + .expect("equal-slot mode transition is accepted"); + + let updated = te.get_account(key).expect("transitioned account exists"); + assert!(updated.is(to), "{from:?} transitions to {to:?}"); + assert_eq!(updated.slot(), SLOT); + assert_eq!(updated.data(), &[2]); + } + + for (from, to) in [ + (AccountMode::Placeholder, AccountMode::Transient), + (AccountMode::Ephemeral, AccountMode::Delegated), + (AccountMode::System, AccountMode::ReadOnly), + ] { + let key = Pubkey::new_unique(); + // Seed the source directly so only the mode-transition invariant is + // under test. + te.accounts() + .store(&[(key, account(owner, vec![1], from, SLOT).into())]) + .unwrap(); + let error = te + .account(key) + .update(account(owner, vec![2], to, SLOT)) + .await + .expect_err("invalid mode transition is rejected"); + assert_invalid_mode_transition(error); + + let unchanged = te.get_account(key).expect("rejected transition preserves the account"); + assert!(unchanged.is(from), "{from:?} does not transition to {to:?}"); + assert_eq!(unchanged.slot(), SLOT); + assert_eq!(unchanged.data(), &[1]); + } + + let key = Pubkey::new_unique(); + te.account(key) + .create(account(owner, vec![3], AccountMode::ReadOnly, SLOT), None) + .await + .expect("baseline account is created"); + + let error = te + .account(key) + .update(account(owner, vec![4], AccountMode::ReadOnly, SLOT)) + .await + .expect_err("equal-slot replacement without a mode change is rejected"); + assert_non_advancing_slot(error); + + let error = te + .account(key) + .update(account(owner, vec![5], AccountMode::Delegated, SLOT - 1)) + .await + .expect_err("a mode change never authorizes an older slot"); + assert_non_advancing_slot(error); + + let unchanged = te.get_account(key).expect("rejected replacements preserve the account"); + assert!(unchanged.is(AccountMode::ReadOnly)); + assert_eq!(unchanged.slot(), SLOT); + assert_eq!(unchanged.data(), &[3]); + + te.close().await; +} + +// Post-finalize actions are invoked via CPI after the account is finalized, so a +// failing action aborts the whole creation (nothing commits), while a benign one +// lets it through. The contrast proves the actions actually execute rather than +// being silently dropped. +#[tokio::test(flavor = "multi_thread")] +async fn create_runs_post_finalize_actions() { + let te = TestEngine::new().await; + + // A successful v42 transfer proves the post-finalize action ran after the + // new account became writable and program-owned. + let source = store_v42(&te, 0, AccountMode::Delegated); + let source_before = load_v42_lamports(&te, source).expect("source exists"); + let ok_key = Pubkey::new_unique(); + let acc = v42_builder(0, AccountMode::Delegated); + let benign = transfer(source, ok_key, 1); + te.account(ok_key) + .create(acc, Some(vec![benign])) + .await + .expect("create with a succeeding post-finalize action"); + assert_eq!( + load_v42_lamports(&te, source).expect("source remains"), + source_before - 1, + "post-finalize action debited its source" + ); + assert_eq!( + load_v42_lamports(&te, ok_key).expect("created account exists"), + source_before + 1, + "post-finalize action credited the created account" + ); + + // An overflowing v42 action errors; the failure propagates and rolls back + // the account creation in the same transaction. + let bad_key = Pubkey::new_unique(); + let failing = (E::lit(i64::MIN) - E::lit(1)).compose(bad_key, &[]); + let acc = v42_builder(0, AccountMode::Delegated); + let result = te.account(bad_key).create(acc, Some(vec![failing])).await; + assert!( + result.is_err(), + "failing post-finalize action surfaces an error" + ); + assert!( + te.get_account(bad_key).is_none(), + "nothing commits when the action fails" + ); + + te.close().await; +} diff --git a/engine/tests/builtins.rs b/engine/tests/builtins.rs new file mode 100644 index 00000000..595da1c4 --- /dev/null +++ b/engine/tests/builtins.rs @@ -0,0 +1,62 @@ +//! Full-engine coverage for native builtins registered during startup. +#![cfg(test)] + +use engine::testkit::TestEngine; +use solana_account::{AccountBuilder, AccountMode, AccountSharedData, ReadableAccount}; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_signer::Signer; +use solana_system_interface::{ + instruction::{allocate, assign, transfer}, + program, +}; +use solana_sysvar::rent::Rent; +use solana_transaction::Transaction; + +#[tokio::test(flavor = "multi_thread")] +async fn system_program_executes_transfer_allocate_and_assign() { + const LAMPORTS: u64 = 42; + const SPACE: usize = 8; + + let te = TestEngine::new().await; + let source = te.authority(); + + let source_before = te.get_account(source).expect("authority account remains"); + assert_eq!(source_before.owner(), &program::ID); + + let destination = Keypair::new(); + let destination_before = Rent::default().minimum_balance(SPACE); + let account: AccountSharedData = AccountBuilder::default() + .lamports(destination_before) + .mode(AccountMode::Delegated) + .build(); + assert_eq!(account.owner(), &program::ID); + te.accounts().store(&[(destination.pubkey(), account)]).unwrap(); + + let owner = Pubkey::new_unique(); + let instructions = [ + transfer(&source, &destination.pubkey(), LAMPORTS), + allocate(&destination.pubkey(), SPACE as u64), + assign(&destination.pubkey(), &owner), + ]; + let transaction = Transaction::new_signed_with_payer( + &instructions, + Some(&source), + &[te.signer(), &destination], + te.blockhash(), + ); + te.execute(transaction).await.expect("failed to execute system ixs"); + + let source_after = te.get_account(source).expect("authority account remains"); + assert_eq!(source_after.lamports(), source_before.lamports() - LAMPORTS); + assert_eq!(source_after.data(), source_before.data()); + assert_eq!(source_after.owner(), source_before.owner()); + + let destination_after = + te.get_account(destination.pubkey()).expect("destination account remains"); + assert_eq!(destination_after.lamports(), destination_before + LAMPORTS); + assert_eq!(destination_after.data(), &[0; SPACE]); + assert_eq!(destination_after.owner(), &owner); + + te.close().await; +} diff --git a/engine/tests/recovery.rs b/engine/tests/recovery.rs new file mode 100644 index 00000000..9130b995 --- /dev/null +++ b/engine/tests/recovery.rs @@ -0,0 +1,146 @@ +//! Full-engine replay recovery — the engine's most distinctive orchestration. +//! After an accountsdb inconsistency the keeper restores an older archived snapshot, +//! leaving durable state behind the ledger tip; the engine then spins a temporary +//! replay sequencer to re-execute the retained ledger entries and rebuild the +//! missing state, checksum-verified at each sealed superblock. Nothing below the +//! engine wires this end to end. Covered here: the healthy restart that must not +//! recover, the replay that crosses a sealed checksum and succeeds, and the +//! replay that diverges from one and must refuse to start. +#![cfg(test)] + +use std::{path::PathBuf, time::Duration}; + +use engine::{EngineError, ReplayError, testkit::TestEngine}; +use keeper::testkit::{corrupt, load_v42_data, store_v42}; +use nucleus::ledger::ACCOUNTSDB_SNAPSHOT_FILE; +use solana_account::AccountMode; +use solana_pubkey::Pubkey; +use tokio::time; +use v42_calculator_interface::builder::Expr as E; + +/// Commits `K = value` through a full transaction and seals the following +/// superblock, returning its archived snapshot path. +async fn commit_and_seal(te: &mut TestEngine, key: Pubkey, value: i64) -> PathBuf { + te.execute(&[E::lit(value).compose(key, &[])]).await.unwrap(); + te.seal_and_archive().await +} + +// Replay must rebuild everything between the restored snapshot and the ledger +// tip: dropping superblock 2's archive forces the restore back onto snapshot 1, +// so re-executing B crosses superblock 2's sealed checksum (the verification +// arm's happy path) before C is rebuilt from the unsealed head. +#[tokio::test(flavor = "multi_thread")] +async fn replay_rebuilds_state_after_counter_lag() { + let mut te = TestEngine::new().await; + let key = store_v42(&te, 0, AccountMode::Delegated); + + // A: K = 10 sealed into superblock 1, whose snapshot the restore lands on. + let s1 = commit_and_seal(&mut te, key, 10).await; + assert!(s1.exists(), "archived accountsdb snapshot exists on disk"); + assert!( + s1.ends_with(ACCOUNTSDB_SNAPSHOT_FILE), + "archive is the compressed accountsdb tarball" + ); + // B: K = 20 sealed into superblock 2; C: K = 30 lives only in the ledger's + // unsealed head, past every archived snapshot. + let s2 = commit_and_seal(&mut te, key, 20).await; + te.execute(&[E::lit(30).compose(key, &[])]).await.expect("C commits"); + te.advance(2).await; + let (dirs, authority) = te.close().await; + + // Lag only accountsdb's durable checkpoint in the closed store, preserving + // valid account content and its checksum. + corrupt(dirs.accounts.path(), 32, 2); + + // Drop the newest archive so recovery falls back to snapshot 1 (K = 10) and + // replays both a sealed successor and the unsealed ledger head. + std::fs::remove_file(&s2).unwrap(); + + let te2 = TestEngine::with(dirs, authority).await; + assert_eq!( + load_v42_data(&te2, key), + Some(30), + "both post-snapshot mutations were rebuilt purely from ledger replay" + ); + // The temporary replay sequencer must hand off to a working live one. + te2.execute(&[E::lit(1).compose(key, &[])]) + .await + .expect("engine is live after replay"); + + te2.close().await; +} + +// A mutation that bypasses the ledger is sealed into superblock 2's checksum but +// can never be rebuilt by replay, so the reopen must refuse to come up with +// `StateMismatch` rather than run on quietly diverged state. +#[tokio::test(flavor = "multi_thread")] +async fn replay_aborts_on_checksum_mismatch() { + let mut te = TestEngine::new().await; + let key = store_v42(&te, 0, AccountMode::Delegated); + commit_and_seal(&mut te, key, 10).await; + // Direct store: lands in persisted state (and superblock 2's checksum) + // without a ledger entry. + store_v42(&te, 7, AccountMode::Delegated); + let s2 = commit_and_seal(&mut te, key, 20).await; + let (dirs, authority) = te.close().await; + + corrupt(dirs.accounts.path(), 8, 0xABAB_ABAB_ABAB_ABAB); + std::fs::remove_file(&s2).unwrap(); + + let result = time::timeout( + Duration::from_secs(4), + TestEngine::try_with(dirs, authority), + ) + .await + .expect("replay aborts in time"); + let error = result.err().expect("diverged checksum refuses startup"); + assert!( + matches!(error, EngineError::Replay(ReplayError::StateMismatch)), + "unexpected startup error: {error:?}" + ); +} + +// A healthy restart opens persisted state as-is and restores the clean-shutdown +// volatile dump. A failed execution still counts on both durable sides without +// writing accounts. The direct-stored delegated account exists in neither +// snapshots nor ledger, while the read-only account exists only in the volatile +// dump; the post-seal transaction write pins the persisted tip alongside them. +#[tokio::test(flavor = "multi_thread")] +async fn clean_restart_reopens_persisted_and_volatile_state() { + let mut te = TestEngine::new().await; + let key = store_v42(&te, 0, AccountMode::Delegated); + commit_and_seal(&mut te, key, 10).await; + te.execute(&[E::lit(20).compose(key, &[])]).await.unwrap(); + let failed = (E::lit(i64::MIN) - E::lit(1)).compose(key, &[]); + assert!( + te.execute(&[failed]).await.is_err(), + "overflow execution fails" + ); + assert_eq!( + load_v42_data(&te, key), + Some(20), + "failed execution writes no state" + ); + let direct = store_v42(&te, 7, AccountMode::Delegated); + let volatile = store_v42(&te, 8, AccountMode::ReadOnly); + let (dirs, authority) = te.close().await; + + let te2 = TestEngine::with(dirs, authority).await; + assert_eq!( + load_v42_data(&te2, key), + Some(20), + "persisted tip state reopened as-is" + ); + assert_eq!( + load_v42_data(&te2, direct), + Some(7), + "ledger-invisible account intact, so no snapshot was restored" + ); + assert_eq!( + load_v42_data(&te2, volatile), + Some(8), + "clean shutdown restores volatile state" + ); + + te2.close().await; +} diff --git a/engine/tests/security.rs b/engine/tests/security.rs new file mode 100644 index 00000000..d8247a11 --- /dev/null +++ b/engine/tests/security.rs @@ -0,0 +1,185 @@ +//! Account-mutability enforcement at the engine boundary. +//! +//! The SVM lets a program write any account it owns; the engine's post-execution +//! guard (`validate_access`) is what rejects writes to accounts that are not in a +//! mutable mode, unless the whole transaction is privileged (every instruction +//! targets MagicRoot). These black-box tests drive the full engine and assert both +//! that the rejection surfaces the right error and that the illegal write never +//! commits. A second enforcement path — MagicRoot's own `post_finalize` check — +//! is covered by `post_finalize_immutable_action_is_rejected`. +#![cfg(test)] + +use engine::{EngineError, testkit::TestEngine}; +use keeper::testkit::{load_v42_data, load_v42_lamports, signed_view, store_v42, v42_builder}; +use magic_root_interface::MagicRootInstruction; +use solana_account::{AccountFieldPatch, AccountMode}; +use solana_instruction::Instruction; +use solana_instruction_error::InstructionError; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_signer::Signer; +use solana_transaction::TransactionError; +use v42_calculator_interface::builder::{Expr as E, transfer}; + +/// Complete v42 account replacement at an explicit non-default slot. +fn compose_v42_replacement(key: Pubkey, mode: AccountMode, slot: u64) -> Vec { + let account = v42_builder(0, mode).slot(slot).build(); + MagicRootInstruction::compose_account(key, account).unwrap() +} + +// The SVM permits the v42 program to write accounts it owns, but the guard +// rejects the commit and discards the mutation whenever the account is immutable +// and the transaction is not privileged. Two branches: a writable operand yields +// InvalidWritableAccount, the fee payer itself yields InvalidAccountForFee. A +// delegated (mutable) account is the positive control. +#[tokio::test(flavor = "multi_thread")] +async fn immutable_writes_are_rejected_and_not_committed() { + let te = TestEngine::new().await; + + // A writable, non-payer immutable source: the transfer dirties both balance + // fields before the guard rejects the source account's engine mode. + let operand = store_v42(&te, 5, AccountMode::ReadOnly); + let recipient = store_v42(&te, 0, AccountMode::Delegated); + let operand_before = load_v42_lamports(&te, operand).expect("operand exists"); + let recipient_before = load_v42_lamports(&te, recipient).expect("recipient exists"); + assert_eq!( + te.execute(&[transfer(operand, recipient, 1)]).await, + Err(TransactionError::InvalidWritableAccount) + ); + assert_eq!( + load_v42_lamports(&te, operand).expect("operand remains"), + operand_before, + "immutable source debit discarded" + ); + assert_eq!( + load_v42_lamports(&te, recipient).expect("recipient remains"), + recipient_before, + "recipient credit rolled back with the transaction" + ); + + // The immutable account is the fee payer itself. The harness `execute` always + // pays with the engine authority, so this branch needs a hand-signed + // transaction: message compilation merges the signer and the writable output + // into account 0. Fees are zero and this SVM does no fee-payer validation, so + // a program-owned payer loads as-is. + let payer = Keypair::new(); + let acc = v42_builder(5, AccountMode::ReadOnly).build(); + te.accounts().store(&[(payer.pubkey(), acc)]).unwrap(); + let (_sig, view) = signed_view(&te, Some(&payer), E::lit(9).compose(payer.pubkey(), &[])); + let result = te.transaction(view).unwrap().execute().await.unwrap(); + assert_eq!(result, Err(TransactionError::InvalidAccountForFee)); + assert_eq!( + load_v42_data(&te, payer.pubkey()), + Some(5), + "fee-payer write discarded" + ); + + // Positive control: a delegated (mutable) account commits normally. + let mutable = store_v42(&te, 0, AccountMode::Delegated); + assert!(te.execute(&[E::lit(9).compose(mutable, &[])]).await.is_ok()); + assert_eq!( + load_v42_data(&te, mutable), + Some(9), + "mutable write commits" + ); + + te.close().await; +} + +// Post-finalize actions are invoked via CPI after an account is created, and +// MagicRoot's `post_finalize` refuses to run them against a writable account that +// is not mutable. Creating a ReadOnly account with an attached v42 write is +// therefore rejected, and the whole creation rolls back — a distinct enforcement +// path from `validate_access` (this fires inside the program, not after). +#[tokio::test(flavor = "multi_thread")] +async fn post_finalize_immutable_action_is_rejected() { + let te = TestEngine::new().await; + + let key = Pubkey::new_unique(); + let mut ixs = compose_v42_replacement(key, AccountMode::ReadOnly, 1); + let post_finalize_idx = ixs.len(); + let post_finalize = MagicRootInstruction::PostFinalize(vec![E::lit(9).compose(key, &[])]); + ixs.push(post_finalize.compose(key).unwrap()); + assert_eq!( + te.execute(ixs.as_slice()).await, + Err(TransactionError::InstructionError( + post_finalize_idx as u8, + InstructionError::Immutable + )), + "MagicRoot's PostFinalize guard rejects the immutable writable account" + ); + assert!( + te.get_account(key).is_none(), + "the rejected creation commits nothing" + ); + + te.close().await; +} + +/// Proves PostFinalize rejects a recursive MagicRoot instruction of a delegated +/// account owned by an unrelated program and rolls back its creation. +#[tokio::test(flavor = "multi_thread")] +async fn post_finalize_rejects_magic_root_ix() { + let te = TestEngine::new().await; + + let key = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let account = v42_builder(0, AccountMode::Delegated).owner(owner); + let patch = MagicRootInstruction::Patch(AccountFieldPatch::DataAt { + offset: 0, + data: 9_i64.to_le_bytes().to_vec(), + }) + .compose(key) + .unwrap(); + + let error = te + .account(key) + .create(account, Some(vec![patch])) + .await + .expect_err("PostFinalize rejects a recursive MagicRoot patch"); + assert!( + matches!( + error, + EngineError::TransactionExecution(TransactionError::InstructionError( + _, + InstructionError::CallDepth + )) + ), + "unexpected recursive invocation error: {error:?}" + ); + assert!( + te.get_account(key).is_none(), + "the rejected recursive action rolls back account creation" + ); + + te.close().await; +} + +// Privilege cannot be laundered through account creation: a single transaction +// that mixes MagicRoot's create-a-ReadOnly-account instructions with a top-level +// (foreign) v42 write is not privileged — `is_privileged` requires *every* +// instruction to be MagicRoot — so the guard runs and the whole transaction, +// creation included, reverts. +#[tokio::test(flavor = "multi_thread")] +async fn mixed_foreign_write_on_created_readonly_is_rejected() { + let te = TestEngine::new().await; + + let key = Pubkey::new_unique(); + // A missing account starts as ReadOnly at slot zero. Advance the replacement + // slot so this test reaches the access guard rather than MagicRoot's + // duplicate-replacement guard. + let mut ixs = compose_v42_replacement(key, AccountMode::ReadOnly, 1); + // The foreign instruction that makes the whole transaction non-privileged. + ixs.push(E::lit(9).compose(key, &[])); + + assert_eq!( + te.execute(ixs.as_slice()).await, + Err(TransactionError::InvalidWritableAccount) + ); + assert!( + te.get_account(key).is_none(), + "the mixed transaction reverts wholesale" + ); + + te.close().await; +} diff --git a/engine/tests/transactions.rs b/engine/tests/transactions.rs new file mode 100644 index 00000000..e0c3ee30 --- /dev/null +++ b/engine/tests/transactions.rs @@ -0,0 +1,249 @@ +//! Transaction submission at the engine boundary: the `execute`, `simulate`, and +//! `schedule` wrappers around the sequencer. The processor suite already proves +//! the SVM commits/simulates correctly; these assert the `TransactionAccessor` +//! ergonomics on top — subscribe-then-await commit, the separate simulation +//! channel that never commits, and fire-and-forget scheduling. +#![cfg(test)] + +use agave_transaction_view::MAX_STANDARD_TRANSACTION_SIZE; +use engine::testkit::TestEngine; +use keeper::testkit::{ + WireVersion, decode_v42, load_v42_data, load_v42_lamports, sign_versioned_instructions, + signed_view, store_v42, v42_padded_value, v42_sum, +}; +use nucleus::KB; +use solana_account::{AccountMode, ReadableAccount}; +use solana_instruction_error::InstructionError; +use solana_packet::PACKET_DATA_SIZE; +use solana_transaction::TransactionError; +use v42_calculator_interface::builder::{Expr as E, transfer}; + +// The Engine accepts the same standard wire formats produced by Solana clients +// on either side of the canonical packet boundary. The wide form reads every +// supplied account, while the batched form independently exercises instruction +// framing instead of relying on one large payload. +#[tokio::test(flavor = "multi_thread")] +async fn client_transaction_formats_execute_below_and_above_packet_limit() { + const WIDE_ACCOUNTS: usize = 32; + const BATCHED_INSTRUCTIONS: usize = 32; + const BATCHED_TERMS: usize = 16; + const FOUR_KIB: usize = 4 * KB; + + let te = TestEngine::new().await; + let operands: Vec<_> = + (0..WIDE_ACCOUNTS).map(|_| store_v42(&te, 1, AccountMode::Delegated)).collect(); + + for version in [WireVersion::Legacy, WireVersion::V0, WireVersion::V1] { + let output = store_v42(&te, 0, AccountMode::Ephemeral); + let (_, small) = sign_versioned_instructions( + te.signer(), + version, + [E::lit(42).compose(output, &[])], + te.blockhash(), + ); + assert!(small.len() < PACKET_DATA_SIZE,); + te.execute(small).await.expect("small v42 transaction succeeds"); + assert_eq!(load_v42_data(&te, output), Some(42)); + + let output = store_v42(&te, 0, AccountMode::Ephemeral); + let (_, wide) = sign_versioned_instructions( + te.signer(), + version, + [v42_sum(output, &operands)], + te.blockhash(), + ); + assert!(wide.len() > PACKET_DATA_SIZE); + assert!(wide.len() < MAX_STANDARD_TRANSACTION_SIZE); + te.execute(wide).await.expect("wide v42 transaction succeeds"); + assert_eq!(load_v42_data(&te, output), Some(WIDE_ACCOUNTS as i64)); + + let output = store_v42(&te, 0, AccountMode::Ephemeral); + let instructions: Vec<_> = (0..BATCHED_INSTRUCTIONS) + .map(|value| v42_padded_value(output, value as i64, BATCHED_TERMS)) + .collect(); + let (_, batched) = + sign_versioned_instructions(te.signer(), version, &instructions, te.blockhash()); + assert!(batched.len() > FOUR_KIB); + assert!(batched.len() < MAX_STANDARD_TRANSACTION_SIZE); + te.execute(batched).await.expect("batched v42 transaction succeeds"); + assert_eq!( + load_v42_data(&te, output), + Some((BATCHED_INSTRUCTIONS - 1) as i64) + ); + } + + te.close().await; +} + +// Simulation runs against live state through the dedicated simulation channel +// but must not commit; execution of the same transfer does. +#[tokio::test(flavor = "multi_thread")] +async fn simulate_does_not_commit_execute_does() { + let te = TestEngine::new().await; + let source = store_v42(&te, 0, AccountMode::Delegated); + let recipient = store_v42(&te, 0, AccountMode::Ephemeral); + let source_before = load_v42_lamports(&te, source).expect("source exists"); + let recipient_before = load_v42_lamports(&te, recipient).expect("recipient exists"); + let ixs = [transfer(source, recipient, 42)]; + + // The record's post-execution account copy proves simulation actually ran + // the program, not merely that the channel round-tripped. + let record = te.simulate(&ixs).await.expect("simulation resolves"); + let executed = record.result.expect("simulated transaction processes"); + assert!(executed.was_successful(), "simulated execution succeeds"); + let (_, simulated_source) = executed + .loaded_transaction + .accounts + .iter() + .find(|(key, _)| *key == source) + .expect("simulation loaded the source account"); + assert_eq!( + simulated_source.lamports(), + source_before - 42, + "simulation debited its source copy" + ); + let (_, simulated_recipient) = executed + .loaded_transaction + .accounts + .iter() + .find(|(key, _)| *key == recipient) + .expect("simulation loaded the recipient account"); + assert_eq!( + simulated_recipient.lamports(), + recipient_before + 42, + "simulation credited its recipient copy" + ); + assert_eq!( + load_v42_lamports(&te, source).expect("source remains"), + source_before, + "simulation leaves the live source untouched" + ); + assert_eq!( + load_v42_lamports(&te, recipient).expect("recipient remains"), + recipient_before, + "simulation leaves the live recipient untouched" + ); + + assert!(te.execute(&ixs).await.is_ok(), "execution resolves"); + assert_eq!( + load_v42_lamports(&te, source).expect("source remains"), + source_before - 42, + "execution commits the source debit" + ); + assert_eq!( + load_v42_lamports(&te, recipient).expect("recipient remains"), + recipient_before + 42, + "execution commits the recipient credit" + ); + + te.close().await; +} + +// A real processed transaction retains its execution artifacts through keeper's +// projection and the ledger's compressed append→index→reader round-trip. +#[tokio::test(flavor = "multi_thread")] +async fn processed_transaction_details_roundtrip() { + let te = TestEngine::new().await; + let slot = te.blocks().current_slot(); + let mut expected = Vec::new(); + + for value in [7, 42, 99] { + let output = store_v42(&te, 0, AccountMode::Ephemeral); + let ix = E::lit(value).cpi().compose(output, &[]); + let (signature, transaction) = signed_view(&te, None, ix.clone()); + let bytes = transaction.inner_data().as_ref().clone(); + + te.execute(&[ix]).await.expect("processed transaction succeeds"); + expected.push((signature, bytes, value)); + } + + te.sync().await; + + for (signature, bytes, value) in expected { + let response = te + .transactions() + .get(signature) + .await + .expect("ledger read succeeds") + .expect("processed transaction is retained"); + assert_eq!(response.transaction, bytes); + assert_eq!(response.execution.header.signature, signature); + assert_eq!(response.execution.header.slot, slot); + assert!(response.execution.header.result.is_ok()); + + let details = response.execution.details.expect("execution details retained"); + assert_eq!( + details.fee, 0, + "the engine does not charge transaction fees" + ); + assert!( + !details.balances.pre.is_empty(), + "native balances were recorded" + ); + assert_eq!( + details.balances.pre, details.balances.post, + "the calculator changes account data, not lamports" + ); + assert!(details.logs.iter().any(|line| line.contains("v42:"))); + assert!(details.compute_units > 0); + assert!( + details + .cpi + .as_ref() + .is_some_and(|groups| groups.iter().any(|group| !group.0.is_empty())), + "the nested expression retains its CPI trace" + ); + let returned = details.return_data.expect("CPI return data retained"); + assert_eq!(returned.program, v42_calculator_interface::ID.to_bytes()); + assert_eq!(returned.data.as_slice(), &value.to_le_bytes()); + } + + te.close().await; +} + +// A transaction that runs but errors resolves as a committed error result and +// leaves its output account untouched — the engine surfaces the failure through +// the outer Ok / inner Err split rather than dropping it. +#[tokio::test(flavor = "multi_thread")] +async fn failed_execution_surfaces_error_result() { + let te = TestEngine::new().await; + let output = store_v42(&te, 5, AccountMode::Ephemeral); + // MIN - 1 overflows the program's checked_sub before any write. + let ixs = [(E::lit(i64::MIN) - E::lit(1)).compose(output, &[])]; + + let error = te.execute(&ixs).await.expect_err("overflow yields an error result"); + // CalcError::Arithmetic = 6; its discriminants are stable for tests. + assert_eq!( + error, + TransactionError::InstructionError(0, InstructionError::Custom(6)), + "the program's own failure is surfaced, not a substitute" + ); + assert_eq!( + load_v42_data(&te, output), + Some(5), + "failed execution commits no writes" + ); + + te.close().await; +} + +// schedule returns before the transaction commits; the write still lands, and an +// account subscription (not a poll loop) observes it. +#[tokio::test(flavor = "multi_thread")] +async fn schedule_is_fire_and_forget() { + let te = TestEngine::new().await; + let output = store_v42(&te, 0, AccountMode::Ephemeral); + let mut updates = te.accounts().subscribe(output).await; + let ixs = [E::lit(7).compose(output, &[])]; + + te.schedule(&ixs).await; + + let account = updates.recv().await.expect("scheduled write reaches the subscriber"); + assert_eq!( + decode_v42(&account), + 7, + "scheduled transaction commits the write" + ); + + te.close().await; +} diff --git a/keeper/Cargo.toml b/keeper/Cargo.toml new file mode 100644 index 00000000..76a8a348 --- /dev/null +++ b/keeper/Cargo.toml @@ -0,0 +1,66 @@ +[package] +name = "magicblock-keeper" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "keeper" + +[features] +# Exposes `keeper::testkit`, the shared keeper-level test harness, as normal code +# so downstream crates can import it via a dev-dependency (no new crate needed). +testkit = ["accountsdb/testkit", "dep:solana-instruction", "ledger/testkit", "nucleus/testkit"] + +[dependencies] +accountsdb = { workspace = true } +ledger = { workspace = true } +nucleus = { workspace = true, features = ["config", "notifier", "runtime"] } + +ahash = { workspace = true } +arc-swap = { workspace = true } +derive_more = { workspace = true, features = ["from"] } +flume = { workspace = true } +oneshot = { workspace = true } +parking_lot = { workspace = true } +scc = { workspace = true } +serde = { workspace = true } +smallvec = { workspace = true } +tar = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt", "sync"] } +tracing = { workspace = true } +zstd = { workspace = true } + +agave-feature-set = { workspace = true, features = ["agave-unstable-api"] } +solana-account = { workspace = true, features = ["bincode"] } +solana-feature-gate-interface = { workspace = true, features = ["bincode"] } +solana-hash = { workspace = true } +solana-instruction = { workspace = true, optional = true } +solana-keypair = { workspace = true } +solana-message = { workspace = true } +solana-program-runtime = { workspace = true } +solana-pubkey = { workspace = true } +solana-sdk-ids = { workspace = true } +solana-signature = { workspace = true } +solana-signer = { workspace = true } +solana-svm = { workspace = true } +solana-sysvar = { workspace = true } +solana-transaction-error = { workspace = true } + +[dev-dependencies] +keeper = { workspace = true, features = ["testkit"] } +nucleus = { workspace = true, features = ["testkit"] } + +solana-instruction = { workspace = true } +solana-keypair = { workspace = true } +solana-signer = { workspace = true } +tokio = { workspace = true, features = ["macros"] } + +[lints] +workspace = true diff --git a/keeper/README.md b/keeper/README.md new file mode 100644 index 00000000..074a0957 --- /dev/null +++ b/keeper/README.md @@ -0,0 +1,79 @@ +# `magicblock-keeper` + +Keeper opens accountsdb and the ledger as one durable state boundary. It also +owns startup account seeding, read-side caches, and live subscription fanout. +Account routing remains in accountsdb and ledger retention remains in ledger. + +## Startup and recovery + +`KeeperBuilder::build` opens both stores and seeds active feature accounts, +native builtins, configured loader-v4 programs, caller-provided accounts, +authority funding, and sysvars. + +Accountsdb is restored from the newest retained snapshot when validation finds +corruption, its sealed superblock trails the ledger head, or its committed +transaction count trails the ledger's durable count. The accountsdb count is a +checkpoint high-water mark and may exceed the locally retained ledger count, +including after replication snapshot bootstrap. The original active tree is +saved until the restored snapshot validates. Engine replay is responsible for +advancing restored state from the successor of its sealed superblock through the +ledger tip and must finish with matching transaction counts when replay runs. + +`nucleus::config::BlockstoreParams` supplies the expected block time and +non-zero superblock interval used by pacing and cache TTL calculation. The +shared accountsdb, blockstore, and ledger parameters are defined by nucleus; +keeper consumes them when opening its durable stores and caches. + +## Authority + +`nucleus::config::Authority::local` is the keypair used for locally signed +messages. When `Authority::remote` is set, `Keeper::authority` returns that +immediate upstream identity instead of the local pubkey, while `Keeper::signer` +continues to return the local signer. Replication followers retain both values +across restart. + +The effective authority also identifies the engine's sponsor account. Keeper +creates this engine-local account only for an empty ledger, persists its spent +balance across restarts, and restores its initial balance on reset. Startup +rejects a non-empty deployment whose configured authority account is absent. + +## Superblock finalization + +`Keeper::finalize_superblock` snapshots accountsdb at the current ledger head, +computes the persisted-account checksum, appends the corresponding +`SuperblockSeal`, and archives the snapshot in the successor superblock +directory. + +Finalization requires exclusive account-store access. Engine obtains that +exclusivity through the sequencer and simulator barriers before calling it. + +## Synchronization + +`Keeper::sync(false)` flushes queued appends and accountsdb while keeping ledger +workers available, as required by replay and replication. `Keeper::sync(true)` +is the irreversible shutdown fence: it closes every reader after earlier queued +requests, flushes and closes the appender, then flushes accountsdb. + +## Caches and subscriptions + +Signature and recent-block caches use slot-based TTLs with lazy eviction on +insertion. The account cache is an LRU that also coordinates concurrent loads of +missing accounts. Only non-authoritative modes enter the eviction LRU; +delegated, ephemeral, and unresolved transient state remains outside it. + +Dedicated channels publish account and program updates, signature results, logs, +processed transactions, blocks, cache evictions, completed snapshots, and +service messages. Signatures have terminal oneshot fanout; persistent multicast +streams give each receiver a bounded queue and disconnect a receiver that falls +behind. Processed transactions, service messages, and cache evictions each have +one process-lifetime receiver and apply producer backpressure when full. +Append rejection notifies only its newest signature waiter, preserving older +waiters for an already accepted transaction; invalid-blockhash status is cached. + +## `testkit` + +The `testkit` feature exposes a keeper backed by throwaway directories plus v42 +account and transaction helpers, including persisted-metadata fault injection. +When enabled, Keeper's build script builds the v42 SBF artifact consumed by the +harness. Downstream tests enable the feature on their dev-dependency instead of +duplicating the setup. diff --git a/keeper/build.rs b/keeper/build.rs new file mode 100644 index 00000000..d285f3d9 --- /dev/null +++ b/keeper/build.rs @@ -0,0 +1,67 @@ +//! Builds the v42 calculator SBF program for runtime tests. + +use std::{env, io, path::PathBuf, process::Command}; + +const PROGRAM_DIR: &str = "programs/v42-calculator-program"; +const PROGRAM: &str = "programs/v42-calculator-program/Cargo.toml"; +const SO: &str = "v42_calculator_program.so"; + +fn main() -> Result<(), Box> { + // The embedded program is only compiled by `keeper::testkit`. + if env::var_os("CARGO_FEATURE_TESTKIT").is_none() { + return Ok(()); + } + + let manifest_dir = PathBuf::from( + env::var_os("CARGO_MANIFEST_DIR") + .ok_or_else(|| io::Error::other("CARGO_MANIFEST_DIR is not set"))?, + ); + let workspace = manifest_dir.parent().ok_or_else(|| { + io::Error::other(format!( + "CARGO_MANIFEST_DIR has no parent: {}", + manifest_dir.display() + )) + })?; + let manifest = workspace.join(PROGRAM); + let artifact = workspace.join("target/deploy").join(SO); + println!( + "cargo:rerun-if-changed={}", + workspace.join(PROGRAM_DIR).display() + ); + + let output = Command::new("cargo") + .arg("build-sbf") + .arg("--manifest-path") + .arg(&manifest) + .arg("--arch") + .arg("v3") + .current_dir(workspace) + // `cargo clippy` exports these wrappers pointing at `clippy-driver`; left in + // place they hijack the SBF toolchain's rustc, which can't resolve the + // `sbpf*-solana` target. Strip them so `build-sbf` uses its own toolchain. + .env_remove("RUSTC_WRAPPER") + .env_remove("RUSTC_WORKSPACE_WRAPPER") + .output() + .map_err(|e| io::Error::other(format!("failed to run `cargo build-sbf`: {e}")))?; + + if !output.status.success() { + return Err(io::Error::other(format!( + "`cargo build-sbf` failed with {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + .into()); + } + if !artifact.is_file() { + Err(io::Error::other(format!( + "missing SBF artifact: {}", + artifact.display() + )))?; + } + println!( + "cargo:rustc-env=V42_CALCULATOR_PROGRAM_SO={}", + artifact.display() + ); + Ok(()) +} diff --git a/keeper/src/accessor.rs b/keeper/src/accessor.rs new file mode 100644 index 00000000..5d729b96 --- /dev/null +++ b/keeper/src/accessor.rs @@ -0,0 +1,372 @@ +//! Namespaced keeper access APIs. + +use std::{ops::Deref, path::PathBuf, sync::Arc}; + +use accountsdb::{AccountEntry, AccountLoader, AccountsDB, AccountsDBError}; +use ledger::{ + LedgerRequestError, + request::*, + schema::{Event, SuperblockSeal, TransactionEntry}, +}; +use nucleus::{ + Slot, + ledger::{Block, BlockstorePosition}, + tls::{EncodedMessage, TlsManager}, +}; +use solana_account::{AccountSharedData, ReadableAccount}; +use solana_hash::Hash; +use solana_pubkey::Pubkey; +use solana_sdk_ids::sysvar; +use solana_signature::Signature; +use solana_svm::{ + transaction_execution_result::ExecutedTransaction, + transaction_processing_result::TransactionProcessingResult, +}; +use solana_sysvar::{ + clock::Clock, + slot_hashes::{SlotHashes, SysvarId}, +}; +use solana_transaction_error::TransactionError; +use tokio::sync::mpsc::Receiver; + +use crate::{ + FullTransaction, Keeper, ResolvedTransaction, + cache::{AccountCache, MissingAccount}, + error::Result, + subscriptions::TransactionLogs, + util::{execution_commit, request}, +}; + +/// Account operations namespace. +pub struct AccountsAccessor<'a> { + pub(crate) keeper: &'a Keeper, +} + +/// Cursor over accounts missing from local storage. +/// +/// Each item either gives the caller load ownership or waits for another +/// caller that already owns the load. +pub struct MissingAccounts<'a> { + index: usize, + accounts: &'a [Pubkey], + loader: AccountLoader<'a>, + cache: &'a Arc, +} + +impl<'a> Iterator for MissingAccounts<'a> { + type Item = MissingAccount; + /// Returns the next account that still needs resolution. + fn next(&mut self) -> Option { + loop { + let pubkey = self.accounts.get(self.index)?; + self.index += 1; + if self.loader.contains(pubkey).unwrap_or_default() { + self.cache.promote(pubkey); + continue; + } + return Some(self.cache.reserve(*pubkey)); + } + } +} + +impl<'a> AccountsAccessor<'a> { + /// Coordinates loading of accounts missing from local storage. + pub fn ensure(&'a self, accounts: &'a [Pubkey]) -> MissingAccounts<'a> { + MissingAccounts { + index: 0, + accounts, + loader: self.keeper.accountsdb.loader(), + cache: &self.keeper.caches.accounts, + } + } + + /// Returns recent transaction signatures that mention the account. + pub async fn signatures( + &self, + params: AccountSignaturesParams, + ) -> Result> { + Ok(request(self.keeper, params, ReadRequest::AccountSignatures).await??) + } + + /// Subscribes to updates for one account pubkey. + pub async fn subscribe(&self, account: Pubkey) -> Receiver { + self.keeper.subscriptions.accounts.subscribe(account).await + } + + /// Subscribes to account updates for accounts owned by `program`. + pub async fn subscribe_program(&self, program: Pubkey) -> Receiver { + self.keeper.subscriptions.programs.subscribe(program).await + } + + /// Subscribes as the sole receiver of account pubkeys evicted from the recent-load cache. + /// + /// Returns an error if the process-lifetime eviction receiver was already registered. + pub fn subscribe_evictions(&self) -> Result> { + self.keeper.caches.accounts.evictions.subscribe() + } + + /// Subscribes to completed accountsdb snapshot archives. + pub fn subscribe_snapshots(&self) -> Receiver { + self.keeper.subscriptions.snapshots.subscribe_sync(()) + } + + /// Updates durable `SlotHashes` and `Clock` sysvar accounts from `block`. + /// + /// If either sysvar account is absent, no account updates are stored. + pub fn update_sysvars(&self, block: Block) -> Result<()> { + let loader = self.loader(); + + let Some(mut hacc) = loader.load(&SlotHashes::id())? else { + return Ok(()); + }; + let mut hashes: SlotHashes = hacc.deserialize_data().map_err(AccountsDBError::from)?; + hashes.add(block.slot, block.hash); + hacc.serialize_data(&hashes).map_err(AccountsDBError::from)?; + let Some(mut cacc) = loader.load(&Clock::id())? else { + return Ok(()); + }; + let mut clock: Clock = cacc.deserialize_data().map_err(AccountsDBError::from)?; + clock.slot = block.slot; + clock.unix_timestamp = block.time; + cacc.serialize_data(&clock).map_err(AccountsDBError::from)?; + self.store(&[(SlotHashes::id(), hacc), (Clock::id(), cacc)]).map_err(Into::into) + } +} + +impl Deref for AccountsAccessor<'_> { + type Target = AccountsDB; + + fn deref(&self) -> &Self::Target { + &self.keeper.accountsdb + } +} + +/// Transaction operations namespace. +pub struct TransactionsAccessor<'a> { + pub(crate) keeper: &'a Keeper, +} + +impl<'a> TransactionsAccessor<'a> { + /// Loads the full retained transaction for `signature`. + pub async fn get(&self, signature: Signature) -> Result> { + Ok(request(self.keeper, signature, ReadRequest::Transaction).await??) + } + + /// Loads the retained execution status for `signature`. + pub async fn status(&self, signature: Signature) -> Result> { + if let Some(status) = self.keeper.caches.signatures.get(&signature) { + return Ok(status); + } + Ok(request(self.keeper, signature, ReadRequest::TransactionStatus).await??) + } + + /// Subscribes as the sole receiver of all processed transactions, including failures. + /// + /// Returns an error if the process-lifetime transaction receiver was already registered. + pub fn subscribe_processed(&self) -> Result> { + self.keeper.subscriptions.transactions.subscribe() + } + + /// Subscribes to status updates for one transaction signature. + pub async fn subscribe_signature( + &self, + signature: Signature, + ) -> oneshot::Receiver { + self.keeper.subscriptions.signatures.subscribe(signature).await + } + + /// Subscribes to log batches mentioning `account`. + pub async fn subscribe_logs(&self, account: Pubkey) -> Receiver> { + self.keeper.subscriptions.logs.subscribe(account).await + } + + /// Subscribes as the sole receiver of encoded service messages. + /// + /// Returns an error if the process-lifetime service receiver was already registered. + pub fn subscribe_service_messages(&self) -> Result> { + self.keeper.subscriptions.services.subscribe() + } + + /// Appends transaction bytes to the ledger, deduplicating by signature. + /// + /// Returns `Ok(true)` when the transaction was appended. On `Ok(false)`, + /// the latest signature subscriber receives `AlreadyProcessed` or + /// `BlockhashNotFound`. Execution details are appended later by + /// `commit_execution`. + pub async fn append(&self, transaction: &ResolvedTransaction) -> Result { + let caches = &self.keeper.caches; + let slot = caches.blocks.latest.load().slot + 1; + let signature = transaction.signatures()[0]; + let mut result = Ok(()); + if !caches.signatures.push(signature, None, slot) { + result = Err(TransactionError::AlreadyProcessed); + } else if !self.keeper.blocks().is_valid(transaction.recent_blockhash()) { + result = Err(TransactionError::BlockhashNotFound); + let status = TransactionStatus { result: result.clone(), slot }; + caches.signatures.update(&signature, Some(status)); + } + if result.is_err() { + let status = TransactionStatus { result, slot }; + self.keeper.subscriptions.signatures.send_last(&signature, &status); + return Ok(false); + } + let event = Event::Transaction(TransactionEntry { + signature, + payload: transaction.inner_data().clone(), + }); + self.keeper.ledger.appender.send_async(event).await?; + Ok(true) + } + + /// Commits execution metadata and publishes resulting account changes. + pub fn commit_execution(&self, mut txn: FullTransaction) -> Result<()> { + let subs = &self.keeper.subscriptions; + let commit = execution_commit(&mut txn); + self.keeper.ledger.appender.send(commit.event)?; + + if let Some(execution) = self.commit_state_transitions(&txn.execution.result)? { + let accounts = &execution.loaded_transaction.accounts; + let mut logs = None; + for (pubkey, acc) in accounts { + if subs.logs.contains(pubkey) { + let logs = logs.get_or_insert_with(|| { + Arc::new(TransactionLogs { + signature: commit.signature, + result: commit.status.result.clone(), + logs: Arc::clone(&commit.logs), + }) + }); + subs.logs.send(pubkey, logs); + } + if !acc.dirty() { + continue; + } + subs.accounts.send(pubkey, acc); + if subs.programs.contains(acc.owner()) { + let account = &(*pubkey, acc.clone()); + subs.programs.send(acc.owner(), account); + } + } + while let Some(msg) = TlsManager::dequeue() { + subs.services.blocking_send(msg); + } + } + subs.transactions.blocking_send(txn); + // Clear TLS unconditionally so unsent messages cannot leak into the next transaction. + TlsManager::clear(); + subs.signatures.send(&commit.signature, &commit.status); + self.keeper.caches.signatures.update(&commit.signature, Some(commit.status)); + Ok(()) + } + + /// Commits one accepted transaction to accountsdb, writing dirty accounts + /// only for successful execution and returning it for downstream fanout. + pub fn commit_state_transitions<'t>( + &self, + result: &'t TransactionProcessingResult, + ) -> Result> { + let execution = result.as_ref().ok().filter(|e| e.was_successful()); + let accounts = execution + .into_iter() + .flat_map(|execution| execution.loaded_transaction.accounts.iter()) + .filter(|(id, a)| a.dirty() && !sysvar::instructions::check_id(id)); + self.keeper.accountsdb.commit(accounts)?; + Ok(execution.map(|execution| &**execution)) + } +} + +/// Block operations namespace. +pub struct BlocksAccessor<'a> { + pub(crate) keeper: &'a Keeper, +} + +impl<'a> BlocksAccessor<'a> { + /// Loads a retained block at the requested detail level. + pub async fn get(&self, params: BlockParams) -> Result> { + Ok(request(self.keeper, params, ReadRequest::Block).await??) + } + + /// Returns the latest block boundary known to keeper. + pub fn latest(&self) -> Block { + **self.keeper.caches.blocks.latest.load() + } + + /// Returns the slot currently being built (one past the latest block). + pub fn current_slot(&self) -> Slot { + self.keeper.caches.blocks.latest.load().slot + 1 + } + + /// Returns whether `hash` is in the recent block hash cache (still valid). + pub fn is_valid(&self, hash: &Hash) -> bool { + self.keeper.caches.blocks.history.contains(hash) + } + + /// Subscribes to newly committed slots. + pub fn subscribe(&self) -> Receiver { + self.keeper.subscriptions.blocks.subscribe_sync(()) + } + + /// Publishes a completed block and advances block-derived account state. + /// + /// Replay skips the ledger append because the block is already stored. + pub fn append(&self, block: Block, replay: bool) -> Result<()> { + let event = Event::Block(block); + if !replay { + self.keeper.ledger.appender.send(event)?; + self.keeper.subscriptions.blocks.send(&(), &block); + } + self.keeper.caches.blocks.push(block); + self.keeper.accounts().update_sysvars(block)?; + self.keeper.accounts().set_slot(block.slot)?; + Ok(()) + } +} + +/// Superblock operations namespace +pub struct SuperblockAccessor<'a> { + pub(crate) keeper: &'a Keeper, +} + +impl SuperblockAccessor<'_> { + /// Id of the superblock accountsdb last sealed. + pub fn sealed(&self) -> SuperblockSeal { + SuperblockSeal { + id: self.keeper.accountsdb.superblock(), + checksum: self.keeper.accountsdb.checksum(), + transactions: self.keeper.accountsdb.transactions(), + } + } + + /// Returns the ledger root containing retained superblock directories. + pub fn directory(&self) -> &PathBuf { + &self.keeper.ledger.directory + } + + /// Follower's current durable blockstore position, reported to the leader at handshake. + pub fn position(&self) -> BlockstorePosition { + self.keeper.ledger.position() + } + + /// Enqueues a seal onto the append stream, sealing the current superblock and + /// rotating to the next. Used by a follower applying a seal received from the leader. + pub fn append(&self, seal: SuperblockSeal) -> Result<()> { + let event = Event::Superblock(seal); + self.keeper.ledger.appender.send(event)?; + self.sync(false) + } + + /// Installs a snapshot seal and adopts its cumulative transaction count. + pub fn bootstrap(&self, seal: SuperblockSeal) -> Result<()> { + let event = Event::Bootstrap(seal); + self.keeper.ledger.appender.send(event)?; + self.sync(false) + } + + /// Blocks until every queued append event has been flushed and made durable. + pub fn sync(&self, is_final: bool) -> Result<()> { + let (response, ack) = oneshot::channel(); + let event = Event::Sync { response, is_final }; + self.keeper.ledger.appender.send(event)?; + ack.recv().map_err(LedgerRequestError::from)?.map_err(Into::into) + } +} diff --git a/keeper/src/builder.rs b/keeper/src/builder.rs new file mode 100644 index 00000000..d50f48c8 --- /dev/null +++ b/keeper/src/builder.rs @@ -0,0 +1,325 @@ +//! Keeper construction, recovery, and startup account seeding. + +use std::{ + collections::HashMap, + fs::{self, File}, + sync::Arc, + time::Duration, +}; + +use accountsdb::{AccountEntry, AccountsDB, AccountsDBError, BackupOp, SnapshotError}; +use agave_feature_set::FeatureSet; +use ledger::{ + Ledger, LedgerHandle, + request::{BlockDetails, BlockParams, ReadRequest, RequestPayload}, +}; +use nucleus::{ + Slot, + config::{AccountsDBParams, Authority, BlockstoreParams, LedgerParams}, + ledger::{ACCOUNTSDB_SNAPSHOT_FILE, Block}, + shutdown::ShutdownManager, +}; +use serde::Serialize; +use solana_account::{AccountBuilder, AccountMode, AccountSharedData, ReadableAccount}; +use solana_feature_gate_interface::Feature; +use solana_program_runtime::invoke_context::BuiltinFunctionWithContext; +use solana_pubkey::Pubkey; +use solana_sdk_ids::sysvar; +#[allow(deprecated)] +use solana_sysvar::fees::Fees; +use solana_sysvar::{ + clock::Clock, + epoch_rewards::EpochRewards, + epoch_schedule::EpochSchedule, + last_restart_slot::LastRestartSlot, + rent::Rent, + slot_hashes::{SlotHashes, SysvarId}, +}; +use tracing::{error, info, warn}; + +use crate::{ + Keeper, + cache::{AccountCache, BlocksCache, Caches, ExpiringCache}, + error::Result, + metrics, + subscriptions::Subscriptions, +}; + +/// Initial balance assigned to the authority account that sponsors account creation. +pub(crate) const SPONSOR_INIT_BALANCE: u64 = u64::MAX / 2; +/// Maximum number of recent hashes retained by the `SlotHashes` sysvar. +const SLOTHASH_ENTRIES: usize = 512; +/// Wall-clock retention window for recently processed signatures. +const SIGNATURE_CACHE_WINDOW: Duration = Duration::from_secs(75); +/// Wall-clock retention window for recently produced blocks. +const BLOCK_CACHE_WINDOW: Duration = Duration::from_secs(60); + +/// Builder for keeper directories and cache timing. +#[derive(Clone)] +pub struct KeeperBuilder { + /// Local signer and optional remote authority represented by the engine. + pub authority: Authority, + /// Accounts database storage parameters. + pub accountsdb: AccountsDBParams, + /// Ledger storage parameters. + pub ledger: LedgerParams, + /// Block production timing and superblock sealing parameters. + pub blockstore: BlockstoreParams, + /// Native builtin program ids to seed as executable accounts. + pub builtins: HashMap, + /// Upgradeable program accounts to seed, paired as `(program id, ELF bytes)`. + pub programs: HashMap>, + /// Plain accounts to seed into storage before startup completes. + pub accounts: HashMap, + /// Rent parameters used to size seeded accounts and the Rent sysvar. + pub rent: Rent, +} + +impl KeeperBuilder { + /// Open durable stores, recover accounts from the latest snapshot if needed, and wire caches. + pub async fn build(mut self, shutdown: &mut ShutdownManager) -> Result { + let ledger = Ledger::init(&self.ledger.directory, self.ledger.size_limit, shutdown)?; + let accountsdb = self.accountsdb(&ledger)?; + let (block, featureset) = self.prepopulate(&accountsdb, &ledger).await?; + let caches = self.caches(block); + metrics::init(); + Ok(Keeper { + authority: self.authority, + featureset, + rent: self.rent, + accountsdb, + ledger, + caches, + subscriptions: Subscriptions::new(shutdown), + }) + } + + /// Seeds accounts needed before the engine starts serving reads. + async fn prepopulate( + &mut self, + accountsdb: &AccountsDB, + ledger: &LedgerHandle, + ) -> Result<(Block, FeatureSet)> { + let mut accounts = Vec::new(); + let featureset = self.seed_featureset(&mut accounts)?; + self.seed_programs(&mut accounts)?; + let block = self.seed_sysvars(accountsdb, ledger, &mut accounts).await?; + let authority = self.authority.pubkey(); + if accountsdb.loader().load(&authority)?.is_none() { + let sponsor = AccountBuilder::default() + .lamports(SPONSOR_INIT_BALANCE) + .mode(AccountMode::Ephemeral); + accounts.push((authority, sponsor.build())); + } + accounts.extend(self.accounts.drain()); + accountsdb.store(&accounts)?; + Ok((block, featureset)) + } + + /// Builds read-side caches using blocktime-derived slot TTLs. + fn caches(&self, latest: Block) -> Caches { + let blocktime = self.blockstore.blocktime; + let ttl = |window: Duration| window.div_duration_f64(blocktime).ceil() as Slot; + let blocks = BlocksCache::new(latest, ttl(BLOCK_CACHE_WINDOW)); + let signatures = ExpiringCache::new(ttl(SIGNATURE_CACHE_WINDOW)); + let accounts = Arc::new(AccountCache::new(self.accountsdb.lru_capacity)); + + Caches { signatures, blocks, accounts } + } + + /// Activates the engine's required feature gates at slot 0, seeds a feature + /// account for each, and returns the resulting [`FeatureSet`]. + fn seed_featureset(&self, accounts: &mut Vec) -> Result { + let mut featureset = FeatureSet::default(); + [ + agave_feature_set::curve25519_syscall_enabled::ID, + agave_feature_set::curve25519_restrict_msm_length::ID, + agave_feature_set::enable_poseidon_syscall::ID, + agave_feature_set::enable_sbpf_v3_deployment_and_execution::ID, + agave_feature_set::virtual_address_space_adjustments::ID, + agave_feature_set::syscall_parameter_address_restrictions::ID, + agave_feature_set::get_sysvar_syscall_enabled::ID, + agave_feature_set::ed25519_program_enabled::ID, + agave_feature_set::secp256k1_program_enabled::ID, + agave_feature_set::enable_secp256r1_precompile::ID, + ] + .iter() + .for_each(|f| featureset.activate(f, 0)); + for (&id, &slot) in featureset.active() { + let feature = &Feature { activated_at: Some(slot) }; + let account = self.account(feature, &solana_feature_gate_interface::ID)?; + accounts.push((id, account.build())); + } + Ok(featureset) + } + + /// Seeds builtin and upgradeable program accounts. + fn seed_programs(&self, accounts: &mut Vec) -> Result<()> { + for &builtin in self.builtins.keys() { + let account = self.account(&(), &solana_sdk_ids::native_loader::ID)?; + let account = account.executable(true).build(); + accounts.push((builtin, account)); + } + + for (&program, elf) in &self.programs { + let lamports = self.rent.minimum_balance(elf.len()); + let account = AccountBuilder::default() + .lamports(lamports) + .mode(AccountMode::System) + .owner(solana_sdk_ids::loader_v4::ID) + .executable(true) + .data(elf.clone()); + accounts.push((program, account.build())); + } + Ok(()) + } + + /// Seeds sysvars derived from retained ledger state and keeper config. + /// + /// Returns the latest available block, resolved from accountsdb or ledger + async fn seed_sysvars( + &self, + accountsdb: &AccountsDB, + ledger: &LedgerHandle, + accounts: &mut Vec, + ) -> Result { + let slot = accountsdb.slot(); + let loader = accountsdb.loader(); + let mut last_block = None; + if let Some(hashes) = loader.load(&SlotHashes::id())? { + let hashes = hashes.deserialize_data::().map_err(AccountsDBError::from)?; + // `SlotHashes` is ordered newest-first, so the latest block is `first` + if let Some(&(slot, hash)) = hashes.first() { + let parent = &slot.saturating_sub(1); + let parent = hashes.get(parent).copied().unwrap_or_default(); + let time = self.blocktime(ledger, slot).await?; + last_block.replace(Block { slot, hash, time, parent }); + } + } else { + let range = slot.saturating_sub(SLOTHASH_ENTRIES as u64)..slot + 1; + let (payload, handle) = RequestPayload::new(range); + ledger.reader.send(ReadRequest::BlockRange(payload))?; + + let mut hashes = SlotHashes::new(&[Default::default(); SLOTHASH_ENTRIES]); + for block in handle.recv_timeout().await?? { + hashes.add(block.slot, block.hash); + last_block.replace(block); + } + let acc = self.account(&hashes, &sysvar::ID)?; + accounts.push((SlotHashes::id(), acc.build())); + } + + let block = last_block.unwrap_or_default(); + // Set the clock slot one ahead from the last + let clock = Clock { + slot: block.slot + 1, + unix_timestamp: block.time, + ..Default::default() + }; + accounts.push((Clock::id(), self.account(&clock, &sysvar::ID)?.build())); + accounts.push((Rent::id(), self.account(&self.rent, &sysvar::ID)?.build())); + #[allow(deprecated)] + accounts.push(( + Fees::id(), + self.account(&Fees::default(), &sysvar::ID)?.build(), + )); + accounts.push(( + sysvar::last_restart_slot::id(), + self.account(&LastRestartSlot::default(), &sysvar::ID)?.build(), + )); + accounts.push(( + sysvar::instructions::id(), + self.account(&(), &sysvar::ID)?.build(), + )); + accounts.push(( + EpochSchedule::id(), + self.account(&EpochSchedule::default(), &sysvar::ID)?.build(), + )); + accounts.push(( + EpochRewards::id(), + self.account(&EpochRewards::default(), &sysvar::ID)?.build(), + )); + Ok(block) + } + + /// Builds a rent-exempt system account containing a serialized sysvar-like state. + fn account(&self, state: &S, owner: &Pubkey) -> Result { + let account = + AccountSharedData::new_data(0, state, owner).map_err(AccountsDBError::from)?; + let lamports = self.rent.minimum_balance(account.data().len()); + Ok(AccountBuilder::from(account).lamports(lamports).mode(AccountMode::System)) + } + + /// Returns the retained block time for the given slot. + async fn blocktime(&self, ledger: &LedgerHandle, slot: Slot) -> Result { + let (payload, handle) = RequestPayload::new(BlockParams { + slot, + details: BlockDetails::None, + }); + ledger.reader.send(ReadRequest::Block(payload))?; + Ok(handle.recv_timeout().await??.map(|r| r.block().time).unwrap_or_default()) + } + /// Opens accountsdb, restoring the newest archived snapshot after corruption. + /// + /// A restored store trails the ledger tip — snapshots are archived at sealed + /// superblocks, not at the tip — so the returned accountsdb is only + /// guaranteed to validate, not to be current. Catching it back up is the + /// caller's job. + fn accountsdb(&self, ledger: &LedgerHandle) -> Result { + let mut backup = None; + loop { + let mut accountsdb = AccountsDB::new(&self.accountsdb.directory)?; + // Seal N opens ledger head N+1, so accountsdb is current at head-1. + let expected = ledger.head().saturating_sub(1); + let restored = backup.is_some(); + let lagging = accountsdb.superblock() < expected; + let count_lagging = accountsdb.transactions() < ledger.transactions(); + match accountsdb.validate() { + Ok(()) if restored || (!lagging && !count_lagging) => { + let reclaimed = accountsdb.compact()?; + info!( + lagging, + count_lagging, reclaimed, "accountsdb validation succeeded" + ); + backup.map(fs::remove_dir_all).transpose()?; + return Ok(accountsdb); + } + validation @ (Err(AccountsDBError::Corruption) | Ok(())) => { + if restored { + error!(?validation, "restored accountsdb is corrupt"); + accountsdb.backup(BackupOp::Restore)?; + return Err(SnapshotError::Missing.into()); + } + warn!( + ?validation, + lagging, count_lagging, "state inconsistency detected" + ); + backup.replace(accountsdb.backup(BackupOp::Save)?); + if let Err(error) = self.unarchive(ledger) { + accountsdb.backup(BackupOp::Restore)?; + return Err(error); + } + } + Err(other) => return Err(other.into()), + } + } + } + + /// Restores the first retained accountsdb snapshot found, from newest to oldest. + fn unarchive(&self, ledger: &LedgerHandle) -> Result<()> { + info!("restoring accountsdb from latest available snapshot"); + for superblock in ledger.iter() { + let src = superblock.directory.join(ACCOUNTSDB_SNAPSHOT_FILE); + if !src.exists() { + continue; + } + let dst = AccountsDB::directory(&self.accountsdb.directory); + let file = File::open(src)?; + let mut tar = tar::Archive::new(zstd::Decoder::new(file)?); + tar.unpack(dst)?; + info!(directory = ?superblock.directory, "restored accountsdb snapshot"); + return Ok(()); + } + Err(SnapshotError::Missing.into()) + } +} diff --git a/keeper/src/cache.rs b/keeper/src/cache.rs new file mode 100644 index 00000000..13526cac --- /dev/null +++ b/keeper/src/cache.rs @@ -0,0 +1,250 @@ +//! Read-side caches owned by keeper. + +use std::{collections::VecDeque, hash::Hash, sync::Arc}; + +use ahash::RandomState; +use arc_swap::ArcSwap; +use ledger::request::TransactionStatus; +use nucleus::{Slot, ledger::Block, notifier::EventNotifier}; +use parking_lot::Mutex; +use scc::{HashCache, HashMap, hash_map::Entry}; +use solana_account::AccountMode; +use solana_hash::Hash as SolanaHash; +use solana_pubkey::Pubkey; +use solana_signature::Signature; + +use crate::{ + metrics, + subscriptions::{Subscription, Unicast}, +}; + +pub(crate) struct Caches { + /// Recent signature statuses keyed by transaction signature. + pub(crate) signatures: ExpiringCache>, + /// Recent block hashes and latest block boundary. + pub(crate) blocks: BlocksCache, + /// Account recency and missing-load coordination. + pub(crate) accounts: Arc, +} + +/// Account access cache along with missing-account load reservations. +pub(crate) struct AccountCache { + /// Committed account loads, ordered by recent access. + pub(crate) lru: HashCache, + /// In-flight account loads keyed by account pubkey. + pub(crate) reservations: HashMap, RandomState>, + /// Pubkeys evicted when a committed load displaces a cold account. + pub(crate) evictions: Unicast, +} + +impl AccountCache { + /// Creates missing-load coordination with the requested recent-load capacity. + pub(crate) fn new(capacity: usize) -> Self { + let lru = HashCache::with_capacity_and_hasher(256, capacity, Default::default()); + Self { + lru, + reservations: Default::default(), + evictions: Unicast::new(32, Subscription::Evictions), + } + } +} + +/// Load guard owned by the task responsible for one missing account. +pub struct AccountLoad { + /// Account this guard is responsible for loading. + pub pubkey: Pubkey, + /// Cache reservation released on commit or drop. + cache: Option>, +} + +/// Wait handle for an account currently being loaded by another task. +pub struct AccountWait(Pubkey, Arc); + +/// Coordination state for one account missing from local storage. +pub enum MissingAccount { + /// Caller owns the load; dropping the guard releases waiters. + Load(AccountLoad), + /// Another caller is already loading the account. + Wait(AccountWait), +} + +impl AccountLoad { + /// Completes the load, wakes waiters, and tracks non-authoritative modes for + /// eviction. + pub async fn complete(mut self, mode: AccountMode) { + let Some((cache, notifier)) = self.release() else { + return; + }; + notifier.notify(true); + if mode.authoritative() { + return; + } + if let Ok(Some(evicted)) = cache.lru.put_sync(self.pubkey, ()) { + metrics::account_cache_eviction(); + cache.evictions.send(evicted.0).await; + } else { + metrics::account_cache_insert() + } + } + + fn release(&mut self) -> Option<(Arc, Arc)> { + let cache = self.cache.take()?; + let (_, notifier) = cache.reservations.remove_sync(&self.pubkey)?; + Some((cache, notifier)) + } +} + +impl Drop for AccountLoad { + /// Cancels the load reservation and wakes waiters without caching. + fn drop(&mut self) { + let Some((_, notifier)) = self.release() else { + return; + }; + notifier.notify(false); + } +} + +impl AccountWait { + /// Waits for the active loader and returns the pubkey plus whether it committed. + pub async fn wait(self) -> (Pubkey, bool) { + let result = self.1.notified().await; + (self.0, result) + } +} + +impl AccountCache { + /// Promotes `pubkey` only if it has already been committed to the cache. + pub(crate) fn promote(&self, pubkey: &Pubkey) { + self.lru.get_sync(pubkey); + } + + /// Reserves a missing account load or returns a waiter for the active load. + pub(crate) fn reserve(self: &Arc, pubkey: Pubkey) -> MissingAccount { + match self.reservations.entry_sync(pubkey) { + Entry::Occupied(e) => { + metrics::account_resolution_race(); + MissingAccount::Wait(AccountWait(pubkey, e.get().clone())) + } + Entry::Vacant(e) => { + let notifier = Arc::new(EventNotifier::default()); + e.insert_entry(notifier); + MissingAccount::Load(AccountLoad { + pubkey, + cache: Some(self.clone()), + }) + } + } + } +} + +/// Block lookup cache with a lock-free latest-block pointer. +pub(crate) struct BlocksCache { + /// Latest block boundary known at keeper startup or after updates. + pub(crate) latest: ArcSwap, + /// Recent block hash to slot lookups. + pub(crate) history: ExpiringCache, +} + +impl BlocksCache { + /// Creates a block cache seeded with the current latest block. + pub(crate) fn new(block: Block, ttl: Slot) -> Self { + let cache = Self { + latest: ArcSwap::new(block.into()), + history: ExpiringCache::new(ttl), + }; + cache.history.push(block.hash, block.slot, block.slot); + cache + } + + /// Records `block` as the latest and adds its hash to the recent history. + pub(crate) fn push(&self, block: Block) { + self.latest.store(block.into()); + self.history.push(block.hash, block.slot, block.slot); + metrics::block_hash_entries(self.history.len()); + } +} + +/// Concurrent cache with slot-based lazy eviction. +/// +/// Entries are evicted only when another entry is pushed. Re-inserting an +/// existing key leaves its value and expiry slot unchanged. +pub(crate) struct ExpiringCache { + /// Cached values by key. + index: HashMap, + /// Expiry order used for lazy eviction. + queue: Mutex>>, + /// Number of slots each entry lives after insertion. + ttl: Slot, +} + +struct ExpiringRecord { + key: K, + expires: Slot, +} + +impl ExpiringCache { + /// Creates a cache whose entries live for `ttl` slots after insertion. + pub(crate) fn new(ttl: Slot) -> Self { + Self { + index: HashMap::default(), + queue: Default::default(), + ttl, + } + } + + /// Insert a key and evict entries expired at `slot`. + /// + /// Returns `false` if `key` already exists. Existing values and expiry slots + /// are left unchanged. + pub(crate) fn push(&self, key: K, value: V, slot: Slot) -> bool { + let mut queue = self.queue.lock(); + // Lazily evict expired entries from the front of the queue. + while let Some(expired) = queue.pop_front_if(|e| e.expired(slot)) { + self.index.remove_sync(&expired.key); + } + + match self.index.entry_sync(key) { + Entry::Occupied(_) => false, + Entry::Vacant(v) => { + v.insert_entry(value); + queue.push_back(ExpiringRecord::new(key, slot + self.ttl)); + true + } + } + } + + /// Entry count, including expired entries no sweep has reached yet, so the + /// metrics gauges fed from this can read above the live entry count. + pub(crate) fn len(&self) -> usize { + self.index.len() + } + + /// May yield an entry already past its expiry slot but not yet swept. + pub(crate) fn get(&self, key: &K) -> Option { + self.index.read_sync(key, |_, v| v.clone()) + } + + /// May report an entry already past its expiry slot but not yet swept. + pub(crate) fn contains(&self, key: &K) -> bool { + self.index.contains_sync(key) + } + + /// Replaces the value for `key`, leaving its expiry slot untouched: an + /// update does not extend the entry's life. No-op if `key` is absent. + pub(crate) fn update(&self, key: &K, value: V) { + let Some(mut entry) = self.index.get_sync(key) else { + return; + }; + entry.insert(value); + } +} + +impl ExpiringRecord { + fn new(key: K, expires: Slot) -> Self { + Self { key, expires } + } + + fn expired(&self, instant: Slot) -> bool { + instant >= self.expires + } +} diff --git a/keeper/src/error.rs b/keeper/src/error.rs new file mode 100644 index 00000000..d8c67c59 --- /dev/null +++ b/keeper/src/error.rs @@ -0,0 +1,48 @@ +//! Keeper error types. + +use accountsdb::{AccountsDBError, SnapshotError}; +use derive_more::From; +use flume::SendError; +use ledger::{LedgerError, LedgerRequestError, request::ReadRequest, schema::Event}; +use nucleus::shutdown::Service; + +/// Errors produced while initializing, finalizing, or serving keeper state. +#[derive(Debug, thiserror::Error, From)] +pub enum KeeperError { + /// Filesystem or archive IO failed. + #[error("io: {0}")] + IO(#[source] std::io::Error), + /// Accounts database operation failed. + #[error("accountsdb: {0}")] + AccountsDB(#[source] AccountsDBError), + /// Snapshot creation, restore, or archive operation failed. + #[error("snapshot: {0}")] + Snapshot(#[source] SnapshotError), + /// Ledger initialization or append failed. + #[error("ledger: {0}")] + Ledger(#[source] LedgerError), + /// A background service is no longer reachable, so the request was dropped. + #[error("service became unavailable: {0:?}")] + ServiceUnavailable(Service), + /// Ledger read request failed before a response was received. + #[error("ledger read request: {0}")] + LedgerRequest(#[source] LedgerRequestError), + /// A process-lifetime unicast stream was already registered. + #[error("subscription already registered: {0}")] + SubscriptionRegistered(&'static str), +} + +impl From> for KeeperError { + fn from(_: SendError) -> Self { + Self::ServiceUnavailable(Service::LedgerAppender) + } +} + +impl From> for KeeperError { + fn from(_: SendError) -> Self { + Self::ServiceUnavailable(Service::LedgerReader) + } +} + +/// Result type used by keeper APIs. +pub type Result = std::result::Result; diff --git a/keeper/src/lib.rs b/keeper/src/lib.rs new file mode 100644 index 00000000..8e54b129 --- /dev/null +++ b/keeper/src/lib.rs @@ -0,0 +1,245 @@ +#![doc = include_str!("../README.md")] + +use std::{ + fs::{self, File}, + path::PathBuf, + sync::Arc, + thread, +}; + +use agave_feature_set::FeatureSet; +use solana_account::AccountBuilder; +use solana_hash::Hash; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use tokio::sync::mpsc; +use tracing::{error, info, warn}; + +use accountsdb::{AccountsDB, SnapshotError}; +use ledger::{ + LedgerHandle, Superblock, + request::{ReadRequest, ReplayHandle, ReplayParams, RequestPayload}, + schema::Event, +}; +use nucleus::{ + Slot, + config::Authority, + ledger::{ACCOUNTSDB_SNAPSHOT_FILE, SuperblockSeal}, +}; +use solana_sysvar::rent::Rent; + +use crate::{ + accessor::{AccountsAccessor, BlocksAccessor, SuperblockAccessor, TransactionsAccessor}, + builder::SPONSOR_INIT_BALANCE, + cache::Caches, + error::Result, + metrics::Operation, + subscriptions::Subscriptions, +}; + +pub use cache::{AccountLoad, AccountWait, MissingAccount}; +/// Re-exported so callers can name what `Keeper::transactions().status()` returns. +pub use ledger::request::TransactionStatus; +pub use nucleus::runtime::{ + ExecutionRecord, FullTransaction, ResolvedTransaction, TransactionView, +}; + +mod accessor; +pub mod builder; +mod cache; +pub mod error; +mod metrics; +mod subscriptions; +mod util; + +#[cfg(feature = "testkit")] +pub mod testkit; + +#[cfg(test)] +mod tests; + +/// Owns the durable state and live access helpers for the execution engine. +pub struct Keeper { + /// Local signer and optional remote authority represented by this engine. + authority: Authority, + /// Active feature set governing runtime behavior. + featureset: FeatureSet, + /// Rent parameters applied during execution. + rent: Rent, + /// Account state store. + accountsdb: AccountsDB, + /// Ledger worker handles and append path. + ledger: LedgerHandle, + /// Read-side caches shared by accessors. + caches: Caches, + /// Subscription fanout maps for live updates. + subscriptions: Arc, +} + +impl Keeper { + /// Returns the account operations namespace. + pub fn accounts(&self) -> AccountsAccessor<'_> { + AccountsAccessor { keeper: self } + } + + /// Returns the transaction operations namespace. + pub fn transactions(&self) -> TransactionsAccessor<'_> { + TransactionsAccessor { keeper: self } + } + + /// Returns the block operations namespace. + pub fn blocks(&self) -> BlocksAccessor<'_> { + BlocksAccessor { keeper: self } + } + + /// Returns the superblock operations namespace. + pub fn superblocks(&self) -> SuperblockAccessor<'_> { + SuperblockAccessor { keeper: self } + } + + /// Returns the configured remote authority, or the local identity when unset. + pub fn authority(&self) -> Pubkey { + self.authority.pubkey() + } + + /// Returns the local signer, which may differ from [`Self::authority`]. + pub fn signer(&self) -> &Keypair { + &self.authority.local + } + + /// Returns the latest block hash + pub fn blockhash(&self) -> Hash { + self.blocks().latest().hash + } + + /// Borrows the handle used for direct ledger reads, appends, and + /// durable-position subscriptions. + pub fn ledger(&self) -> &LedgerHandle { + &self.ledger + } + + /// Streams retained ledger entries after accountsdb's sealed superblock up to + /// the ledger tip, used to rebuild state after snapshot restoration. Returns + /// `None` when accountsdb is already current by slot and transaction count. + pub async fn replay(&self) -> Result> { + let ledger_slot = self.ledger.tip().unwrap_or_default(); + let accountsdb_slot = self.accountsdb.slot(); + let ledger_txns = self.ledger.transactions(); + let accountsdb_txns = self.accountsdb.transactions(); + if accountsdb_slot >= ledger_slot && accountsdb_txns >= ledger_txns { + return Ok(None); + }; + let (tx, rx) = mpsc::channel(16); + let params = ReplayParams { + tx, + superblock: self.accountsdb.superblock(), + }; + let (payload, response) = RequestPayload::new(params); + let handle = ReplayHandle { rx, response }; + self.ledger.reader.send_async(ReadRequest::Replay(payload)).await?; + warn!( + accountsdb_slot, + ledger_slot, accountsdb_txns, ledger_txns, "starting ledger replay" + ); + Ok(Some(handle)) + } + + /// Seal the current superblock and archive the matching accounts snapshot. + /// + /// Must run only when no account store can race the snapshot export; the + /// in-body `SAFETY` note relies on this exclusivity. + pub fn finalize_superblock(&self) -> Result<()> { + let _timer = metrics::time(Operation::FinalizeSuperblock); + let head = self.ledger.head(); + let next = head + 1; + // SAFETY: `snapshot` requires exclusive write access to accountsdb, + // i.e. no store operation may race the export. `finalize_superblock` + // is only run when there're no concurrent mutations taking place + let snapshot = unsafe { self.accountsdb.snapshot(head) }?; + let checksum = self.accountsdb.checksum(); + let transactions = self.accountsdb.transactions(); + let seal = SuperblockSeal { id: head, checksum, transactions }; + self.superblocks().append(seal)?; + let dir = Superblock::init_dir(&self.ledger.directory, next)?; + self.archive(snapshot, dir)?; + info!(head, "finalized superblock"); + Ok(()) + } + + /// Resolved once from the seeded feature accounts at startup and fixed for + /// the engine's lifetime — features never activate mid-run. + pub fn features(&self) -> &FeatureSet { + &self.featureset + } + + /// Supplied by the builder at startup and fixed for the engine's lifetime; + /// the same parameters that sized the seeded accounts. + pub fn rent(&self) -> &Rent { + &self.rent + } + + /// Waits for queued ledger work to become durable, then synchronously + /// flushes persisted account storage. Volatile accounts are not serialized. + /// + /// A final sync closes every ledger reader and the appender. It is + /// irreversible and must only be used during coordinated shutdown. + pub fn sync(&self, is_final: bool) -> Result<()> { + if is_final { + for _ in 0..self.ledger.reader.receiver_count() { + self.ledger.reader.send(ReadRequest::Shutdown)?; + } + } + self.superblocks().sync(is_final)?; + self.accountsdb.flush(true).map_err(Into::into) + } + + /// Appends a reset marker before discarding chain-synchronized volatile + /// accounts and restoring the authority sponsor's initial balance. + /// + /// Internal system accounts and persisted engine-authoritative state remain + /// available. + pub fn reset(&self, slot: Slot) -> Result<()> { + self.ledger.appender.send(Event::Reset(slot))?; + self.accountsdb.reset(); + let authority = self.authority(); + let account = self.accounts().loader().load(&authority)?; + if let Some(account) = account { + let acc = AccountBuilder::from(account).lamports(SPONSOR_INIT_BALANCE); + self.accounts().store(&[(authority, acc.build())])?; + } + info!(slot, "reset volatile state"); + Ok(()) + } + + /// Spawns a background thread that tars and zstd-compresses the accountsdb + /// snapshot at `snapshot` into `target`, removing the snapshot afterward. + fn archive( + &self, + snapshot: PathBuf, + target: PathBuf, + ) -> std::result::Result<(), SnapshotError> { + let path = target.join(ACCOUNTSDB_SNAPSHOT_FILE); + let tmp = target.join(format!("{ACCOUNTSDB_SNAPSHOT_FILE}.tmp")); + let dst = File::options().write(true).create(true).truncate(true).open(&tmp)?; + let subscriptions = self.subscriptions.clone(); + thread::Builder::new().name("snapshot-archiver".into()).spawn(move || { + { + let _timer = metrics::time(Operation::ArchiveSnapshot); + let mut tar = tar::Builder::new(zstd::Encoder::new(dst, 0)?); + tar.append_dir_all(".", &snapshot)?; + { + let archive = tar.into_inner()?.finish()?; + metrics::snapshot_size(archive.metadata()?.len()); + archive.sync_data()?; + } + // Rename only after sync so replication cannot serve a partial archive. + fs::rename(tmp, &path)?; + fs::remove_dir_all(snapshot)?; + subscriptions.snapshots.send(&(), &path); + Ok::<(), SnapshotError>(()) + } + .inspect_err(|error| error!(?error, "snapshot archival failed")) + })?; + Ok(()) + } +} diff --git a/keeper/src/metrics.rs b/keeper/src/metrics.rs new file mode 100644 index 00000000..aa9654be --- /dev/null +++ b/keeper/src/metrics.rs @@ -0,0 +1,152 @@ +//! Prometheus metrics for keeper. + +use std::sync::OnceLock; + +use nucleus::metrics::{self as metric, OperationTimer}; +use nucleus::metrics::{ + IntCounter, IntCounterVec, IntGauge, MetricOperation, MetricSpec, OperationCounters, +}; + +use crate::subscriptions::Subscription; + +/// Process-wide keeper metrics registered in the default Prometheus registry. +static METRICS: OnceLock = OnceLock::new(); + +/// Operation latency histogram recorded in microseconds. +const OPERATION_TIME: MetricSpec = MetricSpec { + name: "keeper_operation_duration_micros", + help: "Keeper operation duration distribution in microseconds.", +}; +/// Account load cache entries. +const ACCOUNT_CACHE_ENTRIES: MetricSpec = MetricSpec { + name: "keeper_account_cache_entries", + help: "Current account load cache entries.", +}; +/// Block hash cache entries. +const BLOCK_HASH_CACHE_ENTRIES: MetricSpec = MetricSpec { + name: "keeper_block_hash_cache_entries", + help: "Current block hash cache entries.", +}; +/// Most recently completed snapshot archive size. +const SNAPSHOT_SIZE: MetricSpec = MetricSpec { + name: "keeper_snapshot_size", + help: "Most recently completed snapshot archive size in bytes.", +}; +/// Account cache eviction counter. +const ACCOUNT_CACHE_EVICTIONS: MetricSpec = MetricSpec { + name: "keeper_account_cache_evictions", + help: "Account cache evictions.", +}; +/// Account resolution conflict counter. +const ACCOUNT_RESOLUTION_RACES: MetricSpec = MetricSpec { + name: "keeper_account_resolution_races", + help: "Account resolution race conditions.", +}; +/// Slow multicast consumer disconnection counter. +const SLOW_CONSUMER_DISCONNECTS: MetricSpec = MetricSpec { + name: "keeper_subscription_slow_consumer_disconnects", + help: "Multicast receivers disconnected because their queue was full.", +}; + +/// Keeper operation used as a low-cardinality operation label. +#[derive(Clone, Copy)] +pub(crate) enum Operation { + /// Superblock finalization path. + FinalizeSuperblock, + /// Idle subscription cleanup path. + Cleanup, + /// Snapshot tar/zstd archival path. + ArchiveSnapshot, +} + +impl MetricOperation for Operation { + /// Returns the Prometheus label value for this operation. + fn label(self) -> &'static str { + match self { + Operation::FinalizeSuperblock => "finalize_superblock", + Operation::Cleanup => "cleanup", + Operation::ArchiveSnapshot => "archive_snapshot", + } + } +} + +/// Registers keeper metrics once in the default Prometheus registry. +pub(crate) fn init() { + METRICS.get_or_init(Default::default); +} + +/// Starts an operation timer that records latency when the returned guard drops. +pub(crate) fn time(op: Operation) -> OperationTimer<'static> { + op.time(METRICS.get().map(|m| &m.operations)) +} + +/// Records an account cache insertion that increases current occupancy. +pub(crate) fn account_cache_insert() { + metric::with_metrics(&METRICS, |m| m.account_cache_entries.inc()); +} + +/// Refreshes block hash cache entry gauge. +pub(crate) fn block_hash_entries(count: usize) { + metric::with_metrics(&METRICS, |m| { + m.block_hash_cache_entries.set(metric::gauge_value(count)) + }); +} + +/// Records the most recently completed snapshot archive size in bytes. +pub(crate) fn snapshot_size(bytes: u64) { + metric::with_metrics(&METRICS, |m| { + m.snapshot_size.set(metric::gauge_value(bytes)) + }); +} + +/// Records one account cache eviction. +pub(crate) fn account_cache_eviction() { + metric::with_metrics(&METRICS, |m| m.account_cache_evictions.inc()); +} + +/// Records one account resolution race condition. +pub(crate) fn account_resolution_race() { + metric::with_metrics(&METRICS, |m| m.account_resolution_race.inc()); +} + +/// Records one receiver disconnected because its queue was full. +pub(crate) fn slow_consumer_disconnect(subscription: Subscription) { + metric::with_metrics(&METRICS, |m| { + m.slow_consumer_disconnects.with_label_values(&[subscription.label()]).inc() + }); +} + +/// Owns all Prometheus collectors registered by keeper. +struct Metrics { + /// Runtime operation duration and completion counters. + operations: OperationCounters, + /// Account cache entry gauge. + account_cache_entries: IntGauge, + /// Block hash cache entry gauge. + block_hash_cache_entries: IntGauge, + /// Most recently completed snapshot archive size in bytes. + snapshot_size: IntGauge, + /// Account cache eviction counter. + account_cache_evictions: IntCounter, + /// Account resolution race conditions counter. + account_resolution_race: IntCounter, + /// Slow consumer disconnections labeled by subscription stream. + slow_consumer_disconnects: IntCounterVec, +} +impl Default for Metrics { + /// Builds collectors and registers them in the default Prometheus registry. + fn default() -> Self { + Self { + operations: OperationCounters::new(OPERATION_TIME), + account_cache_entries: metric::gauge(ACCOUNT_CACHE_ENTRIES, 0), + block_hash_cache_entries: metric::gauge(BLOCK_HASH_CACHE_ENTRIES, 0), + snapshot_size: metric::gauge(SNAPSHOT_SIZE, 0), + account_cache_evictions: metric::counter(ACCOUNT_CACHE_EVICTIONS, 0), + account_resolution_race: metric::counter(ACCOUNT_RESOLUTION_RACES, 0), + slow_consumer_disconnects: metric::counter_vec( + SLOW_CONSUMER_DISCONNECTS, + &["subscription"], + ), + } + } +} diff --git a/keeper/src/subscriptions.rs b/keeper/src/subscriptions.rs new file mode 100644 index 00000000..1fe11373 --- /dev/null +++ b/keeper/src/subscriptions.rs @@ -0,0 +1,317 @@ +//! Live read-side notification channels. + +use std::{ + hash::Hash, + path::PathBuf, + sync::{Arc, OnceLock}, + time::Duration, +}; + +use accountsdb::AccountEntry; +use ahash::RandomState; +use ledger::{request::TransactionStatus, schema::Block}; +use nucleus::{ + shutdown::{Service, ShutdownHandle, ShutdownManager, ShutdownReason}, + tls::EncodedMessage, +}; +use scc::HashMap; +use smallvec::SmallVec; +use solana_account::AccountSharedData; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_transaction_error::TransactionResult; +use tokio::{ + sync::mpsc::{self, error::TrySendError}, + time::{MissedTickBehavior, interval}, +}; + +use crate::{ + FullTransaction, + error::{KeeperError, Result}, + metrics::{self, Operation}, +}; + +type MpscSenders = SmallVec<[mpsc::Sender; 1]>; +type OneshotSenders = SmallVec<[oneshot::Sender; 1]>; + +/// Stable metric identity for a subscription stream. +#[derive(Clone, Copy)] +pub(crate) enum Subscription { + Accounts, + Programs, + Logs, + Blocks, + Transactions, + Snapshots, + Services, + Evictions, +} + +impl Subscription { + pub(crate) const fn label(self) -> &'static str { + match self { + Self::Accounts => "accounts", + Self::Programs => "programs", + Self::Logs => "logs", + Self::Blocks => "blocks", + Self::Transactions => "transactions", + Self::Snapshots => "snapshots", + Self::Services => "services", + Self::Evictions => "evictions", + } + } +} + +/// Live notification channels owned by keeper. +pub(crate) struct Subscriptions { + /// Account updates keyed by account pubkey. + pub(crate) accounts: Multicast, + /// Program account updates keyed by owner pubkey. + pub(crate) programs: Multicast, + /// Signature status updates keyed by transaction signature. + pub(crate) signatures: MulticastOneshot, + /// Log notifications keyed by mentioned program or account pubkey. + pub(crate) logs: Multicast>, + /// Newly committed blocks. + pub(crate) blocks: Multicast<(), Block>, + /// All committed transactions for the sole stream consumer. + pub(crate) transactions: Unicast, + /// Accountsdb snapshot archive completions. + pub(crate) snapshots: Multicast<(), PathBuf>, + /// Encoded service messages for the sole stream consumer. + pub(crate) services: Unicast, +} + +impl Subscriptions { + /// Builds subscription channels and starts cleanup for idle keyed entries. + pub(crate) fn new(shutdown: &mut ShutdownManager) -> Arc { + let subscriptions = Arc::new(Self { + accounts: Multicast::new(8, Subscription::Accounts), + programs: Multicast::new(16, Subscription::Programs), + signatures: Default::default(), + logs: Multicast::new(8, Subscription::Logs), + blocks: Multicast::new(32, Subscription::Blocks), + transactions: Unicast::new(1024, Subscription::Transactions), + snapshots: Multicast::new(4, Subscription::Snapshots), + services: Unicast::new(64, Subscription::Services), + }); + let shutdown = shutdown.handle(Service::SubscriptionsCleanup); + tokio::spawn(cleanup(subscriptions.clone(), shutdown)); + subscriptions + } + + async fn cleanup(&self) { + self.accounts.cleanup().await; + self.programs.cleanup().await; + self.signatures.cleanup().await; + self.logs.cleanup().await; + self.blocks.cleanup().await; + self.snapshots.cleanup().await; + } +} + +/// Composite log notification sent to log subscribers. +#[derive(Clone)] +pub struct TransactionLogs { + /// First transaction signature. + pub signature: Signature, + /// Runtime transaction result (carries the error on failure). + pub result: TransactionResult<()>, + /// Log lines emitted during execution. + pub logs: Arc>, +} + +/// One process-lifetime bounded receiver. +pub(crate) struct Unicast { + sender: OnceLock>, + capacity: usize, + subscription: Subscription, +} + +impl Unicast { + pub(crate) const fn new(capacity: usize, subscription: Subscription) -> Self { + Self { + sender: OnceLock::new(), + capacity, + subscription, + } + } + + /// Creates the process-lifetime receiver, rejecting every later subscriber. + pub(crate) fn subscribe(&self) -> Result> { + let (tx, rx) = mpsc::channel(self.capacity); + self.sender + .set(tx) + .map_err(|_| KeeperError::SubscriptionRegistered(self.subscription.label()))?; + Ok(rx) + } + + /// Sends asynchronously, waiting until the receiver has capacity. + pub(crate) async fn send(&self, value: V) { + let Some(sender) = self.sender.get() else { + return; + }; + let _ = sender.send(value).await; + } + + /// Sends from a synchronous worker, waiting until the receiver has capacity. + pub(crate) fn blocking_send(&self, value: V) { + let Some(sender) = self.sender.get() else { + return; + }; + let _ = sender.blocking_send(value); + } +} + +/// Persistent per-key fanout over one bounded queue per receiver. +pub(crate) struct Multicast { + senders: HashMap, RandomState>, + capacity: usize, + subscription: Subscription, +} + +impl Multicast +where + K: Eq + Hash, +{ + pub(crate) fn new(capacity: usize, subscription: Subscription) -> Self { + Self { + senders: Default::default(), + capacity, + subscription, + } + } + + /// Adds a receiver for `key` with its own bounded queue. + pub(crate) async fn subscribe(&self, key: K) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(self.capacity); + self.senders.entry_async(key).await.or_default().push(tx); + rx + } + + /// Adds a receiver synchronously when the public accessor cannot await. + pub(crate) fn subscribe_sync(&self, key: K) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(self.capacity); + self.senders.entry_sync(key).or_default().push(tx); + rx + } + + /// Returns whether `key` has any live receivers. + pub(crate) fn contains(&self, key: &K) -> bool { + let mut contains = false; + self.senders.remove_if_sync(key, |senders| { + senders.retain(|sender| !sender.is_closed()); + contains = !senders.is_empty(); + !contains + }); + contains + } + + /// Drops closed receivers and keys that no longer have receivers. + async fn cleanup(&self) { + self.senders + .retain_async(|_, senders| { + senders.retain(|sender| !sender.is_closed()); + !senders.is_empty() + }) + .await; + } +} + +impl Multicast +where + K: Eq + Hash, + V: Clone, +{ + /// Fans out without blocking, disconnecting receivers whose queues are full. + pub(crate) fn send(&self, key: &K, value: &V) { + self.senders.remove_if_sync(key, |senders| { + senders.retain(|sender| match sender.try_send(value.clone()) { + Ok(()) => true, + Err(TrySendError::Full(_)) => { + metrics::slow_consumer_disconnect(self.subscription); + false + } + Err(TrySendError::Closed(_)) => false, + }); + senders.is_empty() + }); + } +} + +/// Terminal per-key fanout over one oneshot channel per receiver. +pub(crate) struct MulticastOneshot(HashMap, RandomState>); + +impl Default for MulticastOneshot { + fn default() -> Self { + Self(Default::default()) + } +} + +impl MulticastOneshot +where + K: Eq + Hash, +{ + /// Adds a receiver for the terminal value associated with `key`. + pub(crate) async fn subscribe(&self, key: K) -> oneshot::Receiver { + let (tx, rx) = oneshot::channel(); + self.0.entry_async(key).await.or_default().push(tx); + rx + } + + /// Drops closed receivers and keys that no longer have receivers. + async fn cleanup(&self) { + self.0 + // Keep closed positions while any receiver is live so `send_last` + // cannot mistake an older subscription for the newest one. + .retain_async(|_, senders| senders.iter().any(|sender| !sender.is_closed())) + .await; + } +} + +impl MulticastOneshot +where + K: Eq + Hash, + V: Clone, +{ + /// Sends the terminal value only to the most recently added receiver. + pub(crate) fn send_last(&self, key: &K, value: &V) { + let Some(mut senders) = self.0.get_sync(key) else { + return; + }; + let sender = senders.pop(); + if senders.is_empty() { + let _ = senders.remove_entry(); + } + if let Some(sender) = sender { + let _ = sender.send(value.clone()); + } + } + + /// Removes `key` and sends its terminal value to every current receiver. + pub(crate) fn send(&self, key: &K, value: &V) { + let Some((_, senders)) = self.0.remove_sync(key) else { + return; + }; + for sender in senders { + let _ = sender.send(value.clone()); + } + } +} + +/// Drops abandoned multicast senders after their receivers are gone. +async fn cleanup(subscriptions: Arc, mut shutdown: ShutdownHandle) { + let mut ticker = interval(Duration::from_secs(60)); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + tokio::select! { + biased; + _ = shutdown.signalled() => break, + _ = ticker.tick() => { + let _timer = metrics::time(Operation::Cleanup); + subscriptions.cleanup().await; + } + } + } + shutdown.terminate(ShutdownReason::Signalled); +} diff --git a/keeper/src/testkit.rs b/keeper/src/testkit.rs new file mode 100644 index 00000000..0595ed25 --- /dev/null +++ b/keeper/src/testkit.rs @@ -0,0 +1,273 @@ +//! Keeper-level test harness shared by keeper and processor test suites. +//! +//! Builds a real [`Keeper`] over throwaway directories with the canonical test +//! parameters (retention disabled, 400 ms blocktime, superblock 16), and exposes +//! the loadable v42 calculator program guaranteed by `build.rs`. The low-level, +//! engine-agnostic builders (transactions, blocks, tempdirs) are re-exported from +//! [`nucleus::testkit`]. Compiled only under the `testkit` feature (or a crate's +//! own `cfg(test)`), so it never reaches release builds. +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::{ + collections::HashMap, + fs::{self, File}, + num::NonZeroU64, + path::{Path, PathBuf}, + sync::Arc, + time::Duration, +}; + +use accountsdb::{AccountsDB, STORAGE_FILE}; +use derive_more::Deref; +use nucleus::{ + config::{AccountsDBParams, BlockstoreParams, LedgerParams}, + ledger::ACCOUNTSDB_SNAPSHOT_FILE, + shutdown::ShutdownManager, + testkit::signed_view as compose_view, +}; +use solana_account::{AccountBuilder, AccountMode, ReadableAccount}; +use solana_hash::Hash; +use solana_instruction::{AccountMeta, Instruction}; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_signer::Signer; +use solana_sysvar::rent::Rent; + +pub use nucleus::testkit::{ + TempDir, V42_ID, WireVersion, block, init_tracing, patterned_bytes, + sign_versioned_instructions, tempdir, transaction, v42_padded_value, v42_sum, +}; +use tokio::time; + +use crate::{Keeper, ResolvedTransaction, TransactionView, builder::KeeperBuilder}; + +/// The v42 calculator ELF, built and located by `keeper/build.rs`. +pub const V42_PROGRAM_ELF: &[u8] = include_bytes!(env!("V42_CALCULATOR_PROGRAM_SO")); +/// Slots sealed into each superblock by the standard test engine. +pub const SUPERBLOCK: NonZeroU64 = NonZeroU64::new(4).unwrap(); + +/// Throwaway on-disk homes for the accountsdb and ledger stores. +/// +/// The directories must outlive every keeper opened over them — the stores keep +/// their files open/mmapped — which is why the recovery tests hold `Dirs` across +/// a full close-and-reopen cycle. +pub struct Dirs { + /// Accounts database directory. + pub accounts: TempDir, + /// Ledger directory. + pub ledger: TempDir, +} + +impl Default for Dirs { + fn default() -> Self { + Self { + accounts: tempdir(), + ledger: tempdir(), + } + } +} + +/// A keeper builder over `dirs` with retention disabled and a 100 ms blocktime. +/// +/// `builtins` and `accounts` default to empty and `programs` holds only v42; +/// individual tests fill the rest as needed. [`TestKeeper::new`] is the seeded +/// path, adding a funded payer on top. +pub fn keeper_builder(dirs: &Dirs) -> KeeperBuilder { + let mut programs = HashMap::new(); + programs.insert(V42_ID, V42_PROGRAM_ELF.to_vec()); + init_tracing(); + + KeeperBuilder { + authority: Keypair::new().into(), + accountsdb: AccountsDBParams { + directory: dirs.accounts.path().to_owned(), + lru_capacity: 256, + }, + ledger: LedgerParams { + directory: dirs.ledger.path().to_owned(), + size_limit: u64::MAX, + }, + blockstore: BlockstoreParams { + blocktime: Duration::from_millis(100), + superblock: SUPERBLOCK, + }, + builtins: Default::default(), + programs, + accounts: Default::default(), + rent: Rent::default(), + } +} + +/// A built keeper together with the directories and shutdown manager keeping its +/// background services alive. Derefs to [`Keeper`] for accessor calls. +/// +/// The keeper is seeded at construction with the v42 program and one funded +/// payer, so keeper-backed suites can build and run transactions immediately +/// instead of re-loading the ELF or re-storing a signer per test. +#[derive(Deref)] +pub struct TestKeeper { + /// Directories backing this keeper, returned by [`Self::close`] so a test can + /// reopen over the same on-disk state. + pub dirs: Dirs, + /// Lifecycle manager owning the keeper's background services; exposed so + /// tests can register their own services against the same shutdown. + pub shutdown: ShutdownManager, + #[deref] + keeper: Arc, +} + +impl TestKeeper { + /// Builds a keeper on fresh directories seeded with v42 and a funded payer. + pub async fn new() -> Self { + Self::with(Dirs::default()).await + } + + /// [`Self::new`] over `dirs`, which may already hold state from an earlier + /// keeper closed over the same directories. + pub async fn with(dirs: Dirs) -> Self { + let mut builder = keeper_builder(&dirs); + let payer = Keypair::new(); + builder.accounts.insert( + payer.pubkey(), + AccountBuilder::default().lamports(1_000_000).build(), + ); + Self::from_builder(dirs, builder).await + } + + /// Builds a keeper from a caller-configured builder. + /// + /// `dirs` must own the directories referenced by `builder` and outlive the + /// resulting keeper. Unlike [`Self::with`], nothing is seeded beyond what the + /// builder already carries. + pub async fn from_builder(dirs: Dirs, builder: KeeperBuilder) -> Self { + let mut shutdown = ShutdownManager::default(); + let keeper = Arc::new(builder.build(&mut shutdown).await.unwrap()); + Self { dirs, shutdown, keeper } + } + + /// Flushes durable state, stops every background service, and returns the + /// directories for reopen. + /// + /// The flush republishes a valid accountsdb checksum, so a test that wants a + /// corrupt store must call [`corrupt`] on the returned directories *after* + /// this, never before. + pub async fn close(self) -> Dirs { + let Self { mut shutdown, keeper, dirs } = self; + keeper.sync(true).unwrap(); + shutdown.terminate().await; + dirs + } +} + +/// A signed, resolved no-op transaction and its first signature, built the same +/// way the sequencer resolves inbound transactions. +pub fn signed_tx() -> (Signature, ResolvedTransaction) { + let (signature, bytes) = transaction(&[]); + let view = TransactionView::try_new_sanitized(bytes, true).unwrap(); + let resolved = + ResolvedTransaction::try_new(view, Some(Default::default()), &Default::default()).unwrap(); + (signature, resolved) +} + +/// A resolved transaction whose account metadata matches `accounts`. +/// +/// Each tuple is `(pubkey, writable)`. The transaction is fully sanitized and +/// resolved so scheduling sees the same account flags the keeper resolution path +/// would produce. A fresh random program id per call keeps the referenced account +/// set disjoint from other transactions under test. +pub fn resolved(accounts: &[(Pubkey, bool)]) -> ResolvedTransaction { + let payer = Keypair::new(); + let program = Pubkey::new_unique(); + let metas = accounts + .iter() + .map(|(key, writable)| { + if *writable { + AccountMeta::new(*key, false) + } else { + AccountMeta::new_readonly(*key, false) + } + }) + .collect(); + let ix = Instruction::new_with_bytes(program, &[], metas); + let (_signature, view) = compose_view(&payer, [ix], Hash::default()); + ResolvedTransaction::try_new(view, Some(Default::default()), &Default::default()).unwrap() +} + +/// Configures a v42 account carrying an 8-byte little-endian `i64` and twice +/// its rent-exempt minimum, leaving one reserve available for transfer tests. +pub fn v42_builder(value: i64, mode: AccountMode) -> AccountBuilder { + AccountBuilder::default() + .lamports(Rent::default().minimum_balance(8) * 2) + .owner(V42_ID) + .mode(mode) + .data(value.to_le_bytes().to_vec()) +} + +/// Stores a funded v42 `i64` account in `mode` and returns its pubkey. +pub fn store_v42(keeper: &Keeper, value: i64, mode: AccountMode) -> Pubkey { + let key = Pubkey::new_unique(); + keeper.accounts().store(&[(key, v42_builder(value, mode).build())]).unwrap(); + key +} + +/// Reads the little-endian `i64` payload of a v42 account, or `None` if absent. +pub fn load_v42_data(keeper: &Keeper, key: Pubkey) -> Option { + keeper.accounts().loader().read(&key, decode_v42).unwrap() +} + +/// Reads the lamport balance of a stored v42 account, or `None` if absent. +pub fn load_v42_lamports(keeper: &Keeper, key: Pubkey) -> Option { + keeper.accounts().loader().read(&key, ReadableAccount::lamports).unwrap() +} + +/// Signs `instruction` into the sanitized transaction view consumed by services. +pub fn signed_view( + keeper: &Keeper, + payer: Option<&Keypair>, + instruction: Instruction, +) -> (Signature, TransactionView) { + let payer = payer.unwrap_or(keeper.signer()); + compose_view(payer, [instruction], keeper.blockhash()) +} + +/// Returns the archived accountsdb snapshot path under any retained superblock, +/// or `None` when no superblock directory holds one yet. +pub fn archived_snapshot(keeper: &Keeper) -> Option { + fs::read_dir(&keeper.ledger.directory) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.path().join(ACCOUNTSDB_SNAPSHOT_FILE)) + .find(|p| p.exists()) +} + +/// Waits until a subscribed detached snapshot archiver reports completion. +pub async fn await_archive(keeper: &Keeper) -> PathBuf { + let mut rx = keeper.accounts().subscribe_snapshots(); + time::timeout(Duration::from_secs(8), rx.recv()) + .await + .expect("snapshot archives in time") + .unwrap() +} + +/// Overwrites one `u64` metadata word in the closed persisted store. +/// +/// `DatabaseMeta` starts with version at offset 0, checksum at 8, slot at 16, +/// superblock at 24, and committed transaction count at 32. +/// +/// This must be the last write to the store: `flush(true)` recomputes and +/// republishes the checksum, so this must run *after* the keeper is closed, never +/// against a live one. +pub fn corrupt(root: &Path, offset: u64, value: u64) { + use std::io::{Seek, SeekFrom, Write}; + let path = AccountsDB::directory(root).join(STORAGE_FILE); + let mut file = File::options().write(true).open(&path).unwrap(); + file.seek(SeekFrom::Start(offset)).unwrap(); + file.write_all(&value.to_ne_bytes()).unwrap(); + file.flush().unwrap(); +} + +/// Decodes the little-endian `i64` payload stored in a v42 account. +pub fn decode_v42(account: &impl ReadableAccount) -> i64 { + i64::from_le_bytes(account.data()[..8].try_into().expect("v42 account holds an i64")) +} diff --git a/keeper/src/tests/caches.rs b/keeper/src/tests/caches.rs new file mode 100644 index 00000000..99fd3557 --- /dev/null +++ b/keeper/src/tests/caches.rs @@ -0,0 +1,120 @@ +//! Read-side cache primitives keeper owns: the slot-based `ExpiringCache` and the +//! `AccountCache` missing-load coordination. + +use std::sync::Arc; + +use solana_account::{AccountBuilder, AccountMode}; +use solana_pubkey::Pubkey; + +use super::TestKeeper; +use crate::cache::{AccountCache, AccountLoad, AccountWait, ExpiringCache, MissingAccount}; + +// `ExpiringCache` evicts lazily on push, never on read; re-inserting an existing +// key is a no-op; `update` replaces only present values. +#[test] +fn expiring_cache_lazy_eviction() { + // ttl = 2 slots: a key pushed at slot s expires at s + 2. + let cache: ExpiringCache = ExpiringCache::new(2); + + assert!(cache.push(1, 10, 0)); // inserted, expires at slot 2 + assert!(!cache.push(1, 99, 0)); // re-insert of an existing key is a no-op + assert_eq!( + cache.get(&1), + Some(10), + "value left unchanged by the re-insert" + ); + + // Eviction runs only on push: at slot 5 the entry is well past its expiry but + // stays readable until the next push sweeps the queue. + assert!(cache.contains(&1)); + assert_eq!(cache.get(&1), Some(10)); + + // A push at slot 5 first evicts everything expired at 5 (key 1), then inserts. + assert!(cache.push(2, 20, 5)); + assert!(!cache.contains(&1), "expired key swept on the next push"); + assert_eq!(cache.get(&2), Some(20)); + + // `update` replaces a present value and no-ops for an absent key. + cache.update(&2, 21); + assert_eq!(cache.get(&2), Some(21)); + cache.update(&404, 0); + assert!(!cache.contains(&404)); + + // A key re-admitted after expiry is a fresh insert again. + assert!(cache.push(1, 11, 5)); + assert_eq!(cache.get(&1), Some(11)); +} + +// Two callers racing on the same missing account get exactly one loader and one +// waiter; committing caches the account while dropping the load guard does not. +#[tokio::test] +async fn account_load_release_paths_wake_waiters() { + use AccountMode::*; + let modes = [ReadOnly, Placeholder, Delegated, Ephemeral, Transient, System]; + for mode in modes { + for commit in [true, false] { + let cache = Arc::new(AccountCache::new(256)); + let pk = Pubkey::new_unique(); + let (load, wait) = reserve_load_and_wait(&cache, pk); + + let waiter = tokio::spawn(async move { wait.wait().await }); + if commit { + load.complete(mode).await; + } else { + drop(load) + } + + assert_eq!( + waiter.await.unwrap(), + (pk, commit), + "waiter returns the load outcome" + ); + let tracked = matches!(mode, ReadOnly | Placeholder | System); + assert_eq!(cache.lru.get_sync(&pk).is_some(), tracked && commit); + assert!(matches!(cache.reserve(pk), MissingAccount::Load(_))); + } + } +} + +// `ensure` is the production seam over `AccountCache`: it skips accounts already +// resident in storage (promoting them) and hands back a coordination item only +// for the ones missing, with the first caller owning the load. +#[tokio::test] +async fn ensure_reserves_only_missing_accounts() { + let keeper = TestKeeper::new().await; + let present = Pubkey::new_unique(); + let missing = Pubkey::new_unique(); + keeper + .accounts() + .store(&[(present, AccountBuilder::default().lamports(1).build())]) + .unwrap(); + + // The accessor must outlive the iterator that borrows it. + let accounts = keeper.accounts(); + let reserved: Vec<_> = accounts.ensure(&[present, missing]).collect(); + + // The resident account is skipped entirely; only the missing one surfaces, + // and the first caller to reach it owns the load. + assert_eq!(reserved.len(), 1, "only the missing account is reserved"); + let MissingAccount::Load(load) = &reserved[0] else { + panic!("first reservation of a missing account owns the load"); + }; + assert_eq!(load.pubkey, missing); + + // The reservation stays live while the load guard is held, so a concurrent + // `ensure` of the same account waits instead of racing a second load. + let again: Vec<_> = accounts.ensure(&[missing]).collect(); + assert!(matches!(again.as_slice(), [MissingAccount::Wait(_)])); + + keeper.close().await; +} + +fn reserve_load_and_wait(cache: &Arc, pk: Pubkey) -> (AccountLoad, AccountWait) { + let MissingAccount::Load(load) = cache.reserve(pk) else { + panic!("first reservation must own the load"); + }; + let MissingAccount::Wait(wait) = cache.reserve(pk) else { + panic!("concurrent reservation must wait"); + }; + (load, wait) +} diff --git a/keeper/src/tests/mod.rs b/keeper/src/tests/mod.rs new file mode 100644 index 00000000..11a3dfae --- /dev/null +++ b/keeper/src/tests/mod.rs @@ -0,0 +1,11 @@ +//! Keeper integration and unit tests. +//! +//! These cover the composition layer keeper owns — startup seeding, corruption +//! recovery, the read-side caches, and subscription fanout — and deliberately +//! avoid re-testing the accountsdb/ledger internals already covered below it. + +mod caches; +mod recovery; +mod subscriptions; + +use crate::testkit::{TestKeeper, signed_tx}; diff --git a/keeper/src/tests/recovery.rs b/keeper/src/tests/recovery.rs new file mode 100644 index 00000000..ee18a988 --- /dev/null +++ b/keeper/src/tests/recovery.rs @@ -0,0 +1,132 @@ +//! Startup seeding, corruption recovery + +use solana_account::{AccountBuilder, AccountMode, ReadableAccount}; +use solana_pubkey::Pubkey; +use solana_sdk_ids::{loader_v4, sysvar}; +use solana_sysvar::{ + clock::Clock, epoch_schedule::EpochSchedule, rent::Rent, slot_hashes::SysvarId, +}; + +use super::TestKeeper; +use crate::testkit::{Dirs, archived_snapshot, await_archive, corrupt, keeper_builder}; + +// Startup seeds the engine's required feature gates, the configured upgradeable +// programs, and the sysvars, with the exact ownership/rent/clock-offset shape the +// rest of the engine assumes. +#[tokio::test] +async fn seeds_features_programs_and_sysvars() { + let dirs = Dirs::default(); + let mut builder = keeper_builder(&dirs); + let program = Pubkey::new_unique(); + let elf = vec![1u8, 2, 3, 4, 5, 6, 7, 8]; + builder.programs.insert(program, elf.clone()); + let keeper = TestKeeper::from_builder(dirs, builder).await; + let rent = Rent::default(); + let accounts = keeper.accounts(); + let loader = accounts.loader(); + + // The engine's required curve25519/precompile/sbpf/sysvar gates are all + // active at slot 0, and every active feature is backed by a rent-exempt + // feature-gate-owned account. + let required = [ + agave_feature_set::curve25519_syscall_enabled::ID, + agave_feature_set::enable_sbpf_v3_deployment_and_execution::ID, + agave_feature_set::syscall_parameter_address_restrictions::ID, + agave_feature_set::get_sysvar_syscall_enabled::ID, + agave_feature_set::ed25519_program_enabled::ID, + agave_feature_set::secp256k1_program_enabled::ID, + ]; + for id in required { + assert_eq!( + keeper.features().active().get(&id), + Some(&0), + "required gate active at slot 0" + ); + } + for (&id, &slot) in keeper.features().active() { + assert_eq!(slot, 0, "features activate at slot 0"); + let acc = loader.load(&id).unwrap().expect("feature account seeded"); + assert_eq!(acc.owner(), &solana_feature_gate_interface::ID); + assert!(acc.lamports() >= rent.minimum_balance(acc.data().len())); + } + + // The upgradeable program account carries its ELF verbatim, is executable, + // owned by loader_v4 (not the BPF upgradeable loader), and rent-exempt. + // Builtins are seeded through the same path with an executable native-loader + // account, so they share this shape. + let acc = loader.load(&program).unwrap().expect("program seeded"); + assert!(acc.executable()); + assert_eq!(acc.owner(), &loader_v4::ID); + assert_eq!(acc.data(), elf.as_slice()); + assert_eq!(acc.lamports(), rent.minimum_balance(elf.len())); + + // The Clock is seeded one slot ahead of the last block; a fresh ledger's last + // block defaults to slot 0, so the clock starts at slot 1. + let clock: Clock = loader + .load(&Clock::id()) + .unwrap() + .expect("clock seeded") + .deserialize_data() + .unwrap(); + assert_eq!(clock.slot, 1); + + // Rent and EpochSchedule sysvars are present and sysvar-owned. + for id in [Rent::id(), EpochSchedule::id()] { + let acc = loader.load(&id).unwrap().expect("sysvar seeded"); + assert_eq!(acc.owner(), &sysvar::ID); + } + drop(loader); + keeper.close().await; +} + +// A corrupt accountsdb on open is restored from the newest archived snapshot, +// and the saved corrupt tree is discarded once the restored store revalidates. +// +// The marker takes a distinct value in each state the reopen could land on, so +// the assertion separates all three: 1 is the older snapshot, 2 the newest, and +// 3 lives only in persisted state (stored after the last archive, so no snapshot +// holds it). Recovery must yield 2 — reading 3 back would mean the corruption +// went undetected and nothing was restored at all. +#[tokio::test] +async fn recovers_the_newest_snapshot() { + let marker = Pubkey::new_unique(); + let dirs = Dirs::default(); + let builder = keeper_builder(&dirs); + let keeper = TestKeeper::from_builder(dirs, builder.clone()).await; + + // First snapshot captures marker == 1. + store_marker(&keeper, marker, 1); + keeper.finalize_superblock().expect("first finalize"); + await_archive(&keeper).await; + assert!( + archived_snapshot(&keeper).is_some(), + "snapshot archived under superblock" + ); + // Second snapshot, in a later superblock, captures marker == 2. + store_marker(&keeper, marker, 2); + keeper.finalize_superblock().expect("second finalize"); + await_archive(&keeper).await; + // Past every archive: this value is what an un-restored store would keep. + store_marker(&keeper, marker, 3); + let dirs = keeper.close().await; + + // Corruption must follow the close, whose flush would otherwise republish a + // valid checksum over the poisoned word. + corrupt(dirs.accounts.path(), 8, 0xABAB_ABAB_ABAB_ABAB); + + let keeper = TestKeeper::from_builder(dirs, builder).await; + keeper.accounts().validate().expect("restored store validates"); + let restored = keeper.accounts().loader().load(&marker).unwrap().expect("marker restored"); + assert_eq!(restored.lamports(), 2, "newest snapshot wins"); + // The corrupt tree saved for inspection is removed on successful recovery. + assert!(!keeper.dirs.accounts.path().join("CURRENT.bkp").exists()); + + keeper.close().await; +} + +/// Stores the recovery marker account at `lamports`, the value each snapshot +/// captures and recovery must bring back. +fn store_marker(keeper: &TestKeeper, marker: Pubkey, lamports: u64) { + let account = AccountBuilder::default().lamports(lamports).mode(AccountMode::Delegated); + keeper.accounts().store(&[(marker, account.build())]).unwrap(); +} diff --git a/keeper/src/tests/subscriptions.rs b/keeper/src/tests/subscriptions.rs new file mode 100644 index 00000000..ee063695 --- /dev/null +++ b/keeper/src/tests/subscriptions.rs @@ -0,0 +1,121 @@ +//! Subscription fanout primitives, transaction-append dedup + +use std::sync::Arc; + +use super::{TestKeeper, signed_tx}; +use crate::{ + ResolvedTransaction, + subscriptions::{Multicast, MulticastOneshot, Subscription, Unicast}, +}; +use nucleus::testkit::{V42_ID, signed_view}; +use solana_hash::Hash; +use solana_instruction::Instruction; +use solana_keypair::Keypair; +use solana_transaction_error::TransactionError; + +/// Proves unicast exclusivity, persistent fanout, terminal fanout, and slow-receiver removal. +#[tokio::test] +async fn subscribers_send_semantics() { + let unicast = Arc::new(Unicast::new(1, Subscription::Transactions)); + let mut unicast_rx = unicast.subscribe().unwrap(); + assert!(unicast.subscribe().is_err()); + unicast.send(1).await; + let sender = unicast.clone(); + let send = tokio::spawn(async move { sender.send(2).await }); + tokio::task::yield_now().await; + assert!(!send.is_finished(), "async unicast send waits for capacity"); + assert_eq!(unicast_rx.recv().await, Some(1)); + send.await.unwrap(); + assert_eq!(unicast_rx.recv().await, Some(2)); + + unicast.send(3).await; + let sender = unicast.clone(); + let send = std::thread::spawn(move || sender.blocking_send(4)); + assert_eq!(unicast_rx.recv().await, Some(3)); + send.join().unwrap(); + assert_eq!(unicast_rx.recv().await, Some(4)); + drop(unicast_rx); + assert!(unicast.subscribe().is_err()); + + let multicast = Multicast::new(1, Subscription::Accounts); + multicast.send(&1, &9); + let mut first = multicast.subscribe(1).await; + let mut second = multicast.subscribe(1).await; + multicast.send(&1, &10); + assert_eq!(first.recv().await, Some(10)); + assert_eq!(second.recv().await, Some(10)); + multicast.send(&1, &11); + multicast.send(&1, &12); + assert_eq!(first.recv().await, Some(11)); + assert_eq!(first.recv().await, None); + assert_eq!(second.recv().await, Some(11)); + assert_eq!(second.recv().await, None); + + let oneshot = MulticastOneshot::default(); + let first = oneshot.subscribe(1).await; + let closed = oneshot.subscribe(1).await; + drop(closed); + oneshot.send_last(&1, &18); + assert!(matches!( + first.try_recv(), + Err(oneshot::TryRecvError::Empty) + )); + let second = oneshot.subscribe(1).await; + oneshot.send_last(&1, &19); + assert_eq!(second.await.unwrap(), 19); + oneshot.send(&1, &20); + assert_eq!(first.await.unwrap(), 20); + let third = oneshot.subscribe(1).await; + oneshot.send(&1, &21); + assert_eq!(third.await.unwrap(), 21); +} + +// Appending reserves the signature while rejection wakes only its own latest +// waiter. Invalid blockhash is retained as a terminal cached status. +#[tokio::test] +async fn append_dedup_and_status_sentinel() { + let keeper = TestKeeper::new().await; + let (signature, txn) = signed_tx(); + let slot = keeper.blocks().current_slot(); + let original = keeper.transactions().subscribe_signature(signature).await; + + // First append writes to the ledger; the duplicate is dropped. + assert!( + keeper.transactions().append(&txn).await.unwrap(), + "first append is accepted" + ); + let duplicate = keeper.transactions().subscribe_signature(signature).await; + assert!( + !keeper.transactions().append(&txn).await.unwrap(), + "duplicate is deduplicated" + ); + let status = duplicate.await.unwrap(); + assert_eq!(status.result, Err(TransactionError::AlreadyProcessed)); + assert_eq!(status.slot, slot); + assert!(matches!( + original.try_recv(), + Err(oneshot::TryRecvError::Empty) + )); + + // The sentinel makes status() return None from the cache. + assert!(keeper.transactions().status(signature).await.unwrap().is_none()); + + let payer = Keypair::new(); + let (signature, view) = signed_view( + &payer, + [Instruction::new_with_bytes(V42_ID, &[], vec![])], + Hash::new_from_array([1; 32]), + ); + let txn = + ResolvedTransaction::try_new(view, Some(Default::default()), &Default::default()).unwrap(); + let rejected = keeper.transactions().subscribe_signature(signature).await; + assert!(!keeper.transactions().append(&txn).await.unwrap()); + let status = rejected.await.unwrap(); + assert_eq!(status.result, Err(TransactionError::BlockhashNotFound)); + assert_eq!(status.slot, slot); + let cached = keeper.transactions().status(signature).await.unwrap().unwrap(); + assert_eq!(cached.result, Err(TransactionError::BlockhashNotFound)); + assert_eq!(cached.slot, slot); + + keeper.close().await; +} diff --git a/keeper/src/util.rs b/keeper/src/util.rs new file mode 100644 index 00000000..d73906ec --- /dev/null +++ b/keeper/src/util.rs @@ -0,0 +1,107 @@ +//! Internal helpers shared by keeper accessors. + +use std::sync::Arc; + +use ledger::{ + request::{ReadRequest, RequestPayload, TransactionStatus}, + schema::{ + Balances, CompiledInstruction, Cpis, Event, Execution, ExecutionDetails, ExecutionHeader, + Instruction, ReturnData, + }, +}; +use solana_message::inner_instruction::{InnerInstruction, InnerInstructionsList}; +use solana_signature::Signature; +use solana_svm::{ + transaction_balances::BalanceCollector, transaction_execution_result::ExecutedTransaction, + transaction_processing_result::TransactionProcessingResultExtensions, +}; + +use crate::{FullTransaction, Keeper, Result}; + +/// Ledger event, status-cache entry, and logs derived from one execution result. +pub(crate) struct ExecutionCommit { + /// First transaction signature used for status notifications. + pub(crate) signature: Signature, + /// Status stored in the signature cache and sent to subscribers. + pub(crate) status: TransactionStatus, + /// Ledger event that pairs execution metadata with the appended transaction. + pub(crate) event: Event, + /// Execution logs shared with account log subscribers. + pub(crate) logs: Arc>, +} + +/// Sends a typed read request to the ledger reader and waits for its response. +pub(crate) async fn request(keeper: &Keeper, params: P, request: F) -> Result +where + F: FnOnce(RequestPayload) -> ReadRequest, +{ + let (payload, handle) = RequestPayload::::new(params); + let request = request(payload); + keeper.ledger.reader.send_async(request).await?; + Ok(handle.recv().await?) +} + +/// Builds the ledger and cache records for a completed transaction execution. +pub(crate) fn execution_commit(txn: &mut FullTransaction) -> ExecutionCommit { + let slot = txn.execution.slot; + let result = txn.execution.result.flattened_result(); + let signature = txn.transaction.signatures()[0]; + + let header = ExecutionHeader { + signature, + slot, + result: result.clone(), + }; + let status = TransactionStatus { result, slot }; + let details = txn + .execution + .result + .as_ref() + .ok() + .map(|execution| execution_details(execution, txn.execution.balances.take())); + let logs = details.as_ref().map(|d| Arc::clone(&d.logs)).unwrap_or_default(); + let event = Event::Execution(Execution { header, details }); + + ExecutionCommit { signature, status, event, logs } +} + +/// Projects SVM execution data into the retained ledger format. +fn execution_details( + execution: &ExecutedTransaction, + balances: Option, +) -> ExecutionDetails { + let (pre, post) = balances.map(|bc| bc.into_vecs()).unwrap_or_default(); + let details = &execution.execution_details; + + ExecutionDetails { + fee: execution.loaded_transaction.fee_details.total_fee(), + balances: Balances { pre, post }, + logs: details.log_messages.clone().unwrap_or_default(), + compute_units: details.executed_units, + return_data: details.return_data.as_ref().map(|rd| ReturnData { + program: rd.program_id.to_bytes(), + data: rd.data.clone().into(), + }), + cpi: details.inner_instructions.as_ref().map(cpis), + } +} + +/// Projects grouped SVM inner instructions into ledger CPI records. +fn cpis(groups: &InnerInstructionsList) -> Vec { + groups + .iter() + .map(|group| Cpis(group.iter().map(instruction).collect())) + .collect() +} + +/// Projects one SVM inner instruction into the ledger instruction format. +fn instruction(ix: &InnerInstruction) -> Instruction { + Instruction { + stack_height: ix.stack_height, + compiled: CompiledInstruction { + program_index: ix.instruction.program_id_index, + accounts: ix.instruction.accounts.clone(), + data: ix.instruction.data.clone(), + }, + } +} diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml new file mode 100644 index 00000000..f1077cdd --- /dev/null +++ b/ledger/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "magicblock-ledger" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "ledger" + +[features] +testkit = [] + +[dependencies] +nucleus = { workspace = true, features = ["heed", "ledger", "service"] } + +bitcode = { workspace = true } +bytemuck = { workspace = true, features = ["derive", "extern_crate_std"] } +derive_more = { workspace = true, features = ["deref", "from"] } +flume = { workspace = true } +heed = { workspace = true } +memmap2 = { workspace = true } +num_cpus = { workspace = true } +oneshot = { workspace = true, features = ["std"] } +parking_lot = { workspace = true } +rustix = { workspace = true, features = ["fs"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["sync"] } +tracing = { workspace = true } +wincode = { workspace = true } +zstd = { workspace = true } + +agave-transaction-view = { workspace = true, features = ["agave-unstable-api"] } +solana-pubkey = { workspace = true } +solana-signature = { workspace = true, features = ["wincode"] } +solana-transaction-error = { workspace = true, features = ["wincode"] } + +[dev-dependencies] +ledger = { workspace = true, features = ["testkit"] } +nucleus = { workspace = true, features = ["testkit"] } +tokio = { workspace = true, features = ["macros", "rt"] } + +[lints] +workspace = true diff --git a/ledger/README.md b/ledger/README.md new file mode 100644 index 00000000..a1ac0c60 --- /dev/null +++ b/ledger/README.md @@ -0,0 +1,56 @@ +# `magicblock-ledger` + +The ledger stores transaction bytes, execution metadata, block boundaries, +superblock seals, and volatile-state reset markers. History is partitioned into +self-contained superblock directories so retention removes a complete sealed +segment without compacting the active store. + +```text +ledger.meta +superblock-000000001/ + superblock.meta + blockstore.db + executions.db + index/ +``` + +`blockstore.db` is a wincode stream. Blockstore decoding permits allocations up +to the ledger's 25-bit encoded entry-size bound (33,554,431 bytes); larger +entries are rejected. Execution headers and zstd-compressed bitcode details are +stored separately in `executions.db`. + +## Append and read paths + +One appender owns ordered writes. Transaction bytes are appended first and kept +pending until their execution metadata arrives; only then are transaction and +account indexes inserted. Every durable sync flushes data and indexes, publishes +durable file cursors, transfers the accumulated transaction count, and flushes +ledger metadata. When the sync carries a block boundary, it also publishes that +block's slot and increments the block count. A seal finalizes the active files +and rotates to the next superblock. The successor metadata retains the sealed +snapshot's checksum and cumulative transaction count so it remains +self-describing after retention removes the preceding blockstore. + +Reader requests run on a worker pool. Each worker owns its decode buffers and +reads only through published cursors. The optional `testkit` feature reduces +LMDB map sizes and uses one reader worker without changing the on-disk format. + +During coordinated shutdown, one queue marker per reader closes the pool after +earlier requests. A final appender sync flushes every preceding event, reports +its durability result, and then closes the appender. Intermediate replication +syncs flush without closing either service, and retained sender clones do not +delay terminal shutdown. + +Replay is superblock-based. The consumer supplies the last sealed superblock +already reflected in its state, and the reader streams each retained successor +through the active head in full. + +## Retention + +At a block boundary, the appender checks used bytes on the ledger filesystem. +When the configured limit is reached, `Ledger::truncate` removes the oldest +sealed superblock; the active head is never removed. + +The size check assumes the ledger directory is on a dedicated filesystem. +Unrelated files on that filesystem contribute to the used-byte total and can +trigger earlier retention. diff --git a/ledger/src/appender.rs b/ledger/src/appender.rs new file mode 100644 index 00000000..aa64c13c --- /dev/null +++ b/ledger/src/appender.rs @@ -0,0 +1,389 @@ +//! Ledger append service and writable superblock storage. + +use std::{ + collections::HashMap, + path::Path, + sync::{Arc, atomic::Ordering::*}, +}; + +use agave_transaction_view::transaction_view::TransactionView; +use bitcode::Buffer; +use flume::Receiver; +use heed::{Env, RwTxn}; +use nucleus::{ + Slot, + heed::{DatabaseIndex, OptRwTxn, write_txn}, + ledger::BlockstorePosition, + shutdown::{ShutdownHandle, ShutdownReason}, +}; +use solana_signature::Signature; +use tokio::sync::broadcast::Sender; +use tracing::{info, warn}; +use wincode::Error; +use zstd::bulk::Compressor; + +use crate::{ + Ledger, Superblock, + error::{LedgerError, Result}, + index::{Index, Span, TxSpan}, + metrics::{self, Operation}, + schema::{ + Block, BlockstoreEntry, Event, Execution, ExecutionDetails, MAX_EXECUTION_DETAILS_SIZE, + SuperblockSeal, TransactionEntry, blockstore, + }, + storage::{AppendFile, MetaMap, SuperblockMeta}, +}; + +/// Blockstore stream file name inside a superblock. +pub(crate) const BLOCKSTORE_DB: &str = "blockstore.db"; +/// Execution details file name inside a superblock. +pub(crate) const EXECUTIONS_DB: &str = "executions.db"; +/// Superblock metadata file name. +pub(crate) const SUPERBLOCK_META: &str = "superblock.meta"; + +/// Background service that appends ledger events into the active superblock. +pub(crate) struct LedgerAppender { + /// Shared top-level ledger state. + ledger: Arc, + /// Writable files for the active superblock. + writer: SuperblockWriter, + /// Transactions waiting for their matching execution details. + pending: HashMap, + /// Active superblock index. + index: Arc, + /// Event stream from the execution pipeline. + rx: Receiver, + /// Transactions written since the last durable sync. + transactions: u64, + /// Broadcasts the blockstore write position after each committed block. + position: Sender, +} + +impl LedgerAppender { + /// Opens the active superblock and registers it on the ledger handle. + pub(crate) fn new( + ledger: Arc, + rx: Receiver, + position: Sender, + ) -> Result { + let head = ledger.meta.head(); + let directory = Superblock::init_dir(&ledger.directory, head)?; + let writer = SuperblockWriter::new(&directory)?; + metrics::pending_transactions(0); + let index = ledger + .superblocks + .read() + .get(&head) + .map(|s| s.index.clone()) + .ok_or(LedgerError::Corruption("active superblock missing"))?; + + Ok(Self { + ledger, + writer, + index, + rx, + pending: HashMap::new(), + transactions: 0, + position, + }) + } + + /// Runs until shutdown (when the event stream closes). + pub(crate) fn run(mut self, mut shutdown: ShutdownHandle) { + let reason = loop { + let env = self.index.env().clone(); + match self.run_epoch(&env) { + Ok(Epoch::Rotate) => (), + Ok(Epoch::Shutdown) => break ShutdownReason::Signalled, + Err(err) => break ShutdownReason::Error(Box::new(err)), + } + }; + // Release ledger ownership before the manager can reopen it. + drop(self); + shutdown.terminate(reason); + } + + /// Processes events against one stable superblock environment. + fn run_epoch<'e>(&mut self, env: &'e Env) -> Result { + let mut txn: Option> = None; + loop { + let Ok(event) = self.rx.recv() else { + return Ok(Epoch::Shutdown); + }; + if let Some(epoch) = self.process(event, env, &mut txn)? { + return Ok(epoch); + } + } + } + + /// Rotates to the next superblock directory. + fn rotate(&mut self, seal: SuperblockSeal) -> Result<()> { + let _timer = metrics::time(Operation::Rotate); + let head = seal.id + 1; + let superblock = Superblock::open(&self.ledger.directory, head)?; + // Seal N opens N+1, which stores N's snapshot archive and seal metadata. + superblock.meta.checksum.store(seal.checksum, Release); + superblock.meta.transactions.store(seal.transactions, Release); + self.writer = SuperblockWriter::new(&superblock.directory)?; + self.ledger.meta.head.store(head, Release); + self.ledger.meta.superblocks.fetch_add(1, Release); + self.ledger.meta.flush()?; + self.index = superblock.index.clone(); + superblock.meta.flush()?; + self.ledger.superblocks.write().insert(head, superblock); + info!(head, "opened active superblock"); + Ok(()) + } + + /// Processes one append event, rotating after a superblock seal. + /// + fn process<'e>( + &mut self, + event: Event, + env: &'e Env, + txn: OptRwTxn<'_, 'e>, + ) -> Result> { + match event { + Event::Transaction(transaction) => { + self.write_transaction(transaction)?; + } + Event::Execution(execution) => { + self.write_execution(execution, env, txn)?; + } + Event::Block(block) => { + self.write_block(block, env, txn)?; + } + Event::Superblock(seal) => { + self.seal(seal, false, txn)?; + return Ok(Some(Epoch::Rotate)); + } + Event::Bootstrap(seal) => { + self.seal(seal, true, txn)?; + return Ok(Some(Epoch::Rotate)); + } + Event::Reset(slot) => { + self.write_reset(slot, txn)?; + } + Event::Sync { response, is_final } => { + let _ = response.send(self.sync(None, txn)); + if is_final { + return Ok(Some(Epoch::Shutdown)); + } + } + } + Ok(None) + } + + /// Seals the active superblock, optionally adopting a restored snapshot's + /// cumulative transaction count before publishing the successor metadata. + fn seal(&mut self, seal: SuperblockSeal, bootstrap: bool, txn: OptRwTxn<'_, '_>) -> Result<()> { + self.write_superblock(seal, txn)?; + if bootstrap { + self.ledger.meta.transactions.store(seal.transactions, Release); + } + self.rotate(seal) + } + + /// Writes a raw transaction and keeps it pending until execution arrives. + fn write_transaction(&mut self, transaction: TransactionEntry) -> Result<()> { + let entry = BlockstoreEntry::Transaction(transaction.payload.as_slice()); + let span = self.writer.write_blockstore(&entry)?; + let entry = PendingTx { + transaction: transaction.payload, + span, + }; + self.pending.insert(transaction.signature, entry); + metrics::pending_transactions(self.pending.len()); + self.transactions += 1; + Ok(()) + } + + /// Writes execution details and adds transaction/account indexes. + fn write_execution<'e>( + &mut self, + execution: Execution, + env: &'e Env, + txn: OptRwTxn<'_, 'e>, + ) -> Result<()> { + let signature: Signature = execution.header.signature; + let Some(pending) = self.pending.remove(&signature) else { + warn!(%signature, "ledger execution arrived without a pending transaction; skipping"); + return Ok(()); + }; + metrics::pending_transactions(self.pending.len()); + let execution = self.writer.write_execution(&execution)?; + let span = TxSpan { + blockstore: pending.span, + execution, + }; + let txn = write_txn(env, txn)?; + self.index.insert_transaction(txn, &signature, &span)?; + let view = TransactionView::try_new_unsanitized(pending.transaction)?; + let accounts = view.static_account_keys(); + self.index.insert_accounts(txn, accounts, &span.execution)?; + Ok(()) + } + + /// Writes a block boundary and publishes it after data and indexes are durable. + fn write_block<'e>(&mut self, block: Block, env: &'e Env, txn: OptRwTxn<'_, 'e>) -> Result<()> { + let span = self.writer.write_blockstore(&BlockstoreEntry::Block(block))?; + self.index.insert_block(write_txn(env, txn)?, &block.slot, &span)?; + self.sync(Some(block.slot), txn)?; + if self.ledger.size_exceeded()? { + self.ledger.truncate()?; + } + metrics::ledger_counts(&self.ledger); + Ok(()) + } + + /// Writes a superblock seal and prepares files for read-only access. + fn write_superblock(&mut self, seal: SuperblockSeal, txn: OptRwTxn<'_, '_>) -> Result<()> { + self.writer.write_blockstore(&BlockstoreEntry::Superblock(seal))?; + self.sync(None, txn)?; + self.writer.finalize()?; + info!(superblock = seal.id, "sealed superblock"); + Ok(()) + } + + /// Writes and publishes a volatile-state reset marker. + fn write_reset(&mut self, slot: Slot, txn: OptRwTxn<'_, '_>) -> Result<()> { + self.writer.write_blockstore(&BlockstoreEntry::Reset(slot))?; + self.sync(None, txn)?; + info!(slot, "appended volatile state reset"); + Ok(()) + } + + /// Makes files and indexes durable, publishes their cursors and accumulated + /// transaction count, and broadcasts the new blockstore position. When + /// `slot` is supplied, the same boundary also publishes block metadata. + fn sync(&mut self, slot: Option, txn: OptRwTxn<'_, '_>) -> Result<()> { + let cursors = self.writer.sync()?; + if let Some(txn) = txn.take() { + txn.commit()?; + } + self.index.flush()?; + self.writer.publish(cursors, slot)?; + self.ledger.meta.transactions.fetch_add(self.transactions, Release); + if let Some(slot) = slot { + self.ledger.meta.blocks.fetch_add(1, Release); + self.ledger.meta.range.end.store(slot, Release); + } + self.ledger.meta.flush()?; + self.transactions = 0; + let position = BlockstorePosition { + superblock: self.ledger.meta.head(), + offset: cursors.0, + }; + let _ = self.position.send(position); + Ok(()) + } +} + +/// Boundary reached while processing one superblock environment. +enum Epoch { + Rotate, + Shutdown, +} + +/// Transaction bytes already written but not yet paired with execution details. +struct PendingTx { + /// Transaction bytes retained until execution details arrive for indexing. + transaction: Arc>, + /// Blockstore-file span of the transaction bytes. + span: Span, +} + +/// Writable superblock files and reusable execution-detail encoder. +struct SuperblockWriter { + /// Compressor reused for execution metadata payloads. + compressor: Compressor<'static>, + /// Scratch buffer owned by bitcode while encoding metadata. + buffer: Buffer, + /// Buffered transaction/block/seal blockstore stream. + blockstore: AppendFile, + /// Buffered execution details stream. + executions: AppendFile, + /// Mmap-backed metadata for this superblock. + metadata: MetaMap, +} + +impl SuperblockWriter { + /// Opens writable files and metadata under `directory`. + fn new(directory: &Path) -> Result { + // SAFETY: `SuperblockMeta` and its nested headers have stable C layouts, + // and all fields that can change while mapped are atomic. The ledger + // exclusively creates and updates this superblock metadata file. + let metadata = unsafe { MetaMap::::new(&directory.join(SUPERBLOCK_META)) }?; + + Ok(Self { + blockstore: AppendFile::new( + &directory.join(BLOCKSTORE_DB), + &metadata.cursors.blockstore, + )?, + executions: AppendFile::new( + &directory.join(EXECUTIONS_DB), + &metadata.cursors.executions, + )?, + metadata, + compressor: Compressor::new(0)?, + buffer: Buffer::new(), + }) + } + + /// Appends an entry to the blockstore file. + fn write_blockstore(&mut self, entry: &BlockstoreEntry<&[u8]>) -> Result { + let offset = self.blockstore.cursor; + blockstore::encode(&mut self.blockstore, entry).map_err(Into::::into)?; + let size = self.blockstore.cursor - offset; + Ok(Span::new(offset, size)) + } + + /// Appends execution details and returns their file span. + fn write_execution(&mut self, execution: &Execution) -> Result { + let offset = self.executions.cursor; + wincode::serialize_into(&mut self.executions, &execution.header) + .map_err(Into::::into)?; + let mut details = self.buffer.encode(&execution.details); + if details.len() > MAX_EXECUTION_DETAILS_SIZE { + // Omit oversized details but retain the fixed execution header and status. + details = self.buffer.encode(&None::); + } + self.executions.compress(details, &mut self.compressor)?; + let size = self.executions.cursor - offset; + Ok(Span::new(offset, size)) + } + + /// Syncs data files and returns durable cursors. + fn sync(&mut self) -> Result<(u64, u64)> { + let _timer = metrics::time(Operation::FileSync); + Ok((self.blockstore.sync()?, self.executions.sync()?)) + } + + /// Publishes durable cursors into superblock metadata. + fn publish(&self, cursors: (u64, u64), slot: Option) -> Result<()> { + let (blockstore, executions) = cursors; + self.metadata.cursors.blockstore.store(blockstore, Release); + self.metadata.cursors.executions.store(executions, Release); + if let Some(slot) = slot { + self.metadata.range.end.store(slot, Release); + // The first block of a segment fixes its start + // slot; later blocks only extend the end. + let _ = self.metadata.range.start.compare_exchange(0, slot, Release, Relaxed); + } + self.metadata.flush()?; + Ok(()) + } + + /// Trims preallocated file space after the superblock cursors are durable. + fn finalize(&mut self) -> Result<()> { + let _timer = metrics::time(Operation::FileFinalize); + self.blockstore.finalize()?; + self.executions.finalize()?; + info!( + blockstore = self.blockstore.cursor, + executions = self.executions.cursor, + "sealed superblock files" + ); + Ok(()) + } +} diff --git a/ledger/src/error.rs b/ledger/src/error.rs new file mode 100644 index 00000000..4c899384 --- /dev/null +++ b/ledger/src/error.rs @@ -0,0 +1,54 @@ +//! Error types shared by the ledger crate. + +use std::io; + +use agave_transaction_view::result::TransactionViewError; +use heed::BoxedError; +use oneshot::RecvError; +use tokio::time::error::Elapsed; + +/// Errors returned by ledger storage, codecs, and indexes. +#[derive(Debug, derive_more::From, thiserror::Error)] +pub enum LedgerError { + /// Filesystem I/O failed while opening, writing, or flushing ledger files. + #[error("ledger storage I/O error: {0}")] + IO(#[source] io::Error), + /// Platform filesystem operation failed. + #[error("ledger filesystem operation failed: {0}")] + FS(#[source] rustix::io::Errno), + /// Wincode failed while serializing the blockstore stream. + #[error("ledger blockstore codec error: {0}")] + Wincode(#[source] wincode::Error), + /// Bitcode failed while serializing execution details. + #[error("ledger execution-details codec error: {0}")] + Bitcode(#[source] bitcode::Error), + /// LMDB key/value codec failed while encoding or decoding an index value. + #[error("ledger index key/value codec error: {0}")] + IndexCodec(#[source] BoxedError), + /// LMDB failed while opening or accessing a ledger index. + #[error("ledger index error: {0}")] + Index(#[source] heed::Error), + /// Sanitized transaction view could not be decoded. + #[error("transaction view error: {0:?}")] + TransactionView(TransactionViewError), + /// Ledger index pointed to an entry of the wrong type or invalid shape. + #[error("ledger corruption: {0}")] + #[from(skip)] + Corruption(&'static str), +} + +/// Errors returned while waiting for a ledger reader response. +#[derive(Debug, thiserror::Error)] +pub enum LedgerRequestError { + /// Reader dropped the response channel before sending a result. + #[error("ledger reader response channel closed: {0}")] + ResponseFailed(#[from] RecvError), + /// Reader did not answer within the request timeout. + #[error("ledger reader request timed out: {0}")] + Timeout(#[from] Elapsed), +} + +/// Result type used by the ledger crate. +pub(crate) type Result = std::result::Result; +/// Result type used while waiting for asynchronous ledger reader responses. +pub(crate) type RequestResult = std::result::Result; diff --git a/ledger/src/index.rs b/ledger/src/index.rs new file mode 100644 index 00000000..247d9a5d --- /dev/null +++ b/ledger/src/index.rs @@ -0,0 +1,321 @@ +//! LMDB index schema and codecs for ledger blockstore entries. + +use std::{array, borrow::Cow, fs, path::Path}; + +use bytemuck::{Pod, Zeroable}; +use heed::{ + BoxedError, BytesDecode, BytesEncode, Database, DatabaseFlags, Env, EnvFlags, EnvOpenOptions, + IntegerComparator, RoIter, RwTxn, byteorder::LittleEndian, + iteration_method::MoveOnCurrentKeyDuplicates, types::U64, +}; +use nucleus::{ + Slot, + heed::{DatabaseIndex, OptRoTxn, read_txn}, +}; +use solana_pubkey::Pubkey; +use solana_signature::Signature; + +use crate::schema::Offset; + +/// Index directory below each superblock directory. +const INDEX_SUBDIR: &str = "index"; +/// Maximum LMDB map size for ledger indexes. +#[cfg(feature = "testkit")] +const INDEX_MAP_SIZE: usize = 32 * nucleus::MB; +#[cfg(not(feature = "testkit"))] +const INDEX_MAP_SIZE: usize = 16 * nucleus::GB; +/// Number of LMDB named databases in the ledger index. +const INDEX_DBS: u32 = 3; +/// Transaction signature index name. +const TRANSACTIONS_INDEX: &str = "transactions"; +/// Slot-to-offset index name. +const SLOTS_INDEX: &str = "slots"; +/// Account-to-offset index name. +const ACCOUNTS_INDEX: &str = "accounts"; +/// Bytes kept from wide keys in compact index keys. +const KEY_BYTES: usize = 16; + +/// Result type returned by heed codec hooks. +type CodecResult = Result; +/// Little-endian u64 codec used by LMDB keys and values. +type U64Le = U64; +/// Little-endian slot key with integer ordering. +type SlotKey = U64; +/// Truncated transaction signature key. +/// +/// The 16-byte prefix is used as an index tag, not as a collision-proof +/// identity. Collision probability is negligible for the ledger's expected +/// scale, so the index accepts that risk to keep keys compact. +#[derive(Pod, Zeroable, Clone, Copy)] +#[repr(C)] +pub(crate) struct SignatureKey([u8; KEY_BYTES]); + +/// Truncated account pubkey key. +/// +/// See `SignatureKey`; account history uses the same compact prefix tag. +#[derive(Pod, Zeroable, Clone, Copy)] +#[repr(C)] +pub(crate) struct AccountKey([u8; KEY_BYTES]); + +/// Packed span inside a ledger data file. +/// +/// The high 39 bits store the byte offset. The low 25 bits store the encoded +/// entry size. Transaction entries are bounded by Solana's transaction-size +/// limit, and execution details are expected to stay at a few dozen KiB before +/// compression, so spans stay well below the 32 MiB encoded-size cap. +#[derive(Zeroable, Pod, Clone, Copy, Default, Debug, PartialEq, Eq, PartialOrd, Ord)] +#[repr(C)] +pub(crate) struct Span(u64); + +impl Span { + /// Number of low bits reserved for entry size. + const SIZE_BITS: u64 = 25; + /// Mask for the encoded entry size. + const SIZE_MASK: u64 = (1 << Self::SIZE_BITS) - 1; + /// Largest blockstore entry size that can be packed into an index value. + pub(crate) const MAX_SIZE: u64 = Self::SIZE_MASK; + /// Largest file size addressable by the packed offset. + pub(crate) const MAX_FILE_SIZE: u64 = (u64::MAX >> Self::SIZE_BITS) + 1; + + /// Packs an `offset` and `size` into one integer. + pub(crate) fn new(offset: Offset, size: u64) -> Self { + Self(offset << Self::SIZE_BITS | size) + } + + /// Returns the byte offset in the backing file. + pub(crate) fn offset(&self) -> u64 { + self.0 >> Self::SIZE_BITS + } + + /// Returns the encoded entry size in bytes. + pub(crate) fn size(&self) -> u64 { + self.0 & Self::SIZE_MASK + } +} + +/// Pair of blockstore-file and execution-file spans for a transaction. +#[derive(Zeroable, Pod, Clone, Copy)] +#[repr(C)] +pub(crate) struct TxSpan { + /// Span of the raw transaction entry in the blockstore. + pub(crate) blockstore: Span, + /// Span of the execution details. + pub(crate) execution: Span, +} + +/// Account-index span codec. +pub(crate) enum AccountSpan {} + +/// Duplicate account-index iterator over execution spans. +pub(crate) type AccountIter<'a> = RoIter<'a, AccountKey, AccountSpan, MoveOnCurrentKeyDuplicates>; + +/// LMDB databases used to locate ledger data. +pub(crate) struct Index { + /// Owning LMDB environment. + env: Env, + /// Transaction signature to transaction/execution spans. + transactions: Database, + /// Slot to blockstore-file span. + blocks: Database, + /// Account key to execution-details span. + accounts: Database, +} + +impl Index { + /// Opens or creates the index directory and databases. + pub(crate) fn new(path: &Path) -> heed::Result { + let path = path.join(INDEX_SUBDIR); + fs::create_dir_all(&path)?; + // SAFETY: this process owns the index directory for the lifetime of + // the database, so the backing files are not mutated behind LMDB's back. + let env = unsafe { + EnvOpenOptions::new() + .max_dbs(INDEX_DBS) + .map_size(INDEX_MAP_SIZE) + .flags(EnvFlags::WRITE_MAP) + .flags(EnvFlags::NO_SYNC) + .open(path)? + }; + + let mut txn = env.write_txn()?; + let transactions = + env.database_options().name(TRANSACTIONS_INDEX).types().create(&mut txn)?; + let blocks = env + .database_options() + .name(SLOTS_INDEX) + .key_comparator() + .types() + .create(&mut txn)?; + let accounts = env + .database_options() + .name(ACCOUNTS_INDEX) + .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED | DatabaseFlags::REVERSE_DUP) + .types() + .create(&mut txn)?; + txn.commit()?; + Ok(Self { + env, + transactions, + blocks, + accounts, + }) + } + + /// Indexes a block boundary by slot. + pub(crate) fn insert_block( + &self, + txn: &mut RwTxn<'_>, + slot: &Slot, + span: &Span, + ) -> heed::Result<()> { + self.blocks.put(txn, slot, span) + } + + /// Indexes a transaction and its execution details. + pub(crate) fn insert_transaction( + &self, + txn: &mut RwTxn<'_>, + signature: &Signature, + span: &TxSpan, + ) -> heed::Result<()> { + self.transactions.put(txn, signature, span) + } + + /// Adds account-to-execution entries for all static transaction accounts. + pub(crate) fn insert_accounts( + &self, + txn: &mut RwTxn<'_>, + accounts: &[Pubkey], + span: &Span, + ) -> heed::Result<()> { + for account in accounts { + self.accounts.put(txn, account, span)?; + } + Ok(()) + } + + /// Locates a transaction by its first signature. + pub(crate) fn transaction<'t, 'e>( + &'e self, + signature: &Signature, + txn: OptRoTxn<'t, 'e>, + ) -> heed::Result> { + let txn = read_txn(&self.env, txn)?; + self.transactions.get(txn, signature) + } + + /// Locates a block boundary by slot. + pub(crate) fn block<'t, 'e>( + &'e self, + slot: &Slot, + txn: OptRoTxn<'t, 'e>, + ) -> heed::Result> { + let txn = read_txn(&self.env, txn)?; + self.blocks.get(txn, slot) + } + + /// Returns execution spans that mention `pubkey`. + pub(crate) fn accounts<'t, 'e>( + &'e self, + pubkey: &Pubkey, + txn: OptRoTxn<'t, 'e>, + ) -> heed::Result>> { + let txn = read_txn(&self.env, txn)?; + self.accounts.get_duplicates(txn, pubkey) + } +} + +impl DatabaseIndex for Index { + fn env(&self) -> &Env { + &self.env + } +} + +impl> From for SignatureKey { + fn from(signature: S) -> Self { + Self(array::from_fn(|i| signature.as_ref().as_array()[i])) + } +} + +impl<'a> BytesEncode<'a> for SignatureKey { + type EItem = Signature; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + Ok(item.as_array()[..KEY_BYTES].into()) + } +} + +impl<'a> BytesEncode<'a> for AccountKey { + type EItem = Pubkey; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + Ok(item.as_array()[..KEY_BYTES].into()) + } +} + +impl<'a> BytesDecode<'a> for SignatureKey { + type DItem = &'a Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + bytemuck::try_from_bytes(bytes).map_err(Into::into) + } +} + +impl<'a> BytesDecode<'a> for AccountKey { + type DItem = &'a Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + bytemuck::try_from_bytes(bytes).map_err(Into::into) + } +} + +impl<'a> BytesEncode<'a> for Span { + type EItem = Self; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + U64Le::bytes_encode(&item.0) + } +} + +impl<'a> BytesDecode<'a> for Span { + type DItem = Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + U64Le::bytes_decode(bytes).map(Self) + } +} + +impl<'a> BytesEncode<'a> for AccountSpan { + type EItem = Span; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + // `REVERSE_DUP` compares fixed-size values from the end, which makes + // little-endian u64 values sort numerically ascending. Store the + // inverted span so higher execution offsets are returned first. + Ok((!item.0).to_le_bytes().to_vec().into()) + } +} + +impl<'a> BytesDecode<'a> for AccountSpan { + type DItem = Span; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + U64Le::bytes_decode(bytes).map(|span| Span(!span)) + } +} + +impl<'a> BytesEncode<'a> for TxSpan { + type EItem = Self; + + fn bytes_encode(item: &'a Self::EItem) -> CodecResult> { + Ok(bytemuck::bytes_of(item).into()) + } +} + +impl<'a> BytesDecode<'a> for TxSpan { + type DItem = Self; + + fn bytes_decode(bytes: &'a [u8]) -> CodecResult { + bytemuck::try_pod_read_unaligned(bytes).map_err(Into::into) + } +} diff --git a/ledger/src/lib.rs b/ledger/src/lib.rs new file mode 100644 index 00000000..8af03947 --- /dev/null +++ b/ledger/src/lib.rs @@ -0,0 +1,298 @@ +#![doc = include_str!("../README.md")] + +use std::{ + collections::BTreeMap, + fs::{self, File}, + path::{Path, PathBuf}, + sync::{Arc, atomic::Ordering::*}, + thread, +}; + +pub use crate::error::{LedgerError, LedgerRequestError}; +use derive_more::Deref; +use nucleus::{ + Slot, + ledger::BlockstorePosition, + shutdown::{Service, ShutdownManager}, +}; +use parking_lot::RwLock; +use tokio::sync::broadcast; +use tracing::info; + +mod appender; +mod error; +mod index; +mod metrics; +mod reader; +pub mod request; +pub mod schema; +mod storage; + +#[cfg(test)] +mod tests; + +use crate::{ + appender::{BLOCKSTORE_DB, EXECUTIONS_DB, LedgerAppender, SUPERBLOCK_META}, + error::Result, + index::Index, + reader::LedgerReader, + request::ReaderSender, + schema::Event, + storage::{LedgerMeta, MetaMap, SuperblockMeta}, +}; + +const LEDGER_META: &str = "ledger.meta"; +const SERVICE_QUEUE_CAPACITY: usize = 128; + +/// Top-level ledger handle. +/// +/// `head` in ledger metadata is the active superblock id. Older superblocks +/// stay open in `superblocks` until retention removes their directories. +pub struct Ledger { + /// Mmap-backed ledger metadata shared with the appender. + meta: MetaMap, + /// Retained superblocks keyed by id. + superblocks: RwLock>>, + /// Root ledger directory. + pub directory: PathBuf, + /// Maximum used bytes allowed on the ledger filesystem before retention runs. + size_limit: u64, +} + +impl Ledger { + /// Opens the ledger and starts one appender plus the reader worker pool. + pub fn init( + directory: impl AsRef, + size_limit: u64, + shutdown: &mut ShutdownManager, + ) -> Result { + let directory = directory.as_ref().to_owned(); + let ledger = Arc::new(Self::new(directory, size_limit)?); + metrics::init(&ledger); + let (appender_tx, rx) = flume::bounded(SERVICE_QUEUE_CAPACITY); + let (position, _) = broadcast::channel(256); + let appender = LedgerAppender::new(ledger.clone(), rx, position.clone())?; + let sh = shutdown.handle(Service::LedgerAppender); + thread::Builder::new() + .name("ledger-appender".into()) + .spawn(|| appender.run(sh))?; + let (reader_tx, rx) = flume::bounded(SERVICE_QUEUE_CAPACITY); + + #[cfg(not(feature = "testkit"))] + let readers = num_cpus::get() as u32; + #[cfg(feature = "testkit")] + let readers = 1; + + for id in 0..readers { + let sh = shutdown.handle(Service::LedgerReader); + let reader = LedgerReader::new(ledger.clone(), rx.clone())?; + thread::Builder::new() + .name(format!("ledger-reader-{id}")) + .spawn(|| reader.run(sh))?; + } + info!(readers, "initialized ledger"); + Ok(LedgerHandle { + ledger, + reader: reader_tx, + appender: appender_tx, + position, + }) + } + + /// Iterates retained superblocks from newest to oldest. + pub fn iter(&self) -> impl Iterator> { + let range = self.meta.superblocks(); + range.rev().filter_map(|id| self.superblocks.read().get(&id).cloned()) + } + + /// Returns the highest slot recorded across retained superblocks, or + /// `None` when none are retained. Taken over all superblocks rather than + /// just the head: right after a seal the active head has no blocks yet and + /// its persisted range is still zeroed. + pub fn tip(&self) -> Option { + self.iter().map(|s| s.meta.range.end.load(Acquire)).max() + } + + /// Blockstore write offset of a retained superblock, `None` when it is not retained. + pub fn cursor(&self, superblock: u64) -> Option { + self.iter().find_map(|sb| { + (sb.id == superblock).then_some(sb.meta.cursors.blockstore.load(Acquire)) + }) + } + + /// Iterates retained superblocks after `superblock` through the active head, + /// so replay excludes the sealed snapshot state but includes the unsealed head. + fn iter_after(&self, superblock: u64) -> impl Iterator> { + let range = superblock + 1..=self.meta.head(); + range.filter_map(|id| self.superblocks.read().get(&id).cloned()) + } + + /// Opens ledger metadata and retained superblocks without starting services. + fn new(directory: PathBuf, size_limit: u64) -> Result { + fs::create_dir_all(&directory)?; + let meta = directory.join(LEDGER_META); + // SAFETY: `LedgerMeta` and its nested headers have stable C layouts, + // and all fields that can change while mapped are atomic. This process + // exclusively creates and updates the metadata file at `meta`. + let meta = unsafe { MetaMap::::new(&meta) }?; + let mut superblocks = BTreeMap::new(); + for id in meta.superblocks() { + let superblock = Superblock::open(&directory, id)?; + superblocks.insert(id, superblock); + } + + info!(?directory, superblocks = superblocks.len(), "opened ledger"); + Ok(Self { + meta, + superblocks: superblocks.into(), + directory, + size_limit, + }) + } + + /// Returns true when the ledger filesystem has reached the configured limit. + /// + /// This assumes `directory` is on a filesystem dedicated to the ledger. Any + /// unrelated files on the same filesystem count toward the used byte total. + fn size_exceeded(&self) -> Result { + let st = rustix::fs::statvfs(&self.directory)?; + let total = st.f_blocks * st.f_frsize; + let free = st.f_bfree * st.f_frsize; + Ok(total.saturating_sub(free) >= self.size_limit) + } + + /// Removes the oldest sealed superblock while keeping the active head. + /// + /// Superblock slot ranges are sequential and non-overlapping, so the next + /// retained start slot is the removed superblock end plus one. + /// + /// The appender runs this at a block boundary once the ledger filesystem is + /// over its size limit; calling it directly forces a single retention pass + /// regardless of that limit. + pub fn truncate(&self) -> Result<()> { + let _timer = metrics::time(metrics::Operation::Truncate); + let Some((id, superblock)) = self + .superblocks + .read() + .first_key_value() + .map(|(id, superblock)| (*id, superblock.clone())) + else { + return Ok(()); + }; + if id >= self.meta.head() { + return Ok(()); + } + + let end = superblock.meta.range.end.load(Acquire); + superblock.purge()?; + self.superblocks.write().remove(&id); + self.meta.superblocks.fetch_sub(1, Release); + self.meta.range.start.store(end + 1, Release); + self.meta.flush()?; + info!(id, end, "purged oldest superblock for retention"); + Ok(()) + } +} + +/// Cloneable senders over shared ledger state: append events, read requests, and +/// blockstore-position updates. +#[derive(Deref, Clone)] +pub struct LedgerHandle { + /// Shared ledger state behind the request senders. + #[deref] + ledger: Arc, + /// Read request queue consumed by reader workers. + pub reader: ReaderSender, + /// Append event queue consumed by the appender worker. + pub appender: flume::Sender, + /// Blockstore write position broadcast after every committed block, so the + /// replication path can stream newly appended bytes. + pub position: broadcast::Sender, +} + +impl LedgerHandle { + /// Returns the number of transactions published at durable sync boundaries. + pub fn transactions(&self) -> u64 { + self.ledger.meta.transactions.load(Acquire) + } + + /// Returns the active superblock id. + pub fn head(&self) -> u64 { + self.ledger.meta.head() + } + + /// Position of the next byte to append: active superblock plus its durable cursor. + pub fn position(&self) -> BlockstorePosition { + let superblock = self.head(); + let offset = self.cursor(superblock).unwrap_or_default(); + BlockstorePosition { superblock, offset } + } +} + +/// Opened superblock files kept alive for active and retained readers. +pub struct Superblock { + /// Superblock id, matching its directory suffix under the ledger root. + pub id: u64, + /// Mmap-backed metadata for file cursors and slot range. + meta: MetaMap, + /// Transaction stream delimited by block entries and a superblock seal. + pub blockstore: File, + /// Transaction execution metadata file. + executions: File, + /// LMDB index for this superblock. + index: Arc, + /// Superblock directory path. + pub directory: PathBuf, +} + +impl Superblock { + /// Returns the directory path for `id` under `root`. + pub fn init_dir(root: &Path, id: u64) -> Result { + let dir = root.join(format!("superblock-{id:0>9}")); + fs::create_dir_all(&dir).map_err(Into::into).map(|()| dir) + } + + /// Accountsdb snapshot checksum recorded by the seal that opened this superblock. + pub fn checksum(&self) -> u64 { + self.meta.checksum.load(Acquire) + } + + /// Transaction count recorded by the seal that opened this superblock. + pub fn transactions(&self) -> u64 { + self.meta.transactions.load(Acquire) + } + + /// Opens a superblock directory, creating its data files when needed. + fn open(root: &Path, id: u64) -> Result> { + let directory = Self::init_dir(root, id)?; + let index = Arc::new(Index::new(&directory)?); + let meta = unsafe { MetaMap::::new(&directory.join(SUPERBLOCK_META)) }?; + let blockstore = Self::file(&directory.join(BLOCKSTORE_DB))?; + let executions = Self::file(&directory.join(EXECUTIONS_DB))?; + + Ok(Arc::new(Self { + id, + meta, + blockstore, + executions, + index, + directory, + })) + } + + /// Opens a superblock data file for random reads. + fn file(path: &Path) -> Result { + drop(File::options().write(true).create(true).truncate(false).open(path)?); + let f = File::open(path)?; + #[cfg(target_os = "linux")] + // Access advice is an optional optimization and must not prevent opening the ledger. + let _ = rustix::fs::fadvise(&f, 0, None, rustix::fs::Advice::Random); + + Ok(f) + } + + /// Removes this superblock directory from the ledger root. + fn purge(&self) -> Result<()> { + fs::remove_dir_all(&self.directory).map_err(Into::into) + } +} diff --git a/ledger/src/metrics.rs b/ledger/src/metrics.rs new file mode 100644 index 00000000..30387b5c --- /dev/null +++ b/ledger/src/metrics.rs @@ -0,0 +1,143 @@ +//! Prometheus metrics for ledger. + +use std::sync::{OnceLock, atomic::Ordering::Relaxed}; + +use nucleus::metrics as metric; +use nucleus::metrics::{IntGauge, MetricOperation, MetricSpec, OperationCounters}; + +use crate::Ledger; + +/// Process-wide ledger metrics registered in the default Prometheus registry. +static METRICS: OnceLock = OnceLock::new(); + +/// Operation latency histogram recorded in microseconds. +const OPERATION_TIME: MetricSpec = MetricSpec { + name: "ledger_operation_duration_micros", + help: "Ledger operation duration distribution in microseconds.", +}; +/// Transactions waiting for execution metadata in the appender. +const PENDING_TRANSACTIONS: MetricSpec = MetricSpec { + name: "ledger_pending_transactions", + help: "Current ledger transactions waiting for execution metadata.", +}; +/// Total committed transactions from ledger metadata. +const TRANSACTIONS: MetricSpec = MetricSpec { + name: "ledger_transactions", + help: "Current total transactions committed into ledger metadata.", +}; +/// Total committed blocks from ledger metadata. +const BLOCKS: MetricSpec = MetricSpec { + name: "ledger_blocks", + help: "Current total blocks committed into ledger metadata.", +}; +/// Total superblocks allocated from ledger metadata. +const SUPERBLOCKS: MetricSpec = MetricSpec { + name: "ledger_superblocks", + help: "Current total ledger superblocks allocated from genesis.", +}; + +/// Ledger operation used as a low-cardinality operation label. +#[derive(Clone, Copy)] +pub(crate) enum Operation { + /// Transaction read request. + ReadTransaction, + /// Transaction status read request. + ReadTransactionStatus, + /// Account signature history read request. + ReadAccountSignatures, + /// Single-block read request. + ReadBlock, + /// Block range read request. + ReadBlockRange, + /// Retained blockstore replay request. + Replay, + /// Superblock rotation path. + Rotate, + /// Retention truncation path. + Truncate, + /// Data-file sync path. + FileSync, + /// Data-file finalization path. + FileFinalize, +} + +impl MetricOperation for Operation { + /// Returns the Prometheus label value for this operation. + fn label(self) -> &'static str { + match self { + Operation::ReadTransaction => "read_transaction", + Operation::ReadTransactionStatus => "read_transaction_status", + Operation::ReadAccountSignatures => "read_account_signatures", + Operation::ReadBlock => "read_block", + Operation::ReadBlockRange => "read_block_range", + Operation::Replay => "replay", + Operation::Rotate => "rotate", + Operation::Truncate => "truncate", + Operation::FileSync => "file_sync", + Operation::FileFinalize => "file_finalize", + } + } +} + +/// Registers ledger metrics once and seeds gauges from opened ledger metadata. +pub(crate) fn init(ledger: &Ledger) { + METRICS.get_or_init(|| Metrics::new(ledger)); +} + +/// Starts an operation timer that records latency when the returned guard drops. +pub(crate) fn time(op: Operation) -> metric::OperationTimer<'static> { + op.time(METRICS.get().map(|m| &m.operations)) +} + +/// Refreshes the current pending transaction count. +pub(crate) fn pending_transactions(count: usize) { + metric::with_metrics(&METRICS, |m| { + m.pending_transactions.set(metric::gauge_value(count)) + }); +} + +/// Refreshes ledger-wide count gauges from metadata. +pub(crate) fn ledger_counts(ledger: &Ledger) { + metric::with_metrics(&METRICS, |m| m.ledger_counts(ledger)); +} + +/// Owns all Prometheus collectors registered by ledger. +struct Metrics { + /// Runtime operation duration and completion counters. + operations: OperationCounters, + /// Runtime pending transaction count. + pending_transactions: IntGauge, + /// Runtime transaction total gauge. + transactions: IntGauge, + /// Runtime block total gauge. + blocks: IntGauge, + /// Runtime superblock total gauge. + superblocks: IntGauge, +} + +impl Metrics { + /// Builds collectors and registers them in the default Prometheus registry. + fn new(ledger: &Ledger) -> Self { + Self { + operations: OperationCounters::new(OPERATION_TIME), + pending_transactions: metric::gauge(PENDING_TRANSACTIONS, 0), + transactions: metric::gauge( + TRANSACTIONS, + metric::gauge_value(ledger.meta.transactions.load(Relaxed)), + ), + blocks: metric::gauge( + BLOCKS, + metric::gauge_value(ledger.meta.blocks.load(Relaxed)), + ), + superblocks: metric::gauge(SUPERBLOCKS, metric::gauge_value(ledger.meta.head())), + } + } + + /// Refreshes count gauges from ledger metadata. + fn ledger_counts(&self, ledger: &Ledger) { + self.transactions + .set(metric::gauge_value(ledger.meta.transactions.load(Relaxed))); + self.blocks.set(metric::gauge_value(ledger.meta.blocks.load(Relaxed))); + self.superblocks.set(metric::gauge_value(ledger.meta.head())); + } +} diff --git a/ledger/src/reader.rs b/ledger/src/reader.rs new file mode 100644 index 00000000..cb7750e0 --- /dev/null +++ b/ledger/src/reader.rs @@ -0,0 +1,493 @@ +//! Ledger read-side worker implementation. + +use std::{ + collections::HashMap, + io::{BufRead, BufReader, Read}, + mem, + ops::Range, + os::unix::fs::FileExt, + sync::{Arc, atomic::Ordering::Acquire}, +}; + +use agave_transaction_view::transaction_view::SanitizedTransactionView; +use bitcode::Buffer; +use flume::Receiver; +use nucleus::{ + Slot, + heed::OptRoTxn, + shutdown::{ShutdownHandle, ShutdownReason}, +}; +use solana_signature::Signature; +use tracing::error; +use wincode::{Error, io::Cursor}; +use zstd::bulk::Decompressor; + +use crate::{ + Ledger, LedgerError, Result, Superblock, + index::{AccountIter, Span}, + metrics::{self, Operation}, + request::{ + AccountSignature, AccountSignaturesParams, AccountSignaturesPayload, + AccountSignaturesReadResult, BlockDetails, BlockParams, BlockPayload, BlockReadResult, + BlockResponse, BlockWithSignatures, BlockWithTransactions, FullBlockInfo, ReadRequest, + ReplayParams, ReplayPayload, TransactionPayload, TransactionReadResult, + TransactionResponse, TransactionStatus, TransactionStatusPayload, + TransactionStatusReadResult, + }, + schema::{ + Block, BlockstoreEntry, Execution, ExecutionHeader, MAX_EXECUTION_DETAILS_SIZE, + OwnedBlockstoreEntry, blockstore, + }, +}; + +/// Stateful ledger reader with reusable decoder buffers. +pub(crate) struct LedgerReader { + /// Request stream shared by callers through `LedgerHandle`. + rx: Receiver, + /// Shared top-level ledger state. + ledger: Arc, + /// Reusable zstd decompressor. + decompressor: Decompressor<'static>, + /// Scratch buffers reused across requests. + buffers: ReadBuffers, +} + +impl LedgerReader { + /// Creates a reader over `ledger`. + pub(crate) fn new(ledger: Arc, rx: Receiver) -> Result { + let mut buffers = ReadBuffers::default(); + buffers.details.resize(MAX_EXECUTION_DETAILS_SIZE, 0); + Ok(Self { + rx, + ledger, + decompressor: Decompressor::new()?, + buffers, + }) + } + + /// Serves requests until every sender has been dropped. + pub(crate) fn run(mut self, mut shutdown: ShutdownHandle) { + while let Ok(request) = self.rx.recv() { + match request { + ReadRequest::Shutdown => break, + ReadRequest::Transaction(r) => { + let _timer = metrics::time(Operation::ReadTransaction); + let result = self.transaction(&r); + let _ = r.response.send(result); + } + ReadRequest::TransactionStatus(r) => { + let _timer = metrics::time(Operation::ReadTransactionStatus); + let result = self.transaction_status(&r); + let _ = r.response.send(result); + } + ReadRequest::AccountSignatures(r) => { + let _timer = metrics::time(Operation::ReadAccountSignatures); + let result = self.account_signatures(&r); + let _ = r.response.send(result); + } + ReadRequest::Block(r) => { + let _timer = metrics::time(Operation::ReadBlock); + let result = self.block(&r); + let _ = r.response.send(result); + } + ReadRequest::BlockRange(r) => { + let _timer = metrics::time(Operation::ReadBlockRange); + let result = self.blocks(r.params); + let _ = r.response.send(result); + } + ReadRequest::Replay(r) => { + let _timer = metrics::time(Operation::Replay); + let result = self.replay(&r); + let _ = r.response.send(result); + } + } + } + // Release ledger ownership before the manager can reopen it. + drop(self); + shutdown.terminate(ShutdownReason::Signalled); + } + + /// Reads a full transaction and its execution details. + fn transaction(&mut self, request: &TransactionPayload) -> TransactionReadResult { + for superblock in self.ledger.clone().iter() { + if request.cancelled() { + return Ok(None); + } + let Some(spans) = superblock.index.transaction(&request.params, &mut None)? else { + continue; + }; + return Ok(Some(TransactionResponse { + transaction: self.transaction_entry(&superblock, spans.blockstore)?, + execution: self.execution(&superblock, spans.execution)?, + })); + } + Ok(None) + } + + /// Reads the status header for a transaction. + fn transaction_status( + &mut self, + request: &TransactionStatusPayload, + ) -> TransactionStatusReadResult { + for superblock in self.ledger.clone().iter() { + if request.cancelled() { + return Ok(None); + } + let Some(spans) = superblock.index.transaction(&request.params, &mut None)? else { + continue; + }; + let header = self.header(&superblock, spans.execution)?; + return Ok(Some(TransactionStatus { + result: header.result, + slot: header.slot, + })); + } + Ok(None) + } + + /// Reads recent signatures that mention an account. + fn account_signatures( + &mut self, + request: &AccountSignaturesPayload, + ) -> AccountSignaturesReadResult { + let AccountSignaturesParams { pubkey, limit, mut before, until } = request.params; + let mut signatures = Vec::new(); + if limit == 0 { + return Ok(signatures); + } + let mut blocktimes = HashMap::::new(); + for superblock in self.ledger.clone().iter() { + if request.cancelled() { + return Ok(signatures); + } + let mut txn = None; + let Some(iter) = superblock.index.accounts(&pubkey, &mut txn)? else { + continue; + }; + // SAFETY: `iter` borrows the read transaction stored in `txn`. + // `txn` was populated by `accounts`, remains in this stack frame, + // is not replaced or dropped while `iter` is used, and every later + // index read only reuses that already-open read transaction. + let iter = unsafe { mem::transmute::, AccountIter<'_>>(iter) }; + + let mut upper = None; + if let Some(signature) = &before { + // Skip newest-first segments until `before`, then include every older segment. + let Some(cutoff) = superblock.index.transaction(signature, &mut txn)? else { + continue; + }; + upper.replace(cutoff.execution); + before.take(); + } + + for result in iter { + let span = result?.1; + if let Some(s) = upper + && span >= s + { + continue; + }; + let header = self.header(&superblock, span)?; + let sig: Signature = header.signature; + if let Some(signature) = until + && sig == signature + { + return Ok(signatures); + } + // Several matching signatures can share a slot, so cache block timestamps per slot. + let blocktime = match blocktimes.get(&header.slot) { + Some(time) => *time, + None => { + let time = self.blocktime(&superblock, header.slot, &mut txn)?; + blocktimes.insert(header.slot, time); + time + } + }; + signatures.push(AccountSignature { + signature: sig, + result: header.result, + slot: header.slot, + blocktime, + }); + if signatures.len() >= limit { + return Ok(signatures); + } + } + } + Ok(signatures) + } + + /// Reads a block with the requested transaction detail level. + fn block(&mut self, request: &BlockPayload) -> BlockReadResult { + let slot = request.params.slot; + if !self.ledger.meta.range.contains(&slot) { + return Ok(None); + } + + let mut position = None; + for superblock in self.ledger.clone().iter() { + if request.cancelled() { + return Ok(None); + } + if !superblock.meta.range.contains(&slot) { + continue; + } + let Some(span) = superblock.index.block(&slot, &mut None)? else { + return Ok(None); + }; + position.replace((superblock, span)); + break; + } + let Some((superblock, span)) = position else { return Ok(None) }; + self.populate_block_response(&superblock, request, span).map(Some) + } + + /// Reads block boundaries for a slot range in ascending slot order. + fn blocks(&mut self, range: Range) -> Result> { + let mut blocks = Vec::with_capacity(range.clone().count()); + let mut range = range.into_iter().rev().peekable(); + + for superblock in self.ledger.clone().iter() { + let mut txn = None; + let start = superblock.meta.range.start.load(Acquire); + while let Some(&slot) = range.peek() { + // Slots descend and superblocks go newest to oldest, so a slot + // below this segment's start belongs to an older superblock; + // leave it in the iterator rather than consuming it here. + if slot < start { + break; + } + range.next(); + let Some(span) = superblock.index.block(&slot, &mut txn)? else { + continue; + }; + if let BlockstoreEntry::Block(b) = self.blockstore_entry(&superblock, span)? { + blocks.push(b); + } + } + } + blocks.reverse(); + Ok(blocks) + } + + fn replay(&mut self, request: &ReplayPayload) -> Result<()> { + let ReplayParams { superblock, tx } = &request.params; + for superblock in self.ledger.iter_after(*superblock) { + let limit = superblock.meta.cursors.blockstore.load(Acquire); + let mut reader = BufReader::new((&superblock.blockstore).take(limit)); + loop { + if request.cancelled() || tx.is_closed() { + return Ok(()); + } + if reader.fill_buf()?.is_empty() { + break; + } + let entry = blockstore::decode(&mut reader).map_err(Into::::into)?; + if tx.blocking_send(entry).is_err() { + return Ok(()); + } + } + } + Ok(()) + } + + /// Reads and decodes a raw transaction entry. + fn transaction_entry(&mut self, superblock: &Superblock, span: Span) -> Result> { + let entry = self.blockstore_entry(superblock, span)?; + let BlockstoreEntry::Transaction(transaction) = entry else { + error!( + superblock = superblock.id, + "ledger index points to invalid transaction entry" + ); + return Err(LedgerError::Corruption( + "index points to invalid transaction entry", + )); + }; + Ok(transaction) + } + + /// Reads and decodes a complete execution entry. + fn execution(&mut self, superblock: &Superblock, span: Span) -> Result { + self.buffers.read_executions(superblock, span)?; + let header = self.decode_header()?; + let header_size = wincode::serialized_size(&header).map_err(Into::::into)? as usize; + + let compressed = &self.buffers.executions[header_size..]; + let size = self + .decompressor + .decompress_to_buffer(compressed, self.buffers.details.as_mut_slice())?; + let details = self.buffers.decoder.decode(&self.buffers.details[..size])?; + Ok(Execution { header, details }) + } + + /// Reads and decodes one blockstore entry at an indexed span. + fn blockstore_entry( + &mut self, + superblock: &Superblock, + span: Span, + ) -> Result { + self.buffers.read_blockstore(superblock, span)?; + let bytes = self.buffers.blockstore.as_slice(); + let entry = blockstore::decode(bytes).map_err(Into::::into)?; + Ok(entry) + } + + /// Builds the block response requested by the caller. + fn populate_block_response( + &mut self, + superblock: &Superblock, + request: &BlockPayload, + span: Span, + ) -> Result { + let BlockParams { slot, details } = request.params; + let BlockstoreEntry::Block(block) = self.blockstore_entry(superblock, span)? else { + error!( + superblock = superblock.id, + slot, "ledger index points to invalid block entry" + ); + return Err(LedgerError::Corruption( + "index points to invalid block entry", + )); + }; + + if matches!(details, BlockDetails::None) { + return Ok(BlockResponse::Bare(block)); + } + let mut txn = None; + // Slots are contiguous, so the previous block boundary is always `slot - 1`. + let start = match superblock.index.block(&(slot - 1), &mut txn)? { + Some(previous) => previous.offset() + previous.size(), + None => 0, + }; + self.buffers.read_blockstore_range(superblock, start..span.offset())?; + let blockstore = mem::take(&mut self.buffers.blockstore); + let len = blockstore.len(); + let mut cursor = Cursor::new(blockstore); + let result = (|| { + let mut full = Vec::new(); + let mut transactions = Vec::new(); + let mut signatures = Vec::new(); + while cursor.position() < len && !request.cancelled() { + let entry = blockstore::decode(&mut cursor).map_err(Into::::into)?; + let transaction = match entry { + BlockstoreEntry::Transaction(transaction) => transaction, + BlockstoreEntry::Reset(_) => continue, + _ => { + error!( + superblock = superblock.id, + slot, "ledger blockstore includes invalid block entries" + ); + return Err(LedgerError::Corruption( + "blockstore includes invalid block entries", + )); + } + }; + if matches!(details, BlockDetails::Transactions) { + transactions.push(transaction); + continue; + } + let signature = signature(&transaction)?; + if matches!(details, BlockDetails::Signatures) { + signatures.push(signature); + continue; + } + let Some(spans) = superblock.index.transaction(&signature, &mut txn)? else { + continue; + }; + let execution = self.execution(superblock, spans.execution)?; + full.push(TransactionResponse { transaction, execution }); + } + Ok((full, transactions, signatures)) + })(); + self.buffers.blockstore = cursor.into_inner(); + let (full, transactions, signatures) = result?; + + let response = match details { + BlockDetails::Full => BlockResponse::Full(FullBlockInfo { block, transactions: full }), + BlockDetails::Transactions => { + BlockResponse::WithTransactions(BlockWithTransactions { block, transactions }) + } + BlockDetails::Signatures => { + BlockResponse::WithSignatures(BlockWithSignatures { block, signatures }) + } + BlockDetails::None => BlockResponse::Bare(block), + }; + Ok(response) + } + + /// Reads the timestamp from a block boundary entry. + fn blocktime<'t, 'e>( + &mut self, + superblock: &'e Superblock, + slot: Slot, + txn: OptRoTxn<'t, 'e>, + ) -> Result { + let Some(span) = superblock.index.block(&slot, txn)? else { + return Ok(0); + }; + let entry = self.blockstore_entry(superblock, span)?; + let BlockstoreEntry::Block(block) = entry else { + return Ok(0); + }; + Ok(block.time) + } + + /// Reads only the fixed execution header. + fn header(&mut self, superblock: &Superblock, span: Span) -> Result { + self.buffers.read_executions(superblock, span)?; + self.decode_header() + } + + /// Decodes the execution header currently loaded in the executions buffer. + fn decode_header(&self) -> Result { + let header = wincode::deserialize(self.buffers.executions.as_slice()) + .map_err(Into::::into)?; + Ok(header) + } +} + +/// Reusable read buffers owned by one reader task. +#[derive(Default)] +struct ReadBuffers { + /// Blockstore-file scratch bytes. + blockstore: Vec, + /// Executions-file scratch bytes. + executions: Vec, + /// Bitcode decoder state for execution details. + decoder: Buffer, + /// Decompressed execution-detail payload. + details: Vec, +} + +impl ReadBuffers { + /// Reads blockstore-file bytes at `span` into the reusable buffer. + fn read_blockstore(&mut self, superblock: &Superblock, span: Span) -> Result<()> { + self.blockstore.resize(span.size() as usize, 0); + superblock.blockstore.read_exact_at(&mut self.blockstore, span.offset())?; + Ok(()) + } + + /// Reads executions-file bytes at `span` into the reusable buffer. + fn read_executions(&mut self, superblock: &Superblock, span: Span) -> Result<()> { + self.executions.resize(span.size() as usize, 0); + superblock.executions.read_exact_at(&mut self.executions, span.offset())?; + Ok(()) + } + + /// Reads a byte range from the blockstore file into the reusable buffer. + fn read_blockstore_range(&mut self, superblock: &Superblock, range: Range) -> Result<()> { + self.blockstore.resize((range.end - range.start) as usize, 0); + superblock.blockstore.read_exact_at(&mut self.blockstore, range.start)?; + Ok(()) + } +} + +fn signature(transaction: &[u8]) -> Result { + SanitizedTransactionView::try_new_sanitized(transaction, false) + .map(|transaction| transaction.signatures()[0]) + .map_err(Into::into) +} + +// SAFETY: `LedgerReader` is moved into one background thread and keeps its +// decoder and decompressor state on that thread for the reader lifetime. +unsafe impl Send for LedgerReader {} diff --git a/ledger/src/request.rs b/ledger/src/request.rs new file mode 100644 index 00000000..ca4cf71a --- /dev/null +++ b/ledger/src/request.rs @@ -0,0 +1,264 @@ +//! Ledger read-side request and response types. + +use std::{ + ops::Range, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + time::Duration, +}; + +use flume::Sender as RequestSender; +use nucleus::Slot; +use oneshot::{Receiver, Sender}; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_transaction_error::TransactionResult; +use tokio::{sync::mpsc, time}; + +use crate::{ + Result, + error::RequestResult, + schema::{Block, Execution, OwnedBlockstoreEntry}, +}; + +/// Result returned by a full transaction lookup. +pub(crate) type TransactionReadResult = Result>; +/// Result returned by a transaction-status lookup. +pub(crate) type TransactionStatusReadResult = Result>; +/// Result returned by an account-signature history lookup. +pub(crate) type AccountSignaturesReadResult = Result>; +/// Result returned by a block lookup. +pub(crate) type BlockReadResult = Result>; + +/// Payload for a transaction lookup request. +pub(crate) type TransactionPayload = RequestPayload; +/// Payload for a transaction-status lookup request. +pub(crate) type TransactionStatusPayload = RequestPayload; +/// Payload for an account-signature history request. +pub(crate) type AccountSignaturesPayload = + RequestPayload; +/// Payload for a single-block lookup request. +pub(crate) type BlockPayload = RequestPayload; +/// Payload for a contiguous block-range request. +pub(crate) type BlockRangePayload = RequestPayload, Result>>; +/// Payload for replaying owned blockstore entries after a sealed superblock. +pub(crate) type ReplayPayload = RequestPayload>; + +/// Read request queue consumed by reader workers. +pub type ReaderSender = RequestSender; + +/// Handle for consuming a ledger replay stream. +pub struct ReplayHandle { + /// Entries streamed from retained blockstore data in on-disk order. + pub rx: mpsc::Receiver, + /// Completion result sent after the reader finishes streaming entries. + pub response: RequestHandle>, +} + +/// Signature summary returned for account history queries. +pub struct AccountSignature { + /// Transaction signature. + pub signature: Signature, + /// Slot where the transaction executed. + pub slot: Slot, + /// Runtime transaction result. + pub result: TransactionResult<()>, + /// Block timestamp for `slot`, or `0` when unavailable. + pub blocktime: i64, +} + +/// Full transaction response with optional execution metadata. +pub struct TransactionResponse { + /// Serialized transaction bytes from the blockstore file. + pub transaction: Vec, + /// Execution metadata when requested or available. + pub execution: Execution, +} + +/// Cheap transaction-status response. +#[derive(Clone)] +pub struct TransactionStatus { + /// Runtime transaction result. + pub result: TransactionResult<()>, + /// Slot where the transaction executed. + pub slot: Slot, +} + +/// Account-signature query parameters. +pub struct AccountSignaturesParams { + /// Account pubkey to search for. + pub pubkey: Pubkey, + /// Maximum number of signatures to return. + pub limit: usize, + /// Signature before which results should start. + pub before: Option, + /// Signature at which results should stop. + pub until: Option, +} + +/// Parameters for streaming retained blockstore entries into a replay consumer. +pub struct ReplayParams { + /// Last sealed superblock already reflected in the consumer's state. + pub superblock: u64, + /// Channel receiving blockstore entries in on-disk order. + pub tx: mpsc::Sender, +} + +/// Read request sent to a ledger reader service. +pub enum ReadRequest { + /// Stop this reader after all requests queued before this marker. + Shutdown, + /// Full transaction lookup by signature. + Transaction(TransactionPayload), + /// Transaction status lookup by signature. + TransactionStatus(TransactionStatusPayload), + /// Account-signature history lookup. + AccountSignatures(AccountSignaturesPayload), + /// Block lookup by slot and detail level. + Block(BlockPayload), + /// Contiguous block boundary lookup. + BlockRange(BlockRangePayload), + /// Replay retained blockstore entries after an applied superblock seal. + Replay(ReplayPayload), +} + +/// Block lookup response at the requested detail level. +pub enum BlockResponse { + /// Block boundary only. + Bare(Block), + /// Block boundary with transaction signatures. + WithSignatures(BlockWithSignatures), + /// Block boundary with serialized transactions. + WithTransactions(BlockWithTransactions), + /// Block boundary with serialized transactions and execution metadata. + Full(FullBlockInfo), +} + +impl BlockResponse { + /// Returns the block boundary included in every response variant. + pub fn block(&self) -> &Block { + match self { + Self::Bare(b) => b, + Self::WithSignatures(b) => &b.block, + Self::WithTransactions(b) => &b.block, + Self::Full(b) => &b.block, + } + } +} + +/// Full block response with execution metadata. +pub struct FullBlockInfo { + /// Block boundary. + pub block: Block, + /// Transactions with matching execution metadata. + pub transactions: Vec, +} + +/// Block response containing serialized transactions only. +pub struct BlockWithTransactions { + /// Block boundary. + pub block: Block, + /// Serialized transactions in block order. + pub transactions: Vec>, +} + +/// Block response containing transaction signatures only. +pub struct BlockWithSignatures { + /// Block boundary. + pub block: Block, + /// Transaction signatures in block order. + pub signatures: Vec, +} + +/// Handle used by the request caller to await a reader response. +pub struct RequestHandle { + /// One-shot response receiver. + rx: Receiver, + /// Pending flag guard cleared when the response is no longer needed. + pending: PendingGuard, +} + +impl RequestHandle { + /// Wait up to one minute for a reader response. + /// + /// Dropping the handle before a response clears the request's pending flag + /// so long-running readers can stop work that no caller will receive. + pub async fn recv_timeout(self) -> RequestResult { + const ONE_MINUTE: Duration = Duration::from_secs(60); + time::timeout(ONE_MINUTE, self.recv()).await? + } + + /// Wait for a reader response without applying a timeout. + pub async fn recv(mut self) -> RequestResult { + let result = self.rx.await?; + self.pending.0.take(); + Ok(result) + } +} + +/// Clears the request's pending flag when dropped. +struct PendingGuard(Option>); + +impl Drop for PendingGuard { + fn drop(&mut self) { + if let Some(pending) = self.0.take() { + pending.store(false, Ordering::Release); + } + } +} + +/// Request payload with cancellation and oneshot response channel. +pub struct RequestPayload { + /// Request parameters. + pub params: P, + /// Response channel for the result. + pub response: Sender, + /// Set while the caller is still waiting for the response. + pub pending: Arc, +} + +impl RequestPayload { + /// Creates a request payload and the handle that receives its response. + pub fn new(params: P) -> (Self, RequestHandle) { + let (response, rx) = oneshot::channel(); + let pending = Arc::new(AtomicBool::new(true)); + let payload = Self { + params, + response, + pending: pending.clone(), + }; + let handle = RequestHandle { + rx, + pending: PendingGuard(Some(pending)), + }; + (payload, handle) + } + + /// Returns whether the caller stopped waiting for this response. + pub(crate) fn cancelled(&self) -> bool { + !self.pending.load(Ordering::Acquire) + } +} + +/// Block query parameters. +pub struct BlockParams { + /// Slot to look up. + pub slot: Slot, + /// Amount of transaction data to include in the response. + pub details: BlockDetails, +} + +/// Transaction detail level for a block lookup. +#[derive(Clone, Copy)] +pub enum BlockDetails { + /// Include transactions and execution details. + Full, + /// Include transactions without execution details. + Transactions, + /// Include only transaction signatures. + Signatures, + /// Include only the block boundary entry. + None, +} diff --git a/ledger/src/schema.rs b/ledger/src/schema.rs new file mode 100644 index 00000000..cbbd84d7 --- /dev/null +++ b/ledger/src/schema.rs @@ -0,0 +1,187 @@ +//! Ledger wire, event, and on-disk blockstore formats. +//! +//! The blockstore is a wincode stream of raw transactions in execution order. +//! Block entries delimit the transactions that belong to each Solana-like +//! block, and a superblock seal terminates a truncatable group of blocks. +//! Execution details live in a separate file as a wincode header followed by +//! a zstd-compressed bitcode payload. + +use std::sync::Arc; + +use bitcode::{Decode, Encode}; +use nucleus::Slot; +pub use nucleus::ledger::{Block, SuperblockSeal}; + +use solana_signature::Signature; +use solana_transaction_error::TransactionResult; +use wincode::{SchemaRead, SchemaWrite}; + +use crate::{error::Result, index::Span}; + +/// Byte offset into a ledger data file. +pub(crate) type Offset = u64; + +/// Owned blockstore entry used when replaying persisted ledger data. +pub type OwnedBlockstoreEntry = BlockstoreEntry>; + +/// Largest encoded ledger entry representable by an index span. +pub const MAX_ENTRY_SIZE: usize = Span::MAX_SIZE as usize; + +/// Largest decoded execution-details payload retained by the ledger. +/// +/// Compressed execution records are normally only a few KiB, so 4 MiB leaves +/// substantial headroom while bounding decompression and bitcode decoding. +pub(crate) const MAX_EXECUTION_DETAILS_SIZE: usize = 4 * nucleus::MB; + +/// Codec for entries in the blockstore stream. +/// +/// Owned payloads may allocate up to [`MAX_ENTRY_SIZE`], rather than wincode's +/// smaller default. Writers enforce the encoded-size bound before reaching this +/// codec. +pub mod blockstore { + use super::{BlockstoreEntry, MAX_ENTRY_SIZE, OwnedBlockstoreEntry}; + use wincode::{ + ReadResult, WriteResult, + config::Configuration, + io::{Reader, Writer}, + }; + + /// Decodes the next entry without consuming bytes belonging to a following entry. + pub fn decode<'de>(src: impl Reader<'de>) -> ReadResult { + let config = Configuration::default().with_preallocation_size_limit::(); + wincode::config::deserialize_from(src, config) + } + + /// Writes one entry without changing the default blockstore wire encoding. + pub(crate) fn encode(dst: impl Writer, entry: &BlockstoreEntry<&[u8]>) -> WriteResult<()> { + let config = Configuration::default().with_preallocation_size_limit::(); + wincode::config::serialize_into(dst, entry, config) + } +} + +/// Work item sent to the ledger appender. +// Keep execution inline to avoid one allocation per append event. +#[allow(clippy::large_enum_variant)] +pub enum Event { + /// Transaction bytes accepted for later execution indexing. + Transaction(TransactionEntry), + /// Runtime execution metadata for a previously appended transaction. + Execution(Execution), + /// Block boundary marker and block hash. + Block(Block), + /// Seal the active superblock and rotate to a fresh directory. + Superblock(SuperblockSeal), + /// Install a snapshot seal and adopt its cumulative transaction count. + Bootstrap(SuperblockSeal), + /// Volatile accounts were discarded at `Slot` after upstream synchronization was lost. + Reset(Slot), + /// Flush pending appends and optionally stop the appender after acknowledging. + Sync { + /// Receives the durability result. + response: oneshot::Sender>, + /// Whether this is the terminal engine sync. + is_final: bool, + }, +} + +/// Raw transaction payload queued for blockstore append. +pub struct TransactionEntry { + /// First transaction signature used for status and execution indexes. + pub signature: Signature, + /// Serialized sanitized transaction bytes shared with the appender. + pub payload: Arc>, +} + +/// One typed entry in the blockstore stream. +#[derive(SchemaRead, SchemaWrite)] +pub enum BlockstoreEntry { + /// Delimits the preceding transactions as one block and stores its hash. + Block(Block), + /// Serialized transaction bytes. + Transaction(T), + /// Superblock seal marker and snapshot checksum. + Superblock(SuperblockSeal), + /// Marks a volatile-state reset in the durable replay stream. + Reset(Slot), +} + +/// Fixed execution prefix stored before the compressed bitcode payload. +#[derive(SchemaRead, SchemaWrite)] +pub struct ExecutionHeader { + /// Transaction signature. + pub signature: Signature, + /// Runtime transaction error, if execution failed. + pub result: TransactionResult<()>, + /// Slot where the transaction executed. + pub slot: Slot, +} + +/// Complete transaction execution metadata. +pub struct Execution { + /// Fixed prefix used for cheap signature and error reads. + pub header: ExecutionHeader, + /// Execution details stored in the compressed payload. + pub details: Option, +} + +/// Transaction execution details stored after the execution header. +#[derive(Encode, Decode)] +pub struct ExecutionDetails { + /// Fee charged for the transaction. + pub fee: u64, + /// Pre/post balance deltas encoded in a compact prototype layout. + pub balances: Balances, + /// Log messages emitted during execution. + pub logs: Arc>, + /// Cross-program invocation trace. + pub cpi: Option>, + /// Compute units consumed. + pub compute_units: u64, + /// Program return data, if any. + pub return_data: Option, +} + +/// One cross-program invocation group. +#[derive(Encode, Decode)] +pub struct Cpis( + /// Inner instructions invoked by one transaction-level instruction. + pub Vec, +); + +/// Runtime instruction metadata. +#[derive(Encode, Decode)] +pub struct Instruction { + /// Compiled Solana instruction. + pub compiled: CompiledInstruction, + /// Invocation stack height; transaction-level instructions start at 1. + pub stack_height: u8, +} + +/// Compact compiled instruction form. +#[derive(Encode, Decode)] +pub struct CompiledInstruction { + /// Program account index. + pub program_index: u8, + /// Account indexes referenced by the instruction. + pub accounts: Vec, + /// Opaque instruction data. + pub data: Vec, +} + +/// Program return data. +#[derive(Encode, Decode)] +pub struct ReturnData { + /// Program that produced the return data. + pub program: [u8; 32], + /// Opaque return bytes. + pub data: Arc>, +} + +/// Pre/post balance vectors. +#[derive(Encode, Decode)] +pub struct Balances { + /// Balances before execution. + pub pre: Vec, + /// Balances after execution. + pub post: Vec, +} diff --git a/ledger/src/storage.rs b/ledger/src/storage.rs new file mode 100644 index 00000000..e4fc99d4 --- /dev/null +++ b/ledger/src/storage.rs @@ -0,0 +1,302 @@ +//! Superblock storage primitives. +//! +//! This module keeps the low-level storage surface in one place: buffered +//! append files and mmap-backed metadata headers. + +use std::{ + fs::File, + io::{self, Cursor, Write}, + ops::{Deref, Range}, + os::{ + fd::{AsFd, AsRawFd}, + unix::fs::FileExt, + }, + path::Path, + ptr::NonNull, + sync::atomic::{AtomicU64, Ordering::*}, +}; + +use memmap2::{MmapMut, MmapOptions}; +use nucleus::{MB, Slot}; +use rustix::fs::{self, FallocateFlags}; +use tracing::debug; +use zstd::bulk::Compressor; + +use crate::{Result, index::Span, schema::MAX_ENTRY_SIZE}; + +/// Initial buffer size for append-heavy files. +const FILE_BUFFER_SIZE: usize = 64 * MB; +/// Physical space reserved when an append file is close to its allocated end. +// NOTE: fallocate is slow on non-linux, so we use tiny increments for tests/dev +#[cfg(any(test, target_os = "macos"))] +const PREALLOCATION_SIZE: u64 = MB as u64; +#[cfg(not(any(test, target_os = "macos")))] +const PREALLOCATION_SIZE: u64 = 4 * nucleus::GB as u64; + +/// Remaining allocation that triggers another reservation. +const PREALLOCATION_THRESHOLD: u64 = PREALLOCATION_SIZE / 16; + +/// Buffered append writer that tracks logical file position. +pub(crate) struct AppendFile { + /// Backing file. + pub(crate) file: File, + /// Pending bytes not yet flushed to the backing file. + buffer: Vec, + /// Logical byte position including buffered bytes. + pub(crate) cursor: u64, + len: u64, +} + +impl AppendFile { + /// Opens an append file with spare capacity past its flush threshold. + pub(crate) fn new(path: &Path, cursor: &AtomicU64) -> Result { + let file = + File::options().create(true).truncate(false).read(true).write(true).open(path)?; + let cursor = cursor.load(Acquire); + let len = file.metadata()?.len(); + Ok(Self { + file, + buffer: Vec::with_capacity(FILE_BUFFER_SIZE + MAX_ENTRY_SIZE), + cursor, + len, + }) + } + + /// Compresses directly into the pending buffer and advances the logical cursor. + pub(crate) fn compress( + &mut self, + bytes: &[u8], + compressor: &mut Compressor<'static>, + ) -> io::Result<()> { + let buffered = self.buffer.len(); + let written = { + let mut destination = Cursor::new(&mut self.buffer); + destination.set_position(buffered as u64); + compressor.compress_to_buffer(bytes, &mut destination)? + }; + self.cursor += written as u64; + if self.buffer.len() >= FILE_BUFFER_SIZE { + self.flush()?; + } + Ok(()) + } + + /// Buffers a complete encoded fragment and advances the logical cursor. + #[inline] + pub(crate) fn append(&mut self, bytes: &[u8]) -> io::Result<()> { + self.buffer.extend_from_slice(bytes); + self.cursor += bytes.len() as u64; + if self.buffer.len() >= FILE_BUFFER_SIZE { + self.flush()?; + } + Ok(()) + } + + /// Flushes buffered bytes, syncs file data, and returns the durable cursor. + pub(crate) fn sync(&mut self) -> Result { + if self.len.saturating_sub(self.cursor) < PREALLOCATION_THRESHOLD { + self.preallocate()?; + } + self.flush()?; + self.file.sync_data()?; + Ok(self.cursor) + } + + /// Extends the physical file without changing the logical append cursor. + fn preallocate(&mut self) -> Result<()> { + let offset = self.len; + if (offset + PREALLOCATION_SIZE) > Span::MAX_FILE_SIZE { + Err(io::Error::from(io::ErrorKind::FileTooLarge))?; + } + fs::fallocate( + self.file.as_fd(), + FallocateFlags::empty(), + offset, + PREALLOCATION_SIZE, + )?; + self.file.sync_all()?; + self.len = offset + PREALLOCATION_SIZE; + debug!(len = self.len, "preallocated ledger file space"); + Ok(()) + } + + /// Trims unused preallocated space after the superblock is sealed. + pub(crate) fn finalize(&mut self) -> Result<()> { + self.flush()?; + self.file.set_len(self.cursor)?; + self.file.sync_all()?; + self.len = self.cursor; + Ok(()) + } +} + +impl Write for AppendFile { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.append(buf)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + if self.buffer.is_empty() { + return Ok(()); + } + // `cursor` includes the buffered bytes; indexes use these logical offsets. + let offset = self.cursor - self.buffer.len() as u64; + self.file.write_all_at(&self.buffer, offset)?; + self.buffer.clear(); + self.len = self.len.max(self.cursor); + Ok(()) + } +} + +impl wincode::io::Writer for AppendFile { + #[inline] + fn write(&mut self, src: &[u8]) -> wincode::io::WriteResult<()> { + self.append(src)?; + Ok(()) + } +} + +/// Mmap-backed fixed-size metadata header. +pub(crate) struct MetaMap { + /// Typed pointer into the mapped header. + data: NonNull, + /// Mutable mapping kept alive for `data`. + mmap: MmapMut, +} + +impl MetaMap { + /// Opens `path` and initializes it with `T::default()` when empty. + /// + /// # Safety + /// + /// `T` must be a plain fixed-layout metadata header that can be safely + /// read from bytes previously written by this process. All shared mutation + /// inside `T` must use synchronization such as atomics. + #[allow(unsafe_op_in_unsafe_fn)] + pub(crate) unsafe fn new(path: &Path) -> Result { + let size = size_of::().div_ceil(512) * 512; + let file = + File::options().write(true).read(true).create(true).truncate(false).open(path)?; + if file.metadata()?.len() == 0 { + file.set_len(size as u64)?; + let mut mmap = MmapOptions::new().len(size).map_mut(file.as_raw_fd())?; + mmap.as_mut_ptr().cast::().write(T::default()); + mmap.flush()?; + let data = NonNull::new_unchecked(mmap.as_mut_ptr().cast::()); + return Ok(Self { data, mmap }); + } + + // SAFETY: `size` matches the fixed metadata header size used when the + // file was created, and `mmap` is owned by the returned `MetaMap`. + let mut mmap = MmapOptions::new().len(size).map_mut(file.as_raw_fd())?; + // SAFETY: mmap pointers are non-null for non-empty mappings. + let data = NonNull::new_unchecked(mmap.as_mut_ptr().cast::()); + Ok(Self { data, mmap }) + } + + /// Flushes metadata changes to disk. + #[inline] + pub(crate) fn flush(&self) -> Result<()> { + self.mmap.flush().map_err(Into::into) + } +} + +impl Deref for MetaMap { + type Target = T; + fn deref(&self) -> &Self::Target { + unsafe { self.data.as_ref() } + } +} + +// SAFETY: `MetaMap` owns the mapping that backs `data`; moving the wrapper +// does not invalidate the pointer. Cross-thread access is constrained by `T`. +unsafe impl Send for MetaMap {} + +// SAFETY: shared access exposes only `&T`. The metadata headers used by the +// ledger mutate shared state through atomics. +unsafe impl Sync for MetaMap {} + +/// Ledger-wide metadata header. +#[repr(C)] +pub(crate) struct LedgerMeta { + /// Total transactions committed since genesis. + pub(crate) transactions: AtomicU64, + /// Number of retained superblocks. + pub(crate) superblocks: AtomicU64, + /// Active superblock identifier. + pub(crate) head: AtomicU64, + /// Inclusive slot range covered by retained superblocks. + pub(crate) range: BlockRange, + /// Total blocks committed since genesis. + pub(crate) blocks: AtomicU64, +} + +impl Default for LedgerMeta { + fn default() -> Self { + Self { + transactions: 0.into(), + head: 1.into(), + superblocks: 1.into(), + range: Default::default(), + blocks: 0.into(), + } + } +} + +impl LedgerMeta { + /// Returns the active superblock id. + #[inline] + pub(crate) fn head(&self) -> u64 { + self.head.load(Acquire) + } + + /// Returns the retained superblock id range, including the active head. + #[inline] + pub(crate) fn superblocks(&self) -> Range { + let head = self.head(); + let start = head.saturating_sub(self.superblocks.load(Acquire).saturating_sub(1)); + start..head + 1 + } +} + +/// Metadata header for one superblock directory. +#[derive(Default)] +#[repr(C)] +pub(crate) struct SuperblockMeta { + /// Durable append cursors for files in this superblock. + pub(crate) cursors: FileCursors, + /// Slot range stored in this segment. + pub(crate) range: BlockRange, + /// Accountsdb snapshot checksum carried over from the seal that opened this superblock. + pub(crate) checksum: AtomicU64, + /// Transaction count carried over from the seal that opened this superblock. + pub(crate) transactions: AtomicU64, +} + +/// Durable append cursors for superblock data files. +#[derive(Default)] +#[repr(C)] +pub(crate) struct FileCursors { + /// Durable byte cursor in `blockstore.db`. + pub(crate) blockstore: AtomicU64, + /// Durable byte cursor in `executions.db`. + pub(crate) executions: AtomicU64, +} + +/// Inclusive slot range covered by a superblock. +#[derive(Default)] +#[repr(C)] +pub(crate) struct BlockRange { + /// First slot in the segment. + pub(crate) start: AtomicU64, + /// Last slot in the segment. + pub(crate) end: AtomicU64, +} + +impl BlockRange { + /// Returns true when `slot` is inside the range. + pub(crate) fn contains(&self, slot: &Slot) -> bool { + (self.start.load(Acquire)..=self.end.load(Acquire)).contains(slot) + } +} diff --git a/ledger/src/tests/index.rs b/ledger/src/tests/index.rs new file mode 100644 index 00000000..4307eb36 --- /dev/null +++ b/ledger/src/tests/index.rs @@ -0,0 +1,102 @@ +//! Index unit tests. + +use nucleus::{ + Slot, + heed::DatabaseIndex, + testkit::{TempDir, init_tracing, tempdir}, +}; +use solana_pubkey::Pubkey; +use solana_signature::Signature; + +use crate::index::{Index, Span, TxSpan}; + +/// Opens a fresh index on a throwaway directory kept alive by the returned guard. +fn index() -> (TempDir, Index) { + init_tracing(); + let dir = tempdir(); + let index = Index::new(dir.path()).unwrap(); + (dir, index) +} + +/// Drains every execution span the account index holds for `pubkey`. +fn account_spans(index: &Index, pubkey: &Pubkey) -> Vec { + let mut txn = None; + let Some(iter) = index.accounts(pubkey, &mut txn).unwrap() else { + return Vec::new(); + }; + iter.map(|entry| entry.unwrap().1).collect() +} + +#[test] +fn span_pack_and_order() { + let span = Span::new(1234, 56); + assert_eq!(span.offset(), 1234); + assert_eq!(span.size(), 56); + + // Boundary values: full size field, and an offset filling every high bit. + let max_size = Span::new(0, Span::MAX_SIZE); + assert_eq!((max_size.offset(), max_size.size()), (0, Span::MAX_SIZE)); + let max_offset = u64::MAX >> 25; + let high = Span::new(max_offset, 0); + assert_eq!((high.offset(), high.size()), (max_offset, 0)); + + // Ordering is offset-dominant: a later offset outranks any size at an + // earlier offset, and size only breaks ties within one offset. + assert!(Span::new(11, 0) > Span::new(10, Span::MAX_SIZE)); + assert!(Span::new(10, 7) > Span::new(10, 5)); +} + +#[test] +fn transaction_and_block_roundtrip() { + let (_dir, index) = index(); + let signature = Signature::from([7; 64]); + let txspan = TxSpan { + blockstore: Span::new(10, 20), + execution: Span::new(30, 40), + }; + let (slot_a, slot_b): (Slot, Slot) = (1, 2); + let block_a = Span::new(0, 8); + let block_b = Span::new(8, 16); + + let mut txn = index.env().write_txn().unwrap(); + index.insert_transaction(&mut txn, &signature, &txspan).unwrap(); + index.insert_block(&mut txn, &slot_a, &block_a).unwrap(); + index.insert_block(&mut txn, &slot_b, &block_b).unwrap(); + txn.commit().unwrap(); + + let mut txn = None; + let got = index.transaction(&signature, &mut txn).unwrap().expect("transaction present"); + assert_eq!(got.blockstore, txspan.blockstore); + assert_eq!(got.execution, txspan.execution); + assert_eq!(index.block(&slot_a, &mut txn).unwrap(), Some(block_a)); + assert_eq!(index.block(&slot_b, &mut txn).unwrap(), Some(block_b)); + + // Absent keys resolve to nothing rather than a stale or default hit. + assert!(index.transaction(&Signature::from([9; 64]), &mut txn).unwrap().is_none()); + assert_eq!(index.block(&99, &mut txn).unwrap(), None); +} + +#[test] +fn account_signature_duplicates() { + let (_dir, index) = index(); + let account = Pubkey::new_unique(); + let other = Pubkey::new_unique(); + let spans = [Span::new(100, 10), Span::new(200, 20), Span::new(300, 30)]; + let other_span = Span::new(400, 40); + + let mut txn = index.env().write_txn().unwrap(); + for span in &spans { + index.insert_accounts(&mut txn, &[account], span).unwrap(); + } + index.insert_accounts(&mut txn, &[other], &other_span).unwrap(); + txn.commit().unwrap(); + + // Account duplicate spans are newest-first, so later execution offsets are returned first. + assert_eq!( + account_spans(&index, &account), + vec![spans[2], spans[1], spans[0]] + ); + + // Duplicates stay partitioned per account key. + assert_eq!(account_spans(&index, &other), vec![other_span]); +} diff --git a/ledger/src/tests/integration.rs b/ledger/src/tests/integration.rs new file mode 100644 index 00000000..bceb391b --- /dev/null +++ b/ledger/src/tests/integration.rs @@ -0,0 +1,554 @@ +//! End-to-end tests over the append→seal→read pipeline. +//! +//! Each test drives real [`Event`]s through the [`LedgerAppender`] and reads +//! them back through the [`LedgerReader`] without starting the service pool. +//! Single-response readers run synchronously on the test thread; replay uses a +//! worker thread so the test can drain its bounded channel concurrently. +//! Transactions are genuine wincode-serialized Solana transactions so the +//! appender's account/signature extraction and the reader's block reconstruction +//! exercise the real codecs. +//! +//! The appender only makes data durable at a block boundary (sync + index +//! commit + cursor publish), so every append batch here ends with a `Block`; +//! that also mirrors how a caller must frame writes. + +use std::{ + ops::Range, + sync::{Arc, atomic::Ordering::Acquire}, +}; + +use nucleus::{ + MB, Slot, + ledger::{Block, SuperblockSeal}, + shutdown::{Service, ShutdownManager}, + testkit::{TempDir, init_tracing, tempdir, transaction}, +}; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_transaction_error::TransactionResult; +use tokio::sync::{broadcast, mpsc}; + +use crate::{ + Ledger, + appender::LedgerAppender, + reader::LedgerReader, + request::{ + AccountSignature, AccountSignaturesParams, BlockDetails, BlockParams, BlockResponse, + ReadRequest, ReplayParams, RequestPayload, TransactionResponse, + }, + schema::{ + Balances, Event, Execution, ExecutionDetails, ExecutionHeader, OwnedBlockstoreEntry, + TransactionEntry, + }, +}; + +/// Fresh ledger on a throwaway directory; the `TempDir` must outlive the ledger. +/// +/// `size_limit` gates retention: `u64::MAX` disables it, `0` forces `truncate` +/// to run at every block boundary (the whole ledger filesystem always counts as +/// "over budget"). +fn ledger(size_limit: u64) -> (TempDir, Arc) { + init_tracing(); + let dir = tempdir(); + let ledger = Arc::new(Ledger::new(dir.path().to_owned(), size_limit).unwrap()); + (dir, ledger) +} + +/// Feeds `events` through a freshly opened appender and runs it to completion. +/// +/// The appender resumes from on-disk cursors, so successive calls model both +/// continued appends and a restart of the write side. +fn append(ledger: &Arc, events: Vec) { + let (tx, rx) = flume::bounded(events.len().max(1)); + let (position, _) = broadcast::channel(64); + for event in events { + tx.send(event).unwrap(); + } + drop(tx); + let mut shutdown = ShutdownManager::default(); + LedgerAppender::new(ledger.clone(), rx, position) + .unwrap() + .run(shutdown.handle(Service::LedgerAppender)); +} + +/// Execution metadata carrying a recognizable `fee`/`logs` for read assertions. +fn execution(signature: Signature, slot: Slot, result: TransactionResult<()>) -> Execution { + Execution { + header: ExecutionHeader { signature, result, slot }, + details: Some(ExecutionDetails { + fee: slot * 1000, + balances: Balances { pre: vec![1], post: vec![2] }, + logs: Arc::new(vec![format!("log for {slot}")]), + cpi: None, + compute_units: slot * 7, + return_data: None, + }), + } +} + +/// The `Transaction` + paired `Execution` events that record one transaction in +/// a block at `slot` (execution result `Ok`). +fn recorded(sig: Signature, payload: Arc>, slot: Slot) -> [Event; 2] { + [ + Event::Transaction(TransactionEntry { signature: sig, payload }), + Event::Execution(execution(sig, slot, Ok(()))), + ] +} + +/// Ends superblock `id` and rotates the writer to the next one. +fn seal(id: u64) -> Event { + Event::Superblock(SuperblockSeal { checksum: 0, id, transactions: 0 }) +} + +/// Serves one read request on a reader run synchronously on the test thread. +/// +/// `wrap` is the `ReadRequest` variant to route `params` through — the variants +/// are tuple constructors, so `read(&ledger, sig, ReadRequest::Transaction)` +/// names the request and infers both the parameter and response types. The +/// reader terminates as soon as the single queued request is served. +async fn read( + ledger: &Arc, + params: P, + wrap: impl FnOnce(RequestPayload) -> ReadRequest, +) -> R { + let (payload, handle) = RequestPayload::new(params); + let (tx, reader_rx) = flume::bounded(1); + tx.send(wrap(payload)).unwrap(); + drop(tx); + let mut shutdown = ShutdownManager::default(); + LedgerReader::new(ledger.clone(), reader_rx) + .unwrap() + .run(shutdown.handle(Service::LedgerReader)); + handle.recv().await.unwrap() +} + +/// Reads a full transaction by signature. +async fn read_transaction(ledger: &Arc, sig: Signature) -> Option { + read(ledger, sig, ReadRequest::Transaction).await.unwrap() +} + +/// Reads a block at the requested detail level. +async fn read_block( + ledger: &Arc, + slot: Slot, + details: BlockDetails, +) -> Option { + read(ledger, BlockParams { slot, details }, ReadRequest::Block).await.unwrap() +} + +/// Signatures of the block at `slot`; panics if the read returns any other +/// variant or nothing. +async fn block_signatures(ledger: &Arc, slot: Slot) -> Vec { + match read_block(ledger, slot, BlockDetails::Signatures).await { + Some(BlockResponse::WithSignatures(b)) => b.signatures, + _ => panic!("expected signatures response for slot {slot}"), + } +} + +/// Streams every replayed blockstore entry after `superblock`. +/// +/// The reader runs on its own thread while the test drains the channel, so a +/// small mpsc buffer never deadlocks the `blocking_send` in the replay loop. +async fn replay(ledger: &Arc, superblock: u64) -> Vec { + let (tx, mut rx) = mpsc::channel(4); + let (payload, handle) = RequestPayload::new(ReplayParams { superblock, tx }); + let (reader_tx, reader_rx) = flume::bounded(1); + reader_tx.send(ReadRequest::Replay(payload)).unwrap(); + drop(reader_tx); + let ledger = ledger.clone(); + let worker = std::thread::spawn(move || { + let mut shutdown = ShutdownManager::default(); + LedgerReader::new(ledger, reader_rx) + .unwrap() + .run(shutdown.handle(Service::LedgerReader)); + }); + let mut entries = Vec::new(); + while let Some(entry) = rx.recv().await { + entries.push(entry); + } + handle.recv().await.unwrap().unwrap(); + worker.join().unwrap(); + entries +} + +/// Appends one block at `slot` whose transactions touch `accounts` — one +/// single-account transaction per entry, each paired with its execution — +/// and returns their signatures in append order. +fn block_touching(ledger: &Arc, slot: Slot, accounts: &[Pubkey]) -> Vec { + let mut events = Vec::new(); + let mut signatures = Vec::new(); + for account in accounts { + let (sig, payload) = transaction(&[*account]); + events.extend(recorded(sig, payload, slot)); + signatures.push(sig); + } + events.push(Event::Block(Block::new(slot, slot as i64 * 100))); + append(ledger, events); + signatures +} + +/// [`block_touching`] with `count` transactions over distinct fresh accounts, +/// for tests that care about the block, not about who it touches. +fn block_of(ledger: &Arc, slot: Slot, count: usize) -> Vec { + let accounts: Vec = (0..count).map(|_| Pubkey::new_unique()).collect(); + block_touching(ledger, slot, &accounts) +} + +// A transaction and its execution written in one block come back whole through +// every read surface: full transaction bytes, decompressed execution details, +// and the cheap status header — the core append→index→read roundtrip. +#[tokio::test] +async fn test_transaction_roundtrip() { + let (_dir, ledger) = ledger(u64::MAX); + let account = Pubkey::new_unique(); + let (sig, bytes) = transaction(&[account]); + + let events = vec![ + Event::Transaction(TransactionEntry { + signature: sig, + payload: bytes.clone(), + }), + Event::Execution(execution(sig, 5, Ok(()))), + Event::Block(Block::new(5, 500)), + ]; + append(&ledger, events); + + let response = read_transaction(&ledger, sig).await.expect("transaction present"); + assert_eq!(response.transaction, *bytes); + assert_eq!(response.execution.header.slot, 5); + let details = response.execution.details.expect("details decompressed"); + assert_eq!(details.fee, 5000); + assert_eq!(details.logs.as_slice(), &["log for 5".to_string()]); + + let status = read(&ledger, sig, ReadRequest::TransactionStatus) + .await + .unwrap() + .expect("status present"); + assert_eq!(status.slot, 5); + assert!(status.result.is_ok()); + + // An unknown signature resolves to nothing on both surfaces. + let missing = Signature::from([9; 64]); + assert!(read_transaction(&ledger, missing).await.is_none()); + let status = read(&ledger, missing, ReadRequest::TransactionStatus).await.unwrap(); + assert!(status.is_none(), "unknown signature has no status"); +} + +// Blockstore payloads may exceed wincode's default 4 MiB preallocation limit. +// Persisting, block reconstruction, and replay all use the ledger-specific +// codec bound and must return the original owned bytes unchanged. +#[tokio::test] +async fn test_large_transaction_roundtrip() { + let (_dir, ledger) = ledger(u64::MAX); + let (sig, transaction) = transaction(&[Pubkey::new_unique()]); + let mut payload = (*transaction).clone(); + payload.resize(10 * MB + 1, 0); + let payload = Arc::new(payload); + + let events = vec![ + Event::Transaction(TransactionEntry { + signature: sig, + payload: payload.clone(), + }), + Event::Block(Block::new(1, 0)), + ]; + append(&ledger, events); + + match read_block(&ledger, 1, BlockDetails::Transactions).await { + Some(BlockResponse::WithTransactions(block)) => { + assert_eq!( + block.transactions.as_slice(), + std::slice::from_ref(payload.as_ref()) + ); + } + _ => panic!("expected transactions response"), + } + + let entries = replay(&ledger, 0).await; + match entries.first() { + Some(OwnedBlockstoreEntry::Transaction(transaction)) => { + assert_eq!(transaction, payload.as_ref()); + } + _ => panic!("expected replayed transaction entry"), + } +} + +// A transaction stays pending until its execution arrives: sealed into a block +// without one, it is never indexed (a record is never half-written); and an +// execution whose transaction never appeared is silently dropped. +#[tokio::test] +async fn test_pending_requires_execution() { + let (_dir, ledger) = ledger(u64::MAX); + let (indexed, indexed_bytes) = transaction(&[Pubkey::new_unique()]); + let (orphan, orphan_bytes) = transaction(&[Pubkey::new_unique()]); + let stray = Signature::from([3; 64]); + + let events = vec![ + // Paired transaction: indexed and readable. + Event::Transaction(TransactionEntry { + signature: indexed, + payload: indexed_bytes, + }), + Event::Execution(execution(indexed, 1, Ok(()))), + // Transaction with no execution: written to the blockstore but never + // indexed, so no read surface can resolve it. + Event::Transaction(TransactionEntry { + signature: orphan, + payload: orphan_bytes, + }), + // Execution with no pending transaction: dropped without error. + Event::Execution(execution(stray, 1, Ok(()))), + Event::Block(Block::new(1, 0)), + ]; + append(&ledger, events); + + assert!(read_transaction(&ledger, indexed).await.is_some()); + assert!(read_transaction(&ledger, orphan).await.is_none()); + assert!(read_transaction(&ledger, stray).await.is_none()); +} + +// Block reads reconstruct exactly the transactions between the previous block +// boundary and this one, at every detail level — the reader derives the block's +// transaction range from the `slot - 1` boundary, so later blocks must not leak +// earlier blocks' transactions. +#[tokio::test] +async fn test_block_detail_levels_partition_transactions() { + let (_dir, ledger) = ledger(u64::MAX); + let first = block_of(&ledger, 1, 2); + let second = block_of(&ledger, 2, 3); + + // Each block reports only its own transactions, in append order. + assert_eq!(block_signatures(&ledger, 1).await, first); + assert_eq!(block_signatures(&ledger, 2).await, second); + + // Transactions-only and Full carry the same count without bleed-through. + match read_block(&ledger, 2, BlockDetails::Transactions).await { + Some(BlockResponse::WithTransactions(b)) => assert_eq!(b.transactions.len(), 3), + _ => panic!("expected transactions response"), + } + match read_block(&ledger, 2, BlockDetails::Full).await { + Some(BlockResponse::Full(b)) => { + assert_eq!(b.transactions.len(), 3); + assert!(b.transactions.iter().all(|t| t.execution.details.is_some())); + } + _ => panic!("expected full response"), + } + + // The bare boundary carries the block metadata only. + match read_block(&ledger, 1, BlockDetails::None).await { + Some(BlockResponse::Bare(block)) => assert_eq!(block.time, 100), + _ => panic!("expected bare response"), + } +} + +// A sealed superblock stays readable after the writer rotates to a new segment, +// and retention purges the oldest sealed superblock — dropping its transactions +// while preserving the active head and advancing the retained slot range. +#[tokio::test] +async fn test_superblock_rotation_and_retention() { + // size_limit 0 makes every block boundary trigger a retention pass. + let (_dir, ledger) = ledger(0); + let old = block_of(&ledger, 1, 1); + // Seal superblock 1 and rotate to superblock 2. + let events = vec![seal(1)]; + append(&ledger, events); + assert_eq!(ledger.meta.head(), 2); + + // The sealed superblock is still readable through the newer head. + assert!(read_transaction(&ledger, old[0]).await.is_some()); + + // Writing a block into the new head triggers truncation of superblock 1. + let new = block_of(&ledger, 2, 1); + assert_eq!(ledger.meta.head(), 2, "active head is never purged"); + assert!( + ledger.superblocks.read().get(&1).is_none(), + "oldest sealed segment purged" + ); + // Its transactions are gone; the head's remain. + assert!(read_transaction(&ledger, old[0]).await.is_none()); + assert!(read_transaction(&ledger, new[0]).await.is_some()); + // Retention advances the retained range past the purged superblock's end. + assert_eq!(ledger.meta.range.start.load(Acquire), 2); +} + +// A single-block read resolves a slot living in an older sealed superblock, not +// only the active head: each segment's range must be pinned to its own slots so +// a newer segment does not shadow older ones, and the ledger-wide range must +// track the tip so in-range slots pass the guard and out-of-range ones do not. +#[tokio::test] +async fn test_block_read_across_superblocks() { + let (_dir, ledger) = ledger(u64::MAX); + let first = block_of(&ledger, 1, 1); + let events = vec![seal(1)]; + append(&ledger, events); + block_of(&ledger, 2, 1); + + // Slot 1 lives in the sealed superblock; slot 2 in the head. Both resolve. + assert_eq!(block_signatures(&ledger, 1).await, first); + assert!(read_block(&ledger, 2, BlockDetails::None).await.is_some()); + // A slot past the retained tip is rejected by the ledger-wide range guard. + assert!(read_block(&ledger, 9, BlockDetails::None).await.is_none()); +} + +// Replay streams superblocks in on-disk order after the last applied seal through +// the active head, so nothing committed after the snapshot is lost on recovery. +// Entries come back exactly as written: +// transactions, their block delimiter, then the seal — and the unsealed head's +// entries have no trailing seal. The read is bounded by each superblock's write +// cursor, so the active head's preallocated tail is not decoded. +#[tokio::test] +async fn test_replay_streams_superblocks_through_active_head() { + let (_dir, ledger) = ledger(u64::MAX); + // Two sealed superblocks (slots 1 and 2), then an unsealed head (slot 3). + block_of(&ledger, 1, 2); + let events = vec![seal(1)]; + append(&ledger, events); + block_of(&ledger, 2, 1); + let events = vec![seal(2)]; + append(&ledger, events); + block_of(&ledger, 3, 1); + + let entries = replay(&ledger, 0).await; + use crate::schema::BlockstoreEntry::*; + let shape: Vec<&str> = entries + .iter() + .map(|e| match e { + Transaction(_) => "tx", + Block(_) => "block", + Superblock(_) => "seal", + Reset(_) => "reset", + }) + .collect(); + // Superblock 1 (two txns) and superblock 2 (one txn), each ending in its + // block and seal, followed by the active head (superblock 3: one txn and its + // block, no seal). + assert_eq!( + shape, + ["tx", "tx", "block", "seal", "tx", "block", "seal", "tx", "block"] + ); +} + +// Ledger state survives reopening from disk: committed transactions remain +// readable and a reopened appender resumes at the persisted cursors, appending +// a new block without clobbering the old one. +#[tokio::test] +async fn test_reopen_resumes_state() { + let dir = tempdir(); + let first = { + let ledger = Arc::new(Ledger::new(dir.path().to_owned(), u64::MAX).unwrap()); + let sigs = block_of(&ledger, 1, 2); + sigs[0] + }; + + // Reopen from the same directory. + let ledger = Arc::new(Ledger::new(dir.path().to_owned(), u64::MAX).unwrap()); + assert!( + read_transaction(&ledger, first).await.is_some(), + "prior block survives reopen" + ); + + // A resumed appender writes a second block after the first. + let second = block_of(&ledger, 2, 1)[0]; + assert!( + read_transaction(&ledger, first).await.is_some(), + "old block not clobbered" + ); + assert!(read_transaction(&ledger, second).await.is_some()); + assert_eq!(ledger.meta.blocks.load(Acquire), 2); +} + +// A block-range read returns every block in the range, in ascending slot order, +// even when the range straddles a superblock boundary. The reader walks slots +// descending across superblocks newest-first, so a boundary slot must be handed +// to the older segment instead of being consumed against the newer one. +#[tokio::test] +async fn test_block_range_spans_superblocks() { + let (_dir, ledger) = ledger(u64::MAX); + // Slot 1 lands in superblock 1; the seal rotates slots 2 and 3 into + // superblock 2, so any range over 1..=2 crosses the segment boundary. + block_of(&ledger, 1, 1); + let events = vec![seal(1)]; + append(&ledger, events); + block_of(&ledger, 2, 1); + block_of(&ledger, 3, 1); + + let slots = async |range: Range| { + read(&ledger, range, ReadRequest::BlockRange) + .await + .unwrap() + .iter() + .map(|b| b.slot) + .collect::>() + }; + // The full range comes back once each, in order, across the boundary. + assert_eq!(slots(1..4).await, vec![1, 2, 3]); + // A sub-range inside one segment returns only its blocks. + assert_eq!(slots(2..3).await, vec![2]); + // A tail past the retained tip yields the retained blocks without dropping + // the boundary slot. + assert_eq!(slots(1..9).await, vec![1, 2, 3]); +} + +// The account index keeps one entry per touching transaction, excludes unrelated +// transactions, and account-signature history pages newest-superblock first with +// exclusive `before`/`until` cursors. +#[tokio::test] +async fn test_account_signatures_history_pagination_and_ordering() { + let (_dir, ledger) = ledger(u64::MAX); + let account = Pubkey::new_unique(); + + // Two transactions touch `account` in superblock 1, plus a third that does + // not — the unrelated transaction must stay out of the account's history. + let sb1 = block_touching(&ledger, 1, &[account, account, Pubkey::new_unique()]); + let events = vec![seal(1)]; + append(&ledger, events); + let sb2 = block_touching(&ledger, 2, &[account, account]); + + let read_history = async |pubkey, limit, before, until| { + let params = AccountSignaturesParams { pubkey, limit, before, until }; + read(&ledger, params, ReadRequest::AccountSignatures).await.unwrap() + }; + let signatures = |history: &[AccountSignature]| { + history.iter().map(|s| s.signature).collect::>() + }; + + let expected = vec![sb2[1], sb2[0], sb1[1], sb1[0]]; + let full = read_history(account, 10, None, None).await; + assert_eq!(signatures(&full), expected); + assert!(full.iter().all(|s| s.blocktime == s.slot as i64 * 100)); + + assert_eq!( + signatures(&read_history(account, 2, None, None).await), + expected[..2], + "`limit` caps results" + ); + assert!( + read_history(Pubkey::new_unique(), 10, None, None).await.is_empty(), + "unmentioned account has no history" + ); + + // Newest-superblock first, newest execution first within each superblock. + assert_eq!( + full.iter().map(|s| s.slot).collect::>(), + vec![2, 2, 1, 1], + "newest account history first" + ); + + let history_signatures = + async |before, until| signatures(&read_history(account, 10, before, until).await); + + // `before` is an exclusive upper bound. The newest cursor yields every + // older signature across the superblock boundary in newest-to-oldest order. + assert_eq!(history_signatures(Some(sb2[1]), None).await, expected[1..]); + // The oldest cursor yields nothing, separating an exclusive bound from an + // inclusive one. + assert!(history_signatures(Some(sb1[0]), None).await.is_empty()); + + // `until` stops at, and excludes, its signature: the first match yields + // nothing, the last yields everything above it. + assert!(history_signatures(None, Some(expected[0])).await.is_empty()); + assert_eq!( + history_signatures(None, Some(expected[3])).await, + expected[..3] + ); +} diff --git a/ledger/src/tests/mod.rs b/ledger/src/tests/mod.rs new file mode 100644 index 00000000..794fc572 --- /dev/null +++ b/ledger/src/tests/mod.rs @@ -0,0 +1,7 @@ +//! Ledger test modules. +//! +//! `index` covers the LMDB codec/index in isolation; `integration` drives the +//! append→seal→read pipeline end to end through the appender and reader. + +mod index; +mod integration; diff --git a/nucleus/Cargo.toml b/nucleus/Cargo.toml new file mode 100644 index 00000000..aeae2e35 --- /dev/null +++ b/nucleus/Cargo.toml @@ -0,0 +1,86 @@ +[package] +name = "magicblock-engine-nucleus" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "nucleus" + +[features] +config = [ + "dep:serde", + "dep:serde_with", + "dep:solana-keypair", + "dep:solana-pubkey", + "dep:solana-signer" +] +default = [] +heed = ["dep:heed"] +ledger = ["solana-hash/copy", "solana-hash/wincode", "wincode/derive"] +metrics = ["dep:prometheus", "dep:tracing"] +notifier = ["tokio/sync"] +runtime = [ + "agave-transaction-view/agave-unstable-api", + "dep:derive_more", + "dep:oneshot", + "dep:solana-svm", + "dep:solana-transaction-error", + "ledger", + "service", + "tls", + "tokio/sync" +] +service = ["metrics", "shutdown"] +shutdown = ["dep:futures", "dep:oneshot", "dep:tokio", "dep:tokio-util", "dep:tracing"] +testkit = [ + "dep:solana-instruction", + "dep:solana-keypair", + "dep:solana-message", + "dep:solana-pubkey", + "dep:solana-signature", + "dep:solana-signer", + "dep:tempfile", + "dep:tracing-subscriber", + "dep:v42-calculator-interface", + "runtime", + "solana-transaction/wincode" +] +tls = ["dep:solana-instruction-error", "dep:solana-pubkey", "dep:wincode"] + +[dependencies] +derive_more = { workspace = true, optional = true, features = ["deref", "from"] } +futures = { workspace = true, features = ["alloc"], optional = true } +heed = { workspace = true, optional = true } +oneshot = { workspace = true, features = ["async"], optional = true } +prometheus = { workspace = true, optional = true } +serde = { workspace = true, features = ["derive"], optional = true } +serde_with = { workspace = true, optional = true } +solana-hash = { workspace = true, optional = true } +solana-instruction-error = { workspace = true, optional = true } +tempfile = { workspace = true, optional = true } +tokio = { workspace = true, features = ["macros", "signal", "time"], optional = true } +tokio-util = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } +tracing-subscriber = { workspace = true, features = ["env-filter", "fmt"], optional = true } +wincode = { workspace = true, optional = true } + +agave-transaction-view = { workspace = true, optional = true } +solana-instruction = { workspace = true, optional = true } +solana-keypair = { workspace = true, optional = true } +solana-message = { workspace = true, optional = true } +solana-pubkey = { workspace = true, optional = true } +solana-signature = { workspace = true, optional = true } +solana-signer = { workspace = true, optional = true } +solana-svm = { workspace = true, optional = true } +solana-transaction = { workspace = true, optional = true } +solana-transaction-error = { workspace = true, optional = true } +v42-calculator-interface = { workspace = true, optional = true, features = ["builder"] } + +[lints] +workspace = true diff --git a/nucleus/README.md b/nucleus/README.md new file mode 100644 index 00000000..a463b5b6 --- /dev/null +++ b/nucleus/README.md @@ -0,0 +1,33 @@ +# `magicblock-engine-nucleus` + +Nucleus contains shared engine types that do not own storage or execution +policy. It also exposes byte-size constants and a Unix-time helper that returns +zero when the system clock predates the epoch. Its default feature set is empty. + +## Features + +- `config`: serializable authority, accountsdb, blockstore, and ledger + configuration types. Authority serialization includes the complete local + keypair; consumers must redact it before exposing serialized output. +- `heed`: LMDB transaction aliases, safe environment-bound transaction reuse + helpers, and the shared `DatabaseIndex` trait. +- `shutdown`: ordered cancellation, service handles, and termination reporting. + The pacemaker quiesces execution and terminally syncs the ledger before the + sequencer and appender tier; remaining backing services stop afterward. + Dropping the manager cancels every tier without waiting for services to stop. +- `notifier`: the one-shot, non-resetting `EventNotifier` latch. +- `ledger`: shared block-boundary metadata, including each block's locally + computed hash and parent, plus snapshot checksum/transaction seals and + blockstore positions. +- `metrics`: Prometheus metric construction, `engine_`-namespaced registration, + labels, and timers. +- `service`: the `metrics` and `shutdown` feature bundle. +- `runtime`: transaction views, execution messages, sequencer handles, and the + quiescence barrier; it also enables `ledger`, `service`, and `tls`. +- `tls`: thread-local MagicRoot authority and encoded service-message state. +- `testkit`: engine-independent fixtures, temporary directories, Legacy/V0/V1 + transaction encoding, v42 instructions, transaction views, and tracing setup + used by downstream test targets. It enables `runtime` because `signed_view` + returns the runtime transaction view. + +Keeper-specific harnesses remain in `keeper::testkit`. diff --git a/nucleus/src/config.rs b/nucleus/src/config.rs new file mode 100644 index 00000000..efc26313 --- /dev/null +++ b/nucleus/src/config.rs @@ -0,0 +1,92 @@ +//! Shared engine configuration types. + +use std::{num::NonZeroU64, path::PathBuf, sync::Arc, time::Duration}; + +use serde::{Deserialize, Serialize}; +use solana_keypair::Keypair; +use solana_pubkey::Pubkey; +use solana_signer::Signer; + +/// Local signing identity and optional authority override represented by a replica. +/// +/// Serialization includes the complete local keypair as a base58 string. +/// Consumers must redact the `local` field before exposing serialized output. +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct Authority { + /// Signer used for locally produced messages and transactions. + #[serde(with = "keypair")] + pub local: Arc, + /// Immediate upstream identity exposed as the engine authority when set. + #[serde(default, with = "serde_with::As::>")] + pub remote: Option, +} + +impl Authority { + /// Returns the remote authority when configured, otherwise the local identity. + pub fn pubkey(&self) -> Pubkey { + self.remote.unwrap_or(self.local.pubkey()) + } +} + +impl>> From for Authority { + fn from(local: K) -> Self { + let local = local.into(); + Self { local, remote: None } + } +} + +/// Account storage and recent-load cache parameters. +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct AccountsDBParams { + /// Accounts database root directory. + pub directory: PathBuf, + /// Requested maximum number of resolved account pubkeys retained for + /// recency tracking and eviction notifications. + /// + /// The cache uses at least 256 slots, rounds larger capacities up to a + /// power of two, and may evict earlier under bucket pressure. + pub lru_capacity: usize, +} + +/// Block production timing used by the engine and keeper caches. +#[derive(Clone, Copy, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct BlockstoreParams { + /// Expected wall-clock interval between produced slots. + pub blocktime: Duration, + /// Number of blocks included into each superblock. + pub superblock: NonZeroU64, +} + +/// Ledger storage parameters. +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct LedgerParams { + /// Ledger root directory. + pub directory: PathBuf, + /// Maximum used bytes allowed on the ledger filesystem before eviction runs. + pub size_limit: u64, +} + +mod keypair { + use super::*; + use serde::{Deserializer, Serializer, de::Error as _}; + + pub(super) fn serialize( + keypair: &Arc, + serializer: S, + ) -> Result { + serializer.serialize_str(&keypair.to_base58_string()) + } + + pub(super) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + let encoded = String::deserialize(deserializer)?; + Keypair::try_from_base58_string(&encoded) + .map(Arc::new) + .map_err(D::Error::custom) + } +} diff --git a/nucleus/src/heed.rs b/nucleus/src/heed.rs new file mode 100644 index 00000000..fe95adcf --- /dev/null +++ b/nucleus/src/heed.rs @@ -0,0 +1,37 @@ +//! Shared heed index plumbing. + +use ::heed::{Env, Result, RoTxn, RwTxn, WithTls}; + +/// Read-only transaction using heed thread-local storage. +pub type RoTxnTls<'e> = RoTxn<'e, WithTls>; +/// Optional write transaction used by batched updates. +pub type OptRwTxn<'t, 'e> = &'t mut Option>; +/// Optional read transaction used by batched reads. +pub type OptRoTxn<'t, 'e> = &'t mut Option>; + +/// Common access for heed-backed indexes. +pub trait DatabaseIndex { + /// Returns the owning heed environment. + fn env(&self) -> &Env; + + /// Flushes the index databases to durable storage. + fn flush(&self) -> Result<()> { + self.env().force_sync() + } +} + +/// Uses the supplied write transaction or opens one against `env` on demand. +pub fn write_txn<'t, 'e>(env: &'e Env, txn: OptRwTxn<'t, 'e>) -> Result<&'t mut RwTxn<'e>> { + if let Some(txn) = txn { + return Ok(txn); + } + Ok(txn.insert(env.write_txn()?)) +} + +/// Uses the supplied read transaction or opens one against `env` on demand. +pub fn read_txn<'t, 'e>(env: &'e Env, txn: OptRoTxn<'t, 'e>) -> Result<&'t RoTxnTls<'e>> { + if let Some(txn) = txn { + return Ok(txn); + } + Ok(txn.insert(env.read_txn()?)) +} diff --git a/nucleus/src/ledger.rs b/nucleus/src/ledger.rs new file mode 100644 index 00000000..82b7ed5d --- /dev/null +++ b/nucleus/src/ledger.rs @@ -0,0 +1,51 @@ +//! Ledger block-boundary schema shared by storage-adjacent crates. + +use solana_hash::Hash; +use wincode::{SchemaRead, SchemaWrite}; + +use crate::Slot; + +/// File name of the archived accountsdb snapshot tarball inside a superblock directory. +pub const ACCOUNTSDB_SNAPSHOT_FILE: &str = "accountsdb.tar.zst"; + +/// A byte cursor into the ledger blockstore stream, used by the replication path +/// to mark how far a follower has consumed. Ordering is lexicographic over +/// `(superblock, offset)`, matching the on-disk append order across rotations. +#[derive(Clone, Copy, SchemaRead, SchemaWrite, PartialEq, Eq, PartialOrd, Ord, Debug)] +pub struct BlockstorePosition { + /// Superblock whose blockstore file the offset indexes into. + pub superblock: u64, + /// Byte offset of the write cursor within that superblock's blockstore file. + pub offset: u64, +} + +/// Block boundary entry stored after all transactions in the block. +#[derive(SchemaRead, SchemaWrite, Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct Block { + /// Slot that produced the block. + pub slot: Slot, + /// Block hash for `slot`. + pub hash: Hash, + /// Block timestamp in the producer's time base. + pub time: i64, + /// Hash of the preceding block. + pub parent: Hash, +} + +impl Block { + /// Creates a block boundary whose hash-chain metadata is not yet known. + pub fn new(slot: Slot, time: i64) -> Self { + Self { slot, time, ..Default::default() } + } +} + +/// Superblock boundary entry stored at the end of the blockstore stream. +#[derive(SchemaRead, SchemaWrite, Clone, Copy, Debug, PartialEq, Eq)] +pub struct SuperblockSeal { + /// Id of the superblock this seal closes. + pub id: u64, + /// Checksum of accountsdb at the moment the superblock was sealed. + pub checksum: u64, + /// Total committed transactions represented by the sealed accountsdb snapshot. + pub transactions: u64, +} diff --git a/nucleus/src/lib.rs b/nucleus/src/lib.rs new file mode 100644 index 00000000..61b41626 --- /dev/null +++ b/nucleus/src/lib.rs @@ -0,0 +1,44 @@ +#![doc = include_str!("../README.md")] + +use std::time::{Duration, UNIX_EPOCH}; + +#[cfg(feature = "config")] +pub mod config; + +#[cfg(feature = "heed")] +pub mod heed; + +#[cfg(feature = "shutdown")] +pub mod shutdown; + +#[cfg(feature = "notifier")] +pub mod notifier; + +#[cfg(feature = "ledger")] +pub mod ledger; + +#[cfg(feature = "metrics")] +pub mod metrics; + +#[cfg(feature = "runtime")] +pub mod runtime; + +#[cfg(feature = "testkit")] +pub mod testkit; + +#[cfg(feature = "tls")] +pub mod tls; + +/// Ledger slot number. +pub type Slot = u64; +/// One kibibyte in bytes. +pub const KB: usize = 1024; +/// One mebibyte in bytes. +pub const MB: usize = 1024 * KB; +/// One gibibyte in bytes. +pub const GB: usize = 1024 * MB; + +/// Returns the duration since the Unix epoch, or zero if the clock predates it. +pub fn unix_time() -> Duration { + UNIX_EPOCH.elapsed().unwrap_or_default() +} diff --git a/nucleus/src/metrics.rs b/nucleus/src/metrics.rs new file mode 100644 index 00000000..05692189 --- /dev/null +++ b/nucleus/src/metrics.rs @@ -0,0 +1,186 @@ +//! Prometheus metric helpers shared by engine crates. + +use std::{fmt::Display, sync::OnceLock, time::Instant}; + +use prometheus::{HistogramOpts, HistogramVec, Opts, default_registry}; +pub use prometheus::{IntCounter, IntCounterVec, IntGauge, IntGaugeVec}; +use tracing::{info, warn}; + +/// Prometheus namespace shared by all engine metrics. +const NAMESPACE: &str = "engine"; + +/// Duration logger for the time elapsed between events and their total duration. +pub struct EventTimer { + sequence: &'static str, + start: Instant, + interval: Instant, +} + +impl EventTimer { + /// Initialize a new timer for the given sequence of events + pub fn new(sequence: &'static str) -> Self { + let start = Instant::now(); + let interval = start; + Self { sequence, start, interval } + } + /// Logs the supplied event and time since construction or the previous event, + /// then starts a new interval. + pub fn record(&mut self, event: impl Display) { + let elapsed = self.interval.elapsed(); + self.interval = Instant::now(); + info!(?elapsed, "{}: {event}", self.sequence); + } +} + +impl Drop for EventTimer { + fn drop(&mut self) { + let elapsed = self.start.elapsed(); + info!(?elapsed, "{} is complete", self.sequence); + } +} + +/// Metric name and help text kept together so collector definitions stay grepable. +#[derive(Clone, Copy)] +pub struct MetricSpec { + /// Prometheus collector name. + pub name: &'static str, + /// Prometheus help text. + pub help: &'static str, +} + +/// Creates and registers a counter after applying its initial value. +pub fn counter(spec: MetricSpec, initial: u64) -> IntCounter { + let counter = validate(IntCounter::with_opts(opts(spec))); + counter.inc_by(initial); + register(spec, counter.clone()); + counter +} + +/// Creates and registers a labeled counter. +pub fn counter_vec(spec: MetricSpec, labels: &[&'static str]) -> IntCounterVec { + let counter = validate(IntCounterVec::new(opts(spec), labels)); + register(spec, counter.clone()); + counter +} + +/// Creates and registers a gauge after applying its initial value. +pub fn gauge(spec: MetricSpec, initial: i64) -> IntGauge { + let gauge = validate(IntGauge::with_opts(opts(spec))); + gauge.set(initial); + register(spec, gauge.clone()); + gauge +} + +/// Creates and registers a labeled gauge. +pub fn gauge_vec(spec: MetricSpec, labels: &[&'static str]) -> IntGaugeVec { + let gauge = validate(IntGaugeVec::new(opts(spec), labels)); + register(spec, gauge.clone()); + gauge +} + +/// Converts an unsigned metric value to a saturating Prometheus gauge value. +pub fn gauge_value(value: T) -> i64 +where + i64: TryFrom, +{ + i64::try_from(value).unwrap_or(i64::MAX) +} + +/// Applies `f` when metrics have been initialized; early calls are intentionally no-ops. +pub fn with_metrics(metrics: &OnceLock, f: impl FnOnce(&T)) { + if let Some(metrics) = metrics.get() { + f(metrics); + } +} + +/// Low-cardinality operation label used by duration histograms. +pub trait MetricOperation: Copy { + /// Returns the Prometheus label value for this operation. + fn label(self) -> &'static str; + + /// Starts a timer against `counters`, or a no-op timer before metrics are initialized. + fn time(self, counters: Option<&OperationCounters>) -> OperationTimer<'_> { + counters.map(|c| c.time(self)).unwrap_or_else(|| OperationTimer::noop(self)) + } +} + +/// Runtime operation duration histogram sharing one `op` label. +pub struct OperationCounters(HistogramVec); + +/// Operation duration histogram buckets, in microseconds. +const OPERATION_BUCKETS_MICROS: [f64; 8] = + [50.0, 200.0, 800.0, 3_200.0, 12_800.0, 51_200.0, 204_800.0, 1_000_000.0]; + +impl OperationCounters { + /// Builds the duration histogram collector. + pub fn new(micros: MetricSpec) -> Self { + let opts = HistogramOpts::new(micros.name, micros.help) + .namespace(NAMESPACE) + .buckets(OPERATION_BUCKETS_MICROS.to_vec()); + let counters = Self(validate(HistogramVec::new(opts, &["op"]))); + register(micros, counters.0.clone()); + counters + } + + /// Starts an operation timer that records latency when the returned guard drops. + pub fn time(&self, op: impl MetricOperation) -> OperationTimer<'_> { + OperationTimer { + counters: Some(self), + op: op.label(), + started: Instant::now(), + } + } +} + +/// Drop guard that records elapsed operation time in the metrics registry. +pub struct OperationTimer<'a> { + /// Operation counters to update on drop. + counters: Option<&'a OperationCounters>, + /// Operation label recorded with the duration observation. + op: &'static str, + /// Monotonic start instant captured when the guard is created. + started: Instant, +} + +impl OperationTimer<'static> { + /// Returns a timer that intentionally records nothing. + pub fn noop(op: impl MetricOperation) -> Self { + Self { + counters: None, + op: op.label(), + started: Instant::now(), + } + } +} + +impl Drop for OperationTimer<'_> { + /// Records elapsed microseconds when the timer leaves scope. + fn drop(&mut self) { + let Some(counters) = self.counters else { + return; + }; + let elapsed = self.started.elapsed().as_micros() as f64; + counters.0.with_label_values(&[self.op]).observe(elapsed); + } +} + +/// Builds namespaced Prometheus options for an engine metric. +fn opts(spec: MetricSpec) -> Opts { + Opts::new(spec.name, spec.help).namespace(NAMESPACE) +} + +/// Registers `collector`, logging registry errors without aborting startup. +fn register(spec: MetricSpec, collector: C) +where + C: prometheus::core::Collector + 'static, +{ + if let Err(error) = default_registry().register(Box::new(collector)) { + warn!(metric = spec.name, ?error, "failed to register metric"); + } +} + +/// Unwraps construction of static metric definitions. +#[allow(clippy::expect_used)] +fn validate(result: prometheus::Result) -> T { + result.expect("prometheus metric registration should succeed") +} diff --git a/nucleus/src/notifier.rs b/nucleus/src/notifier.rs new file mode 100644 index 00000000..d51b401a --- /dev/null +++ b/nucleus/src/notifier.rs @@ -0,0 +1,42 @@ +//! One-shot async event notification. + +use std::sync::atomic::{AtomicBool, Ordering::*}; + +use tokio::sync::Notify; + +/// A one-shot latch that wakes every waiter once notified. +/// +/// Waiters that arrive after notification return immediately. The event cannot +/// be reset. +#[derive(Default)] +pub struct EventNotifier { + /// Set after notification so future waiters can return immediately. + done: AtomicBool, + /// Outcome published before `done` makes the event observable. + success: AtomicBool, + /// Wakes tasks that registered before notification. + notify: Notify, +} + +impl EventNotifier { + /// Completes the event with `success` and wakes all current waiters. + pub fn notify(&self, success: bool) { + self.success.store(success, Release); + self.done.store(true, Release); + self.notify.notify_waiters(); + } + + /// Waits until completion and returns the published outcome. + pub async fn notified(&self) -> bool { + loop { + let notified = self.notify.notified(); + // The waiter is created before the second load, so a concurrent + // notify cannot land between observing `false` and registering. + if self.done.load(Acquire) { + return self.success.load(Acquire); + } + + notified.await; + } + } +} diff --git a/nucleus/src/runtime.rs b/nucleus/src/runtime.rs new file mode 100644 index 00000000..06735410 --- /dev/null +++ b/nucleus/src/runtime.rs @@ -0,0 +1,129 @@ +//! Runtime-facing shared types: transaction views, execution I/O, scheduling +//! messages, and the schema of the engine's built-in MagicRoot program (its +//! address, instruction set, and authority). The MagicRoot execution logic +//! lives in the `magic-root-program` crate, which depends on these shared +//! definitions. + +use std::sync::Arc; + +use agave_transaction_view::{ + resolved_transaction_view::ResolvedTransactionView, transaction_view::SanitizedTransactionView, +}; +use derive_more::{Deref, From}; +use solana_svm::{ + transaction_balances::BalanceCollector, + transaction_processing_result::TransactionProcessingResult, +}; +use solana_transaction_error::TransactionResult; +use tokio::sync::mpsc::Sender; + +use crate::{Slot, ledger::Block}; + +/// Sanitized transaction view backed by a shared, immutable payload buffer. +pub type TransactionView = SanitizedTransactionView>>; +/// Sanitized transaction view with its account keys already resolved. +pub type ResolvedTransaction = ResolvedTransactionView>>; +/// Dropping this handle releases a sequencer quiescence barrier. +pub type BarrierHandle = oneshot::Sender<()>; + +/// Cloneable submission handle into the sequencer's execution and simulation +/// channels. +#[derive(Clone, Deref)] +pub struct SequencerHandle { + /// Channel for submitting transactions to be executed and committed. + #[deref] + pub execution: Sender, + /// Channel for submitting transactions to be simulated without committing. + pub simulation: Sender, +} + +/// Work item handed to the transaction sequencer. +#[derive(From)] +pub enum SequencerMessage { + /// A transaction to schedule and execute. + Transaction(TransactionView), + /// A block boundary to seal before scheduling further transactions. + Block(Block), + /// Quiesce the sequencer and all its executors until released — used to take + /// a consistent snapshot at superblock boundaries (see ledger replay and + /// `finalize_superblock`). + Barrier(BarrierGuard), +} + +/// Quiescence barrier handed to a running service. +/// +/// On receipt the service drains its in-flight work, signals `acknowledged`, +/// then blocks on `released` before resuming. Paired with a [`BarrierController`] +/// via [`barrier`]. +pub struct BarrierGuard { + /// Signals the controller that the service is now idle. + pub acknowledged: BarrierHandle, + /// Resolves when the controller permits the service to resume; a dropped + /// controller also releases it. + pub released: oneshot::Receiver<()>, +} + +/// Controller side of a quiescence barrier, held by the caller that raised it. +/// +/// Awaits `acknowledged` to learn the service is quiesced, then sends `released` +/// (or drops) to let it resume. +pub struct BarrierController { + /// Resolves once the service reports it has gone idle. + pub acknowledged: oneshot::Receiver<()>, + /// Releases the service to resume operation. + pub released: BarrierHandle, +} + +/// Constructs a paired [`BarrierController`] and [`BarrierGuard`] over two +/// oneshot channels. +pub fn barrier() -> (BarrierController, BarrierGuard) { + let (acknowledged_tx, acknowledged_rx) = oneshot::channel(); + let (released_tx, released_rx) = oneshot::channel(); + let controller = BarrierController { + acknowledged: acknowledged_rx, + released: released_tx, + }; + let guard = BarrierGuard { + acknowledged: acknowledged_tx, + released: released_rx, + }; + (controller, guard) +} + +/// Work item handed to the transaction simulator. +#[derive(From)] +pub enum SimulatorMessage { + /// A transaction to simulate against the current state. + Transaction(Simulation), + /// A block boundary advancing the simulator's environment. + Block(Block), + /// Quiesce the simulator until released — keeps it idle while a consistent + /// snapshot is taken at superblock boundaries. + Barrier(BarrierGuard), +} + +/// A single simulation request and the channel to deliver its outcome. +pub struct Simulation { + /// The transaction to simulate. + pub transaction: TransactionView, + /// Where the simulation result is returned to the caller. + pub response: oneshot::Sender>, +} + +/// Transaction payload paired with the SVM output needed to finalize state. +pub struct FullTransaction { + /// Sanitized transaction bytes and signatures accepted by the ledger. + pub transaction: TransactionView, + /// SVM execution output for the transaction. + pub execution: ExecutionRecord, +} + +/// SVM output produced by executing a transaction. +pub struct ExecutionRecord { + /// SVM processing result produced for this transaction. + pub result: TransactionProcessingResult, + /// Native pre/post balances collected during execution. + pub balances: Option, + /// Slot assigned to the execution result. + pub slot: Slot, +} diff --git a/nucleus/src/shutdown.rs b/nucleus/src/shutdown.rs new file mode 100644 index 00000000..3dd2396d --- /dev/null +++ b/nucleus/src/shutdown.rs @@ -0,0 +1,257 @@ +//! Cooperative shutdown for engine services. +//! +//! A [`ShutdownManager`] owns ordered cancellation tokens and a set of +//! registered service handles. Each service receives a [`ShutdownHandle`], +//! observes its tier token while running, and reports a [`ShutdownReason`] when +//! it exits. The manager waits for an OS shutdown signal, internal cancellation, +//! or service termination, then cancels each tier in order and gives it a +//! bounded window to stop before moving on. + +use std::{ + error::Error, + io, + time::{Duration, Instant}, +}; + +use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered}; +use oneshot::Sender; +use tokio::time::timeout; +use tracing::{error, info, warn}; + +pub use tokio_util::sync::CancellationToken; + +const TIMEOUT: Duration = Duration::from_secs(4); + +type HandleFuture = BoxFuture<'static, (Service, ShutdownTier, ShutdownReason)>; + +/// Background service tracked by the shutdown manager. +#[derive(Clone, Copy, Debug)] +pub enum Service { + /// Ledger append worker. + LedgerAppender, + /// Ledger read worker. + LedgerReader, + /// Ledger replay worker. + LedgerReplayer, + /// Transaction scheduler service. + Sequencer, + /// Transaction executor worker with its worker index. + TransactionExecutor(u32), + /// Transaction simulation worker. + TransactionSimulator, + /// Subscription map cleanup worker. + SubscriptionsCleanup, + /// Block pacing task. + PaceMaker, + /// Leader-side service streaming blockstore bytes to followers. + ReplicationDispatcher, + /// Follower-side service pulling replicated state from the leader. + ReplicationClient, +} + +impl Service { + /// Shutdown tier for this service; lower tiers are stopped first. + fn tier(&self) -> ShutdownTier { + use Service::*; + match self { + ReplicationClient => ShutdownTier::One, + PaceMaker => ShutdownTier::Two, + // The pacemaker drains the sequencer and sends the appender's final + // sync before either service reaches this tier. + Sequencer | LedgerAppender => ShutdownTier::Three, + _ => ShutdownTier::Four, + } + } +} + +#[derive(Clone, Copy, Debug)] +enum ShutdownTier { + One, + Two, + Three, + Four, +} + +impl ShutdownTier { + const COUNT: usize = 4; + const ORDER: [Self; Self::COUNT] = [Self::One, Self::Two, Self::Three, Self::Four]; +} + +/// Coordinates graceful shutdown across engine services. +#[derive(Default)] +pub struct ShutdownManager { + /// Service cancellation tokens, one per ordered shutdown tier. + tokens: [CancellationToken; ShutdownTier::COUNT], + /// Registered service termination reports. + handles: FuturesUnordered, + /// Number of services that have not reported termination, by tier. + pending: [isize; ShutdownTier::COUNT], +} + +impl ShutdownManager { + /// Wait for an OS shutdown signal, internal cancellation, or service failure. + pub async fn wait(&mut self) -> ShutdownReason { + tokio::select! { + result = graceful_shutdown() => { + match result { + Ok(()) => { + info!("graceful shutdown has been requested"); + ShutdownReason::Signalled + } + Err(error) => ShutdownReason::Error(Box::new(error)), + } + } + Some((service, tier, reason)) = self.handles.next(), if !self.handles.is_empty() => { + self.pending(tier, -1); + error!(?service, ?reason, "terminated prematurely"); + reason + } + } + } + + /// Cancels services one tier at a time and drains their termination reports. + /// + /// Each tier gets `TIMEOUT` to report before the next tier is + /// cancelled. Already terminated services are skipped by their tier. + pub async fn terminate(&mut self) { + info!("initiating graceful shutdown of the engine"); + let start = Instant::now(); + let mut timers = [start; ShutdownTier::COUNT]; + for tier in ShutdownTier::ORDER { + timers[tier as usize] = Instant::now(); + self.tokens[tier as usize].cancel(); + if self.pending[tier as usize] == 0 { + continue; + } + if timeout(TIMEOUT, self.drain(tier, &timers)).await.is_err() { + let remaining = self.pending[tier as usize]; + let elapsed = timers[tier as usize].elapsed(); + warn!(?tier, remaining, ?elapsed, "shutdown tier timed out"); + } + } + info!(elapsed = ?start.elapsed(), "engine shutdown complete"); + } + + /// Register a service and return its cancellation handle. + pub fn handle(&mut self, service: Service) -> ShutdownHandle { + let tier = service.tier(); + let (tx, rx) = oneshot::channel(); + let fut = async move { + let reason = rx.await.unwrap_or_default(); + (service, tier, reason) + }; + self.handles.push(Box::pin(fut)); + self.pending(tier, 1); + ShutdownHandle { + cancel: self.tokens[tier as usize].child_token(), + reason: Some(tx), + } + } + + async fn drain(&mut self, tier: ShutdownTier, timers: &[Instant]) { + while self.pending[tier as usize] != 0 { + let Some((service, tier, reason)) = self.handles.next().await else { + return; + }; + // Another tier may finish while this one drains; debit its own pending count. + self.pending(tier, -1); + let elapsed = timers[tier as usize].elapsed(); + Self::log(service, reason, elapsed); + } + } + + fn pending(&mut self, tier: ShutdownTier, op: isize) { + self.pending[tier as usize] += op; + } + + fn log(service: Service, reason: ShutdownReason, elapsed: Duration) { + match reason { + ShutdownReason::Unexpected => { + warn!(?service, ?elapsed, "terminated unexpectedly") + } + ShutdownReason::Signalled => { + info!(?service, ?elapsed, "terminated gracefully") + } + ShutdownReason::RestartRequired => { + warn!(?service, ?elapsed, "requested a restart") + } + ShutdownReason::Error(error) => { + error!(?service, ?error, ?elapsed, "terminated with error") + } + } + } +} + +impl Drop for ShutdownManager { + fn drop(&mut self) { + for token in &self.tokens { + token.cancel(); + } + } +} + +/// Waits for SIGTERM or Ctrl-C. +async fn graceful_shutdown() -> io::Result<()> { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate())?; + tokio::select! { + signal = term.recv() => signal + .ok_or_else(|| io::Error::other("SIGTERM listener closed")), + result = tokio::signal::ctrl_c() => result, + } +} + +/// Cancellation handle held by a running service. +pub struct ShutdownHandle { + /// Token observed by the running service. + cancel: CancellationToken, + /// One-shot report consumed by the manager when the service exits. + reason: Option>, +} + +/// Reason reported when a service terminates. +#[derive(Debug, Default)] +pub enum ShutdownReason { + /// Service handle was dropped without reporting a reason. + #[default] + Unexpected, + /// Service stopped after being signalled. + Signalled, + /// Service stopped because of an error. + Error(Box), + /// Service staged state that must be installed by restarting the engine. + RestartRequired, +} + +impl ShutdownHandle { + /// Request engine shutdown and report this service's termination reason. + pub fn terminate(&mut self, reason: ShutdownReason) { + self.cancel.cancel(); + if let Some(tx) = self.reason.take() { + let _ = tx.send(reason); + }; + } + + /// Wait until this service's shutdown tier is cancelled. + pub async fn signalled(&self) { + self.cancel.cancelled().await + } + + /// Returns whether this service's shutdown tier has been cancelled. + pub fn requested(&self) -> bool { + self.cancel.is_cancelled() + } + + /// Creates a cancellation token for work owned by this service. + pub fn child(&self) -> CancellationToken { + self.cancel.child_token() + } +} + +impl Drop for ShutdownHandle { + fn drop(&mut self) { + if let Some(tx) = self.reason.take() { + let _ = tx.send(ShutdownReason::Unexpected); + }; + } +} diff --git a/nucleus/src/testkit.rs b/nucleus/src/testkit.rs new file mode 100644 index 00000000..8f05a58b --- /dev/null +++ b/nucleus/src/testkit.rs @@ -0,0 +1,156 @@ +//! Engine-agnostic test fixtures shared across crate test suites. +//! +//! Only the primitives that depend on nothing above nucleus live here — +//! wincode-serialized transactions, block boundaries, and throwaway directories. +//! Keeper-level harness code (building a `Keeper`, loading the v42 ELF) lives in +//! `keeper::testkit`. Compiled only under the `testkit` feature, so it never +//! reaches release builds. +// Test-support code: a panic here fails the test that caused it, which is the +// intended reporting path. Kept out of release builds by the `testkit` feature. +#![allow(clippy::expect_used, clippy::unwrap_used)] + +use std::sync::{Arc, Once}; + +use solana_hash::Hash; +use solana_instruction::{AccountMeta, Instruction}; +use solana_keypair::Keypair; +use solana_message::{Message, VersionedMessage, v0, v1}; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use solana_signer::Signer; +use solana_transaction::versioned::VersionedTransaction; +pub use tempfile::TempDir; +use tracing_subscriber::{EnvFilter, fmt}; +use v42_calculator_interface::builder::Expr as E; + +pub use v42_calculator_interface::ID as V42_ID; + +use crate::{Slot, ledger::Block, runtime::TransactionView}; + +static TRACING: Once = Once::new(); + +/// Standard transaction wire format produced by client SDKs. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum WireVersion { + /// Unversioned legacy message. + Legacy, + /// Version 0 message without address lookup tables. + V0, + /// Version 1 message. + V1, +} + +/// Installs a libtest-aware tracing subscriber for test processes. +/// +/// The default filter is intentionally quiet. Set `RUST_LOG` and run tests with +/// `-- --nocapture` to see lower-level spans and events while debugging. +pub fn init_tracing() { + TRACING.call_once(|| { + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("warn")); + let _ = fmt().with_env_filter(filter).with_test_writer().try_init(); + }); +} + +/// A throwaway on-disk directory; the returned guard must outlive the store +/// opened over it, since these stores keep their files open/mmapped. +pub fn tempdir() -> TempDir { + TempDir::new().unwrap() +} + +/// A block boundary with a distinct hash and time derived from `slot`. +pub fn block(slot: Slot) -> Block { + let mut hash = [0; 32]; + let bytes = slot.to_le_bytes(); + hash[..bytes.len()].copy_from_slice(&bytes); + Block { + slot, + hash: Hash::new_from_array(hash), + time: slot as i64, + parent: Hash::default(), + } +} + +/// Signs `instructions` in a standard client wire format. +pub fn sign_versioned_instructions( + payer: &Keypair, + version: WireVersion, + instructions: impl AsRef<[Instruction]>, + blockhash: Hash, +) -> (Signature, Vec) { + let instructions = instructions.as_ref(); + let message = match version { + WireVersion::Legacy => VersionedMessage::Legacy(Message::new_with_blockhash( + instructions, + Some(&payer.pubkey()), + &blockhash, + )), + WireVersion::V0 => VersionedMessage::V0( + v0::Message::try_compile(&payer.pubkey(), instructions, &[], blockhash).unwrap(), + ), + WireVersion::V1 => VersionedMessage::V1( + v1::Message::try_compile(&payer.pubkey(), instructions, blockhash).unwrap(), + ), + }; + let transaction = VersionedTransaction::try_new(message, &[payer]).unwrap(); + let signature = transaction.signatures[0]; + (signature, wincode::serialize(&transaction).unwrap()) +} + +/// Signs `instructions` from `payer` against `blockhash`, returning the first +/// signature and the wincode-serialized transaction bytes. +pub fn sign_instructions( + payer: &Keypair, + instructions: impl AsRef<[Instruction]>, + blockhash: Hash, +) -> (Signature, Arc>) { + let (signature, transaction) = + sign_versioned_instructions(payer, WireVersion::Legacy, instructions, blockhash); + (signature, Arc::new(transaction)) +} + +/// Builds one v42 instruction that sums every supplied operand into `output`. +pub fn v42_sum(output: Pubkey, operands: &[Pubkey]) -> Instruction { + assert!( + !operands.is_empty(), + "v42 sum requires at least one operand" + ); + let expression = (2..=operands.len()).fold(E::acc(1), |expr, index| expr + E::acc(index as u8)); + expression.compose(output, operands) +} + +/// Builds a v42 instruction retaining `value` while adding evaluator work. +pub fn v42_padded_value(output: Pubkey, value: i64, terms: usize) -> Instruction { + let expression = (1..terms).fold(E::lit(value), |expr, _| expr + E::lit(0)); + expression.compose(output, &[]) +} + +/// Returns deterministic non-uniform bytes for detecting damaged large payloads. +pub fn patterned_bytes(len: usize, seed: u8) -> Vec { + (0..len).map(|index| seed.wrapping_add((index % 251) as u8)).collect() +} + +/// Signs `instructions` and returns the sanitized transaction view consumed by +/// the runtime. +pub fn signed_view( + payer: &Keypair, + instructions: impl AsRef<[Instruction]>, + blockhash: Hash, +) -> (Signature, TransactionView) { + let (signature, bytes) = sign_instructions(payer, instructions, blockhash); + let view = TransactionView::try_new_sanitized(bytes, true).unwrap(); + (signature, view) +} + +/// A signed, wincode-serialized v42 transaction plus its first signature. +/// +/// The instruction references `accounts` as read-only keys so they land in the +/// transaction's static account keys (and thus any account index), and targets +/// the v42 program so the transaction is executable, not just well-formed. A +/// fresh random payer per call keeps signatures unique without varying the +/// blockhash. +pub fn transaction(accounts: &[Pubkey]) -> (Signature, Arc>) { + let payer = Keypair::new(); + let metas = accounts.iter().map(|k| AccountMeta::new_readonly(*k, false)).collect(); + let ix = Instruction::new_with_bytes(V42_ID, &[], metas); + sign_instructions(&payer, [ix], Hash::default()) +} diff --git a/nucleus/src/tls.rs b/nucleus/src/tls.rs new file mode 100644 index 00000000..35968321 --- /dev/null +++ b/nucleus/src/tls.rs @@ -0,0 +1,46 @@ +//! Thread-local execution state shared with runtime-adjacent code. + +use std::{ + cell::{Cell, RefCell}, + collections::VecDeque, +}; + +use solana_instruction_error::InstructionError; +use solana_pubkey::Pubkey; +use wincode::{SchemaWrite, config::Configuration}; + +/// Wincode-encoded message buffered for later handling on the same thread. +pub type EncodedMessage = Vec; + +thread_local! { + /// Per-thread queue for messages emitted while a transaction executes. + pub static TLS: RefCell = RefCell::new(Default::default()); + /// Signer authorized to invoke the MagicRoot program on the current thread. + pub static AUTHORITY: Cell = Cell::new(Default::default()); +} + +/// FIFO queue of encoded messages scoped to the current thread. +#[derive(Default)] +pub struct TlsManager(VecDeque); + +impl TlsManager { + /// Encodes `msg` and appends it to the current thread's queue. + pub fn enqueue(msg: &T) -> Result<(), InstructionError> + where + T: SchemaWrite, + { + let encoded = wincode::serialize(msg).map_err(|_| InstructionError::Custom(u32::MAX))?; + TLS.with_borrow_mut(|tls| tls.0.push_back(encoded)); + Ok(()) + } + + /// Removes the oldest encoded message from the current thread's queue. + pub fn dequeue() -> Option { + TLS.with_borrow_mut(|tls| tls.0.pop_front()) + } + + /// Drops every queued message for the current thread. + pub fn clear() { + TLS.with_borrow_mut(|tls| tls.0.clear()) + } +} diff --git a/processor/Cargo.toml b/processor/Cargo.toml new file mode 100644 index 00000000..25b58017 --- /dev/null +++ b/processor/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "magicblock-processor" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "processor" + +[dependencies] +accountsdb = { workspace = true } +keeper = { workspace = true } +nucleus = { workspace = true, features = ["runtime"] } + +ahash = { workspace = true } +blake3 = { workspace = true } +derive_more = { workspace = true, features = ["deref", "deref_mut", "from"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt", "sync", "time"] } +tracing = { workspace = true } + +agave-feature-set = { workspace = true } +agave-precompiles = { workspace = true, features = ["agave-unstable-api"] } +agave-syscalls = { workspace = true, features = ["agave-unstable-api"] } +agave-transaction-view = { workspace = true } +solana-account = { workspace = true } +solana-compute-budget-instruction = { workspace = true, features = ["agave-unstable-api"] } +solana-hash = { workspace = true } +solana-precompile-error = { workspace = true } +solana-program-runtime = { workspace = true } +solana-pubkey = { workspace = true } +solana-signature = { workspace = true } +solana-svm = { workspace = true } +solana-svm-transaction = { workspace = true } +solana-sysvar = { workspace = true } +solana-transaction-error = { workspace = true } + +[dev-dependencies] +keeper = { workspace = true, features = ["testkit"] } +v42-calculator-interface = { workspace = true, features = ["builder"] } + +oneshot = { workspace = true } +solana-instruction = { workspace = true } +solana-keypair = { workspace = true } +solana-sdk-ids = { workspace = true } + +[lints] +workspace = true diff --git a/processor/README.md b/processor/README.md new file mode 100644 index 00000000..5b052b84 --- /dev/null +++ b/processor/README.md @@ -0,0 +1,28 @@ +# `magicblock-processor` + +The processor schedules transactions across a fixed pool of SVM executors and +commits their results through keeper. + +The sequencer tracks each static account key with executor-holder bits and an +exclusive-write bit. Transactions with non-conflicting read/write sets run in +parallel. Conflicting transactions queue behind the executor holding the +blocking account and are retried when that executor becomes available. + +Block hashes chain the prior block hash with each appended transaction's +canonical signature. Finalization drains executor work before publishing the +ledger boundary, so every transaction's execution metadata precedes the block +that contains it. + +## Quiescence + +The sequencer barrier drains all executor work, acknowledges the caller, and +holds new execution until its guard is released. Engine uses the barrier for +coherent superblock snapshots, replay seal checks, replication handshakes, and +shutdown. + +## Simulation + +Simulation has a separate worker and SVM context. It resolves a transaction, +loads owned account copies, executes against the current block environment, and +returns an `ExecutionRecord` without appending to the ledger or storing account +changes. diff --git a/processor/src/callback.rs b/processor/src/callback.rs new file mode 100644 index 00000000..af1be321 --- /dev/null +++ b/processor/src/callback.rs @@ -0,0 +1,54 @@ +use agave_feature_set::FeatureSet; +use nucleus::Slot; +use solana_account::AccountSharedData; +use solana_precompile_error::PrecompileError; +use solana_pubkey::Pubkey; +use solana_svm::transaction_processing_callback::{ + InvokeContextCallback, TransactionProcessingCallback, +}; +use tracing::error; + +use accountsdb::AccountLoader; + +/// Bridges the SVM to keeper-backed account loads. +/// +/// When `LOAD_OWNED` is set, loaded accounts are copied to owned data; this is +/// used by simulation, which must always operate on owned account copies. +pub(crate) struct SVMCallback<'a, const LOAD_OWNED: bool> { + /// Reads accounts from the engine's accounts store. + pub(crate) loader: AccountLoader<'a>, + /// Active feature set governing precompiles and runtime behavior. + pub(crate) featureset: &'a FeatureSet, +} + +impl<'a, const LO: bool> InvokeContextCallback for SVMCallback<'a, LO> { + fn is_precompile(&self, program_id: &Pubkey) -> bool { + agave_precompiles::is_precompile(program_id, |id| self.featureset.is_active(id)) + } + + fn process_precompile( + &self, + program_id: &Pubkey, + data: &[u8], + instruction_datas: Vec<&[u8]>, + ) -> Result<(), PrecompileError> { + agave_precompiles::get_precompile(program_id, |id| self.featureset.is_active(id)) + .ok_or(PrecompileError::InvalidPublicKey) + .and_then(|p| p.verify(data, &instruction_datas, self.featureset)) + } +} + +impl<'a, const LOAD_OWNED: bool> TransactionProcessingCallback for SVMCallback<'a, LOAD_OWNED> { + fn get_account_shared_data(&self, pubkey: &Pubkey) -> Option<(AccountSharedData, Slot)> { + self.loader + .load(pubkey) + .inspect_err(|error| error!(?error, "accountsdb load error")) + .unwrap_or_default() + .map(|mut acc| { + if LOAD_OWNED { + acc = acc.owned().into(); + } + (acc, 0) + }) + } +} diff --git a/processor/src/error.rs b/processor/src/error.rs new file mode 100644 index 00000000..a251fbd8 --- /dev/null +++ b/processor/src/error.rs @@ -0,0 +1,27 @@ +//! Error type for the transaction processor. + +use std::io; + +use derive_more::From; +use keeper::error::KeeperError; +use nucleus::shutdown::Service; + +/// Convenience result alias for processor operations. +pub type Result = std::result::Result; + +/// Failures raised while scheduling or executing transactions. +#[derive(Debug, thiserror::Error, From)] +pub enum ProcessorError { + /// An I/O error, e.g. while spawning a worker thread or building a runtime. + #[error("i/o error: {0}")] + Io(#[source] io::Error), + /// A failure surfaced by the underlying keeper-backed state. + #[error("state error: {0}")] + State(#[source] KeeperError), + /// A background service is no longer reachable, so work could not be handed off. + #[error("service became unavailable: {0:?}")] + ServiceUnavailable(Service), + /// An internal invariant was violated; carries a human-readable description. + #[error("internal error: {0}")] + Internal(String), +} diff --git a/processor/src/executor.rs b/processor/src/executor.rs new file mode 100644 index 00000000..a5f707b6 --- /dev/null +++ b/processor/src/executor.rs @@ -0,0 +1,180 @@ +//! Transaction execution coordination. + +use std::{ + collections::VecDeque, + sync::{ + Arc, + mpsc::{self, Receiver, SyncSender}, + }, + thread::{self, JoinHandle}, +}; + +use ahash::HashMap; +use derive_more::{Deref, DerefMut}; +use keeper::{ExecutionRecord, FullTransaction, Keeper}; +use nucleus::{ + shutdown::{Service, ShutdownHandle, ShutdownManager, ShutdownReason}, + tls::AUTHORITY, +}; +use solana_program_runtime::loaded_programs::ProgramCache; +use solana_pubkey::Pubkey; +use solana_svm::transaction_processing_result::TransactionProcessingResultExtensions; +use tokio::sync::mpsc::Sender; +use tracing::{error, info}; + +use crate::{ + ExecutorMessage, ExecutorReady, ResolvedTransaction, Result, + callback::SVMCallback, + metrics::{self, FailureKind}, + svm::SvmContext, +}; + +/// Index identifying an executor within the pool. +pub(crate) type ExecutorId = u32; + +/// Sequencer-owned work accumulated for an executor between dispatches. +#[derive(Default)] +pub(crate) struct ExecutorWork { + /// Transactions accumulated for the next dispatch. + pub(crate) batch: Vec, + /// Accounts held on this executor's behalf, mapped to their reference count. + pub(crate) locks: HashMap, + /// Transactions queued behind this executor on lock contention. + pub(crate) blocked: VecDeque, +} + +/// Worker that drives the SVM for batches of conflict-free transactions. +pub(crate) struct TransactionExecutor { + /// This executor's index within the pool. + id: ExecutorId, + /// Inbound channel of execution batches and block boundaries. + rx: Receiver, + /// Channel used to report back to the sequencer once a batch completes. + ready: Sender, + /// SVM batch processor and per-block environment driving execution. + svm: SvmContext, + /// Durable engine state used for account loads and commits. + state: Arc, + /// Handle used to report cooperative shutdown for this worker. + shutdown: ShutdownHandle, + /// Whether the executor runs in ledger-replay mode: when set, only state + /// transitions are committed directly instead of recording full execution. + replay: bool, +} + +/// Sequencer-side handle to a spawned executor: its dispatch channel, pending +/// batch, held locks, and join handle. +#[derive(Deref, DerefMut)] +pub(crate) struct ExecutorHandle { + /// Index of the executor this handle controls. + pub(crate) id: ExecutorId, + /// Mutable sequencing state moved out while completed work is reclaimed. + #[deref] + #[deref_mut] + pub(crate) work: ExecutorWork, + /// Channel for dispatching batches and block boundaries to the worker. + pub(crate) tx: SyncSender, + /// Join handle for the worker thread, taken during shutdown. + pub(crate) task: Option>, +} + +impl TransactionExecutor { + /// Builds the SVM environment, spawns the worker thread, and returns its + /// sequencer-side handle. + pub(crate) fn spawn( + id: ExecutorId, + state: Arc, + cache: Arc, + shutdown: &mut ShutdownManager, + ready: Sender, + replay: bool, + ) -> Result { + let svm = SvmContext::new(&state, cache)?; + let shutdown = shutdown.handle(Service::TransactionExecutor(id)); + let (tx, rx) = mpsc::sync_channel(3); + let executor = Self { + id, + rx, + svm, + ready, + state, + shutdown, + replay, + }; + let task = thread::Builder::new() + .name(format!("transaction-executor-{id}")) + .spawn(move || executor.run())?; + Ok(ExecutorHandle { + id, + task: Some(task), + tx, + work: Default::default(), + }) + } + + /// Worker loop: executes batches and applies block transitions until the + /// channel closes or a batch fails, then reports the termination reason. + fn run(mut self) { + // MagicRoot authorizes callers against this thread-local; publish the + // engine authority before executing any transaction on this thread. + AUTHORITY.set(self.state.authority()); + let mut error = None; + while let Ok(msg) = self.rx.recv() { + match msg { + ExecutorMessage::Transactions(mut batch) => { + let result = self.process(&mut batch); + if let Err(e) = result { + error.replace(e); + drop(self.rx); + let signal = ExecutorReady { id: self.id, batch }; + let _ = self.ready.blocking_send(signal); + break; + } + let signal = ExecutorReady { id: self.id, batch }; + if self.ready.blocking_send(signal).is_err() { + info!(id = self.id, "ready channel closed, executor exiting"); + break; + } + } + ExecutorMessage::Block(block) => self.svm.transition(block), + }; + } + let reason = if let Some(error) = error { + error!(?error, self.id, "executor failed, terminating"); + ShutdownReason::Error(Box::new(error)) + } else { + ShutdownReason::Signalled + }; + self.shutdown.terminate(reason); + } + + /// Loads and executes each transaction in the batch through the SVM, + /// committing either raw state transitions (replay) or full execution. + fn process(&mut self, transactions: &mut Vec) -> Result<()> { + let accessor = self.state.accounts(); + let callback = SVMCallback:: { + loader: accessor.loader(), + featureset: self.state.features(), + }; + for txn in transactions.drain(..) { + let output = self.svm.execute(&callback, &txn, self.state.features()); + if !output.processing_result.was_processed_with_successful_result() { + metrics::failed_transaction(FailureKind::Execution); + } + if self.replay { + self.state.transactions().commit_state_transitions(&output.processing_result)?; + } else { + let txn = FullTransaction { + transaction: txn.into_view(), + execution: ExecutionRecord { + result: output.processing_result, + balances: output.balance_collector, + slot: self.svm.slot(), + }, + }; + self.state.transactions().commit_execution(txn)?; + } + } + Ok(()) + } +} diff --git a/processor/src/lib.rs b/processor/src/lib.rs new file mode 100644 index 00000000..d367ccc7 --- /dev/null +++ b/processor/src/lib.rs @@ -0,0 +1,36 @@ +#![doc = include_str!("../README.md")] + +use keeper::ResolvedTransaction; +use nucleus::ledger::Block; + +use crate::executor::ExecutorId; + +mod callback; +mod error; +mod executor; +mod metrics; +pub mod sequencer; +pub mod simulator; +mod svm; + +#[cfg(test)] +mod tests; + +pub use error::{ProcessorError, Result}; +pub use nucleus::runtime::{SequencerMessage, Simulation, SimulatorMessage}; + +/// Batch of work delivered from the sequencer to a transaction executor. +pub enum ExecutorMessage { + /// A set of conflict-free transactions to execute together. + Transactions(Vec), + /// A block boundary advancing the executor's processing environment. + Block(Block), +} + +/// Notification that an executor finished its batch and is idle again. +struct ExecutorReady { + /// Identifier of the executor that became ready. + id: ExecutorId, + /// The drained batch handed back for reuse + batch: Vec, +} diff --git a/processor/src/metrics.rs b/processor/src/metrics.rs new file mode 100644 index 00000000..b1fccff1 --- /dev/null +++ b/processor/src/metrics.rs @@ -0,0 +1,132 @@ +//! Prometheus metrics for processor. + +use std::sync::OnceLock; + +use nucleus::metrics::{self as metric, OperationTimer}; +use nucleus::metrics::{IntCounter, IntGauge, MetricOperation, MetricSpec, OperationCounters}; + +/// Process-wide processor metrics registered in the default Prometheus registry. +static METRICS: OnceLock = OnceLock::new(); + +/// Operation latency histogram recorded in microseconds. +const OPERATION_TIME: MetricSpec = MetricSpec { + name: "processor_operation_duration_micros", + help: "Processor operation duration distribution in microseconds.", +}; +/// Executors currently running transaction batches. +const BUSY_EXECUTORS: MetricSpec = MetricSpec { + name: "processor_busy_executors", + help: "Current processor executors running transaction batches.", +}; +/// Transactions queued behind account-lock conflicts. +const BLOCKED_TRANSACTIONS: MetricSpec = MetricSpec { + name: "processor_blocked_transactions", + help: "Current processor transactions queued behind account-lock conflicts.", +}; +/// Account lock conflict counter. +const LOCK_CONFLICTS: MetricSpec = MetricSpec { + name: "processor_lock_conflicts", + help: "Processor account-lock conflicts.", +}; +/// Failed transaction counter grouped by terminal failure kind. +const FAILED_TRANSACTIONS: MetricSpec = MetricSpec { + name: "processor_failed_transactions", + help: "Transactions dropped by the sequencer or failed during execution.", +}; + +/// Fixed failure kinds used to select pre-resolved counter handles. +#[derive(Clone, Copy)] +pub(crate) enum FailureKind { + /// Transaction rejected before executor dispatch. + SequencerDrop = 0, + /// Transaction load or execution result was unsuccessful. + Execution = 1, +} + +/// Processor operation used as a low-cardinality operation label. +#[derive(Clone, Copy)] +pub(crate) enum Operation { + /// Block finalization path. + FinalizeBlock, + /// Quiescence barrier drain path. + BarrierDrain, +} + +impl MetricOperation for Operation { + /// Returns the Prometheus label value for this operation. + fn label(self) -> &'static str { + match self { + Operation::FinalizeBlock => "finalize_block", + Operation::BarrierDrain => "barrier_drain", + } + } +} + +/// Registers processor metrics once. +pub(crate) fn init() { + METRICS.get_or_init(Default::default); +} + +/// Starts an operation timer that records latency when the returned guard drops. +pub(crate) fn time(op: Operation) -> OperationTimer<'static> { + op.time(METRICS.get().map(|m| &m.operations)) +} + +/// Refreshes the busy executor gauge. +pub(crate) fn busy_executors(count: usize) { + metric::with_metrics(&METRICS, |m| { + m.busy_executors.set(metric::gauge_value(count)) + }); +} + +/// Records one transaction queued behind an executor. +pub(crate) fn blocked_transaction() { + metric::with_metrics(&METRICS, |m| m.blocked_transactions.inc()); +} + +/// Records one queued transaction leaving an executor backlog. +pub(crate) fn unblocked_transaction() { + metric::with_metrics(&METRICS, |m| m.blocked_transactions.dec()); +} + +/// Records one account-lock conflict. +pub(crate) fn lock_conflict() { + metric::with_metrics(&METRICS, |m| m.lock_conflicts.inc()); +} + +/// Records one terminal transaction failure. +pub(crate) fn failed_transaction(kind: FailureKind) { + metric::with_metrics(&METRICS, |m| m.failed_transactions[kind as usize].inc()); +} + +/// Owns all Prometheus collectors registered by processor. +struct Metrics { + /// Runtime operation duration and completion counters. + operations: OperationCounters, + /// Executors currently running transaction batches. + busy_executors: IntGauge, + /// Transactions queued behind account-lock conflicts. + blocked_transactions: IntGauge, + /// Account-lock conflict counter. + lock_conflicts: IntCounter, + /// Per-kind failed transaction counters resolved during initialization. + failed_transactions: [IntCounter; 2], +} + +impl Default for Metrics { + /// Builds collectors and registers them in the default Prometheus registry. + fn default() -> Self { + let failed_transactions_vec = metric::counter_vec(FAILED_TRANSACTIONS, &["kind"]); + let failed_transactions = [ + failed_transactions_vec.with_label_values(&["dropped"]), + failed_transactions_vec.with_label_values(&["execution"]), + ]; + Self { + operations: OperationCounters::new(OPERATION_TIME), + busy_executors: metric::gauge(BUSY_EXECUTORS, 0), + blocked_transactions: metric::gauge(BLOCKED_TRANSACTIONS, 0), + lock_conflicts: metric::counter(LOCK_CONFLICTS, 0), + failed_transactions, + } + } +} diff --git a/processor/src/sequencer/locks.rs b/processor/src/sequencer/locks.rs new file mode 100644 index 00000000..89d8ea07 --- /dev/null +++ b/processor/src/sequencer/locks.rs @@ -0,0 +1,157 @@ +//! Account lock tracking for scheduled transactions. + +use ahash::HashMap; +use derive_more::{Deref, DerefMut}; +use keeper::ResolvedTransaction; +use solana_pubkey::Pubkey; +use solana_svm_transaction::svm_message::SVMMessage; + +use crate::executor::{ExecutorHandle, ExecutorId}; + +pub(super) const MAX_EXECUTORS: u32 = u64::BITS - 1; +const WRITE_BIT: u64 = 1 << MAX_EXECUTORS; + +/// Global account locks held by in-flight executor work. +#[derive(Default, Deref, DerefMut)] +pub(super) struct LockTable(HashMap); + +/// Tracks active account holders as executor bits plus a top write-mode bit, +/// with an optional contender granted priority on the account. +/// +/// [`WRITE_BIT`] is lock-mode metadata for the active holder set, not an +/// executor. `contender` is priority metadata and is not an active holder. +/// +/// Locks are reentrant per executor: an executor that already holds the account +/// can re-acquire it — including upgrading its read hold to a write — because a +/// conflict is only raised against a *different* executor. +#[derive(Default)] +pub(super) struct AccountLock { + /// Bitset of holding executors; [`WRITE_BIT`] set marks an exclusive lock. + lock: u64, + /// Executor that lost a conflict here and is owed the account next. + contender: Option, +} + +impl LockTable { + /// Acquires all account locks required by `txn` for `executor`. + /// + /// On conflict, locks acquired earlier in the transaction are rolled back + /// while preserving the blocking executor's contender priority. + pub(super) fn acquire( + &mut self, + executor: &mut ExecutorHandle, + txn: &ResolvedTransaction, + ) -> Result<(), ExecutorId> { + let id = executor.id; + let mut locked = 0; + let mut result = Ok(()); + for (i, &acc) in txn.static_account_keys().iter().enumerate() { + let lock = self.entry(acc).or_default(); + result = if txn.is_writable(i) { lock.write(id) } else { lock.read(id) }; + if result.is_err() { + break; + } + *executor.locks.entry(acc).or_default() += 1; + locked += 1; + } + let Err(blocker) = result else { + return Ok(()); + }; + for acc in txn.static_account_keys().iter().take(locked) { + let Some(count) = executor.locks.get_mut(acc) else { + continue; + }; + *count -= 1; + let released = *count == 0; + if released { + executor.locks.remove(acc); + } + let Some(lock) = self.get_mut(acc) else { + continue; + }; + // Retry runs on `blocker`; reserve its acquired prefix against other work. + lock.contend(blocker); + if released { + lock.unlock(id); + } + } + Err(blocker) + } + + /// Releases every account lock recorded in `executor.locks`. + pub(super) fn release(&mut self, executor: &mut ExecutorHandle) { + let id = executor.id; + for (acc, _) in executor.locks.drain() { + let Some(lock) = self.get_mut(&acc) else { + continue; + }; + lock.unlock(id); + if !lock.locked() { + self.remove(&acc); + } + } + } +} + +impl AccountLock { + /// Acquires an exclusive (write) lock for `executor`. + /// + /// Fails with the blocking executor's id if a contender other than + /// `executor` is queued, or if another executor already holds the account. + pub(super) fn write(&mut self, executor: ExecutorId) -> Result<(), ExecutorId> { + if let Some(contender) = self.contender + && contender != executor + { + return Err(contender); + } + self.contender.take(); + let holders = self.lock & !WRITE_BIT; + let bit = 1 << executor; + let others = holders & !bit; + if others != 0 { + return Err(others.trailing_zeros()); + } + self.lock = WRITE_BIT | bit; + + Ok(()) + } + + /// Acquires a shared (read) lock for `executor`. + /// + /// Fails with the blocking executor's id if a contender other than + /// `executor` is queued, or if another executor holds it for writing. + pub(super) fn read(&mut self, executor: ExecutorId) -> Result<(), ExecutorId> { + if let Some(contender) = self.contender + && contender != executor + { + return Err(contender); + } + self.contender.take(); + let holders = self.lock & !WRITE_BIT; + let bit = 1 << executor; + if self.lock & WRITE_BIT != 0 && holders & bit == 0 { + return Err(holders.trailing_zeros()); + } + self.lock |= bit; + + Ok(()) + } + + /// Releases `executor`'s hold on the account and its write mode, if held. + pub(super) fn unlock(&mut self, executor: ExecutorId) { + let bit = 1 << executor; + if self.lock & bit != 0 { + self.lock &= !(bit | WRITE_BIT); + } + } + + /// Records `executor` as the contender owed the account next. + pub(super) fn contend(&mut self, executor: ExecutorId) { + self.contender.replace(executor); + } + + /// Returns whether any executor still actively holds the account. + pub(super) fn locked(&self) -> bool { + self.lock & !WRITE_BIT != 0 + } +} diff --git a/processor/src/sequencer/mod.rs b/processor/src/sequencer/mod.rs new file mode 100644 index 00000000..93d8ce24 --- /dev/null +++ b/processor/src/sequencer/mod.rs @@ -0,0 +1,311 @@ +//! Transaction sequencer: resolves account-lock conflicts and fans +//! non-conflicting transactions out to a pool of executors. + +use std::{mem, sync::Arc, thread}; + +use blake3::Hasher; +use keeper::{Keeper, ResolvedTransaction, TransactionView}; +use nucleus::{ + Slot, + ledger::Block, + runtime::{BarrierGuard, SequencerHandle}, + shutdown::{Service, ShutdownHandle, ShutdownManager, ShutdownReason}, +}; +use solana_hash::Hash; +use solana_program_runtime::loaded_programs::ProgramCache; +use solana_signature::Signature; +use tokio::{runtime::Builder, sync::mpsc}; +use tracing::{debug, error, info, warn}; + +mod locks; +mod pool; + +#[cfg(test)] +mod tests; + +use self::{ + locks::{LockTable, MAX_EXECUTORS}, + pool::Executors, +}; +use crate::{ + ExecutorMessage, ExecutorReady, ProcessorError, Result, SequencerMessage, + executor::TransactionExecutor, + metrics::{self, FailureKind, Operation}, + simulator::TransactionSimulator, +}; + +/// Per-executor blocked-queue threshold that pauses inbound scheduling. +/// Retry requeueing may temporarily grow a queue beyond this threshold. +const MAX_BLOCKED_EXECUTOR_TXNS: usize = 16; + +/// Schedules inbound transactions onto executors, ordering them by per-account +/// lock conflicts and finalizing block boundaries. +pub struct Sequencer { + /// Durable engine state used for appends and account/block lookups. + state: Arc, + /// Per-account locks held by in-flight transactions, used to detect conflicts. + locks: LockTable, + /// Inbound stream of transactions and block boundaries. + rx: mpsc::Receiver, + /// Hash-chain state for the block currently being sequenced. + hasher: BlockHasher, + /// The executor pool and its availability bookkeeping. + executors: Executors, + /// Slot currently being sequenced. + slot: Slot, + /// Handle used to observe and report cooperative shutdown. + shutdown: ShutdownHandle, + /// Whether the sequencer runs in ledger-replay mode; propagated to every + /// spawned executor (see the executor's `replay` field). + replay: bool, +} + +impl Sequencer { + /// Spawns the executor pool and assembles a sequencer over the given state. + pub fn new( + executors: usize, + state: Arc, + cache: Arc, + shutdown: &mut ShutdownManager, + replay: bool, + ) -> Result<(Self, SequencerHandle)> { + let count = executors.min(MAX_EXECUTORS as usize); + metrics::init(); + let (ready_tx, ready_rx) = mpsc::channel(count); + let mut executors = Vec::with_capacity(count); + for id in 0..count as u32 { + let handle = TransactionExecutor::spawn( + id, + state.clone(), + cache.clone(), + shutdown, + ready_tx.clone(), + replay, + )?; + executors.push(handle); + } + + let hasher = BlockHasher::new(state.blockhash()); + + let (execution, rx) = mpsc::channel(1024); + let simulation = TransactionSimulator::spawn(state.clone(), cache, shutdown)?; + let shutdown = shutdown.handle(Service::Sequencer); + + let sequencer = Self { + slot: state.blocks().current_slot(), + state, + locks: Default::default(), + executors: Executors::new(executors, ready_rx), + rx, + hasher, + shutdown, + replay, + }; + let handle = SequencerHandle { execution, simulation }; + info!(executors = count, replay, "sequencer started"); + Ok((sequencer, handle)) + } + + /// Moves the sequencer onto its own thread with a current-thread runtime. + pub fn spawn(self) -> Result<()> { + let runtime = Builder::new_current_thread().build()?; + thread::Builder::new() + .name("transaction-sequencer".into()) + .spawn(move || runtime.block_on(self.run()))?; + Ok(()) + } + + /// Main loop: drains executor-ready signals, inbound messages, and the + /// shutdown signal until cancellation, then joins the executor threads. + async fn run(mut self) { + let reason = loop { + let result = tokio::select! { + biased; + Some(msg) = self.executors.ready.recv() => { + self.handle_ready(msg) + } + _ = self.shutdown.signalled() => { + break ShutdownReason::Signalled; + } + Some(msg) = self.rx.recv(), if self.executors.ready() => { + self.handle_message(msg).await + } + }; + if let Err(error) = result { + error!(?error, "sequencer failed, terminating"); + break ShutdownReason::Error(Box::new(error)); + } + }; + info!("sequencer is draining the in-flight work"); + let _ = self.drain().await; + for mut e in self.executors.handles.drain(..) { + let handle = e.task.take(); + drop(e); + if let Some(handle) = handle { + let _ = handle.join(); + } + } + // Release engine storage before the manager can reopen it. + drop(self.state); + self.shutdown.terminate(reason); + } + + /// Dispatches an inbound message to the transaction or block handler. + async fn handle_message(&mut self, msg: SequencerMessage) -> Result<()> { + match msg { + SequencerMessage::Transaction(txn) => self.schedule(txn).await, + SequencerMessage::Block(block) => self.finalize(block).await, + SequencerMessage::Barrier(guard) => self.barrier(guard).await, + } + } + + /// Resolves account-lock conflicts for a transaction and either dispatches + /// it to a free executor or queues it behind the executor that blocks it. + async fn schedule(&mut self, txn: TransactionView) -> Result<()> { + let Ok(txn) = ResolvedTransaction::try_new(txn, None, &Default::default()) else { + metrics::failed_transaction(FailureKind::SequencerDrop); + return Ok(()); + }; + if !self.replay { + if !self.state.transactions().append(&txn).await? { + metrics::failed_transaction(FailureKind::SequencerDrop); + return Ok(()); + } + self.hasher.update(&txn.signatures()[0]); + } + let Some(executor) = self.executors.available() else { + // All executors are busy; park this unlocked transaction on executor 0. + self.executors.enqueue(txn, 0); + return Ok(()); + }; + if let Err(blocker) = self.locks.acquire(executor, &txn) { + metrics::lock_conflict(); + self.executors.enqueue(txn, blocker); + return Ok(()); + } + executor.batch.push(txn); + let id = executor.id; + self.executors.dispatch(id) + } + + /// Reclaims an executor that finished its batch: releases the account locks + /// it held, then re-tries the transactions queued behind it. Any that can + /// now acquire their locks form a fresh batch dispatched back to it; the + /// rest are re-queued behind whichever executor still blocks them. + fn handle_ready(&mut self, msg: ExecutorReady) -> Result<()> { + let id = msg.id; + let idx = id as usize; + let Some(executor) = self.executors.release(id) else { + warn!(id, "ready signal for unknown executor; ignoring"); + return Ok(()); + }; + executor.batch = msg.batch; + self.locks.release(executor); + let mut blocked = mem::take(&mut executor.blocked); + // Lock acquisition is reentrant per executor, so a retry cannot be + // enqueued back into its own detached queue. + while let Some(txn) = blocked.pop_front() { + metrics::unblocked_transaction(); + let executor = &mut self.executors.handles[idx]; + if let Err(blocker) = self.locks.acquire(executor, &txn) { + metrics::lock_conflict(); + self.executors.enqueue(txn, blocker); + continue; + } + executor.batch.push(txn); + } + self.executors.handles[idx].blocked = blocked; + self.executors.dispatch(id) + } + + /// Raises a quiescence barrier: drains all in-flight work so the sequencer + /// and its executors are idle, acknowledges the controller, then waits to be + /// released before resuming. Used to take a consistent state snapshot at + /// superblock boundaries. + async fn barrier(&mut self, guard: BarrierGuard) -> Result<()> { + info!("sequencer is halting operation"); + self.drain().await?; + let _ = guard.acknowledged.send(()); + let _ = guard.released.await; + info!("sequencer is resuming operation"); + Ok(()) + } + + /// Awaits executor-ready signals, reclaiming each finished executor + /// until the whole pool is idle. Used to ensure that the in-flight + /// work is complete before finalizing a block and during shutdown. + async fn drain(&mut self) -> Result<()> { + let _timer = metrics::time(Operation::BarrierDrain); + while !self.executors.idle() { + let Some(signal) = self.executors.ready.recv().await else { + debug!("executor pool closed during drain; abandoning in-flight work"); + return Ok(()); + }; + self.handle_ready(signal)?; + } + Ok(()) + } + + /// Finalizes the current block: chains its hash, appends it, and notifies + /// every executor of the new block boundary. + async fn finalize(&mut self, mut block: Block) -> Result<()> { + let _timer = metrics::time(Operation::FinalizeBlock); + if self.replay && block.parent != self.hasher.parent { + return Err(ProcessorError::Internal(format!( + "replayed block {} has parent {:?}, expected {:?}", + block.slot, block.parent, self.hasher.parent + ))); + } + if !self.replay { + block.parent = self.hasher.parent; + block.hash = self.hasher.finalize(); + } + + // Block boundaries synchronize executors: + // 1. sysvar writes bypass account locks and must be ordered deterministically + // 2. replaying should schedule transactions in their original block + self.drain().await?; + for e in &self.executors.handles { + e.tx.send(ExecutorMessage::Block(block)) + .map_err(|_| Service::TransactionExecutor(e.id))?; + } + self.state.blocks().append(block, self.replay)?; + self.hasher.advance(block.hash); + self.slot = self.state.blocks().current_slot(); + Ok(()) + } +} + +/// Rolling state for the locally computed block-hash chain. +struct BlockHasher { + /// Hasher seeded with `parent` for the block currently being finalized. + current: Hasher, + /// Latest committed block hash. + parent: Hash, +} + +impl BlockHasher { + /// Starts a block hash from the latest committed block hash. + fn new(parent: Hash) -> Self { + let mut current = Hasher::new(); + current.update(parent.as_ref()); + Self { current, parent } + } + + /// Adds an appended transaction's canonical signature to the current block. + fn update(&mut self, signature: &Signature) { + self.current.update(signature.as_ref()); + } + + /// Finalizes the current block hash without advancing committed state. + fn finalize(&self) -> Hash { + Hash::from(*self.current.finalize().as_bytes()) + } + + /// Advances the chain after the finalized block has committed successfully. + fn advance(&mut self, hash: Hash) { + self.parent = hash; + self.current.reset(); + self.current.update(hash.as_ref()); + } +} diff --git a/processor/src/sequencer/pool.rs b/processor/src/sequencer/pool.rs new file mode 100644 index 00000000..0938ed89 --- /dev/null +++ b/processor/src/sequencer/pool.rs @@ -0,0 +1,137 @@ +//! Executor-pool bookkeeping for the sequencer. + +use std::mem; + +use keeper::ResolvedTransaction; +use nucleus::shutdown::Service; +use tokio::sync::mpsc::Receiver; +use tracing::warn; + +use super::MAX_BLOCKED_EXECUTOR_TXNS; +use crate::{ + ExecutorMessage, ExecutorReady, Result, + executor::{ExecutorHandle, ExecutorId}, + metrics, +}; + +/// The pool of executors and the state needed to dispatch work to them. +pub(super) struct Executors { + /// One handle per executor worker, indexed by [`ExecutorId`]. + pub(super) handles: Vec, + /// Channel on which executors signal they have finished a batch. + pub(super) ready: Receiver, + /// Bitset of executors currently free to accept a batch. + available: AvailableExecutors, +} + +/// Bitset of free executors, one bit per [`ExecutorId`]. +pub(super) struct AvailableExecutors { + /// Set bits are executor IDs currently free to accept work. + bitflags: u64, + /// Number of executor slots represented by `bitflags`. + total: u32, +} + +impl Executors { + /// Builds the pool from spawned executor handles and their readiness channel. + pub(super) fn new(handles: Vec, ready: Receiver) -> Self { + let available = AvailableExecutors::new(handles.len() as u32); + Self { handles, ready, available } + } + + /// Whether the pool can accept more work: at least one executor is free and + /// no executor's blocked queue has reached the backpressure threshold. + pub(super) fn ready(&self) -> bool { + let saturated = |h: &ExecutorHandle| h.blocked.len() >= MAX_BLOCKED_EXECUTOR_TXNS; + !(self.available.empty() || self.handles.iter().any(saturated)) + } + + /// Returns whether every executor is currently free. + pub(super) fn idle(&self) -> bool { + self.available.idle() + } + + /// Queues a transaction behind the executor that currently blocks it, to be + /// retried once that executor releases its conflicting locks. + pub(super) fn enqueue(&mut self, txn: ResolvedTransaction, executor: ExecutorId) { + self.handles[executor as usize].blocked.push_back(txn); + metrics::blocked_transaction(); + } + + /// Returns a handle to a currently free executor, or `None` if all are busy. + /// The executor is not yet marked busy; the caller does that once it commits + /// a batch to it. + pub(super) fn available(&mut self) -> Option<&mut ExecutorHandle> { + self.available.get().and_then(|idx| self.get(idx)) + } + + /// Returns the handle for executor `idx`, if such an executor exists. + fn get(&mut self, idx: ExecutorId) -> Option<&mut ExecutorHandle> { + self.handles.get_mut(idx as usize) + } + + /// Marks executor `idx` as available again and returns its handle. + pub(super) fn release(&mut self, idx: ExecutorId) -> Option<&mut ExecutorHandle> { + self.available.insert(idx); + metrics::busy_executors(self.available.busy()); + self.get(idx) + } + + /// Sends executor `idx`'s accumulated batch to its worker and marks it busy. + /// A no-op if the executor is unknown or its batch is empty. + pub(super) fn dispatch(&mut self, idx: ExecutorId) -> Result<()> { + let Some(executor) = self.get(idx) else { + warn!(idx, "dispatch to unknown executor; ignoring"); + return Ok(()); + }; + if executor.batch.is_empty() { + return Ok(()); + } + let msg = ExecutorMessage::Transactions(mem::take(&mut executor.batch)); + executor.tx.send(msg).map_err(|_| Service::TransactionExecutor(idx))?; + self.available.remove(idx); + metrics::busy_executors(self.available.busy()); + Ok(()) + } +} + +impl AvailableExecutors { + /// Starts with every executor marked available. + pub(super) fn new(executors: u32) -> Self { + Self { + bitflags: (1u64 << executors) - 1, + total: executors, + } + } + + /// Returns the id of an available executor, or `None` if all are busy. + pub(super) fn get(&self) -> Option { + let position = self.bitflags.trailing_zeros(); + (position != u64::BITS).then_some(position) + } + + /// Returns whether no executor is currently available. + pub(super) fn empty(&self) -> bool { + self.bitflags == 0 + } + + /// Returns whether every executor is currently free. + pub(super) fn idle(&self) -> bool { + self.bitflags.count_ones() == self.total + } + + /// Returns how many executors are currently busy. + pub(super) fn busy(&self) -> usize { + (self.total - self.bitflags.count_ones()) as usize + } + + /// Marks an executor as busy. + pub(super) fn remove(&mut self, executor: ExecutorId) { + self.bitflags &= !(1 << executor) + } + + /// Marks an executor as available again. + pub(super) fn insert(&mut self, executor: ExecutorId) { + self.bitflags |= 1 << executor + } +} diff --git a/processor/src/sequencer/tests.rs b/processor/src/sequencer/tests.rs new file mode 100644 index 00000000..95e14d5d --- /dev/null +++ b/processor/src/sequencer/tests.rs @@ -0,0 +1,255 @@ +//! Sequencer unit tests for account locks and executor availability. +//! +//! These tests stay below the public processor surface so they can exercise the +//! scheduling invariants directly: lock fairness, partial-acquire behavior, and +//! the bookkeeping that decides when executor work can be drained. + +use std::{ + collections::VecDeque, + sync::{Arc, mpsc}, +}; + +use keeper::{ + Keeper, + testkit::{TestKeeper, resolved}, +}; +use nucleus::shutdown::Service; +use solana_pubkey::Pubkey; +use tokio::sync::mpsc as tokio_mpsc; + +use super::{ + BlockHasher, Sequencer, + locks::AccountLock, + pool::{AvailableExecutors, Executors}, +}; +use crate::{ + ExecutorMessage, ExecutorReady, + executor::{ExecutorHandle, ExecutorId, ExecutorWork}, +}; + +/// Constructs a bare sequencer over `tk`'s keeper without spawning executors. +/// +/// Tests fill only the fields needed to call scheduling helpers directly; the +/// message channels are intentionally inert. +fn sequencer(tk: &mut TestKeeper) -> Sequencer { + let state: Arc = tk.clone(); + let (_tx, rx) = tokio_mpsc::channel(1); + let (_ready_tx, ready_rx) = tokio_mpsc::channel(1); + let hasher = BlockHasher::new(state.blockhash()); + Sequencer { + slot: state.blocks().current_slot(), + state, + locks: Default::default(), + rx, + hasher, + executors: Executors::new(Vec::new(), ready_rx), + shutdown: tk.shutdown.handle(Service::Sequencer), + replay: false, + } +} + +/// Builds an idle executor handle plus the receiver end of its dispatch channel, +/// so a test can observe the batch `dispatch` actually sends. +fn executor_with_rx(id: ExecutorId) -> (ExecutorHandle, mpsc::Receiver) { + let (tx, rx) = mpsc::sync_channel(1); + let handle = ExecutorHandle { + id, + work: ExecutorWork { + batch: Vec::new(), + locks: Default::default(), + blocked: VecDeque::new(), + }, + tx, + task: None, + }; + (handle, rx) +} + +/// [`executor_with_rx`] for tests that never inspect what was dispatched. +fn executor(id: ExecutorId) -> ExecutorHandle { + executor_with_rx(id).0 +} + +// Readers share a lock until a writer contends, then the writer waits for every +// current reader and is granted before later readers. +#[test] +fn read_locks_share_until_a_writer_arrives() { + let mut lock = AccountLock::default(); + + assert_eq!(lock.read(0), Ok(())); + assert_eq!(lock.read(1), Ok(())); + assert_eq!(lock.write(2), Err(0)); + assert!(lock.locked()); + + lock.unlock(0); + assert_eq!(lock.write(2), Err(1)); + lock.unlock(1); + assert_eq!(lock.write(2), Ok(())); +} + +// One executor may reacquire a lock it already owns and may upgrade its own read +// lock to a write lock without blocking itself. +#[test] +fn same_executor_can_reenter_and_upgrade() { + let mut lock = AccountLock::default(); + + assert_eq!(lock.read(4), Ok(())); + assert_eq!(lock.write(4), Ok(())); + assert_eq!(lock.read(4), Ok(())); + + lock.unlock(4); + assert!(!lock.locked()); + assert_eq!(lock.read(5), Ok(())); +} + +// A contending executor keeps priority after it is queued, preventing unrelated +// readers from slipping ahead while the current writer drains. +#[test] +fn contender_gets_priority_until_granted() { + let mut lock = AccountLock::default(); + + assert_eq!(lock.write(0), Ok(())); + lock.contend(1); + assert_eq!(lock.read(2), Err(1)); + + lock.unlock(0); + assert_eq!(lock.read(1), Ok(())); + assert_eq!(lock.read(2), Ok(())); +} + +// If acquiring a multi-account transaction fails partway through, already-held +// locks keep the blocked executor marked as the contender until its turn. +#[tokio::test(flavor = "current_thread")] +async fn acquire_locks_preserves_contender_priority_after_partial_conflict() { + let mut tk = TestKeeper::new().await; + let mut sequencer = sequencer(&mut tk); + let a = Pubkey::new_unique(); + let b = Pubkey::new_unique(); + let mut blocker = executor(0); + let mut blocked = executor(1); + + sequencer.locks.acquire(&mut blocker, &resolved(&[(b, true)])).unwrap(); + let err = sequencer + .locks + .acquire(&mut blocked, &resolved(&[(a, true), (b, true)])) + .expect_err("b blocks the second transaction"); + + assert_eq!(err, 0); + assert_eq!(blocked.locks.get(&a), None); + // `a` was released after `b` conflicted, but the blocker keeps contender + // priority so unrelated executors cannot acquire it before the retry. + let a_lock = sequencer.locks.get_mut(&a).expect("a lock remains"); + assert!(a_lock.read(1).is_err()); + assert_eq!(a_lock.read(0), Ok(())); + assert!(blocker.locks.contains_key(&b)); +} + +// Executor availability reports the first idle executor, the busy count, and the +// all-idle/all-busy states used by sequencer drain logic. +#[test] +fn available_executors_track_busy_and_idle_state() { + let mut available = AvailableExecutors::new(3); + + assert_eq!(available.get(), Some(0)); + assert!(available.idle()); + available.remove(0); + available.remove(2); + + assert_eq!(available.get(), Some(1)); + assert_eq!(available.busy(), 2); + assert!(!available.empty()); + assert!(!available.idle()); + + available.remove(1); + assert!(available.empty()); + assert_eq!(available.get(), None); + + available.insert(2); + assert_eq!(available.get(), Some(2)); + assert_eq!(available.busy(), 2); +} + +// When a freed executor retries a transaction queued behind it, and that +// transaction re-acquires its now-free lock but then conflicts with a lock still +// held by another executor, it rolls back and is re-queued behind the new blocker. +#[tokio::test(flavor = "current_thread")] +async fn handle_ready_requeues_behind_the_new_blocker() { + let mut tk = TestKeeper::new().await; + let mut sequencer = sequencer(&mut tk); + let x = Pubkey::new_unique(); + let y = Pubkey::new_unique(); + + let mut e0 = executor(0); + let mut e1 = executor(1); + sequencer.locks.acquire(&mut e0, &resolved(&[(x, true)])).unwrap(); + sequencer.locks.acquire(&mut e1, &resolved(&[(y, true)])).unwrap(); + // A transaction needing both x and y is parked behind executor 0, the x holder. + e0.blocked.push_back(resolved(&[(x, true), (y, true)])); + + let (_ready_tx, ready_rx) = tokio_mpsc::channel(1); + sequencer.executors = Executors::new(vec![e0, e1], ready_rx); + + // Executor 0 finishes: x is released, but the retried transaction now conflicts + // with y (still held by executor 1) and is re-parked behind executor 1. + sequencer.handle_ready(ExecutorReady { id: 0, batch: Vec::new() }).unwrap(); + + assert_eq!( + sequencer.executors.handles[1].blocked.len(), + 1, + "requeued behind y's holder" + ); + assert!( + sequencer.executors.handles[0].blocked.is_empty(), + "no longer queued behind x" + ); + assert!( + sequencer.executors.handles[0].batch.is_empty(), + "nothing dispatched to executor 0" + ); + // x was rolled back (no holder) but keeps executor 1 as its priority contender. + let x_lock = sequencer.locks.get_mut(&x).expect("x lock retained"); + assert!(!x_lock.locked()); + assert_eq!( + x_lock.read(2), + Err(1), + "blocker keeps contender priority on x" + ); + // y is still write-held by executor 1. + assert_eq!( + sequencer.locks.get_mut(&y).expect("y lock retained").read(2), + Err(1) + ); +} + +// A freed executor re-dispatches a transaction queued behind it once it can fully +// re-acquire its locks, handing it back to the executor in a fresh batch. +#[tokio::test(flavor = "current_thread")] +async fn handle_ready_redispatches_unblocked_transaction() { + let mut tk = TestKeeper::new().await; + let mut sequencer = sequencer(&mut tk); + let x = Pubkey::new_unique(); + + let (mut e0, rx) = executor_with_rx(0); + sequencer.locks.acquire(&mut e0, &resolved(&[(x, true)])).unwrap(); + e0.blocked.push_back(resolved(&[(x, true)])); + + let (_ready_tx, ready_rx) = tokio_mpsc::channel(1); + sequencer.executors = Executors::new(vec![e0], ready_rx); + + // Executor 0 finishes: x is released, the queued transaction re-acquires it and + // is dispatched straight back to executor 0. + sequencer.handle_ready(ExecutorReady { id: 0, batch: Vec::new() }).unwrap(); + + let ExecutorMessage::Transactions(batch) = rx.try_recv().expect("batch dispatched") else { + panic!("dispatched a transaction batch"); + }; + assert_eq!(batch.len(), 1); + assert!(batch[0].static_account_keys().contains(&x)); + // x is held again for the redispatched transaction, and its queue is empty. + assert_eq!( + sequencer.locks.get_mut(&x).expect("x lock").read(2), + Err(0), + "x re-held by executor 0" + ); + assert!(sequencer.executors.handles[0].blocked.is_empty()); +} diff --git a/processor/src/simulator.rs b/processor/src/simulator.rs new file mode 100644 index 00000000..dee21a32 --- /dev/null +++ b/processor/src/simulator.rs @@ -0,0 +1,113 @@ +//! Transaction simulation: executes transactions against current state on +//! owned account copies, without committing any changes. + +use std::{sync::Arc, thread}; + +use keeper::{ExecutionRecord, Keeper, ResolvedTransaction}; +use nucleus::{ + shutdown::{Service, ShutdownHandle, ShutdownManager, ShutdownReason}, + tls::AUTHORITY, +}; +use solana_program_runtime::loaded_programs::ProgramCache; +use solana_transaction_error::TransactionError; +use tokio::{ + runtime::Builder, + sync::mpsc::{self, Receiver, Sender}, +}; +use tracing::debug; + +use crate::{Result, Simulation, SimulatorMessage, callback::SVMCallback, svm::SvmContext}; + +/// Worker that simulates transactions against current state without committing. +pub struct TransactionSimulator { + /// Inbound channel of simulation requests and block boundaries. + rx: Receiver, + /// SVM batch processor and per-block environment driving simulation. + svm: SvmContext, + /// Durable engine state used for account loads. + state: Arc, + /// Handle used to report cooperative shutdown for this worker. + shutdown: ShutdownHandle, +} + +impl TransactionSimulator { + /// Builds the SVM environment and spawns the simulator on its own thread. + pub fn spawn( + state: Arc, + cache: Arc, + shutdown: &mut ShutdownManager, + ) -> Result> { + let svm = SvmContext::new(&state, cache)?; + let shutdown = shutdown.handle(Service::TransactionSimulator); + let (tx, rx) = mpsc::channel(16); + let executor = Self { rx, svm, state, shutdown }; + let runtime = Builder::new_current_thread().build()?; + thread::Builder::new() + .name("transaction-simulator".into()) + .spawn(move || runtime.block_on(executor.run()))?; + Ok(tx) + } + + /// Worker loop: simulates requests and applies block transitions until the + /// channel closes, then reports cooperative shutdown. + async fn run(mut self) { + // Mirror the executor: simulated MagicRoot calls authorize against the + // same authority published on this simulator thread. + AUTHORITY.set(self.state.authority()); + loop { + tokio::select! { + biased; + _ = self.shutdown.signalled() => { + break; + } + msg = self.rx.recv() => { + let Some(msg) = msg else { + break; + }; + self.handle_message(msg); + } + } + } + // Release engine storage before the manager can reopen it. + drop(self.state); + self.shutdown.terminate(ShutdownReason::Signalled); + } + + fn handle_message(&mut self, msg: SimulatorMessage) { + match msg { + SimulatorMessage::Transaction(simulation) => { + self.process(simulation); + } + SimulatorMessage::Block(block) => self.svm.transition(block), + SimulatorMessage::Barrier(guard) => { + let _ = guard.acknowledged.send(()); + let _ = guard.released.recv(); + } + }; + } + + /// Resolves and executes a single transaction on owned account copies, + /// producing a result without persisting any state. + fn process(&mut self, simulation: Simulation) { + let result = + ResolvedTransaction::try_new(simulation.transaction, None, &Default::default()); + let Ok(txn) = result else { + debug!("simulation rejected: transaction resolution failed"); + let error = Err(TransactionError::InvalidAddressLookupTableData); + let _ = simulation.response.send(error); + return; + }; + let accessor = self.state.accounts(); + let callback = SVMCallback:: { + loader: accessor.loader(), + featureset: self.state.features(), + }; + let output = self.svm.execute(&callback, &txn, self.state.features()); + let execution = ExecutionRecord { + result: output.processing_result, + balances: output.balance_collector, + slot: self.svm.slot(), + }; + let _ = simulation.response.send(Ok(execution)); + } +} diff --git a/processor/src/svm.rs b/processor/src/svm.rs new file mode 100644 index 00000000..8ee8ca60 --- /dev/null +++ b/processor/src/svm.rs @@ -0,0 +1,136 @@ +//! Shared SVM execution context for the transaction executor and simulator. + +use std::sync::Arc; + +use agave_feature_set::FeatureSet; +use agave_transaction_view::{ + MAGICBLOCK_INSTRUCTION_TRACE_LENGTH, transaction_version::TransactionVersion, +}; +use keeper::{Keeper, ResolvedTransaction}; +use nucleus::{Slot, ledger::Block}; +use solana_compute_budget_instruction::instructions_processor::process_compute_budget_instructions; +use solana_program_runtime::loaded_programs::{ProgramCache, ProgramRuntimeEnvironments}; +use solana_svm::{ + account_loader::CheckedTransactionDetails, + transaction_processing_callback::TransactionProcessingCallback, + transaction_processor::{ + ExecutionRecordingConfig, LoadAndExecuteSanitizedTransactionOutput, + TransactionBatchProcessor, TransactionProcessingConfig, TransactionProcessingEnvironment, + }, +}; +use solana_sysvar::clock::Clock; +use solana_transaction_error::TransactionResult; + +use crate::{Result, callback::SVMCallback}; + +/// SVM batch processor together with its per-block environment and recording +/// config, shared verbatim by the executor and simulator workers. +pub(crate) struct SvmContext { + /// SVM batch processor that loads and executes transactions. + processor: TransactionBatchProcessor, + /// Per-block processing environment (blockhash, features, rent, ...). + env: TransactionProcessingEnvironment, + /// Recording and logging configuration for execution. + config: TransactionProcessingConfig, +} + +impl SvmContext { + /// Builds the SVM runtime environment and batch processor from current state. + pub(crate) fn new(state: &Keeper, cache: Arc) -> Result { + let budget = Default::default(); + let runtime_env = agave_syscalls::create_program_runtime_environment( + &state.features().runtime_features(), + &budget, + true, + false, + ) + .map_err(|e| e.to_string())?; + let envs = ProgramRuntimeEnvironments::new(runtime_env); + let block = state.blocks().latest(); + let env = TransactionProcessingEnvironment { + blockhash: block.hash, + feature_set: state.features().runtime_features(), + program_runtime_environments_for_execution: envs, + rent: state.rent().clone(), + blockhash_lamports_per_signature: 0, + epoch_total_stake: 0, + }; + let mut processor = TransactionBatchProcessor::new(block.slot + 1, cache); + let accessor = state.accounts(); + let callback = SVMCallback:: { + loader: accessor.loader(), + featureset: state.features(), + }; + processor.fill_missing_sysvar_cache_entries(&callback); + let config = TransactionProcessingConfig { + log_messages_bytes_limit: None, + recording_config: ExecutionRecordingConfig::new_single_setting(true), + }; + Ok(Self { processor, env, config }) + } + + /// Loads and executes a single transaction through the SVM. + pub(crate) fn execute( + &self, + callback: &impl TransactionProcessingCallback, + txn: &ResolvedTransaction, + features: &FeatureSet, + ) -> LoadAndExecuteSanitizedTransactionOutput { + let details = match self.parse_details(txn, features) { + Ok(d) => d, + Err(e) => { + return LoadAndExecuteSanitizedTransactionOutput { + processing_result: Err(e), + balance_collector: None, + }; + } + }; + self.processor.load_and_execute_sanitized_transaction( + callback, + txn, + details, + &self.env, + &self.config, + ) + } + + /// Advances the context to a new block: bumps the slot and blockhash and + /// refreshes the cached clock sysvar. + pub(crate) fn transition(&mut self, block: Block) { + let slot = block.slot + 1; + let hash = block.hash; + self.processor.slot = slot; + self.env.blockhash = hash; + let clock = Clock { + slot, + unix_timestamp: block.time, + ..Default::default() + }; + self.processor.sysvar_cache_mut().set_clock(&clock); + } + + /// Slot the context is currently executing against. + pub(crate) fn slot(&self) -> Slot { + self.processor.slot + } + + /// Derives the compute budget and limits from the transaction's compute-budget + /// instructions. Fees are forced to zero and depth-8 CPIs disabled on the ER. + pub(crate) fn parse_details( + &self, + txn: &ResolvedTransaction, + features: &FeatureSet, + ) -> TransactionResult { + let ixs = txn.program_instructions_iter(); + let limits = process_compute_budget_instructions(ixs, features)?; + let mut limits = limits.get_compute_budget_and_limits( + limits.loaded_accounts_bytes, + Default::default(), // Fee is always zero on ER + false, // Depth-8 CPIs are disabled on solana + ); + if matches!(txn.version(), TransactionVersion::Magicblock) { + limits.budget.max_instruction_trace_length = MAGICBLOCK_INSTRUCTION_TRACE_LENGTH; + } + Ok(CheckedTransactionDetails::new(None, limits)) + } +} diff --git a/processor/src/tests.rs b/processor/src/tests.rs new file mode 100644 index 00000000..735f95b4 --- /dev/null +++ b/processor/src/tests.rs @@ -0,0 +1,390 @@ +//! End-to-end processor checks driven by the loadable v42 calculator program. + +use std::{sync::Arc, time::Duration}; + +use crate::{SequencerMessage, SimulatorMessage, sequencer::Sequencer}; +use derive_more::Deref; +use keeper::{ + TransactionStatus, TransactionView, + testkit::{TestKeeper, V42_ID, load_v42_data, load_v42_lamports, signed_view, store_v42}, +}; +use nucleus::{ + ledger::Block, + runtime::{SequencerHandle, Simulation, barrier}, +}; +use solana_account::{AccountMode, ReadableAccount}; +use solana_hash::Hash; +use solana_instruction::Instruction; +use solana_keypair::Keypair; +use solana_program_runtime::loaded_programs::ProgramCache; +use solana_pubkey::Pubkey; +use solana_sdk_ids::loader_v4; +use solana_signature::Signature; +use solana_svm::transaction_processing_result::{ + TransactionProcessingResult, TransactionProcessingResultExtensions, +}; +use tokio::time::timeout; +use v42_calculator_interface::builder::{Expr as E, transfer}; + +/// End-to-end fixture with a real keeper, sequencer, simulator, and v42 program. +#[derive(Deref)] +struct Harness { + /// Seeded keeper (v42 + funded payer) owning the engine state along with the + /// directories and shutdown lifecycle its background services run on. + #[deref] + keeper: TestKeeper, + /// Channels used by tests to drive execution and simulation paths. + handle: SequencerHandle, +} + +impl Harness { + /// Builds a seeded keeper and wires a sequencer/simulator onto its lifecycle. + /// + /// `replay` selects whether the sequencer records transaction status while + /// still committing account state, matching the processor replay mode. + async fn new(replay: bool) -> Self { + let (harness, sequencer) = Self::unspawned(replay).await; + sequencer.spawn().unwrap(); + harness + } + + /// Builds a seeded keeper and sequencer without starting the sequencer loop. + /// + /// This lets tests fill the execution channel before the sequencer can + /// consume from it, forcing contention resolution to happen from a backlog. + async fn unspawned(replay: bool) -> (Self, Sequencer) { + let mut keeper = TestKeeper::new().await; + let (sequencer, handle) = Sequencer::new( + 2, + keeper.clone(), + Arc::new(ProgramCache::default()), + &mut keeper.shutdown, + replay, + ) + .unwrap(); + (Self { keeper, handle }, sequencer) + } + + /// A signed transaction view paid for by a fresh payer, not the shared one. + /// + /// The shared payer is writable in every transaction it signs, which would + /// make all of them conflict on that one account and mask the account + /// intersections the contention tests exist to stress. + fn fresh_payer_view(&self, instruction: Instruction) -> (Signature, TransactionView) { + signed_view(self, Some(&Keypair::new()), instruction) + } + + /// Queues a transaction on the execution path without waiting for commit. + async fn execute(&self, tx: TransactionView) { + self.handle.execution.send(SequencerMessage::Transaction(tx)).await.unwrap(); + } + + /// Runs a transaction through simulation and returns the raw processing result. + /// + /// Simulation shares the keeper state but must not persist account writes or + /// transaction status. + async fn simulate(&self, tx: TransactionView) -> TransactionProcessingResult { + let (response, rx) = oneshot::channel(); + self.handle + .simulation + .send(SimulatorMessage::Transaction(Simulation { + transaction: tx, + response, + })) + .await + .unwrap(); + rx.await.unwrap().expect("simulation resolves").result + } + + /// Sends the same block transition to execution and simulation services. + /// + /// Both paths maintain sysvar caches, so block metadata must be delivered to + /// both before comparing simulated and committed behavior. + async fn set_block(&self, block: Block) { + self.handle.execution.send(SequencerMessage::Block(block)).await.unwrap(); + self.handle.simulation.send(SimulatorMessage::Block(block)).await.unwrap(); + } + + /// Waits for all previously submitted execution work to finish, failing + /// quickly if lock contention stops making progress. + async fn barrier(&self) { + let (controller, guard) = barrier(); + self.handle.execution.send(SequencerMessage::Barrier(guard)).await.unwrap(); + timeout(Duration::from_secs(8), controller.acknowledged) + .await + .expect("barrier timed out") + .unwrap(); + controller.released.send(()).unwrap(); + } + + /// Returns the status recorded for `signature`, which must exist. + async fn status(&self, signature: Signature) -> TransactionStatus { + self.transactions() + .status(signature) + .await + .unwrap() + .expect("executed transaction records a status") + } + + /// Drops service handles and waits for background tasks to stop. + /// + /// [`TestKeeper::close`] flushes, so nothing needs flushing here. + async fn close(self) { + let Self { keeper, handle } = self; + drop(handle); + keeper.close().await; + } +} + +/// Asserts that SVM processing accepted the transaction. +fn assert_success(result: &TransactionProcessingResult) { + assert!(result.flattened_result().is_ok(), "{result:?}"); +} + +/// Wraps an expression in nested self-CPI calls. +fn with_cpi_depth(mut expr: E, depth: usize) -> E { + for _ in 0..depth { + expr = expr.cpi(); + } + expr +} + +/// Picks a read-only operand different from `output`. +fn operand(accounts: &[Pubkey], output: Pubkey, index: usize) -> Pubkey { + let mut key = accounts[index % accounts.len()]; + if key == output { + key = accounts[(index + 1) % accounts.len()]; + } + key +} + +// Simulation and execution both accept the same v42 transaction, but only +// execution commits account state and records transaction status. +#[tokio::test(flavor = "current_thread")] +async fn execution_commits_and_simulation_does_not() { + let harness = Harness::new(false).await; + let output = store_v42(&harness, 0, AccountMode::Delegated); + let input = store_v42(&harness, 40, AccountMode::Delegated); + let ix = (E::acc(1) + E::lit(2)).compose(output, &[input]); + let (_, sim_tx) = signed_view(&harness, None, ix.clone()); + + assert_success(&harness.simulate(sim_tx).await); + assert_eq!( + load_v42_data(&harness, output), + Some(0), + "simulation leaves state untouched" + ); + + let (signature, tx) = signed_view(&harness, None, ix); + harness.execute(tx).await; + harness.barrier().await; + + assert_eq!(load_v42_data(&harness, output), Some(42)); + harness.status(signature).await.result.expect("successful execution"); + harness.close().await; +} + +/// Proves a finalized block hashes the canonical signature of an appended transaction. +#[tokio::test(flavor = "current_thread")] +async fn block_hash_includes_appended_transaction_signature() { + let harness = Harness::new(false).await; + let parent = harness.blockhash(); + let output = store_v42(&harness, 0, AccountMode::Delegated); + let (signature, tx) = signed_view(&harness, None, E::lit(42).compose(output, &[])); + let mut hasher = blake3::Hasher::new(); + hasher.update(parent.as_ref()); + hasher.update(signature.as_ref()); + let expected = Hash::from(*hasher.finalize().as_bytes()); + + harness.execute(tx).await; + harness.set_block(Block::new(1, 1234)).await; + harness.barrier().await; + + assert_eq!(harness.blockhash(), expected); + harness.status(signature).await.result.expect("successful execution"); + harness.close().await; +} + +// Program seeding installs the v42 ELF under loader-v4, and recursive CPI +// preserves return-data flow through a committed execution. +#[tokio::test(flavor = "current_thread")] +async fn seeded_program_and_recursive_cpi_return_data_work() { + let harness = Harness::new(false).await; + let account = harness.accounts().loader().load(&V42_ID).unwrap().expect("v42 program seeded"); + + assert!(account.executable()); + assert_eq!(*account.owner(), loader_v4::ID); + assert_eq!(account.data().get(..4), Some(&[0x7f, b'E', b'L', b'F'][..])); + + let output = store_v42(&harness, 0, AccountMode::Delegated); + let expr = E::lit(42) + (E::lit(31) * E::lit(4)).cpi() - E::lit(56); + let (_, tx) = signed_view(&harness, None, expr.compose(output, &[])); + + harness.execute(tx).await; + harness.barrier().await; + + assert_eq!(load_v42_data(&harness, output), Some(110)); + harness.close().await; +} + +// A block transition updates the Clock sysvar for both simulation and execution, +// while committed execution lands in the next slot. +#[tokio::test(flavor = "current_thread")] +async fn block_transition_updates_execution_and_simulation_sysvars() { + let harness = Harness::new(false).await; + let block = Block::new(7, 1234); + harness.set_block(block).await; + + let output = store_v42(&harness, 0, AccountMode::Delegated); + let ix = E::clock().compose(output, &[]); + let (_, sim_tx) = signed_view(&harness, None, ix.clone()); + assert_success(&harness.simulate(sim_tx).await); + + let (signature, tx) = signed_view(&harness, None, ix); + harness.execute(tx).await; + harness.barrier().await; + + assert_eq!( + load_v42_data(&harness, output), + Some(1234), + "both paths see the block's clock" + ); + assert_eq!( + harness.status(signature).await.slot, + 8, + "committed execution lands in the slot after the block" + ); + harness.close().await; +} + +// Replay mode still applies balance writes, but it must not publish transaction +// status because replayed entries were already recorded by the original run. +#[tokio::test(flavor = "current_thread")] +async fn replay_mode_commits_state_without_recording_status() { + let harness = Harness::new(true).await; + let source = store_v42(&harness, 0, AccountMode::Delegated); + let recipient = store_v42(&harness, 0, AccountMode::Delegated); + let source_before = load_v42_lamports(&harness, source).expect("source exists"); + let recipient_before = load_v42_lamports(&harness, recipient).expect("recipient exists"); + let (signature, tx) = signed_view(&harness, None, transfer(source, recipient, 1)); + + harness.execute(tx).await; + harness.barrier().await; + + assert_eq!( + load_v42_lamports(&harness, source).expect("source remains"), + source_before - 1, + "replay commits the source debit" + ); + assert_eq!( + load_v42_lamports(&harness, recipient).expect("recipient remains"), + recipient_before + 1, + "replay commits the recipient credit" + ); + assert!( + harness.transactions().status(signature).await.unwrap().is_none(), + "replay records no status" + ); + harness.close().await; +} + +// A transaction that runs but fails still commits a status receipt carrying the +// error, and leaves its output account untouched — the failure is recorded, not +// silently dropped like an unresolvable transaction. +#[tokio::test(flavor = "current_thread")] +async fn failed_execution_records_an_error_status() { + let harness = Harness::new(false).await; + let output = store_v42(&harness, 5, AccountMode::Delegated); + // MIN - 1 overflows the program's checked_sub, so it returns an error before + // ever writing the output account. + let (signature, tx) = signed_view( + &harness, + None, + (E::lit(i64::MIN) - E::lit(1)).compose(output, &[]), + ); + + harness.execute(tx).await; + harness.barrier().await; + + // The failed run is committed as a status with an error result, unlike the + // success cases the other tests assert. + assert!( + harness.status(signature).await.result.is_err(), + "recorded result reflects the failure" + ); + assert_eq!( + load_v42_data(&harness, output), + Some(5), + "failed execution commits no writes" + ); + harness.close().await; +} + +// A backlog of writes to one account must keep making progress even when every +// transaction initially contends for the same lock. +#[tokio::test(flavor = "current_thread")] +async fn prefilled_same_writable_account_backlog_drains() { + const TRANSACTIONS: usize = 128; + + let (harness, sequencer) = Harness::unspawned(false).await; + let output = store_v42(&harness, 0, AccountMode::Delegated); + let mut signatures = Vec::with_capacity(TRANSACTIONS); + + for i in 0..TRANSACTIONS { + let expr = with_cpi_depth(E::lit((i + 1) as i64), i % 5); + let (signature, tx) = harness.fresh_payer_view(expr.compose(output, &[])); + signatures.push(signature); + harness.execute(tx).await; + } + + sequencer.spawn().unwrap(); + harness.barrier().await; + + assert_eq!( + load_v42_data(&harness, output), + Some(TRANSACTIONS as i64), + "the last write of the drained backlog wins" + ); + for signature in signatures { + harness.status(signature).await.result.expect("successful execution"); + } + harness.close().await; +} + +// Mixed read-only and writable intersections should eventually resolve even +// when the sequencer starts with more conflicted work than it can keep unblocked +// at once. +#[tokio::test(flavor = "current_thread")] +async fn prefilled_mixed_read_write_contention_stress_drains() { + const ACCOUNTS: usize = 16; + const TRANSACTIONS: usize = 512; + + let (harness, sequencer) = Harness::unspawned(false).await; + let accounts: Vec<_> = (0..ACCOUNTS) + .map(|i| store_v42(&harness, i as i64 + 1, AccountMode::Delegated)) + .collect(); + let mut signatures = Vec::with_capacity(TRANSACTIONS); + + for i in 0..TRANSACTIONS { + let output = match i % 4 { + 0 => accounts[0], + 1 => accounts[i % ACCOUNTS], + 2 => accounts[(i + 3) % ACCOUNTS], + _ => accounts[(i * 7 + 5) % ACCOUNTS], + }; + let left = operand(&accounts, output, i + 1); + let right = operand(&accounts, output, i * 3 + 2); + let expr = with_cpi_depth(E::acc(1) + E::lit((i % 5) as i64), i % 5); + let (signature, tx) = harness.fresh_payer_view(expr.compose(output, &[left, right])); + signatures.push(signature); + harness.execute(tx).await; + } + + sequencer.spawn().unwrap(); + harness.barrier().await; + + for signature in signatures { + harness.status(signature).await.result.expect("successful execution"); + } + harness.close().await; +} diff --git a/programs/magic-root-interface/Cargo.toml b/programs/magic-root-interface/Cargo.toml new file mode 100644 index 00000000..6797c532 --- /dev/null +++ b/programs/magic-root-interface/Cargo.toml @@ -0,0 +1,18 @@ +[package] +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +name = "magic-root-interface" +repository.workspace = true +version.workspace = true + +[dependencies] +wincode = { workspace = true } + +solana-account = { workspace = true, features = ["wincode"] } +solana-instruction = { workspace = true, features = ["wincode"] } +solana-pubkey = { workspace = true } + +[lints] +workspace = true diff --git a/programs/magic-root-interface/README.md b/programs/magic-root-interface/README.md new file mode 100644 index 00000000..049f2e95 --- /dev/null +++ b/programs/magic-root-interface/README.md @@ -0,0 +1,20 @@ +# `magic-root-interface` + +This crate defines the MagicRoot program id and `MagicRootInstruction` wire +schema shared by `magic-root-program` and `engine`. Instruction builders can +depend on the interface without linking the native program runtime. + +`MagicRootInstruction::compose` prepends the writable target account and +serializes the instruction with wincode. `PostFinalize` also appends each +follow-up program id and its account metas. Signer bits are cleared in the outer +instruction; the native program supplies the declared follow-up signers when it +invokes each action. + +`MagicRootInstruction::compose_account` builds the ordered patch sequence for a +complete account's non-flag fields and appends a finalization instruction that +installs its complete flag value without changing lamports. The underlying +patch sequence applies mode before slot so the program can validate +lifecycle-aware slot progression. Callers are responsible for composing current +account state. Internal create composition appends `PostFinalize` immediately +after finalization; MagicRoot relies on this ordering after authenticating the +engine authority and caller. diff --git a/programs/magic-root-interface/src/lib.rs b/programs/magic-root-interface/src/lib.rs new file mode 100644 index 00000000..6d0d2811 --- /dev/null +++ b/programs/magic-root-interface/src/lib.rs @@ -0,0 +1,73 @@ +#![doc = include_str!("../README.md")] + +use solana_account::{AccountFieldPatch, OwnedAccount, StateFlags}; +use solana_instruction::{AccountMeta, Instruction, error::InstructionError}; +use solana_pubkey::{Pubkey, declare_id}; +use wincode::{SchemaRead, SchemaWrite}; + +declare_id!("MagicRootDRJ5atQjSJUxFjXzjeZXMADHUDznbk22gy"); + +/// Instructions accepted by the MagicRoot built-in program. +#[derive(SchemaRead, SchemaWrite)] +pub enum MagicRootInstruction { + /// Apply a single-field patch to the target account. + Patch(AccountFieldPatch), + /// Replace the target's complete flag value and, when executable, load it + /// into the transaction's program cache. Does not change lamports. + Finalize(StateFlags), + /// Close the target account and hide any cached executable immediately. + Delete, + /// Run follow-up instructions immediately after finalizing the same target + /// (e.g. initializing a freshly created account); each is invoked via CPI + /// against the accounts it declares. + PostFinalize(Vec), +} + +impl MagicRootInstruction { + /// Composes this variant into an [`Instruction`] targeting the MagicRoot + /// program: prepends the target `account` meta, appends any metas the + /// variant requires, and serializes the instruction data. + pub fn compose(&self, account: Pubkey) -> Result { + let mut accounts = vec![AccountMeta::new(account, false)]; + self.extend_metas(&mut accounts); + // NOTE this code can never error, wincode serialization for instruction is infallible + let data = wincode::serialize(self).map_err(|_| InstructionError::BorshIoError)?; + Ok(Instruction { program_id: ID, accounts, data }) + } + + /// Composes `account` into the ordered instruction sequence that patches + /// every non-flag field on `target`, then finalizes it with `account`'s + /// complete flag value. + pub fn compose_account( + target: Pubkey, + account: OwnedAccount, + ) -> Result, InstructionError> { + let flags = account.flags(); + let patches = AccountFieldPatch::sequence(account); + let mut instructions = Vec::with_capacity(patches.len() + 1); + for patch in patches { + instructions.push(Self::Patch(patch).compose(target)?); + } + instructions.push(Self::Finalize(flags).compose(target)?); + Ok(instructions) + } + + /// Appends the extra account metas a variant requires. Only [`PostFinalize`] + /// contributes any: the program id and de-signed accounts of each follow-up + /// instruction. + /// + /// [`PostFinalize`]: MagicRootInstruction::PostFinalize + fn extend_metas(&self, accounts: &mut Vec) { + let Self::PostFinalize(ixs) = self else { + return; + }; + for ix in ixs { + accounts.push(AccountMeta::new_readonly(ix.program_id, false)); + let metas = ix.accounts.clone().into_iter().map(|mut meta| { + meta.is_signer = false; + meta + }); + accounts.extend(metas) + } + } +} diff --git a/programs/magic-root-program/Cargo.toml b/programs/magic-root-program/Cargo.toml new file mode 100644 index 00000000..9df75161 --- /dev/null +++ b/programs/magic-root-program/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "magic-root-program" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +magic-root-interface = { workspace = true } + +nucleus = { workspace = true, features = ["tls"] } +wincode = { workspace = true } + +solana-account = { workspace = true } +solana-instruction = { workspace = true } +solana-instruction-error = { workspace = true } +solana-program-runtime = { workspace = true } +solana-svm-log-collector = { workspace = true } +solana-transaction-context = { workspace = true } + +[dev-dependencies] +solana-pubkey = { workspace = true } +solana-sdk-ids = { workspace = true } +solana-svm-callback = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-feature-set = { workspace = true, features = ["agave-unstable-api"] } + +[lints] +workspace = true diff --git a/programs/magic-root-program/README.md b/programs/magic-root-program/README.md new file mode 100644 index 00000000..ca2abe63 --- /dev/null +++ b/programs/magic-root-program/README.md @@ -0,0 +1,46 @@ +# `magic-root-program` + +MagicRoot is the engine's privileged native program for patching, finalizing, +and closing accounts. Its wire schema and program id are defined by +`magic-root-interface`. + +Every invocation must use the engine `AUTHORITY` as transaction payer/signer. +Direct transaction instructions are accepted. CPI is accepted only when the +immediate caller is registered in the transaction program cache as a builtin; +MagicRoot itself cannot be the caller. Native-loader account metadata alone does +not grant access. Caller authorization and decoding complete before target-state +authorization. The SVM's separate top-level-only privilege rule is unchanged. + +## Instructions + +- `Patch` applies one `AccountFieldPatch`. Lamport changes are balanced against + the authority account, including no-op balance patches. Slot patches must + advance the stored slot; an equal slot is accepted only when an earlier patch + in the same transaction genuinely changed the account mode. Older slots are + always rejected. Data patches may produce at most 10 MiB of account data; + larger lengths return `InvalidRealloc`. Account mode transitions use the + policy owned by `AccountSharedData`: read-only and placeholder accounts may + transition to any mode except transient, transient accounts may resolve only + to read-only, and delegated accounts may transition only to transient. + Ephemeral accounts may close. Other mode changes are rejected. +- `Finalize` atomically installs the caller-supplied complete flag value and + loads an executable target into the transaction program cache. It does not + change lamports; failed executable loading rolls back the installed flags. +- `Delete` transitions read-only, placeholder, or ephemeral targets to + `AccountMode::Closed`, immediately hides any transaction-local cached program, + and removes its shared cache entry after successful execution and access + validation. Accountsdb removes closed accounts during writeback. Modes that + cannot transition to closed are rejected. +- `PostFinalize` invokes follow-up instructions through native CPI and is + placed immediately after the target's `Finalize` by internal composers. It + rejects any immutable instruction account marked writable and any action that + targets MagicRoot itself. + +After authority and caller checks pass, MagicRoot does not determine whether a +complete account image is stale. Callers must supply current state; slot and +lifecycle validation still apply. + +Complete-account patch sequences apply mode before slot. `AccountSharedData` +marks mode dirty only when its value changes, so a no-op mode patch cannot make +an equal-slot replacement appear fresh. Rejection aborts the transaction, and +therefore rolls back every earlier field patch in that sequence. diff --git a/programs/magic-root-program/src/account.rs b/programs/magic-root-program/src/account.rs new file mode 100644 index 00000000..325bee70 --- /dev/null +++ b/programs/magic-root-program/src/account.rs @@ -0,0 +1,91 @@ +use solana_account::{ + AccountFieldPatch, AccountMode, ReadableAccount, StateFlags, WritableAccount, +}; +use solana_instruction_error::InstructionError; +use solana_program_runtime::{invoke_context::InvokeContext, loaded_programs::ProgramCacheEntry}; +use solana_svm_log_collector::ic_msg; +use solana_transaction_context::{IndexOfAccount, MAX_ACCOUNT_DATA_LEN}; + +/// Applies a single-field patch to the target account. +pub(crate) fn patch( + ctx: &InvokeContext<'_, '_>, + target: IndexOfAccount, + patch: AccountFieldPatch, +) -> Result<(), InstructionError> { + let mut account = ctx.transaction_context.accounts().try_borrow_mut(target)?; + ic_msg!(ctx, "MagicRoot: patch {:?}", patch); + let data_len = match &patch { + AccountFieldPatch::DataAt { offset, data } => { + let end = offset.checked_add(data.len()).ok_or(InstructionError::InvalidRealloc)?; + Some(account.data().len().max(end)) + } + AccountFieldPatch::DataLen(len) => Some(*len), + _ => None, + }; + if data_len.is_some_and(|len| len > MAX_ACCOUNT_DATA_LEN as usize) { + return Err(InstructionError::InvalidRealloc); + } + let old = account.lamports(); + if let Err(error) = patch.apply(&mut account) { + ic_msg!(ctx, "MagicRoot: {}", error); + return Err(InstructionError::InvalidArgument); + } + let new = account.lamports(); + let mut authority = ctx.transaction_context.accounts().try_borrow_mut(0)?; + // Balance target changes against the authority to preserve total lamports. + if new > old { + authority.checked_sub_lamports(new - old)?; + } else if new < old { + authority.checked_add_lamports(old - new)?; + } + Ok(()) +} + +/// Loads an executable target into the transaction's program cache. +pub(crate) fn finalize( + ctx: &mut InvokeContext<'_, '_>, + target: IndexOfAccount, + flags: StateFlags, +) -> Result<(), InstructionError> { + let mut account = ctx.transaction_context.accounts().try_borrow_mut(target)?; + account.set_flags(flags); + if !account.executable() { + return Ok(()); + } + let pubkey = ctx.transaction_context.get_key_of_account_at_index(target)?; + let entry = ProgramCacheEntry::new( + ctx.environment_config + .program_runtime_environments_for_execution + .get_env_for_execution() + .clone(), + account.data(), + ) + .map_err(|_| { + ic_msg!(ctx, "MagicRoot: program load failed {}", pubkey); + InstructionError::ProgramEnvironmentSetupFailure + })? + .into(); + ctx.program_cache_for_tx_batch.store_modified_entry(*pubkey, entry); + ic_msg!(ctx, "MagicRoot: finalized program load"); + Ok(()) +} + +/// Marks the target account closed for removal from storage. +pub(crate) fn delete( + ctx: &mut InvokeContext<'_, '_>, + target: IndexOfAccount, +) -> Result<(), InstructionError> { + let mut acc = ctx.transaction_context.accounts().try_borrow_mut(target)?; + + if let Err(error) = acc.set_mode(AccountMode::Closed) { + ic_msg!(ctx, "MagicRoot: {}", error); + return Err(InstructionError::InvalidArgument); + } + let pubkey = *ctx.transaction_context.get_key_of_account_at_index(target)?; + if acc.executable() { + let entry = ProgramCacheEntry::default().into(); + ctx.program_cache_for_tx_batch.store_modified_entry(pubkey, entry); + } + ic_msg!(ctx, "MagicRoot: removed"); + Ok(()) +} diff --git a/programs/magic-root-program/src/lib.rs b/programs/magic-root-program/src/lib.rs new file mode 100644 index 00000000..f057996d --- /dev/null +++ b/programs/magic-root-program/src/lib.rs @@ -0,0 +1,25 @@ +#![doc = include_str!("../README.md")] + +mod account; +mod post_finalize; +mod processor; +#[cfg(test)] +mod tests; + +use solana_instruction_error::InstructionError; +use solana_program_runtime::invoke_context::InvokeContext; + +/// Instruction-account index of the account a MagicRoot instruction operates on. +pub const TARGET_ACCOUNT_IDX: u16 = 0; + +/// Executes a MagicRoot instruction: authorizes the caller, decodes the +/// instruction, then dispatches it to the matching handler. +pub fn process(ctx: &mut InvokeContext<'_, '_>) -> Result<(), InstructionError> { + processor::process(ctx) +} + +#[allow(missing_docs)] +pub mod entrypoint { + use solana_program_runtime::declare_process_instruction; + declare_process_instruction!(MagicRootEntrypoint, 150, |ctx| { super::process(ctx) }); +} diff --git a/programs/magic-root-program/src/post_finalize.rs b/programs/magic-root-program/src/post_finalize.rs new file mode 100644 index 00000000..e4725171 --- /dev/null +++ b/programs/magic-root-program/src/post_finalize.rs @@ -0,0 +1,39 @@ +use solana_instruction::Instruction; +use solana_instruction_error::InstructionError; +use solana_program_runtime::invoke_context::InvokeContext; +use solana_svm_log_collector::ic_msg; + +/// Runs the post-finalize follow-up instructions, invoking each action via CPI +/// while vouching for exactly the signers that action itself declares, then +/// rejects the whole instruction if an account exposed as writable did not end +/// in a mode this engine may mutate. +pub(crate) fn process( + ctx: &mut InvokeContext<'_, '_>, + actions: Vec, +) -> Result<(), InstructionError> { + ic_msg!(ctx, "MagicRoot: post-finalize {} action(s)", actions.len()); + for action in actions { + let signers: Vec<_> = action + .accounts + .iter() + .filter_map(|meta| meta.is_signer.then_some(meta.pubkey)) + .collect(); + ctx.native_invoke(action, &signers)?; + } + + let instruction = ctx.transaction_context.get_current_instruction_context()?; + let count = instruction.get_number_of_instruction_accounts(); + for i in 0..count { + let index = instruction.get_index_of_instruction_account_in_transaction(i)?; + let account = ctx.transaction_context.accounts().try_borrow(index)?; + if instruction.is_instruction_account_writable(i)? && !account.mutable() { + ic_msg!( + ctx, + "MagicRoot: post-finalize rejected immutable writable account: {}", + ctx.transaction_context.get_key_of_account_at_index(index)? + ); + return Err(InstructionError::Immutable); + } + } + Ok(()) +} diff --git a/programs/magic-root-program/src/processor.rs b/programs/magic-root-program/src/processor.rs new file mode 100644 index 00000000..6fd15313 --- /dev/null +++ b/programs/magic-root-program/src/processor.rs @@ -0,0 +1,84 @@ +use magic_root_interface::MagicRootInstruction; +use nucleus::tls::AUTHORITY; +use solana_instruction::TRANSACTION_LEVEL_STACK_HEIGHT; +use solana_instruction_error::InstructionError; +use solana_program_runtime::{ + invoke_context::InvokeContext, loaded_programs::ProgramCacheEntryType, +}; +use solana_svm_log_collector::ic_msg; + +use crate::{TARGET_ACCOUNT_IDX, account, post_finalize}; + +/// Authorizes and decodes a MagicRoot instruction before validating and +/// dispatching its target operation. +pub(crate) fn process(ctx: &mut InvokeContext<'_, '_>) -> Result<(), InstructionError> { + authorize(ctx)?; + let instruction = decode(ctx)?; + dispatch(ctx, instruction) +} + +/// Rejects recursive or non-builtin CPI callers and callers other than [`AUTHORITY`]. +/// +/// MagicRoot is engine-internal: it must run either at the transaction level or +/// directly below a builtin program. The transaction must be signed by the +/// authority in either case. Both conditions are checked before any account is +/// touched. +/// +/// Authorizing on account 0's key alone is sound because the engine verifies the +/// fee-payer signature at its transaction ingress (`engine::transaction`), so a +/// transaction reaching execution with `AUTHORITY` as account 0 was signed by it. +pub(crate) fn authorize(ctx: &InvokeContext<'_, '_>) -> Result<(), InstructionError> { + let height = ctx.get_stack_height(); + if height != TRANSACTION_LEVEL_STACK_HEIGHT { + let instruction = ctx.transaction_context.get_current_instruction_context()?; + let caller = ctx + .transaction_context + .get_instruction_context_at_index_in_trace(instruction.get_index_of_caller())?; + let caller_id = caller.get_program_key()?; + if caller_id == &magic_root_interface::ID { + ic_msg!(ctx, "MagicRoot: recursive caller"); + return Err(InstructionError::CallDepth); + } + let is_builtin = ctx + .program_cache_for_tx_batch + .find(caller_id) + .filter(|e| matches!(e.program, ProgramCacheEntryType::Builtin(_))) + .is_some(); + if !is_builtin { + ic_msg!(ctx, "MagicRoot: non-builtin caller {}", caller_id); + return Err(InstructionError::CallDepth); + } + } + let signer = *ctx.transaction_context.get_key_of_account_at_index(0)?; + if signer != AUTHORITY.get() { + ic_msg!(ctx, "MagicRoot: unauthorized signer {}", signer); + return Err(InstructionError::MissingRequiredSignature); + } + Ok(()) +} + +/// Dispatches the decoded instruction to the matching handler. +fn dispatch( + ctx: &mut InvokeContext<'_, '_>, + instruction: MagicRootInstruction, +) -> Result<(), InstructionError> { + let target = ctx + .transaction_context + .get_current_instruction_context()? + .get_index_of_instruction_account_in_transaction(TARGET_ACCOUNT_IDX)?; + match instruction { + MagicRootInstruction::Patch(patch) => account::patch(ctx, target, patch), + MagicRootInstruction::Finalize(flags) => account::finalize(ctx, target, flags), + MagicRootInstruction::Delete => account::delete(ctx, target), + MagicRootInstruction::PostFinalize(actions) => post_finalize::process(ctx, actions), + } +} + +/// Decodes the current instruction's data into a [`MagicRootInstruction`]. +fn decode(ctx: &InvokeContext<'_, '_>) -> Result { + let instruction = ctx.transaction_context.get_current_instruction_context()?; + wincode::deserialize(instruction.get_instruction_data()).map_err(|_| { + ic_msg!(ctx, "MagicRoot: malformed instruction data"); + InstructionError::InvalidInstructionData + }) +} diff --git a/programs/magic-root-program/src/tests.rs b/programs/magic-root-program/src/tests.rs new file mode 100644 index 00000000..44598c59 --- /dev/null +++ b/programs/magic-root-program/src/tests.rs @@ -0,0 +1,115 @@ +use { + crate::{entrypoint::MagicRootEntrypoint, processor::authorize}, + nucleus::tls::AUTHORITY, + solana_account::AccountSharedData, + solana_instruction_error::InstructionError, + solana_program_runtime::{ + loaded_programs::{ProgramCacheEntry, ProgramCacheEntryType}, + solana_sbpf::program::BuiltinFunctionDefinition, + with_mock_invoke_context, + }, + solana_pubkey::Pubkey, + solana_sdk_ids::native_loader, + std::sync::Arc, +}; + +#[derive(Clone, Copy)] +enum Caller { + Builtin, + User, +} + +fn authorize_chain( + authority: Pubkey, + signer: Pubkey, + callers: &[Caller], +) -> Result<(), InstructionError> { + AUTHORITY.set(authority); + + let caller_ids = callers.iter().map(|_| Pubkey::new_unique()).collect::>(); + let mut accounts = Vec::with_capacity(caller_ids.len().saturating_add(2)); + accounts.push((signer, AccountSharedData::default())); + accounts.extend( + caller_ids + .iter() + .map(|id| (*id, AccountSharedData::new(1, 0, &native_loader::ID))), + ); + accounts.push(( + magic_root_interface::ID, + AccountSharedData::new(1, 0, &native_loader::ID), + )); + + with_mock_invoke_context!(ctx, transaction_context, accounts); + let mut cache = ProgramCacheForTxBatch::default(); + let environments = ProgramRuntimeEnvironments::default(); + for (&id, caller) in caller_ids.iter().zip(callers) { + let entry = match caller { + Caller::Builtin => ProgramCacheEntry::new_builtin(( + MagicRootEntrypoint::vm, + MagicRootEntrypoint::codegen, + )), + Caller::User => ProgramCacheEntry { + program: ProgramCacheEntryType::Unloaded( + environments.get_env_for_execution().clone(), + ), + }, + }; + cache.replenish(id, Arc::new(entry)); + } + ctx.program_cache_for_tx_batch = &mut cache; + + ctx.transaction_context + .configure_top_level_instruction_for_tests(1, Vec::new(), Vec::new())?; + ctx.push()?; + for program_index in 2..=callers.len().saturating_add(1) { + ctx.transaction_context.configure_next_cpi_for_tests( + program_index as u16, + Vec::new(), + Vec::new(), + )?; + ctx.push()?; + } + + authorize(&ctx) +} + +#[test] +fn authorizes_top_level_and_builtin_cpi() { + let authority = Pubkey::new_unique(); + assert_eq!(authorize_chain(authority, authority, &[]), Ok(())); + assert_eq!( + authorize_chain(authority, authority, &[Caller::Builtin]), + Ok(()) + ); + assert_eq!( + authorize_chain(authority, authority, &[Caller::Builtin, Caller::Builtin],), + Ok(()) + ); +} + +#[test] +fn rejects_a_non_builtin_caller() { + let authority = Pubkey::new_unique(); + assert_eq!( + authorize_chain(authority, authority, &[Caller::User]), + Err(InstructionError::CallDepth) + ); +} + +#[test] +fn authorizes_an_immediate_builtin_caller() { + let authority = Pubkey::new_unique(); + assert_eq!( + authorize_chain(authority, authority, &[Caller::User, Caller::Builtin]), + Ok(()) + ); +} + +#[test] +fn rejects_the_wrong_authority() { + let authority = Pubkey::new_unique(); + assert_eq!( + authorize_chain(authority, Pubkey::new_unique(), &[Caller::Builtin]), + Err(InstructionError::MissingRequiredSignature) + ); +} diff --git a/programs/v42-calculator-interface/Cargo.toml b/programs/v42-calculator-interface/Cargo.toml new file mode 100644 index 00000000..c0ffd3c5 --- /dev/null +++ b/programs/v42-calculator-interface/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "v42-calculator-interface" + +authors.workspace = true +# Compiled to BPF as the program's opcode dependency, so it stays within the +# SBF toolchain's rustc: edition 2021 and no elevated workspace MSRV. +edition = "2021" +homepage.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true + +[dependencies] +solana-instruction = { workspace = true, optional = true } +solana-pubkey = { workspace = true } + +[features] +builder = ["dep:solana-instruction"] +default = ["builder"] + +[lints] +workspace = true diff --git a/programs/v42-calculator-interface/README.md b/programs/v42-calculator-interface/README.md new file mode 100644 index 00000000..3359c7b3 --- /dev/null +++ b/programs/v42-calculator-interface/README.md @@ -0,0 +1,20 @@ +# `v42-calculator-interface` + +This crate defines the v42 calculator program id, instruction wire constants, +and optional off-chain builders. The SBF program depends on the wire definitions +with the default `builder` feature disabled. + +`Expr` produces postfix instruction data from signed `i64` literals, account +operands, the `Clock` sysvar, arithmetic operators, and recursive self-CPI +subexpressions. Expression composition concatenates existing postfix byte +streams. + +`Expr::compose` builds an instruction with the writable output at account zero, +read-only operands after it, and the calculator program id last for recursive +CPI. `Expr::acc` indexes the full instruction account list, so operand indexes +start at one and remain stable across nested calls. + +`builder::transfer` applies a signed delta between distinct writable v42 accounts +at indexes 0 and 1. Its data is `TRANSFER` followed by a little-endian `i64`; +positive values move lamports and calculator value from account 0 to account 1, +while negative values reverse the direction. diff --git a/programs/v42-calculator-interface/src/builder.rs b/programs/v42-calculator-interface/src/builder.rs new file mode 100644 index 00000000..5ffb0127 --- /dev/null +++ b/programs/v42-calculator-interface/src/builder.rs @@ -0,0 +1,117 @@ +//! Off-chain expression builder. + +use core::ops::{Add, Div, Mul, Sub}; + +use solana_instruction::{AccountMeta, Instruction}; +use solana_pubkey::Pubkey; + +use crate::{opcodes::*, ID, TRANSFER}; + +/// Build a transfer between distinct v42-owned accounts. A positive `delta` +/// moves value from `from` to `to`; a negative delta reverses the direction. +/// Neither account is required to sign, but both are writable. +pub fn transfer(from: Pubkey, to: Pubkey, delta: i64) -> Instruction { + let mut data = Vec::with_capacity(9); + data.push(TRANSFER); + data.extend_from_slice(&delta.to_le_bytes()); + Instruction { + program_id: ID, + accounts: vec![AccountMeta::new(from, false), AccountMeta::new(to, false)], + data, + } +} + +/// A calculator expression, built the way you'd write the math and lowered to +/// the postfix byte stream the program evaluates. Construct leaves with the +/// associated functions, combine them with `+`, `-`, `*`, `/`, and wrap any +/// subtree in [`Expr::cpi`] to force it through a recursive self-CPI (its result +/// then travels back through return data instead of being computed inline). +/// +/// ``` +/// use solana_pubkey::Pubkey; +/// use v42_calculator_interface::builder::Expr as E; +/// +/// // (42 + (31 * 4) - 56) * 2, with the product evaluated via CPI. +/// let program = (E::lit(42) + (E::lit(31) * E::lit(4)).cpi() - E::lit(56)) * E::lit(2); +/// let ix = program.compose(Pubkey::default(), &[]); +/// assert_eq!(ix.program_id, v42_calculator_interface::ID); +/// ``` +/// +/// The wrapped bytes are already in postfix order, so combining expressions is +/// just concatenation — there is no intermediate tree to walk. +#[derive(Clone, Debug)] +pub struct Expr(Vec); + +impl Expr { + /// An immediate signed literal operand. + pub fn lit(value: i64) -> Self { + let mut bytes = Vec::with_capacity(9); + bytes.push(PUSH_LIT); + bytes.extend_from_slice(&value.to_le_bytes()); + Self(bytes) + } + + /// The `i64` LE stored in instruction account `index`. Account 0 is the + /// output account; the operands passed to [`Expr::compose`] start at 1. + pub fn acc(index: u8) -> Self { + Self(vec![PUSH_ACC, index]) + } + + /// The current clock unix timestamp. + pub fn clock() -> Self { + Self(vec![PUSH_CLOCK]) + } + + /// Evaluate this subexpression through a recursive self-CPI rather than + /// inline. All instruction accounts are forwarded to the callee, so + /// [`Expr::acc`] indices are unchanged at any depth. + pub fn cpi(self) -> Self { + let mut bytes = Vec::with_capacity(3 + self.0.len()); + bytes.push(CALL); + bytes.extend_from_slice(&(self.0.len() as u16).to_le_bytes()); + bytes.extend_from_slice(&self.0); + Self(bytes) + } + + /// The raw postfix byte stream — i.e. the instruction data. + pub fn program(&self) -> Vec { + self.0.clone() + } + + /// Build the instruction: account 0 is the writable `output` the final + /// result is written to, followed by the read-only `operands` referenced by + /// [`Expr::acc`], followed by this program id for recursive CPI. + pub fn compose(&self, output: Pubkey, operands: &[Pubkey]) -> Instruction { + let mut accounts = Vec::with_capacity(2 + operands.len()); + accounts.push(AccountMeta::new(output, false)); + accounts.extend(operands.iter().map(|key| AccountMeta::new_readonly(*key, false))); + accounts.push(AccountMeta::new_readonly(ID, false)); + Instruction { + program_id: ID, + accounts, + data: self.0.clone(), + } + } + + fn binary(mut self, rhs: Expr, op: u8) -> Self { + self.0.extend_from_slice(&rhs.0); + self.0.push(op); + self + } +} + +macro_rules! bin_op { + ($trait:ident, $method:ident, $op:expr) => { + impl $trait for Expr { + type Output = Expr; + fn $method(self, rhs: Expr) -> Expr { + self.binary(rhs, $op) + } + } + }; +} + +bin_op!(Add, add, ADD); +bin_op!(Sub, sub, SUB); +bin_op!(Mul, mul, MUL); +bin_op!(Div, div, DIV); diff --git a/programs/v42-calculator-interface/src/lib.rs b/programs/v42-calculator-interface/src/lib.rs new file mode 100644 index 00000000..216fef44 --- /dev/null +++ b/programs/v42-calculator-interface/src/lib.rs @@ -0,0 +1,14 @@ +#![doc = include_str!("../README.md")] + +use solana_pubkey::declare_id; + +pub mod opcodes; + +#[cfg(feature = "builder")] +pub mod builder; + +declare_id!("V42CaLcu1atormagicb1ock11111111111111111111"); + +/// Transfer lamports and calculator value between instruction accounts 0 and +/// 1; an exact little-endian `i64` delta follows. +pub const TRANSFER: u8 = 0x30; diff --git a/programs/v42-calculator-interface/src/opcodes.rs b/programs/v42-calculator-interface/src/opcodes.rs new file mode 100644 index 00000000..219fc1f4 --- /dev/null +++ b/programs/v42-calculator-interface/src/opcodes.rs @@ -0,0 +1,30 @@ +//! Opcodes for the v42-calculator RPN byte stream — the single source of truth +//! for the wire format, shared by the off-chain `Expr` builder and the on-chain +//! evaluator so the two can never drift. Adding an operation is one constant +//! here plus one match arm in the program. +//! +//! A program is a flat sequence of tokens evaluated left-to-right against a +//! `i64` stack: `PUSH_*` tokens push one value, the arithmetic tokens pop two +//! and push one, and `CALL` evaluates a nested program through a self-CPI and +//! pushes its result. Exactly one value must remain when the stream ends. + +/// Push an immediate `i64`; 8 little-endian bytes follow. +pub const PUSH_LIT: u8 = 0x00; +/// Push the `i64` LE held in the first 8 data bytes of an instruction account; +/// a 1-byte account index follows. +pub const PUSH_ACC: u8 = 0x01; +/// Push the current clock unix timestamp. +pub const PUSH_CLOCK: u8 = 0x03; + +/// Pop `b`, pop `a`, push `a + b` (checked). +pub const ADD: u8 = 0x10; +/// Pop `b`, pop `a`, push `a - b` (checked). +pub const SUB: u8 = 0x11; +/// Pop `b`, pop `a`, push `a * b` (checked). +pub const MUL: u8 = 0x12; +/// Pop `b`, pop `a`, push checked `a / b` (`b == 0` is an error). +pub const DIV: u8 = 0x13; + +/// Evaluate a nested program via self-CPI and push its return-data `i64`. A +/// `u16` LE byte length follows, then that many bytes of nested program. +pub const CALL: u8 = 0x20; diff --git a/programs/v42-calculator-program/Cargo.toml b/programs/v42-calculator-program/Cargo.toml new file mode 100644 index 00000000..3f979414 --- /dev/null +++ b/programs/v42-calculator-program/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "v42-calculator-program" + +authors.workspace = true +edition = "2021" +homepage.workspace = true +license.workspace = true +repository.workspace = true +version.workspace = true + +[dependencies] +v42-calculator-interface = { workspace = true } + +solana-account-info = { workspace = true } +solana-cpi = { workspace = true } +solana-instruction = { workspace = true, features = ["syscalls"] } +solana-msg = { workspace = true } +solana-program-entrypoint = { workspace = true } +solana-program-error = { workspace = true } +solana-pubkey = { workspace = true } +solana-sysvar = { workspace = true, features = ["bytemuck"] } + +# Compiled to BPF with `cargo build-sbf`; the `rlib` lets it also build on the +# host so the workspace's `cargo build`/`clippy` stay green. +[lib] +crate-type = ["cdylib", "rlib"] + +[lints.rust] +# The `entrypoint!` macro emits `target_os = "solana"` and `custom-heap`/ +# `custom-panic` feature gates that only exist under the SBF toolchain; silence +# the noise they produce on host builds. +unexpected_cfgs = { level = "allow" } diff --git a/programs/v42-calculator-program/README.md b/programs/v42-calculator-program/README.md new file mode 100644 index 00000000..2b3ff24c --- /dev/null +++ b/programs/v42-calculator-program/README.md @@ -0,0 +1,36 @@ +# `v42-calculator-program` + +This SBF test program evaluates the postfix wire format from +`v42-calculator-interface`. It exercises account-data reads, the `Clock` sysvar, +checked arithmetic, recursive CPI, return data, writable account output, and +direct lamport transfers. + +The crate uses Rust 2021 and upstream minimal program crates so it remains +compatible with `cargo build-sbf`. Keeper's build script produces +`target/deploy/v42_calculator_program.so` for its testkit. + +## Execution model + +The crate entrypoint dispatches to the calculator and transfer domains. The +calculator owns expression evaluation and value encoding, transfer owns signed +balance and value movement, and error owns the stable calculator error codes. + +The evaluator processes literals, account `i64` values, clock timestamps, +arithmetic opcodes, and length-prefixed nested calls on a fixed-capacity stack. +Malformed input, stack errors, arithmetic errors, missing accounts, and short +buffers return stable `ProgramError::Custom` codes from `CalcError`. + +A top-level invocation writes the final little-endian `i64` to account zero. A +nested invocation publishes the value through return data after all child calls +finish, because a subsequent CPI would clear earlier return data. + +The program emits `v42:` trace messages for entry, CPI, clock access, results, +and failures. Result messages distinguish `account0` from `return_data` routing. + +A `TRANSFER` instruction instead reads an exact little-endian `i64` delta from +its data and applies it to two distinct writable accounts owned by this program. +Account 0 receives the negated delta and account 1 receives the delta, updating +both lamports and the little-endian `i64` calculator value in their first eight +data bytes. Both updates use wrapping two's-complement arithmetic, so a negative +delta reverses the transfer direction. Neither account needs to sign. Transfer +does not run the evaluator or publish return data. diff --git a/programs/v42-calculator-program/src/calculator.rs b/programs/v42-calculator-program/src/calculator.rs new file mode 100644 index 00000000..972dfc25 --- /dev/null +++ b/programs/v42-calculator-program/src/calculator.rs @@ -0,0 +1,197 @@ +//! Postfix calculator evaluation and result routing. + +use solana_account_info::AccountInfo; +use solana_cpi::{get_return_data, invoke, set_return_data}; +use solana_instruction::{AccountMeta, Instruction, TRANSACTION_LEVEL_STACK_HEIGHT}; +use solana_msg::msg; +use solana_program_error::{ProgramError, ProgramResult}; +use solana_sysvar::clock::Clock; +use solana_sysvar::Sysvar; +use v42_calculator_interface::{opcodes::*, ID}; + +use crate::error::CalcError; + +/// Evaluates the RPN program in `data` and routes the result: a **top-level** +/// invocation writes it (LE `i64`) into the output account; a **CPI** +/// invocation returns it through return data. The two are told apart by the +/// stack height. +/// +/// A nested call must publish its result *after* its own child `CALL`s complete, +/// because every CPI clears the return-data register — which is exactly what +/// happens here, since the result is emitted only once evaluation is done. +pub(crate) fn process(accounts: &[AccountInfo], data: &[u8], height: usize) -> ProgramResult { + let result = eval(accounts, data)?; + if height == TRANSACTION_LEVEL_STACK_HEIGHT { + msg!("v42: result={} -> account0", result); + let output = accounts.first().ok_or(CalcError::MissingOutput)?; + write_account_value(&mut output.try_borrow_mut_data()?, result) + } else { + msg!("v42: result={} -> return_data", result); + set_return_data(&result.to_le_bytes()); + Ok(()) + } +} + +/// Runs the token stream against an operand stack and returns the single value +/// left at the end. +fn eval(accounts: &[AccountInfo], data: &[u8]) -> Result { + let mut cursor = Cursor(data); + let mut stack = Stack::new(); + while !cursor.is_empty() { + match cursor.u8()? { + PUSH_LIT => stack.push(cursor.i64()?)?, + PUSH_ACC => stack.push(account_i64(accounts, cursor.u8()?)?)?, + PUSH_CLOCK => stack.push(clock_ts()?)?, + op @ (ADD | SUB | MUL | DIV) => { + let b = stack.pop()?; + let a = stack.pop()?; + stack.push(arithmetic(op, a, b)?)?; + } + CALL => { + let len = cursor.u16()? as usize; + let nested = cursor.take(len)?; + stack.push(call(accounts, nested)?)?; + } + _ => return Err(CalcError::BadOpcode.into()), + } + } + stack.into_result().map_err(Into::into) +} + +fn arithmetic(op: u8, a: i64, b: i64) -> Result { + match op { + ADD => a.checked_add(b).ok_or(CalcError::Arithmetic), + SUB => a.checked_sub(b).ok_or(CalcError::Arithmetic), + MUL => a.checked_mul(b).ok_or(CalcError::Arithmetic), + DIV if b == 0 => Err(CalcError::DivByZero), + DIV => a.checked_div(b).ok_or(CalcError::Arithmetic), + _ => Err(CalcError::BadOpcode), + } +} + +/// Reads the calculator value from the first eight account data bytes. +pub(crate) fn read_account_value(bytes: &[u8]) -> Result { + read_head_i64(bytes, CalcError::ShortAccount).map_err(Into::into) +} + +/// Writes the calculator value to the first eight account data bytes. +pub(crate) fn write_account_value(bytes: &mut [u8], value: i64) -> ProgramResult { + bytes + .get_mut(..8) + .ok_or(CalcError::ShortAccount)? + .copy_from_slice(&value.to_le_bytes()); + Ok(()) +} + +/// Decodes the `i64` LE stored in the first 8 bytes of `bytes`, reporting `short` +/// if there aren't that many. +fn read_head_i64(bytes: &[u8], short: CalcError) -> Result { + let head = bytes.get(..8).ok_or(short)?; + Ok(i64::from_le_bytes(head.try_into().expect("length checked"))) +} + +/// Reads the `i64` LE operand from the first 8 data bytes of an instruction +/// account (borrowed, not copied). +fn account_i64(accounts: &[AccountInfo], index: u8) -> Result { + let account = accounts.get(index as usize).ok_or(CalcError::BadAccountIndex)?; + read_account_value(&account.try_borrow_data()?) +} + +fn clock_ts() -> Result { + let ts = Clock::get()?.unix_timestamp; + msg!("v42: clock.ts={}", ts); + Ok(ts) +} + +/// Evaluates `nested` by invoking this same program, forwarding every account +/// unchanged so `PUSH_ACC` indices are identical at every recursion depth, and +/// returns the callee's return-data `i64`. +fn call(accounts: &[AccountInfo], nested: &[u8]) -> Result { + let metas = accounts + .iter() + .map(|a| AccountMeta { + pubkey: *a.key, + is_signer: a.is_signer, + is_writable: a.is_writable, + }) + .collect(); + let instruction = Instruction { + program_id: ID, + accounts: metas, + data: nested.to_vec(), + }; + msg!("v42: cpi len={}", nested.len()); + invoke(&instruction, accounts)?; + + let (_, bytes) = get_return_data().ok_or(CalcError::MissingReturnData)?; + Ok(read_head_i64(&bytes, CalcError::ShortReturnData)?) +} + +/// A forward cursor over the instruction byte stream. Every read is bounds- +/// checked and advances the cursor; operands are read in place, never copied. +struct Cursor<'a>(&'a [u8]); + +impl<'a> Cursor<'a> { + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn u8(&mut self) -> Result { + let (&byte, rest) = self.0.split_first().ok_or(CalcError::Truncated)?; + self.0 = rest; + Ok(byte) + } + + fn u16(&mut self) -> Result { + Ok(u16::from_le_bytes(self.array()?)) + } + + fn i64(&mut self) -> Result { + Ok(i64::from_le_bytes(self.array()?)) + } + + fn array(&mut self) -> Result<[u8; N], CalcError> { + Ok(self.take(N)?.try_into().expect("length checked")) + } + + fn take(&mut self, len: usize) -> Result<&'a [u8], CalcError> { + if self.0.len() < len { + return Err(CalcError::Truncated); + } + let (head, rest) = self.0.split_at(len); + self.0 = rest; + Ok(head) + } +} + +/// Fixed-capacity operand stack — deep enough for any expression a test would +/// build, and allocation-free. +struct Stack { + slots: [i64; 64], + len: usize, +} + +impl Stack { + fn new() -> Self { + Self { slots: [0; 64], len: 0 } + } + + fn push(&mut self, value: i64) -> Result<(), CalcError> { + *self.slots.get_mut(self.len).ok_or(CalcError::StackOverflow)? = value; + self.len += 1; + Ok(()) + } + + fn pop(&mut self) -> Result { + self.len = self.len.checked_sub(1).ok_or(CalcError::StackUnderflow)?; + Ok(self.slots[self.len]) + } + + /// The single value a well-formed program leaves behind. + fn into_result(self) -> Result { + match self.len { + 1 => Ok(self.slots[0]), + _ => Err(CalcError::UnbalancedProgram), + } + } +} diff --git a/programs/v42-calculator-program/src/error.rs b/programs/v42-calculator-program/src/error.rs new file mode 100644 index 00000000..7faf6e53 --- /dev/null +++ b/programs/v42-calculator-program/src/error.rs @@ -0,0 +1,29 @@ +//! Stable calculator errors exposed through `ProgramError::Custom`. + +use solana_msg::msg; +use solana_program_error::ProgramError; + +/// Failure modes with stable discriminants for callers and tests. +#[derive(Debug)] +#[repr(u32)] +pub(crate) enum CalcError { + Truncated = 1, + BadOpcode, + StackOverflow, + StackUnderflow, + UnbalancedProgram, + Arithmetic, + DivByZero, + BadAccountIndex, + ShortAccount, + MissingOutput, + MissingReturnData = 12, + ShortReturnData, +} + +impl From for ProgramError { + fn from(error: CalcError) -> Self { + msg!("v42: err {:?}", error); + ProgramError::Custom(error as u32) + } +} diff --git a/programs/v42-calculator-program/src/lib.rs b/programs/v42-calculator-program/src/lib.rs new file mode 100644 index 00000000..38b6501d --- /dev/null +++ b/programs/v42-calculator-program/src/lib.rs @@ -0,0 +1,25 @@ +#![doc = include_str!("../README.md")] + +mod calculator; +mod error; +mod transfer; + +use solana_account_info::AccountInfo; +use solana_instruction::syscalls::get_stack_height; +use solana_msg::msg; +use solana_program_entrypoint::entrypoint; +use solana_program_error::ProgramResult; +use solana_pubkey::Pubkey; +use v42_calculator_interface::TRANSFER; + +entrypoint!(process_instruction); + +fn process_instruction(_: &Pubkey, accounts: &[AccountInfo], data: &[u8]) -> ProgramResult { + let height = get_stack_height(); + msg!("v42: enter height={} len={}", height, data.len()); + if data.first() == Some(&TRANSFER) { + transfer::process(accounts, data) + } else { + calculator::process(accounts, data, height) + } +} diff --git a/programs/v42-calculator-program/src/transfer.rs b/programs/v42-calculator-program/src/transfer.rs new file mode 100644 index 00000000..7bd18a03 --- /dev/null +++ b/programs/v42-calculator-program/src/transfer.rs @@ -0,0 +1,41 @@ +//! Signed value transfers between calculator accounts. + +use solana_account_info::AccountInfo; +use solana_msg::msg; +use solana_program_error::{ProgramError, ProgramResult}; + +use crate::calculator::{read_account_value, write_account_value}; + +/// Applies the encoded signed delta to the lamports and calculator values of +/// two distinct accounts. Both accounts must be owned by this program and +/// writable. +pub(crate) fn process(accounts: &[AccountInfo], data: &[u8]) -> ProgramResult { + let [from, to, ..] = accounts else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + if from.key == to.key { + return Err(ProgramError::InvalidArgument); + } + let delta = data + .get(1..) + .map(TryInto::<[u8; 8]>::try_into) + .ok_or(ProgramError::InvalidInstructionData)? + .map(i64::from_le_bytes) + .map_err(|_| ProgramError::InvalidInstructionData)?; + + msg!("v42: transfer delta={}", delta); + apply_delta(from, delta.wrapping_neg())?; + apply_delta(to, delta) +} + +/// Adds the same two's-complement delta to an account's balance and stored +/// calculator value. +fn apply_delta(account: &AccountInfo, delta: i64) -> ProgramResult { + let mut lamports = account.try_borrow_mut_lamports()?; + **lamports = (**lamports).wrapping_add(delta as u64); + drop(lamports); + + let mut data = account.try_borrow_mut_data()?; + let result = read_account_value(&data)?.wrapping_add(delta); + write_account_value(&mut data, result) +} diff --git a/replicator/Cargo.toml b/replicator/Cargo.toml new file mode 100644 index 00000000..fbd20941 --- /dev/null +++ b/replicator/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "magicblock-replicator" + +authors.workspace = true +edition.workspace = true +homepage.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +name = "replicator" + +[dependencies] +engine = { workspace = true } +keeper = { workspace = true } +ledger = { workspace = true } +nucleus = { workspace = true, features = ["ledger", "service"] } + +derive_more = { workspace = true, features = ["from"] } +scc = { workspace = true } +snedfile = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["io-util", "macros", "net", "sync", "time"] } +tracing = { workspace = true } +wincode = { workspace = true } + +solana-keypair = { workspace = true } +solana-pubkey = { workspace = true } +solana-signature = { workspace = true } + +[dev-dependencies] +engine = { workspace = true, features = ["testkit"] } +keeper = { workspace = true, features = ["testkit"] } +nucleus = { workspace = true, features = ["testkit"] } +tokio = { workspace = true, features = ["rt-multi-thread", "time"] } +v42-calculator-interface = { workspace = true, features = ["builder"] } + +solana-account = { workspace = true } +solana-sysvar = { workspace = true } + +[lints] +workspace = true diff --git a/replicator/README.md b/replicator/README.md new file mode 100644 index 00000000..9f6fae63 --- /dev/null +++ b/replicator/README.md @@ -0,0 +1,57 @@ +# `magicblock-replicator` + +Replicator transfers durable execution state over TCP between independent +engine deployments, typically running on different machines. The follower +reports its durable `BlockstorePosition`; the server either resumes the retained +blockstore stream or sends the newest available accountsdb snapshot. + +## Protocol + +Protocol version 1 uses wincode control messages prefixed by a little-endian +`u32` length. Control frames are limited to 65,535 bytes before allocation. +Snapshot archives and blockstore bytes follow the selected response without +additional framing. + +Handshake requests and responses are signed with the sender's local key and +must be within 30 seconds of the receiver's clock. A server accepts only local +follower identities in its allowlist; an empty allowlist denies all followers. +A follower identity may hold only one active transfer at a time; its reservation +is released when that connection's worker exits. Stream workers detect peer +disconnects through writes triggered by durable cursor updates, which are +published at least every time block is produced while the engine is running. +A follower verifies responses against `Engine::authority()`, which must be +configured with the source authority through `nucleus::config::Authority::remote`. + +Every dispatcher must sign with that same canonical authority key. A follower +whose local signer differs from `Engine::authority()` is therefore a terminal +leaf and dispatcher startup rejects it before binding a listener. Any number of +such leaves may follow the source or a relay. Every relay instead holds the +shared private key, so the source and all relays have one compromise and key +rotation boundary. + +The async dispatcher accepts sockets and assigns each connection to a blocking +thread. File and socket operations on that thread use bounded blocking I/O. +Published ledger cursors are transfer boundaries, including sealed tails and +intermediate superblocks. + +## Follower recovery + +Before each handshake, the follower quiesces execution, flushes queued ledger +appends, and reports the resulting cursor. A received snapshot is written to the +successor superblock directory and its seal is appended synchronously. The +seal's cumulative transaction count replaces the follower ledger baseline, +including when a nonempty follower falls behind retention. The client then +reports `RestartRequired`; keeper restores the staged snapshot on the next +startup and engine replay advances it to the ledger tip. + +Externally paced shutdown flushes the cursor before writing +`CURRENT/volatile.db`. Internally paced origins instead append one reset marker +at startup before producing their first new block, so followers clear +chain-mirrored volatile state at the same stream position while retaining +internal system accounts. + +A shared-key follower may also serve downstream followers. It derives and +validates superblock seals from replicated block boundaries and archives its own +snapshots, while downstream clients continue to verify every response against +the original source authority. A distinct-key follower can consume the same +state but cannot relay it. diff --git a/replicator/src/client.rs b/replicator/src/client.rs new file mode 100644 index 00000000..687e26a4 --- /dev/null +++ b/replicator/src/client.rs @@ -0,0 +1,224 @@ +use std::{ + fs::{self, File}, + io::{self, BufReader, Read}, + net::{SocketAddr, TcpStream}, + thread, +}; + +use derive_more::Deref; +use engine::{Engine, EngineError, ReplayError, pacemaker::ExternalBlock}; +use ledger::{ + Superblock, + schema::{Block, OwnedBlockstoreEntry, blockstore}, +}; +use nucleus::{ + KB, + ledger::{ACCOUNTSDB_SNAPSHOT_FILE, BlockstorePosition}, + shutdown::{Service, ShutdownHandle, ShutdownManager, ShutdownReason}, +}; +use tokio::{ + runtime, + sync::mpsc::{Receiver, Sender}, + time, +}; +use tracing::{error, info, warn}; + +use crate::{ + IO_TIMEOUT, MAX_RECONNECT_ATTEMPTS, RETRY_DELAY, ReplicationError, Result, + metrics::{self, Operation}, + protocol::{ + self, Handshake, HandshakeRequest, HandshakeResponse, PROTO_VERSION, SnapshotMetadata, + }, +}; + +type ReplicationStream = BufReader; + +/// Pulls a leader blockstore stream into an externally paced follower engine. +#[derive(Deref)] +pub struct ReplicationClient { + /// Engine receiving replicated transactions, boundaries, seals, and resets. + #[deref] + engine: Engine, + /// Leader endpoint reused after transport loss. + addr: SocketAddr, + /// External pacemaker channel used to preserve block-boundary ordering. + pacer: Sender, + /// Locally committed block boundaries used to verify replicated output. + blocks: Receiver, +} + +impl ReplicationClient { + /// Starts the follower worker; connection failures are reported through shutdown management. + pub fn spawn( + addr: SocketAddr, + engine: Engine, + pacer: Sender, + shutdown: &mut ShutdownManager, + ) -> Result<()> { + metrics::init(); + let shutdown = shutdown.handle(Service::ReplicationClient); + let mut blocks = engine.blocks().subscribe(); + // drain the channel from potential leftovers + while blocks.try_recv().is_ok() {} + let client = Self { engine, addr, pacer, blocks }; + let rt = runtime::Builder::new_current_thread().enable_time().build()?; + thread::Builder::new() + .name("replication-client".into()) + .spawn(move || rt.block_on(client.run(shutdown)))?; + Ok(()) + } + + /// Consumes the leader stream and reports why the client stopped. + async fn run(self, mut shutdown: ShutdownHandle) { + let result = self.consume(&shutdown).await; + if shutdown.requested() || result.is_ok() { + shutdown.terminate(ShutdownReason::Signalled); + return; + } + match result { + Err(ReplicationError::RestartRequired(slot)) => { + info!(%slot, "replication client has requested node restart"); + shutdown.terminate(ShutdownReason::RestartRequired); + } + Err(error) => { + shutdown.terminate(ShutdownReason::Error(Box::new(error))); + } + Ok(()) => (), + } + } + + /// Reads blockstore entries from the leader, reconnecting on transport loss, + /// until shutdown is requested or a non-recoverable error occurs. + async fn consume(mut self, shutdown: &ShutdownHandle) -> Result<()> { + let mut stream = self.reconnect(shutdown).await?; + let mut connected = metrics::client_connection(); + loop { + if shutdown.requested() { + return Ok(()); + } + match blockstore::decode(&mut stream) { + Ok(entry) => self.process(entry).await?, + Err(wincode::error::ReadError::Io(error)) => { + warn!(?error, "replication stream disconnected"); + drop(connected); + stream = self.reconnect(shutdown).await?; + connected = metrics::client_connection(); + } + Err(error) => Err(wincode::Error::from(error))?, + } + } + } + + /// Applies one blockstore entry to the follower engine, holding block-boundary + /// ordering through the pacemaker and flagging superblock seal mismatches. + async fn process(&mut self, entry: OwnedBlockstoreEntry) -> Result<()> { + match entry { + OwnedBlockstoreEntry::Block(block) => { + let (external, guard) = ExternalBlock::new(block); + self.pacer.send(external).await.map_err(EngineError::from)?; + let pending = time::timeout(IO_TIMEOUT, self.blocks.recv()); + let observed = pending.await?.ok_or(ReplicationError::StreamClosed)?; + if block != observed { + // Mismatches are diagnostic until recovery policy is implemented. + error!(?block, ?observed, "replication block divergence detected"); + } + guard.await.map_err(EngineError::from)?; + } + OwnedBlockstoreEntry::Superblock(expected) => { + // The preceding boundary finalized local state; this seal only validates it. + let observed = self.superblocks().sealed(); + if observed != expected { + error!(?expected, ?observed, "replication state mismatch detected"); + metrics::client_state_mismatch(); + Err(EngineError::Replay(ReplayError::StateMismatch))?; + } + } + entry => self.engine.replay(entry).await?, + } + Ok(()) + } + + /// Handshakes with the leader at `position`; either stages a snapshot and + /// signals a required restart, or returns the resumed byte stream. + fn connect(&self, position: BlockstorePosition) -> Result { + let _timer = metrics::time(Operation::ClientConnect); + let mut connection = TcpStream::connect_timeout(&self.addr, IO_TIMEOUT)?; + connection.set_read_timeout(Some(IO_TIMEOUT))?; + connection.set_write_timeout(Some(IO_TIMEOUT))?; + let request = HandshakeRequest { version: PROTO_VERSION, position }; + let handshake = Handshake::new(self.signer(), request)?; + protocol::write(&mut connection, &handshake)?; + let handshake = protocol::read::>(&mut connection)?; + handshake.verify()?; + let expected = self.authority(); + if handshake.identity != expected { + let message = format!( + "unexpected replication server identity {}; expected {expected}", + handshake.identity + ); + return Err(ReplicationError::Handshake(message)); + } + + match handshake.payload { + HandshakeResponse::Snapshot(meta) => { + self.stage_snapshot(&mut connection, meta)?; + Err(ReplicationError::RestartRequired(meta.id)) + } + HandshakeResponse::Stream(remote) => { + info!(?position, ?remote, "replication handshake accepted"); + Ok(BufReader::with_capacity(256 * KB, connection)) + } + HandshakeResponse::Err(message) => Err(ReplicationError::Handshake(message)), + } + } + + /// Reconnects from a quiesced local cursor. + async fn reconnect(&self, shutdown: &ShutdownHandle) -> Result { + // Hold quiescence so every retry uses the same flushed position. + let _guard = self.barrier().await?; + self.sync(false)?; + let position = self.superblocks().position(); + for attempt in 1..=MAX_RECONNECT_ATTEMPTS { + if shutdown.requested() { + return Err(ReplicationError::StreamClosed); + } + metrics::client_connection_attempt(); + match self.connect(position) { + Ok(stream) => { + info!(attempt, ?position, "replication stream connected"); + return Ok(stream); + } + Err(ReplicationError::IO(error)) => { + warn!(attempt, ?error, "replication reconnect failed"); + } + Err(error) => return Err(error), + } + let timeout = RETRY_DELAY * attempt as u32; + if time::timeout(timeout, shutdown.signalled()).await.is_ok() { + return Err(ReplicationError::StreamClosed); + } + } + Err(ReplicationError::ReconnectExhausted) + } + + /// Writes the incoming snapshot archive into a fresh superblock directory and + /// records its seal, readying the follower to restart from that state. + fn stage_snapshot(&self, connection: &mut TcpStream, meta: SnapshotMetadata) -> Result<()> { + let _timer = metrics::time(Operation::ClientStageSnapshot); + // Stage in the successor before seal rotation so restart can find it. + let dir = Superblock::init_dir(self.superblocks().directory(), meta.id + 1)?; + let archive = dir.join(ACCOUNTSDB_SNAPSHOT_FILE); + let temporary = dir.join(format!("{ACCOUNTSDB_SNAPSHOT_FILE}.tmp")); + let mut file = File::options().write(true).create(true).truncate(true).open(&temporary)?; + let written = io::copy(&mut connection.take(meta.len), &mut file)?; + if written != meta.len { + return Err(ReplicationError::Snapshot(meta.len, written)); + } + file.sync_all()?; + drop(file); + fs::rename(temporary, archive)?; + self.superblocks().bootstrap(meta.superblock)?; + info!(?meta, "replication snapshot staged"); + Ok(()) + } +} diff --git a/replicator/src/error.rs b/replicator/src/error.rs new file mode 100644 index 00000000..24e48b33 --- /dev/null +++ b/replicator/src/error.rs @@ -0,0 +1,54 @@ +use std::io; + +use engine::EngineError; +use keeper::error::KeeperError; +use ledger::LedgerError; +use nucleus::ledger::BlockstorePosition; +use tokio::time::error::Elapsed; + +/// Failure while negotiating or transferring replicated state. +#[derive(thiserror::Error, Debug)] +pub enum ReplicationError { + /// Socket or replicated-file access failed. + #[error("replication I/O failed: {0}")] + IO(#[from] io::Error), + /// Applying replicated state through the keeper failed. + #[error("failed to apply replicated state: {0}")] + State(#[from] KeeperError), + /// Applying a replicated entry through the execution engine failed. + #[error("replication engine operation failed: {0}")] + Engine(#[from] EngineError), + /// Reading or advancing replicated ledger storage failed. + #[error("replication ledger operation failed: {0}")] + Ledger(#[from] LedgerError), + /// A control message could not be encoded or decoded. + #[error("invalid replication control message: {0}")] + Serde(#[from] wincode::Error), + /// The peer uses a protocol version this crate cannot read. + #[error("replication protocol version mismatch; expected version {0}")] + VersionMismatch(u32), + /// The requested or published blockstore cursor is unavailable locally. + #[error("replication position is unavailable: {0:?}")] + PositionNotFound(BlockstorePosition), + /// The leader rejected the client's handshake. + #[error("replication handshake rejected: {0}")] + Handshake(String), + /// The snapshot connection ended before the advertised byte count arrived. + #[error("incomplete replication snapshot: expected {0} bytes, received {1}")] + Snapshot(u64, u64), + /// No complete retained snapshot can satisfy an unavailable cursor. + #[error("no complete replication snapshot is available")] + SnapshotUnavailable, + /// All bounded attempts to reconnect to the leader failed. + #[error("replication reconnect attempts exhausted")] + ReconnectExhausted, + /// A staged snapshot must be installed by restarting the engine. + #[error("replication snapshot for superblock {0} is staged; restart required")] + RestartRequired(u64), + /// A replication event stream closed before the transfer completed. + #[error("replication event stream closed")] + StreamClosed, + /// Waiting for a locally committed block boundary timed out. + #[error("timed out waiting for a replicated block boundary: {0}")] + Timeout(#[from] Elapsed), +} diff --git a/replicator/src/lib.rs b/replicator/src/lib.rs new file mode 100644 index 00000000..899fd4ea --- /dev/null +++ b/replicator/src/lib.rs @@ -0,0 +1,23 @@ +#![doc = include_str!("../README.md")] + +mod client; +mod error; +mod metrics; +mod protocol; +mod server; + +use std::time::Duration; + +pub use client::ReplicationClient; +pub use error::ReplicationError; +pub use protocol::PROTO_VERSION; +pub use server::ReplicationDispatcher; + +type Result = std::result::Result; + +/// Read/write timeout applied to replication sockets on both sides. +const IO_TIMEOUT: Duration = Duration::from_secs(4); +/// Delay a follower waits between reconnect attempts. +const RETRY_DELAY: Duration = Duration::from_secs(1); +/// Reconnect attempts a follower makes before giving up. +const MAX_RECONNECT_ATTEMPTS: usize = 10; diff --git a/replicator/src/metrics.rs b/replicator/src/metrics.rs new file mode 100644 index 00000000..c7ab18b9 --- /dev/null +++ b/replicator/src/metrics.rs @@ -0,0 +1,136 @@ +//! Prometheus metrics for replication clients and servers. + +use std::sync::OnceLock; + +use nucleus::metrics::{self as metric, OperationTimer}; +use nucleus::metrics::{IntCounter, IntGauge, MetricOperation, MetricSpec, OperationCounters}; + +/// Process-wide replicator metrics registered in the default Prometheus registry. +static METRICS: OnceLock = OnceLock::new(); + +const OPERATION_TIME: MetricSpec = MetricSpec { + name: "replicator_operation_duration_micros", + help: "Replication operation duration distribution in microseconds.", +}; +const CLIENT_STREAM_CONNECTED: MetricSpec = MetricSpec { + name: "replicator_client_stream_connected", + help: "Whether the replication client currently holds a live blockstore stream.", +}; +const SERVER_CONNECTIONS: MetricSpec = MetricSpec { + name: "replicator_server_connections", + help: "Current replication server connection workers.", +}; +const CLIENT_CONNECTION_ATTEMPTS: MetricSpec = MetricSpec { + name: "replicator_client_connection_attempts", + help: "Replication client connection attempts.", +}; +const CLIENT_STATE_MISMATCHES: MetricSpec = MetricSpec { + name: "replicator_client_state_mismatches", + help: "Superblock seal mismatches detected by the replication client.", +}; +const SERVER_CURSOR_UPDATES_SKIPPED: MetricSpec = MetricSpec { + name: "replicator_server_cursor_updates_skipped", + help: "Replication server cursor updates skipped after receiver lag.", +}; + +/// Replication operation used as a fixed low-cardinality label. +#[derive(Clone, Copy)] +pub(crate) enum Operation { + ClientConnect, + ClientStageSnapshot, + ServerHandshake, + ServerAdvance, + ServerSendSnapshot, +} + +impl MetricOperation for Operation { + fn label(self) -> &'static str { + match self { + Self::ClientConnect => "client_connect", + Self::ClientStageSnapshot => "client_stage_snapshot", + Self::ServerHandshake => "server_handshake", + Self::ServerAdvance => "server_advance", + Self::ServerSendSnapshot => "server_send_snapshot", + } + } +} + +/// Registers all replicator metrics once. +pub(crate) fn init() { + METRICS.get_or_init(Default::default); +} + +/// Starts an operation timer that records latency when the returned guard drops. +pub(crate) fn time(op: Operation) -> OperationTimer<'static> { + op.time(METRICS.get().map(|m| &m.operations)) +} + +/// Records a client connection attempt. +pub(crate) fn client_connection_attempt() { + metric::with_metrics(&METRICS, |m| m.client_connection_attempts.inc()); +} + +/// Marks a client blockstore stream live until the returned guard drops. +pub(crate) fn client_connection() -> ClientConnection { + metric::with_metrics(&METRICS, |m| m.client_stream_connected.set(1)); + ClientConnection +} + +/// Records a superblock seal mismatch. +pub(crate) fn client_state_mismatch() { + metric::with_metrics(&METRICS, |m| m.client_state_mismatches.inc()); +} + +/// Counts a server worker until the returned guard drops. +pub(crate) fn server_connection() -> ServerConnection { + metric::with_metrics(&METRICS, |m| m.server_connections.inc()); + ServerConnection +} + +/// Records durable cursor updates skipped by a lagged receiver. +pub(crate) fn server_cursor_updates_skipped(skipped: u64) { + metric::with_metrics(&METRICS, |m| { + m.server_cursor_updates_skipped.inc_by(skipped) + }); +} + +/// Clears the live-client gauge on every stream exit path. +pub(crate) struct ClientConnection; + +impl Drop for ClientConnection { + fn drop(&mut self) { + metric::with_metrics(&METRICS, |m| m.client_stream_connected.set(0)); + } +} + +/// Decrements the active-server-worker gauge on every worker exit path. +pub(crate) struct ServerConnection; + +impl Drop for ServerConnection { + fn drop(&mut self) { + metric::with_metrics(&METRICS, |m| m.server_connections.dec()); + } +} + +/// Owns all Prometheus collectors registered by replicator. +struct Metrics { + operations: OperationCounters, + client_stream_connected: IntGauge, + server_connections: IntGauge, + client_connection_attempts: IntCounter, + client_state_mismatches: IntCounter, + server_cursor_updates_skipped: IntCounter, +} + +impl Default for Metrics { + fn default() -> Self { + Self { + operations: OperationCounters::new(OPERATION_TIME), + client_stream_connected: metric::gauge(CLIENT_STREAM_CONNECTED, 0), + server_connections: metric::gauge(SERVER_CONNECTIONS, 0), + client_connection_attempts: metric::counter(CLIENT_CONNECTION_ATTEMPTS, 0), + client_state_mismatches: metric::counter(CLIENT_STATE_MISMATCHES, 0), + server_cursor_updates_skipped: metric::counter(SERVER_CURSOR_UPDATES_SKIPPED, 0), + } + } +} diff --git a/replicator/src/protocol.rs b/replicator/src/protocol.rs new file mode 100644 index 00000000..846a545a --- /dev/null +++ b/replicator/src/protocol.rs @@ -0,0 +1,156 @@ +use std::{ + io::{Read, Write}, + time::Duration, +}; + +use derive_more::Deref; +use ledger::schema::SuperblockSeal; +use nucleus::{ledger::BlockstorePosition, unix_time}; +use solana_keypair::{Keypair, Signer}; +use solana_pubkey::Pubkey; +use solana_signature::Signature; +use wincode::{SchemaRead, SchemaWrite, config::DefaultConfig}; + +use crate::{ReplicationError, Result}; + +/// Wire protocol version accepted by this crate. +pub const PROTO_VERSION: u32 = 1; + +/// Encoded length prefix preceding every control frame. +const HEADER_LENGTH: usize = size_of::(); +/// Largest control frame accepted before allocating its payload. +const MAX_CONTROL_FRAME_LENGTH: u32 = u16::MAX as u32; +/// Maximum clock difference accepted for a signed control message. +const MAX_CLOCK_SKEW: Duration = Duration::from_secs(30); + +/// Signed, freshness-bounded control message exchanged during negotiation. +#[derive(SchemaRead, SchemaWrite)] +pub(crate) struct Handshake

{ + /// Direction-specific handshake payload. + pub(crate) payload: P, + /// Public identity of the signer. + pub(crate) identity: Pubkey, + /// Unix timestamp in microseconds covered by the signature. + pub(crate) timestamp: u64, + /// Signature over the payload and timestamp. + pub(crate) signature: Signature, +} + +/// Initial follower request identifying its last durable blockstore byte. +#[derive(SchemaRead, SchemaWrite)] +pub(crate) struct HandshakeRequest { + /// Wire version understood by the follower. + pub(crate) version: u32, + /// Next durable blockstore byte required by the follower. + pub(crate) position: BlockstorePosition, +} + +/// Leader decision following a valid handshake. +#[derive(SchemaRead, SchemaWrite)] +pub(crate) enum HandshakeResponse { + /// Snapshot that must be staged before replication can resume. + Snapshot(SnapshotMetadata), + /// Leader cursor from which live streaming begins. + Stream(BlockstorePosition), + /// Reason the leader rejected negotiation. + Err(String), +} + +/// Describes the accountsdb snapshot a follower must stage before it can stream. +#[derive(SchemaRead, SchemaWrite, Debug, Clone, Copy, Deref)] +pub(crate) struct SnapshotMetadata { + /// Length of the snapshot archive in bytes. + pub(crate) len: u64, + /// Seal the snapshot restores accountsdb to. + #[deref] + pub(crate) superblock: SuperblockSeal, +} + +impl

Handshake

+where + for<'de> P: SchemaRead<'de, DefaultConfig, Dst = P>, + P: SchemaWrite, +{ + /// Signs the payload with a fresh timestamp and publishes the signer's identity. + pub(crate) fn new(keypair: &Keypair, payload: P) -> Result { + let identity = keypair.pubkey(); + let timestamp = timestamp(); + let data = message(timestamp, &payload)?; + let signature = keypair.sign_message(&data); + Ok(Self { + payload, + identity, + timestamp, + signature, + }) + } + + /// Rejects altered messages and timestamps outside the accepted clock-skew window. + /// + /// Freshness is time-based; duplicate messages inside the window are not tracked. + pub(crate) fn verify(&self) -> Result<()> { + let data = message(self.timestamp, &self.payload)?; + if !self.signature.verify(self.identity.as_ref(), &data) { + let msg = "invalid handshake signature"; + return Err(ReplicationError::Handshake(msg.into())); + } + let skew = timestamp().abs_diff(self.timestamp); + if skew > MAX_CLOCK_SKEW.as_micros() as u64 { + let msg = "handshake timestamp exceeds maximum clock skew"; + return Err(ReplicationError::Handshake(msg.into())); + } + Ok(()) + } +} + +/// Reads and decodes one length-prefixed control message. +pub(crate) fn read(reader: &mut impl Read) -> Result +where + for<'de> T: SchemaRead<'de, DefaultConfig, Dst = T>, +{ + let mut header = [0; HEADER_LENGTH]; + reader.read_exact(&mut header)?; + let len = u32::from_le_bytes(header); + if len > MAX_CONTROL_FRAME_LENGTH { + return Err(ReplicationError::Handshake(format!( + "replication control frame len {len} exceeds max allowed" + ))); + } + + let mut payload = vec![0; len as usize]; + reader.read_exact(&mut payload)?; + wincode::deserialize_exact(&payload) + .map_err(wincode::Error::from) + .map_err(Into::into) +} + +/// Encodes and writes one length-prefixed control message. +pub(crate) fn write(writer: &mut impl Write, message: &T) -> Result<()> +where + T: SchemaWrite + ?Sized, +{ + let payload = wincode::serialize(message).map_err(wincode::Error::from)?; + let len = payload.len() as u32; + if len > MAX_CONTROL_FRAME_LENGTH { + return Err(ReplicationError::Handshake(format!( + "replication control frame len {len} exceeds max allowed", + ))); + } + writer.write_all(&len.to_le_bytes())?; + writer.write_all(&payload)?; + writer.flush().map_err(Into::into) +} + +fn timestamp() -> u64 { + unix_time().as_micros() as u64 +} + +/// Builds the protocol byte string covered by a handshake signature. +fn message

(ts: u64, payload: &P) -> Result> +where + P: SchemaWrite, +{ + let mut data = wincode::serialize(payload).map_err(wincode::Error::from)?; + data.extend_from_slice(&ts.to_le_bytes()); + Ok(data) +} diff --git a/replicator/src/server.rs b/replicator/src/server.rs new file mode 100644 index 00000000..7000a76b --- /dev/null +++ b/replicator/src/server.rs @@ -0,0 +1,353 @@ +use std::{ + fs::File, + io::Write, + net::{SocketAddr, TcpStream}, + sync::Arc, + thread, +}; + +use derive_more::Deref; +use engine::Engine; +use ledger::schema::SuperblockSeal; +use nucleus::{ + ledger::{ACCOUNTSDB_SNAPSHOT_FILE, BlockstorePosition}, + shutdown::{CancellationToken, Service, ShutdownHandle, ShutdownManager, ShutdownReason}, +}; +use scc::HashMap; +use solana_keypair::Signer; +use solana_pubkey::Pubkey; +use tokio::{ + net::{TcpListener, TcpStream as AsyncTcpStream}, + runtime, + sync::broadcast, +}; +use tracing::{error, info, warn}; + +use crate::{ + IO_TIMEOUT, ReplicationError, Result, + metrics::{self, Operation}, + protocol::{ + self, Handshake, HandshakeRequest, HandshakeResponse, PROTO_VERSION, SnapshotMetadata, + }, +}; + +/// Accepts follower connections and assigns each one a blocking transfer worker. +pub struct ReplicationDispatcher { + /// Accepts inbound follower connections. + listener: TcpListener, + /// Engine whose local signer authenticates responses and whose ledger is served. + engine: Engine, + /// List of follower identities permitted to replicate. + allowed: Arc>>, + /// Cancels the accept loop and parents every per-connection worker. + shutdown: ShutdownHandle, +} + +/// Serves one follower from its requested durable cursor onward. +#[derive(Deref)] +struct ReplicationServer { + /// Blocking, timeout-bounded socket to the follower. + connection: TcpStream, + /// Cursor of the next byte owed to the follower. + position: ReplicationPosition, + /// Engine whose local signer authenticates responses and whose ledger is served. + #[deref] + engine: Engine, + /// Local follower identities permitted to replicate. + allowed: Arc>>, + /// Fires when the dispatcher shuts down. + cancellation: CancellationToken, +} + +/// Open blockstore and cursor from which the next byte must be sent. +struct ReplicationPosition { + /// Open blockstore file for `current.superblock`. + blockstore: File, + /// Durable-cursor updates broadcast by the appender. + stream: broadcast::Receiver, + /// Position of the next byte to send. + current: BlockstorePosition, +} + +/// Initial transfer selected after validating the follower cursor. +enum ReplicationAction { + /// Send a full accountsdb snapshot; the follower restarts from it. + Snapshot { archive: File, meta: SnapshotMetadata }, + /// Resume the blockstore stream from a still-retained cursor. + Stream { from: BlockstorePosition, blockstore: File }, +} + +impl ReplicationDispatcher { + /// Verifies the canonical signer, then binds `addr` and starts the accept loop. + pub async fn spawn( + addr: SocketAddr, + engine: Engine, + allowed: Arc<[Pubkey]>, + shutdown: &mut ShutdownManager, + ) -> Result<()> { + metrics::init(); + if engine.signer().pubkey() != engine.authority() { + warn!("dispatcher is disabled: node cannot act as replication relay"); + return Ok(()); + } + let listener = TcpListener::bind(addr).await?; + let shutdown = shutdown.handle(Service::ReplicationDispatcher); + let allowed = Arc::new(allowed.iter().map(|&identity| (identity, Arc::new(()))).collect()); + let service = Self { + listener, + engine, + allowed, + shutdown, + }; + tokio::spawn(service.run()); + info!(%addr, "replication dispatcher started"); + Ok(()) + } + + /// Accepts until cancellation or a listener failure; connection failures stay isolated. + async fn run(mut self) { + let reason = loop { + tokio::select! { + biased; + _ = self.shutdown.signalled() => break ShutdownReason::Signalled, + result = self.listener.accept() => match result { + Ok((stream, peer)) => { + if let Err(error) = self.dispatch(stream, peer) { + warn!(%peer, ?error, "failed to dispatch replication connection"); + } + } + Err(error) => break ShutdownReason::Error(Box::new(error)), + } + } + }; + // Release owned resources before reporting service termination. + drop(self.listener); + drop(self.engine); + self.shutdown.terminate(reason); + } + + /// Converts the accepted async socket into a blocking, timeout-bounded stream + /// and hands it to a dedicated blocking server worker. + fn dispatch(&self, stream: AsyncTcpStream, peer: SocketAddr) -> Result<()> { + let stream = stream.into_std()?; + stream.set_nonblocking(false)?; + stream.set_read_timeout(Some(IO_TIMEOUT))?; + stream.set_write_timeout(Some(IO_TIMEOUT))?; + let engine = self.engine.clone(); + let cancellation = self.shutdown.child(); + let allowed = self.allowed.clone(); + ReplicationServer::spawn(stream, peer, engine, allowed, cancellation) + } +} + +impl ReplicationServer { + /// Starts a blocking worker without blocking the async dispatcher. + fn spawn( + connection: TcpStream, + peer: SocketAddr, + engine: Engine, + allowed: Arc>>, + cancellation: CancellationToken, + ) -> Result<()> { + // Subscribe before sampling so racing cursor updates remain queued. + let stream = engine.ledger().position.subscribe(); + let current = engine.ledger().position(); + let blockstore = blockstore(&engine, current.superblock)?; + let position = ReplicationPosition { stream, current, blockstore }; + let server = Self { + connection, + position, + engine, + allowed, + cancellation, + }; + let runtime = runtime::Builder::new_current_thread().build()?; + thread::Builder::new().name("replication-server".into()).spawn(move || { + let _connection = metrics::server_connection(); + runtime + .block_on(server.run()) + .inspect_err(|error| warn!(%peer, %error, "replication connection failed")) + })?; + Ok(()) + } + + /// Negotiates an initial transfer, catches up immediately, then follows durable cursors. + async fn run(mut self) -> Result<()> { + // Cursor updates arrive at every block and write new durable bytes. Those writes + // detect peer disconnects, exit this worker, and release its identity lease. + let (action, _lease) = match self.handshake() { + Ok(handshake) => handshake, + Err(error) => { + warn!(?error, "replication handshake rejected"); + let response = HandshakeResponse::Err(error.to_string()); + self.respond(response)?; + return Ok(()); + } + }; + + match action { + ReplicationAction::Snapshot { mut archive, meta } => { + let _timer = metrics::time(Operation::ServerSendSnapshot); + info!(?meta, "sending replication snapshot"); + let response = HandshakeResponse::Snapshot(meta); + self.respond(response)?; + send_range(&mut archive, &mut self.connection, 0, meta.len)?; + self.connection.flush()?; + return Ok(()); + } + ReplicationAction::Stream { from, blockstore } => { + let response = HandshakeResponse::Stream(self.position.current); + self.respond(response)?; + self.position.current = from; + self.position.blockstore = blockstore; + let through = self.ledger().position(); + self.advance(through)?; + info!(?from, ?through, "replication caught up"); + } + } + + loop { + tokio::select! { + biased; + _ = self.cancellation.cancelled() => return Ok(()), + result = self.position.stream.recv() => match result { + Ok(position) => self.advance(position)?, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + let position = self.ledger().position(); + // Cursors are cumulative; advancing to latest covers skipped updates. + warn!(skipped, ?position, "replication cursor lagged"); + metrics::server_cursor_updates_skipped(skipped); + self.advance(position)?; + } + Err(broadcast::error::RecvError::Closed) => { + error!("ledger position stream closed unexpectedly"); + return Err(ReplicationError::StreamClosed); + } + } + } + } + } + + /// Signs and writes a leader handshake response with the local engine signer. + fn respond(&mut self, response: HandshakeResponse) -> Result<()> { + let handshake = Handshake::new(self.signer(), response)?; + protocol::write(&mut self.connection, &handshake) + } + + /// Selects retained streaming when possible, otherwise the newest ready snapshot. + fn handshake(&mut self) -> Result<(ReplicationAction, Arc<()>)> { + let _timer = metrics::time(Operation::ServerHandshake); + let handshake: Handshake = protocol::read(&mut self.connection)?; + handshake.verify()?; + let lease = self.reserve(handshake.identity)?; + if handshake.payload.version != PROTO_VERSION { + return Err(ReplicationError::VersionMismatch(PROTO_VERSION)); + } + + let requested = handshake.payload.position; + if requested > self.position.current { + return Err(ReplicationError::PositionNotFound(requested)); + } + let action = match self.ledger().cursor(requested.superblock) { + Some(end) if requested.offset <= end => ReplicationAction::Stream { + from: requested, + blockstore: blockstore(self, requested.superblock)?, + }, + Some(_) => return Err(ReplicationError::PositionNotFound(requested)), + None => self.snapshot()?, + }; + Ok((action, lease)) + } + + /// Falls back to the newest retained superblock that has a staged accountsdb + /// snapshot, used when the follower's cursor is no longer streamable. + fn snapshot(&self) -> Result { + for superblock in self.ledger().iter() { + let path = superblock.directory.join(ACCOUNTSDB_SNAPSHOT_FILE); + if !path.exists() { + continue; + } + let archive = File::open(path)?; + let meta = SnapshotMetadata { + len: archive.metadata()?.len(), + // Successor archives carry the predecessor seal metadata. + superblock: SuperblockSeal { + id: superblock.id.saturating_sub(1), + checksum: superblock.checksum(), + transactions: superblock.transactions(), + }, + }; + return Ok(ReplicationAction::Snapshot { archive, meta }); + } + Err(ReplicationError::SnapshotUnavailable) + } + + /// Sends every durable byte between the current cursor and `target`. + fn advance(&mut self, target: BlockstorePosition) -> Result<()> { + let _timer = metrics::time(Operation::ServerAdvance); + if target <= self.position.current { + return Ok(()); + } + while self.position.current.superblock < target.superblock { + let end = self + .ledger() + .cursor(self.position.current.superblock) + .ok_or(ReplicationError::PositionNotFound(self.position.current))?; + self.send(end)?; + let superblock = self.position.current.superblock + 1; + self.position.blockstore = blockstore(self, superblock)?; + self.position.current = BlockstorePosition { superblock, offset: 0 }; + } + self.send(target.offset) + } + + /// Sends blockstore bytes from the current offset up to `end`, advancing the cursor. + fn send(&mut self, end: u64) -> Result<()> { + let start = self.position.current.offset; + send_range( + &mut self.position.blockstore, + &mut self.connection, + start, + end - start, + )?; + self.position.current.offset = end; + Ok(()) + } + + /// Reserves an allowed identity until the returned lease drops. + fn reserve(&self, identity: Pubkey) -> Result> { + let Some(entry) = self.allowed.get_sync(&identity) else { + let msg = "replication access not allowed"; + return Err(ReplicationError::Handshake(msg.into())); + }; + if Arc::strong_count(entry.get()) > 1 { + let msg = "replication stream already active"; + return Err(ReplicationError::Handshake(msg.into())); + } + Ok(entry.get().clone()) + } +} + +/// Copies `len` bytes from `file` at `offset` to the socket, looping until the +/// full range is transferred (each `send_exact` may transfer only part of it). +fn send_range( + file: &mut File, + stream: &mut TcpStream, + mut offset: u64, + mut len: u64, +) -> Result<()> { + while len != 0 { + let sent = snedfile::send_exact(file, stream, len, offset)?; + offset += sent; + len -= sent; + } + Ok(()) +} + +/// Clones the blockstore file handle of a retained superblock for independent seeking. +fn blockstore(engine: &Engine, superblock: u64) -> Result { + let position = BlockstorePosition { superblock, offset: 0 }; + let candidate = engine.ledger().iter().find(|sb| sb.id == superblock); + let sb = candidate.ok_or(ReplicationError::PositionNotFound(position))?; + sb.blockstore.try_clone().map_err(Into::into) +} diff --git a/replicator/tests/integration.rs b/replicator/tests/integration.rs new file mode 100644 index 00000000..b545f489 --- /dev/null +++ b/replicator/tests/integration.rs @@ -0,0 +1,587 @@ +//! Full-stack, byte-exact replication invariants over real engines and loopback TCP. +//! +//! Followers must match the leader's cursor and seal checksums, not just state. +//! Mutations are non-idempotent so duplicate application is observable. + +#![cfg(test)] + +use std::{ + net::{Ipv4Addr, SocketAddr, TcpListener}, + sync::Arc, + time::Duration, +}; + +use engine::testkit::{Pacing, TestEngine}; +use keeper::testkit::{ + Dirs, WireVersion, keeper_builder, load_v42_data, patterned_bytes, sign_versioned_instructions, + v42_builder, v42_padded_value, +}; +use ledger::request::{BlockDetails, BlockParams, BlockResponse}; +use nucleus::{KB, Slot, config::Authority, ledger::BlockstorePosition, shutdown::ShutdownManager}; +use replicator::{ReplicationClient, ReplicationDispatcher}; +use solana_account::{AccountBuilder, AccountMode, ReadableAccount}; +use solana_keypair::{Keypair, Signer}; +use solana_pubkey::Pubkey; +use solana_sysvar::rent::Rent; +use tokio::{sync::broadcast, time}; +use v42_calculator_interface::builder::Expr as E; + +/// Bound for every asynchronous replication assertion. +const TIMEOUT: Duration = Duration::from_secs(4); + +type AccountSeed = (Pubkey, i64, AccountMode); + +/// Returns an unused loopback address, released immediately after discovery. +fn loopback_addr() -> SocketAddr { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap(); + listener.local_addr().unwrap() +} + +/// Starts an engine over throwaway directories seeded with `accounts`. +async fn engine(authority: Authority, accounts: &[AccountSeed], pacing: Pacing) -> TestEngine { + let dirs = Dirs::default(); + let mut builder = keeper_builder(&dirs); + builder.authority = authority; + for &(key, value, mode) in accounts { + builder.accounts.insert(key, v42_builder(value, mode).build()); + } + TestEngine::from_builder(dirs, builder, pacing).await +} + +/// Starts a leader and follower with distinct local identities and direct trust. +async fn engines( + leader_accounts: &[AccountSeed], + follower_accounts: &[AccountSeed], +) -> (TestEngine, TestEngine) { + let leader_authority: Authority = Keypair::new().into(); + let follower_authority = Authority { + local: Arc::new(Keypair::new()), + remote: Some(leader_authority.local.pubkey()), + }; + let leader = engine(leader_authority, leader_accounts, Pacing::External).await; + let follower = engine(follower_authority, follower_accounts, Pacing::External).await; + (leader, follower) +} + +/// Starts a leader-side dispatcher at an already selected address. +async fn dispatcher(addr: SocketAddr, leader: &TestEngine, allowed: &[Pubkey]) -> ShutdownManager { + let mut shutdown = ShutdownManager::default(); + let engine = (*leader).clone(); + ReplicationDispatcher::spawn(addr, engine, Arc::from(allowed), &mut shutdown) + .await + .unwrap(); + shutdown +} + +/// Registers a replication client with the follower lifecycle manager. +fn replicate(addr: SocketAddr, follower: &mut TestEngine) { + let engine = follower.clone(); + let pacer = follower.pacer(); + ReplicationClient::spawn(addr, engine, pacer, follower.shutdown()).unwrap(); +} + +/// Subscribes before starting replication so the first published position cannot be missed. +fn stream(addr: SocketAddr, follower: &mut TestEngine) -> broadcast::Receiver { + let positions = follower.ledger().position.subscribe(); + replicate(addr, follower); + positions +} + +/// Stages a snapshot through replication and installs it on restart. +async fn restart_from_snapshot(addr: SocketAddr, mut follower: TestEngine) -> TestEngine { + replicate(addr, &mut follower); + follower.shutdown().wait().await; + let (dirs, authority) = follower.close().await; + TestEngine::with(dirs, authority).await +} + +/// Applies a non-idempotent mutation so duplicate replication changes the result. +/// Calls must be separated by a block advance to produce distinct signatures. +async fn increment(engine: &TestEngine, state: Pubkey) { + let ix = (E::acc(1) + E::lit(1)).compose(state, &[state]); + engine.execute(&[ix]).await.expect("increment commits"); +} + +/// Commits one increment and publishes its enclosing block cursor. +async fn commit_increment(engine: &mut TestEngine, state: Pubkey) -> BlockstorePosition { + increment(engine, state).await; + engine.advance(1).await; + engine.sync().await +} + +/// Waits for the exact cursor, flushes the follower, and verifies state and position. +async fn await_replication( + positions: &mut broadcast::Receiver, + follower: &TestEngine, + expected: BlockstorePosition, + state: Pubkey, + value: i64, +) { + time::timeout(TIMEOUT, async { + loop { + let observed = positions.recv().await.expect("position stream is open"); + if observed < expected { + continue; + } + assert_eq!(observed, expected, "follower advanced past the leader"); + return; + } + }) + .await + .expect("replication reaches the synced cursor in time"); + follower.sync().await; + assert_eq!(load_v42_data(follower, state), Some(value)); + assert_eq!(follower.superblocks().position(), expected); +} + +/// Loads the serialized transactions committed in `slot`, preserving ledger order. +async fn block_transactions(engine: &TestEngine, slot: Slot) -> Vec> { + let response = engine + .blocks() + .get(BlockParams { + slot, + details: BlockDetails::Transactions, + }) + .await + .expect("block read succeeds") + .expect("committed block exists"); + let BlockResponse::WithTransactions(block) = response else { + panic!("transaction detail request returns transactions"); + }; + block.transactions +} + +/// Large raw transaction frames survive both retained catch-up and live replay. +#[tokio::test(flavor = "multi_thread")] +async fn replays_large_transactions_during_catch_up_and_live_streaming() { + const BATCHED_INSTRUCTIONS: usize = 32; + const BATCHED_TERMS: usize = 16; + const CREATE_DATA_LEN: usize = 64 * KB + 1; + const UPDATE_DATA_LEN: usize = 128 * KB + 1; + const ACCOUNT_SLOT: Slot = 42; + + let state = Pubkey::new_unique(); + let seed = [(state, 0, AccountMode::Delegated)]; + let (mut leader, mut follower) = engines(&seed, &seed).await; + let account = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let lamports = Rent::default().minimum_balance(UPDATE_DATA_LEN); + + // Persist a standard client transaction and a large private create before + // connecting, forcing both frames through retained-ledger catch-up. + let instructions: Vec<_> = (0..BATCHED_INSTRUCTIONS) + .map(|value| v42_padded_value(state, value as i64, BATCHED_TERMS)) + .collect(); + let (_, client_transaction) = sign_versioned_instructions( + leader.signer(), + WireVersion::Legacy, + &instructions, + leader.blockhash(), + ); + let client_transaction_len = client_transaction.len(); + assert!(client_transaction_len > 4 * KB); + assert!(client_transaction_len < u16::MAX as usize); + leader + .execute(client_transaction) + .await + .expect("large client transaction commits"); + + let created = AccountBuilder::default() + .lamports(lamports) + .owner(owner) + .mode(AccountMode::ReadOnly) + .slot(ACCOUNT_SLOT) + .data(patterned_bytes(CREATE_DATA_LEN, 1)); + leader + .account(account) + .create(created, None) + .await + .expect("large account creation commits"); + + let catch_up_slot = leader.blocks().current_slot(); + leader.advance(1).await; + let expected = leader.sync().await; + let transactions = block_transactions(&leader, catch_up_slot).await; + assert_eq!(transactions.len(), 2); + assert_eq!(transactions[0].len(), client_transaction_len); + assert!(transactions[1].len() > 64 * KB); + + let addr = loopback_addr(); + let follower_identity = follower.signer().pubkey(); + let mut dispatcher = dispatcher(addr, &leader, &[follower_identity]).await; + let mut positions = stream(addr, &mut follower); + await_replication( + &mut positions, + &follower, + expected, + state, + (BATCHED_INSTRUCTIONS - 1) as i64, + ) + .await; + assert_eq!(follower.get_account(account), leader.get_account(account)); + + // Grow and replace the same account while connected, exercising the live + // tail with a different multi-chunk payload and advancing account slot. + let updated = AccountBuilder::default() + .lamports(lamports) + .owner(owner) + .mode(AccountMode::ReadOnly) + .slot(ACCOUNT_SLOT + 1) + .data(patterned_bytes(UPDATE_DATA_LEN, 2)); + leader + .account(account) + .update(updated) + .await + .expect("large account update commits"); + + let live_slot = leader.blocks().current_slot(); + leader.advance(1).await; + let expected = leader.sync().await; + let transactions = block_transactions(&leader, live_slot).await; + assert_eq!(transactions.len(), 1); + assert!(transactions[0].len() > 64 * KB); + + await_replication( + &mut positions, + &follower, + expected, + state, + (BATCHED_INSTRUCTIONS - 1) as i64, + ) + .await; + assert_eq!(follower.get_account(account), leader.get_account(account)); + + dispatcher.terminate().await; + follower.close().await; + leader.close().await; +} + +/// An internally paced leader advances a follower whose delegated state survives restart. +#[tokio::test(flavor = "multi_thread")] +async fn internally_paced_replication_persists_across_restart() { + let state = Pubkey::new_unique(); + let volatile = Pubkey::new_unique(); + let seed = [(state, 0, AccountMode::Delegated), (volatile, 7, AccountMode::ReadOnly)]; + let leader_authority: Authority = Keypair::new().into(); + let follower_authority = Authority { + local: Arc::new(Keypair::new()), + remote: Some(leader_authority.local.pubkey()), + }; + let leader = engine(leader_authority, &seed, Pacing::Internal).await; + let mut follower = engine(follower_authority, &seed, Pacing::External).await; + + let addr = loopback_addr(); + let initial = follower.superblocks().position(); + let follower_identity = follower.signer().pubkey(); + let mut dispatcher = dispatcher(addr, &leader, &[follower_identity]).await; + let mut positions = stream(addr, &mut follower); + let observed = time::timeout(TIMEOUT, async { + loop { + let observed = positions.recv().await.expect("position stream is open"); + if observed > initial { + return observed; + } + } + }) + .await + .expect("follower observes an internally paced block in time"); + + let (dirs, follower_authority) = follower.close().await; + let follower = TestEngine::with(dirs, follower_authority).await; + assert!(follower.get_account(volatile).is_none()); + assert_eq!(load_v42_data(&follower, state), Some(0)); + assert!(follower.superblocks().position() >= observed); + + dispatcher.terminate().await; + follower.close().await; + leader.close().await; +} + +/// Catches up across sealed superblocks, resumes from the durable cursor without +/// re-applying, and drops volatile state on reset. +#[tokio::test(flavor = "multi_thread")] +async fn streams_and_resumes_without_duplicate_application() { + let state = Pubkey::new_unique(); + // Only reset may remove the read-only account; delegated state must survive it. + let volatile = Pubkey::new_unique(); + let seed = [(state, 0, AccountMode::Delegated), (volatile, 7, AccountMode::ReadOnly)]; + let (mut leader, mut follower) = engines(&seed, &seed).await; + + // Catch-up crosses two sealed superblocks and a live tail from byte zero. + for _ in 0..2 { + increment(&leader, state).await; + leader.seal_and_archive().await; + } + let expected = commit_increment(&mut leader, state).await; + + let upstream = loopback_addr(); + let follower_identity = follower.signer().pubkey(); + + // A valid follower signature is still rejected until its local identity is allowed. + let initial = follower.superblocks().position(); + let mut denied_dispatcher = dispatcher(upstream, &leader, &[]).await; + let mut rejected = ShutdownManager::default(); + ReplicationClient::spawn(upstream, follower.clone(), follower.pacer(), &mut rejected).unwrap(); + time::timeout(TIMEOUT, rejected.wait()) + .await + .expect("denied replication client terminates in time"); + assert_eq!(follower.superblocks().position(), initial); + rejected.terminate().await; + denied_dispatcher.terminate().await; + + let mut first_dispatcher = dispatcher(upstream, &leader, &[follower_identity]).await; + let mut positions = stream(upstream, &mut follower); + + // Replay must reproduce the leader's seals and block height, not just state. + await_replication(&mut positions, &follower, expected, state, 3).await; + assert_eq!( + follower.superblocks().sealed(), + leader.superblocks().sealed() + ); + assert_eq!( + follower.blocks().latest().slot, + leader.blocks().latest().slot + ); + + let expected = commit_increment(&mut leader, state).await; + await_replication(&mut positions, &follower, expected, state, 4).await; + + // Resume after an outage from the durable cursor. Exactly-once application + // yields 5; applying the same entry twice would yield 6. + first_dispatcher.terminate().await; + let expected = commit_increment(&mut leader, state).await; + let mut second_dispatcher = dispatcher(upstream, &leader, &[follower_identity]).await; + await_replication(&mut positions, &follower, expected, state, 5).await; + + // Account creation debits the sponsor on both nodes before reset replenishes it. + let authority = leader.authority(); + let authority_before = leader.get_account(authority).expect("leader sponsor exists").lamports(); + let sponsored = Pubkey::new_unique(); + leader + .account(sponsored) + .create(v42_builder(0, AccountMode::Delegated), None) + .await + .expect("sponsored account creation commits"); + let authority_after = leader.get_account(authority).expect("leader sponsor remains").lamports(); + assert!( + authority_after < authority_before, + "account creation debits the leader sponsor" + ); + leader.advance(1).await; + let expected = leader.sync().await; + await_replication(&mut positions, &follower, expected, state, 5).await; + assert_eq!( + follower.get_account(authority).expect("follower sponsor remains").lamports(), + authority_after, + "sponsor debit replicates to the follower" + ); + + // Reset discards volatile accounts, retains delegated state, and replenishes the sponsor. + leader.reset(99).expect("leader reset records"); + assert_eq!( + leader + .get_account(authority) + .expect("leader sponsor exists after reset") + .lamports(), + authority_before, + "reset replenishes the leader sponsor" + ); + let expected = leader.sync().await; + await_replication(&mut positions, &follower, expected, state, 5).await; + assert!(follower.get_account(volatile).is_none()); + assert_eq!( + follower + .get_account(authority) + .expect("follower sponsor exists after reset") + .lamports(), + authority_before, + "reset replenishes the follower sponsor" + ); + + second_dispatcher.terminate().await; + follower.close().await; + leader.close().await; +} + +/// Models a real leader restart end to end: the connection drops, the engine +/// goes down and comes back from durable state, and the still-active follower +/// reconnects and resumes byte-exactly. An update committed just before the +/// restart is recovered, and a further update produced after it streams live — +/// each applied exactly once (the non-idempotent increment would overshoot on a +/// duplicate). +#[tokio::test(flavor = "multi_thread")] +async fn resumes_after_leader_restart() { + let state = Pubkey::new_unique(); + let seed = [(state, 0, AccountMode::Delegated)]; + let (mut leader, mut follower) = engines(&seed, &seed).await; + + let addr = loopback_addr(); + let follower_identity = follower.signer().pubkey(); + let mut first_dispatcher = dispatcher(addr, &leader, &[follower_identity]).await; + let mut positions = stream(addr, &mut follower); + + // Bring the follower current before the leader goes down. + let expected = commit_increment(&mut leader, state).await; + await_replication(&mut positions, &follower, expected, state, 1).await; + + // Take the dispatcher down, then commit an update the follower cannot see. + first_dispatcher.terminate().await; + let expected = commit_increment(&mut leader, state).await; + + // Restart the leader; the reopened engine must durably reload the update. + let (dirs, authority) = leader.close().await; + let mut leader = TestEngine::with(dirs, authority).await; + assert_eq!( + load_v42_data(&leader, state), + Some(2), + "reopened leader durably reloaded the update" + ); + + // Replication resumes on the same address; the still-active follower + // reconnects from its durable cursor and recovers the pre-restart update. + let mut second_dispatcher = dispatcher(addr, &leader, &[follower_identity]).await; + await_replication(&mut positions, &follower, expected, state, 2).await; + + // The reopened leader keeps producing: a new update streams live over the + // resumed connection, proving block production continues from the durable + // slot rather than restarting and stalling the follower. + let expected = commit_increment(&mut leader, state).await; + await_replication(&mut positions, &follower, expected, state, 3).await; + + second_dispatcher.terminate().await; + follower.close().await; + leader.close().await; +} + +/// Installs the newest retained snapshot on restart, then streams every durable +/// ledger entry committed after that snapshot. +#[tokio::test(flavor = "multi_thread")] +async fn restores_the_newest_snapshot_then_streams_its_tail() { + let state = Pubkey::new_unique(); + let seed = [(state, 0, AccountMode::Delegated)]; + // The empty follower can acquire `state` only through snapshot restoration. + let (mut leader, follower) = engines(&seed, &[]).await; + let set = |value| E::lit(value).compose(state, &[]); + + // Distinct values distinguish the newest snapshot from its streamed tail. + for value in [10, 20] { + leader.execute(&[set(value)]).await.unwrap(); + leader.seal_and_archive().await; + } + leader.execute(&[set(30)]).await.unwrap(); + leader.advance(1).await; + let expected = leader.sync().await; + // Make the newest snapshot the only possible handshake response. + leader.ledger().truncate().unwrap(); // superblock 0 + leader.ledger().truncate().unwrap(); // superblock 1 + assert!( + leader.ledger().cursor(0).is_none(), + "follower cursor was retained away" + ); + assert!( + leader.ledger().cursor(1).is_none(), + "older snapshot history was retained away" + ); + + let addr = loopback_addr(); + let follower_identity = follower.signer().pubkey(); + let mut dispatcher = dispatcher(addr, &leader, &[follower_identity]).await; + let mut follower = restart_from_snapshot(addr, follower).await; + assert_eq!( + load_v42_data(&follower, state), + Some(20), + "restart restores the newest completed snapshot, not the post-snapshot tail" + ); + + let mut positions = stream(addr, &mut follower); + await_replication(&mut positions, &follower, expected, state, 30).await; + + dispatcher.terminate().await; + follower.close().await; + leader.close().await; +} + +/// Replicated state is itself replicable: a follower seals and archives from +/// replicated blocks alone, and its reconstruction serves a further follower +/// both as a live stream and as a snapshot bootstrap. +#[tokio::test(flavor = "multi_thread")] +async fn cascades_replication_through_a_follower() { + let state = Pubkey::new_unique(); + let seed = [(state, 0, AccountMode::Delegated)]; + let shared = Arc::new(Keypair::new()); + let leader_authority: Authority = shared.clone().into(); + let middle_authority = Authority { + local: shared.clone(), + remote: Some(shared.pubkey()), + }; + let tail_authority = Authority { + local: Arc::new(Keypair::new()), + remote: Some(shared.pubkey()), + }; + let mut leader = engine(leader_authority, &seed, Pacing::External).await; + let mut middle = engine(middle_authority, &seed, Pacing::External).await; + let tail = engine(tail_authority, &[], Pacing::External).await; + + let leader_addr = loopback_addr(); + let middle_addr = loopback_addr(); + let middle_identity = middle.signer().pubkey(); + let tail_identity = tail.signer().pubkey(); + let mut leader_dispatcher = dispatcher(leader_addr, &leader, &[middle_identity]).await; + let mut middle_positions = stream(leader_addr, &mut middle); + let mut middle_dispatcher = dispatcher(middle_addr, &middle, &[tail_identity]).await; + + // Subscribe before the boundary: the seal is driven by replication, so + // `await_archive` would subscribe after the archiver it means to observe. + let mut archives = middle.accounts().subscribe_snapshots(); + + // Cross the seal live rather than during handshake catch-up. + increment(&leader, state).await; + leader.seal_and_archive().await; + let expected = leader.sync().await; + await_replication(&mut middle_positions, &middle, expected, state, 1).await; + assert_eq!( + middle.superblocks().sealed(), + leader.superblocks().sealed(), + "middle sealed the replicated boundary to the leader's checksum" + ); + time::timeout(TIMEOUT, archives.recv()) + .await + .expect("middle archives its replicated seal in time") + .unwrap(); + + // Keep the archive behind live state so restore and streaming are distinguishable. + let expected = commit_increment(&mut leader, state).await; + await_replication(&mut middle_positions, &middle, expected, state, 2).await; + + // The successor archive must become the only answer to the tail's cursor. + middle.ledger().truncate().unwrap(); // superblock 0 + assert!( + middle.ledger().cursor(0).is_none(), + "tail cursor was retained away" + ); + + let mut tail = restart_from_snapshot(middle_addr, tail).await; + assert_eq!( + load_v42_data(&tail, state), + Some(1), + "tail restores the middle's own archive, not the state streamed past it" + ); + + let mut tail_positions = stream(middle_addr, &mut tail); + await_replication(&mut tail_positions, &tail, expected, state, 2).await; + + // Both hops cross the next seal live. + increment(&leader, state).await; + leader.seal_and_archive().await; + let expected = leader.sync().await; + await_replication(&mut middle_positions, &middle, expected, state, 3).await; + await_replication(&mut tail_positions, &tail, expected, state, 3).await; + assert_eq!(middle.superblocks().sealed(), leader.superblocks().sealed()); + assert_eq!(tail.superblocks().sealed(), leader.superblocks().sealed()); + + leader_dispatcher.terminate().await; + middle_dispatcher.terminate().await; + tail.close().await; + middle.close().await; + leader.close().await; +} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 48701262..bf5eb1a8 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "1.94.1" +channel = "1.96.1" components = ["clippy", "rustfmt"] profile = "minimal" diff --git a/solana/README.md b/solana/README.md new file mode 100644 index 00000000..bd0ee9c8 --- /dev/null +++ b/solana/README.md @@ -0,0 +1,173 @@ +# Engine Runtime Differences from Agave + +This directory contains the Agave runtime forks required by the engine. These +crates execute caller-loaded transactions and return account changes; they do +not own consensus, fork choice, confirmation, persistence, or validator commit +policy. + +The differences below are intentional compatibility constraints for account +representation, transaction context, serialization, VM mapping, and CPI. + +## Runtime boundary + +- `solana-svm` loads accounts through a caller callback and returns execution + results and mutated accounts. +- Persistence, commit decisions, and deployment policy remain outside the fork. +- Program loading is limited to programs required by the transaction. Callers + supply executable SBF account data as raw ELF bytes; decoding loader-specific + headers or indirection remains outside the runtime. +- Deprecated SBF programs retain their loader owner for ABI v0. Other normalized + SBF programs use loader-v4 as their nominal owner and ABI v1. Native programs + retain native-loader ownership. +- Rent-state and lamport-balance checks remain part of execution. + +## Account representation + +`solana-account` replaces the shared-data representation with copy-on-write +storage. `AccountSharedData` contains either an owned `Arc>` or a borrowed +view into an aligned external buffer. `DirtyMarkers` record changes to data, +owner, lamports, slot, mode, and state flags for higher-layer writeback. + +Borrowed storage has these invariants: + +- The buffer is 8-byte aligned and remains live for the borrow. +- One header and pubkey prefix are followed by two account images. +- `AccountHeader::sequence` selects the active image. +- `translate` copies active state into the shadow image before mutation. +- `commit` publishes the shadow image; `reset` abandons it. +- `rollback` is valid only after `commit`. + +Writes remain borrowed while they fit the image capacity. Growth beyond that +capacity promotes the account to owned storage. Shared owned data becomes unique +through `Arc::make_mut` before mutation. + +`AccountMode` contains `ReadOnly`, `Placeholder`, `System`, `Delegated`, +`Ephemeral`, `Transient`, and `Closed`. Only delegated and ephemeral accounts are +mutable by user programs. Transient accounts remain persistent but immutable +after the transaction that legally transitions them from delegated. The +transaction access guard recognizes that transition through the mode dirty +marker; a freshly loaded transient account has a clean marker and remains +immutable. The same transaction-local exception lets a legal mode transition +close an account. Ephemeral accounts remain persistent until then. `StateFlags` +contains `EXECUTABLE`. Complete-account patch sequences cover non-flag fields; +MagicRoot finalization installs the caller's complete flag value without +changing lamports. Replacement freshness remains the caller's responsibility. +`AccountSharedData` does not store `rent_epoch`; compatibility APIs return or +ignore the masked value required by their interface. + +## Transaction context + +`solana-transaction-context` stores accounts in `UnsafeCell`s guarded by explicit +borrow counters. This permits the VM access handler to remap account data while +runtime borrow rules remain enforced. + +`TransactionAccounts` records touched accounts, total account-data resize, and +instruction lamport deltas. `AccountRef` and `AccountRefMut` release their +counters on drop. `ExecutionRecord` returns keyed accounts, return data, touched +count, and resize delta. All references must be released before context +deconstruction; failure of `Rc::try_unwrap` indicates a lifetime bug. + +## Transaction parsing and Engine-private transactions + +`agave-transaction-view` parses Legacy, v0, V1, and Engine-private Magicblock +transactions directly from their serialized bytes. Legacy and v0 retain the +standard Solana wire layouts, while V1 retains the Agave V1 layout. All three +accept serialized sizes through `u16::MAX` bytes, inclusive. The compact-u16 +parser supports the complete canonical one-, two-, and three-byte encoding, so +instruction data and other framed arrays are no longer limited by the former +two-byte parser assumption. + +Frame offsets and total lengths are stored as `u32`. Fallible parsing uses +checked range arithmetic and validates every frame before unchecked iterators +or typed views access the original bytes. The engine is guaranteed not to run +on 16-bit targets, so conversion from validated `u32` offsets to `usize` is +direct. + +Magicblock is private transaction version 127 and reuses the V1 layout with a +distinct prefix. Its signatures follow the V1 message at the end of the byte +stream. The Engine transaction composer compiles account operations as V1, +writes the Magicblock prefix, signs the exact message range, and verifies that +the first static account is the configured Engine authority. Magicblock +transactions may be at most 16 MiB and raise only the SVM instruction-trace +limit to 255; CPI invocations remain limited to 64 and reserve capacity for all +top-level instructions. Standard versions retain their existing structural +limits. The account accessor uses this private path so a 64 KiB account payload +can be split into patch instructions and executed atomically without relaxing +standard transaction policy. Its V1-shaped address count remains encodable at a +maximum of 255. + +Address lookup tables are intentionally disabled. Any transaction containing a +lookup table entry fails sanitization with `AddressLookupMismatch`; an empty v0 +lookup list remains valid and resolves without loaded addresses. Sequencing and +simulation therefore resolve transactions without supplying loaded addresses. + +The crate-specific wire and safety contracts are documented in +[`transaction-view/README.md`](transaction-view/README.md). Keep its version +prefix, signed message range, size limits, sanitizer, Engine composer, and SVM +trace-limit override synchronized. + +## VM account mapping + +Account data is always mapped directly into the SBF VM. Do not restore the +removed `virtual_address_space_adjustments` or `account_data_direct_mapping` +branches that copied account data through serialized program input. + +Serialization retains loader ABI metadata: + +- Deprecated-loader accounts use ABI v0. +- Loader-v2 and loader-v3 accounts use ABI v1. +- ABI v1 optionally includes direct account pointers. + +The serialized input contains metadata, lamports, lengths, owners, instruction +data, and program id. Account data resides in separate `MemoryRegion`s. +Deprecated-loader regions reserve the current length; newer loaders also reserve +`MAX_PERMITTED_DATA_INCREASE`. Deserialization reads mutable metadata but does +not copy account bytes back from the input buffer. + +## Access-violation growth + +Writable borrowed or shared-owned account data may initially be mapped +read-only. The first VM store enters the transaction-context handler, which: + +- handles stores only and requires an account-index region payload; +- rejects accesses outside the account's reserved address range; +- records touch and resize deltas before growing data; +- grows only to the requested access length; and +- replaces the region host pointer, length, and writability. + +Keep serialization, `TransactionContext::access_violation_handler`, and VM error +mapping synchronized. They jointly map growth failures to account-specific +readonly, size, and realloc errors. + +## CPI synchronization + +`CallerAccount::serialized_data` remains empty. CPI entry and exit synchronize +lamports, owner, and data length, while account bytes remain directly mapped. +When storage can move, CPI replaces the caller `MemoryRegion` with one created +from the current account. + +Strict syscall parameter-address checks are always enforced. CPI rejects +`AccountInfo` fields whose key, owner, lamports, data, or data-length pointers do +not reference the canonical VM locations for the passed account. This is required +because account bytes are mapped directly into the VM and cannot be protected by +copy-back serialization. + +Inner-instruction growth uses the caller's original length plus the permitted +increase. Deprecated loaders reserve only the original length. Any account-region +layout change must update CPI pointer checks, region replacement, and VM access +handling together. + +## Maintenance constraints + +- Preserve direct account-region mapping as the only runtime path. +- Preserve ABI v0 and ABI v1 metadata compatibility. +- Keep borrowed layout changes synchronized across account, transaction-context, + serialization, and mapping code. +- Preserve full compact-u16 parsing and checked `u32` transaction framing. +- Keep Magicblock construction and execution policy synchronized with + `agave-transaction-view`. +- Do not enable address lookup resolution without revisiting ingress, + sanitization, scheduling, and simulation together. +- Treat dirty markers and touched flags as the caller's writeback signal. +- Keep persistence, consensus, validator fee policy, and batch commit decisions + outside these runtime crates. diff --git a/solana/account/Cargo.toml b/solana/account/Cargo.toml new file mode 100644 index 00000000..9bf0208e --- /dev/null +++ b/solana/account/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "solana-account" + +authors = { workspace = true } +description = "Solana Account type" +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +readme = "README.md" +repository = { workspace = true } +version = "4.3.1" + +[features] +bincode = ["dep:bincode", "dep:solana-sysvar", "serde"] +serde = [ + "bitflags/serde", + "dep:serde", + "dep:serde_bytes", + "serde/derive", + "serde/rc", + "solana-pubkey/serde" +] +testkit = ["bincode"] +wincode = ["bincode", "dep:wincode", "solana-pubkey/wincode"] + +[dependencies] +bincode = { workspace = true, optional = true } +bitflags = { workspace = true } +serde = { workspace = true, optional = true } +serde_bytes = { workspace = true, optional = true } +solana-account-info = { workspace = true } +solana-clock = { workspace = true } +solana-instruction-error = { workspace = true } +solana-pubkey = { workspace = true } +solana-sdk-ids = { workspace = true } +solana-sysvar = { workspace = true, features = ["bincode"], optional = true } +thiserror = { workspace = true } +wincode = { workspace = true, features = ["alloc"], optional = true } + +[dev-dependencies] +solana-account = { path = ".", features = ["testkit"] } +solana-pubkey = { workspace = true, features = ["std"] } + +[lints] +workspace = true diff --git a/solana/account/README.md b/solana/account/README.md new file mode 100644 index 00000000..2c8a76bf --- /dev/null +++ b/solana/account/README.md @@ -0,0 +1,49 @@ +# `solana-account` + +This fork defines the engine's account representation. `Account` is the +fully-owned compatibility form. `AccountSharedData` uses either a heap-owned +`Arc>` or a borrowed view into aligned external storage and records +field-level dirty markers. + +Equality compares core state and data bytes, ignoring storage form and dirty +markers. + +`AccountMode::mutable()` identifies modes intrinsically writable by user +programs. `AccountSharedData::mutable()` also accepts transient and closed +accounts when its mode dirty marker records the lifecycle transition in the +current transaction. +`AccountMode::authoritative()` separately identifies delegated, ephemeral, and +transient state that the engine owns and higher layers retain in persistent +storage. +`AccountSharedData::set_mode()` is the authoritative lifecycle transition +check: read-only and placeholder accounts may enter any mode except transient, +delegated accounts may enter transient, and transient accounts may resolve to +read-only. Ephemeral accounts may close. Reapplying the current mode is a clean +no-op; invalid mode and slot transitions return `AccountPatchError` with their +source and target context without changing the account. + +Slot patches must advance the stored slot. An equal slot is accepted only after +the mode genuinely changed in the same transaction. + +Full-account patch sequences cover non-flag fields, establish the exact data +length, and then write data in bounded chunks. MagicRoot finalization installs +the caller-supplied complete flag value without changing lamports. `StateFlags` +currently contains only `EXECUTABLE`; replacement freshness is enforced by the +caller rather than an account flag. + +## Borrowed layout + +| Part | Position | Contents | +| --- | --- | --- | +| header | start | sequence and image size | +| pubkey | after header | shared account pubkey | +| image A | after pubkey | core state and data | +| image B | after image A | core state and data | + +Borrowed buffers must be 8-byte aligned, match this layout, remain live, and have +unique mutable access for the duration of the borrow. The source may be an mmap, +arena, or test buffer. + +The sequence counter selects the active image. Mutation translates active state +into the shadow image; commit advances the sequence to publish it. Writes that +exceed borrowed capacity promote the account to owned storage. diff --git a/solana/account/src/account.rs b/solana/account/src/account.rs new file mode 100644 index 00000000..38fb014e --- /dev/null +++ b/solana/account/src/account.rs @@ -0,0 +1,260 @@ +use { + crate::{AccountSharedData, ReadableAccount, traits::debug_fmt}, + solana_account_info::AccountInfo, + solana_clock::Epoch, + solana_pubkey::Pubkey, + solana_sdk_ids::{ + bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, loader_v4, native_loader, + }, + std::{cell::RefCell, fmt, rc::Rc}, +}; + +/// An on-chain account with owned data and an explicit rent epoch. +#[repr(C)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize), serde(rename_all = "camelCase"))] +#[cfg_attr(feature = "wincode", derive(wincode::SchemaRead, wincode::SchemaWrite))] +#[derive(PartialEq, Eq, Clone, Default)] +pub struct Account { + /// Lamports in the account. + pub lamports: u64, + /// Data held in the account. + #[cfg_attr(feature = "serde", serde(with = "serde_bytes"))] + pub data: Vec, + /// The program that owns this account. + pub owner: Pubkey, + /// Whether the account contains executable program data. + pub executable: bool, + /// The epoch at which this account next owes rent. + pub rent_epoch: Epoch, +} + +#[cfg(feature = "serde")] +mod account_serialize { + use { + crate::ReadableAccount, + serde::{Serialize, ser::Serializer}, + solana_clock::Epoch, + solana_pubkey::Pubkey, + }; + + #[repr(C)] + #[derive(serde::Serialize)] + #[serde(rename_all = "camelCase")] + /// Serialization shape shared by `Account` and `AccountSharedData`. + struct Account<'a> { + lamports: u64, + #[serde(with = "serde_bytes")] + data: &'a [u8], + owner: &'a Pubkey, + executable: bool, + rent_epoch: Epoch, + } + + /// Serializes any readable account using the canonical `Account` layout. + pub(crate) fn serialize_account( + account: &impl ReadableAccount, + serializer: S, + ) -> Result + where + S: Serializer, + { + let account = Account { + lamports: account.lamports(), + data: account.data(), + owner: account.owner(), + executable: account.executable(), + rent_epoch: account.rent_epoch(), + }; + account.serialize(serializer) + } +} + +#[cfg(feature = "serde")] +impl serde::ser::Serialize for Account { + fn serialize(&self, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + account_serialize::serialize_account(self, serializer) + } +} + +#[cfg(feature = "serde")] +impl serde::ser::Serialize for AccountSharedData { + fn serialize(&self, serializer: S) -> Result + where + S: serde::ser::Serializer, + { + account_serialize::serialize_account(self, serializer) + } +} + +impl From for Account { + fn from(other: AccountSharedData) -> Self { + Self { + lamports: other.lamports(), + data: other.data().to_vec(), + owner: *other.owner(), + executable: other.executable(), + rent_epoch: other.rent_epoch(), + } + } +} + +impl fmt::Debug for Account { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_fmt(self, f, |_| {}) + } +} + +impl fmt::Debug for AccountSharedData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + debug_fmt(self, f, |f| { + f.field("slot", &self.slot()) + .field("mode", &self.mode) + .field("flags", self.flags()) + .field("dirty", self.markers()); + }) + } +} + +impl Account { + /// Builds an account from its exact field set. + /// + /// Used by the constructors to keep the owned layout in one place. + fn from_parts( + lamports: u64, + data: Vec, + owner: Pubkey, + executable: bool, + rent_epoch: Epoch, + ) -> Self { + Self { + lamports, + data, + owner, + executable, + rent_epoch, + } + } + + /// Creates a new account with zero-filled data. + pub fn new(lamports: u64, space: usize, owner: &Pubkey) -> Self { + Self::new_rent_epoch(lamports, space, owner, Epoch::default()) + } + + /// Creates a new account wrapped in a `RefCell`. + pub fn new_ref(lamports: u64, space: usize, owner: &Pubkey) -> Rc> { + Rc::new(RefCell::new(Self::new(lamports, space, owner))) + } + + /// Creates a new account whose data is the serialized state. + #[cfg(feature = "bincode")] + pub fn new_data( + lamports: u64, + state: &T, + owner: &Pubkey, + ) -> Result { + let data = bincode::serialize(state)?; + Ok(Self::from_parts( + lamports, + data, + *owner, + false, + Epoch::default(), + )) + } + + /// Creates a new serialized account wrapped in a `RefCell`. + #[cfg(feature = "bincode")] + pub fn new_ref_data( + lamports: u64, + state: &T, + owner: &Pubkey, + ) -> Result, bincode::Error> { + Self::new_data(lamports, state, owner).map(RefCell::new) + } + + /// Creates a new account with fixed space and serialized state. + #[cfg(feature = "bincode")] + pub fn new_data_with_space( + lamports: u64, + state: &T, + space: usize, + owner: &Pubkey, + ) -> Result { + let mut account = Self::new(lamports, space, owner); + crate::codec::serialize_data(&mut account, state)?; + Ok(account) + } + + /// Creates a new fixed-size serialized account wrapped in a `RefCell`. + #[cfg(feature = "bincode")] + pub fn new_ref_data_with_space( + lamports: u64, + state: &T, + space: usize, + owner: &Pubkey, + ) -> Result, bincode::Error> { + Self::new_data_with_space(lamports, state, space, owner).map(RefCell::new) + } + + /// Creates a new account with an explicit rent epoch. + pub fn new_rent_epoch(lamports: u64, space: usize, owner: &Pubkey, rent_epoch: Epoch) -> Self { + Self::from_parts(lamports, vec![0; space], *owner, false, rent_epoch) + } + + /// Deserializes the account data as `T`. + #[cfg(feature = "bincode")] + pub fn deserialize_data(&self) -> Result { + crate::codec::deserialize_data(self) + } + + /// Serializes `state` into the existing account data buffer. + #[cfg(feature = "bincode")] + pub fn serialize_data(&mut self, state: &T) -> Result<(), bincode::Error> { + crate::codec::serialize_data(self, state) + } +} + +impl solana_account_info::Account for Account { + fn get(&mut self) -> (&mut u64, &mut [u8], &Pubkey, bool) { + ( + &mut self.lamports, + &mut self.data, + &self.owner, + self.executable, + ) + } +} + +/// Builds `AccountInfo` values for accounts and signer bits. +/// +/// The returned infos borrow the provided accounts directly. +pub fn create_is_signer_account_infos<'a>( + accounts: &'a mut [(&'a Pubkey, bool, &'a mut Account)], +) -> Vec> { + accounts + .iter_mut() + .map(|(key, is_signer, account)| { + AccountInfo::new( + key, + *is_signer, + false, + &mut account.lamports, + &mut account.data, + &account.owner, + account.executable, + ) + }) + .collect() +} + +/// Owners that imply the account contains a loaded program. +pub const PROGRAM_OWNERS: &[Pubkey] = &[ + native_loader::id(), + bpf_loader_upgradeable::id(), + bpf_loader::id(), + bpf_loader_deprecated::id(), + loader_v4::id(), +]; diff --git a/solana/account/src/codec.rs b/solana/account/src/codec.rs new file mode 100644 index 00000000..c8aa14cd --- /dev/null +++ b/solana/account/src/codec.rs @@ -0,0 +1,23 @@ +//! Shared bincode helpers for account data. + +use serde::{Serialize, de::DeserializeOwned}; + +use crate::{ReadableAccount, WritableAccount}; + +/// Deserializes typed state from an account data slice. +pub(crate) fn deserialize_data( + account: &U, +) -> Result { + bincode::deserialize(account.data()) +} + +/// Serializes typed state into an existing account data buffer. +pub(crate) fn serialize_data( + account: &mut U, + state: &T, +) -> Result<(), bincode::Error> { + if bincode::serialized_size(state)? > account.data().len() as u64 { + return Err(Box::new(bincode::ErrorKind::SizeLimit)); + } + bincode::serialize_into(account.data_as_mut_slice(), state) +} diff --git a/solana/account/src/cow/borrowed.rs b/solana/account/src/cow/borrowed.rs new file mode 100644 index 00000000..e41fb510 --- /dev/null +++ b/solana/account/src/cow/borrowed.rs @@ -0,0 +1,324 @@ +//! Raw layout used by the borrowed zero-copy account view. +//! +//! The buffer is 8-byte aligned and contains a header followed by two images. +//! `AccountHeader::sequence` selects the active image; `translate` copies it to the shadow +//! image, `reset` repoints the view to the active image, `commit` publishes the shadow image, +//! and `rollback` undoes that publication by decrementing the sequence counter. + +#![allow(unsafe_op_in_unsafe_fn)] + +use std::{ + ops::{Deref, DerefMut}, + ptr::NonNull, + slice, + sync::atomic::{AtomicU32, Ordering::*}, +}; + +use solana_pubkey::Pubkey; + +use super::owned::OwnedAccount; +use super::{ALIGNMENT, AccountCore, STORAGE_UNIT, StorageUnit}; + +/// Fixed bytes in one image after the shared pubkey prefix: core and data header. +pub(super) const STATIC_SIZE: usize = size_of::() + size_of::(); +/// Storage-unit offset from the header to the first image payload, including the pubkey prefix. +pub(super) const IMAGE_OFFSET: usize = + (size_of::() + size_of::()) / STORAGE_UNIT; + +/// Header that prefixes a double-allocation borrowed account buffer. +#[repr(C, align(8))] +pub(crate) struct AccountHeader { + /// Sequence counter; parity selects the active image. + pub(crate) sequence: AtomicU32, + /// Image size measured in `AccountHeader` units. + pub(crate) space: u32, +} + +impl AccountHeader { + /// Creates a header for one image size in storage units. + pub(crate) fn new(space: u32) -> Self { + // `space` stays in storage units so the active + // image can be indexed with one multiply. + Self { sequence: 0.into(), space } + } +} + +/// Pointer arithmetic relies on these size and alignment invariants. +const _: () = assert!(size_of::() == ALIGNMENT); +const _: () = assert!(size_of::() == STORAGE_UNIT); +const _: () = assert!((size_of::() + STORAGE_UNIT) / ALIGNMENT == IMAGE_OFFSET); + +/// Borrowed zero-copy account view into an aligned external buffer. +#[derive(Eq, PartialEq)] +pub struct BorrowedAccount { + /// Header pointer for the borrowed buffer. + pub(crate) header: NonNull, + /// Pointer to the active image's account core. + pub(crate) core: NonNull, + /// Borrowed data bytes for the active image. + pub(crate) data: DataSlice, + /// Sequence used to select this view's image. + pub(crate) version: u32, +} + +/// Returns the byte offset for the active or shadow image. +#[inline] +fn offset(space: u32, sequence: u32, active: bool) -> usize { + // Even sequence => image A is active, odd sequence => image B is active. + let even = sequence.is_multiple_of(2); + // Flip to the shadow image when `active` does not match the current parity. + let step = (active ^ even) as u32; + (step * space) as usize + IMAGE_OFFSET +} + +impl BorrowedAccount { + /// Returns the sequence value that selects the active image. + pub(crate) fn sequence(&self) -> u32 { + // SAFETY: borrowed account headers live for the account view. + unsafe { self.header.as_ref() }.sequence.load(Acquire) + } + /// Returns the total borrowed span in `StorageUnit`s. + /// + /// # Safety + /// + /// `ptr` must point to a valid borrowed buffer created by + /// [`OwnedAccount::serialize`]. + pub unsafe fn span(ptr: NonNull) -> u32 { + let space = ptr.cast::().as_ref().space; + space * 2 + IMAGE_OFFSET as u32 + } + + /// Reads the account's pubkey stored in the image prefix. + /// + /// # Safety + /// + /// `ptr` must point to a valid borrowed buffer created by + /// [`OwnedAccount::serialize`]. + pub unsafe fn pubkey(ptr: NonNull) -> Pubkey { + *ptr.add(1).cast().as_ref() + } + + /// Builds a borrowed account view from an aligned account buffer. + /// + /// # Safety + /// + /// `buffer` must be 8-byte aligned and point to a valid borrowed account + /// buffer whose first bytes are the account header, followed by two + /// image-sized payloads. The active image is selected from the header + /// sequence parity. + pub unsafe fn init(buffer: NonNull) -> Self { + let header = buffer.cast::(); + let version = header.as_ref().sequence.load(Acquire); + let offset = offset(header.as_ref().space, version, true); + + let core = header.add(offset).cast(); + let data = DataSlice::init(core.add(1).cast()); + + Self { header, core, data, version } + } + + /// Copies the active image into the shadow image and switches to it. + /// + /// # Safety + /// + /// The borrowed image must still be the one selected by `init`. + pub unsafe fn translate(&mut self) { + let offset = offset(self.header.as_ref().space, self.version, false); + + // Copy bytes in bulk from active image to the shadow + let dst = self.header.add(offset).cast(); + let src = self.core.cast::(); + if src == dst { + return; + } + let count = self.header.as_ref().space as usize; + dst.copy_from_nonoverlapping(src, count); + // Switch the pointers to the shadow view + self.core = dst.cast(); + self.data = DataSlice::init(self.core.add(1).cast()); + } + + /// Publishes the shadow image if it was prepared against the current sequence. + pub fn commit(&self) { + // SAFETY: the header is part of the borrowed buffer for the lifetime of `self`. + let header = unsafe { self.header.as_ref() }; + let shadow = unsafe { + self.header.add(offset(header.space, self.version, false)).cast::() + }; + if self.core != shadow { + return; + } + let next = self.version.wrapping_add(1); + let _ = header.sequence.compare_exchange(self.version, next, Release, Relaxed); + } + + /// Repoints this view to the currently active image without copying data. + /// + /// # Safety + /// + /// The header must remain live, and `self` must be a view previously produced + /// by [`Self::init`] or [`Self::translate`] for that borrowed buffer. + pub unsafe fn reset(&mut self) { + self.version = self.header.as_ref().sequence.load(Acquire); + let offset = offset(self.header.as_ref().space, self.version, true); + self.core = self.header.add(offset).cast(); + self.data = DataSlice::init(self.core.add(1).cast()); + } + + /// Undoes the latest commit, by adjusting the sequence counter + /// + /// # Safety + /// + /// Call this only after `commit` to avoid data corruption; + pub unsafe fn rollback(&self) { + // SAFETY: the header is part of the borrowed buffer for the lifetime of `self`. + unsafe { self.header.as_ref().sequence.fetch_sub(1, Release) }; + } + + /// Returns the owner pubkey from the active image. + pub fn owner(&self) -> Pubkey { + // SAFETY: `core` points at a live `AccountCore` inside the borrowed buffer. + unsafe { self.core.as_ref() }.owner + } + + /// Returns the serialized active image bytes that define account state. + /// + /// The slice starts at `AccountCore`, includes the `DataHeader`, and stops + /// after initialized data. It excludes the shared header, pubkey prefix, + /// inactive shadow image, and spare data capacity. + pub fn storage(&self) -> &[u8] { + let len = STATIC_SIZE + self.data.len(); + // SAFETY: `core` points at the active image and `len` only covers its + // initialized state bytes: core, data header, and initialized data. + unsafe { slice::from_raw_parts(self.core.as_ptr().cast(), len) } + } +} + +impl From<&BorrowedAccount> for OwnedAccount { + fn from(value: &BorrowedAccount) -> Self { + Self { + // SAFETY: `BorrowedAccount` guarantees `core` points at a live account + // header inside the borrowed buffer for the lifetime of the borrow. + core: *unsafe { value.core.as_ref() }, + data: value.data.deref().to_vec().into(), + } + } +} + +/// Mutable byte slice backed by a borrowed account buffer. +#[derive(Clone, Eq, PartialEq)] +pub(crate) struct DataSlice { + /// Header carrying length and capacity. + header: NonNull, + /// Pointer to the first data byte. + ptr: NonNull, +} + +/// Data header stored immediately before the raw byte slice. +#[repr(C)] +pub(crate) struct DataHeader { + /// Initialized data length. + len: u32, + /// Total writable capacity. + cap: u32, +} + +impl DataHeader { + /// Creates a data header for one image. + pub(crate) fn new(len: u32, allocation: u32) -> Self { + // `cap` is the writable tail after `AccountCore` and `DataHeader`. + let cap = (allocation as usize * STORAGE_UNIT - STATIC_SIZE) as u32; + Self { len, cap } + } +} + +impl DataSlice { + /// Builds a borrowed slice from a data header. + /// + /// # Safety + /// + /// `header` must point at a valid `DataHeader` followed by initialized data. + unsafe fn init(header: NonNull) -> Self { + let ptr = header.add(1).cast(); + Self { header, ptr } + } + + /// Returns the initialized byte length. + pub(crate) fn len(&self) -> usize { + // SAFETY: `header` points at the live data header for this borrowed slice. + let header = unsafe { self.header.as_ref() }; + header.len.min(header.cap) as usize + } + + /// Returns the total writable capacity. + pub(crate) fn capacity(&self) -> usize { + // SAFETY: `header` points at the live data header for this borrowed slice. + let header = unsafe { self.header.as_ref() }; + header.cap as usize + } + + /// Returns the remaining writable capacity. + pub(crate) fn spare(&self) -> usize { + self.capacity() - self.len() + } + + /// Resizes the initialized range in place. + /// + /// # Safety + /// + /// `len` must not exceed the borrowed capacity. + pub(crate) unsafe fn resize(&mut self, len: usize, val: u8) { + let prev = self.len(); + debug_assert!(prev <= self.capacity()); + debug_assert!(len <= self.capacity()); + let delta = len.saturating_sub(prev); + if delta > 0 { + self.ptr.as_ptr().add(prev).write_bytes(val, delta); + } + self.header.as_mut().len = len as u32; + } + + /// Appends bytes in place. + /// + /// # Safety + /// + /// `data` must fit in the remaining borrowed capacity and not overlap. + pub(crate) unsafe fn extend(&mut self, data: &[u8]) { + let len = self.len(); + let dst = self.ptr.as_ptr().add(len); + dst.copy_from_nonoverlapping(data.as_ptr(), data.len()); + self.header.as_mut().len += data.len() as u32; + } + + /// Replaces the initialized bytes in place. + /// + /// # Safety + /// + /// `data` must fit in the borrowed capacity and not overlap. + pub(crate) unsafe fn set(&mut self, data: &[u8]) { + self.ptr.as_ptr().copy_from_nonoverlapping(data.as_ptr(), data.len()); + self.header.as_mut().len = data.len() as u32; + } +} + +impl Deref for DataSlice { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { + // SAFETY: `len` bytes from `ptr` are initialized account data owned by + // the borrowed buffer described by this `DataSlice`. + unsafe { slice::from_raw_parts(self.ptr.as_ptr(), self.len()) } + } +} + +impl DerefMut for DataSlice { + fn deref_mut(&mut self) -> &mut Self::Target { + // SAFETY: the borrowed buffer grants unique mutable access through this borrow. + unsafe { slice::from_raw_parts_mut(self.ptr.as_ptr(), self.len()) } + } +} + +// SAFETY: `BorrowedAccount` points into external storage and only exposes +// shared reads unless the caller holds `&mut self`; moving the view to another +// thread does not weaken the buffer lifetime and aliasing requirements. +unsafe impl Send for BorrowedAccount {} diff --git a/solana/account/src/cow/mod.rs b/solana/account/src/cow/mod.rs new file mode 100644 index 00000000..2162c6fa --- /dev/null +++ b/solana/account/src/cow/mod.rs @@ -0,0 +1,708 @@ +//! Copy-on-write account data with zero-copy access to aligned external storage. +//! +//! `borrowed` defines the raw buffer layout and `owned` holds the heap-backed form. +#![allow(unsafe_op_in_unsafe_fn)] + +mod borrowed; +mod owned; + +pub use borrowed::BorrowedAccount; +pub use owned::{AccountBuilder, OwnedAccount}; + +use crate::{Account, ReadableAccount, WritableAccount, patch::AccountPatchError}; +use solana_clock::{Epoch, Slot}; +use solana_pubkey::Pubkey; +use std::{ + cell::RefCell, + ops::{Deref, DerefMut}, + rc::Rc, + sync::Arc, +}; + +use CoWAccount::*; + +/// Borrowed buffers must be aligned to this many bytes. +pub const ALIGNMENT: usize = 8; +/// Bytes in one storage unit. +pub const STORAGE_UNIT: usize = size_of::(); +/// Minimum addressable storage unit for borrowed account images. +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct StorageUnit(pub u64); + +/// Shared account data that borrows directly from an aligned external buffer +/// until a write requires promotion to owned heap storage. +/// +/// Higher layers use `mutable()` to enforce transaction write permissions. +#[cfg_attr(feature = "serde", derive(serde::Deserialize), serde(from = "Account"))] +#[derive(Clone, Default)] +pub struct AccountSharedData { + /// Backing storage, borrowed until promotion or direct construction. + pub(crate) cow: CoWAccount, + /// Fields changed through the writable APIs. + pub(crate) dirty: DirtyMarkers, +} + +/// Core account state shared by the borrowed and owned representations. +#[repr(C)] +#[derive(Clone, Copy, Default, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct AccountCore { + /// Lamport balance. + pub(crate) lamports: u64, + /// Account owner. + pub(crate) owner: Pubkey, + /// On-chain slot, at which the account was cloned. + pub(crate) slot: Slot, + /// Mutually exclusive mode of existence for the account. + pub(crate) mode: AccountMode, + /// Account state modifier flags. + pub(crate) flags: StateFlags, + /// Reserved bytes that make the serialized representation deterministic. + _padding: [u8; 6], +} + +impl Deref for AccountSharedData { + type Target = AccountCore; + + fn deref(&self) -> &Self::Target { + match &self.cow { + Borrowed(account) => { + // SAFETY: `BorrowedAccount` owns the invariant that `core` points at + // a live `AccountCore` inside the borrowed buffer. + unsafe { account.core.as_ref() } + } + Owned(account) => &account.core, + } + } +} + +impl DerefMut for AccountSharedData { + fn deref_mut(&mut self) -> &mut Self::Target { + match &mut self.cow { + Borrowed(account) => { + // SAFETY: `&mut self` guarantees unique access to the borrowed image. + unsafe { account.core.as_mut() } + } + Owned(account) => &mut account.core, + } + } +} + +impl PartialEq for AccountSharedData { + fn eq(&self, other: &Self) -> bool { + self.deref() == other.deref() && self.cow.data() == other.cow.data() + } +} + +impl Eq for AccountSharedData {} + +impl AccountSharedData { + /// Returns a reference to the inner copy-on-write representation. + pub fn cow(&self) -> &CoWAccount { + &self.cow + } + + /// Returns mutable access to the inner copy-on-write representation. + pub fn cow_mut(&mut self) -> &mut CoWAccount { + &mut self.cow + } + + /// Returns the account's on-chain slot. + pub fn slot(&self) -> Slot { + self.slot + } + + /// Copies a clean borrowed image into the shadow buffer before mutation. + pub fn translate(&mut self) { + if self.dirty() { + return; + } + if let Borrowed(ref mut acc) = self.cow { + // SAFETY: this runs before the first dirty marker, so the borrowed + // view still points at the active image selected by `init`. + unsafe { acc.translate() }; + } + } + + /// Returns an owned copy of the current account state. + pub fn owned(&self) -> OwnedAccount { + match self.cow() { + Borrowed(a) => a.into(), + Owned(a) => a.clone(), + } + } + + /// Returns whether the current transaction may leave the account modified. + /// + /// Mutable modes are always accepted. `Transient` and `Closed` are accepted + /// only when this transaction performed the corresponding mode transition. + pub fn mutable(&self) -> bool { + self.mode.mutable() + || matches!(self.mode, AccountMode::Transient | AccountMode::Closed) + && self.dirty.contains(DirtyMarkers::MODE) + } + + /// Returns the account's exact lifecycle mode. + pub fn mode(&self) -> AccountMode { + self.mode + } + + /// Returns `true` when the account is in `mode`. + pub fn is(&self, mode: AccountMode) -> bool { + self.mode == mode + } + + /// Returns the account modifier flags. + pub fn flags(&self) -> &StateFlags { + &self.flags + } + + /// Returns the dirty-field markers. + pub fn markers(&self) -> &DirtyMarkers { + &self.dirty + } + + /// Marks the data buffer as modified. + pub(crate) fn mark_data_dirty(&mut self) { + self.dirty.insert(DirtyMarkers::DATA); + } + + /// Returns `true` when the owned buffer has more than one strong reference. + pub fn is_shared(&self) -> bool { + self.cow.is_shared() + } + + /// Returns `true` if any field has been modified. + pub fn dirty(&self) -> bool { + self.dirty.intersects(DirtyMarkers::all()) + } + + /// Returns the current data capacity. + pub fn capacity(&self) -> usize { + self.cow.capacity() + } + + /// Returns a shared owned copy of the current data bytes. + pub fn data_clone(&self) -> Arc> { + self.cow.data_clone() + } + + /// Resizes the account data. + pub fn resize(&mut self, len: usize, val: u8) { + self.translate(); + self.mark_data_dirty(); + self.cow.resize(len, val); + } + + /// Appends bytes to the account data. + pub fn extend_from_slice(&mut self, data: &[u8]) { + self.translate(); + self.mark_data_dirty(); + self.cow.extend_from_slice(data); + } + + /// Replaces the account data with the provided bytes. + pub fn set_data_from_slice(&mut self, data: &[u8]) { + self.translate(); + self.mark_data_dirty(); + self.cow.set_data_from_slice(data); + } + + /// Sets a legal account mode transition and marks the mode dirty. + /// + /// Reapplying the current mode is a no-op so callers can distinguish a + /// genuine lifecycle transition from an unchanged account. Invalid + /// transitions leave the account and its dirty markers unchanged. + pub fn set_mode(&mut self, to: AccountMode) -> Result<(), AccountPatchError> { + use AccountMode::*; + let from = self.mode; + + if from == to { + return Ok(()); + } + let allowed = match (from, to) { + (ReadOnly | Placeholder, to) => to != Transient, + (Delegated, Transient) | (Transient, ReadOnly) | (Ephemeral, Closed) => true, + _ => false, + }; + if !allowed { + Err(AccountPatchError::InvalidModeTransition { from, to })?; + } + self.translate(); + self.dirty.set(DirtyMarkers::MODE, true); + self.mode = to; + Ok(()) + } + + /// Writes bytes at `offset`, extending and zero-filling as needed. + pub(crate) fn set_data_at(&mut self, offset: usize, data: &[u8]) { + self.translate(); + self.mark_data_dirty(); + let len = self.data().len(); + if offset > len { + // Grow to `offset`, zero-filling the gap; the write below then + // appends `data` past it via `extend_from_slice`. + self.resize(offset, 0); + } + + // Write the overlap in place, then append any remaining tail. This + // keeps borrowed buffers on the fast path when the write fits. + let n = self.data().len().saturating_sub(offset).min(data.len()); + self.data_as_mut_slice()[offset..offset + n].copy_from_slice(&data[..n]); + self.extend_from_slice(&data[n..]); + } + + /// Sets a non-regressing account slot. + /// + /// Reapplying the current slot is accepted only after a genuine mode change + /// in the same transaction. Rejected transitions leave the account and its + /// dirty markers unchanged. + pub(crate) fn set_slot(&mut self, to: Slot) -> Result<(), AccountPatchError> { + let from = self.slot; + let mode_changed = self.dirty.contains(DirtyMarkers::MODE); + if to < from || (to == from && !mode_changed) { + Err(AccountPatchError::InvalidSlotTransition { from, to })?; + } + self.translate(); + self.dirty.set(DirtyMarkers::SLOT, true); + self.slot = to; + Ok(()) + } + + /// Replaces all state flags and marks them dirty when the value changes. + pub fn set_flags(&mut self, flags: StateFlags) { + if self.flags == flags { + return; + } + self.translate(); + self.dirty.set(DirtyMarkers::FLAGS, true); + self.flags = flags; + } + + /// Creates a new owned shared-data account with zero-filled data. + pub fn new(lamports: u64, space: usize, owner: &Pubkey) -> Self { + AccountBuilder::default() + .lamports(lamports) + .data(vec![0; space]) + .owner(*owner) + .build() + } + /// Creates a new shared-data account wrapped in a `RefCell`. + pub fn new_ref(lamports: u64, space: usize, owner: &Pubkey) -> Rc> { + Rc::new(RefCell::new(Self::new(lamports, space, owner))) + } + + /// Creates a new account with serialized data. + #[cfg(feature = "bincode")] + pub fn new_data( + lamports: u64, + state: &T, + owner: &Pubkey, + ) -> Result { + let data = bincode::serialize(state)?; + Ok(Self::create_from_existing_shared_data( + lamports, + Arc::new(data), + *owner, + false, + Epoch::default(), + )) + } + + /// Creates a new serialized account wrapped in a `RefCell`. + #[cfg(feature = "bincode")] + pub fn new_ref_data( + lamports: u64, + state: &T, + owner: &Pubkey, + ) -> Result, bincode::Error> { + Self::new_data(lamports, state, owner).map(RefCell::new) + } + + /// Creates a new fixed-size account with serialized data. + #[cfg(feature = "bincode")] + pub fn new_data_with_space( + lamports: u64, + state: &T, + space: usize, + owner: &Pubkey, + ) -> Result { + let mut account = Self::new(lamports, space, owner); + crate::codec::serialize_data(&mut account, state)?; + Ok(account) + } + + /// Creates a new fixed-size serialized account wrapped in a `RefCell`. + #[cfg(feature = "bincode")] + pub fn new_ref_data_with_space( + lamports: u64, + state: &T, + space: usize, + owner: &Pubkey, + ) -> Result, bincode::Error> { + Self::new_data_with_space(lamports, state, space, owner).map(RefCell::new) + } + + /// Creates a new shared-data account. + /// + /// `rent_epoch` is ignored because this type does not store it. + pub fn new_rent_epoch(lamports: u64, space: usize, owner: &Pubkey, _: Epoch) -> Self { + Self::new(lamports, space, owner) + } + + /// Deserializes the account data as `T`. + #[cfg(feature = "bincode")] + pub fn deserialize_data(&self) -> Result { + crate::codec::deserialize_data(self) + } + + /// Serializes `state` into the existing account data buffer. + #[cfg(feature = "bincode")] + pub fn serialize_data(&mut self, state: &T) -> Result<(), bincode::Error> { + crate::codec::serialize_data(self, state) + } + + /// Creates an owned shared-data account from existing shared bytes. + /// + /// `rent_epoch` is ignored because this type does not store it. + pub fn create_from_existing_shared_data( + lamports: u64, + data: Arc>, + owner: Pubkey, + executable: bool, + _: Epoch, + ) -> Self { + AccountBuilder::default() + .lamports(lamports) + .data(data) + .owner(owner) + .executable(executable) + .build() + } +} + +bitflags::bitflags! { + /// Account state modifier flags. + #[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub struct StateFlags: u8 { + /// Executable account data. + const EXECUTABLE = 1 << 0; + } + + /// Bits that record which fields changed through `AccountSharedData`. + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] + pub struct DirtyMarkers: u8 { + /// Owner changed. + const OWNER = 1 << 0; + /// Lamports changed. + const LAMPORTS = 1 << 1; + /// Mode changed. + const MODE = 1 << 2; + /// State flags changed. + const FLAGS = 1 << 3; + /// Slot changed. + const SLOT = 1 << 4; + /// Data bytes changed. + const DATA = 1 << 5; + } +} + +/// `wincode` codec for `StateFlags`, which is a `bitflags!` newtype and so +/// cannot use the derives. Routed through `bincode`/`serde`, which encodes the +/// single bits byte identically to a plain `u8`. +#[cfg(feature = "wincode")] +const _: () = { + use core::mem::MaybeUninit; + use wincode::{ + ReadError, ReadResult, SchemaRead, SchemaWrite, TypeMeta, WriteError, WriteResult, + config::ConfigCore, + io::{Reader, Writer}, + }; + + // SAFETY: encodes exactly one byte; matches `TYPE_META` / `size_of`. + unsafe impl SchemaWrite for StateFlags { + type Src = StateFlags; + const TYPE_META: TypeMeta = TypeMeta::Static { size: 1, zero_copy: false }; + + fn size_of(_: &Self::Src) -> WriteResult { + Ok(1) + } + + fn write(mut writer: impl Writer, src: &Self::Src) -> WriteResult<()> { + let bytes = bincode::serialize(src).map_err(|_| WriteError::Custom("StateFlags"))?; + writer.write(&bytes)?; + Ok(()) + } + } + + // SAFETY: consumes exactly one byte; matches `TYPE_META`. + unsafe impl<'de, C: ConfigCore> SchemaRead<'de, C> for StateFlags { + type Dst = StateFlags; + const TYPE_META: TypeMeta = TypeMeta::Static { size: 1, zero_copy: false }; + + fn read(mut reader: impl Reader<'de>, dst: &mut MaybeUninit) -> ReadResult<()> { + let bytes = reader.take_array::<1>()?; + dst.write(bincode::deserialize(&bytes).map_err(|_| ReadError::Custom("StateFlags"))?); + Ok(()) + } + } +}; + +/// Backing storage for `AccountSharedData`. +#[derive(PartialEq, Eq)] +pub enum CoWAccount { + /// Borrowed image, a view into static backing buffer. + Borrowed(BorrowedAccount), + /// Heap-owned image. + Owned(OwnedAccount), +} + +impl Clone for CoWAccount { + fn clone(&self) -> Self { + match self { + Borrowed(acc) => Self::Owned(acc.into()), + Owned(acc) => Self::Owned(acc.clone()), + } + } +} + +impl CoWAccount { + /// Promotes borrowed storage to the owned form. + pub(crate) fn promote(&mut self) { + let Self::Borrowed(account) = self else { + return; + }; + *self = Self::Owned(account.deref().into()); + } + + /// Returns the current data slice. + pub(crate) fn data(&self) -> &[u8] { + match self { + Self::Borrowed(account) => &account.data, + Self::Owned(account) => &account.data, + } + } + + /// Returns `true` when the heap buffer has multiple owners. + pub(crate) fn is_shared(&self) -> bool { + match self { + Self::Borrowed(_) => false, + Self::Owned(account) => Arc::strong_count(&account.data) > 1, + } + } + + /// Returns the current data capacity. + pub(crate) fn capacity(&self) -> usize { + match self { + Self::Borrowed(account) => account.data.capacity(), + Self::Owned(account) => account.data.capacity(), + } + } + + /// Returns a shared owned copy of the current data bytes. + pub(crate) fn data_clone(&self) -> Arc> { + match self { + Self::Borrowed(account) => Arc::new(account.data.to_vec()), + Self::Owned(account) => Arc::clone(&account.data), + } + } + + /// Returns mutable data, promoting borrowed storage only when needed. + pub(crate) fn data_mut(&mut self) -> &mut [u8] { + match self { + Self::Borrowed(account) => &mut account.data, + Self::Owned(account) => Arc::>::make_mut(&mut account.data).as_mut_slice(), + } + } + + /// Reserves additional space for the account data. + pub fn reserve(&mut self, additional: usize) { + if let Self::Borrowed(a) = self + && a.data.spare() >= additional + { + return; + } + self.promote(); + if let Self::Owned(account) = self { + Arc::make_mut(&mut account.data).reserve(additional); + } + } + + /// Resizes the account data. + pub(crate) fn resize(&mut self, len: usize, val: u8) { + if let Self::Borrowed(a) = self + && len <= a.data.capacity() + { + // SAFETY: this stays in the borrowed image only while the resized + // range fits within the borrowed capacity. + unsafe { a.data.resize(len, val) }; + return; + } + + self.promote(); + if let Self::Owned(account) = self { + Arc::make_mut(&mut account.data).resize(len, val); + } + } + + /// Appends bytes to the account data. + pub(crate) fn extend_from_slice(&mut self, data: &[u8]) { + self.reserve(data.len()); + + match self { + Self::Borrowed(account) => { + // SAFETY: `reserve` keeps the borrowed image only when the appended + // bytes fit in the remaining borrowed capacity. + unsafe { account.data.extend(data) }; + } + Self::Owned(account) => Arc::make_mut(&mut account.data).extend_from_slice(data), + } + } + + /// Replaces the account data with the provided bytes. + pub(crate) fn set_data_from_slice(&mut self, data: &[u8]) { + let additional = data.len().saturating_sub(self.data().len()); + self.reserve(additional); + + match self { + Self::Borrowed(account) => { + // SAFETY: `reserve` keeps the borrowed image only when the + // replacement bytes fit in the borrowed capacity. + unsafe { account.data.set(data) }; + } + Self::Owned(account) => { + let data_buf = Arc::make_mut(&mut account.data); + data_buf.clear(); + data_buf.extend_from_slice(data); + } + } + } +} + +/// Mutually exclusive modes an account can occupy in the ephemeral rollup (ER). +#[derive(Clone, Copy, PartialEq, Eq, Default, Debug)] +#[repr(u8)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +#[cfg_attr(feature = "wincode", derive(wincode::SchemaRead, wincode::SchemaWrite))] +pub enum AccountMode { + /// Empty account (not found on chain) used to avoid frequent chain syncs. + #[default] + Placeholder = 0, + /// Not writable by users (exists on chain, but not delegated) + ReadOnly = 1, + /// Internal account used for sysvars, features, and precompiles. + System, + /// Account delegated to the current ER node instance. + Delegated, + /// Account that exists only inside the ER. + Ephemeral, + /// Temporary state during mode transitions (e.g. delegated -> readonly). + Transient, + /// Closed account that should be removed from storage. + Closed = 255, +} + +impl AccountMode { + /// Returns `true` for modes that may be mutated by user programs. + pub fn mutable(&self) -> bool { + use AccountMode::*; + matches!(self, Delegated | Ephemeral) + } + + /// Returns `true` for modes whose state is authoritative in this engine. + pub fn authoritative(&self) -> bool { + use AccountMode::*; + matches!(self, Delegated | Ephemeral | Transient) + } +} + +/// Read wrapper that retries borrowed account reads when a concurrent publish +/// changes the backing image. +pub struct AccountSeqLock { + account: AccountSharedData, + sequence: Option, +} + +impl AccountSeqLock { + /// Creates a read lock with the sequence that matches the current account view. + pub fn new(account: AccountSharedData) -> Self { + let mut sequence = None; + if let Borrowed(ref acc) = account.cow { + sequence.replace(acc.version); + } + Self { account, sequence } + } + + /// Runs `reader` against a stable account image. + /// + /// For borrowed accounts, the sequence is checked after the read. If a + /// writer published a new image meanwhile, the account view is reset to that + /// active image and the read is retried. + pub fn read(&mut self, reader: F) -> R + where + F: Fn(&AccountSharedData) -> R, + { + loop { + // sequence is always present for borrowed accounts + let pre = self.sequence.unwrap_or_default(); + let result = reader(&self.account); + match self.account.cow_mut() { + Borrowed(acc) => { + let post = acc.sequence(); + if pre == post { + return result; + } + // SAFETY: a changed sequence means the active image may have + // moved, so the borrowed view must be repointed before retrying. + unsafe { acc.reset() }; + self.sequence = Some(acc.version); + } + Owned(_) => return result, + } + } + } +} + +impl Default for CoWAccount { + fn default() -> Self { + Self::Owned(OwnedAccount::default()) + } +} + +/// Wraps an owned account in `AccountSharedData`. +impl From for AccountSharedData { + fn from(value: OwnedAccount) -> Self { + Self { + cow: Owned(value), + dirty: DirtyMarkers::default(), + } + } +} + +/// Wraps a borrowed account in `AccountSharedData`. +impl From for AccountSharedData { + fn from(value: BorrowedAccount) -> Self { + Self { + cow: Borrowed(value), + dirty: DirtyMarkers::default(), + } + } +} + +/// Converts a plain `Account` into shared data. +impl From for AccountSharedData { + fn from(value: Account) -> Self { + AccountBuilder::default() + .lamports(value.lamports) + .data(value.data) + .owner(value.owner) + .executable(value.executable) + .build() + } +} + +/// We only access AccountSharedData via transaction lock in the +/// execution layer or with a SeqLock semantics outside of execution +unsafe impl Sync for AccountSharedData {} diff --git a/solana/account/src/cow/owned.rs b/solana/account/src/cow/owned.rs new file mode 100644 index 00000000..5243fa8e --- /dev/null +++ b/solana/account/src/cow/owned.rs @@ -0,0 +1,177 @@ +use super::borrowed::{AccountHeader, DataHeader, STATIC_SIZE}; +use super::{ALIGNMENT, StateFlags}; +use crate::cow::AccountCore; +use crate::cow::borrowed::IMAGE_OFFSET; +use crate::{Account, AccountMode, AccountSharedData, StorageUnit}; +use solana_clock::Slot; +use solana_pubkey::Pubkey; +use std::{ptr::NonNull, sync::Arc}; + +/// Heap-backed account, used after promotion from borrowed or direct construction. +#[derive(Clone, Default, Eq, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct OwnedAccount { + /// Core account fields. + pub(crate) core: AccountCore, + /// Heap-owned data buffer. + pub(crate) data: Arc>, +} + +impl OwnedAccount { + /// Returns the exact storage units needed to serialize this account. + pub fn units(&self) -> u32 { + self.allocation() * 2 + IMAGE_OFFSET as u32 + } + + /// Returns the storage units needed for one image, rounded up to alignment. + fn allocation(&self) -> u32 { + (STATIC_SIZE + self.data.len()).div_ceil(ALIGNMENT) as u32 + } + + /// Writes the account into a buffer sized by `units`. + /// + /// # Safety + /// + /// `buf` must be exactly `units()` storage units long. + /// `pubkey` is written into the image prefix so borrowed iteration can + /// recover the full account key without consulting the index. + pub unsafe fn serialize(&self, buf: &mut [StorageUnit], pubkey: &Pubkey) { + let ptr = NonNull::new_unchecked(buf.as_mut_ptr()); + debug_assert_eq!(self.units() as usize, buf.len()); + + fn write(ptr: NonNull, v: T) -> NonNull { + // SAFETY: `serialize` requires a buffer sized for the full layout. + unsafe { + ptr.cast().write(v); + ptr.cast().add(1) + } + } + + let allocation = self.allocation(); + let ptr = write(ptr, AccountHeader::new(allocation)); + // The image prefix stores the account pubkey for later iteration. + let ptr = write(ptr, *pubkey); + let ptr = write(ptr, self.core); + let len = self.data.len(); + let ptr = write(ptr, DataHeader::new(len as u32, allocation)).cast(); + self.data.as_ptr().copy_to_nonoverlapping(ptr.as_ptr(), len); + } + + /// Tests the exact account mode without grouping modes by mutability. + pub fn is(&self, mode: AccountMode) -> bool { + self.core.mode == mode + } + + /// Returns the owner pubkey. + pub fn owner(&self) -> Pubkey { + self.core.owner + } + + /// Returns the lamport balance. + pub fn lamports(&self) -> u64 { + self.core.lamports + } + + /// Returns the account's exact lifecycle mode. + pub fn mode(&self) -> AccountMode { + self.core.mode + } + + /// Returns the account's on-chain slot. + pub fn slot(&self) -> u64 { + self.core.slot + } + + /// Returns the account modifier flags. + pub fn flags(&self) -> StateFlags { + self.core.flags + } + + /// Returns the account data. + pub fn data(&self) -> &[u8] { + &self.data + } +} + +/// Builder for an owned account representation. +/// +/// Use this when the account does not start from a borrowed external buffer. +#[derive(Default, Clone)] +pub struct AccountBuilder(OwnedAccount); + +impl AccountBuilder { + /// Sets the lamport balance. + pub fn lamports(mut self, lamports: u64) -> Self { + self.0.core.lamports = lamports; + self + } + + /// Sets the data buffer. + pub fn data(mut self, data: impl Into>>) -> Self { + self.0.data = data.into(); + self + } + + /// Sets the owner. + pub fn owner(mut self, owner: Pubkey) -> Self { + self.0.core.owner = owner; + self + } + + /// Sets the account persistence mode of the account + pub fn mode(mut self, mode: AccountMode) -> Self { + self.0.core.mode = mode; + self + } + + /// Sets the executable flag. + pub fn executable(mut self, executable: bool) -> Self { + self.0.core.flags.set(StateFlags::EXECUTABLE, executable); + self + } + + /// Sets the on chain slot. + pub fn slot(mut self, slot: Slot) -> Self { + self.0.core.slot = slot; + self + } + + /// Borrows the account under construction. + pub fn read(&self) -> &OwnedAccount { + &self.0 + } + + /// Finishes building the owned account. + pub fn build>(self) -> A { + self.0.into() + } +} + +impl From for OwnedAccount { + fn from(value: Account) -> Self { + AccountBuilder::default() + .lamports(value.lamports) + .data(value.data) + .owner(value.owner) + .executable(value.executable) + .build() + } +} + +impl From for OwnedAccount { + fn from(value: AccountBuilder) -> Self { + value.0 + } +} + +impl From for AccountBuilder { + fn from(value: Account) -> Self { + Self(value.into()) + } +} + +impl From for AccountBuilder { + fn from(value: AccountSharedData) -> Self { + Self(value.owned()) + } +} diff --git a/solana/account/src/lib.rs b/solana/account/src/lib.rs new file mode 100644 index 00000000..1d2cfc9c --- /dev/null +++ b/solana/account/src/lib.rs @@ -0,0 +1,33 @@ +#![cfg_attr(docsrs, feature(doc_cfg))] +#![doc = include_str!("../README.md")] + +mod account; +#[cfg(feature = "bincode")] +mod codec; +mod cow; +mod patch; +#[cfg(feature = "bincode")] +pub mod state_traits; +#[cfg(feature = "bincode")] +mod sysvar; +/// Test-only helpers for borrowed account buffers. +#[cfg(feature = "testkit")] +pub mod testkit; +mod traits; + +pub use account::{Account, PROGRAM_OWNERS, create_is_signer_account_infos}; +pub use cow::{ + ALIGNMENT, AccountBuilder, AccountMode, AccountSeqLock, AccountSharedData, BorrowedAccount, + CoWAccount, DirtyMarkers, OwnedAccount, STORAGE_UNIT, StateFlags, StorageUnit, +}; +pub use patch::{AccountFieldPatch, AccountPatchError}; +#[cfg(feature = "bincode")] +pub use sysvar::{ + DUMMY_INHERITABLE_ACCOUNT_FIELDS, InheritableAccountFields, create_account_for_test, + create_account_shared_data_for_test, create_account_shared_data_with_fields, + create_account_with_fields, from_account, to_account, +}; +pub use traits::{ReadableAccount, WritableAccount, accounts_equal}; + +#[cfg(test)] +mod tests; diff --git a/solana/account/src/patch.rs b/solana/account/src/patch.rs new file mode 100644 index 00000000..60a33579 --- /dev/null +++ b/solana/account/src/patch.rs @@ -0,0 +1,107 @@ +use core::fmt; + +use solana_clock::Slot; +use solana_pubkey::Pubkey; + +use crate::{AccountMode, AccountSharedData, OwnedAccount, WritableAccount}; + +const MAX_DATA_CHUNK_SIZE: usize = (u16::MAX - 256) as usize; + +/// Failure to apply an account lifecycle or ordering patch. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub enum AccountPatchError { + /// The requested account mode transition is not part of the lifecycle. + #[error("invalid account mode transition: {from:?} -> {to:?}")] + InvalidModeTransition { + /// Current account mode. + from: AccountMode, + /// Requested account mode. + to: AccountMode, + }, + /// The requested slot neither advances nor accompanies a mode transition. + #[error("invalid account slot transition: {from} -> {to}")] + InvalidSlotTransition { + /// Current account slot. + from: Slot, + /// Requested account slot. + to: Slot, + }, +} + +/// A single-field account patch. +#[cfg_attr(feature = "wincode", derive(wincode::SchemaRead, wincode::SchemaWrite))] +pub enum AccountFieldPatch { + /// Replaces the lamport balance. + Lamports(u64), + /// Replaces the owner. + Owner(Pubkey), + /// Writes bytes starting at `offset`, extending the account data if needed. + DataAt { + /// Byte offset into the current data buffer. + offset: usize, + /// Bytes to write. + data: Vec, + }, + /// Replaces the slot. + Slot(Slot), + /// Replaces the account mode. + Mode(AccountMode), + /// Resizes the data buffer to an exact length, zero-filling when growing. + DataLen(usize), +} + +impl AccountFieldPatch { + /// Applies this patch to `account`. + /// + /// The account methods mark dirtiness and preserve the writable invariants. + /// Invalid mode and slot transitions leave the account unchanged and return + /// their transition context. + pub fn apply(self, account: &mut AccountSharedData) -> Result<(), AccountPatchError> { + match self { + Self::Lamports(v) => account.set_lamports(v), + Self::Slot(v) => return account.set_slot(v), + Self::Owner(v) => account.set_owner(v), + Self::Mode(v) => return account.set_mode(v), + Self::DataAt { offset, data } => account.set_data_at(offset, &data), + Self::DataLen(len) => account.resize(len, 0), + } + Ok(()) + } + + /// Decomposes an owned account into the ordered sequence of patches that + /// reconstruct its non-flag fields: lamports, mode, slot, owner, the exact + /// data length, then the data in `MAX_DATA_CHUNK_SIZE`-sized chunks. + /// + /// Mode precedes slot so consumers can distinguish an equal-slot mode + /// transition from a duplicate replacement by inspecting the mode dirty + /// marker when they apply the slot patch. + pub fn sequence(account: OwnedAccount) -> Vec { + let mut sequence = Vec::with_capacity(6); + sequence.push(Self::Lamports(account.core.lamports)); + sequence.push(Self::Mode(account.core.mode)); + sequence.push(Self::Slot(account.core.slot)); + sequence.push(Self::Owner(account.core.owner)); + sequence.push(Self::DataLen(account.data.len())); + let mut offset = 0; + for data in account.data.chunks(MAX_DATA_CHUNK_SIZE) { + sequence.push(Self::DataAt { offset, data: data.into() }); + offset += data.len(); + } + sequence + } +} + +/// Concise, log-friendly rendering: scalar fields show their value, `DataAt` +/// shows only `offset+len` (never the raw bytes). +impl fmt::Debug for AccountFieldPatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Lamports(v) => write!(f, "lamports={v}"), + Self::Owner(v) => write!(f, "owner={v}"), + Self::Slot(v) => write!(f, "slot={v}"), + Self::Mode(v) => write!(f, "mode={v:?}"), + Self::DataAt { offset, data } => write!(f, "data@{offset}+{}", data.len()), + Self::DataLen(len) => write!(f, "data_len={len}"), + } + } +} diff --git a/solana/account/src/state_traits.rs b/solana/account/src/state_traits.rs new file mode 100644 index 00000000..4441f445 --- /dev/null +++ b/solana/account/src/state_traits.rs @@ -0,0 +1,65 @@ +//! Typed bincode access to account data. + +use { + crate::{AccountSharedData, ReadableAccount, WritableAccount}, + bincode::ErrorKind, + solana_instruction_error::InstructionError, + std::cell::Ref, +}; + +/// Reads and writes typed account state through a mutable account handle. +pub trait StateMut { + /// Deserializes the account data as `T`. + fn state(&self) -> Result; + + /// Serializes `state` into the existing account data buffer. + fn set_state(&mut self, state: &T) -> Result<(), InstructionError>; +} + +/// Deserializes typed state from account data. +/// +/// Invalid bytes map to `InstructionError::InvalidAccountData`. +fn state(account: &impl ReadableAccount) -> Result +where + T: serde::de::DeserializeOwned, +{ + crate::codec::deserialize_data(account).map_err(|_| InstructionError::InvalidAccountData) +} + +/// Serializes typed state into an existing account buffer. +/// +/// Oversized payloads map to `AccountDataTooSmall`; all other failures map to `GenericError`. +fn set_state(account: &mut impl WritableAccount, state: &T) -> Result<(), InstructionError> +where + T: serde::Serialize, +{ + crate::codec::serialize_data(account, state).map_err(|err| match *err { + ErrorKind::SizeLimit => InstructionError::AccountDataTooSmall, + _ => InstructionError::GenericError, + }) +} + +impl StateMut for A +where + A: ReadableAccount + WritableAccount, + T: serde::Serialize + serde::de::DeserializeOwned, +{ + fn state(&self) -> Result { + state(self) + } + fn set_state(&mut self, state: &T) -> Result<(), InstructionError> { + set_state(self, state) + } +} + +impl StateMut for Ref<'_, AccountSharedData> +where + T: serde::Serialize + serde::de::DeserializeOwned, +{ + fn state(&self) -> Result { + state(&**self) + } + fn set_state(&mut self, _state: &T) -> Result<(), InstructionError> { + Err(InstructionError::ReadonlyDataModified) + } +} diff --git a/solana/account/src/sysvar.rs b/solana/account/src/sysvar.rs new file mode 100644 index 00000000..ad108e3a --- /dev/null +++ b/solana/account/src/sysvar.rs @@ -0,0 +1,70 @@ +use { + crate::{Account, AccountSharedData, ReadableAccount, WritableAccount}, + solana_clock::{Epoch, INITIAL_RENT_EPOCH}, + solana_sysvar::SysvarSerialize, +}; + +/// Fields copied into test sysvar accounts. +pub type InheritableAccountFields = (u64, Epoch); + +/// Default lamports and rent epoch used by test sysvar account helpers. +pub const DUMMY_INHERITABLE_ACCOUNT_FIELDS: InheritableAccountFields = (1, INITIAL_RENT_EPOCH); + +/// Serializes a sysvar into account data and pads it to the declared size. +/// +/// Serialization failure falls back to zeroed bytes sized to `S::size_of()`. +/// That keeps the helper infallible while preserving the advertised layout. +fn account_data(sysvar: &S) -> Vec { + let mut data = bincode::serialize(sysvar).unwrap_or_default(); + data.resize(data.len().max(S::size_of()), 0); + data +} + +/// Creates an [`Account`] that contains a serialized sysvar value. +pub fn create_account_with_fields( + sysvar: &S, + (lamports, rent_epoch): InheritableAccountFields, +) -> Account { + Account { + lamports, + data: account_data(sysvar), + owner: solana_sdk_ids::sysvar::id(), + executable: false, + rent_epoch, + } +} + +/// Creates a test sysvar [`Account`]. +pub fn create_account_for_test(sysvar: &S) -> Account { + create_account_with_fields(sysvar, DUMMY_INHERITABLE_ACCOUNT_FIELDS) +} + +/// Creates an [`AccountSharedData`] that contains a serialized sysvar value. +pub fn create_account_shared_data_with_fields( + sysvar: &S, + fields: InheritableAccountFields, +) -> AccountSharedData { + AccountSharedData::from(create_account_with_fields(sysvar, fields)) +} + +/// Creates a test sysvar [`AccountSharedData`]. +pub fn create_account_shared_data_for_test(sysvar: &S) -> AccountSharedData { + create_account_shared_data_with_fields(sysvar, DUMMY_INHERITABLE_ACCOUNT_FIELDS) +} + +/// Deserializes a sysvar value from account data. +/// +/// Returns `None` on decode failure. +pub fn from_account(account: &T) -> Option { + bincode::deserialize(account.data()).ok() +} + +/// Serializes a sysvar value into account data. +/// +/// Returns `None` on encode failure. +pub fn to_account( + sysvar: &S, + account: &mut T, +) -> Option<()> { + bincode::serialize_into(account.data_as_mut_slice(), sysvar).ok() +} diff --git a/solana/account/src/testkit.rs b/solana/account/src/testkit.rs new file mode 100644 index 00000000..f78ba8de --- /dev/null +++ b/solana/account/src/testkit.rs @@ -0,0 +1,47 @@ +#![allow(clippy::expect_used)] + +use { + crate::{ + AccountBuilder, AccountSharedData, BorrowedAccount, OwnedAccount, ReadableAccount, + StorageUnit, + }, + solana_pubkey::Pubkey, + std::ptr::NonNull, +}; + +/// Builds a borrowed account image backed by serialized owned state. +pub fn borrowed_account_buffer(data: Vec, owner: Pubkey) -> Vec { + let pubkey = Pubkey::new_unique(); + let owned = AccountBuilder::default() + .lamports(1) + .data(data) + .owner(owner) + .build::(); + serialize_account_buffer(&owned, &pubkey) +} + +/// Serializes an owned account into the borrowed layout used by tests. +pub fn serialize_account_buffer(owned: &OwnedAccount, pubkey: &Pubkey) -> Vec { + let mut buf = vec![Default::default(); owned.units() as usize]; + // SAFETY: `buf` is sized from `units` and allocated with the alignment + // required by the borrowed account layout. + unsafe { owned.serialize(&mut buf, pubkey) }; + buf +} + +/// Builds a borrowed account view over a test buffer. +pub fn init_borrowed_account(buf: &mut [StorageUnit]) -> BorrowedAccount { + let ptr = NonNull::from(&mut buf[..]).cast(); + // SAFETY: test buffers are created with the borrowed account layout. + unsafe { BorrowedAccount::init(ptr) } +} + +/// Wraps a borrowed account test buffer in `AccountSharedData`. +pub fn borrowed_shared_data(buf: &mut [StorageUnit]) -> AccountSharedData { + AccountSharedData::from(init_borrowed_account(buf)) +} + +/// Reinitializes a borrowed test buffer and returns its active data image. +pub fn active_borrowed_data(buf: &mut [StorageUnit]) -> Vec { + borrowed_shared_data(buf).data().to_vec() +} diff --git a/solana/account/src/tests/account.rs b/solana/account/src/tests/account.rs new file mode 100644 index 00000000..4cf2e602 --- /dev/null +++ b/solana/account/src/tests/account.rs @@ -0,0 +1,504 @@ +use crate::{ + Account, AccountBuilder, AccountFieldPatch, AccountMode, AccountPatchError, AccountSeqLock, + AccountSharedData, BorrowedAccount, CoWAccount, DirtyMarkers, OwnedAccount, ReadableAccount, + StorageUnit, WritableAccount, accounts_equal, + testkit::{init_borrowed_account, serialize_account_buffer}, +}; +use bincode::ErrorKind; +use solana_clock::Epoch; +use solana_instruction_error::LamportsError; +use solana_pubkey::Pubkey; +use std::{ + cell::{Cell, RefCell}, + ptr::NonNull, +}; + +// Builds matching owned and shared accounts for baseline assertions. +fn make_two_accounts() -> (Pubkey, Account, AccountSharedData) { + let key = Pubkey::new_unique(); + let mut account = Account::new(1, 2, &key); + account.executable = true; + account.rent_epoch = Epoch::MAX; + + let mut shared = AccountSharedData::new(1, 2, &key); + shared.set_executable(true); + shared.set_rent_epoch(4); + + assert!(accounts_equal(&account, &shared)); + (key, account, shared) +} + +// Builds a borrowed account image backed by serialized owned state. +fn make_borrowed(data: Vec) -> (Vec, AccountSharedData) { + let pubkey = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let owned = AccountBuilder::default().lamports(5).data(data).owner(owner).build(); + let mut buf = serialize_account_buffer(&owned, &pubkey); + let borrowed = init_borrowed_account(&mut buf); + (buf, AccountSharedData::from(borrowed)) +} + +fn assert_add_err(mut account: T) { + assert!(matches!( + account.checked_add_lamports(u64::MAX), + Err(LamportsError::ArithmeticOverflow) + )); +} + +fn assert_sub_err(mut account: T) { + assert!(matches!( + account.checked_sub_lamports(u64::MAX), + Err(LamportsError::ArithmeticUnderflow) + )); +} + +fn assert_saturating_add( + mut account: T, + start: u64, + add: u64, + expected: u64, +) { + account.set_lamports(start); + account.saturating_add_lamports(add); + assert_eq!(account.lamports(), expected); +} + +fn assert_saturating_sub( + mut account: T, + start: u64, + sub: u64, + expected: u64, +) { + account.set_lamports(start); + account.saturating_sub_lamports(sub); + assert_eq!(account.lamports(), expected); +} + +#[test] +// Owner bytes should copy into both account representations identically. +fn test_account_data_copy_as_slice() { + let key2 = Pubkey::new_unique(); + let (_, mut account1, mut account2) = make_two_accounts(); + account1.copy_into_owner_from_slice(key2.as_ref()); + account2.copy_into_owner_from_slice(key2.as_ref()); + assert!(accounts_equal(&account1, &account2)); + assert_eq!(account1.owner(), &key2); +} + +#[test] +// set_data_from_slice should overwrite, grow, shrink, and preserve contents. +fn test_account_set_data_from_slice() { + let (_, _, mut account) = make_two_accounts(); + assert_eq!(account.data(), &[0, 0]); + account.set_data_from_slice(&[1, 2]); + assert_eq!(account.data(), &[1, 2]); + account.set_data_from_slice(&[1, 2, 3]); + assert_eq!(account.data(), &[1, 2, 3]); + account.set_data_from_slice(&[4, 5, 6]); + assert_eq!(account.data(), &[4, 5, 6]); + account.set_data_from_slice(&[4, 5, 6, 0]); + assert_eq!(account.data(), &[4, 5, 6, 0]); + account.set_data_from_slice(&[]); + assert_eq!(account.data(), &[]); + account.set_data_from_slice(&[44]); + assert_eq!(account.data(), &[44]); + account.set_data_from_slice(&[44]); + assert_eq!(account.data(), &[44]); +} + +#[test] +// set_data_at only writes and extends, so an empty write leaves the buffer as is. +fn test_account_set_data_at_never_truncates() { + let (_, _, mut account) = make_two_accounts(); + assert_eq!(account.data(), &[0, 0]); + account.set_data_at(0, &[1, 2]); + assert_eq!(account.data(), &[1, 2]); + account.set_data_at(0, &[]); + assert_eq!(account.data(), &[1, 2]); +} + +#[test] +// Data patches should write in place and extend when needed. +fn test_account_field_patch_data_at() { + let owner = Pubkey::new_unique(); + let mut account = AccountSharedData::new(1, 2, &owner); + account.set_data_from_slice(&[1, 2, 3, 4]); + + AccountFieldPatch::DataAt { + offset: 1, + data: vec![9, 8, 7, 6], + } + .apply(&mut account) + .unwrap(); + assert_eq!(account.data(), &[1, 9, 8, 7, 6]); + + AccountFieldPatch::DataAt { offset: 6, data: vec![5, 4] } + .apply(&mut account) + .unwrap(); + assert_eq!(account.data(), &[1, 9, 8, 7, 6, 0, 5, 4]); +} + +#[test] +fn test_account_patch_transition_errors() { + let mut account = AccountBuilder::default() + .mode(AccountMode::Delegated) + .slot(10) + .build::(); + + assert_eq!( + account.set_mode(AccountMode::ReadOnly), + Err(AccountPatchError::InvalidModeTransition { + from: AccountMode::Delegated, + to: AccountMode::ReadOnly, + }) + ); + assert!(account.is(AccountMode::Delegated)); + assert!(account.markers().is_empty()); + + assert_eq!( + AccountFieldPatch::Slot(10).apply(&mut account), + Err(AccountPatchError::InvalidSlotTransition { from: 10, to: 10 }) + ); + assert_eq!(account.slot(), 10); + assert!(account.markers().is_empty()); + + account.set_mode(AccountMode::Transient).unwrap(); + AccountFieldPatch::Slot(10).apply(&mut account).unwrap(); + assert!(account.markers().contains(DirtyMarkers::MODE)); + assert!(account.markers().contains(DirtyMarkers::SLOT)); + + let markers = *account.markers(); + assert_eq!( + AccountFieldPatch::Slot(9).apply(&mut account), + Err(AccountPatchError::InvalidSlotTransition { from: 10, to: 9 }) + ); + assert_eq!(account.slot(), 10); + assert_eq!(*account.markers(), markers); + + let mut ephemeral = AccountBuilder::default() + .mode(AccountMode::Ephemeral) + .build::(); + ephemeral.set_mode(AccountMode::Closed).unwrap(); + assert!(ephemeral.is(AccountMode::Closed)); +} + +#[test] +// Deserialization should fail on a non-bincode payload. +fn test_account_deserialize() { + let (_, account1, _) = make_two_accounts(); + assert!(account1.deserialize_data::().is_err()); +} + +#[test] +// Serialization should reject values larger than the data buffer. +fn test_account_serialize() { + let (_, mut account1, _) = make_two_accounts(); + let err = account1.serialize_data(&"hello world").unwrap_err(); + assert!(matches!(*err, ErrorKind::SizeLimit)); +} + +#[test] +// Shared accounts should fail deserialization on the same invalid payload. +fn test_account_cow_deserialize() { + let (_, _, account2) = make_two_accounts(); + assert!(account2.deserialize_data::().is_err()); +} + +#[test] +// Shared accounts should reject oversized serialization too. +fn test_account_cow_serialize() { + let (_, _, mut account2) = make_two_accounts(); + let err = account2.serialize_data(&"hello world").unwrap_err(); + assert!(matches!(*err, ErrorKind::SizeLimit)); +} + +#[test] +// Account and AccountSharedData should expose the same visible state. +fn test_account_cow() { + let (key, account1, account2) = make_two_accounts(); + assert!(accounts_equal(&account1, &account2)); + + assert_eq!(account1.lamports, 1); + assert_eq!(account1.lamports(), 1); + assert_eq!(account1.data.len(), 2); + assert_eq!(account1.data().len(), 2); + assert_eq!(account1.owner, key); + assert_eq!(account1.owner(), &key); + assert!(account1.executable); + assert!(account1.executable()); + assert_eq!(account1.rent_epoch, Epoch::MAX); + assert_eq!(account1.rent_epoch(), Epoch::MAX); + + assert_eq!(account2.lamports(), 1); + assert_eq!(account2.data().len(), 2); + assert_eq!(account2.owner(), &key); + assert!(account2.executable()); + assert_eq!(account2.rent_epoch(), Epoch::MAX); +} + +#[test] +// Checked lamport mutation should keep both account forms in sync. +fn test_account_add_sub_lamports() { + let (_, mut account1, mut account2) = make_two_accounts(); + assert!(accounts_equal(&account1, &account2)); + assert!(matches!(account1.checked_add_lamports(1), Ok(()))); + assert!(matches!(account2.checked_add_lamports(1), Ok(()))); + assert!(accounts_equal(&account1, &account2)); + assert_eq!(account1.lamports(), 2); + assert!(matches!(account1.checked_sub_lamports(2), Ok(()))); + assert!(matches!(account2.checked_sub_lamports(2), Ok(()))); + assert!(accounts_equal(&account1, &account2)); + assert_eq!(account1.lamports(), 0); +} + +#[test] +// Checked lamport arithmetic should report overflow and underflow. +fn test_account_checked_lamport_errors() { + let (_, account1, account2) = make_two_accounts(); + + assert_add_err(account1.clone()); + assert_sub_err(account1); + assert_add_err(account2.clone()); + assert_sub_err(account2); +} + +#[test] +// Saturating lamport arithmetic should clamp on both account forms. +fn test_account_saturating_lamports() { + let (_, account1, account2) = make_two_accounts(); + + assert_saturating_add(account1.clone(), u64::MAX - 22, 44, u64::MAX); + assert_saturating_add(account2.clone(), u64::MAX - 22, 44, u64::MAX); + assert_saturating_sub(account1, 33, 66, 0); + assert_saturating_sub(account2, 33, 66, 0); +} + +#[test] +// Shrinking data should replace the contents and allow regrowth. +fn test_account_cow_set_data_from_slice_shrinks() { + let owner = Pubkey::new_unique(); + let mut shared = AccountSharedData::new(1, 4, &owner); + + shared.set_data_from_slice(&[1, 2, 3, 4]); + assert_eq!(shared.data(), &[1, 2, 3, 4]); + + shared.set_data_from_slice(&[]); + assert_eq!(shared.data(), &[]); + + shared.set_data_from_slice(&[9]); + assert_eq!(shared.data(), &[9]); +} + +#[test] +// Cloning should share storage until a write forces promotion. +fn test_account_cow_is_copy_on_write() { + let owner = Pubkey::new_unique(); + let mut shared = AccountSharedData::new(1, 2, &owner); + shared.set_data_from_slice(&[1, 2]); + + let cloned = shared.clone(); + assert!(shared.is_shared()); + assert!(cloned.is_shared()); + + shared.extend_from_slice(&[3]); + assert_eq!(shared.data(), &[1, 2, 3]); + assert_eq!(cloned.data(), &[1, 2]); +} + +#[test] +// Borrowed serialization should round-trip back to shared state. +fn test_account_cow_borrowed_round_trip() { + let pubkey = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let owned = AccountBuilder::default() + .lamports(5) + .data(vec![7, 8]) + .owner(owner) + .executable(true) + .build::(); + let expected: AccountSharedData = owned.clone().into(); + let mut buf = serialize_account_buffer(&owned, &pubkey); + let borrowed = init_borrowed_account(&mut buf); + let shared = AccountSharedData::from(borrowed); + + assert_eq!(shared, expected); +} + +#[test] +// Writing past borrowed capacity should promote to owned storage. +fn test_account_cow_borrowed_extend_promotes() { + let (_buf, mut shared) = make_borrowed(vec![7, 8]); + let len = shared.data().len(); + let extra = vec![9; shared.capacity() - len + 1]; + + shared.extend_from_slice(&extra); + + assert_eq!(&shared.data()[..len], &[7, 8]); + assert_eq!(&shared.data()[len..], extra.as_slice()); + assert!(matches!(shared.cow(), CoWAccount::Owned(_))); +} + +#[test] +// Exact-capacity borrowed writes should keep the existing bytes intact. +fn test_account_cow_borrowed_exact_capacity_writes() { + let (_buf, mut shared) = make_borrowed(vec![1, 2]); + let snap = shared.data_clone(); + let cap = shared.capacity(); + + assert!(cap > shared.data().len()); + + shared.resize(cap, 0x55); + assert_eq!(shared.data().len(), cap); + assert_eq!(&shared.data()[..2], &[1, 2]); + assert!(shared.data()[2..].iter().all(|&b| b == 0x55)); + + let repl = vec![0x9a; cap]; + shared.set_data_from_slice(&repl); + assert_eq!(shared.data(), repl.as_slice()); + assert_eq!(snap.as_ref(), &[1, 2]); +} + +#[test] +// Borrowed resize must write the shadow image before commit publishes it. +fn test_account_cow_borrowed_resize_survives_commit() { + let (mut buf, mut shared) = make_borrowed(vec![1, 2]); + let cap = shared.capacity(); + + shared.resize(cap, 0x55); + let CoWAccount::Borrowed(borrowed) = shared.cow() else { + panic!("resize within borrowed capacity should not promote"); + }; + borrowed.commit(); + drop(shared); + + let borrowed = init_borrowed_account(&mut buf); + assert_eq!(borrowed.data.len(), cap); + assert_eq!(&borrowed.data[..2], &[1, 2]); + assert!(borrowed.data[2..].iter().all(|&b| b == 0x55)); +} + +#[test] +// Overflowing borrowed writes should preserve existing bytes through promotion. +fn test_account_cow_borrowed_overflow_promotes_without_corruption() { + let (_buf, mut shared) = make_borrowed(vec![3, 4]); + let snap = shared.data_clone(); + let cap = shared.capacity(); + let extra = vec![0xab; cap - shared.data().len() + 1]; + let mut exp = vec![3, 4]; + exp.extend_from_slice(&extra); + + shared.extend_from_slice(&extra); + + assert_eq!(shared.data(), exp.as_slice()); + assert_eq!(snap.as_ref(), &[3, 4]); +} + +#[test] +fn test_cow_set_data_at_borrowed_promotes_once() { + // In-place overlap: `offset < len` writes entirely through `data_as_mut_slice`. + let (_buf, mut shared) = make_borrowed(vec![1, 2, 3, 4]); + shared.set_data_at(1, &[9, 9]); + assert_eq!(shared.data(), &[1, 9, 9, 4]); + assert!(matches!(shared.cow(), CoWAccount::Borrowed(_))); + + // An overlapping write with a tail beyond capacity must promote once. + let (_buf, mut shared) = make_borrowed(vec![1, 2, 3]); + let offset = 2; + let data = vec![7; shared.capacity() - offset + 1]; + shared.set_data_at(offset, &data); + assert_eq!(&shared.data()[..offset], &[1, 2]); + assert_eq!(&shared.data()[offset..], data.as_slice()); + assert!(matches!(shared.cow(), CoWAccount::Owned(_))); +} + +#[test] +// `init` should read the active image without changing the sequence. +fn test_cow_init_reads_active_image() { + let pubkey = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let owned = AccountBuilder::default().lamports(5).data(vec![1, 2, 3]).owner(owner).build(); + let mut buf = serialize_account_buffer(&owned, &pubkey); + let borrowed = init_borrowed_account(&mut buf); + + assert_eq!(&*borrowed.data, &[1, 2, 3]); + assert_eq!(borrowed.sequence(), 0); +} + +#[test] +// `translate` should copy the active image into the shadow view, and `commit` should publish it. +fn test_cow_translate_commit_publishes_shadow_image() { + let pubkey = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let owned = AccountBuilder::default().lamports(5).data(vec![1, 2, 3]).owner(owner).build(); + let mut buf = serialize_account_buffer(&owned, &pubkey); + let mut borrowed = init_borrowed_account(&mut buf); + + // SAFETY: `borrowed` still points at the live borrowed image selected by `init`. + unsafe { borrowed.translate() }; + assert_eq!(borrowed.sequence(), 0); + + borrowed.data[0] = 9; + borrowed.commit(); + assert_eq!(borrowed.sequence(), 1); + + let borrowed = init_borrowed_account(&mut buf); + assert_eq!(&*borrowed.data, &[9, 2, 3]); +} + +#[test] +// Reset should discard shadow writes and re-read the active image. +fn test_cow_translate_rollback_discards_shadow_writes() { + let pubkey = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let owned = AccountBuilder::default().lamports(5).data(vec![4, 5, 6]).owner(owner).build(); + let mut buf = serialize_account_buffer(&owned, &pubkey); + let mut borrowed = init_borrowed_account(&mut buf); + + // SAFETY: `borrowed` still points at the live borrowed image selected by `init`. + unsafe { borrowed.translate() }; + borrowed.data[0] = 8; + + // SAFETY: `reset` only repoints this view back to the active image. + unsafe { borrowed.reset() }; + assert_eq!(borrowed.sequence(), 0); + assert_eq!(&*borrowed.data, &[4, 5, 6]); + + let borrowed = init_borrowed_account(&mut buf); + assert_eq!(&*borrowed.data, &[4, 5, 6]); +} + +#[test] +// AccountSeqLock should retry against the newly published borrowed image. +fn test_account_seq_lock_read_retries_after_borrowed_publish() { + let pubkey = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let owned = AccountBuilder::default().lamports(5).data(vec![1, 2, 3]).owner(owner).build(); + let mut buf = serialize_account_buffer(&owned, &pubkey); + let ptr = NonNull::from(buf.as_mut_slice()).cast(); + // SAFETY: `ptr` names the live serialized buffer for the duration of both + // views. Their intentionally aliased access is sequenced by the test: the + // reader is idle while the writer publishes, then resets before retrying. + let borrowed = unsafe { BorrowedAccount::init(ptr) }; + // SAFETY: same buffer and access protocol as the reader view above. + let writer = RefCell::new(unsafe { BorrowedAccount::init(ptr) }); + let mut lock = AccountSeqLock::new(AccountSharedData::from(borrowed)); + let calls = Cell::new(0); + + let data = lock.read(|account| { + let call = calls.get(); + calls.set(call + 1); + + if call == 0 { + let mut writer = writer.borrow_mut(); + // SAFETY: the writer still points at the image selected by `init`. + unsafe { writer.translate() }; + writer.data[0] = 9; + writer.commit(); + } + + account.data().to_vec() + }); + + assert_eq!(calls.get(), 2); + assert_eq!(data, vec![9, 2, 3]); +} diff --git a/solana/account/src/tests/mod.rs b/solana/account/src/tests/mod.rs new file mode 100644 index 00000000..5a5acba7 --- /dev/null +++ b/solana/account/src/tests/mod.rs @@ -0,0 +1,4 @@ +mod account; +mod state_traits; +#[cfg(feature = "bincode")] +mod sysvar; diff --git a/solana/account/src/tests/state_traits.rs b/solana/account/src/tests/state_traits.rs new file mode 100644 index 00000000..1490cea8 --- /dev/null +++ b/solana/account/src/tests/state_traits.rs @@ -0,0 +1,18 @@ +use { + crate::{AccountSharedData, state_traits::StateMut}, + solana_instruction_error::InstructionError, + solana_pubkey::Pubkey, +}; + +#[test] +fn test_account_state() { + let state = 42; + assert!(AccountSharedData::default().set_state(&state).is_err()); + let res = AccountSharedData::default().state() as Result; + assert!(res.is_err()); + + let mut account = AccountSharedData::new(0, size_of::(), &Pubkey::default()); + + assert!(account.set_state(&state).is_ok()); + assert_eq!(account.state(), Ok(state)); +} diff --git a/solana/account/src/tests/sysvar.rs b/solana/account/src/tests/sysvar.rs new file mode 100644 index 00000000..c5b539ca --- /dev/null +++ b/solana/account/src/tests/sysvar.rs @@ -0,0 +1,15 @@ +use { + crate::{create_account_with_fields, from_account}, + solana_clock::{Clock, Epoch}, +}; + +#[test] +fn test_create_account_with_fields_round_trips_sysvar() { + let clock = Clock { epoch: 7, ..Clock::default() }; + + let account = create_account_with_fields(&clock, (3, Epoch::MAX)); + + assert_eq!(account.lamports, 3); + assert_eq!(account.rent_epoch, Epoch::MAX); + assert_eq!(from_account::(&account), Some(clock)); +} diff --git a/solana/account/src/traits.rs b/solana/account/src/traits.rs new file mode 100644 index 00000000..3078689b --- /dev/null +++ b/solana/account/src/traits.rs @@ -0,0 +1,246 @@ +use { + crate::{ + Account, AccountSharedData, + cow::{DirtyMarkers, StateFlags}, + }, + solana_account_info::debug_account_data::debug_account_data, + solana_clock::Epoch, + solana_instruction_error::LamportsError, + solana_pubkey::Pubkey, + std::{fmt, ops::Deref}, +}; + +/// Read-only access to account state. +pub trait ReadableAccount: Sized { + /// Returns the lamport balance. + fn lamports(&self) -> u64; + + /// Returns the account data. + fn data(&self) -> &[u8]; + + /// Returns the account owner. + fn owner(&self) -> &Pubkey; + + /// Returns whether the account is executable. + fn executable(&self) -> bool; + + /// Returns the rent epoch view for this account. + fn rent_epoch(&self) -> Epoch; +} + +/// Writable access to account state. +pub trait WritableAccount: ReadableAccount { + /// Replaces the lamport balance. + fn set_lamports(&mut self, lamports: u64); + + /// Adds lamports or returns an overflow error. + fn checked_add_lamports(&mut self, lamports: u64) -> Result<(), LamportsError> { + self.set_lamports( + self.lamports().checked_add(lamports).ok_or(LamportsError::ArithmeticOverflow)?, + ); + Ok(()) + } + + /// Subtracts lamports or returns an underflow error. + fn checked_sub_lamports(&mut self, lamports: u64) -> Result<(), LamportsError> { + self.set_lamports( + self.lamports() + .checked_sub(lamports) + .ok_or(LamportsError::ArithmeticUnderflow)?, + ); + Ok(()) + } + + /// Adds lamports and saturates on overflow. + fn saturating_add_lamports(&mut self, lamports: u64) { + self.set_lamports(self.lamports().saturating_add(lamports)) + } + + /// Subtracts lamports and saturates on underflow. + fn saturating_sub_lamports(&mut self, lamports: u64) { + self.set_lamports(self.lamports().saturating_sub(lamports)) + } + + /// Returns mutable access to the account data. + fn data_as_mut_slice(&mut self) -> &mut [u8]; + + /// Replaces the owner. + fn set_owner(&mut self, owner: Pubkey); + + /// Copies 32 raw bytes into the owner pubkey. + fn copy_into_owner_from_slice(&mut self, source: &[u8]); + + /// Sets the executable flag. + fn set_executable(&mut self, executable: bool); + + /// Sets the rent epoch view if the implementation stores one. + /// + /// Implementations that do not store rent epoch may ignore this. + fn set_rent_epoch(&mut self, epoch: Epoch); +} + +/// Returns `true` when the readable account fields match. +/// +/// This ignores storage form and any non-readable metadata. +pub fn accounts_equal(me: &T, other: &U) -> bool { + me.lamports() == other.lamports() + && me.executable() == other.executable() + && me.rent_epoch() == other.rent_epoch() + && me.owner() == other.owner() + && me.data() == other.data() +} + +/// Formats readable accounts with the same debug shape as `Account`. +pub(crate) fn debug_fmt( + item: &T, + f: &mut fmt::Formatter<'_>, + add: impl FnOnce(&mut fmt::DebugStruct<'_, '_>), +) -> fmt::Result { + let mut f = f.debug_struct("Account"); + + f.field("lamports", &item.lamports()) + .field("data.len", &item.data().len()) + .field("owner", &item.owner()) + .field("executable", &item.executable()) + .field("rent_epoch", &item.rent_epoch()); + add(&mut f); + debug_account_data(item.data(), &mut f); + + f.finish() +} + +impl ReadableAccount for T +where + T: Deref, + T::Target: ReadableAccount, +{ + fn lamports(&self) -> u64 { + self.deref().lamports() + } + + fn data(&self) -> &[u8] { + self.deref().data() + } + + fn owner(&self) -> &Pubkey { + self.deref().owner() + } + + fn executable(&self) -> bool { + self.deref().executable() + } + + fn rent_epoch(&self) -> Epoch { + self.deref().rent_epoch() + } +} + +impl ReadableAccount for Account { + fn lamports(&self) -> u64 { + self.lamports + } + + fn data(&self) -> &[u8] { + &self.data + } + + fn owner(&self) -> &Pubkey { + &self.owner + } + + fn executable(&self) -> bool { + self.executable + } + + fn rent_epoch(&self) -> Epoch { + self.rent_epoch + } +} + +impl WritableAccount for Account { + fn set_lamports(&mut self, lamports: u64) { + self.lamports = lamports; + } + + fn data_as_mut_slice(&mut self) -> &mut [u8] { + &mut self.data + } + + fn set_owner(&mut self, owner: Pubkey) { + self.owner = owner; + } + + fn copy_into_owner_from_slice(&mut self, source: &[u8]) { + self.owner.as_mut().copy_from_slice(source); + } + + fn set_executable(&mut self, executable: bool) { + self.executable = executable; + } + + fn set_rent_epoch(&mut self, epoch: Epoch) { + self.rent_epoch = epoch; + } +} + +impl ReadableAccount for AccountSharedData { + fn lamports(&self) -> u64 { + self.lamports + } + + fn data(&self) -> &[u8] { + self.cow.data() + } + + fn owner(&self) -> &Pubkey { + &self.owner + } + + fn executable(&self) -> bool { + self.flags.contains(StateFlags::EXECUTABLE) + } + + fn rent_epoch(&self) -> Epoch { + Epoch::MAX + } +} + +impl WritableAccount for AccountSharedData { + fn set_lamports(&mut self, lamports: u64) { + if self.lamports == lamports { + return; + } + self.translate(); + self.dirty.insert(DirtyMarkers::LAMPORTS); + self.lamports = lamports; + } + + fn data_as_mut_slice(&mut self) -> &mut [u8] { + self.translate(); + self.mark_data_dirty(); + self.cow.data_mut() + } + + fn set_owner(&mut self, owner: Pubkey) { + if self.owner == owner { + return; + } + self.translate(); + self.dirty.insert(DirtyMarkers::OWNER); + self.owner = owner; + } + + fn copy_into_owner_from_slice(&mut self, source: &[u8]) { + self.translate(); + self.dirty.insert(DirtyMarkers::OWNER); + self.owner.as_mut().copy_from_slice(source); + } + + fn set_executable(&mut self, executable: bool) { + let mut flags = self.flags; + flags.set(StateFlags::EXECUTABLE, executable); + self.set_flags(flags); + } + + fn set_rent_epoch(&mut self, _: Epoch) {} +} diff --git a/solana/program-runtime/Cargo.toml b/solana/program-runtime/Cargo.toml new file mode 100644 index 00000000..e9ffae90 --- /dev/null +++ b/solana/program-runtime/Cargo.toml @@ -0,0 +1,90 @@ +[package] +name = "solana-program-runtime" + +authors = { workspace = true } +description = "Solana program runtime" +documentation = "https://docs.rs/solana-program-runtime" +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = "4.1.1" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +crate-type = ["lib"] +name = "solana_program_runtime" + +[features] +# No-op stub retained only so external (patched-in) crates that reference +# `solana-program-runtime/agave-unstable-api` still resolve; the lib is no +# longer gated on it, and the upstream svm-* deps enable their unstable API +# unconditionally below. +agave-unstable-api = [] +dev-context-only-utils = [] +dummy-for-ci-check = ["metrics"] +# Compatibility stub: this fork does not derive or consume frozen ABI metadata. +frozen-abi = [] +metrics = [] +sbpf-debugger = ["solana-sbpf/debugger"] +shuttle-test = ["solana-sbpf/shuttle-test", "solana-svm-type-overrides/shuttle-test"] +svm-internal = ["dep:qualifier_attr"] + +[dependencies] +base64 = { workspace = true } +bincode = { workspace = true } +cfg-if = { workspace = true } +itertools = { workspace = true } +qualifier_attr = { workspace = true, optional = true } +scc = { workspace = true } +serde = { workspace = true } +solana-account = { workspace = true, features = ["bincode"] } +solana-account-info = { workspace = true } +solana-clock = { workspace = true } +solana-epoch-rewards = { workspace = true } +solana-epoch-schedule = { workspace = true } +solana-fee-structure = { workspace = true } +solana-hash = { workspace = true } +solana-instruction = { workspace = true } +solana-last-restart-slot = { workspace = true } +solana-loader-v3-interface = { workspace = true } +solana-program-entrypoint = { workspace = true } +solana-pubkey = { workspace = true } +solana-rent = { workspace = true } +solana-sbpf = { workspace = true, features = ["jit"] } +solana-sdk-ids = { workspace = true } +solana-slot-hashes = { workspace = true } +solana-stable-layout = { workspace = true } +solana-svm-callback = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-feature-set = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-log-collector = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-measure = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-timings = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-transaction = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-type-overrides = { workspace = true, features = ["agave-unstable-api"] } +solana-system-interface = { workspace = true } +solana-sysvar = { workspace = true, features = ["bincode"] } +solana-sysvar-id = { workspace = true } +solana-transaction-context = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +assert_matches = { workspace = true } +solana-account = { workspace = true, features = ["testkit"] } +solana-account-info = { workspace = true } +solana-instruction = { workspace = true, features = ["bincode"] } +solana-keypair = { workspace = true } +solana-program-runtime = { path = ".", features = ["dev-context-only-utils"] } +solana-pubkey = { workspace = true, features = ["rand"] } +solana-signer = { workspace = true } +solana-transaction = { workspace = true, features = ["dev-context-only-utils"] } +solana-transaction-context = { workspace = true, features = ["dev-context-only-utils"] } +test-case = "3.3.1" + +[lints.rust] +unexpected_cfgs = "allow" + +[lints.clippy] +too_many_arguments = "allow" diff --git a/solana/program-runtime/README.md b/solana/program-runtime/README.md new file mode 100644 index 00000000..ed2e2aab --- /dev/null +++ b/solana/program-runtime/README.md @@ -0,0 +1,16 @@ +# `solana-program-runtime` + +This Agave fork implements invocation state, CPI translation, SBF VM setup, +sysvar access, logging, serialization, and program-cache primitives. Workspace +dependencies that select `solana-program-runtime` use this workspace copy. + +Account loading and transaction-level policy belong to `solana-svm`. The +engine-specific direct account mapping, access-violation growth, and CPI +synchronization contracts are documented in [`../README.md`](../README.md). + +Changes to `serialization`, CPI account-region replacement, or `vm` error +mapping must remain synchronized with the transaction-context access-violation +handler. + +The `frozen-abi` feature is retained as a no-op compatibility stub; this fork +does not derive or consume frozen ABI metadata. diff --git a/solana/program-runtime/fixtures/noop_aligned.so b/solana/program-runtime/fixtures/noop_aligned.so new file mode 100644 index 00000000..f00ca366 Binary files /dev/null and b/solana/program-runtime/fixtures/noop_aligned.so differ diff --git a/solana/program-runtime/src/cpi.rs b/solana/program-runtime/src/cpi.rs new file mode 100644 index 00000000..1dc92638 --- /dev/null +++ b/solana/program-runtime/src/cpi.rs @@ -0,0 +1,1795 @@ +//! Cross-program invocation translation and dispatch. + +use { + crate::{ + invoke_context::{InvokeContext, SerializedAccountMetadata}, + memory::{ + MemoryTranslationError, address_is_aligned, translate_slice, translate_type, + translate_type_mut_for_cpi, translate_vm_slice, + }, + serialization::{account_data_region_size, create_memory_region_of_account}, + }, + solana_account_info::AccountInfo, + solana_instruction::{AccountMeta, Instruction, error::InstructionError}, + solana_loader_v3_interface::instruction as bpf_loader_upgradeable, + solana_pubkey::{MAX_SEEDS, Pubkey, PubkeyError}, + solana_sbpf::{ebpf, memory_region::MemoryMapping}, + solana_sdk_ids::{bpf_loader, bpf_loader_deprecated, native_loader}, + solana_stable_layout::stable_instruction::StableInstruction, + solana_svm_log_collector::ic_msg, + solana_svm_measure::measure::Measure, + solana_transaction_context::{ + IndexOfAccount, MAX_ACCOUNTS_PER_INSTRUCTION, MAX_INSTRUCTION_DATA_LEN, + instruction_accounts::BorrowedInstructionAccount, vm_slice::VmSlice, + }, + std::mem, + thiserror::Error, +}; + +/// Errors produced while translating or dispatching a CPI request. +#[derive(Debug, Error, PartialEq, Eq)] +pub enum CpiError { + #[error("Invalid pointer")] + InvalidPointer, + #[error("Too many signers")] + TooManySigners, + #[error("Could not create program address with signer seeds: {0}")] + BadSeeds(PubkeyError), + #[error("InvalidLength")] + InvalidLength, + #[error("Invoked an instruction with too many accounts ({num_accounts} > {max_accounts})")] + MaxInstructionAccountsExceeded { num_accounts: u64, max_accounts: u64 }, + #[error("Invoked an instruction with data that is too large ({data_len} > {max_data_len})")] + MaxInstructionDataLenExceeded { data_len: u64, max_data_len: u64 }, + #[error( + "Invoked an instruction with too many account info's ({num_account_infos} > \ + {max_account_infos})" + )] + MaxInstructionAccountInfosExceeded { num_account_infos: u64, max_account_infos: u64 }, + #[error("Program {0} not supported by inner instructions")] + ProgramNotSupported(Pubkey), +} + +type Error = Box; + +const SUCCESS: u64 = 0; +/// Maximum signer seed groups accepted by CPI. +const MAX_SIGNERS: usize = 16; +/// SIMD-0339 `AccountInfo` translation byte size. +/// +/// Fixed size of **80 bytes** for each AccountInfo, broken down as: +/// - 32 bytes for account address +/// - 32 bytes for owner address +/// - 8 bytes for lamport balance +/// - 8 bytes for data length +const ACCOUNT_INFO_BYTE_SIZE: usize = 80; + +/// Rust representation of C's SolInstruction +#[derive(Debug)] +#[repr(C)] +struct SolInstruction { + pub program_id_addr: u64, + pub accounts_addr: u64, + pub accounts_len: u64, + pub data_addr: u64, + pub data_len: u64, +} + +/// Rust representation of C's SolAccountMeta +#[derive(Debug)] +#[repr(C)] +struct SolAccountMeta { + pub pubkey_addr: u64, + pub is_writable: bool, + pub is_signer: bool, +} + +/// Byte-valid representation of an account meta supplied by an SBF program. +#[repr(C)] +struct UntrustedAccountMeta { + pubkey: [u8; 32], + is_signer: u8, + is_writable: u8, +} + +/// Byte-valid representation of C's `SolAccountMeta`. +#[repr(C)] +struct UntrustedSolAccountMeta { + pubkey_addr: [u8; 8], + is_writable: u8, + is_signer: u8, + padding: [u8; 6], +} + +const _: () = assert!(std::mem::size_of::() == size_of::()); +const _: () = + assert!(std::mem::size_of::() == size_of::()); + +fn translate_bool(value: u8) -> Result { + match value { + 0 => Ok(false), + 1 => Ok(true), + _ => Err(Box::new(InstructionError::InvalidArgument)), + } +} + +/// Rust representation of C's SolAccountInfo +#[derive(Debug)] +#[repr(C)] +struct SolAccountInfo { + pub key_addr: u64, + pub lamports_addr: u64, + pub data_len: u64, + pub data_addr: u64, + pub owner_addr: u64, + pub rent_epoch: u64, + pub is_signer: bool, + pub is_writable: bool, + pub executable: bool, +} + +/// Rust representation of C's SolSignerSeed +#[derive(Debug)] +#[repr(C)] +struct SolSignerSeedC { + pub addr: u64, + pub len: u64, +} + +/// Rust representation of C's SolSignerSeeds +#[derive(Debug)] +#[repr(C)] +struct SolSignerSeedsC { + pub addr: u64, + pub len: u64, +} + +/// Maximum number of account info structs that can be used in a single CPI invocation +const MAX_CPI_ACCOUNT_INFOS: usize = 128; +/// Maximum number of account info structs that can be used in a single CPI invocation with SIMD-0339 active +const MAX_CPI_ACCOUNT_INFOS_SIMD_0339: usize = 255; + +/// Check that an account info pointer field points to the expected address +fn check_account_info_pointer( + invoke_context: &InvokeContext, + vm_addr: u64, + expected_vm_addr: u64, + field: &str, +) -> Result<(), Error> { + if vm_addr != expected_vm_addr { + ic_msg!( + invoke_context, + "Invalid account info pointer `{}': {:#x} != {:#x}", + field, + vm_addr, + expected_vm_addr + ); + return Err(Box::new(CpiError::InvalidPointer)); + } + Ok(()) +} + +/// Check that an instruction's account and data lengths are within limits +fn check_instruction_size(num_accounts: usize, data_len: usize) -> Result<(), Error> { + if num_accounts > MAX_ACCOUNTS_PER_INSTRUCTION { + return Err(Box::new(CpiError::MaxInstructionAccountsExceeded { + num_accounts: num_accounts as u64, + max_accounts: MAX_ACCOUNTS_PER_INSTRUCTION as u64, + })); + } + if data_len > MAX_INSTRUCTION_DATA_LEN { + return Err(Box::new(CpiError::MaxInstructionDataLenExceeded { + data_len: data_len as u64, + max_data_len: MAX_INSTRUCTION_DATA_LEN as u64, + })); + } + Ok(()) +} + +/// Check that the number of account infos is within the CPI limit +fn check_account_infos( + num_account_infos: usize, + invoke_context: &InvokeContext, +) -> Result<(), Error> { + let max_cpi_account_infos = if invoke_context.get_feature_set().increase_cpi_account_info_limit + { + MAX_CPI_ACCOUNT_INFOS_SIMD_0339 + } else if invoke_context.get_feature_set().increase_tx_account_lock_limit { + MAX_CPI_ACCOUNT_INFOS + } else { + 64 + }; + let num_account_infos = num_account_infos as u64; + let max_account_infos = max_cpi_account_infos as u64; + if num_account_infos > max_account_infos { + return Err(Box::new(CpiError::MaxInstructionAccountInfosExceeded { + num_account_infos, + max_account_infos, + })); + } + Ok(()) +} + +/// Check whether a program is authorized for CPI +fn check_authorized_program( + program_id: &Pubkey, + instruction_data: &[u8], + invoke_context: &InvokeContext, +) -> Result<(), Error> { + if native_loader::check_id(program_id) + || bpf_loader::check_id(program_id) + || bpf_loader_deprecated::check_id(program_id) + || (solana_sdk_ids::bpf_loader_upgradeable::check_id(program_id) + && !(bpf_loader_upgradeable::is_upgrade_instruction(instruction_data) + || bpf_loader_upgradeable::is_set_authority_instruction(instruction_data) + || (invoke_context.get_feature_set().enable_bpf_loader_set_authority_checked_ix + && bpf_loader_upgradeable::is_set_authority_checked_instruction( + instruction_data, + )) + || bpf_loader_upgradeable::is_close_instruction(instruction_data))) + || invoke_context.is_precompile(program_id) + { + return Err(Box::new(CpiError::ProgramNotSupported(*program_id))); + } + Ok(()) +} + +/// Host side representation of AccountInfo or SolAccountInfo passed to the CPI syscall. +/// +/// At the start of a CPI, this can be different from the data stored in the +/// corresponding BorrowedAccount, and needs to be synched. +#[derive(Debug)] +pub struct CallerAccount<'a> { + pub lamports: &'a mut u64, + pub owner: &'a mut Pubkey, + // The original data length of the account at the start of the current + // instruction. We use this to determine whether an account was shrunk or + // grown before or after CPI, and to derive the vm address of the realloc + // region. + pub original_data_len: usize, + // This points to the data section for this account, as serialized and + // mapped inside the vm (see serialize_parameters() in + // BpfExecutor::execute). + // + // Empty when account data is directly mapped. + pub serialized_data: &'a mut [u8], + // Given the corresponding input AccountInfo::data, vm_data_addr points to + // the pointer field and ref_to_len_in_vm points to the length field. + pub vm_data_addr: u64, + pub ref_to_len_in_vm: &'a mut u64, +} + +impl<'a> CallerAccount<'a> { + pub fn get_serialized_data( + check_aligned: bool, + original_data_len: usize, + len: usize, + ) -> Result<&'a mut [u8], Error> { + let address_space_reserved_for_account = + account_data_region_size(!check_aligned, original_data_len); + if len > address_space_reserved_for_account { + return Err(InstructionError::InvalidRealloc.into()); + } + Ok(&mut []) + } + + // Create a CallerAccount given an AccountInfo. + pub fn from_account_info( + invoke_context: &InvokeContext, + memory_mapping: &solana_sbpf::memory_region::MemoryMapping, + check_aligned: bool, + account_info: &solana_account_info::AccountInfo, + account_metadata: &crate::invoke_context::SerializedAccountMetadata, + ) -> Result, Error> { + use crate::memory::{translate_type, translate_type_mut_for_cpi}; + + check_account_info_pointer( + invoke_context, + account_info.key as *const _ as u64, + account_metadata.vm_key_addr, + "key", + )?; + check_account_info_pointer( + invoke_context, + account_info.owner as *const _ as u64, + account_metadata.vm_owner_addr, + "owner", + )?; + + // account_info points to host memory. The addresses used internally are + // in vm space so they need to be translated. + let lamports = { + // Double translate lamports out of RefCell + let ptr = translate_type::( + memory_mapping, + account_info.lamports.as_ptr() as u64, + check_aligned, + )?; + if account_info.lamports.as_ptr() as u64 >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); + } + + check_account_info_pointer( + invoke_context, + *ptr, + account_metadata.vm_lamports_addr, + "lamports", + )?; + translate_type_mut_for_cpi::(memory_mapping, *ptr, check_aligned)? + }; + + let owner = translate_type_mut_for_cpi::( + memory_mapping, + account_info.owner as *const _ as u64, + check_aligned, + )?; + + let (serialized_data, vm_data_addr, ref_to_len_in_vm) = { + if account_info.data.as_ptr() as u64 >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); + } + + // Double translate data out of RefCell + let data = *translate_type::<&[u8]>( + memory_mapping, + account_info.data.as_ptr() as *const _ as u64, + check_aligned, + )?; + check_account_info_pointer( + invoke_context, + data.as_ptr() as u64, + account_metadata.vm_data_addr, + "data", + )?; + + let vm_len_addr = (account_info.data.as_ptr() as *const u64 as u64) + .saturating_add(size_of::() as u64); + // In the same vein as the other check_account_info_pointer() checks, we don't lock + // this pointer to a specific address but we don't want it to be inside accounts, or + // callees might be able to write to the pointed memory. + if vm_len_addr >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); + } + let ref_to_len_in_vm = + translate_type_mut_for_cpi::(memory_mapping, vm_len_addr, false)?; + let vm_data_addr = data.as_ptr() as u64; + let serialized_data = CallerAccount::get_serialized_data( + check_aligned, + account_metadata.original_data_len, + *ref_to_len_in_vm as usize, + )?; + (serialized_data, vm_data_addr, ref_to_len_in_vm) + }; + + Ok(CallerAccount { + lamports, + owner, + original_data_len: account_metadata.original_data_len, + serialized_data, + vm_data_addr, + ref_to_len_in_vm, + }) + } + + // Create a CallerAccount given a SolAccountInfo. + fn from_sol_account_info( + invoke_context: &InvokeContext, + memory_mapping: &solana_sbpf::memory_region::MemoryMapping, + check_aligned: bool, + vm_addr: u64, + account_info: &SolAccountInfo, + account_metadata: &crate::invoke_context::SerializedAccountMetadata, + ) -> Result, Error> { + use crate::memory::translate_type_mut_for_cpi; + + check_account_info_pointer( + invoke_context, + account_info.key_addr, + account_metadata.vm_key_addr, + "key", + )?; + + check_account_info_pointer( + invoke_context, + account_info.owner_addr, + account_metadata.vm_owner_addr, + "owner", + )?; + + check_account_info_pointer( + invoke_context, + account_info.lamports_addr, + account_metadata.vm_lamports_addr, + "lamports", + )?; + + check_account_info_pointer( + invoke_context, + account_info.data_addr, + account_metadata.vm_data_addr, + "data", + )?; + + // account_info points to host memory. The addresses used internally are + // in vm space so they need to be translated. + let lamports = translate_type_mut_for_cpi::( + memory_mapping, + account_info.lamports_addr, + check_aligned, + )?; + let owner = translate_type_mut_for_cpi::( + memory_mapping, + account_info.owner_addr, + check_aligned, + )?; + + // we already have the host addr we want: &mut account_info.data_len. + // The account info might be read only in the vm though, so we translate + // to ensure we can write. This is tested by programs/sbf/rust/ro_modify + // which puts SolAccountInfo in rodata. + let vm_len_addr = vm_addr + .saturating_add(&account_info.data_len as *const u64 as u64) + .saturating_sub(account_info as *const _ as *const u64 as u64); + // In the same vein as the other check_account_info_pointer() checks, we don't lock + // this pointer to a specific address but we don't want it to be inside accounts, or + // callees might be able to write to the pointed memory. + if vm_len_addr >= solana_sbpf::ebpf::MM_INPUT_START { + return Err(Box::new(CpiError::InvalidPointer)); + } + let ref_to_len_in_vm = + translate_type_mut_for_cpi::(memory_mapping, vm_len_addr, false)?; + let serialized_data = CallerAccount::get_serialized_data( + check_aligned, + account_metadata.original_data_len, + *ref_to_len_in_vm as usize, + )?; + + Ok(CallerAccount { + lamports, + owner, + original_data_len: account_metadata.original_data_len, + serialized_data, + vm_data_addr: account_info.data_addr, + ref_to_len_in_vm, + }) + } +} + +/// Implemented by language specific data structure translators +pub trait SyscallInvokeSigned { + fn translate_instruction( + addr: u64, + invoke_context: &InvokeContext, + ) -> Result; + fn translate_accounts<'a>( + account_infos_addr: u64, + account_infos_len: u64, + invoke_context: &InvokeContext, + ) -> Result>, Error>; + fn translate_signers( + program_id: &Pubkey, + signers_seeds_addr: u64, + signers_seeds_len: u64, + invoke_context: &InvokeContext, + ) -> Result, Error>; +} + +pub fn translate_instruction_rust( + addr: u64, + invoke_context: &InvokeContext, +) -> Result { + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; + let ix = translate_type::(memory_mapping, addr, check_aligned)?; + let account_metas = translate_slice::( + memory_mapping, + ix.accounts.as_vaddr(), + ix.accounts.len(), + check_aligned, + )?; + let data = translate_slice::( + memory_mapping, + ix.data.as_vaddr(), + ix.data.len(), + check_aligned, + )?; + + check_instruction_size(account_metas.len(), data.len())?; + + let mut total_cu_translation_cost: u64 = (data.len() as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX); + + if invoke_context.get_feature_set().increase_cpi_account_info_limit { + // Each account meta is 34 bytes (32 for pubkey, 1 for is_signer, 1 for is_writable) + let account_meta_translation_cost = + (account_metas.len().saturating_mul(size_of::()) as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX); + + total_cu_translation_cost = + total_cu_translation_cost.saturating_add(account_meta_translation_cost); + } + + consume_compute_meter(invoke_context, total_cu_translation_cost)?; + + let mut accounts = Vec::with_capacity(account_metas.len()); + #[allow(clippy::needless_range_loop)] + for account_index in 0..account_metas.len() { + #[allow(clippy::indexing_slicing)] + let account_meta = &account_metas[account_index]; + accounts.push(AccountMeta { + pubkey: Pubkey::new_from_array(account_meta.pubkey), + is_signer: translate_bool(account_meta.is_signer)?, + is_writable: translate_bool(account_meta.is_writable)?, + }); + } + + Ok(Instruction { + accounts, + data: data.to_vec(), + program_id: ix.program_id, + }) +} + +pub fn translate_accounts_rust<'a>( + account_infos_addr: u64, + account_infos_len: u64, + invoke_context: &InvokeContext, +) -> Result>, Error> { + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; + let (account_infos, account_info_keys) = translate_account_infos( + account_infos_addr, + account_infos_len, + |account_info: &AccountInfo| account_info.key as *const _ as u64, + memory_mapping, + invoke_context, + check_aligned, + )?; + + translate_accounts_common( + &account_info_keys, + account_infos, + account_infos_addr, + invoke_context, + memory_mapping, + check_aligned, + |invoke_context, memory_mapping, check_aligned, _, account_info, account_metadata| { + CallerAccount::from_account_info( + invoke_context, + memory_mapping, + check_aligned, + account_info, + account_metadata, + ) + }, + ) +} + +pub fn translate_signers_rust( + program_id: &Pubkey, + signers_seeds_addr: u64, + signers_seeds_len: u64, + invoke_context: &InvokeContext, +) -> Result, Error> { + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; + let mut signers = Vec::new(); + if signers_seeds_len > 0 { + let signers_seeds = translate_slice::>>( + memory_mapping, + signers_seeds_addr, + signers_seeds_len, + check_aligned, + )?; + if signers_seeds.len() > MAX_SIGNERS { + return Err(Box::new(CpiError::TooManySigners)); + } + for signer_seeds in signers_seeds.iter() { + let untranslated_seeds = translate_slice::>( + memory_mapping, + signer_seeds.ptr(), + signer_seeds.len(), + check_aligned, + )?; + if untranslated_seeds.len() > MAX_SEEDS { + return Err(Box::new(InstructionError::MaxSeedLengthExceeded)); + } + let seeds = untranslated_seeds + .iter() + .map(|untranslated_seed| { + translate_vm_slice(untranslated_seed, memory_mapping, check_aligned) + }) + .collect::, Error>>()?; + let signer = + Pubkey::create_program_address(&seeds, program_id).map_err(CpiError::BadSeeds)?; + signers.push(signer); + } + Ok(signers) + } else { + Ok(vec![]) + } +} + +pub fn translate_instruction_c( + addr: u64, + invoke_context: &InvokeContext, +) -> Result { + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; + let ix_c = translate_type::(memory_mapping, addr, check_aligned)?; + + let program_id = translate_type::(memory_mapping, ix_c.program_id_addr, check_aligned)?; + let account_metas = translate_slice::( + memory_mapping, + ix_c.accounts_addr, + ix_c.accounts_len, + check_aligned, + )?; + if check_aligned && !address_is_aligned::(account_metas.as_ptr() as u64) { + return Err(Box::new(MemoryTranslationError::UnalignedPointer)); + } + let data = translate_slice::(memory_mapping, ix_c.data_addr, ix_c.data_len, check_aligned)?; + + check_instruction_size(ix_c.accounts_len as usize, data.len())?; + + let mut total_cu_translation_cost: u64 = (data.len() as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX); + + if invoke_context.get_feature_set().increase_cpi_account_info_limit { + // Each account meta is 34 bytes (32 for pubkey, 1 for is_signer, 1 for is_writable) + let account_meta_translation_cost = + (ix_c.accounts_len.saturating_mul(size_of::() as u64)) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX); + + total_cu_translation_cost = + total_cu_translation_cost.saturating_add(account_meta_translation_cost); + } + + consume_compute_meter(invoke_context, total_cu_translation_cost)?; + + let mut accounts = Vec::with_capacity(ix_c.accounts_len as usize); + #[allow(clippy::needless_range_loop)] + for account_index in 0..ix_c.accounts_len as usize { + #[allow(clippy::indexing_slicing)] + let account_meta = &account_metas[account_index]; + let is_signer = translate_bool(account_meta.is_signer)?; + let is_writable = translate_bool(account_meta.is_writable)?; + let pubkey_addr = u64::from_ne_bytes(account_meta.pubkey_addr); + let pubkey = translate_type::(memory_mapping, pubkey_addr, check_aligned)?; + accounts.push(AccountMeta { + pubkey: *pubkey, + is_signer, + is_writable, + }); + } + + Ok(Instruction { + accounts, + data: data.to_vec(), + program_id: *program_id, + }) +} + +pub fn translate_accounts_c<'a>( + account_infos_addr: u64, + account_infos_len: u64, + invoke_context: &InvokeContext, +) -> Result>, Error> { + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; + let (account_infos, account_info_keys) = translate_account_infos( + account_infos_addr, + account_infos_len, + |account_info: &SolAccountInfo| account_info.key_addr, + memory_mapping, + invoke_context, + check_aligned, + )?; + + translate_accounts_common( + &account_info_keys, + account_infos, + account_infos_addr, + invoke_context, + memory_mapping, + check_aligned, + CallerAccount::from_sol_account_info, + ) +} + +pub fn translate_signers_c( + program_id: &Pubkey, + signers_seeds_addr: u64, + signers_seeds_len: u64, + invoke_context: &InvokeContext, +) -> Result, Error> { + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping()?; + if signers_seeds_len > 0 { + let signers_seeds = translate_slice::( + memory_mapping, + signers_seeds_addr, + signers_seeds_len, + check_aligned, + )?; + if signers_seeds.len() > MAX_SIGNERS { + return Err(Box::new(CpiError::TooManySigners)); + } + Ok(signers_seeds + .iter() + .map(|signer_seeds| { + let seeds = translate_slice::( + memory_mapping, + signer_seeds.addr, + signer_seeds.len, + check_aligned, + )?; + if seeds.len() > MAX_SEEDS { + return Err(Box::new(InstructionError::MaxSeedLengthExceeded) as Error); + } + let seeds_bytes = seeds + .iter() + .map(|seed| { + translate_slice::(memory_mapping, seed.addr, seed.len, check_aligned) + }) + .collect::, Error>>()?; + Pubkey::create_program_address(&seeds_bytes, program_id) + .map_err(|err| Box::new(CpiError::BadSeeds(err)) as Error) + }) + .collect::, Error>>()?) + } else { + Ok(vec![]) + } +} + +/// Call process instruction, common to both Rust and C +pub fn cpi_common( + invoke_context: &mut InvokeContext, + instruction_addr: u64, + account_infos_addr: u64, + account_infos_len: u64, + signers_seeds_addr: u64, + signers_seeds_len: u64, +) -> Result { + // CPI entry. + // + // Translate the inputs to the syscall and synchronize the caller's account + // changes so the callee can see them. + consume_compute_meter( + invoke_context, + invoke_context.get_execution_cost().invoke_units, + )?; + if let Some(execute_time) = invoke_context.execute_time.as_mut() { + execute_time.stop(); + invoke_context.timings.execute_us += execute_time.as_us(); + } + let check_aligned = invoke_context.get_check_aligned(); + let memory_mapping = invoke_context.memory_contexts.memory_mapping_mut()? as *mut MemoryMapping; + + let instruction = S::translate_instruction(instruction_addr, invoke_context)?; + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + let caller_program_id = instruction_context.get_program_key()?; + let signers = S::translate_signers( + caller_program_id, + signers_seeds_addr, + signers_seeds_len, + invoke_context, + )?; + check_authorized_program(&instruction.program_id, &instruction.data, invoke_context)?; + invoke_context.prepare_next_cpi_instruction(instruction, &signers)?; + + let mut accounts = + S::translate_accounts(account_infos_addr, account_infos_len, invoke_context)?; + let memory_mapping = unsafe { &mut *memory_mapping }; + + // Before initiating CPI, synchronize caller changes into the account seen + // by the callee. + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + for translated_account in accounts.iter_mut() { + let callee_account = instruction_context + .try_borrow_instruction_account(translated_account.index_in_caller)?; + let update_caller = + update_callee_account(&translated_account.caller_account, callee_account)?; + translated_account.update_caller_account_region = + translated_account.update_caller_account_info || update_caller; + } + + // Process the callee instruction + let mut compute_units_consumed = 0; + invoke_context.process_instruction(&mut compute_units_consumed)?; + + // re-bind to please the borrow checker + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + + // CPI exit. + // + // Synchronize the callee's account changes so the caller can see them. + for translated_account in accounts.iter_mut() { + let mut callee_account = instruction_context + .try_borrow_instruction_account(translated_account.index_in_caller)?; + if translated_account.update_caller_account_info { + update_caller_account( + invoke_context, + memory_mapping, + check_aligned, + &mut translated_account.caller_account, + &mut callee_account, + )?; + } + } + + for translated_account in accounts.iter() { + let mut callee_account = instruction_context + .try_borrow_instruction_account(translated_account.index_in_caller)?; + if translated_account.update_caller_account_region { + update_caller_account_region( + memory_mapping, + check_aligned, + &translated_account.caller_account, + &mut callee_account, + )?; + } + } + + invoke_context.execute_time = Some(Measure::start("execute")); + Ok(SUCCESS) +} + +/// Account data and metadata that has been translated from caller space. +pub struct TranslatedAccount<'a> { + pub index_in_caller: IndexOfAccount, + pub caller_account: CallerAccount<'a>, + pub update_caller_account_region: bool, + pub update_caller_account_info: bool, +} + +fn translate_account_infos<'a, T, F>( + account_infos_addr: u64, + account_infos_len: u64, + key_addr: F, + memory_mapping: &'a MemoryMapping, + invoke_context: &InvokeContext, + check_aligned: bool, +) -> Result<(&'a [T], Vec<&'a Pubkey>), Error> +where + F: Fn(&T) -> u64, +{ + // In the same vein as the other check_account_info_pointer() checks, we don't lock + // this pointer to a specific address but we don't want it to be inside accounts, or + // callees might be able to write to the pointed memory. + if account_infos_addr.saturating_add(account_infos_len.saturating_mul(size_of::() as u64)) + >= ebpf::MM_INPUT_START + { + return Err(CpiError::InvalidPointer.into()); + } + + let account_infos = translate_slice::( + memory_mapping, + account_infos_addr, + account_infos_len, + check_aligned, + )?; + check_account_infos(account_infos.len(), invoke_context)?; + + if invoke_context.get_feature_set().increase_cpi_account_info_limit { + let account_infos_bytes = account_infos.len().saturating_mul(ACCOUNT_INFO_BYTE_SIZE); + + consume_compute_meter( + invoke_context, + (account_infos_bytes as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX), + )?; + } + + let mut account_info_keys = Vec::with_capacity(account_infos_len as usize); + #[allow(clippy::needless_range_loop)] + for account_index in 0..account_infos_len as usize { + #[allow(clippy::indexing_slicing)] + let account_info = &account_infos[account_index]; + account_info_keys.push(translate_type::( + memory_mapping, + key_addr(account_info), + check_aligned, + )?); + } + Ok((account_infos, account_info_keys)) +} + +// Finish translating accounts and build TranslatedAccount from CallerAccount. +fn translate_accounts_common<'a, T, F>( + account_info_keys: &[&Pubkey], + account_infos: &[T], + account_infos_addr: u64, + invoke_context: &InvokeContext, + memory_mapping: &MemoryMapping, + check_aligned: bool, + do_translate: F, +) -> Result>, Error> +where + F: Fn( + &InvokeContext, + &MemoryMapping, + bool, + u64, + &T, + &SerializedAccountMetadata, + ) -> Result, Error>, +{ + let transaction_context = &invoke_context.transaction_context; + let next_instruction_context = transaction_context.get_next_instruction_context()?; + let next_instruction_accounts = next_instruction_context.instruction_accounts(); + let instruction_context = transaction_context.get_current_instruction_context()?; + let mut accounts = Vec::with_capacity(next_instruction_accounts.len()); + + // unwrapping here is fine: we're in a syscall and the method below fails + // only outside syscalls + let accounts_metadata = &invoke_context.get_syscall_context().unwrap().accounts_metadata; + + for (instruction_account_index, instruction_account) in + next_instruction_accounts.iter().enumerate() + { + if next_instruction_context + .is_instruction_account_duplicate(instruction_account_index as IndexOfAccount)? + .is_some() + { + continue; // Skip duplicate account + } + + let index_in_caller = instruction_context + .get_index_of_account_in_instruction(instruction_account.index_in_transaction)?; + let callee_account = instruction_context.try_borrow_instruction_account(index_in_caller)?; + let account_key = invoke_context + .transaction_context + .get_key_of_account_at_index(instruction_account.index_in_transaction)?; + + #[allow(deprecated)] + if callee_account.is_executable() { + // Use the known account + consume_compute_meter( + invoke_context, + (callee_account.get_data().len() as u64) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX), + )?; + } else if let Some(caller_account_index) = + account_info_keys.iter().position(|key| *key == account_key) + { + let serialized_metadata = + accounts_metadata.get(index_in_caller as usize).ok_or_else(|| { + ic_msg!( + invoke_context, + "Internal error: index mismatch for account {}", + account_key + ); + Box::new(InstructionError::MissingAccount) as Error + })?; + + // build the CallerAccount corresponding to this account. + if caller_account_index >= account_infos.len() { + return Err(Box::new(CpiError::InvalidLength)); + } + #[allow(clippy::indexing_slicing)] + let caller_account = + do_translate( + invoke_context, + memory_mapping, + check_aligned, + account_infos_addr.saturating_add( + caller_account_index.saturating_mul(mem::size_of::()) as u64, + ), + &account_infos[caller_account_index], + serialized_metadata, + )?; + + consume_compute_meter( + invoke_context, + (*caller_account.ref_to_len_in_vm) + .checked_div(invoke_context.get_execution_cost().cpi_bytes_per_unit) + .unwrap_or(u64::MAX), + )?; + + accounts.push(TranslatedAccount { + index_in_caller, + caller_account, + update_caller_account_region: true, + update_caller_account_info: instruction_account.is_writable(), + }); + } else { + ic_msg!( + invoke_context, + "Instruction references an unknown account {}", + account_key + ); + return Err(Box::new(InstructionError::MissingAccount)); + } + } + + Ok(accounts) +} + +fn consume_compute_meter(invoke_context: &InvokeContext, amount: u64) -> Result<(), Error> { + invoke_context.consume_checked(amount)?; + Ok(()) +} + +// Update the given account before executing CPI. +// +// caller_account and callee_account describe the same account. At CPI entry +// caller_account might include changes the caller has made to the account +// before executing CPI. +// +// This method updates callee_account so the CPI callee can see the caller's +// changes. +// +// When true is returned, the caller account's mapped data region must be +// updated after CPI because the pointer may have changed. +fn update_callee_account( + caller_account: &CallerAccount, + mut callee_account: BorrowedInstructionAccount<'_, '_>, +) -> Result { + let mut must_update_caller = false; + + if callee_account.get_lamports() != *caller_account.lamports { + callee_account.set_lamports(*caller_account.lamports)?; + } + + let prev_len = callee_account.get_data().len(); + let post_len = *caller_account.ref_to_len_in_vm as usize; + if prev_len != post_len { + callee_account.set_data_length(post_len)?; + // pointer to data may have changed, so caller must be updated + must_update_caller = true; + } + + // Change the owner at the end so that we are allowed to change the lamports and data before + if callee_account.get_owner() != caller_account.owner { + callee_account.set_owner(caller_account.owner.as_ref())?; + // caller gave ownership and thus write access away, so caller must be updated + must_update_caller = true; + } + + Ok(must_update_caller) +} + +fn update_caller_account_region( + memory_mapping: &mut MemoryMapping, + check_aligned: bool, + caller_account: &CallerAccount, + callee_account: &mut BorrowedInstructionAccount<'_, '_>, +) -> Result<(), Error> { + let address_space_reserved_for_account = + account_data_region_size(!check_aligned, caller_account.original_data_len); + + if address_space_reserved_for_account > 0 { + // We can trust vm_data_addr to point to the correct region because we + // enforce that in CallerAccount::from_(sol_)account_info. + let (region_index, region) = memory_mapping + .find_region(caller_account.vm_data_addr) + .ok_or_else(|| Box::new(InstructionError::MissingAccount) as Error)?; + // vm_data_addr must always point to the beginning of the region + debug_assert_eq!(region.vm_addr, caller_account.vm_data_addr); + let new_region = create_memory_region_of_account(callee_account, region.vm_addr)?; + unsafe { + memory_mapping.replace_region(region_index, new_region)?; + } + } + + Ok(()) +} + +// Update the given account after executing CPI. +// +// caller_account and callee_account describe to the same account. At CPI exit +// callee_account might include changes the callee has made to the account +// after executing. +// +// This method updates caller_account so the CPI caller can see the callee's +// changes. +// +// Safety: Pointer validation guarantees that all fields of [CallerAccount] +// used here point outside the address space reserved for accounts, regardless +// of an account's current size. +fn update_caller_account( + invoke_context: &InvokeContext, + memory_mapping: &MemoryMapping, + check_aligned: bool, + caller_account: &mut CallerAccount<'_>, + callee_account: &mut BorrowedInstructionAccount<'_, '_>, +) -> Result<(), Error> { + *caller_account.lamports = callee_account.get_lamports(); + *caller_account.owner = *callee_account.get_owner(); + + let prev_len = *caller_account.ref_to_len_in_vm as usize; + let post_len = callee_account.get_data().len(); + let address_space_reserved_for_account = + account_data_region_size(!check_aligned, caller_account.original_data_len); + + if post_len > address_space_reserved_for_account { + let max_increase = + address_space_reserved_for_account.saturating_sub(caller_account.original_data_len); + ic_msg!( + invoke_context, + "Account data size realloc limited to {max_increase} in inner instructions", + ); + return Err(Box::new(InstructionError::InvalidRealloc)); + } + + if prev_len != post_len { + // this is the len field in the AccountInfo::data slice + *caller_account.ref_to_len_in_vm = post_len as u64; + + // this is the len field in the serialized parameters + let serialized_len_ptr = translate_type_mut_for_cpi::( + memory_mapping, + caller_account.vm_data_addr.saturating_sub(size_of::() as u64), + check_aligned, + )?; + *serialized_len_ptr = post_len as u64; + } + + Ok(()) +} + +#[allow(clippy::indexing_slicing)] +#[allow(clippy::arithmetic_side_effects)] +#[cfg(test)] +mod tests { + use { + super::*, + crate::{ + invoke_context::BpfAllocator, memory_context::MemoryContext, + with_mock_invoke_context_with_feature_set, + }, + assert_matches::assert_matches, + solana_account::{Account, AccountSharedData}, + solana_program_entrypoint::MAX_PERMITTED_DATA_INCREASE, + solana_sbpf::{ + ebpf::MM_INPUT_START, memory_region::MemoryRegion, program::SBPFVersion, vm::Config, + }, + solana_sdk_ids::{bpf_loader, system_program}, + solana_svm_feature_set::SVMFeatureSet, + solana_transaction_context::{ + IndexOfAccount, instruction_accounts::InstructionAccount, + transaction_accounts::KeyedAccountSharedData, + }, + std::{mem, ptr, slice}, + }; + + macro_rules! mock_invoke_context { + ($invoke_context:ident, + $transaction_context:ident, + $instruction_data:expr, + $transaction_accounts:expr, + $program_account:expr, + $instruction_accounts:expr) => { + let instruction_data = $instruction_data; + let instruction_accounts = $instruction_accounts + .iter() + .map(|index_in_transaction| { + InstructionAccount::new( + *index_in_transaction as IndexOfAccount, + false, + $transaction_accounts[*index_in_transaction as usize].2, + ) + }) + .collect::>(); + let transaction_accounts = $transaction_accounts + .into_iter() + .map(|a| (a.0, a.1)) + .collect::>(); + let feature_set = SVMFeatureSet::all_enabled(); + let feature_set = &feature_set; + with_mock_invoke_context_with_feature_set!( + $invoke_context, + $transaction_context, + feature_set, + transaction_accounts + ); + $invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + $program_account, + instruction_accounts, + instruction_data.to_vec(), + ) + .unwrap(); + $invoke_context.push().unwrap(); + }; + } + + macro_rules! borrow_instruction_account { + ($borrowed_account:ident, $invoke_context:expr, $index:expr) => { + let instruction_context = + $invoke_context.transaction_context.get_current_instruction_context().unwrap(); + let $borrowed_account = + instruction_context.try_borrow_instruction_account($index).unwrap(); + }; + } + + struct MockCallerAccount { + lamports: u64, + owner: Pubkey, + vm_addr: u64, + data: Vec, + len: u64, + regions: Vec, + } + + impl MockCallerAccount { + fn new(lamports: u64, owner: Pubkey, data: &[u8]) -> MockCallerAccount { + let vm_addr = MM_INPUT_START; + let mut region_addr = vm_addr; + let region_len = mem::size_of::(); + let mut d = vec![0; region_len]; + let mut regions = vec![]; + + unsafe { ptr::write_unaligned::(d.as_mut_ptr().cast(), data.len() as u64) }; + + regions.push(MemoryRegion::new(&raw mut d[..], vm_addr)); + region_addr += region_len as u64; + + regions.push(MemoryRegion::new(&raw const data[..], region_addr)); + + MockCallerAccount { + lamports, + owner, + vm_addr, + data: d, + len: data.len() as u64, + regions, + } + } + + fn caller_account(&mut self) -> CallerAccount<'_> { + CallerAccount { + lamports: &mut self.lamports, + owner: &mut self.owner, + original_data_len: self.len as usize, + serialized_data: &mut [], + vm_data_addr: self.vm_addr + mem::size_of::() as u64, + ref_to_len_in_vm: &mut self.len, + } + } + } + + struct MockInstruction { + program_id: Pubkey, + accounts: Vec, + data: Vec, + } + + impl MockInstruction { + fn into_region(self, vm_addr: u64) -> (Vec, MemoryRegion) { + let accounts_len = mem::size_of::() * self.accounts.len(); + + let size = mem::size_of::() + accounts_len + self.data.len(); + + let mut data = vec![0; size]; + + let vm_addr = vm_addr as usize; + let accounts_addr = vm_addr + mem::size_of::(); + let data_addr = accounts_addr + accounts_len; + + let ins = Instruction { + program_id: self.program_id, + accounts: unsafe { + Vec::from_raw_parts( + accounts_addr as *mut _, + self.accounts.len(), + self.accounts.len(), + ) + }, + data: unsafe { + Vec::from_raw_parts(data_addr as *mut _, self.data.len(), self.data.len()) + }, + }; + let ins = StableInstruction::from(ins); + + unsafe { + ptr::write_unaligned(data.as_mut_ptr().cast(), ins); + data[accounts_addr - vm_addr..][..accounts_len].copy_from_slice( + slice::from_raw_parts(self.accounts.as_ptr().cast(), accounts_len), + ); + data[data_addr - vm_addr..].copy_from_slice(&self.data); + } + + let region = MemoryRegion::new(&raw mut data[..], vm_addr as u64); + (data, region) + } + } + + type TestTransactionAccount = (Pubkey, AccountSharedData, bool); + + fn transaction_with_one_writable_instruction_account( + data: Vec, + ) -> Vec { + let program_id = Pubkey::new_unique(); + let account = AccountSharedData::from(Account { + lamports: 1, + data, + owner: program_id, + executable: false, + rent_epoch: 100, + }); + vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + false, + ), + (Pubkey::new_unique(), account, true), + ] + } + + fn transaction_with_one_readonly_instruction_account( + data: Vec, + ) -> Vec { + let program_id = Pubkey::new_unique(); + let account_owner = Pubkey::new_unique(); + let account = AccountSharedData::from(Account { + lamports: 1, + data, + owner: account_owner, + executable: false, + rent_epoch: 100, + }); + vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + false, + ), + (Pubkey::new_unique(), account, true), + ] + } + + fn mock_signers(signers: &[&[u8]], vm_addr: u64) -> (Vec, MemoryRegion) { + let vm_addr = vm_addr as usize; + + // calculate size + let fat_ptr_size_of_slice = mem::size_of::<&[()]>(); // pointer size + length size + let singers_length = signers.len(); + let sum_signers_data_length: usize = signers.iter().map(|s| s.len()).sum(); + + // init data vec + let total_size = fat_ptr_size_of_slice + + singers_length * fat_ptr_size_of_slice + + sum_signers_data_length; + let mut data = vec![0; total_size]; + + // data is composed by 3 parts + // A. + // [ singers address, singers length, ..., + // B. | + // signer1 address, signer1 length, signer2 address ..., + // ^ p1 ---> + // C. | + // signer1 data, signer2 data, ... ] + // ^ p2 ---> + + // A. + data[..fat_ptr_size_of_slice / 2] + .clone_from_slice(&(fat_ptr_size_of_slice + vm_addr).to_le_bytes()); + data[fat_ptr_size_of_slice / 2..fat_ptr_size_of_slice] + .clone_from_slice(&(singers_length).to_le_bytes()); + + // B. + C. + let (mut p1, mut p2) = ( + fat_ptr_size_of_slice, + fat_ptr_size_of_slice + singers_length * fat_ptr_size_of_slice, + ); + for signer in signers.iter() { + let signer_length = signer.len(); + + // B. + data[p1..p1 + fat_ptr_size_of_slice / 2] + .clone_from_slice(&(p2 + vm_addr).to_le_bytes()); + data[p1 + fat_ptr_size_of_slice / 2..p1 + fat_ptr_size_of_slice] + .clone_from_slice(&(signer_length).to_le_bytes()); + p1 += fat_ptr_size_of_slice; + + // C. + data[p2..p2 + signer_length].clone_from_slice(signer); + p2 += signer_length; + } + + let region = MemoryRegion::new(&raw mut data[..], vm_addr as u64); + (data, region) + } + + #[test] + fn test_translate_instruction() { + let transaction_accounts = + transaction_with_one_writable_instruction_account(b"foo".to_vec()); + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let program_id = Pubkey::new_unique(); + let accounts = vec![AccountMeta { + pubkey: Pubkey::new_unique(), + is_signer: true, + is_writable: false, + }]; + let data = b"ins data".to_vec(); + let vm_addr = MM_INPUT_START; + let (_mem, region) = MockInstruction { + program_id, + accounts: accounts.clone(), + data: data.clone(), + } + .into_region(vm_addr); + + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let memory_mapping = + unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3) }.unwrap(); + invoke_context + .memory_contexts + .set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(solana_program_entrypoint::HEAP_LENGTH as u64), + Vec::new(), + memory_mapping, + )) + .unwrap(); + + let ins = translate_instruction_rust(vm_addr, &invoke_context).unwrap(); + assert_eq!(ins.program_id, program_id); + assert_eq!(ins.accounts, accounts); + assert_eq!(ins.data, data); + } + + #[test] + fn test_translate_signers() { + let transaction_accounts = + transaction_with_one_writable_instruction_account(b"foo".to_vec()); + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let program_id = Pubkey::new_unique(); + let (derived_key, bump_seed) = Pubkey::find_program_address(&[b"foo"], &program_id); + + let vm_addr = MM_INPUT_START; + let (_mem, region) = mock_signers(&[b"foo", &[bump_seed]], vm_addr); + + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let memory_mapping = + unsafe { MemoryMapping::new(vec![region], &config, SBPFVersion::V3) }.unwrap(); + invoke_context + .memory_contexts + .set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(solana_program_entrypoint::HEAP_LENGTH as u64), + Vec::new(), + memory_mapping, + )) + .unwrap(); + + let signers = translate_signers_rust(&program_id, vm_addr, 1, &invoke_context).unwrap(); + assert_eq!(signers[0], derived_key); + } + + #[test] + fn test_get_serialized_data() { + let transaction_accounts = + transaction_with_one_writable_instruction_account(b"foo".to_vec()); + let account = transaction_accounts[1].1.clone(); + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + assert_matches!( + CallerAccount::get_serialized_data( + true, // check_aligned + account.data().len(), + account.data().len().saturating_add(MAX_PERMITTED_DATA_INCREASE).saturating_add(1), + ), + Err(error) if error.downcast_ref::().unwrap() == &InstructionError::InvalidRealloc + ); + } + + #[test] + fn test_update_caller_account_lamports_owner() { + let transaction_accounts = transaction_with_one_writable_instruction_account(vec![]); + let account = transaction_accounts[1].1.clone(); + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let mut mock_caller_account = + MockCallerAccount::new(1234, *account.owner(), account.data()); + + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let memory_mapping = unsafe { + MemoryMapping::new( + mock_caller_account.regions.split_off(0), + &config, + SBPFVersion::V3, + ) + } + .unwrap(); + + let mut caller_account = mock_caller_account.caller_account(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + let mut callee_account = instruction_context.try_borrow_instruction_account(0).unwrap(); + callee_account.set_lamports(42).unwrap(); + callee_account.set_owner(Pubkey::new_unique().as_ref()).unwrap(); + + update_caller_account( + &invoke_context, + &memory_mapping, + true, // check_aligned + &mut caller_account, + &mut callee_account, + ) + .unwrap(); + + assert_eq!(*caller_account.lamports, 42); + assert_eq!(caller_account.owner, callee_account.get_owner()); + } + + #[test] + fn test_update_caller_account_data() { + let transaction_accounts = + transaction_with_one_writable_instruction_account(b"foobar".to_vec()); + let account = transaction_accounts[1].1.clone(); + let original_data_len = account.data().len(); + + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let mut mock_caller_account = + MockCallerAccount::new(account.lamports(), *account.owner(), account.data()); + + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let memory_mapping = unsafe { + MemoryMapping::new( + mock_caller_account.regions.clone(), + &config, + SBPFVersion::V3, + ) + } + .unwrap(); + + let len_ptr = mock_caller_account.data.as_ptr(); + let serialized_len = || unsafe { *len_ptr.cast::() as usize }; + let mut caller_account = mock_caller_account.caller_account(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + let mut callee_account = instruction_context.try_borrow_instruction_account(0).unwrap(); + + for new_value in [b"foo".to_vec(), b"foobaz".to_vec(), b"foobazbad".to_vec()] { + assert!(caller_account.serialized_data.is_empty()); + callee_account.set_data_from_slice(&new_value).unwrap(); + + update_caller_account( + &invoke_context, + &memory_mapping, + true, // check_aligned + &mut caller_account, + &mut callee_account, + ) + .unwrap(); + + let data_len = callee_account.get_data().len(); + assert_eq!(data_len, *caller_account.ref_to_len_in_vm as usize); + assert_eq!(data_len, serialized_len()); + assert!(caller_account.serialized_data.is_empty()); + } + + callee_account + .set_data_length(original_data_len + MAX_PERMITTED_DATA_INCREASE) + .unwrap(); + update_caller_account( + &invoke_context, + &memory_mapping, + true, // check_aligned + &mut caller_account, + &mut callee_account, + ) + .unwrap(); + let data_len = callee_account.get_data().len(); + assert_eq!(data_len, serialized_len()); + + callee_account + .set_data_length(original_data_len + MAX_PERMITTED_DATA_INCREASE + 1) + .unwrap(); + assert_matches!( + update_caller_account( + &invoke_context, + &memory_mapping, + true, // check_aligned + &mut caller_account, + &mut callee_account, + ), + Err(error) if error.downcast_ref::().unwrap() == &InstructionError::InvalidRealloc + ); + + // close the account + callee_account.set_data_length(0).unwrap(); + callee_account.set_owner(system_program::id().as_ref()).unwrap(); + update_caller_account( + &invoke_context, + &memory_mapping, + true, // check_aligned + &mut caller_account, + &mut callee_account, + ) + .unwrap(); + let data_len = callee_account.get_data().len(); + assert_eq!(data_len, 0); + } + + #[test] + fn test_update_callee_account_lamports_owner() { + let transaction_accounts = transaction_with_one_writable_instruction_account(vec![]); + let account = transaction_accounts[1].1.clone(); + + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let mut mock_caller_account = + MockCallerAccount::new(1234, *account.owner(), account.data()); + let caller_account = mock_caller_account.caller_account(); + + borrow_instruction_account!(callee_account, invoke_context, 0); + + *caller_account.lamports = 42; + *caller_account.owner = Pubkey::new_unique(); + + update_callee_account(&caller_account, callee_account).unwrap(); + + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_eq!(callee_account.get_lamports(), 42); + assert_eq!(caller_account.owner, callee_account.get_owner()); + } + + #[test] + fn test_update_callee_account_data_writable() { + let transaction_accounts = + transaction_with_one_writable_instruction_account(b"foobar".to_vec()); + let account = transaction_accounts[1].1.clone(); + + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let mut mock_caller_account = + MockCallerAccount::new(1234, *account.owner(), account.data()); + let mut caller_account = mock_caller_account.caller_account(); + borrow_instruction_account!(callee_account, invoke_context, 0); + + assert!(caller_account.serialized_data.is_empty()); + assert!(!update_callee_account(&caller_account, callee_account).unwrap()); + + // growing resize + *caller_account.ref_to_len_in_vm = b"foobarbaz".len() as u64; + borrow_instruction_account!(callee_account, invoke_context, 0); + assert!(update_callee_account(&caller_account, callee_account).unwrap(),); + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_eq!(callee_account.get_data().len(), b"foobarbaz".len()); + drop(callee_account); + + // truncating resize + *caller_account.ref_to_len_in_vm = b"baz".len() as u64; + borrow_instruction_account!(callee_account, invoke_context, 0); + assert!(update_callee_account(&caller_account, callee_account).unwrap(),); + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_eq!(callee_account.get_data().len(), b"baz".len()); + drop(callee_account); + + // close the account + *caller_account.ref_to_len_in_vm = 0; + let mut owner = system_program::id(); + caller_account.owner = &mut owner; + borrow_instruction_account!(callee_account, invoke_context, 0); + update_callee_account(&caller_account, callee_account).unwrap(); + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_eq!(callee_account.get_data(), b""); + } + + #[test] + fn test_update_callee_account_data_readonly() { + let transaction_accounts = + transaction_with_one_readonly_instruction_account(b"foobar".to_vec()); + let account = transaction_accounts[1].1.clone(); + + mock_invoke_context!( + invoke_context, + transaction_context, + b"instruction data", + transaction_accounts, + 0, + &[1] + ); + + let mut mock_caller_account = + MockCallerAccount::new(1234, *account.owner(), account.data()); + let caller_account = mock_caller_account.caller_account(); + + // growing resize + *caller_account.ref_to_len_in_vm = b"foobarbaz".len() as u64; + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_matches!( + update_callee_account(&caller_account, callee_account), + Err(error) if error.downcast_ref::().unwrap() == &InstructionError::AccountDataSizeChanged + ); + + // truncating resize + *caller_account.ref_to_len_in_vm = b"baz".len() as u64; + borrow_instruction_account!(callee_account, invoke_context, 0); + assert_matches!( + update_callee_account(&caller_account, callee_account), + Err(error) if error.downcast_ref::().unwrap() == &InstructionError::AccountDataSizeChanged + ); + } +} diff --git a/solana/program-runtime/src/deploy.rs b/solana/program-runtime/src/deploy.rs new file mode 100644 index 00000000..ec744c3b --- /dev/null +++ b/solana/program-runtime/src/deploy.rs @@ -0,0 +1,105 @@ +//! Program deployment functionality. + +use { + crate::{ + invoke_context::InvokeContext, + loaded_programs::{ProgramCacheEntry, ProgramCacheForTxBatch, ProgramRuntimeEnvironment}, + }, + solana_instruction::error::InstructionError, + solana_pubkey::Pubkey, + solana_sbpf::{ + elf::{ElfError, Executable}, + program::BuiltinProgram, + verifier::RequisiteVerifier, + }, + solana_svm_log_collector::{LogCollector, ic_logger_msg}, + solana_svm_type_overrides::sync::Arc, + std::{cell::RefCell, rc::Rc}, +}; + +fn morph_into_deployment_environment_v1<'a>( + from: Arc>>, +) -> Result>, ElfError> { + let mut config = from.get_config().clone(); + config.reject_broken_elfs = true; + // Once the tests are being build using a toolchain which supports the newer SBPF versions, + // the deployment of older versions will be disabled: + // config.enabled_sbpf_versions = + // *config.enabled_sbpf_versions.end()..=*config.enabled_sbpf_versions.end(); + + let mut result = BuiltinProgram::new_loader(config); + + for (_key, (name, value)) in from.get_function_registry().iter() { + // Deployment of programs with sol_alloc_free is disabled. So do not register the syscall. + if name != *b"sol_alloc_free_" { + result.register_function(unsafe { std::str::from_utf8_unchecked(name) }, value)?; + } + } + + Ok(result) +} + +/// Directly deploy a program using a provided invoke context. +/// This function should only be invoked from the runtime, since it does not +/// provide any account loads or checks. +#[allow(clippy::too_many_arguments)] +pub fn deploy_program( + log_collector: Option>>, + program_cache_for_tx_batch: &mut ProgramCacheForTxBatch, + program_runtime_environment: ProgramRuntimeEnvironment, + program_id: &Pubkey, + programdata: &[u8], +) -> Result<(), InstructionError> { + let deployment_program_runtime_environment = morph_into_deployment_environment_v1(Arc::clone( + &*program_runtime_environment, + )) + .map_err(|e| { + ic_logger_msg!(log_collector, "Failed to register syscalls: {}", e); + InstructionError::ProgramEnvironmentSetupFailure + })?; + // Verify using stricter deployment_program_runtime_environment + let executable = Executable::::load( + programdata, + Arc::new(deployment_program_runtime_environment), + ) + .map_err(|err| { + ic_logger_msg!(log_collector, "{}", err); + InstructionError::InvalidAccountData + })?; + executable.verify::().map_err(|err| { + ic_logger_msg!(log_collector, "{}", err); + InstructionError::InvalidAccountData + })?; + // Reload but with program_runtime_environment + let executor = unsafe { + // SAFETY: The executable has been verified just above. + ProgramCacheEntry::reload(program_runtime_environment, programdata) + } + .map_err(|err| { + ic_logger_msg!(log_collector, "{}", err); + InstructionError::InvalidAccountData + })?; + program_cache_for_tx_batch.store_modified_entry(*program_id, Arc::new(executor)); + Ok(()) +} + +#[macro_export] +macro_rules! deploy_program { + ($invoke_context:expr, $program_id:expr, $_loader_key:expr, $_account_size:expr, $programdata:expr, $deployment_slot:expr $(,)?) => { + assert_eq!( + $deployment_slot, + $invoke_context.program_cache_for_tx_batch.slot() + ); + $crate::deploy::deploy_program( + $invoke_context.get_log_collector(), + $invoke_context.program_cache_for_tx_batch, + $invoke_context + .environment_config + .program_runtime_environments_for_execution + .get_env_for_execution() + .clone(), + $program_id, + $programdata, + )?; + }; +} diff --git a/solana/program-runtime/src/execution_budget.rs b/solana/program-runtime/src/execution_budget.rs new file mode 100644 index 00000000..3a4b9449 --- /dev/null +++ b/solana/program-runtime/src/execution_budget.rs @@ -0,0 +1,342 @@ +use { + solana_fee_structure::FeeDetails, solana_program_entrypoint::HEAP_LENGTH, + solana_transaction_context::MAX_INSTRUCTION_TRACE_LENGTH, std::num::NonZeroU32, +}; + +/// Max instruction stack depth. This is the maximum nesting of instructions that can happen during +/// a transaction. +pub const MAX_INSTRUCTION_STACK_DEPTH: usize = 5; +/// Max instruction stack depth with SIMD-0268 enabled. Allows 8 nested CPIs. +pub const MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268: usize = 9; + +fn get_max_instruction_stack_depth(simd_0268_active: bool) -> usize { + if simd_0268_active { + MAX_INSTRUCTION_STACK_DEPTH_SIMD_0268 + } else { + MAX_INSTRUCTION_STACK_DEPTH + } +} + +/// Default CPI invocation cost. +pub const DEFAULT_INVOCATION_COST: u64 = 1000; +/// CPI invocation cost with SIMD-0339 active. +pub const INVOKE_UNITS_COST_SIMD_0339: u64 = 946; + +fn get_invoke_unit_cost(simd_0339_active: bool) -> u64 { + if simd_0339_active { + INVOKE_UNITS_COST_SIMD_0339 + } else { + DEFAULT_INVOCATION_COST + } +} + +/// Max call depth. This is the maximum nesting of SBF to SBF call that can happen within a program. +pub const MAX_CALL_DEPTH: usize = 64; + +/// The size of one SBF stack frame. +pub const STACK_FRAME_SIZE: usize = 4096; + +/// Maximum compute units a transaction may request. +pub const MAX_COMPUTE_UNIT_LIMIT: u32 = 1_400_000; + +/// Roughly 0.5us/page, where page is 32K; given roughly 15CU/us, the +/// default heap page cost = 0.5 * 15 ~= 8CU/page +pub const DEFAULT_HEAP_COST: u64 = 8; +/// Default compute-unit limit for an instruction. +pub const DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT: u32 = 200_000; +/// Maximum compute units allocated to native builtins that have not moved to SBF. +pub const MAX_BUILTIN_ALLOCATION_COMPUTE_UNIT_LIMIT: u32 = 3_000; +/// Maximum heap frame size a transaction may request. +pub const MAX_HEAP_FRAME_BYTES: u32 = 256 * 1024; +/// Minimum heap frame size. +pub const MIN_HEAP_FRAME_BYTES: u32 = HEAP_LENGTH as u32; + +/// Default loaded account data limit for one transaction. +pub const MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES: NonZeroU32 = + NonZeroU32::new(64 * 1024 * 1024).unwrap(); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SVMTransactionExecutionBudget { + /// Number of compute units that a transaction or individual instruction is + /// allowed to consume. Compute units are consumed by program execution, + /// resources they use, etc... + pub compute_unit_limit: u64, + /// Maximum program instruction invocation stack depth. Invocation stack + /// depth starts at 1 for transaction instructions and the stack depth is + /// incremented each time a program invokes an instruction and decremented + /// when a program returns. + pub max_instruction_stack_depth: usize, + /// Maximum cross-program invocation and instructions per transaction + pub max_instruction_trace_length: usize, + /// Maximum number of slices hashed per syscall + pub sha256_max_slices: u64, + /// Maximum SBF to BPF call depth + pub max_call_depth: usize, + /// Size of a stack frame in bytes, must match the size specified in the LLVM SBF backend + pub stack_frame_size: usize, + /// program heap region size, default: solana_program_entrypoint::HEAP_LENGTH + pub heap_size: u32, +} + +impl Default for SVMTransactionExecutionBudget { + fn default() -> Self { + Self::new_with_defaults(/* simd_0268_active */ false) + } +} + +impl SVMTransactionExecutionBudget { + /// Creates the default execution budget for the selected feature state. + pub fn new_with_defaults(simd_0268_active: bool) -> Self { + SVMTransactionExecutionBudget { + compute_unit_limit: u64::from(MAX_COMPUTE_UNIT_LIMIT), + max_instruction_stack_depth: get_max_instruction_stack_depth(simd_0268_active), + max_instruction_trace_length: MAX_INSTRUCTION_TRACE_LENGTH, + sha256_max_slices: 20_000, + max_call_depth: MAX_CALL_DEPTH, + stack_frame_size: STACK_FRAME_SIZE, + heap_size: u32::try_from(solana_program_entrypoint::HEAP_LENGTH).unwrap(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SVMTransactionExecutionCost { + /// Number of compute units consumed by a log_u64 call + pub log_64_units: u64, + /// Number of compute units consumed by a create_program_address call + pub create_program_address_units: u64, + /// Number of compute units consumed by an invoke call (not including the cost incurred by + /// the called program) + pub invoke_units: u64, + /// Base number of compute units consumed to call SHA256 + pub sha256_base_cost: u64, + /// Incremental number of units consumed by SHA256 (based on bytes) + pub sha256_byte_cost: u64, + /// Number of compute units consumed by logging a `Pubkey` + pub log_pubkey_units: u64, + /// Number of account data bytes per compute unit charged during a cross-program invocation + pub cpi_bytes_per_unit: u64, + /// Base number of compute units consumed to get a sysvar + pub sysvar_base_cost: u64, + /// Number of compute units consumed to call secp256k1_recover + pub secp256k1_recover_cost: u64, + /// Number of compute units consumed to do a syscall without any work + pub syscall_base_cost: u64, + /// Number of compute units consumed to validate a curve25519 edwards point + pub curve25519_edwards_validate_point_cost: u64, + /// Number of compute units consumed to add two curve25519 edwards points + pub curve25519_edwards_add_cost: u64, + /// Number of compute units consumed to subtract two curve25519 edwards points + pub curve25519_edwards_subtract_cost: u64, + /// Number of compute units consumed to multiply a curve25519 edwards point + pub curve25519_edwards_multiply_cost: u64, + /// Number of compute units consumed for a multiscalar multiplication (msm) of edwards points. + /// The total cost is calculated as `msm_base_cost + (length - 1) * msm_incremental_cost`. + pub curve25519_edwards_msm_base_cost: u64, + /// Number of compute units consumed for a multiscalar multiplication (msm) of edwards points. + /// The total cost is calculated as `msm_base_cost + (length - 1) * msm_incremental_cost`. + pub curve25519_edwards_msm_incremental_cost: u64, + /// Number of compute units consumed to validate a curve25519 ristretto point + pub curve25519_ristretto_validate_point_cost: u64, + /// Number of compute units consumed to add two curve25519 ristretto points + pub curve25519_ristretto_add_cost: u64, + /// Number of compute units consumed to subtract two curve25519 ristretto points + pub curve25519_ristretto_subtract_cost: u64, + /// Number of compute units consumed to multiply a curve25519 ristretto point + pub curve25519_ristretto_multiply_cost: u64, + /// Number of compute units consumed for a multiscalar multiplication (msm) of ristretto points. + /// The total cost is calculated as `msm_base_cost + (length - 1) * msm_incremental_cost`. + pub curve25519_ristretto_msm_base_cost: u64, + /// Number of compute units consumed for a multiscalar multiplication (msm) of ristretto points. + /// The total cost is calculated as `msm_base_cost + (length - 1) * msm_incremental_cost`. + pub curve25519_ristretto_msm_incremental_cost: u64, + /// Number of compute units per additional 32k heap above the default (~.5 + /// us per 32k at 15 units/us rounded up) + pub heap_cost: u64, + /// Memory operation syscall base cost + pub mem_op_base_cost: u64, + /// Number of compute units consumed to call alt_bn128_g1_addition + pub alt_bn128_g1_addition_cost: u64, + /// Number of compute units consumed to call alt_bn128_g2_addition + pub alt_bn128_g2_addition_cost: u64, + /// Number of compute units consumed to call alt_bn128_g1_multiplication. + pub alt_bn128_g1_multiplication_cost: u64, + /// Number of compute units consumed to call alt_bn128_g2_multiplication. + pub alt_bn128_g2_multiplication_cost: u64, + /// Total cost will be alt_bn128_pairing_one_pair_cost_first + /// + alt_bn128_pairing_one_pair_cost_other * (num_elems - 1) + pub alt_bn128_pairing_one_pair_cost_first: u64, + pub alt_bn128_pairing_one_pair_cost_other: u64, + /// Big integer modular exponentiation base cost + pub big_modular_exponentiation_base_cost: u64, + /// Big integer moduler exponentiation cost divisor + /// The modular exponentiation cost is computed as + /// `input_length`/`big_modular_exponentiation_cost_divisor` + `big_modular_exponentiation_base_cost` + pub big_modular_exponentiation_cost_divisor: u64, + /// Coefficient `a` of the quadratic function which determines the number + /// of compute units consumed to call poseidon syscall for a given number + /// of inputs. + pub poseidon_cost_coefficient_a: u64, + /// Coefficient `c` of the quadratic function which determines the number + /// of compute units consumed to call poseidon syscall for a given number + /// of inputs. + pub poseidon_cost_coefficient_c: u64, + /// Number of compute units consumed for accessing the remaining compute units. + pub get_remaining_compute_units_cost: u64, + /// Number of compute units consumed to call alt_bn128_g1_compress. + pub alt_bn128_g1_compress: u64, + /// Number of compute units consumed to call alt_bn128_g1_decompress. + pub alt_bn128_g1_decompress: u64, + /// Number of compute units consumed to call alt_bn128_g2_compress. + pub alt_bn128_g2_compress: u64, + /// Number of compute units consumed to call alt_bn128_g2_decompress. + pub alt_bn128_g2_decompress: u64, + /// Number of compute units consumed to add two bls12_381 g1 points. + pub bls12_381_g1_add_cost: u64, + /// Number of compute units consumed to add two bls12_381 g2 points. + pub bls12_381_g2_add_cost: u64, + /// Number of compute units consumed to subtract two bls12_381 g1 points. + pub bls12_381_g1_subtract_cost: u64, + /// Number of compute units consumed to subtract two bls12_381 g2 points. + pub bls12_381_g2_subtract_cost: u64, + /// Number of compute units consumed to multiply a bls12_381 g1 point. + pub bls12_381_g1_multiply_cost: u64, + /// Number of compute units consumed to multiply a bls12_381 g2 point. + pub bls12_381_g2_multiply_cost: u64, + /// Number of compute units consumed to decompress a bls12_381 g1 point. + pub bls12_381_g1_decompress_cost: u64, + /// Number of compute units consumed to decompress a bls12_381 g2 point. + pub bls12_381_g2_decompress_cost: u64, + /// Number of compute units consumed to validate a bls12_381 g1 point. + pub bls12_381_g1_validate_cost: u64, + /// Number of compute units consumed to validate a bls12_381 g2 point. + pub bls12_381_g2_validate_cost: u64, + /// Base number of compute units consumed to perform a bls12_381 pairing. + pub bls12_381_one_pair_cost: u64, + /// Incremental number of compute units consumed per pair in a bls12_381 pairing. + pub bls12_381_additional_pair_cost: u64, +} + +impl Default for SVMTransactionExecutionCost { + fn default() -> Self { + Self::new_with_defaults(/* simd_0339_active */ false) + } +} + +impl SVMTransactionExecutionCost { + pub fn new_with_defaults(simd_0339_active: bool) -> Self { + SVMTransactionExecutionCost { + log_64_units: 100, + create_program_address_units: 1500, + invoke_units: get_invoke_unit_cost(simd_0339_active), + sha256_base_cost: 85, + sha256_byte_cost: 1, + log_pubkey_units: 100, + cpi_bytes_per_unit: 250, // ~50MB at 200,000 units + sysvar_base_cost: 100, + secp256k1_recover_cost: 25_000, + syscall_base_cost: 100, + curve25519_edwards_validate_point_cost: 159, + curve25519_edwards_add_cost: 473, + curve25519_edwards_subtract_cost: 475, + curve25519_edwards_multiply_cost: 2_177, + curve25519_edwards_msm_base_cost: 2_273, + curve25519_edwards_msm_incremental_cost: 758, + curve25519_ristretto_validate_point_cost: 169, + curve25519_ristretto_add_cost: 521, + curve25519_ristretto_subtract_cost: 519, + curve25519_ristretto_multiply_cost: 2_208, + curve25519_ristretto_msm_base_cost: 2303, + curve25519_ristretto_msm_incremental_cost: 788, + heap_cost: DEFAULT_HEAP_COST, + mem_op_base_cost: 10, + alt_bn128_g1_addition_cost: 334, + alt_bn128_g2_addition_cost: 535, + alt_bn128_g1_multiplication_cost: 3_840, + alt_bn128_g2_multiplication_cost: 15_670, + alt_bn128_pairing_one_pair_cost_first: 36_364, + alt_bn128_pairing_one_pair_cost_other: 12_121, + big_modular_exponentiation_base_cost: 190, + big_modular_exponentiation_cost_divisor: 2, + poseidon_cost_coefficient_a: 61, + poseidon_cost_coefficient_c: 542, + get_remaining_compute_units_cost: 100, + alt_bn128_g1_compress: 30, + alt_bn128_g1_decompress: 398, + alt_bn128_g2_compress: 86, + alt_bn128_g2_decompress: 13610, + bls12_381_g1_add_cost: 128, + bls12_381_g2_add_cost: 203, + bls12_381_g1_subtract_cost: 129, + bls12_381_g2_subtract_cost: 204, + bls12_381_g1_multiply_cost: 4_627, + bls12_381_g2_multiply_cost: 8_255, + bls12_381_g1_decompress_cost: 2_100, + bls12_381_g2_decompress_cost: 3_050, + bls12_381_g1_validate_cost: 1_565, + bls12_381_g2_validate_cost: 1_968, + bls12_381_one_pair_cost: 25_445, + bls12_381_additional_pair_cost: 13_023, + } + } + + /// Returns cost of the Poseidon hash function for the given number of + /// inputs is determined by the following quadratic function: + /// + /// 61*n^2 + 542 + /// + /// Which approximates the results of benchmarks of light-posiedon + /// library[0]. These results assume 1 CU per 33 ns. Examples: + /// + /// * 1 input + /// * light-poseidon benchmark: `18,303 / 33 ≈ 555` + /// * function: `61*1^2 + 542 = 603` + /// * 2 inputs + /// * light-poseidon benchmark: `25,866 / 33 ≈ 784` + /// * function: `61*2^2 + 542 = 786` + /// * 3 inputs + /// * light-poseidon benchmark: `37,549 / 33 ≈ 1,138` + /// * function; `61*3^2 + 542 = 1091` + /// + /// [0] https://github.com/Lightprotocol/light-poseidon#performance + pub fn poseidon_cost(&self, nr_inputs: u64) -> Option { + let squared_inputs = nr_inputs.checked_pow(2)?; + let mul_result = self.poseidon_cost_coefficient_a.checked_mul(squared_inputs)?; + let final_result = mul_result.checked_add(self.poseidon_cost_coefficient_c)?; + + Some(final_result) + } +} + +/// Execution budget, loaded-data limit, and fee metadata for one transaction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SVMTransactionExecutionAndFeeBudgetLimits { + /// Compute and invocation limits. + pub budget: SVMTransactionExecutionBudget, + /// Maximum loaded account data size. + pub loaded_accounts_data_size_limit: u32, + /// Fee metadata carried with the transaction. + pub fee_details: FeeDetails, +} + +#[cfg(feature = "dev-context-only-utils")] +impl Default for SVMTransactionExecutionAndFeeBudgetLimits { + fn default() -> Self { + Self { + budget: SVMTransactionExecutionBudget::default(), + loaded_accounts_data_size_limit: MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + fee_details: FeeDetails::default(), + } + } +} + +#[cfg(feature = "dev-context-only-utils")] +impl SVMTransactionExecutionAndFeeBudgetLimits { + /// Creates default limits with explicit fee metadata. + pub fn with_fee(fee_details: FeeDetails) -> Self { + Self { + fee_details, + ..SVMTransactionExecutionAndFeeBudgetLimits::default() + } + } +} diff --git a/solana/program-runtime/src/invoke_context.rs b/solana/program-runtime/src/invoke_context.rs new file mode 100644 index 00000000..8666552e --- /dev/null +++ b/solana/program-runtime/src/invoke_context.rs @@ -0,0 +1,1891 @@ +use { + crate::{ + execution_budget::{SVMTransactionExecutionBudget, SVMTransactionExecutionCost}, + loaded_programs::{ + ProgramCacheEntry, ProgramCacheEntryType, ProgramCacheForTxBatch, + ProgramRuntimeEnvironments, + }, + memory_context::{MemoryContext, MemoryContexts}, + stable_log, + sysvar_cache::SysvarCache, + }, + solana_account::{AccountSharedData, create_account_shared_data_for_test}, + solana_epoch_schedule::EpochSchedule, + solana_hash::Hash, + solana_instruction::{AccountMeta, Instruction, error::InstructionError}, + solana_pubkey::Pubkey, + solana_sbpf::{ + ebpf::MM_HEAP_START, + elf::Executable as GenericExecutable, + error::{EbpfError, ProgramResult}, + memory_region::MemoryMapping, + program::{BuiltinCodegen, BuiltinFunction, SBPFVersion}, + vm::{Config, ContextObject, EbpfVm}, + }, + solana_sdk_ids::{ + bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable, loader_v4, native_loader, sysvar, + }, + solana_svm_callback::InvokeContextCallback, + solana_svm_feature_set::SVMFeatureSet, + solana_svm_log_collector::{LogCollector, ic_msg}, + solana_svm_measure::measure::Measure, + solana_svm_timings::ExecuteDetailsTimings, + solana_svm_transaction::{instruction::SVMInstruction, svm_message::SVMMessage}, + solana_svm_type_overrides::sync::Arc, + solana_transaction_context::{ + IndexOfAccount, MAX_ACCOUNTS_PER_TRANSACTION, instruction::InstructionContext, + instruction_accounts::InstructionAccount, transaction::TransactionContext, + transaction_accounts::KeyedAccountSharedData, + }, + std::{ + alloc::Layout, + borrow::Cow, + cell::{Cell, RefCell}, + fmt::{self, Debug}, + ptr::NonNull, + rc::Rc, + }, +}; + +pub use crate::memory_context::SerializedAccountMetadata; + +pub type BuiltinFunctionWithContext = ( + BuiltinFunction>, + BuiltinCodegen>, +); +pub type Executable = GenericExecutable>; +pub type RegisterTrace<'a> = &'a [[u64; 12]]; + +/// Adapter so we can unify the interfaces of built-in programs and syscalls +#[macro_export] +macro_rules! declare_process_instruction { + ($process_instruction:ident, $cu_to_consume:expr, |$invoke_context:ident| $inner:tt) => { + $crate::solana_sbpf::declare_builtin_function!( + $process_instruction, + fn rust( + invoke_context: &mut $crate::invoke_context::InvokeContext<'_, '_>, + _arg0: u64, + _arg1: u64, + _arg2: u64, + _arg3: u64, + _arg4: u64, + ) -> Result> { + fn process_instruction_inner( + $invoke_context: &mut $crate::invoke_context::InvokeContext<'_, '_>, + ) -> std::result::Result<(), $crate::__private::InstructionError> + $inner + + let consumption_result = if $cu_to_consume > 0 + { + invoke_context.consume_checked($cu_to_consume) + } else { + Ok(()) + }; + consumption_result + .and_then(|_| { + process_instruction_inner(invoke_context) + .map(|_| 0) + .map_err(|err| Box::new(err) as Box) + }) + .into() + } + ); + }; +} + +impl ContextObject for InvokeContext<'_, '_> { + fn consume(&mut self, amount: u64) { + // 1 to 1 instruction to compute unit mapping + // ignore overflow, Ebpf will bail if exceeded + let compute_meter = self.compute_meter.0.get(); + self.compute_meter.0.set(compute_meter.saturating_sub(amount)); + } + + fn get_remaining(&self) -> u64 { + self.compute_meter.0.get() + } + + fn active_mapping_ptr(&mut self) -> NonNull { + let memory_mapping = self + .memory_contexts + .memory_mapping_mut() + .expect("memory context must be set for the current instruction"); + NonNull::from(memory_mapping) + } +} + +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct AllocErr; +impl fmt::Display for AllocErr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("Error: Memory allocation failed") + } +} + +pub struct BpfAllocator { + len: u64, + pos: u64, +} + +impl BpfAllocator { + pub fn new(len: u64) -> Self { + Self { len, pos: 0 } + } + + pub fn alloc(&mut self, layout: Layout) -> Result { + let bytes_to_align = (self.pos as *const u8).align_offset(layout.align()) as u64; + if self.pos.saturating_add(bytes_to_align).saturating_add(layout.size() as u64) <= self.len + { + self.pos = self.pos.saturating_add(bytes_to_align); + let addr = MM_HEAP_START.saturating_add(self.pos); + self.pos = self.pos.saturating_add(layout.size() as u64); + Ok(addr) + } else { + Err(AllocErr) + } + } +} + +pub struct EnvironmentConfig<'a> { + pub blockhash: Hash, + pub blockhash_lamports_per_signature: u64, + epoch_stake_callback: &'a dyn InvokeContextCallback, + feature_set: &'a SVMFeatureSet, + pub program_runtime_environments_for_execution: &'a ProgramRuntimeEnvironments, + sysvar_cache: &'a SysvarCache, +} +impl<'a> EnvironmentConfig<'a> { + pub fn new( + blockhash: Hash, + blockhash_lamports_per_signature: u64, + epoch_stake_callback: &'a dyn InvokeContextCallback, + feature_set: &'a SVMFeatureSet, + program_runtime_environments_for_execution: &'a ProgramRuntimeEnvironments, + sysvar_cache: &'a SysvarCache, + ) -> Self { + Self { + blockhash, + blockhash_lamports_per_signature, + epoch_stake_callback, + feature_set, + program_runtime_environments_for_execution, + sysvar_cache, + } + } + + /// Get cached sysvars. + pub fn sysvar_cache(&self) -> &SysvarCache { + self.sysvar_cache + } +} + +pub struct SyscallContext { + pub allocator: BpfAllocator, + pub accounts_metadata: Vec, +} + +pub struct ComputeMeter(Cell); + +impl ComputeMeter { + /// Consume compute units. + pub fn consume_checked(&self, amount: u64) -> Result<(), Box> { + let compute_meter = self.0.get(); + let exceeded = compute_meter < amount; + self.0.set(compute_meter.saturating_sub(amount)); + if exceeded { + return Err(Box::new(InstructionError::ComputationalBudgetExceeded)); + } + Ok(()) + } + + /// Set compute units. Only use for tests and benchmarks. + pub fn mock_set_remaining(&self, remaining: u64) { + self.0.set(remaining); + } +} + +/// Main pipeline from runtime to program execution. +pub struct InvokeContext<'a, 'ix_data> { + /// Information about the currently executing transaction. + pub transaction_context: &'a mut TransactionContext<'ix_data>, + /// The local program cache for the transaction batch. + pub program_cache_for_tx_batch: &'a mut ProgramCacheForTxBatch, + /// Runtime configurations used to provision the invocation environment. + pub environment_config: EnvironmentConfig<'a>, + /// The compute budget for the current invocation. + compute_budget: SVMTransactionExecutionBudget, + /// The compute cost for the current invocation. + execution_cost: SVMTransactionExecutionCost, + /// Instruction compute meter, for tracking compute units consumed against + /// the designated compute budget during program execution. + pub compute_meter: ComputeMeter, + log_collector: Option>>, + /// Latest measurement not yet accumulated in [ExecuteDetailsTimings::execute_us] + pub execute_time: Option, + pub timings: ExecuteDetailsTimings, + pub syscall_context: Vec>, + pub memory_contexts: MemoryContexts, + /// Pairs of index in TX instruction trace and VM register trace + register_traces: Vec<(usize, Vec<[u64; 12]>)>, +} + +impl<'a, 'ix_data> InvokeContext<'a, 'ix_data> { + #[allow(clippy::too_many_arguments)] + pub fn new( + transaction_context: &'a mut TransactionContext<'ix_data>, + program_cache_for_tx_batch: &'a mut ProgramCacheForTxBatch, + environment_config: EnvironmentConfig<'a>, + log_collector: Option>>, + compute_budget: SVMTransactionExecutionBudget, + execution_cost: SVMTransactionExecutionCost, + ) -> Self { + Self { + transaction_context, + program_cache_for_tx_batch, + environment_config, + log_collector, + compute_budget, + execution_cost, + compute_meter: ComputeMeter(Cell::new(compute_budget.compute_unit_limit)), + execute_time: None, + timings: ExecuteDetailsTimings::default(), + syscall_context: Vec::new(), + memory_contexts: MemoryContexts::new(), + register_traces: Vec::new(), + } + } + + /// Push a stack frame onto the invocation stack + pub fn push(&mut self) -> Result<(), InstructionError> { + let instruction_context = self.transaction_context.get_next_instruction_context()?; + let program_id = instruction_context + .get_program_key() + .map_err(|_| InstructionError::UnsupportedProgramId)?; + if self.transaction_context.get_instruction_stack_height() != 0 { + let contains = + (0..self.transaction_context.get_instruction_stack_height()).any(|level| { + self.transaction_context + .get_instruction_context_at_nesting_level(level) + .and_then(|instruction_context| instruction_context.get_program_key()) + .map(|program_key| program_key == program_id) + .unwrap_or(false) + }); + let is_last = self + .transaction_context + .get_current_instruction_context() + .and_then(|instruction_context| instruction_context.get_program_key()) + .map(|program_key| program_key == program_id) + .unwrap_or(false); + if contains && !is_last { + // Reentrancy not allowed unless caller is calling itself + return Err(InstructionError::ReentrancyNotAllowed); + } + } + + self.syscall_context.push(None); + self.memory_contexts.push_placeholder(); + self.transaction_context.push() + } + + /// Pop a stack frame from the invocation stack + pub(crate) fn pop(&mut self) -> Result<(), InstructionError> { + self.syscall_context.pop(); + self.memory_contexts.pop(); + self.transaction_context.pop() + } + + /// Current height of the invocation stack, top level instructions are height + /// `solana_instruction::TRANSACTION_LEVEL_STACK_HEIGHT` + pub fn get_stack_height(&self) -> usize { + self.transaction_context.get_instruction_stack_height() + } + + /// Entrypoint for a cross-program invocation from a builtin program. + /// + /// Takes signer seeds and derives PDAs internally via + /// `create_program_address`, mirroring the SBF CPI path. This makes + /// it structurally impossible for a builtin to vouch for a non-PDA + /// address (e.g. a user wallet) as a signer. + pub fn native_invoke_signed( + &mut self, + instruction: Instruction, + signer_seeds: &[&[&[u8]]], + ) -> Result<(), InstructionError> { + let caller_program_id = + *self.transaction_context.get_current_instruction_context()?.get_program_key()?; + // The conversion from `PubkeyError` to `InstructionError` through + // num-traits is incorrect, but it's the existing behavior. + let signers = signer_seeds + .iter() + .map(|seeds| Pubkey::create_program_address(seeds, &caller_program_id)) + .collect::, solana_pubkey::PubkeyError>>() + .map_err(|e| e as u64)?; + self.prepare_next_cpi_instruction(instruction, &signers)?; + self.process_instruction(&mut 0)?; + Ok(()) + } + + /// Deprecated entrypoint for a cross-program invocation from a builtin program + // NOTE: + // we only keep it around for one special case of CPI with post-delegation actions + pub fn native_invoke( + &mut self, + instruction: Instruction, + signers: &[Pubkey], + ) -> Result<(), InstructionError> { + self.prepare_next_cpi_instruction(instruction, signers)?; + self.process_instruction(&mut 0)?; + Ok(()) + } + + /// Helper to prepare for process_instruction() when the instruction is not a top level one, + /// and depends on `AccountMeta`s + pub fn prepare_next_cpi_instruction( + &mut self, + instruction: Instruction, + signers: &[Pubkey], + ) -> Result<(), InstructionError> { + // We reference accounts by an u8 index, so we have a total of 256 accounts. + let mut transaction_callee_map: Vec = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; + let mut instruction_accounts: Vec = + Vec::with_capacity(instruction.accounts.len()); + + // This code block is necessary to restrict the scope of the immutable borrow of + // transaction context (the `instruction_context` variable). At the end of this + // function, we must borrow it again as mutable. + let program_account_index = { + let instruction_context = self.transaction_context.get_current_instruction_context()?; + + for account_meta in instruction.accounts.iter() { + let index_in_transaction = self + .transaction_context + .find_index_of_account(&account_meta.pubkey) + .ok_or_else(|| { + ic_msg!( + self, + "Instruction references an unknown account {}", + account_meta.pubkey, + ); + InstructionError::MissingAccount + })?; + + debug_assert!((index_in_transaction as usize) < transaction_callee_map.len()); + let index_in_callee = + transaction_callee_map.get_mut(index_in_transaction as usize).unwrap(); + + if (*index_in_callee as usize) < instruction_accounts.len() { + let cloned_account = { + let instruction_account = instruction_accounts + .get_mut(*index_in_callee as usize) + .ok_or(InstructionError::MissingAccount)?; + instruction_account.set_is_signer( + instruction_account.is_signer() || account_meta.is_signer, + ); + instruction_account.set_is_writable( + instruction_account.is_writable() || account_meta.is_writable, + ); + *instruction_account + }; + instruction_accounts.push(cloned_account); + } else { + *index_in_callee = instruction_accounts.len() as u16; + instruction_accounts.push(InstructionAccount::new( + index_in_transaction, + account_meta.is_signer, + account_meta.is_writable, + )); + } + } + + for current_index in 0..instruction_accounts.len() { + let instruction_account = instruction_accounts.get(current_index).unwrap(); + let index_in_callee = *transaction_callee_map + .get(instruction_account.index_in_transaction as usize) + .unwrap() as usize; + + if current_index != index_in_callee { + let (is_signer, is_writable) = { + let reference_account = instruction_accounts + .get(index_in_callee) + .ok_or(InstructionError::MissingAccount)?; + ( + reference_account.is_signer(), + reference_account.is_writable(), + ) + }; + + let current_account = instruction_accounts.get_mut(current_index).unwrap(); + current_account.set_is_signer(current_account.is_signer() || is_signer); + current_account.set_is_writable(current_account.is_writable() || is_writable); + // This account is repeated, so there is no need to check for permissions + continue; + } + + let index_in_caller = instruction_context.get_index_of_account_in_instruction( + instruction_account.index_in_transaction, + )?; + + // This unwrap is safe because instruction.accounts.len() == instruction_accounts.len() + let account_key = &instruction.accounts.get(current_index).unwrap().pubkey; + // get_index_of_account_in_instruction has already checked if the index is valid. + let caller_instruction_account = instruction_context + .instruction_accounts() + .get(index_in_caller as usize) + .unwrap(); + + // Readonly in caller cannot become writable in callee + if instruction_account.is_writable() && !caller_instruction_account.is_writable() { + ic_msg!(self, "{}'s writable privilege escalated", account_key,); + return Err(InstructionError::PrivilegeEscalation); + } + + // To be signed in the callee, + // it must be either signed in the caller or by the program + if instruction_account.is_signer() + && !(caller_instruction_account.is_signer() || signers.contains(account_key)) + { + ic_msg!(self, "{}'s signer privilege escalated", account_key,); + return Err(InstructionError::PrivilegeEscalation); + } + } + + // Find and validate executables / program accounts + let callee_program_id = &instruction.program_id; + let program_account_index_in_transaction = + self.transaction_context.find_index_of_account(callee_program_id); + let program_account_index_in_instruction = program_account_index_in_transaction + .map(|index| instruction_context.get_index_of_account_in_instruction(index)); + + // We first check if the account exists in the transaction, and then see if it is part + // of the instruction. + if program_account_index_in_instruction.is_none() + || program_account_index_in_instruction.unwrap().is_err() + { + ic_msg!(self, "Unknown program {}", callee_program_id); + return Err(InstructionError::MissingAccount); + } + + // SAFETY: This unwrap is safe, because we checked the index in instruction in the + // previous if-condition. + program_account_index_in_transaction.unwrap() + }; + + // This ? operator should not error out because `fn get_current_instruction_index` is also called + // in `get_current_instruction_context` + let caller_index = self.transaction_context.get_current_instruction_index()?; + self.transaction_context.configure_instruction_at_index( + self.transaction_context.get_instruction_trace_length(), + program_account_index, + instruction_accounts, + transaction_callee_map, + Cow::Owned(instruction.data), + Some(caller_index as u16), + )?; + Ok(()) + } + + /// Helper to prepare for process_instruction()/process_precompile() when the instruction is + /// a top level one + pub fn prepare_next_top_level_instruction( + &mut self, + message: &impl SVMMessage, + instruction: &SVMInstruction, + program_account_index: IndexOfAccount, + data: &'ix_data [u8], + ) -> Result<(), InstructionError> { + // We reference accounts by an u8 index, so we have a total of 256 accounts. + let mut transaction_callee_map: Vec = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; + + let mut instruction_accounts: Vec = + Vec::with_capacity(instruction.accounts.len()); + for index_in_transaction in instruction.accounts.iter() { + debug_assert!((*index_in_transaction as usize) < transaction_callee_map.len()); + + let index_in_callee = + transaction_callee_map.get_mut(*index_in_transaction as usize).unwrap(); + + if (*index_in_callee as usize) > instruction_accounts.len() { + *index_in_callee = instruction_accounts.len() as u16; + } + + let index_in_transaction = *index_in_transaction as usize; + instruction_accounts.push(InstructionAccount::new( + index_in_transaction as IndexOfAccount, + message.is_signer(index_in_transaction), + message.is_writable(index_in_transaction), + )); + } + + self.transaction_context.configure_instruction_at_index( + self.transaction_context.get_instruction_trace_length(), + program_account_index, + instruction_accounts, + transaction_callee_map, + Cow::Borrowed(data), + None, + )?; + Ok(()) + } + + /// Processes an instruction and returns how many compute units were used + pub fn process_instruction( + &mut self, + compute_units_consumed: &mut u64, + ) -> Result<(), InstructionError> { + *compute_units_consumed = 0; + self.push()?; + self.process_executable_chain(compute_units_consumed) + // MUST pop if and only if `push` succeeded, independent of `result`. + // Thus, the `.and()` instead of an `.and_then()`. + .and(self.pop()) + } + + /// Processes a precompile instruction + pub fn process_precompile( + &mut self, + program_id: &Pubkey, + instruction_data: &[u8], + message_instruction_datas_iter: impl Iterator, + ) -> Result<(), InstructionError> { + self.push()?; + let instruction_datas: Vec<_> = message_instruction_datas_iter.collect(); + self.environment_config + .epoch_stake_callback + .process_precompile(program_id, instruction_data, instruction_datas) + .map_err(InstructionError::from) + .and(self.pop()) + } + + /// Calls the instruction's program entrypoint method + fn process_executable_chain( + &mut self, + compute_units_consumed: &mut u64, + ) -> Result<(), InstructionError> { + let instruction_context = self.transaction_context.get_current_instruction_context()?; + let program_id = *instruction_context.get_program_key()?; + let owner_id = instruction_context.get_program_owner()?; + let process_executable_chain_time = Measure::start("process_executable_chain_time"); + + let cache_id = if native_loader::check_id(&owner_id) + || bpf_loader_deprecated::check_id(&owner_id) + || bpf_loader::check_id(&owner_id) + || bpf_loader_upgradeable::check_id(&owner_id) + || loader_v4::check_id(&owner_id) + { + program_id + } else { + return Err(InstructionError::UnsupportedProgramId); + }; + + let pre_remaining_units = self.get_remaining(); + let entry = self + .program_cache_for_tx_batch + .find(&cache_id) + .ok_or(InstructionError::UnsupportedProgramId)?; + let result = match &entry.program { + ProgramCacheEntryType::Builtin(program) => { + // The Murmur3 hash value (used by RBPF) of the string "entrypoint". + const ENTRYPOINT_KEY: u32 = 0x71E3CF81; + let function = program + .get_function_registry() + .lookup_by_key(ENTRYPOINT_KEY) + .map(|(_name, (function, _codegen))| function) + .ok_or(InstructionError::UnsupportedProgramId)?; + + self.transaction_context.set_return_data(program_id, Vec::new())?; + let logger = self.get_log_collector(); + stable_log::program_invoke(&logger, &program_id, self.get_stack_height()); + self.set_syscall_context(SyscallContext { + allocator: BpfAllocator::new(0), + accounts_metadata: Vec::new(), + })?; + self.memory_contexts.set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(0), + Vec::new(), + // Built-ins do not dereference VM memory, but the invoke + // context still requires an active mapping. + unsafe { + MemoryMapping::new(Vec::new(), &Config::default(), SBPFVersion::Reserved) + .unwrap() + }, + ))?; + let mut vm = EbpfVm::new( + Arc::clone( + &**self + .environment_config + .program_runtime_environments_for_execution + .get_env_for_execution(), + ), + SBPFVersion::V0, + // Removes lifetime tracking. + unsafe { std::mem::transmute::<&mut InvokeContext, &mut InvokeContext>(self) }, + 0, + ); + vm.invoke_function(function); + match vm.program_result { + ProgramResult::Ok(_) => { + stable_log::program_success(&logger, &program_id); + Ok(()) + } + ProgramResult::Err(ref err) => { + if let EbpfError::SyscallError(syscall_error) = err { + if let Some(instruction_err) = + syscall_error.downcast_ref::() + { + stable_log::program_failure(&logger, &program_id, instruction_err); + Err(instruction_err.clone()) + } else { + stable_log::program_failure(&logger, &program_id, syscall_error); + Err(InstructionError::ProgramFailedToComplete) + } + } else { + stable_log::program_failure(&logger, &program_id, err); + Err(InstructionError::ProgramFailedToComplete) + } + } + } + } + ProgramCacheEntryType::Loaded(program) => { + self.transaction_context.set_return_data(program_id, Vec::new())?; + let logger = self.get_log_collector(); + stable_log::program_invoke(&logger, &program_id, self.get_stack_height()); + let result = crate::vm::execute(program, self).map_err(|error| { + error + .downcast_ref::() + .cloned() + .unwrap_or(InstructionError::ProgramFailedToComplete) + }); + match &result { + Ok(()) => stable_log::program_success(&logger, &program_id), + Err(error) => stable_log::program_failure(&logger, &program_id, error), + } + result + } + _ => Err(InstructionError::UnsupportedProgramId), + }; + let post_remaining_units = self.get_remaining(); + *compute_units_consumed = pre_remaining_units.saturating_sub(post_remaining_units); + + if matches!(&entry.program, ProgramCacheEntryType::Builtin(_)) + && result.is_ok() + && *compute_units_consumed == 0 + { + return Err(InstructionError::BuiltinProgramsMustConsumeComputeUnits); + } + + process_executable_chain_time.end_as_us(); + result + } + + /// Get this invocation's LogCollector + pub fn get_log_collector(&self) -> Option>> { + self.log_collector.clone() + } + + /// Consume compute units + pub fn consume_checked(&self, amount: u64) -> Result<(), Box> { + self.compute_meter.consume_checked(amount) + } + + /// Set compute units + /// + /// Only use for tests and benchmarks + pub fn mock_set_remaining(&self, remaining: u64) { + self.compute_meter.mock_set_remaining(remaining); + } + + /// Get this invocation's compute budget + pub fn get_compute_budget(&self) -> &SVMTransactionExecutionBudget { + &self.compute_budget + } + + /// Get this invocation's compute budget + pub fn get_execution_cost(&self) -> &SVMTransactionExecutionCost { + &self.execution_cost + } + + /// Get the current feature set. + pub fn get_feature_set(&self) -> &SVMFeatureSet { + self.environment_config.feature_set + } + + pub fn is_deprecate_legacy_vote_ixs_active(&self) -> bool { + self.environment_config.feature_set.deprecate_legacy_vote_ixs + } + + /// Get cached sysvars + pub fn get_sysvar_cache(&self) -> &SysvarCache { + self.environment_config.sysvar_cache + } + + /// Get cached epoch total stake. + pub fn get_epoch_stake(&self) -> u64 { + self.environment_config.epoch_stake_callback.get_epoch_stake() + } + + /// Get cached stake for the epoch vote account. + pub fn get_epoch_stake_for_vote_account(&self, pubkey: &'a Pubkey) -> u64 { + self.environment_config + .epoch_stake_callback + .get_epoch_stake_for_vote_account(pubkey) + } + + pub fn is_precompile(&self, pubkey: &Pubkey) -> bool { + self.environment_config.epoch_stake_callback.is_precompile(pubkey) + } + + // Should alignment be enforced during user pointer translation + pub fn get_check_aligned(&self) -> bool { + self.transaction_context + .get_current_instruction_context() + .and_then(|instruction_context| { + let owner_id = instruction_context.get_program_owner(); + debug_assert!(owner_id.is_ok()); + owner_id + }) + .map(|owner_key| owner_key != bpf_loader_deprecated::id()) + .unwrap_or(true) + } + + // Set this instruction syscall context + pub fn set_syscall_context( + &mut self, + syscall_context: SyscallContext, + ) -> Result<(), InstructionError> { + *self.syscall_context.last_mut().ok_or(InstructionError::CallDepth)? = + Some(syscall_context); + Ok(()) + } + + // Get this instruction's SyscallContext + pub fn get_syscall_context(&self) -> Result<&SyscallContext, InstructionError> { + self.syscall_context + .last() + .and_then(std::option::Option::as_ref) + .ok_or(InstructionError::CallDepth) + } + + // Get this instruction's SyscallContext + pub fn get_syscall_context_mut(&mut self) -> Result<&mut SyscallContext, InstructionError> { + self.syscall_context + .last_mut() + .and_then(|syscall_context| syscall_context.as_mut()) + .ok_or(InstructionError::CallDepth) + } + + /// Insert a VM register trace + pub fn insert_register_trace(&mut self, register_trace: Vec<[u64; 12]>) { + if register_trace.is_empty() { + return; + } + let Ok(instruction_context) = self.transaction_context.get_current_instruction_context() + else { + return; + }; + self.register_traces + .push((instruction_context.get_index_in_trace(), register_trace)); + } + + /// Iterates over all VM register traces (including CPI) + pub fn iterate_vm_traces( + &self, + callback: &dyn Fn(InstructionContext, &Executable, RegisterTrace), + ) { + for (index_in_trace, register_trace) in &self.register_traces { + let Ok(instruction_context) = self + .transaction_context + .get_instruction_context_at_index_in_trace(*index_in_trace) + else { + continue; + }; + let Ok(program_id) = instruction_context.get_program_key() else { + continue; + }; + let Some(entry) = self.program_cache_for_tx_batch.find(program_id) else { + continue; + }; + let ProgramCacheEntryType::Loaded(ref executable) = entry.program else { + continue; + }; + callback(instruction_context, executable, register_trace.as_slice()); + } + } +} + +#[macro_export] +macro_rules! with_mock_invoke_context_with_feature_set { + ( + $invoke_context:ident, + $transaction_context:ident, + $feature_set:ident, + $top_level_instructions:literal, + $transaction_accounts:expr $(,)? + ) => { + use { + solana_svm_callback::InvokeContextCallback, + solana_svm_log_collector::LogCollector, + $crate::{ + __private::{Hash, ReadableAccount, Rent, TransactionContext}, + execution_budget::{SVMTransactionExecutionBudget, SVMTransactionExecutionCost}, + invoke_context::{EnvironmentConfig, InvokeContext}, + loaded_programs::{ProgramCacheForTxBatch, ProgramRuntimeEnvironments}, + sysvar_cache::SysvarCache, + }, + }; + + struct MockInvokeContextCallback {} + impl InvokeContextCallback for MockInvokeContextCallback {} + + let compute_budget = SVMTransactionExecutionBudget::new_with_defaults( + $feature_set.raise_cpi_nesting_limit_to_8, + ); + let mut $transaction_context = TransactionContext::new( + $transaction_accounts, + Rent::default(), + compute_budget.max_instruction_stack_depth, + compute_budget.max_instruction_trace_length, + $top_level_instructions, + ); + let mut sysvar_cache = SysvarCache::default(); + sysvar_cache.fill_missing_entries(|pubkey, callback| { + for index in 0..$transaction_context.get_number_of_accounts() { + if $transaction_context.get_key_of_account_at_index(index).unwrap() == pubkey { + callback($transaction_context.accounts().try_borrow(index).unwrap().data()); + } + } + }); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockInvokeContextCallback {}, + $feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + let mut $invoke_context = InvokeContext::new( + &mut $transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + Some(LogCollector::new_ref()), + compute_budget, + SVMTransactionExecutionCost::new_with_defaults( + $feature_set.increase_cpi_account_info_limit, + ), + ); + }; + ( + $invoke_context:ident, + $transaction_context:ident, + $feature_set:ident, + $transaction_accounts:expr $(,)? + ) => { + with_mock_invoke_context_with_feature_set!( + $invoke_context, + $transaction_context, + $feature_set, + 1, + $transaction_accounts + ); + }; +} + +#[macro_export] +macro_rules! with_mock_invoke_context { + ( + $invoke_context:ident, + $transaction_context:ident, + $top_level_instructions:literal, + $transaction_accounts:expr $(,)? + ) => { + let feature_set = &solana_svm_feature_set::SVMFeatureSet::default(); + $crate::with_mock_invoke_context_with_feature_set!( + $invoke_context, + $transaction_context, + feature_set, + $top_level_instructions, + $transaction_accounts + ) + }; + ( + $invoke_context:ident, + $transaction_context:ident, + $transaction_accounts:expr $(,)? + ) => { + with_mock_invoke_context!( + $invoke_context, + $transaction_context, + 1, + $transaction_accounts + ); + }; +} + +#[allow(clippy::too_many_arguments)] +pub fn mock_process_instruction_with_feature_set< + F: FnMut(&mut InvokeContext), + G: FnMut(&mut InvokeContext), +>( + loader_id: &Pubkey, + program_index: Option, + instruction_data: &[u8], + mut transaction_accounts: Vec, + instruction_account_metas: Vec, + expected_result: Result<(), InstructionError>, + builtin_function: BuiltinFunctionWithContext, + mut pre_adjustments: F, + mut post_adjustments: G, + feature_set: &SVMFeatureSet, +) -> Vec { + let mut instruction_accounts: Vec = + Vec::with_capacity(instruction_account_metas.len()); + for account_meta in instruction_account_metas.iter() { + let index_in_transaction = transaction_accounts + .iter() + .position(|(key, _account)| *key == account_meta.pubkey) + .unwrap_or(transaction_accounts.len()) + as IndexOfAccount; + instruction_accounts.push(InstructionAccount::new( + index_in_transaction, + account_meta.is_signer, + account_meta.is_writable, + )); + } + + let (program_index, pop_loader_account) = if let Some(index) = program_index { + (index, false) + } else { + let processor_account = AccountSharedData::new(0, 0, &native_loader::id()); + transaction_accounts.push((*loader_id, processor_account)); + ( + transaction_accounts.len().saturating_sub(1) as IndexOfAccount, + true, + ) + }; + let pop_epoch_schedule_account = + if !transaction_accounts.iter().any(|(key, _)| *key == sysvar::epoch_schedule::id()) { + transaction_accounts.push(( + sysvar::epoch_schedule::id(), + create_account_shared_data_for_test(&EpochSchedule::default()), + )); + true + } else { + false + }; + with_mock_invoke_context_with_feature_set!( + invoke_context, + transaction_context, + feature_set, + transaction_accounts + ); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + *loader_id, + Arc::new(ProgramCacheEntry::new_builtin(builtin_function)), + ); + program_cache_for_tx_batch.set_slot_for_tests( + invoke_context + .get_sysvar_cache() + .get_clock() + .map(|clock| clock.slot) + .unwrap_or(1), + ); + invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; + pre_adjustments(&mut invoke_context); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + program_index, + instruction_accounts, + instruction_data.to_vec(), + ) + .unwrap(); + let result = invoke_context.process_instruction(&mut 0); + assert_eq!(result, expected_result); + post_adjustments(&mut invoke_context); + let mut transaction_accounts = transaction_context.deconstruct_without_keys().unwrap(); + if pop_epoch_schedule_account { + transaction_accounts.pop(); + } + if pop_loader_account { + transaction_accounts.pop(); + } + transaction_accounts +} + +pub fn mock_process_instruction( + loader_id: &Pubkey, + program_index: Option, + instruction_data: &[u8], + transaction_accounts: Vec, + instruction_account_metas: Vec, + expected_result: Result<(), InstructionError>, + builtin_function: BuiltinFunctionWithContext, + pre_adjustments: F, + post_adjustments: G, +) -> Vec { + mock_process_instruction_with_feature_set( + loader_id, + program_index, + instruction_data, + transaction_accounts, + instruction_account_metas, + expected_result, + builtin_function, + pre_adjustments, + post_adjustments, + &SVMFeatureSet::all_enabled(), + ) +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::execution_budget::DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, + serde::{Deserialize, Serialize}, + solana_account::WritableAccount, + solana_instruction::Instruction, + solana_keypair::Keypair, + solana_rent::Rent, + solana_sbpf::program::BuiltinFunctionDefinition, + solana_sdk_ids::system_program, + solana_signer::Signer, + solana_transaction::{Transaction, sanitized::SanitizedTransaction}, + solana_transaction_context::MAX_ACCOUNTS_PER_INSTRUCTION, + std::collections::HashSet, + test_case::test_case, + }; + + #[derive(Debug, Serialize, Deserialize)] + enum MockInstruction { + NoopSuccess, + NoopFail, + ModifyOwned, + ModifyNotOwned, + ModifyReadonly, + UnbalancedPush, + UnbalancedPop, + ConsumeComputeUnits { + compute_units_to_consume: u64, + desired_result: Result<(), InstructionError>, + }, + Resize { + new_len: u64, + }, + } + + const MOCK_BUILTIN_COMPUTE_UNIT_COST: u64 = 1; + + declare_process_instruction!( + MockBuiltin, + MOCK_BUILTIN_COMPUTE_UNIT_COST, + |invoke_context| { + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + let instruction_data = instruction_context.get_instruction_data(); + let program_id = instruction_context.get_program_key()?; + let instruction_accounts = (0..4) + .map(|instruction_account_index| { + InstructionAccount::new(instruction_account_index, false, false) + }) + .collect::>(); + assert_eq!( + program_id, + instruction_context.try_borrow_instruction_account(0)?.get_owner() + ); + assert_ne!( + instruction_context.try_borrow_instruction_account(1)?.get_owner(), + instruction_context.get_key_of_instruction_account(0)? + ); + + if let Ok(instruction) = bincode::deserialize(instruction_data) { + match instruction { + MockInstruction::NoopSuccess => (), + MockInstruction::NoopFail => return Err(InstructionError::GenericError), + MockInstruction::ModifyOwned => instruction_context + .try_borrow_instruction_account(0)? + .set_data_from_slice(&[1])?, + MockInstruction::ModifyNotOwned => instruction_context + .try_borrow_instruction_account(1)? + .set_data_from_slice(&[1])?, + MockInstruction::ModifyReadonly => instruction_context + .try_borrow_instruction_account(2)? + .set_data_from_slice(&[1])?, + MockInstruction::UnbalancedPush => { + instruction_context + .try_borrow_instruction_account(0)? + .checked_add_lamports(1)?; + let program_id = *transaction_context.get_key_of_account_at_index(3)?; + let metas = vec![ + AccountMeta::new_readonly( + *transaction_context.get_key_of_account_at_index(0)?, + false, + ), + AccountMeta::new_readonly( + *transaction_context.get_key_of_account_at_index(1)?, + false, + ), + ]; + let inner_instruction = Instruction::new_with_bincode( + program_id, + &MockInstruction::NoopSuccess, + metas, + ); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 3, + instruction_accounts, + vec![], + ) + .unwrap(); + let result = invoke_context.push(); + assert_eq!(result, Err(InstructionError::UnbalancedInstruction)); + result?; + invoke_context + .native_invoke_signed(inner_instruction, &[]) + .and(invoke_context.pop())?; + } + MockInstruction::UnbalancedPop => instruction_context + .try_borrow_instruction_account(0)? + .checked_add_lamports(1)?, + MockInstruction::ConsumeComputeUnits { + compute_units_to_consume, + desired_result, + } => { + invoke_context + .consume_checked(compute_units_to_consume) + .map_err(|_| InstructionError::ComputationalBudgetExceeded)?; + return desired_result; + } + MockInstruction::Resize { new_len } => instruction_context + .try_borrow_instruction_account(0)? + .set_data_from_slice(&vec![0; new_len as usize])?, + } + } else { + return Err(InstructionError::InvalidInstructionData); + } + Ok(()) + } + ); + + #[test_case(false; "SIMD-0268 disabled")] + #[test_case(true; "SIMD-0268 enabled")] + fn test_instruction_stack_height(simd_0268_active: bool) { + let one_more_than_max_depth = + SVMTransactionExecutionBudget::new_with_defaults(simd_0268_active) + .max_instruction_stack_depth + .saturating_add(1); + let mut invoke_stack = vec![]; + let mut transaction_accounts = vec![]; + let mut instruction_accounts = vec![]; + for index in 0..one_more_than_max_depth { + invoke_stack.push(solana_pubkey::new_rand()); + transaction_accounts.push(( + solana_pubkey::new_rand(), + AccountSharedData::new(index as u64, 1, invoke_stack.get(index).unwrap()), + )); + instruction_accounts.push(InstructionAccount::new( + index as IndexOfAccount, + false, + true, + )); + } + for (index, program_id) in invoke_stack.iter().enumerate() { + transaction_accounts.push(( + *program_id, + AccountSharedData::new(1, 1, &solana_pubkey::Pubkey::default()), + )); + instruction_accounts.push(InstructionAccount::new( + index as IndexOfAccount, + false, + false, + )); + } + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + + // Check call depth increases and has a limit + let mut depth_reached: usize = 0; + for _ in 0..invoke_stack.len() { + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + one_more_than_max_depth.saturating_add(depth_reached) as IndexOfAccount, + instruction_accounts.clone(), + vec![], + ) + .unwrap(); + if Err(InstructionError::CallDepth) == invoke_context.push() { + break; + } + depth_reached = depth_reached.saturating_add(1); + } + assert_ne!(depth_reached, 0); + assert!(depth_reached < one_more_than_max_depth); + } + + #[test] + fn test_max_instruction_trace_length_top_level() { + const MAX_INSTRUCTIONS: usize = 8; + let mut transaction_context = TransactionContext::new( + vec![( + Pubkey::new_unique(), + AccountSharedData::new(1, 1, &Pubkey::new_unique()), + )], + Rent::default(), + 1, + MAX_INSTRUCTIONS, + MAX_INSTRUCTIONS, + ); + for _ in 0..MAX_INSTRUCTIONS { + transaction_context.push().unwrap(); + transaction_context + .configure_top_level_instruction_for_tests( + 0, + vec![InstructionAccount::new(0, false, false)], + vec![], + ) + .unwrap(); + transaction_context.pop().unwrap(); + } + assert_eq!( + transaction_context.push(), + Err(InstructionError::MaxInstructionTraceLengthExceeded) + ); + } + + #[test] + fn test_max_instruction_trace_length_cpi() { + // Hitting the limit with CPIs + const MAX_INSTRUCTIONS: usize = 8; + let mut transaction_context = TransactionContext::new( + vec![( + Pubkey::new_unique(), + AccountSharedData::new(1, 1, &Pubkey::new_unique()), + )], + Rent::default(), + 256, + MAX_INSTRUCTIONS, + 1, + ); + + for _ in 0..MAX_INSTRUCTIONS { + transaction_context.push().unwrap(); + transaction_context + .configure_next_cpi_for_tests( + 0, + vec![InstructionAccount::new(0, false, false)], + Vec::new(), + ) + .unwrap(); + } + + assert_eq!( + transaction_context.push(), + Err(InstructionError::MaxInstructionTraceLengthExceeded) + ); + } + + #[test_case(MockInstruction::NoopSuccess, Ok(()); "NoopSuccess")] + #[test_case(MockInstruction::NoopFail, Err(InstructionError::GenericError); "NoopFail")] + #[test_case(MockInstruction::ModifyOwned, Ok(()); "ModifyOwned")] + #[test_case(MockInstruction::ModifyNotOwned, Err(InstructionError::ExternalAccountDataModified); "ModifyNotOwned")] + #[test_case(MockInstruction::ModifyReadonly, Err(InstructionError::ReadonlyDataModified); "ModifyReadonly")] + #[test_case(MockInstruction::UnbalancedPush, Err(InstructionError::UnbalancedInstruction); "UnbalancedPush")] + #[test_case(MockInstruction::UnbalancedPop, Err(InstructionError::UnbalancedInstruction); "UnbalancedPop")] + fn test_process_instruction_account_modifications( + instruction: MockInstruction, + expected_result: Result<(), InstructionError>, + ) { + let callee_program_id = solana_pubkey::new_rand(); + let owned_account = AccountSharedData::new(42, 1, &callee_program_id); + let not_owned_account = AccountSharedData::new(84, 1, &solana_pubkey::new_rand()); + let readonly_account = AccountSharedData::new(168, 1, &solana_pubkey::new_rand()); + let loader_account = AccountSharedData::new(0, 1, &native_loader::id()); + let mut program_account = AccountSharedData::new(1, 1, &native_loader::id()); + program_account.set_executable(true); + let transaction_accounts = vec![ + (solana_pubkey::new_rand(), owned_account), + (solana_pubkey::new_rand(), not_owned_account), + (solana_pubkey::new_rand(), readonly_account), + (callee_program_id, program_account), + (solana_pubkey::new_rand(), loader_account), + ]; + let metas = vec![ + AccountMeta::new(transaction_accounts.first().unwrap().0, false), + AccountMeta::new(transaction_accounts.get(1).unwrap().0, false), + AccountMeta::new_readonly(transaction_accounts.get(2).unwrap().0, false), + ]; + let instruction_accounts = (0..4) + .map(|instruction_account_index| { + InstructionAccount::new( + instruction_account_index, + false, + instruction_account_index < 2, + ) + }) + .collect::>(); + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + callee_program_id, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; + + // Account modification tests + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(4, instruction_accounts, vec![]) + .unwrap(); + invoke_context.push().unwrap(); + let inner_instruction = + Instruction::new_with_bincode(callee_program_id, &instruction, metas); + let result = invoke_context + .native_invoke_signed(inner_instruction, &[]) + .and(invoke_context.pop()); + assert_eq!(result, expected_result); + } + + #[test_case(Ok(()); "Ok")] + #[test_case(Err(InstructionError::GenericError); "GenericError")] + fn test_process_instruction_compute_unit_consumption( + expected_result: Result<(), InstructionError>, + ) { + let callee_program_id = solana_pubkey::new_rand(); + let owned_account = AccountSharedData::new(42, 1, &callee_program_id); + let not_owned_account = AccountSharedData::new(84, 1, &solana_pubkey::new_rand()); + let readonly_account = AccountSharedData::new(168, 1, &solana_pubkey::new_rand()); + let loader_account = AccountSharedData::new(0, 1, &native_loader::id()); + let mut program_account = AccountSharedData::new(1, 1, &native_loader::id()); + program_account.set_executable(true); + let transaction_accounts = vec![ + (solana_pubkey::new_rand(), owned_account), + (solana_pubkey::new_rand(), not_owned_account), + (solana_pubkey::new_rand(), readonly_account), + (callee_program_id, program_account), + (solana_pubkey::new_rand(), loader_account), + ]; + let metas = vec![ + AccountMeta::new(transaction_accounts.first().unwrap().0, false), + AccountMeta::new(transaction_accounts.get(1).unwrap().0, false), + AccountMeta::new_readonly(transaction_accounts.get(2).unwrap().0, false), + ]; + let instruction_accounts = (0..4) + .map(|instruction_account_index| { + InstructionAccount::new( + instruction_account_index, + false, + instruction_account_index < 2, + ) + }) + .collect::>(); + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + callee_program_id, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; + + // Compute unit consumption tests + let compute_units_to_consume = 10; + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(4, instruction_accounts, vec![]) + .unwrap(); + invoke_context.push().unwrap(); + let inner_instruction = Instruction::new_with_bincode( + callee_program_id, + &MockInstruction::ConsumeComputeUnits { + compute_units_to_consume, + desired_result: expected_result.clone(), + }, + metas, + ); + invoke_context.prepare_next_cpi_instruction(inner_instruction, &[]).unwrap(); + + let mut compute_units_consumed = 0; + let result = invoke_context.process_instruction(&mut compute_units_consumed); + + // Because the instruction had compute cost > 0, then regardless of the execution result, + // the number of compute units consumed should be a non-default which is something greater + // than zero. + assert!(compute_units_consumed > 0); + assert_eq!( + compute_units_consumed, + compute_units_to_consume.saturating_add(MOCK_BUILTIN_COMPUTE_UNIT_COST), + ); + assert_eq!(result, expected_result); + + invoke_context.pop().unwrap(); + } + + #[test] + fn test_invoke_context_compute_budget() { + let transaction_accounts = vec![(solana_pubkey::new_rand(), AccountSharedData::default())]; + let execution_budget = SVMTransactionExecutionBudget { + compute_unit_limit: u64::from(DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT), + ..SVMTransactionExecutionBudget::default() + }; + + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context.compute_budget = execution_budget; + + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(0, vec![], vec![]) + .unwrap(); + invoke_context.push().unwrap(); + assert_eq!(*invoke_context.get_compute_budget(), execution_budget); + invoke_context.pop().unwrap(); + } + + #[test_case(0; "Resize the account to *the same size*, so not consuming any additional size")] + #[test_case(1; "Resize the account larger")] + #[test_case(-1; "Resize the account smaller")] + fn test_process_instruction_accounts_resize_delta(resize_delta: i64) { + let program_key = Pubkey::new_unique(); + let user_account_data_len = 123u64; + let user_account = + AccountSharedData::new(100, user_account_data_len as usize, &program_key); + let dummy_account = AccountSharedData::new(10, 0, &program_key); + let mut program_account = AccountSharedData::new(500, 500, &native_loader::id()); + program_account.set_executable(true); + let transaction_accounts = vec![ + (Pubkey::new_unique(), user_account), + (Pubkey::new_unique(), dummy_account), + (program_key, program_account), + ]; + let instruction_accounts = vec![ + InstructionAccount::new(0, false, true), + InstructionAccount::new(1, false, false), + ]; + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + program_key, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; + + let new_len = (user_account_data_len as i64).saturating_add(resize_delta) as u64; + let instruction_data = bincode::serialize(&MockInstruction::Resize { new_len }).unwrap(); + + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(2, instruction_accounts, instruction_data) + .unwrap(); + let result = invoke_context.process_instruction(&mut 0); + + assert!(result.is_ok()); + assert_eq!( + invoke_context.transaction_context.accounts().resize_delta(), + resize_delta + ); + } + + #[test] + fn test_prepare_instruction_maximum_accounts() { + const MAX_ACCOUNTS_REFERENCED: usize = u16::MAX as usize; + let mut transaction_accounts: Vec = + Vec::with_capacity(MAX_ACCOUNTS_PER_TRANSACTION); + let mut account_metas: Vec = Vec::with_capacity(MAX_ACCOUNTS_REFERENCED); + + // Fee-payer + let fee_payer = Keypair::new(); + transaction_accounts.push(( + fee_payer.pubkey(), + AccountSharedData::new(1, 1, &Pubkey::new_unique()), + )); + account_metas.push(AccountMeta::new(fee_payer.pubkey(), true)); + + let program_id = Pubkey::new_unique(); + let mut program_account = AccountSharedData::new(1, 1, &Pubkey::new_unique()); + program_account.set_executable(true); + transaction_accounts.push((program_id, program_account)); + account_metas.push(AccountMeta::new_readonly(program_id, false)); + + for i in 2..MAX_ACCOUNTS_REFERENCED { + // Let's reference 256 unique accounts, and the rest is repeated. + if i < MAX_ACCOUNTS_PER_TRANSACTION { + let key = Pubkey::new_unique(); + transaction_accounts + .push((key, AccountSharedData::new(1, 1, &Pubkey::new_unique()))); + account_metas.push(AccountMeta::new_readonly(key, false)); + } else { + let repeated_key = + transaction_accounts.get(i % MAX_ACCOUNTS_PER_TRANSACTION).unwrap().0; + account_metas.push(AccountMeta::new_readonly(repeated_key, false)); + } + } + + with_mock_invoke_context!(invoke_context, transaction_context, 2, transaction_accounts); + + let instruction_1 = Instruction::new_with_bytes(program_id, &[20], account_metas.clone()); + + let instruction_2 = Instruction::new_with_bytes( + program_id, + &[20], + account_metas.iter().rev().cloned().collect(), + ); + + let transaction = Transaction::new_with_payer( + &[instruction_1.clone(), instruction_2.clone()], + Some(&fee_payer.pubkey()), + ); + + let sanitized = + SanitizedTransaction::try_from_legacy_transaction(transaction, &HashSet::new()) + .unwrap(); + + fn test_case_1(invoke_context: &InvokeContext) { + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); + for index_in_instruction in 0..MAX_ACCOUNTS_REFERENCED as IndexOfAccount { + let index_in_transaction = instruction_context + .get_index_of_instruction_account_in_transaction(index_in_instruction) + .unwrap(); + let other_ix_index = instruction_context + .get_index_of_account_in_instruction(index_in_transaction) + .unwrap(); + if (index_in_instruction as usize) < MAX_ACCOUNTS_PER_TRANSACTION { + assert_eq!(index_in_instruction, index_in_transaction); + assert_eq!(index_in_instruction, other_ix_index); + } else { + assert_eq!( + index_in_instruction as usize % MAX_ACCOUNTS_PER_TRANSACTION, + index_in_transaction as usize + ); + assert_eq!( + index_in_instruction as usize % MAX_ACCOUNTS_PER_TRANSACTION, + other_ix_index as usize + ); + } + } + } + + fn test_case_2(invoke_context: &InvokeContext) { + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); + for index_in_instruction in 0..MAX_ACCOUNTS_REFERENCED as IndexOfAccount { + let index_in_transaction = instruction_context + .get_index_of_instruction_account_in_transaction(index_in_instruction) + .unwrap(); + let other_ix_index = instruction_context + .get_index_of_account_in_instruction(index_in_transaction) + .unwrap(); + assert_eq!( + index_in_transaction, + (MAX_ACCOUNTS_REFERENCED as u16) + .saturating_sub(index_in_instruction) + .saturating_sub(1) + .overflowing_rem(MAX_ACCOUNTS_PER_TRANSACTION as u16) + .0 + ); + if (index_in_instruction as usize) < MAX_ACCOUNTS_PER_TRANSACTION { + assert_eq!(index_in_instruction, other_ix_index); + } else { + assert_eq!( + index_in_instruction as usize % MAX_ACCOUNTS_PER_TRANSACTION, + other_ix_index as usize + ); + } + } + } + + let svm_instruction = + SVMInstruction::from(sanitized.message().instructions().first().unwrap()); + invoke_context + .prepare_next_top_level_instruction( + &sanitized, + &svm_instruction, + 90, + svm_instruction.data, + ) + .unwrap(); + + test_case_1(&invoke_context); + + invoke_context.transaction_context.push().unwrap(); + let svm_instruction = + SVMInstruction::from(sanitized.message().instructions().get(1).unwrap()); + invoke_context + .prepare_next_top_level_instruction( + &sanitized, + &svm_instruction, + 90, + svm_instruction.data, + ) + .unwrap(); + + test_case_2(&invoke_context); + + invoke_context.transaction_context.push().unwrap(); + invoke_context + .prepare_next_cpi_instruction(instruction_1, &[fee_payer.pubkey()]) + .unwrap(); + test_case_1(&invoke_context); + + invoke_context.transaction_context.push().unwrap(); + invoke_context + .prepare_next_cpi_instruction(instruction_2, &[fee_payer.pubkey()]) + .unwrap(); + test_case_2(&invoke_context); + } + + #[test] + fn test_duplicated_accounts() { + let mut transaction_accounts: Vec = + Vec::with_capacity(MAX_ACCOUNTS_PER_TRANSACTION); + let mut account_metas: Vec = + Vec::with_capacity(MAX_ACCOUNTS_PER_INSTRUCTION.saturating_sub(1)); + + // Fee-payer + let fee_payer = Keypair::new(); + transaction_accounts.push(( + fee_payer.pubkey(), + AccountSharedData::new(1, 1, &Pubkey::new_unique()), + )); + account_metas.push(AccountMeta::new(fee_payer.pubkey(), true)); + + let program_id = Pubkey::new_unique(); + let mut program_account = AccountSharedData::new(1, 1, &Pubkey::new_unique()); + program_account.set_executable(true); + transaction_accounts.push((program_id, program_account)); + account_metas.push(AccountMeta::new_readonly(program_id, false)); + + for i in 2..account_metas.capacity() { + if i % 2 == 0 { + let key = Pubkey::new_unique(); + transaction_accounts + .push((key, AccountSharedData::new(1, 1, &Pubkey::new_unique()))); + account_metas.push(AccountMeta::new_readonly(key, false)); + } else { + let last_key = transaction_accounts.last().unwrap().0; + account_metas.push(AccountMeta::new_readonly(last_key, false)); + } + } + + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + + let instruction = Instruction::new_with_bytes(program_id, &[20], account_metas.clone()); + + let transaction = Transaction::new_with_payer(&[instruction], Some(&fee_payer.pubkey())); + + let sanitized = + SanitizedTransaction::try_from_legacy_transaction(transaction, &HashSet::new()) + .unwrap(); + let svm_instruction = + SVMInstruction::from(sanitized.message().instructions().first().unwrap()); + + invoke_context + .prepare_next_top_level_instruction( + &sanitized, + &svm_instruction, + 90, + svm_instruction.data, + ) + .unwrap(); + + { + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); + for index_in_instruction in 2..account_metas.len() as IndexOfAccount { + let is_duplicate = instruction_context + .is_instruction_account_duplicate(index_in_instruction) + .unwrap(); + if index_in_instruction % 2 == 0 { + assert!(is_duplicate.is_none()); + } else { + assert_eq!(is_duplicate, Some(index_in_instruction.saturating_sub(1))); + } + } + } + + invoke_context.transaction_context.push().unwrap(); + + let instruction = Instruction::new_with_bytes( + program_id, + &[20], + account_metas.iter().cloned().rev().collect(), + ); + + invoke_context + .prepare_next_cpi_instruction(instruction, &[fee_payer.pubkey()]) + .unwrap(); + let instruction_context = + invoke_context.transaction_context.get_next_instruction_context().unwrap(); + for index_in_instruction in 2..account_metas.len().saturating_sub(1) as u16 { + let is_duplicate = instruction_context + .is_instruction_account_duplicate(index_in_instruction) + .unwrap(); + if index_in_instruction % 2 == 0 { + assert!(is_duplicate.is_none()); + } else { + assert_eq!(is_duplicate, Some(index_in_instruction.saturating_sub(1))); + } + } + } + + // Used for native_invoke_signed tests below. + const TEST_CALLER_PROGRAM_ID: Pubkey = Pubkey::new_from_array([1u8; 32]); + const TEST_CALLEE_PROGRAM_ID: Pubkey = Pubkey::new_from_array([2u8; 32]); + const TEST_WRONG_PROGRAM_ID: Pubkey = Pubkey::new_from_array([3u8; 32]); + const TEST_MOCK_EXTRA_KEY: Pubkey = Pubkey::new_from_array([4u8; 32]); + const TEST_ACCOUNT_KEY: Pubkey = Pubkey::new_from_array([5u8; 32]); + + /// Runs a `native_invoke_signed` call with the standard test setup and returns + /// the result. + /// + /// Same layout for all tests: + /// 0: target account (writable, signer iff `target_is_signer`) + /// 1: caller program (executable) + /// 2: mock extra (satisfies MockBuiltin's 2-account requirement) + /// 3: callee program (executable) + fn run_native_invoke_signed_test( + target_key: Pubkey, + target_is_signer: bool, + inner_instruction: Instruction, + signer_seeds: &[&[&[u8]]], + ) -> Result<(), InstructionError> { + let target_account = AccountSharedData::new(100, 0, &TEST_CALLEE_PROGRAM_ID); + let mock_extra_account = AccountSharedData::new(0, 1, &system_program::id()); + let mut caller_program_account = AccountSharedData::new(1, 1, &native_loader::id()); + caller_program_account.set_executable(true); + let mut callee_program_account = AccountSharedData::new(1, 1, &native_loader::id()); + callee_program_account.set_executable(true); + let transaction_accounts = vec![ + (target_key, target_account), + (TEST_CALLER_PROGRAM_ID, caller_program_account), + (TEST_MOCK_EXTRA_KEY, mock_extra_account), + (TEST_CALLEE_PROGRAM_ID, callee_program_account), + ]; + + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + TEST_CALLEE_PROGRAM_ID, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + invoke_context.program_cache_for_tx_batch = &mut program_cache_for_tx_batch; + + let instruction_accounts = (0..4) + .map(|i| InstructionAccount::new(i, i == 0 && target_is_signer, i < 2)) + .collect::>(); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(1, instruction_accounts, vec![]) + .unwrap(); + invoke_context.push().unwrap(); + + let result = invoke_context.native_invoke_signed(inner_instruction, signer_seeds); + invoke_context.pop().unwrap(); + result + } + + // Valid PDA seeds grant signer privilege to the derived address. + #[test] + fn test_native_invoke_signed_with_valid_pda_signer() { + let (pda_key, bump_seed) = + Pubkey::find_program_address(&[b"seed"], &TEST_CALLER_PROGRAM_ID); + let instruction = Instruction::new_with_bincode( + TEST_CALLEE_PROGRAM_ID, + &MockInstruction::NoopSuccess, + vec![ + AccountMeta::new(pda_key, true), + AccountMeta::new_readonly(TEST_MOCK_EXTRA_KEY, false), + ], + ); + let result = + run_native_invoke_signed_test(pda_key, false, instruction, &[&[b"seed", &[bump_seed]]]); + assert!( + result.is_ok(), + "valid PDA signer should succeed: {result:?}" + ); + } + + // Oversized seeds (>MAX_SEED_LEN) hit `MaxSeedLengthExceeded` + // (discriminant 0) which the broken `as u64` num-traits conversion + // maps to `Custom(0)`. + #[test] + fn test_native_invoke_signed_with_invalid_seeds() { + let instruction = Instruction::new_with_bincode( + TEST_CALLEE_PROGRAM_ID, + &MockInstruction::NoopSuccess, + vec![AccountMeta::new(TEST_ACCOUNT_KEY, true)], + ); + let oversized_seed = [0u8; 33]; + let result = run_native_invoke_signed_test( + TEST_ACCOUNT_KEY, + false, + instruction, + &[&[&oversized_seed]], + ); + assert_eq!(result, Err(InstructionError::Custom(0))); + } + + // CPI marks an account as signer but caller provides no seeds — + // signer privilege escalation. + #[test] + fn test_native_invoke_signed_pda_privilege_escalation_without_seeds() { + let (pda_key, _bump_seed) = + Pubkey::find_program_address(&[b"seed"], &TEST_CALLER_PROGRAM_ID); + let instruction = Instruction::new_with_bincode( + TEST_CALLEE_PROGRAM_ID, + &MockInstruction::NoopSuccess, + vec![AccountMeta::new(pda_key, true)], + ); + let result = run_native_invoke_signed_test(pda_key, false, instruction, &[]); + assert_eq!(result, Err(InstructionError::PrivilegeEscalation)); + } + + // Seeds valid for a different program ID don't grant signer privilege + // because native_invoke_signed derives against the caller's own program ID. + #[test] + fn test_native_invoke_signed_uses_caller_program_id_for_pda() { + let (pda_key, bump_seed) = Pubkey::find_program_address(&[b"seed"], &TEST_WRONG_PROGRAM_ID); + let instruction = Instruction::new_with_bincode( + TEST_CALLEE_PROGRAM_ID, + &MockInstruction::NoopSuccess, + vec![AccountMeta::new(pda_key, true)], + ); + let result = + run_native_invoke_signed_test(pda_key, false, instruction, &[&[b"seed", &[bump_seed]]]); + assert_eq!(result, Err(InstructionError::PrivilegeEscalation)); + } + + // Top-level signer privilege carries through CPI without needing seeds. + #[test] + fn test_native_invoke_signed_top_level_signer_needs_no_seeds() { + let (pda_key, _bump_seed) = + Pubkey::find_program_address(&[b"seed"], &TEST_CALLER_PROGRAM_ID); + let instruction = Instruction::new_with_bincode( + TEST_CALLEE_PROGRAM_ID, + &MockInstruction::NoopSuccess, + vec![ + AccountMeta::new(pda_key, true), + AccountMeta::new_readonly(TEST_MOCK_EXTRA_KEY, false), + ], + ); + let result = run_native_invoke_signed_test(pda_key, true, instruction, &[]); + assert!( + result.is_ok(), + "top-level signer should not need seeds: {result:?}" + ); + } +} diff --git a/solana/program-runtime/src/lib.rs b/solana/program-runtime/src/lib.rs new file mode 100644 index 00000000..de3c8450 --- /dev/null +++ b/solana/program-runtime/src/lib.rs @@ -0,0 +1,30 @@ +#![allow(clippy::disallowed_methods)] +#![deny(clippy::arithmetic_side_effects)] +#![deny(clippy::indexing_slicing)] +#![doc = include_str!("../README.md")] + +pub use solana_sbpf; +pub mod cpi; +pub mod deploy; +pub mod execution_budget; +pub mod invoke_context; +pub mod loaded_programs; +pub mod mem_pool; +pub mod memory; +pub mod memory_context; +pub mod serialization; +pub mod stable_log; +pub mod sysvar_cache; +pub mod vm; + +// re-exports for macros +pub mod __private { + pub use { + crate::vm::{MEMORY_POOL, calculate_heap_cost, create_vm}, + solana_account::ReadableAccount, + solana_hash::Hash, + solana_instruction::error::InstructionError, + solana_rent::Rent, + solana_transaction_context::transaction::TransactionContext, + }; +} diff --git a/solana/program-runtime/src/loaded_programs.rs b/solana/program-runtime/src/loaded_programs.rs new file mode 100644 index 00000000..fd8b62ff --- /dev/null +++ b/solana/program-runtime/src/loaded_programs.rs @@ -0,0 +1,340 @@ +use { + crate::invoke_context::{BuiltinFunctionWithContext, InvokeContext}, + solana_clock::Slot, + solana_pubkey::Pubkey, + solana_sbpf::{ + elf::Executable, program::BuiltinProgram, verifier::RequisiteVerifier, vm::Config, + }, + solana_svm_type_overrides::sync::Arc, + std::{ + collections::HashMap, + fmt::{Debug, Formatter}, + hash::{Hash, Hasher}, + ops::Deref, + }, +}; + +#[repr(transparent)] +pub struct ProgramRuntimeEnvironment(Arc>>); + +impl ProgramRuntimeEnvironment { + pub fn from(program: BuiltinProgram>) -> Self { + Self(Arc::new(program)) + } + + /// Converts a loader reference into its transparent runtime-environment wrapper. + /// + /// # Safety + /// + /// `ProgramRuntimeEnvironment` is `repr(transparent)` over the same `Arc` + /// type, so the reference layout is identical. + pub unsafe fn from_ref<'a>( + program: &'a Arc>>, + ) -> &'a Self { + unsafe { &*(program as *const Arc<_> as *const Self) } + } +} + +impl Clone for ProgramRuntimeEnvironment { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl Debug for ProgramRuntimeEnvironment { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ProgramRuntimeEnvironment").field(&Arc::as_ptr(&self.0)).finish() + } +} + +impl Deref for ProgramRuntimeEnvironment { + type Target = Arc>>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Hash for ProgramRuntimeEnvironment { + fn hash(&self, state: &mut H) { + std::ptr::hash(Arc::as_ptr(&self.0), state); + } +} + +impl PartialEq for ProgramRuntimeEnvironment { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} + +impl Eq for ProgramRuntimeEnvironment {} + +/// Actual payload of [ProgramCacheEntry]. +#[derive(Default)] +pub enum ProgramCacheEntryType { + /// Program failed verification for the current runtime environment. + FailedVerification(ProgramRuntimeEnvironment), + /// Program is unavailable or intentionally closed. + #[default] + Closed, + /// Retained for API compatibility with older delayed-visibility flows. + DelayVisibility, + /// Program was verified but is not currently compiled. + Unloaded(ProgramRuntimeEnvironment), + /// Verified and compiled program. + Loaded(Executable>), + /// Builtin program shipped with the runtime. + Builtin(BuiltinProgram>), +} + +impl Debug for ProgramCacheEntryType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + ProgramCacheEntryType::FailedVerification(_) => { + write!(f, "ProgramCacheEntryType::FailedVerification") + } + ProgramCacheEntryType::Closed => write!(f, "ProgramCacheEntryType::Closed"), + ProgramCacheEntryType::DelayVisibility => { + write!(f, "ProgramCacheEntryType::DelayVisibility") + } + ProgramCacheEntryType::Unloaded(_) => write!(f, "ProgramCacheEntryType::Unloaded"), + ProgramCacheEntryType::Loaded(_) => write!(f, "ProgramCacheEntryType::Loaded"), + ProgramCacheEntryType::Builtin(_) => write!(f, "ProgramCacheEntryType::Builtin"), + } + } +} + +impl ProgramCacheEntryType { + /// Returns the runtime environment when this entry keeps one. + pub fn get_environment(&self) -> Option<&ProgramRuntimeEnvironment> { + match self { + ProgramCacheEntryType::Loaded(program) => { + // SAFETY: `ProgramRuntimeEnvironment` is transparent over the loader Arc. + Some(unsafe { ProgramRuntimeEnvironment::from_ref(program.get_loader()) }) + } + ProgramCacheEntryType::FailedVerification(env) + | ProgramCacheEntryType::Unloaded(env) => Some(env), + _ => None, + } + } +} + +/// Single cache entry for a program address. +#[derive(Debug, Default)] +pub struct ProgramCacheEntry { + pub program: ProgramCacheEntryType, +} + +impl ProgramCacheEntry { + /// Creates a new user program. + pub fn new( + program_runtime_environment: ProgramRuntimeEnvironment, + elf_bytes: &[u8], + ) -> Result> { + Self::new_internal(program_runtime_environment, elf_bytes, false) + } + + /// Reloads a previously verified user program without re-running the verifier. + /// + /// # Safety + /// + /// Callers must ensure `elf_bytes` were already verified for the provided + /// runtime environment. + pub unsafe fn reload( + program_runtime_environment: ProgramRuntimeEnvironment, + elf_bytes: &[u8], + ) -> Result> { + Self::new_internal(program_runtime_environment, elf_bytes, true) + } + + fn new_internal( + program_runtime_environment: ProgramRuntimeEnvironment, + elf_bytes: &[u8], + reloading: bool, + ) -> Result> { + // Some architectures build without JIT support. + #[allow(unused_mut)] + let mut executable = + Executable::load(elf_bytes, Arc::clone(&*program_runtime_environment))?; + if !reloading { + executable.verify::()?; + } + + #[cfg(all(not(target_os = "windows"), target_arch = "x86_64"))] + executable.jit_compile()?; + + Ok(Self { + program: ProgramCacheEntryType::Loaded(executable), + }) + } + + /// Creates a new built-in program. + pub fn new_builtin(builtin_function: BuiltinFunctionWithContext) -> Self { + let mut program = BuiltinProgram::new_builtin(); + program.register_function("entrypoint", builtin_function).unwrap(); + Self { + program: ProgramCacheEntryType::Builtin(program), + } + } +} + +/// Shared runtime environments keyed by feature configuration. +#[derive(Clone, Debug)] +pub struct ProgramRuntimeEnvironments { + execution: ProgramRuntimeEnvironment, +} + +impl ProgramRuntimeEnvironments { + pub fn new(execution: ProgramRuntimeEnvironment) -> Self { + Self { execution } + } + + pub fn get_env_for_execution(&self) -> &ProgramRuntimeEnvironment { + &self.execution + } +} + +impl Default for ProgramRuntimeEnvironments { + fn default() -> Self { + let empty_loader = + ProgramRuntimeEnvironment::from(BuiltinProgram::new_loader(Config::default())); + Self::new(empty_loader.clone()) + } +} + +/// Global program cache shared across transaction batches. +#[derive(Default)] +pub struct ProgramCache { + index: scc::HashMap>, +} + +impl Debug for ProgramCache { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ProgramCache").field("index_len", &self.index.len()).finish() + } +} + +/// Local view into [ProgramCache] used by a transaction batch. +#[derive(Clone, Debug, Default)] +pub struct ProgramCacheForTxBatch { + entries: HashMap>, + modified_entries: HashMap>, + slot: Slot, + pub hit_max_limit: bool, + pub loaded_missing: bool, + pub merged_modified: bool, +} + +impl ProgramCacheForTxBatch { + pub fn new(slot: Slot) -> Self { + Self { + entries: HashMap::new(), + modified_entries: HashMap::new(), + slot, + hit_max_limit: false, + loaded_missing: false, + merged_modified: false, + } + } + + pub fn replenish(&mut self, key: Pubkey, entry: Arc) { + self.entries.insert(key, entry); + } + + pub fn store_modified_entry(&mut self, key: Pubkey, entry: Arc) { + self.modified_entries.insert(key, entry); + } + + pub fn drain_modified_entries(&mut self) -> HashMap> { + std::mem::take(&mut self.modified_entries) + } + + pub fn find(&self, key: &Pubkey) -> Option> { + self.modified_entries.get(key).or_else(|| self.entries.get(key)).cloned() + } + + pub fn slot(&self) -> Slot { + self.slot + } + + pub fn set_slot_for_tests(&mut self, slot: Slot) { + self.slot = slot; + } + + pub fn merge(&mut self, modified_entries: &HashMap>) { + for (key, entry) in modified_entries { + self.merged_modified = true; + self.replenish(*key, entry.clone()); + } + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +impl ProgramCache { + pub fn assign_program(&self, key: Pubkey, entry: Arc) { + self.index.upsert_sync(key, entry); + } + + pub fn get(&self, key: &Pubkey) -> Option> { + self.index.read_sync(key, |_, entry| entry.clone()) + } + + pub fn merge(&self, modified_entries: &HashMap>) { + for (key, entry) in modified_entries { + if matches!(&entry.program, ProgramCacheEntryType::Closed) { + self.index.remove_sync(key); + } else { + self.assign_program(*key, entry.clone()); + } + } + } +} + +#[cfg(test)] +mod tests { + use { + super::{ + ProgramCache, ProgramCacheEntry, ProgramCacheEntryType, ProgramRuntimeEnvironment, + }, + solana_pubkey::Pubkey, + solana_sbpf::{elf::Executable, program::BuiltinProgram}, + solana_svm_type_overrides::sync::Arc, + }; + + static MOCK_ENVIRONMENT: std::sync::OnceLock = + std::sync::OnceLock::new(); + + fn mock_env() -> ProgramRuntimeEnvironment { + MOCK_ENVIRONMENT + .get_or_init(|| ProgramRuntimeEnvironment::from(BuiltinProgram::new_mock())) + .clone() + } + + fn test_entry() -> Arc { + let elf = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/fixtures/noop_aligned.so" + )) + .unwrap(); + let environment = mock_env(); + let executable = Executable::load(&elf, Arc::clone(&*environment)).unwrap(); + Arc::new(ProgramCacheEntry { + program: ProgramCacheEntryType::Loaded(executable), + }) + } + + #[test] + fn assign_program_makes_entry_retrievable() { + let cache = ProgramCache::default(); + let key = Pubkey::new_unique(); + let entry = test_entry(); + cache.assign_program(key, entry.clone()); + + let fetched = cache.get(&key); + assert!(fetched.is_some()); + assert!(Arc::ptr_eq(&fetched.unwrap(), &entry)); + } +} diff --git a/solana/program-runtime/src/mem_pool.rs b/solana/program-runtime/src/mem_pool.rs new file mode 100644 index 00000000..6f133d84 --- /dev/null +++ b/solana/program-runtime/src/mem_pool.rs @@ -0,0 +1,173 @@ +use { + crate::execution_budget::{ + MAX_CALL_DEPTH, MAX_HEAP_FRAME_BYTES, MAX_INSTRUCTION_STACK_DEPTH, MIN_HEAP_FRAME_BYTES, + STACK_FRAME_SIZE, + }, + solana_sbpf::{aligned_memory::AlignedMemory, ebpf::HOST_ALIGN, vm::CallFrame}, + std::array, +}; + +trait Reset { + fn reset(&mut self); +} + +struct Pool { + items: [Option; SIZE], + next_empty: usize, +} + +impl Pool { + fn new(items: [T; SIZE]) -> Self { + Self { + items: items.map(|i| Some(i)), + next_empty: SIZE, + } + } + + fn len(&self) -> usize { + SIZE + } + + fn get(&mut self) -> Option { + if self.next_empty == 0 { + return None; + } + self.next_empty = self.next_empty.saturating_sub(1); + self.items.get_mut(self.next_empty).and_then(|item| item.take()) + } + + fn put(&mut self, mut value: T) -> bool { + self.items + .get_mut(self.next_empty) + .map(|item| { + value.reset(); + item.replace(value); + self.next_empty = self.next_empty.saturating_add(1); + true + }) + .unwrap_or(false) + } +} + +impl Reset for AlignedMemory<{ HOST_ALIGN }> { + fn reset(&mut self) { + self.as_slice_mut().fill(0) + } +} + +impl Reset for Vec { + fn reset(&mut self) { + self.fill(CallFrame::default()) + } +} + +/// Fixed-size pools of reusable SBF VM buffers. +pub struct VmMemoryPool { + stack: Pool, MAX_INSTRUCTION_STACK_DEPTH>, + heap: Pool, MAX_INSTRUCTION_STACK_DEPTH>, + call_frames: Pool, MAX_INSTRUCTION_STACK_DEPTH>, +} + +impl VmMemoryPool { + /// Allocates a pool sized for the maximum instruction stack depth. + pub fn new() -> Self { + Self { + stack: Pool::new(array::from_fn(|_| { + AlignedMemory::zero_filled(STACK_FRAME_SIZE * MAX_CALL_DEPTH) + })), + heap: Pool::new(array::from_fn(|_| { + AlignedMemory::zero_filled(MAX_HEAP_FRAME_BYTES as usize) + })), + call_frames: Pool::new(array::from_fn(|_| { + std::iter::repeat_with(CallFrame::default).take(MAX_CALL_DEPTH).collect() + })), + } + } + + /// Number of stack buffers managed by the pool. + pub fn stack_len(&self) -> usize { + self.stack.len() + } + + /// Number of heap buffers managed by the pool. + pub fn heap_len(&self) -> usize { + self.heap.len() + } + + /// Returns a zeroed stack buffer, allocating one if the pool is empty. + pub fn get_stack(&mut self, size: usize) -> AlignedMemory<{ HOST_ALIGN }> { + debug_assert!(size == STACK_FRAME_SIZE * MAX_CALL_DEPTH); + self.stack.get().unwrap_or_else(|| AlignedMemory::zero_filled(size)) + } + + /// Returns a stack buffer to the pool after clearing it. + pub fn put_stack(&mut self, stack: AlignedMemory<{ HOST_ALIGN }>) -> bool { + self.stack.put(stack) + } + + /// Returns a maximum-sized zeroed heap buffer; callers slice it to `heap_size`. + pub fn get_heap(&mut self, heap_size: u32) -> AlignedMemory<{ HOST_ALIGN }> { + debug_assert!((MIN_HEAP_FRAME_BYTES..=MAX_HEAP_FRAME_BYTES).contains(&heap_size)); + self.heap + .get() + .unwrap_or_else(|| AlignedMemory::zero_filled(MAX_HEAP_FRAME_BYTES as usize)) + } + + /// Returns a heap buffer to the pool after clearing it. + pub fn put_heap(&mut self, heap: AlignedMemory<{ HOST_ALIGN }>) -> bool { + let heap_size = heap.len(); + debug_assert!( + heap_size >= MIN_HEAP_FRAME_BYTES as usize + && heap_size <= MAX_HEAP_FRAME_BYTES as usize + ); + self.heap.put(heap) + } + + /// Returns a zeroed call-frame buffer sized for the executable. + pub(crate) fn get_call_frames(&mut self, max_call_depth: usize) -> Vec { + let mut call_frames = self.call_frames.get().unwrap_or_default(); + call_frames.resize_with(max_call_depth, CallFrame::default); + call_frames.truncate(max_call_depth); + call_frames + } + + /// Returns a call-frame buffer to the pool after clearing it. + pub(crate) fn put_call_frames(&mut self, call_frames: Vec) -> bool { + self.call_frames.put(call_frames) + } +} + +impl Default for VmMemoryPool { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[derive(Debug, Eq, PartialEq)] + struct Item(u8, u8); + impl Reset for Item { + fn reset(&mut self) { + self.1 = 0; + } + } + + #[test] + fn test_pool() { + let mut pool = Pool::::new([Item(0, 1), Item(1, 1)]); + assert_eq!(pool.get(), Some(Item(1, 1))); + assert_eq!(pool.get(), Some(Item(0, 1))); + assert_eq!(pool.get(), None); + pool.put(Item(1, 1)); + assert_eq!(pool.get(), Some(Item(1, 0))); + pool.put(Item(2, 2)); + pool.put(Item(3, 3)); + assert!(!pool.put(Item(4, 4))); + assert_eq!(pool.get(), Some(Item(3, 0))); + assert_eq!(pool.get(), Some(Item(2, 0))); + assert_eq!(pool.get(), None); + } +} diff --git a/solana/program-runtime/src/memory.rs b/solana/program-runtime/src/memory.rs new file mode 100644 index 00000000..725ccd56 --- /dev/null +++ b/solana/program-runtime/src/memory.rs @@ -0,0 +1,136 @@ +//! Memory translation utilities. + +use { + solana_sbpf::memory_region::{AccessType, MemoryMapping}, + solana_transaction_context::vm_slice::VmSlice, + std::{mem::align_of, slice::from_raw_parts_mut}, +}; + +/// Error types for memory translation operations. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum MemoryTranslationError { + #[error("Unaligned pointer")] + UnalignedPointer, + #[error("Invalid length")] + InvalidLength, +} + +pub fn address_is_aligned(address: u64) -> bool { + (address as *mut T as usize) + .checked_rem(align_of::()) + .map(|rem| rem == 0) + .expect("T to be non-zero aligned") +} + +// Do not use this directly +#[macro_export] +macro_rules! translate_inner { + ($memory_mapping:expr, $map:ident, $access_type:expr, $vm_addr:expr, $len:expr $(,)?) => { + Result::>::from( + $memory_mapping.$map($access_type, $vm_addr, $len).map_err(|err| err.into()), + ) + }; +} + +// Do not use this directly +#[macro_export] +macro_rules! translate_type_inner { + ($memory_mapping:expr, $access_type:expr, $vm_addr:expr, $T:ty, $check_aligned:expr $(,)?) => {{ + let host_addr = $crate::translate_inner!( + $memory_mapping, + map, + $access_type, + $vm_addr, + size_of::<$T>() as u64 + )?; + if !$check_aligned { + Ok(unsafe { std::mem::transmute::(host_addr) }) + } else if !$crate::memory::address_is_aligned::<$T>(host_addr) { + Err($crate::memory::MemoryTranslationError::UnalignedPointer.into()) + } else { + Ok(unsafe { &mut *(host_addr as *mut $T) }) + } + }}; +} + +// Do not use this directly +#[macro_export] +macro_rules! translate_slice_inner { + ($memory_mapping:expr, $access_type:expr, $vm_addr:expr, $len:expr, $T:ty, $check_aligned:expr $(,)?) => {{ + if $len == 0 { + return Ok(&mut []); + } + let total_size = $len.saturating_mul(size_of::<$T>() as u64); + if isize::try_from(total_size).is_err() { + return Err($crate::memory::MemoryTranslationError::InvalidLength.into()); + } + let host_addr = + $crate::translate_inner!($memory_mapping, map, $access_type, $vm_addr, total_size)?; + if $check_aligned && !$crate::memory::address_is_aligned::<$T>(host_addr) { + return Err($crate::memory::MemoryTranslationError::UnalignedPointer.into()); + } + Ok(unsafe { from_raw_parts_mut(host_addr as *mut $T, $len as usize) }) + }}; +} + +pub fn translate_type<'a, T>( + memory_mapping: &MemoryMapping, + vm_addr: u64, + check_aligned: bool, +) -> Result<&'a T, Box> { + translate_type_inner!(memory_mapping, AccessType::Load, vm_addr, T, check_aligned) + .map(|value| &*value) +} + +pub fn translate_slice( + memory_mapping: &MemoryMapping, + vm_addr: u64, + len: u64, + check_aligned: bool, +) -> Result<&[T], Box> { + translate_slice_inner!( + memory_mapping, + AccessType::Load, + vm_addr, + len, + T, + check_aligned, + ) + .map(|value| &*value) +} + +/// CPI-specific version with intentionally different lifetime signature. +/// This version is missing lifetime 'a of the return type in the parameter &MemoryMapping. +pub fn translate_type_mut_for_cpi<'a, T>( + memory_mapping: &MemoryMapping, + vm_addr: u64, + check_aligned: bool, +) -> Result<&'a mut T, Box> { + translate_type_inner!(memory_mapping, AccessType::Store, vm_addr, T, check_aligned) +} + +/// CPI-specific version with intentionally different lifetime signature. +/// This version is missing lifetime 'a of the return type in the parameter &MemoryMapping. +pub fn translate_slice_mut_for_cpi<'a, T>( + memory_mapping: &MemoryMapping, + vm_addr: u64, + len: u64, + check_aligned: bool, +) -> Result<&'a mut [T], Box> { + translate_slice_inner!( + memory_mapping, + AccessType::Store, + vm_addr, + len, + T, + check_aligned, + ) +} + +pub fn translate_vm_slice<'a, T>( + slice: &VmSlice, + memory_mapping: &'a MemoryMapping, + check_aligned: bool, +) -> Result<&'a [T], Box> { + translate_slice::(memory_mapping, slice.ptr(), slice.len(), check_aligned) +} diff --git a/solana/program-runtime/src/memory_context.rs b/solana/program-runtime/src/memory_context.rs new file mode 100644 index 00000000..b82d5565 --- /dev/null +++ b/solana/program-runtime/src/memory_context.rs @@ -0,0 +1,87 @@ +use { + crate::invoke_context::BpfAllocator, solana_instruction::error::InstructionError, + solana_sbpf::memory_region::MemoryMapping, +}; + +enum MemoryContextType { + ABIv1(MemoryContext), + Placeholder, +} + +pub struct MemoryContexts { + contexts: Vec, +} + +impl MemoryContexts { + pub(crate) fn new() -> Self { + Self { contexts: Vec::new() } + } + + pub fn set_memory_context_abi_v1( + &mut self, + memory_context: MemoryContext, + ) -> Result<(), InstructionError> { + *self.contexts.last_mut().ok_or(InstructionError::CallDepth)? = + MemoryContextType::ABIv1(memory_context); + Ok(()) + } + + pub fn memory_context_mut_abi_v1(&mut self) -> Result<&mut MemoryContext, InstructionError> { + match self.contexts.last_mut().ok_or(InstructionError::CallDepth)? { + MemoryContextType::ABIv1(context) => Ok(context), + MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), + } + } + + pub fn memory_mapping(&self) -> Result<&MemoryMapping, InstructionError> { + match self.contexts.last().ok_or(InstructionError::CallDepth)? { + MemoryContextType::ABIv1(context) => Ok(&context.memory_mapping), + MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), + } + } + + pub fn memory_mapping_mut(&mut self) -> Result<&mut MemoryMapping, InstructionError> { + match self.contexts.last_mut().ok_or(InstructionError::CallDepth)? { + MemoryContextType::ABIv1(context) => Ok(&mut context.memory_mapping), + MemoryContextType::Placeholder => Err(InstructionError::ProgramEnvironmentSetupFailure), + } + } + + pub fn push_placeholder(&mut self) { + self.contexts.push(MemoryContextType::Placeholder); + } + + pub fn pop(&mut self) { + self.contexts.pop(); + } +} + +pub struct MemoryContext { + pub allocator: BpfAllocator, + pub accounts_metadata: Vec, + memory_mapping: Box, +} + +impl MemoryContext { + pub fn new( + allocator: BpfAllocator, + accounts_metadata: Vec, + memory_mapping: MemoryMapping, + ) -> Self { + Self { + allocator, + accounts_metadata, + memory_mapping: Box::new(memory_mapping), + } + } +} + +#[derive(Debug, Clone)] +pub struct SerializedAccountMetadata { + pub vm_addr: u64, + pub original_data_len: usize, + pub vm_data_addr: u64, + pub vm_key_addr: u64, + pub vm_lamports_addr: u64, + pub vm_owner_addr: u64, +} diff --git a/solana/program-runtime/src/serialization.rs b/solana/program-runtime/src/serialization.rs new file mode 100644 index 00000000..add4f868 --- /dev/null +++ b/solana/program-runtime/src/serialization.rs @@ -0,0 +1,1702 @@ +#![allow(clippy::arithmetic_side_effects)] + +use { + crate::invoke_context::SerializedAccountMetadata, + solana_instruction::error::InstructionError, + solana_program_entrypoint::{BPF_ALIGN_OF_U128, MAX_PERMITTED_DATA_INCREASE, NON_DUP_MARKER}, + solana_pubkey::Pubkey, + solana_sbpf::{ + aligned_memory::{AlignedMemory, Pod}, + ebpf::{HOST_ALIGN, MM_INPUT_START}, + memory_region::MemoryRegion, + }, + solana_sdk_ids::bpf_loader_deprecated, + solana_system_interface::MAX_PERMITTED_DATA_LENGTH, + solana_transaction_context::{ + IndexOfAccount, MAX_ACCOUNTS_PER_INSTRUCTION, instruction::InstructionContext, + instruction_accounts::BorrowedInstructionAccount, + }, + std::mem::{self, size_of}, +}; + +/// Modifies an existing memory mapping region to point at account data. +pub fn modify_memory_region_of_account( + account: &mut BorrowedInstructionAccount<'_, '_>, + region: &mut MemoryRegion, +) { + region.len = account.get_data().len() as u64; + if account.can_data_be_changed().is_ok() { + region.writable = true; + region.access_violation_handler_payload = Some(account.get_index_in_transaction()); + } else { + region.writable = false; + region.access_violation_handler_payload = None; + } +} + +/// Creates a memory region that directly maps account data for serialization and CPI return. +pub fn create_memory_region_of_account( + account: &mut BorrowedInstructionAccount<'_, '_>, + vaddr: u64, +) -> Result { + let can_data_be_changed = account.can_data_be_changed().is_ok(); + let mut memory_region = if can_data_be_changed && !account.is_shared() { + MemoryRegion::new(&raw mut account.get_data_mut()?[..], vaddr) + } else { + MemoryRegion::new(&raw const account.get_data()[..], vaddr) + }; + if can_data_be_changed { + memory_region.access_violation_handler_payload = Some(account.get_index_in_transaction()); + } + Ok(memory_region) +} + +/// Returns the VM address space reserved for an account's data. +pub(crate) fn account_data_region_size(is_loader_deprecated: bool, data_len: usize) -> usize { + if is_loader_deprecated { + data_len + } else { + data_len.saturating_add(MAX_PERMITTED_DATA_INCREASE) + } +} + +#[allow(dead_code)] +enum SerializeAccount<'a, 'ix_data> { + Account(IndexOfAccount, BorrowedInstructionAccount<'a, 'ix_data>), + Duplicate(IndexOfAccount), +} + +struct Serializer { + buffer: AlignedMemory, + regions: Vec, + vaddr: u64, + region_start: usize, + is_loader_v1: bool, +} + +impl Serializer { + fn new(size: usize, start_addr: u64, is_loader_v1: bool) -> Serializer { + Serializer { + buffer: AlignedMemory::with_capacity(size), + regions: Vec::new(), + region_start: 0, + vaddr: start_addr, + is_loader_v1, + } + } + + fn fill_write(&mut self, num: usize, value: u8) -> std::io::Result<()> { + self.buffer.fill_write(num, value) + } + + fn write(&mut self, value: T) -> u64 { + self.debug_assert_alignment::(); + let vaddr = self + .vaddr + .saturating_add(self.buffer.len() as u64) + .saturating_sub(self.region_start as u64); + // Safety: + // in serialize_parameters_(aligned|unaligned) first we compute the + // required size then we write into the newly allocated buffer. There's + // no need to check bounds at every write. + // + // AlignedMemory::write_unchecked _does_ debug_assert!() that the capacity + // is enough, so in the unlikely case we introduce a bug in the size + // computation, tests will abort. + unsafe { + self.buffer.write_unchecked(value); + } + + vaddr + } + + fn write_all(&mut self, value: &[u8]) -> u64 { + let vaddr = self + .vaddr + .saturating_add(self.buffer.len() as u64) + .saturating_sub(self.region_start as u64); + // Safety: + // see write() - the buffer is guaranteed to be large enough + unsafe { + self.buffer.write_all_unchecked(value); + } + + vaddr + } + + fn write_account( + &mut self, + account: &mut BorrowedInstructionAccount<'_, '_>, + ) -> Result { + self.push_region(); + let vm_data_addr = self.vaddr; + let address_space_reserved_for_account = + account_data_region_size(self.is_loader_v1, account.get_data().len()); + if address_space_reserved_for_account > 0 { + let new_region = create_memory_region_of_account(account, self.vaddr)?; + self.vaddr += address_space_reserved_for_account as u64; + self.regions.push(new_region); + } + if !self.is_loader_v1 { + let align_offset = + (account.get_data().len() as *const u8).align_offset(BPF_ALIGN_OF_U128); + // The deserialization code is going to align the vm_addr to + // BPF_ALIGN_OF_U128. Always add one BPF_ALIGN_OF_U128 worth of + // padding and shift the start of the next region, so that once + // vm_addr is aligned, the corresponding host_addr is aligned too. + self.fill_write(BPF_ALIGN_OF_U128, 0) + .map_err(|_| InstructionError::InvalidArgument)?; + self.region_start += BPF_ALIGN_OF_U128.saturating_sub(align_offset); + } + Ok(vm_data_addr) + } + + fn push_region(&mut self) { + let range = self.region_start..self.buffer.len(); + self.regions.push(MemoryRegion::new( + &raw mut self.buffer.as_slice_mut().get_mut(range.clone()).unwrap()[..], + self.vaddr, + )); + self.region_start = range.end; + self.vaddr += range.len() as u64; + } + + fn finish(mut self) -> (AlignedMemory, Vec) { + self.push_region(); + debug_assert_eq!(self.region_start, self.buffer.len()); + (self.buffer, self.regions) + } + + fn debug_assert_alignment(&self) { + debug_assert!( + self.is_loader_v1 + || self.buffer.as_slice().as_ptr_range().end.align_offset(mem::align_of::()) + == 0 + ); + } +} + +pub fn serialize_parameters( + instruction_context: &InstructionContext, + direct_account_pointers_in_program_input: bool, +) -> Result< + ( + AlignedMemory, + Vec, + Vec, + usize, + ), + InstructionError, +> { + let num_ix_accounts = instruction_context.get_number_of_instruction_accounts(); + if num_ix_accounts > MAX_ACCOUNTS_PER_INSTRUCTION as IndexOfAccount { + return Err(InstructionError::MaxAccountsExceeded); + } + + let program_id = *instruction_context.get_program_key()?; + let is_loader_deprecated = + instruction_context.get_program_owner()? == bpf_loader_deprecated::id(); + + let accounts = (0..instruction_context.get_number_of_instruction_accounts()) + .map(|instruction_account_index| { + if let Some(index) = instruction_context + .is_instruction_account_duplicate(instruction_account_index) + .unwrap() + { + SerializeAccount::Duplicate(index) + } else { + let account = instruction_context + .try_borrow_instruction_account(instruction_account_index) + .unwrap(); + SerializeAccount::Account(instruction_account_index, account) + } + }) + // fun fact: jemalloc is good at caching tiny allocations like this one, + // so collecting here is actually faster than passing the iterator + // around, since the iterator does the work to produce its items each + // time it's iterated on. + .collect::>(); + + if is_loader_deprecated { + // Used by loader-v1 (bpf_loader_deprecated) + serialize_parameters_for_abiv0( + accounts, + instruction_context.get_instruction_data(), + &program_id, + ) + } else { + // Used by loader-v2 (bpf_loader) and loader-v3 (bpf_loader_upgradeable) + serialize_parameters_for_abiv1( + accounts, + instruction_context.get_instruction_data(), + &program_id, + // SIMD-0449: only available on ABIv1 + direct_account_pointers_in_program_input, + ) + } +} + +pub fn deserialize_parameters( + instruction_context: &InstructionContext, + buffer: &[u8], + accounts_metadata: &[SerializedAccountMetadata], +) -> Result<(), InstructionError> { + let is_loader_deprecated = + instruction_context.get_program_owner()? == bpf_loader_deprecated::id(); + let account_lengths = accounts_metadata.iter().map(|a| a.original_data_len); + if is_loader_deprecated { + // Used by loader-v1 (bpf_loader_deprecated) + deserialize_parameters_for_abiv0(instruction_context, buffer, account_lengths) + } else { + // Used by loader-v2 (bpf_loader) and loader-v3 (bpf_loader_upgradeable) + deserialize_parameters_for_abiv1(instruction_context, buffer, account_lengths) + } +} + +fn serialize_parameters_for_abiv0( + accounts: Vec, + instruction_data: &[u8], + program_id: &Pubkey, +) -> Result< + ( + AlignedMemory, + Vec, + Vec, + usize, + ), + InstructionError, +> { + // Calculate size in order to alloc once + let mut size = size_of::(); + for account in &accounts { + size += 1; // dup + match account { + SerializeAccount::Duplicate(_) => {} + SerializeAccount::Account(_, _) => { + size += size_of::() // is_signer + + size_of::() // is_writable + + size_of::() // key + + size_of::() // lamports + + size_of::() // data len + + size_of::() // owner + + size_of::() // executable + + size_of::(); // rent_epoch + } + } + } + size += size_of::() // instruction data len + + instruction_data.len() // instruction data + + size_of::(); // program id + + let mut s = Serializer::new(size, MM_INPUT_START, true); + + let mut accounts_metadata: Vec = Vec::with_capacity(accounts.len()); + s.write::((accounts.len() as u64).to_le()); + for account in accounts { + match account { + SerializeAccount::Duplicate(position) => { + accounts_metadata.push(accounts_metadata.get(position as usize).unwrap().clone()); + s.write(position as u8); + } + SerializeAccount::Account(_, mut account) => { + let vm_addr = s.write::(NON_DUP_MARKER); + s.write::(account.is_signer() as u8); + s.write::(account.is_writable() as u8); + let vm_key_addr = s.write_all(account.get_key().as_ref()); + let vm_lamports_addr = s.write::(account.get_lamports().to_le()); + s.write::((account.get_data().len() as u64).to_le()); + let vm_data_addr = s.write_account(&mut account)?; + let vm_owner_addr = s.write_all(account.get_owner().as_ref()); + #[allow(deprecated)] + s.write::(account.is_executable() as u8); + let rent_epoch = u64::MAX; + s.write::(rent_epoch.to_le()); + accounts_metadata.push(SerializedAccountMetadata { + vm_addr, + original_data_len: account.get_data().len(), + vm_key_addr, + vm_lamports_addr, + vm_owner_addr, + vm_data_addr, + }); + } + }; + } + s.write::((instruction_data.len() as u64).to_le()); + let instruction_data_offset = s.write_all(instruction_data); + s.write_all(program_id.as_ref()); + + let (mem, regions) = s.finish(); + Ok(( + mem, + regions, + accounts_metadata, + instruction_data_offset as usize, + )) +} + +fn deserialize_parameters_for_abiv0>( + instruction_context: &InstructionContext, + buffer: &[u8], + account_lengths: I, +) -> Result<(), InstructionError> { + let mut start = size_of::(); // number of accounts + for (instruction_account_index, pre_len) in + (0..instruction_context.get_number_of_instruction_accounts()).zip(account_lengths) + { + let duplicate = + instruction_context.is_instruction_account_duplicate(instruction_account_index)?; + start += 1; // is_dup + if duplicate.is_none() { + let mut borrowed_account = + instruction_context.try_borrow_instruction_account(instruction_account_index)?; + start += size_of::(); // is_signer + start += size_of::(); // is_writable + start += size_of::(); // key + let lamports = buffer + .get(start..start.saturating_add(8)) + .map(<[u8; 8]>::try_from) + .and_then(Result::ok) + .map(u64::from_le_bytes) + .ok_or(InstructionError::InvalidArgument)?; + if borrowed_account.get_lamports() != lamports { + borrowed_account.set_lamports(lamports)?; + } + start += size_of::() // lamports + + size_of::(); // data length + if borrowed_account.get_data().len() != pre_len { + borrowed_account.set_data_length(pre_len)?; + } + start += size_of::() // owner + + size_of::() // executable + + size_of::(); // rent_epoch + } + } + Ok(()) +} + +fn serialize_parameters_for_abiv1( + accounts: Vec, + instruction_data: &[u8], + program_id: &Pubkey, + direct_account_pointers_program_input: bool, +) -> Result< + ( + AlignedMemory, + Vec, + Vec, + usize, + ), + InstructionError, +> { + let mut accounts_metadata = Vec::with_capacity(accounts.len()); + // Calculate size in order to alloc once + let mut size = size_of::(); + for account in &accounts { + size += 1; // dup + match account { + SerializeAccount::Duplicate(_) => size += 7, // padding to 64-bit aligned + SerializeAccount::Account(_, _) => { + size += size_of::() // is_signer + + size_of::() // is_writable + + size_of::() // executable + + size_of::() // original_data_len + + size_of::() // key + + size_of::() // owner + + size_of::() // lamports + + size_of::() // data len + + size_of::(); // rent epoch + size += BPF_ALIGN_OF_U128; + } + } + } + size += size_of::() // data len + + instruction_data.len() + + size_of::(); // program id; + + // reserve space for account pointer array if SIMD-0449 is enabled + let account_pointers_offset = if direct_account_pointers_program_input { + let offset = (size as *const u8).align_offset(BPF_ALIGN_OF_U128); + size += offset + accounts.len() * size_of::(); + Some(offset) + } else { + None + }; + + let mut s = Serializer::new(size, MM_INPUT_START, false); + + // Serialize into the buffer + s.write::((accounts.len() as u64).to_le()); + for account in accounts { + match account { + SerializeAccount::Account(_, mut borrowed_account) => { + let vm_addr = s.write::(NON_DUP_MARKER); + s.write::(borrowed_account.is_signer() as u8); + s.write::(borrowed_account.is_writable() as u8); + #[allow(deprecated)] + s.write::(borrowed_account.is_executable() as u8); + s.write_all(&[0u8, 0, 0, 0]); + let vm_key_addr = s.write_all(borrowed_account.get_key().as_ref()); + let vm_owner_addr = s.write_all(borrowed_account.get_owner().as_ref()); + let vm_lamports_addr = s.write::(borrowed_account.get_lamports().to_le()); + s.write::((borrowed_account.get_data().len() as u64).to_le()); + let vm_data_addr = s.write_account(&mut borrowed_account)?; + let rent_epoch = u64::MAX; + s.write::(rent_epoch.to_le()); + accounts_metadata.push(SerializedAccountMetadata { + vm_addr, + original_data_len: borrowed_account.get_data().len(), + vm_key_addr, + vm_owner_addr, + vm_lamports_addr, + vm_data_addr, + }); + } + SerializeAccount::Duplicate(position) => { + accounts_metadata.push(accounts_metadata.get(position as usize).unwrap().clone()); + s.write::(position as u8); + s.write_all(&[0u8, 0, 0, 0, 0, 0, 0]); + } + }; + } + s.write::((instruction_data.len() as u64).to_le()); + let instruction_data_offset = s.write_all(instruction_data); + s.write_all(program_id.as_ref()); + + if let Some(offset) = account_pointers_offset { + // Add padding before the account pointer array to reach 8-byte alignment + // (BPF_ALIGN_OF_U128). + s.fill_write(offset, 0).map_err(|_| InstructionError::InvalidArgument)?; + for entry in accounts_metadata.iter() { + s.write::(entry.vm_addr.to_le()); + } + } + + let (mem, regions) = s.finish(); + Ok(( + mem, + regions, + accounts_metadata, + instruction_data_offset as usize, + )) +} + +fn deserialize_parameters_for_abiv1>( + instruction_context: &InstructionContext, + buffer: &[u8], + account_lengths: I, +) -> Result<(), InstructionError> { + let mut start = size_of::(); // number of accounts + for (instruction_account_index, pre_len) in + (0..instruction_context.get_number_of_instruction_accounts()).zip(account_lengths) + { + let duplicate = + instruction_context.is_instruction_account_duplicate(instruction_account_index)?; + start += size_of::(); // position + if duplicate.is_some() { + start += 7; // padding to 64-bit aligned + } else { + let mut borrowed_account = + instruction_context.try_borrow_instruction_account(instruction_account_index)?; + start += size_of::() // is_signer + + size_of::() // is_writable + + size_of::() // executable + + size_of::() // original_data_len + + size_of::(); // key + let owner = buffer + .get(start..start + size_of::()) + .ok_or(InstructionError::InvalidArgument)?; + start += size_of::(); // owner + let lamports = buffer + .get(start..start.saturating_add(8)) + .map(<[u8; 8]>::try_from) + .and_then(Result::ok) + .map(u64::from_le_bytes) + .ok_or(InstructionError::InvalidArgument)?; + if borrowed_account.get_lamports() != lamports { + borrowed_account.set_lamports(lamports)?; + } + start += size_of::(); // lamports + let post_len = buffer + .get(start..start.saturating_add(8)) + .map(<[u8; 8]>::try_from) + .and_then(Result::ok) + .map(u64::from_le_bytes) + .ok_or(InstructionError::InvalidArgument)? as usize; + start += size_of::(); // data length + if post_len.saturating_sub(pre_len) > MAX_PERMITTED_DATA_INCREASE + || post_len > MAX_PERMITTED_DATA_LENGTH as usize + { + return Err(InstructionError::InvalidRealloc); + } + if borrowed_account.get_data().len() != post_len { + borrowed_account.set_data_length(post_len)?; + } + // See Serializer::write_account() as to why we have this padding. + start += BPF_ALIGN_OF_U128; + start += size_of::(); // rent_epoch + if borrowed_account.get_owner().to_bytes() != owner { + // Change the owner at the end so that we are allowed to change the lamports and data before + borrowed_account.set_owner(owner)?; + } + } + } + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::indexing_slicing)] +mod tests { + use { + super::*, + crate::with_mock_invoke_context, + solana_account::{ + Account, AccountSharedData, CoWAccount, ReadableAccount, + testkit::{active_borrowed_data, borrowed_account_buffer, borrowed_shared_data}, + }, + solana_account_info::AccountInfo, + solana_program_entrypoint::deserialize, + solana_rent::Rent, + solana_sbpf::{memory_region::MemoryMapping, program::SBPFVersion, vm::Config}, + solana_sdk_ids::bpf_loader, + solana_system_interface::MAX_PERMITTED_ACCOUNTS_DATA_ALLOCATIONS_PER_TRANSACTION, + solana_transaction_context::{ + MAX_ACCOUNTS_PER_TRANSACTION, instruction_accounts::InstructionAccount, + transaction::TransactionContext, + }, + std::{ + borrow::Cow, + cell::RefCell, + mem::transmute, + rc::Rc, + slice::{self, from_raw_parts, from_raw_parts_mut}, + }, + test_case::test_case, + }; + + fn deduplicated_instruction_accounts( + transaction_indexes: &[IndexOfAccount], + is_writable: fn(usize) -> bool, + ) -> Vec { + transaction_indexes + .iter() + .enumerate() + .map(|(index_in_instruction, index_in_transaction)| { + InstructionAccount::new( + *index_in_transaction, + false, + is_writable(index_in_instruction), + ) + }) + .collect() + } + + #[test_case(false; "direct_account_pointers_in_program_input disabled")] + #[test_case(true; "direct_account_pointers_in_program_input enabled")] + fn test_serialize_parameters_with_many_accounts( + direct_account_pointers_in_program_input: bool, + ) { + struct TestCase { + num_ix_accounts: usize, + append_dup_account: bool, + expected_err: Option, + name: &'static str, + } + + for TestCase { + num_ix_accounts, + append_dup_account, + expected_err, + name, + } in [ + TestCase { + name: "serialize max accounts with cap", + num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION, + append_dup_account: false, + expected_err: None, + }, + TestCase { + name: "serialize too many accounts with cap", + num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION + 1, + append_dup_account: false, + expected_err: Some(InstructionError::MaxAccountsExceeded), + }, + TestCase { + name: "serialize too many accounts and append dup with cap", + num_ix_accounts: MAX_ACCOUNTS_PER_INSTRUCTION, + append_dup_account: true, + expected_err: Some(InstructionError::MaxAccountsExceeded), + }, + ] { + let program_id = solana_pubkey::new_rand(); + let mut transaction_accounts = vec![( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + )]; + for _ in 0..num_ix_accounts { + transaction_accounts.push(( + Pubkey::new_unique(), + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: program_id, + executable: false, + rent_epoch: 0, + }), + )); + } + + let transaction_accounts_indexes: Vec = + (0..num_ix_accounts as u16).collect(); + let mut instruction_accounts = + deduplicated_instruction_accounts(&transaction_accounts_indexes, |_| false); + if append_dup_account { + instruction_accounts.push(instruction_accounts.last().cloned().unwrap()); + } + let instruction_data = vec![]; + + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + if instruction_accounts.len() > MAX_ACCOUNTS_PER_INSTRUCTION { + // Special case implementation of configure_next_instruction_for_tests() + // which avoids the overflow when constructing the dedup_map + // by simply not filling it. + let dedup_map = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; + invoke_context + .transaction_context + .configure_instruction_at_index( + 0, + 0, + instruction_accounts, + dedup_map, + Cow::Owned(instruction_data.clone()), + Some(0), + ) + .unwrap(); + } else { + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + instruction_accounts, + instruction_data.clone(), + ) + .unwrap(); + } + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let serialization_result = serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + ); + assert_eq!( + serialization_result.as_ref().err(), + expected_err.as_ref(), + "{name} test case failed", + ); + if expected_err.is_some() { + continue; + } + + let (_serialized, regions, _account_lengths, _instruction_data_offset) = + serialization_result.unwrap(); + let mut serialized_regions = concat_regions(®ions); + let (de_program_id, de_accounts, de_instruction_data) = unsafe { + deserialize(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + assert_eq!(de_program_id, &program_id); + assert_eq!(de_instruction_data, &instruction_data); + for account_info in de_accounts { + let index_in_transaction = invoke_context + .transaction_context + .find_index_of_account(account_info.key) + .unwrap(); + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction) + .unwrap(); + assert_eq!(account.lamports(), account_info.lamports()); + assert_eq!(account.data(), &account_info.data.borrow()[..]); + assert_eq!(account.owner(), account_info.owner); + assert_eq!(account.executable(), account_info.executable); + #[allow(deprecated)] + { + // Using the sdk entrypoint, the rent-epoch is skipped + assert_eq!(0, account_info._unused); + } + } + } + } + + #[test_case(false; "direct_account_pointers_in_program_input disabled")] + #[test_case(true; "direct_account_pointers_in_program_input enabled")] + fn test_serialize_parameters(direct_account_pointers_in_program_input: bool) { + let program_id = solana_pubkey::new_rand(); + let transaction_accounts = vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 1, + data: vec![1u8, 2, 3, 4, 5], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 2, + data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 200, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 3, + data: vec![], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 3100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 4, + data: vec![1u8, 2, 3, 4, 5], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 5, + data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 200, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 6, + data: vec![], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 3100, + }), + ), + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader_deprecated::id(), + executable: true, + rent_epoch: 0, + }), + ), + ]; + let instruction_accounts = + deduplicated_instruction_accounts(&[1, 1, 2, 3, 4, 4, 5, 6], |index| index >= 4); + let instruction_data = vec![1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + let original_accounts = transaction_accounts.clone(); + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + instruction_accounts.clone(), + instruction_data.clone(), + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + // check serialize_parameters_for_abiv1 + let (serialized, regions, accounts_metadata, _instruction_data_offset) = + serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + ) + .unwrap(); + + let mut serialized_regions = concat_regions(®ions); + let (de_program_id, de_accounts, de_instruction_data) = unsafe { + deserialize(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + + assert_eq!(&program_id, de_program_id); + assert_eq!(instruction_data, de_instruction_data); + assert_eq!( + (de_instruction_data.first().unwrap() as *const u8).align_offset(BPF_ALIGN_OF_U128), + 0 + ); + for account_info in de_accounts { + let index_in_transaction = invoke_context + .transaction_context + .find_index_of_account(account_info.key) + .unwrap(); + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction) + .unwrap(); + assert_eq!(account.lamports(), account_info.lamports()); + assert_eq!(account.data(), &account_info.data.borrow()[..]); + assert_eq!(account.owner(), account_info.owner); + assert_eq!(account.executable(), account_info.executable); + #[allow(deprecated)] + { + // Using the sdk entrypoint, the rent-epoch is skipped + assert_eq!(0, account_info._unused); + } + + assert_eq!( + (*account_info.lamports.borrow() as *const u64).align_offset(BPF_ALIGN_OF_U128), + 0 + ); + assert_eq!( + account_info.data.borrow().as_ptr().align_offset(BPF_ALIGN_OF_U128), + 0 + ); + } + + deserialize_parameters( + &instruction_context, + serialized.as_slice(), + &accounts_metadata, + ) + .unwrap(); + for (index_in_transaction, (_key, original_account)) in original_accounts.iter().enumerate() + { + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction as IndexOfAccount) + .unwrap(); + assert_eq!(&*account, original_account); + } + + invoke_context.pop().unwrap(); + // check serialize_parameters_for_abiv0 + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 7, + instruction_accounts, + instruction_data.clone(), + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let (serialized, regions, account_lengths, _instruction_data_offset) = + serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + ) + .unwrap(); + let mut serialized_regions = concat_regions(®ions); + + let (de_program_id, de_accounts, de_instruction_data) = unsafe { + deserialize_for_abiv0(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + assert_eq!(&program_id, de_program_id); + assert_eq!(instruction_data, de_instruction_data); + for account_info in de_accounts { + let index_in_transaction = invoke_context + .transaction_context + .find_index_of_account(account_info.key) + .unwrap(); + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction) + .unwrap(); + assert_eq!(account.lamports(), account_info.lamports()); + assert_eq!(account.data(), &account_info.data.borrow()[..]); + assert_eq!(account.owner(), account_info.owner); + assert_eq!(account.executable(), account_info.executable); + #[allow(deprecated)] + { + assert_eq!(u64::MAX, account_info._unused); + } + } + + deserialize_parameters( + &instruction_context, + serialized.as_slice(), + &account_lengths, + ) + .unwrap(); + for (index_in_transaction, (_key, original_account)) in original_accounts.iter().enumerate() + { + let account = invoke_context + .transaction_context + .accounts() + .try_borrow(index_in_transaction as IndexOfAccount) + .unwrap(); + assert_eq!(&*account, original_account); + } + } + + #[test_case(false; "direct_account_pointers_in_program_input disabled")] + #[test_case(true; "direct_account_pointers_in_program_input enabled")] + fn test_serialize_parameters_mask_out_rent_epoch_in_vm_serialization( + direct_account_pointers_in_program_input: bool, + ) { + let transaction_accounts = vec![ + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 1, + data: vec![1u8, 2, 3, 4, 5], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 2, + data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 200, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 3, + data: vec![], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 300, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 4, + data: vec![1u8, 2, 3, 4, 5], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 5, + data: vec![11u8, 12, 13, 14, 15, 16, 17, 18, 19], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 200, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 6, + data: vec![], + owner: bpf_loader::id(), + executable: false, + rent_epoch: 3100, + }), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader_deprecated::id(), + executable: true, + rent_epoch: 0, + }), + ), + ]; + let instruction_accounts = + deduplicated_instruction_accounts(&[1, 1, 2, 3, 4, 4, 5, 6], |index| index >= 4); + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(0, instruction_accounts.clone(), vec![]) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + // check serialize_parameters_for_abiv1 + let (_serialized, regions, _accounts_metadata, _instruction_data_offset) = + serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + ) + .unwrap(); + + let mut serialized_regions = concat_regions(®ions); + let (_de_program_id, de_accounts, _de_instruction_data) = unsafe { + deserialize(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + + for account_info in de_accounts { + // Using program-entrypoint, the rent-epoch will always be 0 + #[allow(deprecated)] + { + assert_eq!(0, account_info._unused); + } + } + + // check serialize_parameters_for_abiv0 + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests(7, instruction_accounts, vec![]) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let (_serialized, regions, _account_lengths, _instruction_data_offset) = + serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + ) + .unwrap(); + let mut serialized_regions = concat_regions(®ions); + + let (_de_program_id, de_accounts, _de_instruction_data) = unsafe { + deserialize_for_abiv0(serialized_regions.as_slice_mut().first_mut().unwrap() as *mut u8) + }; + for account_info in de_accounts { + #[allow(deprecated)] + { + assert_eq!(account_info._unused, u64::MAX); + } + } + } + + // the old bpf_loader in-program deserializer bpf_loader::id() + /// + /// # Safety + /// + /// `input` must point to a valid ABI-v0 serialized instruction buffer laid + /// out exactly as the legacy loader expects for the duration of the returned + /// borrows. + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn deserialize_for_abiv0<'a>( + input: *mut u8, + ) -> (&'a Pubkey, Vec>, &'a [u8]) { + // this boring boilerplate struct is needed until inline const... + struct Ptr(std::marker::PhantomData); + impl Ptr { + const COULD_BE_UNALIGNED: bool = std::mem::align_of::() > 1; + + #[inline(always)] + fn read_possibly_unaligned(input: *mut u8, offset: usize) -> T { + unsafe { + let src = input.add(offset) as *const T; + if Self::COULD_BE_UNALIGNED { src.read_unaligned() } else { src.read() } + } + } + + // rustc inserts debug_assert! for misaligned pointer dereferences when + // deserializing, starting from [1]. so, use std::mem::transmute as the last resort + // while preventing clippy from complaining to suggest not to use it. + // [1]: https://github.com/rust-lang/rust/commit/22a7a19f9333bc1fcba97ce444a3515cb5fb33e6 + // as for the ub nature of the misaligned pointer dereference, this is + // acceptable in this code, given that this is cfg(test) and it's cared only with + // x86-64 and the target only incurs some performance penalty, not like segfaults + // in other targets. + #[inline(always)] + fn ref_possibly_unaligned<'a>(input: *mut u8, offset: usize) -> &'a T { + #[allow(clippy::transmute_ptr_to_ref)] + unsafe { + transmute(input.add(offset) as *const T) + } + } + + // See ref_possibly_unaligned's comment + #[inline(always)] + fn mut_possibly_unaligned<'a>(input: *mut u8, offset: usize) -> &'a mut T { + #[allow(clippy::transmute_ptr_to_ref)] + unsafe { + transmute(input.add(offset) as *mut T) + } + } + } + + let mut offset: usize = 0; + + // number of accounts present + + let num_accounts = Ptr::::read_possibly_unaligned(input, offset) as usize; + offset += size_of::(); + + // account Infos + + let mut accounts = Vec::with_capacity(num_accounts); + for _ in 0..num_accounts { + let dup_info = Ptr::::read_possibly_unaligned(input, offset); + offset += size_of::(); + if dup_info == NON_DUP_MARKER { + let is_signer = Ptr::::read_possibly_unaligned(input, offset) != 0; + offset += size_of::(); + + let is_writable = Ptr::::read_possibly_unaligned(input, offset) != 0; + offset += size_of::(); + + let key = Ptr::::ref_possibly_unaligned(input, offset); + offset += size_of::(); + + let lamports = Rc::new(RefCell::new(Ptr::mut_possibly_unaligned(input, offset))); + offset += size_of::(); + + let data_len = Ptr::::read_possibly_unaligned(input, offset) as usize; + offset += size_of::(); + + let data = Rc::new(RefCell::new(unsafe { + from_raw_parts_mut(input.add(offset), data_len) + })); + offset += data_len; + + let owner: &Pubkey = Ptr::::ref_possibly_unaligned(input, offset); + offset += size_of::(); + + let executable = Ptr::::read_possibly_unaligned(input, offset) != 0; + offset += size_of::(); + + let unused = Ptr::::read_possibly_unaligned(input, offset); + offset += size_of::(); + + #[allow(deprecated)] + accounts.push(AccountInfo { + key, + is_signer, + is_writable, + lamports, + data, + owner, + executable, + _unused: unused, + }); + } else { + // duplicate account, clone the original + accounts.push(accounts.get(dup_info as usize).unwrap().clone()); + } + } + + // instruction data + + let instruction_data_len = Ptr::::read_possibly_unaligned(input, offset) as usize; + offset += size_of::(); + + let instruction_data = unsafe { from_raw_parts(input.add(offset), instruction_data_len) }; + offset += instruction_data_len; + + // program Id + + let program_id = Ptr::::ref_possibly_unaligned(input, offset); + + (program_id, accounts, instruction_data) + } + + fn concat_regions(regions: &[MemoryRegion]) -> AlignedMemory { + let last_region = regions.last().unwrap(); + let mut mem = AlignedMemory::zero_filled( + (last_region.vm_addr - MM_INPUT_START + last_region.len) as usize, + ); + for region in regions { + let host_slice = unsafe { + slice::from_raw_parts(region.host_addr as *const u8, region.len as usize) + }; + mem.as_slice_mut()[(region.vm_addr - MM_INPUT_START) as usize..][..region.len as usize] + .copy_from_slice(host_slice) + } + mem + } + + fn write_vm_data_len( + serialized: &mut AlignedMemory, + account_metadata: &SerializedAccountMetadata, + len: usize, + ) { + let offset = account_metadata + .vm_data_addr + .saturating_sub(MM_INPUT_START) + .saturating_sub(size_of::() as u64) as usize; + serialized.as_slice_mut()[offset..offset + size_of::()] + .copy_from_slice(&(len as u64).to_le_bytes()); + } + + #[test] + fn test_vas_serialization_direct_maps_account_data_only() { + let program_id = Pubkey::new_unique(); + let account_data = b"direct-account-data-is-not-copied".to_vec(); + let transaction_accounts = vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + Pubkey::new_unique(), + AccountSharedData::from(Account { + lamports: 1, + data: account_data.clone(), + owner: program_id, + executable: false, + rent_epoch: 0, + }), + ), + ]; + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + deduplicated_instruction_accounts(&[1], |_| true), + vec![], + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let (serialized, regions, accounts_metadata, _instruction_data_offset) = + serialize_parameters(&instruction_context, false).unwrap(); + + assert!( + !serialized + .as_slice() + .windows(account_data.len()) + .any(|window| window == account_data) + ); + let data_region = regions + .iter() + .find(|region| region.vm_addr == accounts_metadata[0].vm_data_addr) + .unwrap(); + assert_eq!(data_region.len, account_data.len() as u64); + assert!(data_region.writable); + let mapped_data = unsafe { + slice::from_raw_parts(data_region.host_addr as *const u8, data_region.len as usize) + }; + assert_eq!(mapped_data, account_data); + } + + #[test_case(4, 8, Ok(4); "unchanged vm length restores original after transient growth")] + #[test_case(7, 8, Ok(7); "vm length grow wins over larger transient backing")] + #[test_case(2, 4, Ok(2); "vm length shrink truncates")] + #[test_case( + 4 + MAX_PERMITTED_DATA_INCREASE + 1, + 4, + Err(InstructionError::InvalidRealloc); + "invalid realloc limit errors" + )] + fn test_vas_deserialize_reconciles_direct_mapped_length( + vm_len: usize, + transient_len: usize, + expected: Result, + ) { + let program_id = Pubkey::new_unique(); + let transaction_accounts = vec![ + ( + program_id, + AccountSharedData::from(Account { + lamports: 0, + data: vec![], + owner: bpf_loader::id(), + executable: true, + rent_epoch: 0, + }), + ), + ( + Pubkey::new_unique(), + AccountSharedData::from(Account { + lamports: 1, + data: vec![1, 2, 3, 4], + owner: program_id, + executable: false, + rent_epoch: 0, + }), + ), + ]; + with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts); + invoke_context + .transaction_context + .configure_top_level_instruction_for_tests( + 0, + deduplicated_instruction_accounts(&[1], |_| true), + vec![], + ) + .unwrap(); + invoke_context.push().unwrap(); + let instruction_context = + invoke_context.transaction_context.get_current_instruction_context().unwrap(); + + let (mut serialized, _regions, accounts_metadata, _instruction_data_offset) = + serialize_parameters(&instruction_context, false).unwrap(); + write_vm_data_len(&mut serialized, &accounts_metadata[0], vm_len); + { + let mut account = instruction_context.try_borrow_instruction_account(0).unwrap(); + account.set_data_length(transient_len).unwrap(); + } + + let result = deserialize_parameters( + &instruction_context, + serialized.as_slice(), + &accounts_metadata, + ); + assert_eq!(result, expected.clone().map(|_| ())); + if let Ok(expected_len) = expected { + let account = instruction_context.try_borrow_instruction_account(0).unwrap(); + assert_eq!(account.get_data().len(), expected_len); + } + } + + #[test] + fn test_vas_borrowed_writable_account_store_uses_shadow_image() { + let program_id = Pubkey::new_unique(); + let initial_data = vec![1, 2, 3]; + let mut borrowed_buf = borrowed_account_buffer(initial_data.clone(), program_id); + let borrowed_account = borrowed_shared_data(&mut borrowed_buf); + let mut transaction_context = TransactionContext::new( + vec![ + (Pubkey::new_unique(), borrowed_account), + (program_id, AccountSharedData::default()), + ], + Rent::default(), + /* max_instruction_stack_depth */ 1, + /* max_instruction_trace_length */ 1, + /* number_of_top_level_instructions */ 1, + ); + transaction_context + .configure_top_level_instruction_for_tests( + 1, + vec![InstructionAccount::new(0, false, true)], + vec![], + ) + .unwrap(); + transaction_context.push().unwrap(); + let instruction_context = transaction_context.get_current_instruction_context().unwrap(); + let account_start_offset = MM_INPUT_START; + let region = create_memory_region_of_account( + &mut instruction_context.try_borrow_instruction_account(0).unwrap(), + account_start_offset, + ) + .unwrap(); + + assert_eq!(region.len, initial_data.len() as u64); + assert!(!region.writable); + assert_eq!(region.access_violation_handler_payload, Some(0)); + + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let mut memory_mapping = unsafe { + MemoryMapping::new_with_access_violation_handler( + vec![region], + &config, + SBPFVersion::V3, + transaction_context.access_violation_handler(), + ) + } + .unwrap(); + + assert_eq!(memory_mapping.load::(account_start_offset).unwrap(), 1); + assert_eq!(active_borrowed_data(&mut borrowed_buf), initial_data); + + memory_mapping.store::(9, account_start_offset).unwrap(); + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().data(), + &[9, 2, 3], + ); + assert_eq!(active_borrowed_data(&mut borrowed_buf), initial_data); + + { + let account = transaction_context.accounts().try_borrow(0).unwrap(); + match account.cow() { + CoWAccount::Borrowed(account) => account.commit(), + CoWAccount::Owned(_) => panic!("borrowed account should stay borrowed"), + } + } + assert_eq!(active_borrowed_data(&mut borrowed_buf), vec![9, 2, 3]); + } + + #[test] + fn test_vas_borrowed_writable_account_store_without_commit_keeps_active_image() { + let program_id = Pubkey::new_unique(); + let initial_data = vec![4, 5, 6]; + let mut borrowed_buf = borrowed_account_buffer(initial_data.clone(), program_id); + let borrowed_account = borrowed_shared_data(&mut borrowed_buf); + let mut transaction_context = TransactionContext::new( + vec![ + (Pubkey::new_unique(), borrowed_account), + (program_id, AccountSharedData::default()), + ], + Rent::default(), + /* max_instruction_stack_depth */ 1, + /* max_instruction_trace_length */ 1, + /* number_of_top_level_instructions */ 1, + ); + transaction_context + .configure_top_level_instruction_for_tests( + 1, + vec![InstructionAccount::new(0, false, true)], + vec![], + ) + .unwrap(); + transaction_context.push().unwrap(); + let instruction_context = transaction_context.get_current_instruction_context().unwrap(); + let account_start_offset = MM_INPUT_START; + let region = create_memory_region_of_account( + &mut instruction_context.try_borrow_instruction_account(0).unwrap(), + account_start_offset, + ) + .unwrap(); + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let mut memory_mapping = unsafe { + MemoryMapping::new_with_access_violation_handler( + vec![region], + &config, + SBPFVersion::V3, + transaction_context.access_violation_handler(), + ) + } + .unwrap(); + + memory_mapping.store::(7, account_start_offset).unwrap(); + + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().data(), + &[7, 5, 6], + ); + assert_eq!(active_borrowed_data(&mut borrowed_buf), initial_data); + } + + #[test] + fn test_access_violation_handler() { + let program_id = Pubkey::new_unique(); + let shared_account = AccountSharedData::new(0, 4, &program_id); + let mut transaction_context = TransactionContext::new( + vec![ + ( + Pubkey::new_unique(), + AccountSharedData::new(0, 4, &program_id), + ), // readonly + (Pubkey::new_unique(), shared_account.clone()), // writable shared + ( + Pubkey::new_unique(), + AccountSharedData::new(0, 0, &program_id), + ), // another writable account + ( + Pubkey::new_unique(), + AccountSharedData::new( + 0, + MAX_PERMITTED_DATA_LENGTH as usize - 0x100, + &program_id, + ), + ), // almost max sized writable account + ( + Pubkey::new_unique(), + AccountSharedData::new(0, 0, &program_id), + ), // writable dummy to burn accounts_resize_delta + ( + Pubkey::new_unique(), + AccountSharedData::new(0, 0x3000, &program_id), + ), // writable dummy to burn accounts_resize_delta + ( + Pubkey::new_unique(), + AccountSharedData::new(0, 0, &program_id), + ), // writable dummy to burn accounts_resize_delta + (program_id, AccountSharedData::default()), // program + ], + Rent::default(), + /* max_instruction_stack_depth */ 1, + /* max_instruction_trace_length */ 1, + /* number_of_top_level_instructions */ 1, + ); + let transaction_accounts_indexes = [0, 1, 2, 3, 4, 5, 6]; + let instruction_accounts = + deduplicated_instruction_accounts(&transaction_accounts_indexes, |index| index > 0); + transaction_context + .configure_top_level_instruction_for_tests(7, instruction_accounts, vec![]) + .unwrap(); + transaction_context.push().unwrap(); + let instruction_context = transaction_context.get_current_instruction_context().unwrap(); + let account_start_offsets = [ + MM_INPUT_START, + MM_INPUT_START + 4 + MAX_PERMITTED_DATA_INCREASE as u64, + MM_INPUT_START + (4 + MAX_PERMITTED_DATA_INCREASE as u64) * 2, + MM_INPUT_START + (4 + MAX_PERMITTED_DATA_INCREASE as u64) * 3, + ]; + let regions = account_start_offsets + .iter() + .enumerate() + .map(|(index_in_instruction, account_start_offset)| { + create_memory_region_of_account( + &mut instruction_context + .try_borrow_instruction_account(index_in_instruction as IndexOfAccount) + .unwrap(), + *account_start_offset, + ) + .unwrap() + }) + .collect::>(); + let config = Config { + aligned_memory_mapping: false, + ..Config::default() + }; + let mut memory_mapping = unsafe { + MemoryMapping::new_with_access_violation_handler( + regions, + &config, + SBPFVersion::V3, + transaction_context.access_violation_handler(), + ) + } + .unwrap(); + + // Reading readonly account is allowed + memory_mapping.load::(account_start_offsets[0]).unwrap(); + + // Reading writable account is allowed + memory_mapping.load::(account_start_offsets[1]).unwrap(); + + // Reading beyond readonly accounts current size is denied + memory_mapping.load::(account_start_offsets[0] + 4).unwrap_err(); + + // Writing to readonly account is denied + memory_mapping.store::(0, account_start_offsets[0]).unwrap_err(); + + // Writing to shared writable account makes it unique (CoW logic.) + // It has been previously been made non-unique at the beginning of + // the test through a clone. + let _shared_account_ref = shared_account; + assert!(transaction_context.accounts().try_borrow_mut(1).unwrap().is_shared()); + memory_mapping.store::(0, account_start_offsets[1]).unwrap(); + assert!(!transaction_context.accounts().try_borrow_mut(1).unwrap().is_shared()); + assert_eq!( + transaction_context.accounts().try_borrow(1).unwrap().data().len(), + 4, + ); + + // Reading beyond writable accounts current size grows is denied + memory_mapping.load::(account_start_offsets[1] + 4).unwrap_err(); + + // Writing beyond writable accounts current size grows it only to the + // requested access length. + memory_mapping.store::(0, account_start_offsets[1] + 4).unwrap(); + assert_eq!( + transaction_context.accounts().try_borrow(1).unwrap().data().len(), + 8, + ); + assert!(transaction_context.accounts().try_borrow(1).unwrap().data().len() < 0x3000); + + // Writing beyond almost max sized writable accounts current size only grows it + // to MAX_PERMITTED_DATA_LENGTH + memory_mapping + .store::(0, account_start_offsets[3] + MAX_PERMITTED_DATA_LENGTH - 4) + .unwrap(); + assert_eq!( + transaction_context.accounts().try_borrow(3).unwrap().data().len(), + MAX_PERMITTED_DATA_LENGTH as usize, + ); + + // Accessing the rest of the address space reserved for + // the almost max sized writable account is denied + memory_mapping + .load::(account_start_offsets[3] + MAX_PERMITTED_DATA_LENGTH) + .unwrap_err(); + memory_mapping + .store::(0, account_start_offsets[3] + MAX_PERMITTED_DATA_LENGTH) + .unwrap_err(); + + // Burn through most of the accounts_resize_delta budget + let remaining_allowed_growth: usize = 0x700; + let target_resize_delta = MAX_PERMITTED_ACCOUNTS_DATA_ALLOCATIONS_PER_TRANSACTION + - remaining_allowed_growth as i64; + for index_in_instruction in 4..7 { + let burn = target_resize_delta + .saturating_sub(transaction_context.accounts().resize_delta()) + as usize; + if burn == 0 { + break; + } + let mut borrowed_account = instruction_context + .try_borrow_instruction_account(index_in_instruction) + .unwrap(); + let old_len = borrowed_account.get_data().len(); + let new_len = old_len.saturating_add(burn).min(MAX_PERMITTED_DATA_LENGTH as usize); + borrowed_account.set_data_length(new_len).unwrap(); + } + assert_eq!( + transaction_context.accounts().resize_delta(), + target_resize_delta, + ); + + // Writing beyond empty writable accounts current size grows to the + // requested access length while it fits in the remaining transaction budget. + memory_mapping.store::(0, account_start_offsets[2] + 0x500).unwrap(); + assert_eq!( + transaction_context.accounts().try_borrow(2).unwrap().data().len(), + 0x504, + ); + + // A write that would need more than the remaining transaction budget is denied. + memory_mapping + .store::( + 0, + account_start_offsets[2] + remaining_allowed_growth as u64, + ) + .unwrap_err(); + } +} diff --git a/solana/program-runtime/src/stable_log.rs b/solana/program-runtime/src/stable_log.rs new file mode 100644 index 00000000..bf31feac --- /dev/null +++ b/solana/program-runtime/src/stable_log.rs @@ -0,0 +1,110 @@ +//! Stable program log messages +//! +//! The format of these log messages should not be modified to avoid breaking downstream consumers +//! of program logging +use { + base64::{Engine, prelude::BASE64_STANDARD}, + itertools::Itertools, + solana_pubkey::Pubkey, + solana_svm_log_collector::{LogCollector, ic_logger_msg}, + std::{cell::RefCell, rc::Rc}, +}; + +/// Log a program invoke. +/// +/// The general form is: +/// +/// ```notrust +/// "Program

invoke []" +/// ``` +pub fn program_invoke( + log_collector: &Option>>, + program_id: &Pubkey, + invoke_depth: usize, +) { + ic_logger_msg!( + log_collector, + "Program {} invoke [{}]", + program_id, + invoke_depth + ); +} + +/// Log a message from the program itself. +/// +/// The general form is: +/// +/// ```notrust +/// "Program log: " +/// ``` +/// +/// That is, any program-generated output is guaranteed to be prefixed by "Program log: " +pub fn program_log(log_collector: &Option>>, message: &str) { + ic_logger_msg!(log_collector, "Program log: {}", message); +} + +/// Emit a program data. +/// +/// The general form is: +/// +/// ```notrust +/// "Program data: *" +/// ``` +/// +/// That is, any program-generated output is guaranteed to be prefixed by "Program data: " +pub fn program_data(log_collector: &Option>>, data: &[&[u8]]) { + ic_logger_msg!( + log_collector, + "Program data: {}", + data.iter().map(|v| BASE64_STANDARD.encode(v)).join(" ") + ); +} + +/// Log return data as from the program itself. This line will not be present if no return +/// data was set, or if the return data was set to zero length. +/// +/// The general form is: +/// +/// ```notrust +/// "Program return: " +/// ``` +/// +/// That is, any program-generated output is guaranteed to be prefixed by "Program return: " +pub fn program_return( + log_collector: &Option>>, + program_id: &Pubkey, + data: &[u8], +) { + ic_logger_msg!( + log_collector, + "Program return: {} {}", + program_id, + BASE64_STANDARD.encode(data) + ); +} + +/// Log successful program execution. +/// +/// The general form is: +/// +/// ```notrust +/// "Program
success" +/// ``` +pub fn program_success(log_collector: &Option>>, program_id: &Pubkey) { + ic_logger_msg!(log_collector, "Program {} success", program_id); +} + +/// Log program execution failure +/// +/// The general form is: +/// +/// ```notrust +/// "Program
failed: " +/// ``` +pub fn program_failure( + log_collector: &Option>>, + program_id: &Pubkey, + err: &E, +) { + ic_logger_msg!(log_collector, "Program {} failed: {}", program_id, err); +} diff --git a/solana/program-runtime/src/sysvar_cache.rs b/solana/program-runtime/src/sysvar_cache.rs new file mode 100644 index 00000000..e8e23826 --- /dev/null +++ b/solana/program-runtime/src/sysvar_cache.rs @@ -0,0 +1,310 @@ +use solana_epoch_rewards::EpochRewards; +#[allow(deprecated)] +use solana_sysvar::fees::Fees; +#[allow(deprecated)] +use solana_sysvar::recent_blockhashes::RecentBlockhashes; +use { + crate::invoke_context::InvokeContext, + serde::de::DeserializeOwned, + solana_clock::Clock, + solana_epoch_schedule::EpochSchedule, + solana_instruction::error::InstructionError, + solana_last_restart_slot::LastRestartSlot, + solana_pubkey::Pubkey, + solana_rent::Rent, + solana_slot_hashes::SlotHashes, + solana_svm_type_overrides::sync::Arc, + solana_sysvar_id::SysvarId, + solana_transaction_context::{IndexOfAccount, instruction::InstructionContext}, +}; + +/// Serialized sysvars exposed to programs during execution. +#[derive(Default, Debug)] +pub struct SysvarCache { + // Full account data, including any trailing zero bytes. + clock: Option>, + epoch_schedule: Option>, + epoch_rewards: Option>, + rent: Option>, + slot_hashes: Option>, + last_restart_slot: Option>, + + // Object representations of large sysvars used by native builtins. + slot_hashes_obj: Option, + + #[allow(deprecated)] + fees: Option, + #[allow(deprecated)] + recent_blockhashes: Option, +} + +impl SysvarCache { + /// Returns the serialized sysvar buffer for `SyscallGetSysvar`. + pub fn sysvar_id_to_buffer(&self, sysvar_id: &Pubkey) -> &Option> { + if Clock::check_id(sysvar_id) { + &self.clock + } else if EpochSchedule::check_id(sysvar_id) { + &self.epoch_schedule + } else if EpochRewards::check_id(sysvar_id) { + &self.epoch_rewards + } else if Rent::check_id(sysvar_id) { + &self.rent + } else if SlotHashes::check_id(sysvar_id) { + &self.slot_hashes + } else if LastRestartSlot::check_id(sysvar_id) { + &self.last_restart_slot + } else { + &None + } + } + + fn get_sysvar_obj( + &self, + sysvar_id: &Pubkey, + ) -> Result, InstructionError> { + if let Some(sysvar_buf) = self.sysvar_id_to_buffer(sysvar_id) { + bincode::deserialize(sysvar_buf) + .map(Arc::new) + .map_err(|_| InstructionError::UnsupportedSysvar) + } else { + Err(InstructionError::UnsupportedSysvar) + } + } + + /// Stores a serialized clock sysvar. + pub fn set_clock(&mut self, clock: &Clock) { + let buffer = self.clock.get_or_insert_default(); + buffer.clear(); + // bincode doesn't fail when writing to a correctly sized sysvar buffer. + let _ = bincode::serialize_into(buffer, clock); + } + + /// Returns the cached clock sysvar. + pub fn get_clock(&self) -> Result, InstructionError> { + self.get_sysvar_obj(&Clock::id()) + } + + /// Returns the cached rent sysvar. + pub fn get_rent(&self) -> Result, InstructionError> { + self.get_sysvar_obj(&Rent::id()) + } + + /// Returns the cached last-restart-slot sysvar. + pub fn get_last_restart_slot(&self) -> Result, InstructionError> { + self.get_sysvar_obj(&LastRestartSlot::id()) + } + + /// Returns the cached slot hashes sysvar. + pub fn get_slot_hashes(&self) -> Result, InstructionError> { + self.slot_hashes_obj + .as_ref() + .map(|s| Arc::new(SlotHashes::new(s.slot_hashes()))) + .ok_or(InstructionError::UnsupportedSysvar) + } + + #[deprecated] + #[allow(deprecated)] + /// Returns the cached recent-blockhashes sysvar. + pub fn get_recent_blockhashes(&self) -> Result, InstructionError> { + self.recent_blockhashes + .clone() + .ok_or(InstructionError::UnsupportedSysvar) + .map(Arc::new) + } + + /// Returns the cached deprecated fees sysvar. + #[allow(deprecated)] + pub fn get_fees(&self) -> Result, InstructionError> { + self.fees.clone().map(Arc::new).ok_or(InstructionError::UnsupportedSysvar) + } + + /// Typed epoch-schedule access is unsupported; use the serialized sysvar cache. + pub fn get_epoch_schedule(&self) -> Result, InstructionError> { + Err(InstructionError::UnsupportedSysvar) + } + + /// Typed epoch-rewards access is unsupported; use the serialized sysvar cache. + pub fn get_epoch_rewards(&self) -> Result, InstructionError> { + Err(InstructionError::UnsupportedSysvar) + } + + /// Fills missing sysvars by asking the caller for serialized account data. + pub fn fill_missing_entries( + &mut self, + mut get_account_data: F, + ) { + if self.clock.is_none() { + get_account_data(&Clock::id(), &mut |data: &[u8]| { + if bincode::deserialize::(data).is_ok() { + self.clock = Some(data.to_vec()); + } + }); + } + + if self.epoch_schedule.is_none() { + get_account_data(&EpochSchedule::id(), &mut |data: &[u8]| { + if bincode::deserialize::(data).is_ok() { + self.epoch_schedule = Some(data.to_vec()); + } + }); + } + + if self.epoch_rewards.is_none() { + get_account_data(&EpochRewards::id(), &mut |data: &[u8]| { + if bincode::deserialize::(data).is_ok() { + self.epoch_rewards = Some(data.to_vec()); + } + }); + } + + if self.rent.is_none() { + get_account_data(&Rent::id(), &mut |data: &[u8]| { + if bincode::deserialize::(data).is_ok() { + self.rent = Some(data.to_vec()); + } + }); + } + + if self.slot_hashes.is_none() { + get_account_data(&SlotHashes::id(), &mut |data: &[u8]| { + if let Ok(obj) = bincode::deserialize::(data) { + self.slot_hashes = Some(data.to_vec()); + self.slot_hashes_obj = Some(obj); + } + }); + } + + if self.last_restart_slot.is_none() { + get_account_data(&LastRestartSlot::id(), &mut |data: &[u8]| { + if bincode::deserialize::(data).is_ok() { + self.last_restart_slot = Some(data.to_vec()); + } + }); + } + + #[allow(deprecated)] + if self.fees.is_none() { + get_account_data(&Fees::id(), &mut |data: &[u8]| { + if let Ok(fees) = bincode::deserialize(data) { + self.fees = Some(fees); + } + }); + } + + #[allow(deprecated)] + if self.recent_blockhashes.is_none() { + get_account_data(&RecentBlockhashes::id(), &mut |data: &[u8]| { + if let Ok(recent_blockhashes) = bincode::deserialize(data) { + self.recent_blockhashes = Some(recent_blockhashes); + } + }); + } + } + + /// Clears all cached sysvars. + pub fn reset(&mut self) { + *self = Self::default(); + } +} + +/// Sysvar accessors that also verify the instruction account matches the +/// requested sysvar id. +pub mod get_sysvar_with_account_check { + use super::*; + + fn check_sysvar_account( + instruction_context: &InstructionContext, + instruction_account_index: IndexOfAccount, + ) -> Result<(), InstructionError> { + if !S::check_id( + instruction_context.get_key_of_instruction_account(instruction_account_index)?, + ) { + return Err(InstructionError::InvalidArgument); + } + Ok(()) + } + + /// Returns the clock sysvar after checking the provided instruction account. + pub fn clock( + invoke_context: &InvokeContext, + instruction_context: &InstructionContext, + instruction_account_index: IndexOfAccount, + ) -> Result, InstructionError> { + check_sysvar_account::(instruction_context, instruction_account_index)?; + invoke_context.get_sysvar_cache().get_clock() + } + + /// Returns the rent sysvar after checking the provided instruction account. + pub fn rent( + invoke_context: &InvokeContext, + instruction_context: &InstructionContext, + instruction_account_index: IndexOfAccount, + ) -> Result, InstructionError> { + check_sysvar_account::(instruction_context, instruction_account_index)?; + invoke_context.get_sysvar_cache().get_rent() + } + + /// Returns slot hashes after checking the provided instruction account. + pub fn slot_hashes( + invoke_context: &InvokeContext, + instruction_context: &InstructionContext, + instruction_account_index: IndexOfAccount, + ) -> Result, InstructionError> { + check_sysvar_account::(instruction_context, instruction_account_index)?; + invoke_context.get_sysvar_cache().get_slot_hashes() + } + + #[allow(deprecated)] + /// Returns recent blockhashes after checking the provided instruction account. + pub fn recent_blockhashes( + invoke_context: &InvokeContext, + instruction_context: &InstructionContext, + instruction_account_index: IndexOfAccount, + ) -> Result, InstructionError> { + check_sysvar_account::(instruction_context, instruction_account_index)?; + invoke_context.get_sysvar_cache().get_recent_blockhashes() + } + + pub fn last_restart_slot( + invoke_context: &InvokeContext, + instruction_context: &InstructionContext, + instruction_account_index: IndexOfAccount, + ) -> Result, InstructionError> { + check_sysvar_account::(instruction_context, instruction_account_index)?; + invoke_context.get_sysvar_cache().get_last_restart_slot() + } +} + +#[cfg(test)] +mod tests { + use {super::*, solana_sysvar::SysvarSerialize, test_case::test_case}; + + // sysvar cache provides the full account data of a sysvar + // the setters MUST NOT be changed to serialize an object representation + // it is required that the syscall be able to access the full buffer as it exists onchain + // this is meant to cover the cases: + // * account data is larger than struct sysvar + // * vector sysvar has fewer than its maximum entries + // if at any point the data is roundtripped through bincode, the vector will shrink + #[test_case(Clock::default(); "clock")] + #[test_case(Rent::default(); "rent")] + #[test_case(SlotHashes::default(); "slot_hashes")] + #[test_case(LastRestartSlot::default(); "last_restart_slot")] + fn test_sysvar_cache_preserves_bytes(_: T) { + let id = T::id(); + let size = T::size_of().saturating_mul(2); + let in_buf = vec![0; size]; + + let mut sysvar_cache = SysvarCache::default(); + sysvar_cache.fill_missing_entries(|pubkey, callback| { + if *pubkey == id { + callback(&in_buf) + } + }); + let sysvar_cache = sysvar_cache; + + let out_buf = sysvar_cache.sysvar_id_to_buffer(&id).clone().unwrap(); + + assert_eq!(out_buf, in_buf); + } +} diff --git a/solana/program-runtime/src/vm.rs b/solana/program-runtime/src/vm.rs new file mode 100644 index 00000000..b07b2f37 --- /dev/null +++ b/solana/program-runtime/src/vm.rs @@ -0,0 +1,384 @@ +//! SBF virtual machine provisioning and execution. + +#[cfg(feature = "svm-internal")] +use qualifier_attr::qualifiers; +use { + crate::{ + execution_budget::MAX_INSTRUCTION_STACK_DEPTH, + invoke_context::{BpfAllocator, InvokeContext, SerializedAccountMetadata, SyscallContext}, + mem_pool::VmMemoryPool, + memory_context::MemoryContext, + serialization, stable_log, + }, + cfg_if::cfg_if, + solana_instruction::error::InstructionError, + solana_program_entrypoint::{MAX_PERMITTED_DATA_INCREASE, SUCCESS}, + solana_sbpf::{ + ebpf::{self, MM_HEAP_START}, + elf::Executable, + error::{EbpfError, ProgramResult}, + memory_region::{AccessType, MemoryMapping, MemoryRegion}, + vm::{ContextObject, EbpfVm, ExecutionMode}, + }, + solana_sdk_ids::bpf_loader_deprecated, + solana_svm_log_collector::ic_logger_msg, + solana_svm_measure::measure::Measure, + solana_transaction_context::{IndexOfAccount, transaction::TransactionContext}, + std::{cell::RefCell, mem}, +}; + +thread_local! { + pub static MEMORY_POOL: RefCell = RefCell::new(VmMemoryPool::new()); +} + +/// Calculates extra compute units charged for a requested heap size. +pub fn calculate_heap_cost(heap_size: u32, heap_cost: u64) -> u64 { + const KIBIBYTE: u64 = 1024; + const PAGE_SIZE_KB: u64 = 32; + let mut rounded_heap_size = u64::from(heap_size); + rounded_heap_size = + rounded_heap_size.saturating_add(PAGE_SIZE_KB.saturating_mul(KIBIBYTE).saturating_sub(1)); + rounded_heap_size + .checked_div(PAGE_SIZE_KB.saturating_mul(KIBIBYTE)) + .expect("PAGE_SIZE_KB * KIBIBYTE > 0") + .saturating_sub(1) + .saturating_mul(heap_cost) +} + +/// Creates an SBF VM bound to the current invocation context. +#[cfg_attr(feature = "svm-internal", qualifiers(pub))] +pub fn create_vm<'a, 'b, 'c>( + program: &'a Executable>, + regions: Vec, + accounts_metadata: Vec, + invoke_context: &'a mut InvokeContext<'b, 'c>, + stack: &mut [u8], + heap: &mut [u8], +) -> Result>, Box> { + let stack_size = stack.len(); + let heap_size = heap.len(); + let memory_mapping = create_memory_mapping( + program, + stack, + heap, + regions, + invoke_context.transaction_context, + )?; + invoke_context.set_syscall_context(SyscallContext { + allocator: BpfAllocator::new(heap_size as u64), + accounts_metadata: accounts_metadata.clone(), + })?; + invoke_context.memory_contexts.set_memory_context_abi_v1(MemoryContext::new( + BpfAllocator::new(heap_size as u64), + accounts_metadata, + memory_mapping, + ))?; + Ok(EbpfVm::new( + program.get_loader().clone(), + program.get_sbpf_version(), + invoke_context, + stack_size, + )) +} + +fn create_memory_mapping<'a, C: ContextObject>( + executable: &Executable, + stack: &'a mut [u8], + heap: &'a mut [u8], + additional_regions: Vec, + transaction_context: &TransactionContext, +) -> Result> { + let config = executable.get_config(); + let sbpf_version = executable.get_sbpf_version(); + let regions: Vec = vec![ + executable.get_ro_region(), + MemoryRegion::new_gapped( + &raw mut stack[..], + ebpf::MM_STACK_START, + if sbpf_version.stack_frame_gaps() && config.enable_stack_frame_gaps { + config.stack_frame_size as u64 + } else { + 0 + }, + ), + MemoryRegion::new(&raw mut heap[..], MM_HEAP_START), + ] + .into_iter() + .chain(additional_regions) + .collect(); + + Ok(unsafe { + MemoryMapping::new_with_access_violation_handler( + regions, + config, + sbpf_version, + transaction_context.access_violation_handler(), + )? + }) +} + +/// Create the SBF virtual machine +#[macro_export] +macro_rules! create_vm { + ($vm:ident, $program:expr, $regions:expr, $accounts_metadata:expr, $invoke_context:expr $(,)?) => { + let invoke_context = &*$invoke_context; + let stack_size = $program.get_config().stack_size(); + let heap_size = invoke_context.get_compute_budget().heap_size; + let heap_cost_result = + invoke_context.consume_checked($crate::__private::calculate_heap_cost( + heap_size, + invoke_context.get_execution_cost().heap_cost, + )); + let $vm = heap_cost_result.and_then(|_| { + let (mut stack, mut heap) = $crate::__private::MEMORY_POOL + .with_borrow_mut(|pool| (pool.get_stack(stack_size), pool.get_heap(heap_size))); + let vm = $crate::__private::create_vm( + $program, + $regions, + $accounts_metadata, + $invoke_context, + stack.as_slice_mut().get_mut(..stack_size).expect("invalid stack size"), + heap.as_slice_mut().get_mut(..heap_size as usize).expect("invalid heap size"), + ); + vm.map(|vm| (vm, stack, heap)) + }); + }; +} + +#[cfg_attr(feature = "svm-internal", qualifiers(pub))] +pub fn execute<'a, 'b, 'c>( + executable: &'a Executable>, + invoke_context: &'a mut InvokeContext<'b, 'c>, +) -> Result<(), Box> { + // We dropped the lifetime tracking in the Executor by setting it to 'static, + // thus we need to reintroduce the correct lifetime of InvokeContext here again. + let executable = unsafe { + mem::transmute::< + &'a Executable>, + &'a Executable>, + >(executable) + }; + let log_collector = invoke_context.get_log_collector(); + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + let program_id = *instruction_context.get_program_key()?; + let is_loader_deprecated = + instruction_context.get_program_owner()? == bpf_loader_deprecated::id(); + cfg_if! { + if #[cfg(any( + target_os = "windows", + not(target_arch = "x86_64"), + feature = "sbpf-debugger" + ))] { + let use_jit = false; + #[cfg(feature = "sbpf-debugger")] + let debug_metadata = format!( + "program_id={};cpi_level={};caller={}", + program_id, + instruction_context.get_stack_height().saturating_sub(1), + invoke_context + .get_stack_height() + .checked_sub(2) + .and_then(|nesting_level| { + transaction_context + .get_instruction_context_at_nesting_level(nesting_level) + .ok() + }) + .and_then(|ctx| ctx.get_program_key().ok()) + .map(|key| key.to_string()) + .unwrap_or_else(|| "none".into()) + ); + } else { + let use_jit = executable.get_compiled_program().is_some(); + } + } + let direct_account_pointers_in_program_input = + invoke_context.get_feature_set().direct_account_pointers_in_program_input; + + let mut serialize_time = Measure::start("serialize"); + let (parameter_bytes, regions, accounts_metadata, instruction_data_offset) = + serialization::serialize_parameters( + &instruction_context, + direct_account_pointers_in_program_input, + )?; + serialize_time.stop(); + + // save the account addresses so in case we hit an AccessViolation error we + // can map to a more specific error + let account_region_addrs = accounts_metadata + .iter() + .map(|m| { + let vm_end = m.vm_data_addr.saturating_add(m.original_data_len as u64).saturating_add( + if !is_loader_deprecated { MAX_PERMITTED_DATA_INCREASE as u64 } else { 0 }, + ); + m.vm_data_addr..vm_end + }) + .collect::>(); + + let mut create_vm_time = Measure::start("create_vm"); + let execution_result = { + let compute_meter_prev = invoke_context.get_remaining(); + create_vm!(vm, executable, regions, accounts_metadata, invoke_context); + let (mut vm, stack, heap) = match vm { + Ok(info) => info, + Err(e) => { + ic_logger_msg!(log_collector, "Failed to create SBF VM: {}", e); + return Err(Box::new(InstructionError::ProgramEnvironmentSetupFailure)); + } + }; + create_vm_time.stop(); + + #[cfg(feature = "sbpf-debugger")] + { + vm.debug_metadata = Some(debug_metadata); + } + let mut execute_time = Measure::start("execute"); + vm.registers[1] = ebpf::MM_INPUT_START; + vm.registers[2] = instruction_data_offset as u64; + let mut execution_mode = + if use_jit { ExecutionMode::PreferJit } else { ExecutionMode::Interpreted }; + let mut call_frames = MEMORY_POOL.with_borrow_mut(|memory_pool| { + memory_pool.get_call_frames(executable.get_config().max_call_depth) + }); + let (compute_units_consumed, result) = + vm.execute_program(executable, &mut execution_mode, &mut call_frames); + let register_trace = std::mem::take(&mut vm.register_trace); + MEMORY_POOL.with_borrow_mut(|memory_pool| { + memory_pool.put_stack(stack); + memory_pool.put_heap(heap); + memory_pool.put_call_frames(call_frames); + debug_assert!(memory_pool.stack_len() <= MAX_INSTRUCTION_STACK_DEPTH); + debug_assert!(memory_pool.heap_len() <= MAX_INSTRUCTION_STACK_DEPTH); + }); + drop(vm); + invoke_context.insert_register_trace(register_trace); + execute_time.stop(); + invoke_context.timings.execute_us += execute_time.as_us(); + + ic_logger_msg!( + log_collector, + "Program {} consumed {} of {} compute units", + &program_id, + compute_units_consumed, + compute_meter_prev + ); + let (_returned_from_program_id, return_data) = + invoke_context.transaction_context.get_return_data(); + if !return_data.is_empty() { + stable_log::program_return(&log_collector, &program_id, return_data); + } + match result { + ProgramResult::Ok(status) if status != SUCCESS => { + let error: InstructionError = status.into(); + Err(Box::new(error) as Box) + } + ProgramResult::Err(mut error) => { + // Don't clean me up!! + // This feature is active on all networks, but we still toggle + // it off during fuzzing. + if invoke_context.get_feature_set().deplete_cu_meter_on_vm_failure + && !matches!(error, EbpfError::SyscallError(_)) + { + // when an exception is thrown during the execution of a + // Basic Block (e.g., a null memory dereference or other + // faults), determining the exact number of CUs consumed + // up to the point of failure requires additional effort + // and is unnecessary since these cases are rare. + // + // In order to simplify CU tracking, simply consume all + // remaining compute units so that the block cost + // tracker uses the full requested compute unit cost for + // this failed transaction. + invoke_context.consume(invoke_context.get_remaining()); + } + + if let EbpfError::SyscallError(err) = error { + error = err + .downcast::() + .map(|err| *err) + .unwrap_or_else(EbpfError::SyscallError); + } + if let EbpfError::AccessViolation(access_type, vm_addr, len, _section_name) = error + { + // Account data is directly mapped. A write to readonly data + // or an access into the reserved growth range appears as a + // memory violation; map it to the account-specific error. + if let Some((instruction_account_index, vm_addr_range)) = account_region_addrs + .iter() + .enumerate() + .find(|(_, vm_addr_range)| vm_addr_range.contains(&vm_addr)) + { + let transaction_context = &invoke_context.transaction_context; + let instruction_context = + transaction_context.get_current_instruction_context()?; + let account = instruction_context.try_borrow_instruction_account( + instruction_account_index as IndexOfAccount, + )?; + if vm_addr.saturating_add(len) <= vm_addr_range.end { + // The access was within the range of the account address space, + // but it might not be within the range of the actual data. + let is_access_outside_of_data = + vm_addr.saturating_add(len).saturating_sub(vm_addr_range.start) + as usize + > account.get_data().len(); + error = EbpfError::SyscallError(Box::new( + #[allow(deprecated)] + match access_type { + AccessType::Store => { + if let Err(err) = account.can_data_be_changed() { + err + } else { + // The store was allowed but failed, + // thus it must have been an attempt to grow the account. + debug_assert!(is_access_outside_of_data); + InstructionError::InvalidRealloc + } + } + AccessType::Load => { + // Loads should only fail when they are outside of the account data. + debug_assert!(is_access_outside_of_data); + if account.can_data_be_changed().is_err() { + // Load beyond readonly account data happened because the program + // expected more data than there actually is. + InstructionError::AccountDataTooSmall + } else { + // Load beyond writable account data also attempted to grow. + InstructionError::InvalidRealloc + } + } + }, + )); + } + } + } + Err(if let EbpfError::SyscallError(err) = error { err } else { error.into() }) + } + _ => Ok(()), + } + }; + + fn deserialize_parameters( + invoke_context: &mut InvokeContext, + parameter_bytes: &[u8], + ) -> Result<(), InstructionError> { + serialization::deserialize_parameters( + &invoke_context.transaction_context.get_current_instruction_context()?, + parameter_bytes, + &invoke_context.get_syscall_context()?.accounts_metadata, + ) + } + + let mut deserialize_time = Measure::start("deserialize"); + let execute_or_deserialize_result = execution_result.and_then(|_| { + deserialize_parameters(invoke_context, parameter_bytes.as_slice()) + .map_err(|error| Box::new(error) as Box) + }); + deserialize_time.stop(); + + // Update the timings + invoke_context.timings.serialize_us += serialize_time.as_us(); + invoke_context.timings.create_vm_us += create_vm_time.as_us(); + invoke_context.timings.deserialize_us += deserialize_time.as_us(); + + execute_or_deserialize_result +} diff --git a/solana/svm/Cargo.toml b/solana/svm/Cargo.toml new file mode 100644 index 00000000..a778afb8 --- /dev/null +++ b/solana/svm/Cargo.toml @@ -0,0 +1,84 @@ +[package] +name = "solana-svm" + +authors = { workspace = true } +description = "Solana SVM" +documentation = "https://docs.rs/solana-svm" +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = "4.1.1" + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[lib] +crate-type = ["lib"] +name = "solana_svm" + +[features] +# No-op stub retained only so external (patched-in) crates that reference +# `solana-svm/agave-unstable-api` still resolve; the lib is no longer gated on +# it, and the upstream svm-* deps enable their unstable API unconditionally below. +agave-unstable-api = [] +dev-context-only-utils = ["dep:qualifier_attr", "solana-program-runtime/dev-context-only-utils"] +# Compatibility stub forwarded to program-runtime; neither fork consumes frozen +# ABI metadata. +frozen-abi = ["solana-program-runtime/frozen-abi"] +shuttle-test = ["solana-program-runtime/shuttle-test", "solana-svm-type-overrides/shuttle-test"] +svm-internal = ["dep:qualifier_attr"] + +[dependencies] +magic-root-interface = { workspace = true } + +ahash = { workspace = true } +qualifier_attr = { workspace = true, optional = true } +serde = { workspace = true, features = ["rc"] } + +solana-account = { workspace = true } +solana-clock = { workspace = true } +solana-fee-structure = { workspace = true } +solana-hash = { workspace = true } +solana-instruction = { workspace = true, features = ["std"] } +solana-instructions-sysvar = { workspace = true } +solana-loader-v3-interface = { workspace = true, features = ["bincode"] } +solana-message = { workspace = true } +solana-program-runtime = { workspace = true } +solana-pubkey = { workspace = true } +solana-rent = { workspace = true } +solana-sdk-ids = { workspace = true } +solana-svm-callback = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-feature-set = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-log-collector = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-transaction = { workspace = true, features = ["agave-unstable-api"] } +solana-svm-type-overrides = { workspace = true, features = ["agave-unstable-api"] } +solana-system-interface = { workspace = true, features = ["bincode"] } +solana-transaction-context = { workspace = true } +solana-transaction-error = { workspace = true } + +[dev-dependencies] +bincode = { workspace = true } +env_logger = { workspace = true } +rand = { workspace = true } +solana-clock = { workspace = true } +solana-ed25519-program = { workspace = true } +solana-epoch-schedule = { workspace = true } +solana-fee-calculator = { workspace = true } +solana-keypair = { workspace = true } +solana-native-token = { workspace = true } +solana-precompile-error = { workspace = true } +solana-program-runtime = { workspace = true, features = ["dev-context-only-utils"] } +solana-pubkey = { workspace = true, features = ["rand"] } +solana-rent = { workspace = true } +solana-sbpf = { workspace = true, features = ["jit"] } +solana-signature = { workspace = true, features = ["rand"] } +solana-signer = { workspace = true } +# See order-crates-for-publishing.py for using this unusual `path = "."` +solana-svm = { path = ".", features = ["dev-context-only-utils", "svm-internal"] } +solana-sysvar = { workspace = true } +solana-transaction = { workspace = true, features = ["dev-context-only-utils"] } +solana-transaction-context = { workspace = true, features = ["dev-context-only-utils"] } + +[lints.rust] +unexpected_cfgs = "allow" diff --git a/solana/svm/README.md b/solana/svm/README.md new file mode 100644 index 00000000..1afa450b --- /dev/null +++ b/solana/svm/README.md @@ -0,0 +1,16 @@ +# `solana-svm` + +This Agave fork is the transaction-level execution entry point. Workspace +`[patch.crates-io]` entries force the dependency graph to use this copy. + +The SVM loads required accounts through the caller's +`transaction_processing_callback`, loads required programs, executes through +`solana-program-runtime`, and returns processing results and mutated accounts. +It owns no account storage. + +Persistence, commit decisions, deployment policy, and validator batch behavior +remain above this crate. Engine-specific runtime differences are documented in +[`../README.md`](../README.md). + +The `frozen-abi` feature is retained as a no-op compatibility stub and forwards +to the corresponding program-runtime feature. diff --git a/solana/svm/src/access_permissions.rs b/solana/svm/src/access_permissions.rs new file mode 100644 index 00000000..c3ae2b2c --- /dev/null +++ b/solana/svm/src/access_permissions.rs @@ -0,0 +1,267 @@ +use solana_svm_transaction::svm_message::SVMMessage; +use solana_transaction_error::TransactionError; +use std::sync::Arc; + +use crate::transaction_execution_result::ExecutedTransaction; + +impl ExecutedTransaction { + /// Enforces engine account mutability after successful execution. + pub(crate) fn access_is_valid(&mut self, tx: &impl SVMMessage) -> bool { + if !self.was_successful() { + return false; + } + let privileged = is_privileged(tx); + let mut accounts = self.loaded_transaction.accounts.iter().enumerate(); + let Some((_, payer)) = accounts.next() else { + // Sanitized transactions always carry a fee payer. + return false; + }; + let logs = Arc::make_mut(self.execution_details.log_messages.get_or_insert_default()); + for (i, (pk, acc)) in accounts { + if !tx.is_writable(i) || acc.mutable() || privileged { + continue; + } + let error = format!("Program log: Immutable account {pk} has been modified"); + logs.push(error); + self.execution_details.status = Err(TransactionError::InvalidWritableAccount); + return false; + } + // The payer is writable in Solana messages even when fees are disabled + // here; reject it only if execution actually changed immutable state. + if payer.1.dirty() && !payer.1.mutable() { + let error = format!("Program log: ({}) Feepayer account is readonly", payer.0); + logs.push(error); + self.execution_details.status = Err(TransactionError::InvalidAccountForFee); + return false; + } + true + } +} + +/// Returns true when every instruction is handled by the MagicRoot authority path. +fn is_privileged(tx: &impl SVMMessage) -> bool { + tx.program_instructions_iter().all(|(id, _)| *id == magic_root_interface::ID) +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::{ + account_loader::LoadedTransaction, + transaction_execution_result::{ExecutedTransaction, TransactionExecutionDetails}, + }, + solana_account::{AccountBuilder, AccountMode, AccountSharedData}, + solana_hash::Hash, + solana_message::{ + LegacyMessage, Message, MessageHeader, SanitizedMessage, + compiled_instruction::CompiledInstruction, + }, + solana_pubkey::Pubkey, + solana_signature::Signature, + solana_transaction::sanitized::SanitizedTransaction, + solana_transaction_error::TransactionResult, + std::collections::HashSet, + }; + + /// A dirtied account in the requested mode. + fn account(mode: AccountMode) -> AccountSharedData { + let mut acc: AccountSharedData = AccountBuilder::default().mode(mode).build(); + acc.set_data_from_slice(&[1]); + acc + } + + /// Builds a sanitized transaction over `account_keys` whose only writable + /// non-signer accounts are the first `writable_non_signers` after the payer, + /// invoking one instruction per entry in `program_indices`. + /// + /// Layout is `[payer, non-signers.., program..]`: the payer signs and is + /// writable, and `is_writable(i)` follows directly from the header math the + /// engine guard relies on. + fn sanitized_tx( + account_keys: Vec, + writable_non_signers: u8, + program_indices: &[u8], + ) -> SanitizedTransaction { + let non_signers = account_keys.len() as u8 - 1; + let header = MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: non_signers - writable_non_signers, + }; + let instructions = program_indices + .iter() + .map(|&program_id_index| CompiledInstruction { + program_id_index, + accounts: vec![], + data: vec![], + }) + .collect(); + let message = Message { + account_keys, + header, + instructions, + recent_blockhash: Hash::default(), + }; + let sanitized = SanitizedMessage::Legacy(LegacyMessage::new(message, &HashSet::new())); + SanitizedTransaction::new_for_tests(sanitized, vec![Signature::new_unique()], false) + } + + /// Wraps executed account state and a status into an `ExecutedTransaction`. + fn executed( + accounts: Vec<(Pubkey, AccountSharedData)>, + status: TransactionResult<()>, + ) -> ExecutedTransaction { + ExecutedTransaction { + loaded_transaction: LoadedTransaction { accounts, ..Default::default() }, + execution_details: TransactionExecutionDetails { + status, + log_messages: None, + inner_instructions: None, + return_data: None, + executed_units: 0, + accounts_data_len_delta: 0, + }, + } + } + + /// Runs the guard over a `[payer, target, program]` transaction, returning + /// its verdict and the (possibly rewritten) execution details. + /// + /// `program` is MagicRoot when `privileged`, so the whole tx takes the + /// authority path; only `target` (index 1) is ever writable in the message. + fn run( + payer: AccountSharedData, + target: AccountSharedData, + writable_target: bool, + privileged: bool, + status: TransactionResult<()>, + ) -> (bool, ExecutedTransaction) { + let payer_key = Pubkey::new_unique(); + let target_key = Pubkey::new_unique(); + let program = if privileged { magic_root_interface::ID } else { Pubkey::new_unique() }; + let tx = sanitized_tx( + vec![payer_key, target_key, program], + writable_target as u8, + &[2], + ); + let mut executed = executed( + vec![ + (payer_key, payer), + (target_key, target), + (program, AccountSharedData::default()), + ], + status, + ); + let verdict = executed.access_is_valid(&tx); + (verdict, executed) + } + + /// Asserts a log line containing `needle` was recorded. + fn assert_logged(tx: &ExecutedTransaction, needle: &str) { + let logs: &[String] = tx + .execution_details + .log_messages + .as_deref() + .map(Vec::as_slice) + .unwrap_or_default(); + assert!( + logs.iter().any(|l| l.contains(needle)), + "expected {needle:?} in {logs:?}" + ); + } + + #[test] + fn writable_account_guard() { + let mut transitioned = account(AccountMode::Delegated); + transitioned.set_mode(AccountMode::Transient).unwrap(); + let mut closed = account(AccountMode::Ephemeral); + closed.set_mode(AccountMode::Closed).unwrap(); + + // A dirty, writable, immutable operand is rejected — unless the tx is + // privileged or it legally entered a transaction-final mode. Mutable or + // message-read-only operands are always fine. + let cases = [ + // (target, writable, privileged, accepted) + (account(AccountMode::ReadOnly), true, false, false), + (account(AccountMode::Transient), true, false, false), + (transitioned, true, false, true), + (closed, true, false, true), + (account(AccountMode::Delegated), true, false, true), + (account(AccountMode::ReadOnly), false, false, true), // read-only in the message + (account(AccountMode::ReadOnly), true, true, true), // MagicRoot bypass + ]; + for (i, (target, writable, privileged, accepted)) in cases.into_iter().enumerate() { + let (verdict, executed) = run( + AccountSharedData::default(), + target, + writable, + privileged, + Ok(()), + ); + assert_eq!(verdict, accepted, "case {i}"); + let expected = + if accepted { Ok(()) } else { Err(TransactionError::InvalidWritableAccount) }; + assert_eq!(executed.execution_details.status, expected, "case {i}"); + if !accepted { + assert_logged(&executed, "Immutable account"); + } + } + } + + #[test] + fn fee_payer_guard() { + // A dirty immutable fee payer is rejected; a mutable + // payer is always accepted. + let cases = [ + // (payer, privileged, accepted) + (account(AccountMode::ReadOnly), false, false), + (account(AccountMode::Transient), false, false), + (account(AccountMode::Delegated), false, true), + ]; + for (i, (payer, privileged, accepted)) in cases.into_iter().enumerate() { + let (verdict, executed) = run( + payer, + AccountSharedData::default(), + false, + privileged, + Ok(()), + ); + assert_eq!(verdict, accepted, "case {i}"); + let expected = + if accepted { Ok(()) } else { Err(TransactionError::InvalidAccountForFee) }; + assert_eq!(executed.execution_details.status, expected, "case {i}"); + if !accepted { + assert_logged(&executed, "Feepayer account"); + } + } + } + + #[test] + fn guard_edge_cases() { + // A failed run committed nothing, so a dirty immutable account must not + // rewrite the original error into an access error. + let (verdict, failed) = run( + AccountSharedData::default(), + account(AccountMode::ReadOnly), + true, + false, + Err(TransactionError::AccountInUse), + ); + assert!(!verdict); + assert_eq!( + failed.execution_details.status, + Err(TransactionError::AccountInUse) + ); + + // A transaction without a fee payer account cannot be validated. + let tx = sanitized_tx(vec![Pubkey::new_unique(), Pubkey::new_unique()], 0, &[1]); + let mut payerless = executed(vec![], Ok(())); + assert!(!payerless.access_is_valid(&tx)); + + // Privilege requires *every* instruction to invoke MagicRoot. + let keys = vec![Pubkey::new_unique(), magic_root_interface::ID, Pubkey::new_unique()]; + assert!(is_privileged(&sanitized_tx(keys.clone(), 0, &[1, 1]))); + assert!(!is_privileged(&sanitized_tx(keys, 0, &[1, 2]))); + } +} diff --git a/solana/svm/src/account_loader.rs b/solana/svm/src/account_loader.rs new file mode 100644 index 00000000..5cfea00d --- /dev/null +++ b/solana/svm/src/account_loader.rs @@ -0,0 +1,1421 @@ +#[cfg(feature = "dev-context-only-utils")] +use qualifier_attr::{field_qualifiers, qualifiers}; +use { + solana_account::{Account, AccountSharedData, PROGRAM_OWNERS, ReadableAccount}, + solana_fee_structure::FeeDetails, + solana_instruction::{BorrowedAccountMeta, BorrowedInstruction}, + solana_instructions_sysvar::construct_instructions_data, + solana_program_runtime::execution_budget::{ + SVMTransactionExecutionAndFeeBudgetLimits, SVMTransactionExecutionBudget, + }, + solana_pubkey::Pubkey, + solana_sdk_ids::sysvar, + solana_svm_callback::TransactionProcessingCallback, + solana_svm_transaction::svm_message::SVMMessage, + solana_transaction_context::{IndexOfAccount, transaction_accounts::KeyedAccountSharedData}, + solana_transaction_error::{TransactionError, TransactionResult as Result}, +}; + +// Per SIMD-0186, all accounts are assigned a base size of 64 bytes to cover +// the storage cost of metadata. +#[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] +pub(crate) const TRANSACTION_ACCOUNT_BASE_SIZE: usize = 64; + +// Per SIMD-0186, resolved address lookup tables are assigned a base size of 8248 +// bytes: 8192 bytes for the maximum table size plus 56 bytes for metadata. +const ADDRESS_LOOKUP_TABLE_BASE_SIZE: usize = 8248; + +/// Result of transaction prechecking before account loading. +pub type TransactionCheckResult = Result; + +#[derive(PartialEq, Eq, Debug)] +pub(crate) enum TransactionLoadResult { + /// All transaction accounts and executable program accounts were resolved. + Loaded(LoadedTransaction), + /// Loading failed before execution could start. + NotLoaded(TransactionError), +} + +/// Transaction limits and metadata computed before account loading. +#[derive(PartialEq, Eq, Debug, Clone)] +#[cfg_attr(feature = "svm-internal", qualifier_attr::field_qualifiers(nonce_address(pub)))] +pub struct CheckedTransactionDetails { + pub(crate) nonce_address: Option, + pub(crate) compute_budget_and_limits: SVMTransactionExecutionAndFeeBudgetLimits, +} + +impl Default for CheckedTransactionDetails { + fn default() -> Self { + Self { + nonce_address: None, + compute_budget_and_limits: SVMTransactionExecutionAndFeeBudgetLimits { + budget: SVMTransactionExecutionBudget::default(), + loaded_accounts_data_size_limit: 32, + fee_details: FeeDetails::default(), + }, + } + } +} + +impl CheckedTransactionDetails { + /// Creates checked transaction details from caller-provided validation. + pub fn new( + nonce_address: Option, + compute_budget_and_limits: SVMTransactionExecutionAndFeeBudgetLimits, + ) -> Self { + Self { + nonce_address, + compute_budget_and_limits, + } + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +pub(crate) struct ValidatedTransactionDetails { + pub(crate) compute_budget: SVMTransactionExecutionBudget, + pub(crate) loaded_accounts_bytes_limit: u32, + pub(crate) fee_details: FeeDetails, +} + +#[cfg(feature = "dev-context-only-utils")] +impl Default for ValidatedTransactionDetails { + fn default() -> Self { + Self { + compute_budget: SVMTransactionExecutionBudget::default(), + loaded_accounts_bytes_limit: + solana_program_runtime::execution_budget::MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + fee_details: FeeDetails::default(), + } + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +#[cfg_attr(feature = "dev-context-only-utils", derive(Default))] +pub(crate) struct LoadedTransactionAccount { + pub(crate) account: AccountSharedData, + pub(crate) loaded_size: usize, +} + +impl LoadedTransactionAccount { + fn new(account: AccountSharedData) -> Self { + Self { + loaded_size: TRANSACTION_ACCOUNT_BASE_SIZE.saturating_add(account.data().len()), + account, + } + } +} + +/// Accounts and execution metadata needed to run one transaction. +#[derive(PartialEq, Eq, Debug, Clone, Default)] +#[cfg_attr( + feature = "dev-context-only-utils", + field_qualifiers(program_indices(pub), compute_budget(pub)) +)] +pub struct LoadedTransaction { + /// Transaction accounts in message account-key order. + pub accounts: Vec, + pub(crate) program_indices: Vec, + /// Fee metadata carried through for callers that still consume it. + pub fee_details: FeeDetails, + pub(crate) compute_budget: SVMTransactionExecutionBudget, + /// Total loaded account data size charged against the transaction limit. + pub loaded_accounts_data_size: u32, +} + +pub(crate) fn load_transaction( + account_loader: &CB, + message: &impl SVMMessage, + validation_details: ValidatedTransactionDetails, +) -> TransactionLoadResult { + let load_result = load_transaction_accounts( + account_loader, + message, + validation_details.loaded_accounts_bytes_limit, + ); + + match load_result { + Ok(loaded_tx_accounts) => TransactionLoadResult::Loaded(LoadedTransaction { + accounts: loaded_tx_accounts.accounts, + program_indices: loaded_tx_accounts.program_indices, + fee_details: validation_details.fee_details, + compute_budget: validation_details.compute_budget, + loaded_accounts_data_size: loaded_tx_accounts.loaded_accounts_data_size, + }), + Err(err) => TransactionLoadResult::NotLoaded(err), + } +} + +#[derive(PartialEq, Eq, Debug, Clone)] +struct LoadedTransactionAccounts { + pub(crate) accounts: Vec, + pub(crate) program_indices: Vec, + pub(crate) loaded_accounts_data_size: u32, +} + +impl LoadedTransactionAccounts { + fn increase_calculated_data_size( + &mut self, + data_size_delta: usize, + requested_loaded_accounts_data_size_limit: u32, + ) -> Result<()> { + let Ok(data_size_delta) = u32::try_from(data_size_delta) else { + return Err(TransactionError::MaxLoadedAccountsDataSizeExceeded); + }; + + self.loaded_accounts_data_size = + self.loaded_accounts_data_size.saturating_add(data_size_delta); + + if self.loaded_accounts_data_size > requested_loaded_accounts_data_size_limit { + Err(TransactionError::MaxLoadedAccountsDataSizeExceeded) + } else { + Ok(()) + } + } +} + +fn load_transaction_accounts( + account_loader: &CB, + message: &impl SVMMessage, + loaded_accounts_bytes_limit: u32, +) -> Result { + let account_keys = message.account_keys(); + + let mut loaded_transaction_accounts = LoadedTransactionAccounts { + accounts: Vec::with_capacity(account_keys.len()), + program_indices: Vec::with_capacity(message.num_instructions()), + loaded_accounts_data_size: 0, + }; + + // Transactions pay a base fee per address lookup table. + loaded_transaction_accounts.increase_calculated_data_size( + message.num_lookup_tables().saturating_mul(ADDRESS_LOOKUP_TABLE_BASE_SIZE), + loaded_accounts_bytes_limit, + )?; + + let mut collect_loaded_account = |key: &Pubkey, loaded_account| -> Result<()> { + let LoadedTransactionAccount { account, loaded_size } = loaded_account; + + loaded_transaction_accounts + .increase_calculated_data_size(loaded_size, loaded_accounts_bytes_limit)?; + + loaded_transaction_accounts.accounts.push((*key, account)); + + Ok(()) + }; + + // Attempt to load all of the transaction accounts + for account_key in account_keys.iter() { + let loaded_account = load_transaction_account(account_loader, message, account_key); + collect_loaded_account(account_key, loaded_account)?; + } + + for (program_id, instruction) in message.program_instructions_iter() { + let Some(program_account) = account_loader.get_account_shared_data(program_id) else { + return Err(TransactionError::ProgramAccountNotFound); + }; + + let owner_id = program_account.0.owner(); + if !PROGRAM_OWNERS.contains(owner_id) { + return Err(TransactionError::InvalidProgramForExecution); + } + + loaded_transaction_accounts + .program_indices + .push(instruction.program_id_index as IndexOfAccount); + } + + Ok(loaded_transaction_accounts) +} + +fn load_transaction_account( + account_loader: &CB, + message: &impl SVMMessage, + account_key: &Pubkey, +) -> LoadedTransactionAccount { + if solana_sdk_ids::sysvar::instructions::check_id(account_key) { + // Since the instructions sysvar is constructed by the SVM and modified + // for each transaction instruction, it cannot be loaded. + return LoadedTransactionAccount { + loaded_size: 0, + account: construct_instructions_account(message), + }; + } + account_loader + .get_account_shared_data(account_key) + .map(|a| LoadedTransactionAccount::new(a.0)) + .unwrap_or_else(|| LoadedTransactionAccount::new(Default::default())) +} + +fn construct_instructions_account(message: &impl SVMMessage) -> AccountSharedData { + let account_keys = message.account_keys(); + let mut decompiled_instructions = Vec::with_capacity(message.num_instructions()); + for (program_id, instruction) in message.program_instructions_iter() { + let accounts = instruction + .accounts + .iter() + .map(|account_index| { + let account_index = usize::from(*account_index); + BorrowedAccountMeta { + is_signer: message.is_signer(account_index), + is_writable: message.is_writable(account_index), + pubkey: account_keys.get(account_index).unwrap(), + } + }) + .collect(); + + decompiled_instructions.push(BorrowedInstruction { + accounts, + data: instruction.data, + program_id, + }); + } + + AccountSharedData::from(Account { + data: construct_instructions_data(&decompiled_instructions).unwrap_or_default(), + owner: sysvar::id(), + ..Account::default() + }) +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::{ + rent_calculator::RENT_EXEMPT_RENT_EPOCH, + transaction_account_state_info::TransactionAccountStateInfo, + }, + ahash::AHashMap, + rand::prelude::*, + solana_account::{ + Account, AccountSharedData, ReadableAccount, WritableAccount, state_traits::StateMut, + }, + solana_clock::Slot, + solana_hash::Hash, + solana_instruction::{AccountMeta, Instruction}, + solana_keypair::Keypair, + solana_loader_v3_interface::state::UpgradeableLoaderState, + solana_message::{ + LegacyMessage, Message, MessageHeader, SanitizedMessage, + compiled_instruction::CompiledInstruction, + v0::{LoadedAddresses, LoadedMessage}, + }, + solana_native_token::LAMPORTS_PER_SOL, + solana_program_runtime::execution_budget::{ + DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT, MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES, + }, + solana_pubkey::Pubkey, + solana_rent::Rent, + solana_sdk_ids::{ + bpf_loader, bpf_loader_upgradeable, native_loader, system_program, sysvar, + }, + solana_signature::Signature, + solana_signer::Signer, + solana_svm_callback::{AccountState, InvokeContextCallback, TransactionProcessingCallback}, + solana_system_interface::instruction as system_instruction, + solana_transaction::{Transaction, sanitized::SanitizedTransaction}, + solana_transaction_context::{ + transaction::TransactionContext, transaction_accounts::KeyedAccountSharedData, + }, + solana_transaction_error::TransactionError, + std::{ + borrow::Cow, + cell::RefCell, + collections::{HashMap, HashSet}, + sync::Arc, + }, + }; + + fn setup_test_logger() { + let _ = env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("error")) + .format_timestamp_nanos() + .is_test(true) + .try_init(); + } + + #[derive(Clone, Default)] + struct TestCallbacks { + accounts_map: HashMap, + #[allow(clippy::type_complexity)] + inspected_accounts: + RefCell, /* is_writable */ bool)>>>, + } + + impl InvokeContextCallback for TestCallbacks {} + + impl TransactionProcessingCallback for TestCallbacks { + fn get_account_shared_data(&self, pubkey: &Pubkey) -> Option<(AccountSharedData, Slot)> { + self.accounts_map.get(pubkey).map(|(account, slot)| (account.clone(), *slot)) + } + + fn inspect_account( + &self, + address: &Pubkey, + account_state: AccountState, + is_writable: bool, + ) { + let account = match account_state { + AccountState::Dead => None, + AccountState::Alive(account) => Some(account.clone()), + }; + self.inspected_accounts + .borrow_mut() + .entry(*address) + .or_default() + .push((account, is_writable)); + } + } + + fn load_accounts_with_features_and_rent( + tx: Transaction, + accounts: &[KeyedAccountSharedData], + ) -> TransactionLoadResult { + let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); + let mut accounts_map = HashMap::new(); + for (pubkey, account) in accounts { + accounts_map.insert(*pubkey, (account.clone(), 1)); + } + let callbacks = TestCallbacks { + accounts_map, + ..Default::default() + }; + load_transaction( + &callbacks, + &sanitized_tx, + ValidatedTransactionDetails::default(), + ) + } + + fn new_unchecked_sanitized_message(message: Message) -> SanitizedMessage { + SanitizedMessage::Legacy(LegacyMessage::new(message, &HashSet::new())) + } + + #[test] + fn test_load_accounts_unknown_program_id() { + let mut accounts: Vec = Vec::new(); + + let keypair = Keypair::new(); + let key0 = keypair.pubkey(); + let key1 = Pubkey::from([5u8; 32]); + + let account = AccountSharedData::new(1, 0, &Pubkey::default()); + accounts.push((key0, account)); + + let account = AccountSharedData::new(2, 1, &Pubkey::default()); + accounts.push((key1, account)); + + let instructions = vec![CompiledInstruction::new(1, &(), vec![0])]; + let tx = Transaction::new_with_compiled_instructions( + &[&keypair], + &[], + Hash::default(), + vec![Pubkey::default()], + instructions, + ); + + let load_results = load_accounts_with_features_and_rent(tx, &accounts); + + assert!(matches!( + load_results, + TransactionLoadResult::NotLoaded(TransactionError::ProgramAccountNotFound), + )); + } + + #[test] + fn test_load_accounts_no_loaders() { + let mut accounts: Vec = Vec::new(); + + let keypair = Keypair::new(); + let key0 = keypair.pubkey(); + let key1 = Pubkey::from([5u8; 32]); + + let mut account = AccountSharedData::new(1, 0, &Pubkey::default()); + account.set_rent_epoch(1); + accounts.push((key0, account)); + + let mut account = AccountSharedData::new(2, 1, &Pubkey::default()); + account.set_rent_epoch(1); + accounts.push((key1, account)); + + let instructions = vec![CompiledInstruction::new(2, &(), vec![0, 1])]; + let tx = Transaction::new_with_compiled_instructions( + &[&keypair], + &[key1], + Hash::default(), + vec![native_loader::id()], + instructions, + ); + + let loaded_accounts = load_accounts_with_features_and_rent(tx, &accounts); + + match &loaded_accounts { + TransactionLoadResult::NotLoaded(err) => { + assert_eq!(*err, TransactionError::ProgramAccountNotFound); + } + result => panic!("unexpected result: {result:?}"), + } + } + + #[test] + fn test_load_accounts_bad_owner() { + let mut accounts: Vec = Vec::new(); + + let keypair = Keypair::new(); + let key0 = keypair.pubkey(); + let key1 = Pubkey::from([5u8; 32]); + + let account = AccountSharedData::new(1, 0, &Pubkey::default()); + accounts.push((key0, account)); + + let mut account = AccountSharedData::new(40, 1, &Pubkey::default()); + account.set_executable(true); + accounts.push((key1, account)); + + let instructions = vec![CompiledInstruction::new(1, &(), vec![0])]; + let tx = Transaction::new_with_compiled_instructions( + &[&keypair], + &[], + Hash::default(), + vec![key1], + instructions, + ); + + let load_results = load_accounts_with_features_and_rent(tx, &accounts); + + assert!(matches!( + load_results, + TransactionLoadResult::NotLoaded(TransactionError::InvalidProgramForExecution), + )); + } + + #[test] + fn test_load_accounts_not_executable() { + let mut accounts: Vec = Vec::new(); + + let keypair = Keypair::new(); + let key0 = keypair.pubkey(); + let key1 = Pubkey::from([5u8; 32]); + + let account = AccountSharedData::new(1, 0, &Pubkey::default()); + accounts.push((key0, account)); + + let account = AccountSharedData::new(40, 0, &native_loader::id()); + accounts.push((key1, account)); + + let instructions = vec![CompiledInstruction::new(1, &(), vec![0])]; + let tx = Transaction::new_with_compiled_instructions( + &[&keypair], + &[], + Hash::default(), + vec![key1], + instructions, + ); + + let load_results = load_accounts_with_features_and_rent(tx, &accounts); + + match &load_results { + TransactionLoadResult::Loaded(loaded_transaction) => { + assert_eq!(loaded_transaction.accounts.len(), 2); + assert_eq!(loaded_transaction.accounts[0].1, accounts[0].1); + assert_eq!(loaded_transaction.accounts[1].1, accounts[1].1); + assert_eq!(loaded_transaction.program_indices.len(), 1); + assert_eq!(loaded_transaction.program_indices[0], 1); + } + TransactionLoadResult::NotLoaded(e) => panic!("{e}"), + } + } + + #[test] + fn test_load_accounts_multiple_loaders() { + let mut accounts: Vec = Vec::new(); + + let keypair = Keypair::new(); + let key0 = keypair.pubkey(); + let key1 = bpf_loader_upgradeable::id(); + let key2 = Pubkey::from([6u8; 32]); + + let mut account = AccountSharedData::new(1, 0, &Pubkey::default()); + account.set_rent_epoch(1); + accounts.push((key0, account)); + + let mut account = AccountSharedData::new(40, 1, &Pubkey::default()); + account.set_executable(true); + account.set_rent_epoch(1); + account.set_owner(native_loader::id()); + accounts.push((key1, account)); + + let mut account = AccountSharedData::new(41, 1, &Pubkey::default()); + account.set_executable(true); + account.set_rent_epoch(1); + account.set_owner(key1); + accounts.push((key2, account)); + + let instructions = vec![ + CompiledInstruction::new(1, &(), vec![0]), + CompiledInstruction::new(2, &(), vec![0]), + ]; + let tx = Transaction::new_with_compiled_instructions( + &[&keypair], + &[], + Hash::default(), + vec![key1, key2], + instructions, + ); + + let loaded_accounts = load_accounts_with_features_and_rent(tx, &accounts); + + match &loaded_accounts { + TransactionLoadResult::Loaded(loaded_transaction) => { + assert_eq!(loaded_transaction.accounts.len(), 3); + assert_eq!(loaded_transaction.accounts[0].1, accounts[0].1); + assert_eq!(loaded_transaction.program_indices.len(), 2); + assert_eq!(loaded_transaction.program_indices[0], 1); + assert_eq!(loaded_transaction.program_indices[1], 2); + } + TransactionLoadResult::NotLoaded(e) => panic!("{e}"), + } + } + + fn load_accounts_no_store( + accounts: &[KeyedAccountSharedData], + tx: Transaction, + ) -> TransactionLoadResult { + let tx = SanitizedTransaction::from_transaction_for_tests(tx); + + let mut accounts_map = HashMap::new(); + for (pubkey, account) in accounts { + accounts_map.insert(*pubkey, (account.clone(), 1)); + } + let callbacks = TestCallbacks { + accounts_map, + ..Default::default() + }; + load_transaction(&callbacks, &tx, ValidatedTransactionDetails::default()) + } + + #[test] + fn test_instructions() { + setup_test_logger(); + let instructions_key = solana_sdk_ids::sysvar::instructions::id(); + let keypair = Keypair::new(); + let instructions = vec![CompiledInstruction::new(1, &(), vec![0, 1])]; + let tx = Transaction::new_with_compiled_instructions( + &[&keypair], + &[solana_pubkey::new_rand(), instructions_key], + Hash::default(), + vec![native_loader::id()], + instructions, + ); + + let load_results = load_accounts_no_store(&[], tx); + assert!(matches!( + load_results, + TransactionLoadResult::NotLoaded(TransactionError::ProgramAccountNotFound), + )); + } + + #[test] + fn test_increase_calculated_data_size() { + let mut acc = LoadedTransactionAccounts { + accounts: vec![], + program_indices: vec![], + loaded_accounts_data_size: 0, + }; + + let data_size: usize = 123; + let requested_data_size_limit = data_size as u32; + + // OK - loaded data size is up to limit + assert!(acc.increase_calculated_data_size(data_size, requested_data_size_limit).is_ok()); + assert_eq!(data_size as u32, acc.loaded_accounts_data_size); + + // fail - loading more data that would exceed limit + let another_byte: usize = 1; + assert_eq!( + acc.increase_calculated_data_size(another_byte, requested_data_size_limit), + Err(TransactionError::MaxLoadedAccountsDataSizeExceeded) + ); + } + + #[test] + fn test_construct_instructions_account() { + let loaded_message = LoadedMessage { + message: Cow::Owned(solana_message::v0::Message::default()), + loaded_addresses: Cow::Owned(LoadedAddresses::default()), + is_writable_account_cache: vec![false], + }; + let message = SanitizedMessage::V0(loaded_message); + let shared_data = construct_instructions_account(&message); + let expected = AccountSharedData::from(Account { + data: construct_instructions_data(&message.decompile_instructions()).unwrap(), + owner: sysvar::id(), + ..Account::default() + }); + assert_eq!(shared_data, expected); + } + + #[test] + fn test_load_transaction_accounts_fee_payer() { + let fee_payer_address = Pubkey::new_unique(); + let message = Message { + account_keys: vec![fee_payer_address], + header: MessageHeader::default(), + instructions: vec![], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + + let fee_payer_balance = 200; + let mut fee_payer_account = AccountSharedData::default(); + fee_payer_account.set_lamports(fee_payer_balance); + mock_bank.accounts_map.insert(fee_payer_address, (fee_payer_account.clone(), 1)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + assert_eq!( + result.unwrap(), + LoadedTransactionAccounts { + accounts: vec![(fee_payer_address, fee_payer_account)], + program_indices: vec![], + loaded_accounts_data_size: TRANSACTION_ACCOUNT_BASE_SIZE as u32, + } + ); + } + + #[test] + fn test_load_transaction_accounts_native_loader() { + let key1 = Keypair::new(); + let message = Message { + account_keys: vec![key1.pubkey(), native_loader::id()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + mock_bank + .accounts_map + .insert(native_loader::id(), (AccountSharedData::default(), 0)); + let mut fee_payer_account = AccountSharedData::default(); + fee_payer_account.set_lamports(200); + mock_bank.accounts_map.insert(key1.pubkey(), (fee_payer_account.clone(), 1)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + assert_eq!( + result.unwrap_err(), + TransactionError::InvalidProgramForExecution + ); + } + + #[test] + fn test_load_transaction_accounts_program_account_no_data() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + + let message = Message { + account_keys: vec![key1.pubkey(), key2.pubkey()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0, 1], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 1)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + assert_eq!(result.err(), Some(TransactionError::ProgramAccountNotFound)); + } + + #[test] + fn test_load_transaction_accounts_invalid_program_for_execution() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + + let message = Message { + account_keys: vec![key1.pubkey(), key2.pubkey()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![0, 1], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 1)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + assert_eq!( + result.err(), + Some(TransactionError::InvalidProgramForExecution) + ); + } + + #[test] + fn test_load_transaction_accounts_native_loader_owner() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_owner(native_loader::id()); + account_data.set_lamports(1); + account_data.set_executable(true); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 1)); + + let mut fee_payer_account = AccountSharedData::default(); + fee_payer_account.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), (fee_payer_account.clone(), 1)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + let loaded_accounts_data_size = TRANSACTION_ACCOUNT_BASE_SIZE as u32 * 2; + + assert_eq!( + result.unwrap(), + LoadedTransactionAccounts { + accounts: vec![ + (key2.pubkey(), fee_payer_account), + ( + key1.pubkey(), + mock_bank.accounts_map[&key1.pubkey()].0.clone() + ), + ], + program_indices: vec![1], + loaded_accounts_data_size, + } + ); + } + + #[test] + fn test_load_transaction_accounts_program_account_not_found_after_all_checks() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_executable(true); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 1)); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), (account_data, 1)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + assert_eq!( + result.err(), + Some(TransactionError::InvalidProgramForExecution) + ); + } + + #[test] + fn test_load_transaction_accounts_program_account_invalid_program_for_execution_last_check() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(key3.pubkey()); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 1)); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), (account_data, 1)); + mock_bank.accounts_map.insert(key3.pubkey(), (AccountSharedData::default(), 0)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + assert_eq!( + result.err(), + Some(TransactionError::InvalidProgramForExecution) + ); + } + + #[test] + fn test_load_transaction_accounts_program_success_complete() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey()], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(bpf_loader::id()); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 1)); + + let mut fee_payer_account = AccountSharedData::default(); + fee_payer_account.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), (fee_payer_account.clone(), 1)); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(native_loader::id()); + mock_bank.accounts_map.insert(bpf_loader::id(), (account_data, 0)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + let loaded_accounts_data_size = TRANSACTION_ACCOUNT_BASE_SIZE as u32 * 2; + + assert_eq!( + result.unwrap(), + LoadedTransactionAccounts { + accounts: vec![ + (key2.pubkey(), fee_payer_account), + ( + key1.pubkey(), + mock_bank.accounts_map[&key1.pubkey()].0.clone() + ), + ], + program_indices: vec![1], + loaded_accounts_data_size, + } + ); + } + + #[test] + fn test_load_transaction_accounts_program_builtin_saturating_add() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key3.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(bpf_loader::id()); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 0)); + + let mut fee_payer_account = AccountSharedData::default(); + fee_payer_account.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), (fee_payer_account.clone(), 1)); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(native_loader::id()); + mock_bank.accounts_map.insert(bpf_loader::id(), (account_data, 0)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let result = load_transaction_accounts( + &mock_bank, + sanitized_transaction.message(), + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ); + + let loaded_accounts_data_size = TRANSACTION_ACCOUNT_BASE_SIZE as u32 * 3; + + let mut account_data = AccountSharedData::default(); + account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + assert_eq!( + result.unwrap(), + LoadedTransactionAccounts { + accounts: vec![ + (key2.pubkey(), fee_payer_account), + ( + key1.pubkey(), + mock_bank.accounts_map[&key1.pubkey()].0.clone() + ), + (key3.pubkey(), account_data), + ], + program_indices: vec![1, 1], + loaded_accounts_data_size, + } + ); + } + + #[test] + fn test_rent_state_list_len() { + let mint_keypair = Keypair::new(); + let mut bank = TestCallbacks::default(); + let recipient = Pubkey::new_unique(); + let last_block_hash = Hash::new_unique(); + + let mut system_data = AccountSharedData::default(); + system_data.set_lamports(1); + system_data.set_executable(true); + system_data.set_owner(native_loader::id()); + bank.accounts_map.insert(Pubkey::new_from_array([0u8; 32]), (system_data, 0)); + + let mut mint_data = AccountSharedData::default(); + mint_data.set_lamports(2); + bank.accounts_map.insert(mint_keypair.pubkey(), (mint_data, 0)); + bank.accounts_map.insert(recipient, (AccountSharedData::default(), 1)); + let mut tx = Transaction::new_with_payer( + &[system_instruction::transfer( + &mint_keypair.pubkey(), + &recipient, + LAMPORTS_PER_SOL, + )], + Some(&mint_keypair.pubkey()), + ); + tx.sign(&[&mint_keypair], last_block_hash); + let num_accounts = tx.message().account_keys.len(); + let sanitized_tx = SanitizedTransaction::from_transaction_for_tests(tx); + let load_result = + load_transaction(&bank, &sanitized_tx, ValidatedTransactionDetails::default()); + + let TransactionLoadResult::Loaded(loaded_transaction) = load_result else { + panic!("transaction loading failed"); + }; + + let compute_budget = SVMTransactionExecutionBudget { + compute_unit_limit: u64::from(DEFAULT_INSTRUCTION_COMPUTE_UNIT_LIMIT), + ..SVMTransactionExecutionBudget::default() + }; + let rent = Rent::default(); + let transaction_context = TransactionContext::new( + loaded_transaction.accounts, + rent.clone(), + compute_budget.max_instruction_stack_depth, + compute_budget.max_instruction_trace_length, + 1, + ); + + assert_eq!( + TransactionAccountStateInfo::new(&transaction_context, sanitized_tx.message(), &rent,) + .len(), + num_accounts, + ); + } + + #[test] + fn test_load_accounts_success() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key3.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut mock_bank = TestCallbacks::default(); + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(bpf_loader::id()); + mock_bank.accounts_map.insert(key1.pubkey(), (account_data, 0)); + + let mut fee_payer_account = AccountSharedData::default(); + fee_payer_account.set_lamports(200); + mock_bank.accounts_map.insert(key2.pubkey(), (fee_payer_account.clone(), 1)); + + let mut account_data = AccountSharedData::default(); + account_data.set_lamports(1); + account_data.set_executable(true); + account_data.set_owner(native_loader::id()); + mock_bank.accounts_map.insert(bpf_loader::id(), (account_data, 0)); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let load_result = load_transaction( + &mock_bank, + &sanitized_transaction, + ValidatedTransactionDetails::default(), + ); + + let loaded_accounts_data_size = TRANSACTION_ACCOUNT_BASE_SIZE as u32 * 3; + + let mut account_data = AccountSharedData::default(); + account_data.set_rent_epoch(RENT_EXEMPT_RENT_EPOCH); + + let TransactionLoadResult::Loaded(loaded_transaction) = load_result else { + panic!("transaction loading failed"); + }; + assert_eq!( + loaded_transaction, + LoadedTransaction { + accounts: vec![ + ( + key2.pubkey(), + mock_bank.accounts_map[&key2.pubkey()].0.clone() + ), + ( + key1.pubkey(), + mock_bank.accounts_map[&key1.pubkey()].0.clone() + ), + (key3.pubkey(), account_data), + ], + program_indices: vec![1, 1], + fee_details: FeeDetails::default(), + compute_budget: SVMTransactionExecutionBudget::default(), + loaded_accounts_data_size, + } + ); + } + + #[test] + fn test_load_accounts_error() { + let mock_bank = TestCallbacks::default(); + let message = Message { + account_keys: vec![Pubkey::new_from_array([0; 32])], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let load_result = load_transaction( + &mock_bank, + &sanitized_transaction, + ValidatedTransactionDetails::default(), + ); + + assert!(matches!( + load_result, + TransactionLoadResult::NotLoaded(TransactionError::ProgramAccountNotFound), + )); + } + + // note all magic numbers (how many accounts, how many instructions, how big to size buffers) are arbitrary + // other than trying not to swamp programs with blank accounts and keep transaction size below the 64mb limit + #[test] + fn test_load_transaction_accounts_data_sizes() { + let mut rng = rand::rng(); + let mut mock_bank = TestCallbacks::default(); + + // arbitrary accounts + for _ in 0..128 { + let account = AccountSharedData::create_from_existing_shared_data( + 1, + Arc::new(vec![0; rng.random_range(0..128)]), + Pubkey::new_unique(), + rng.random(), + u64::MAX, + ); + mock_bank.accounts_map.insert(Pubkey::new_unique(), (account, 1)); + } + + // fee-payers + let mut fee_payers = vec![]; + for _ in 0..8 { + let fee_payer = Pubkey::new_unique(); + let account = AccountSharedData::create_from_existing_shared_data( + LAMPORTS_PER_SOL, + Arc::new(vec![0; rng.random_range(0..32)]), + system_program::id(), + rng.random(), + u64::MAX, + ); + mock_bank.accounts_map.insert(fee_payer, (account, 1)); + fee_payers.push(fee_payer); + } + + // programs + let mut loader_owned_accounts = vec![]; + let mut programdata_tracker = AHashMap::new(); + for loader in PROGRAM_OWNERS { + for _ in 0..16 { + let program_id = Pubkey::new_unique(); + let mut account = AccountSharedData::create_from_existing_shared_data( + 1, + Arc::new(vec![0; rng.random_range(0..512)]), + *loader, + rng.random(), + u64::MAX, + ); + + // give half loaderv3 accounts (if they're long enough) a valid programdata + // a quarter a dead pointer and a quarter nothing + // we set executable like a program because after the flag is disabled... + // ...programdata and buffer accounts can be used as program ids without aborting loading + // this will always fail at execution but we are merely testing the data size accounting here + if *loader == bpf_loader_upgradeable::id() && account.data().len() >= 64 { + let programdata_address = Pubkey::new_unique(); + let has_programdata = rng.random(); + + if has_programdata { + let programdata_account = + AccountSharedData::create_from_existing_shared_data( + 1, + Arc::new(vec![0; rng.random_range(0..512)]), + *loader, + rng.random(), + u64::MAX, + ); + programdata_tracker.insert( + program_id, + (programdata_address, programdata_account.data().len()), + ); + mock_bank + .accounts_map + .insert(programdata_address, (programdata_account, 1)); + loader_owned_accounts.push(programdata_address); + } + + if has_programdata || rng.random() { + account + .set_state(&UpgradeableLoaderState::Program { programdata_address }) + .unwrap(); + } + } + + mock_bank.accounts_map.insert(program_id, (account, 1)); + loader_owned_accounts.push(program_id); + } + } + + let mut all_accounts = mock_bank.accounts_map.keys().copied().collect::>(); + + // Append some missing accounts. The current loader materializes them as + // default accounts, so they still contribute the base account size. + for _ in 0..32 { + all_accounts.push(Pubkey::new_unique()); + } + + // now generate arbitrary transactions using this accounts + // we ensure valid fee-payers and that all program ids are loader-owned + // otherwise any account can appear anywhere + // some edge cases we hope to hit (not necessarily all in every run): + // * programs used multiple times as program ids and/or normal accounts are counted once + // * loaderv3 programdata used explicitly zero one or multiple times is counted once + // * loaderv3 programs with missing programdata are allowed through + // * loaderv3 programdata used as program id does nothing weird + // * loaderv3 programdata used as a regular account does nothing weird + // * the programdata conditions hold regardless of ordering + for _ in 0..1024 { + let mut instructions = vec![]; + for _ in 0..rng.random_range(1..8) { + let mut accounts = vec![]; + for _ in 0..rng.random_range(1..16) { + all_accounts.shuffle(&mut rng); + let pubkey = all_accounts[0]; + + accounts.push(AccountMeta { + pubkey, + is_writable: rng.random(), + is_signer: rng.random() && rng.random(), + }); + } + + loader_owned_accounts.shuffle(&mut rng); + let program_id = loader_owned_accounts[0]; + instructions.push(Instruction { + accounts, + program_id, + data: vec![], + }); + } + + fee_payers.shuffle(&mut rng); + let fee_payer = fee_payers[0]; + let transaction = SanitizedTransaction::from_transaction_for_tests( + Transaction::new_with_payer(&instructions, Some(&fee_payer)), + ); + + let mut expected_size = 0; + for pubkey in transaction.account_keys().iter() { + let account_data_len = mock_bank + .accounts_map + .get(pubkey) + .map(|(account, _last_modification_slot)| account.data().len()) + .unwrap_or_default(); + expected_size += TRANSACTION_ACCOUNT_BASE_SIZE + account_data_len; + } + + assert!(expected_size <= MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get() as usize); + + let loaded_transaction_accounts = load_transaction_accounts( + &mock_bank, + &transaction, + MAX_LOADED_ACCOUNTS_DATA_SIZE_BYTES.get(), + ) + .unwrap(); + + assert_eq!( + loaded_transaction_accounts.loaded_accounts_data_size, + expected_size as u32, + ); + } + } +} diff --git a/solana/svm/src/lib.rs b/solana/svm/src/lib.rs new file mode 100644 index 00000000..e54b7a89 --- /dev/null +++ b/solana/svm/src/lib.rs @@ -0,0 +1,16 @@ +#![allow(clippy::arithmetic_side_effects)] +#![allow(clippy::arc_with_non_send_sync)] +#![allow(clippy::disallowed_methods)] +#![doc = include_str!("../README.md")] + +mod access_permissions; +pub mod account_loader; +pub mod message_processor; +pub mod program_loader; +pub mod rent_calculator; +pub mod transaction_account_state_info; +pub mod transaction_balances; +pub mod transaction_execution_result; +pub mod transaction_processing_callback; +pub mod transaction_processing_result; +pub mod transaction_processor; diff --git a/solana/svm/src/message_processor.rs b/solana/svm/src/message_processor.rs new file mode 100644 index 00000000..301963e9 --- /dev/null +++ b/solana/svm/src/message_processor.rs @@ -0,0 +1,614 @@ +use { + solana_program_runtime::invoke_context::InvokeContext, + solana_svm_transaction::svm_message::SVMMessage, solana_transaction_context::IndexOfAccount, + solana_transaction_error::TransactionError, +}; + +/// Process each top-level instruction in a message. +/// +/// The caller provides a transaction context containing the loaded accounts. +/// This function advances instruction state, dispatches either a precompile or +/// program entrypoint, and maps instruction failures to transaction errors. +pub(crate) fn process_message<'ix_data>( + message: &'ix_data impl SVMMessage, + program_indices: &[IndexOfAccount], + invoke_context: &mut InvokeContext<'_, 'ix_data>, + accumulated_consumed_units: &mut u64, +) -> Result<(), TransactionError> { + debug_assert_eq!(program_indices.len(), message.num_instructions()); + for (top_level_instruction_index, ((program_id, instruction), program_account_index)) in + message.program_instructions_iter().zip(program_indices.iter()).enumerate() + { + invoke_context + .prepare_next_top_level_instruction( + message, + &instruction, + *program_account_index, + instruction.data, + ) + .map_err(|err| { + TransactionError::InstructionError(top_level_instruction_index as u8, err) + })?; + + let mut compute_units_consumed = 0; + let result = if invoke_context.is_precompile(program_id) { + invoke_context.process_precompile( + program_id, + instruction.data, + message.instructions_iter().map(|ix| ix.data), + ) + } else { + invoke_context.process_instruction(&mut compute_units_consumed) + }; + + *accumulated_consumed_units = + accumulated_consumed_units.saturating_add(compute_units_consumed); + + result.map_err(|err| { + TransactionError::InstructionError(top_level_instruction_index as u8, err) + })?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use { + super::*, + solana_account::{ + Account, AccountSharedData, DUMMY_INHERITABLE_ACCOUNT_FIELDS, ReadableAccount, + }, + solana_ed25519_program::new_ed25519_instruction_with_signature, + solana_hash::Hash, + solana_instruction::{AccountMeta, Instruction, error::InstructionError}, + solana_keypair::Keypair, + solana_message::{AccountKeys, Message, SanitizedMessage}, + solana_precompile_error::PrecompileError, + solana_program_runtime::{ + declare_process_instruction, + execution_budget::{SVMTransactionExecutionBudget, SVMTransactionExecutionCost}, + invoke_context::EnvironmentConfig, + loaded_programs::{ + ProgramCacheEntry, ProgramCacheForTxBatch, ProgramRuntimeEnvironments, + }, + solana_sbpf::program::BuiltinFunctionDefinition, + sysvar_cache::SysvarCache, + }, + solana_pubkey::Pubkey, + solana_rent::Rent, + solana_sdk_ids::{ed25519_program, native_loader}, + solana_signer::Signer, + solana_svm_callback::InvokeContextCallback, + solana_svm_feature_set::SVMFeatureSet, + solana_transaction_context::transaction::TransactionContext, + std::{collections::HashSet, sync::Arc}, + }; + + struct MockCallback {} + impl InvokeContextCallback for MockCallback {} + + fn create_loadable_account_for_test(name: &str) -> AccountSharedData { + let (lamports, rent_epoch) = DUMMY_INHERITABLE_ACCOUNT_FIELDS; + AccountSharedData::from(Account { + lamports, + owner: native_loader::id(), + data: name.as_bytes().to_vec(), + executable: true, + rent_epoch, + }) + } + + fn new_sanitized_message(message: Message) -> SanitizedMessage { + SanitizedMessage::try_from_legacy_message(message, &HashSet::new()).unwrap() + } + + fn ed25519_instruction_for_test() -> Instruction { + let keypair = Keypair::new(); + let signature = keypair.sign_message(b"hello"); + let pubkey = keypair.pubkey().to_bytes(); + new_ed25519_instruction_with_signature(b"hello", signature.as_array(), &pubkey) + } + + #[test] + fn test_process_message_readonly_handling() { + #[derive(serde::Serialize, serde::Deserialize)] + enum MockSystemInstruction { + Correct, + TransferLamports { lamports: u64 }, + ChangeData { data: u8 }, + } + + declare_process_instruction!(MockBuiltin, 1, |invoke_context| { + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + let instruction_data = instruction_context.get_instruction_data(); + if let Ok(instruction) = bincode::deserialize(instruction_data) { + match instruction { + MockSystemInstruction::Correct => Ok(()), + MockSystemInstruction::TransferLamports { lamports } => { + instruction_context + .try_borrow_instruction_account(0)? + .checked_sub_lamports(lamports)?; + instruction_context + .try_borrow_instruction_account(1)? + .checked_add_lamports(lamports)?; + Ok(()) + } + MockSystemInstruction::ChangeData { data } => { + instruction_context + .try_borrow_instruction_account(1)? + .set_data_from_slice(&[data])?; + Ok(()) + } + } + } else { + Err(InstructionError::InvalidInstructionData) + } + }); + + let writable_pubkey = Pubkey::new_unique(); + let readonly_pubkey = Pubkey::new_unique(); + let mock_system_program_id = Pubkey::new_unique(); + + let accounts = vec![ + ( + writable_pubkey, + AccountSharedData::new(100, 1, &mock_system_program_id), + ), + ( + readonly_pubkey, + AccountSharedData::new(0, 1, &mock_system_program_id), + ), + ( + mock_system_program_id, + create_loadable_account_for_test("mock_system_program"), + ), + ]; + let mut transaction_context = + TransactionContext::new(accounts.clone(), Rent::default(), 1, 3, 1); + let program_indices = vec![2]; + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + mock_system_program_id, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + let account_keys = (0..transaction_context.get_number_of_accounts()) + .map(|index| *transaction_context.get_key_of_account_at_index(index).unwrap()) + .collect::>(); + let account_metas = vec![ + AccountMeta::new(writable_pubkey, true), + AccountMeta::new_readonly(readonly_pubkey, false), + ]; + + let message = new_sanitized_message(Message::new_with_compiled_instructions( + 1, + 0, + 2, + account_keys.clone(), + Hash::default(), + AccountKeys::new(&account_keys, None).compile_instructions(&[ + Instruction::new_with_bincode( + mock_system_program_id, + &MockSystemInstruction::Correct, + account_metas.clone(), + ), + ]), + )); + let sysvar_cache = SysvarCache::default(); + let feature_set = SVMFeatureSet::all_enabled(); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &program_indices, &mut invoke_context, &mut 0); + assert!(result.is_ok()); + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().lamports(), + 100 + ); + assert_eq!( + transaction_context.accounts().try_borrow(1).unwrap().lamports(), + 0 + ); + + let message = new_sanitized_message(Message::new_with_compiled_instructions( + 1, + 0, + 2, + account_keys.clone(), + Hash::default(), + AccountKeys::new(&account_keys, None).compile_instructions(&[ + Instruction::new_with_bincode( + mock_system_program_id, + &MockSystemInstruction::TransferLamports { lamports: 50 }, + account_metas.clone(), + ), + ]), + )); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut transaction_context = + TransactionContext::new(accounts.clone(), Rent::default(), 1, 3, 1); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &program_indices, &mut invoke_context, &mut 0); + assert_eq!( + result, + Err(TransactionError::InstructionError( + 0, + InstructionError::ReadonlyLamportChange + )) + ); + + let message = new_sanitized_message(Message::new_with_compiled_instructions( + 1, + 0, + 2, + account_keys.clone(), + Hash::default(), + AccountKeys::new(&account_keys, None).compile_instructions(&[ + Instruction::new_with_bincode( + mock_system_program_id, + &MockSystemInstruction::ChangeData { data: 50 }, + account_metas, + ), + ]), + )); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut transaction_context = TransactionContext::new(accounts, Rent::default(), 1, 3, 1); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &program_indices, &mut invoke_context, &mut 0); + assert_eq!( + result, + Err(TransactionError::InstructionError( + 0, + InstructionError::ReadonlyDataModified + )) + ); + } + + #[test] + fn test_process_message_duplicate_accounts() { + #[derive(serde::Serialize, serde::Deserialize)] + enum MockSystemInstruction { + BorrowFail, + MultiBorrowMut, + DoWork { lamports: u64, data: u8 }, + } + + declare_process_instruction!(MockBuiltin, 1, |invoke_context| { + let transaction_context = &invoke_context.transaction_context; + let instruction_context = transaction_context.get_current_instruction_context()?; + let instruction_data = instruction_context.get_instruction_data(); + let mut to_account = instruction_context.try_borrow_instruction_account(1)?; + if let Ok(instruction) = bincode::deserialize(instruction_data) { + match instruction { + MockSystemInstruction::BorrowFail => { + let from_account = instruction_context.try_borrow_instruction_account(0)?; + let dup_account = instruction_context.try_borrow_instruction_account(2)?; + if from_account.get_lamports() != dup_account.get_lamports() { + return Err(InstructionError::InvalidArgument); + } + Ok(()) + } + MockSystemInstruction::MultiBorrowMut => { + let lamports_a = + instruction_context.try_borrow_instruction_account(0)?.get_lamports(); + let lamports_b = + instruction_context.try_borrow_instruction_account(2)?.get_lamports(); + if lamports_a != lamports_b { + return Err(InstructionError::InvalidArgument); + } + Ok(()) + } + MockSystemInstruction::DoWork { lamports, data } => { + let mut dup_account = + instruction_context.try_borrow_instruction_account(2)?; + dup_account.checked_sub_lamports(lamports)?; + to_account.checked_add_lamports(lamports)?; + dup_account.set_data_from_slice(&[data])?; + drop(dup_account); + let mut from_account = + instruction_context.try_borrow_instruction_account(0)?; + from_account.checked_sub_lamports(lamports)?; + to_account.checked_add_lamports(lamports)?; + Ok(()) + } + } + } else { + Err(InstructionError::InvalidInstructionData) + } + }); + let mock_program_id = Pubkey::from([2u8; 32]); + let accounts = vec![ + ( + solana_pubkey::new_rand(), + AccountSharedData::new(100, 1, &mock_program_id), + ), + ( + solana_pubkey::new_rand(), + AccountSharedData::new(0, 1, &mock_program_id), + ), + ( + mock_program_id, + create_loadable_account_for_test("mock_system_program"), + ), + ]; + let mut transaction_context = + TransactionContext::new(accounts.clone(), Rent::default(), 1, 3, 1); + let program_indices = vec![2]; + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + mock_program_id, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + let account_metas = vec![ + AccountMeta::new( + *transaction_context.get_key_of_account_at_index(0).unwrap(), + true, + ), + AccountMeta::new( + *transaction_context.get_key_of_account_at_index(1).unwrap(), + false, + ), + AccountMeta::new( + *transaction_context.get_key_of_account_at_index(0).unwrap(), + false, + ), + ]; + + // Try to borrow mut the same account + let message = new_sanitized_message(Message::new( + &[Instruction::new_with_bincode( + mock_program_id, + &MockSystemInstruction::BorrowFail, + account_metas.clone(), + )], + Some(transaction_context.get_key_of_account_at_index(0).unwrap()), + )); + let sysvar_cache = SysvarCache::default(); + let feature_set = SVMFeatureSet::all_enabled(); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &program_indices, &mut invoke_context, &mut 0); + assert_eq!( + result, + Err(TransactionError::InstructionError( + 0, + InstructionError::AccountBorrowFailed + )) + ); + + // Try to borrow mut the same account in a safe way + let message = new_sanitized_message(Message::new( + &[Instruction::new_with_bincode( + mock_program_id, + &MockSystemInstruction::MultiBorrowMut, + account_metas.clone(), + )], + Some(transaction_context.get_key_of_account_at_index(0).unwrap()), + )); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut transaction_context = + TransactionContext::new(accounts.clone(), Rent::default(), 1, 3, 1); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &program_indices, &mut invoke_context, &mut 0); + assert!(result.is_ok()); + + // Do work on the same transaction account but at different instruction accounts + let message = new_sanitized_message(Message::new( + &[Instruction::new_with_bincode( + mock_program_id, + &MockSystemInstruction::DoWork { lamports: 10, data: 42 }, + account_metas, + )], + Some(transaction_context.get_key_of_account_at_index(0).unwrap()), + )); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut transaction_context = TransactionContext::new(accounts, Rent::default(), 1, 3, 1); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &program_indices, &mut invoke_context, &mut 0); + assert!(result.is_ok()); + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().lamports(), + 80 + ); + assert_eq!( + transaction_context.accounts().try_borrow(1).unwrap().lamports(), + 20 + ); + assert_eq!( + transaction_context.accounts().try_borrow(0).unwrap().data(), + &vec![42] + ); + } + + #[test] + fn test_precompile() { + let mock_program_id = Pubkey::new_unique(); + declare_process_instruction!(MockBuiltin, 1, |_invoke_context| { + Err(InstructionError::Custom(0xbabb1e)) + }); + + let payer = Pubkey::new_unique(); + let accounts = vec![ + (payer, AccountSharedData::new(1, 0, &native_loader::id())), + ( + ed25519_program::id(), + create_loadable_account_for_test("ed25519_program"), + ), + ( + mock_program_id, + create_loadable_account_for_test("mock_program"), + ), + ]; + let mut transaction_context = TransactionContext::new(accounts, Rent::default(), 1, 2, 2); + + let account_keys = (0..transaction_context.get_number_of_accounts()) + .map(|index| *transaction_context.get_key_of_account_at_index(index).unwrap()) + .collect::>(); + let instructions = vec![ + ed25519_instruction_for_test(), + Instruction::new_with_bytes(mock_program_id, &[], vec![]), + ]; + let message = new_sanitized_message(Message::new_with_compiled_instructions( + 1, + 0, + account_keys.len() as u8 - 1, + account_keys.clone(), + Hash::default(), + AccountKeys::new(&account_keys, None).compile_instructions(&instructions), + )); + + let sysvar_cache = SysvarCache::default(); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + program_cache_for_tx_batch.replenish( + mock_program_id, + Arc::new(ProgramCacheEntry::new_builtin(( + MockBuiltin::vm, + MockBuiltin::codegen, + ))), + ); + + struct MockCallback {} + impl InvokeContextCallback for MockCallback { + fn is_precompile(&self, program_id: &Pubkey) -> bool { + program_id == &ed25519_program::id() + } + + fn process_precompile( + &self, + program_id: &Pubkey, + _data: &[u8], + _instruction_datas: Vec<&[u8]>, + ) -> std::result::Result<(), PrecompileError> { + if self.is_precompile(program_id) { + Ok(()) + } else { + Err(PrecompileError::InvalidPublicKey) + } + } + } + + let feature_set = SVMFeatureSet::all_enabled(); + let program_runtime_environments = ProgramRuntimeEnvironments::default(); + let environment_config = EnvironmentConfig::new( + Hash::default(), + 0, + &MockCallback {}, + &feature_set, + &program_runtime_environments, + &sysvar_cache, + ); + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + &mut program_cache_for_tx_batch, + environment_config, + None, + SVMTransactionExecutionBudget::default(), + SVMTransactionExecutionCost::default(), + ); + let result = process_message(&message, &[1, 2], &mut invoke_context, &mut 0); + + assert_eq!( + result, + Err(TransactionError::InstructionError( + 1, + InstructionError::Custom(0xbabb1e) + )) + ); + assert_eq!(transaction_context.get_instruction_trace_length(), 2); + } +} diff --git a/solana/svm/src/program_loader.rs b/solana/svm/src/program_loader.rs new file mode 100644 index 00000000..01544799 --- /dev/null +++ b/solana/svm/src/program_loader.rs @@ -0,0 +1,29 @@ +use { + solana_account::{AccountSharedData, ReadableAccount}, + solana_program_runtime::loaded_programs::{ + ProgramCacheEntry, ProgramCacheEntryType, ProgramRuntimeEnvironments, + }, + solana_svm_type_overrides::sync::Arc, +}; + +/// Builds a program-cache entry from a normalized executable account. +/// +/// The caller must supply raw ELF bytes in `program.data()`. Decoding +/// loader-specific account headers or indirection is outside the SVM boundary. +/// +/// Invalid program data is kept as a failed-verification entry so execution +/// can report the normal program failure path. +pub fn load_program( + environments: &ProgramRuntimeEnvironments, + program: &AccountSharedData, +) -> Arc { + let environment = environments.get_env_for_execution().clone(); + ProgramCacheEntry::new(environment.clone(), program.data()) + .map(Arc::new) + .unwrap_or_else(|_| { + ProgramCacheEntry { + program: ProgramCacheEntryType::FailedVerification(environment), + } + .into() + }) +} diff --git a/solana/svm/src/rent_calculator.rs b/solana/svm/src/rent_calculator.rs new file mode 100644 index 00000000..915cb9a6 --- /dev/null +++ b/solana/svm/src/rent_calculator.rs @@ -0,0 +1,110 @@ +//! Solana SVM Rent Calculator. +//! +//! Rent management for SVM. + +use { + solana_account::{AccountMode, AccountSharedData, ReadableAccount}, + solana_clock::Epoch, + solana_pubkey::Pubkey, + solana_rent::Rent, + solana_transaction_context::{IndexOfAccount, transaction::TransactionContext}, + solana_transaction_error::{TransactionError, TransactionResult}, +}; + +/// When rent is collected from an exempt account, rent_epoch is set to this +/// value. The idea is to have a fixed, consistent value for rent_epoch for all accounts that do not collect rent. +/// This enables us to get rid of the field completely. +pub const RENT_EXEMPT_RENT_EPOCH: Epoch = Epoch::MAX; + +/// Rent state of a Solana account. +#[derive(Debug, PartialEq, Eq)] +pub enum RentState { + /// account.lamports == 0 + Uninitialized, + /// 0 < account.lamports < rent-exempt-minimum + RentPaying { + lamports: u64, // account.lamports() + data_size: usize, // account.data().len() + }, + /// account.lamports >= rent-exempt-minimum + RentExempt, +} + +/// Checks a writable account rent-state transition inside a transaction. +pub fn check_rent_state( + pre_rent_state: Option<&RentState>, + post_rent_state: Option<&RentState>, + transaction_context: &TransactionContext, + index: IndexOfAccount, +) -> TransactionResult<()> { + if let Some((pre_rent_state, post_rent_state)) = pre_rent_state.zip(post_rent_state) { + let expect_msg = "account must exist at TransactionContext index if rent-states are Some"; + check_rent_state_with_account( + pre_rent_state, + post_rent_state, + transaction_context.get_key_of_account_at_index(index).expect(expect_msg), + index, + )?; + } + Ok(()) +} + +/// Checks a rent-state transition for a known account address. +/// +/// The incinerator is exempt from this check. +pub fn check_rent_state_with_account( + pre_rent_state: &RentState, + post_rent_state: &RentState, + address: &Pubkey, + account_index: IndexOfAccount, +) -> TransactionResult<()> { + if !solana_sdk_ids::incinerator::check_id(address) + && !transition_allowed(pre_rent_state, post_rent_state) + { + let account_index = account_index as u8; + Err(TransactionError::InsufficientFundsForRent { account_index }) + } else { + Ok(()) + } +} + +/// Determines the rent state of an account from lamports and data size. +/// +/// A nonzero-lamport ephemeral account is rent-exempt even below the normal +/// minimum. An account with zero lamports remains uninitialized. +pub fn get_account_rent_state(rent: &Rent, acc: &AccountSharedData) -> RentState { + let (lamports, len) = (acc.lamports(), acc.data().len()); + if lamports == 0 { + RentState::Uninitialized + } else if rent.is_exempt(lamports, len) || acc.is(AccountMode::Ephemeral) { + RentState::RentExempt + } else { + RentState::RentPaying { data_size: len, lamports } + } +} + +/// Returns whether a pre/post rent-state transition is valid. +/// +/// Any state may become uninitialized or rent-exempt. A rent-paying account +/// may remain rent-paying only if it keeps the same data size and is not +/// credited. +pub fn transition_allowed(pre_rent_state: &RentState, post_rent_state: &RentState) -> bool { + match post_rent_state { + RentState::Uninitialized | RentState::RentExempt => true, + RentState::RentPaying { + data_size: post_data_size, + lamports: post_lamports, + } => { + match pre_rent_state { + RentState::Uninitialized | RentState::RentExempt => false, + RentState::RentPaying { + data_size: pre_data_size, + lamports: pre_lamports, + } => { + // Cannot remain RentPaying if resized or credited. + post_data_size == pre_data_size && post_lamports <= pre_lamports + } + } + } + } +} diff --git a/solana/svm/src/transaction_account_state_info.rs b/solana/svm/src/transaction_account_state_info.rs new file mode 100644 index 00000000..06e20edb --- /dev/null +++ b/solana/svm/src/transaction_account_state_info.rs @@ -0,0 +1,222 @@ +use { + crate::rent_calculator::{RentState, check_rent_state, get_account_rent_state}, + solana_rent::Rent, + solana_svm_transaction::svm_message::SVMMessage, + solana_transaction_context::{IndexOfAccount, transaction::TransactionContext}, + solana_transaction_error::TransactionResult as Result, +}; + +#[derive(PartialEq, Debug)] +pub(crate) struct TransactionAccountStateInfo { + rent_state: Option, // None: readonly account +} + +impl TransactionAccountStateInfo { + pub(crate) fn new( + transaction_context: &TransactionContext, + message: &impl SVMMessage, + rent: &Rent, + ) -> Vec { + (0..message.account_keys().len()) + .map(|i| { + let rent_state = if message.is_writable(i) { + let state = transaction_context + .accounts() + .try_borrow(i as IndexOfAccount) + .map(|acc| get_account_rent_state(rent, &acc)) + .ok(); + debug_assert!( + state.is_some(), + "message and transaction context out of sync, fatal" + ); + state + } else { + None + }; + Self { rent_state } + }) + .collect() + } + + pub(crate) fn verify_changes( + pre_state_infos: &[Self], + post_state_infos: &[Self], + transaction_context: &TransactionContext, + ) -> Result<()> { + for (i, (pre_state_info, post_state_info)) in + pre_state_infos.iter().zip(post_state_infos).enumerate() + { + check_rent_state( + pre_state_info.rent_state.as_ref(), + post_state_info.rent_state.as_ref(), + transaction_context, + i as IndexOfAccount, + )?; + } + Ok(()) + } +} + +#[cfg(test)] +mod test { + use { + super::*, + solana_account::AccountSharedData, + solana_hash::Hash, + solana_keypair::Keypair, + solana_message::{ + LegacyMessage, Message, MessageHeader, SanitizedMessage, + compiled_instruction::CompiledInstruction, + }, + solana_rent::Rent, + solana_signer::Signer, + solana_transaction_context::transaction::TransactionContext, + solana_transaction_error::TransactionError, + std::collections::HashSet, + }; + + #[test] + fn test_new() { + let rent = Rent::default(); + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + let key4 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = + SanitizedMessage::Legacy(LegacyMessage::new(message, &HashSet::new())); + + let transaction_accounts = vec![ + (key1.pubkey(), AccountSharedData::default()), + (key2.pubkey(), AccountSharedData::default()), + (key3.pubkey(), AccountSharedData::default()), + ]; + + let context = TransactionContext::new(transaction_accounts, rent.clone(), 20, 20, 1); + let result = TransactionAccountStateInfo::new(&context, &sanitized_message, &rent); + assert_eq!( + result, + vec![ + TransactionAccountStateInfo { + rent_state: Some(RentState::Uninitialized) + }, + TransactionAccountStateInfo { rent_state: None }, + TransactionAccountStateInfo { + rent_state: Some(RentState::Uninitialized) + } + ] + ); + } + + #[test] + #[should_panic(expected = "message and transaction context out of sync, fatal")] + fn test_new_panic() { + let rent = Rent::default(); + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let key3 = Keypair::new(); + let key4 = Keypair::new(); + + let message = Message { + account_keys: vec![key2.pubkey(), key1.pubkey(), key4.pubkey(), key3.pubkey()], + header: MessageHeader::default(), + instructions: vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![2], + data: vec![], + }, + ], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = + SanitizedMessage::Legacy(LegacyMessage::new(message, &HashSet::new())); + + let transaction_accounts = vec![ + (key1.pubkey(), AccountSharedData::default()), + (key2.pubkey(), AccountSharedData::default()), + (key3.pubkey(), AccountSharedData::default()), + ]; + + let context = TransactionContext::new(transaction_accounts, rent.clone(), 20, 20, 1); + let _result = TransactionAccountStateInfo::new(&context, &sanitized_message, &rent); + } + + #[test] + fn test_verify_changes() { + let key1 = Keypair::new(); + let key2 = Keypair::new(); + let pre_rent_state = vec![ + TransactionAccountStateInfo { + rent_state: Some(RentState::Uninitialized), + }, + TransactionAccountStateInfo { + rent_state: Some(RentState::Uninitialized), + }, + ]; + let post_rent_state = vec![TransactionAccountStateInfo { + rent_state: Some(RentState::Uninitialized), + }]; + + let transaction_accounts = vec![ + (key1.pubkey(), AccountSharedData::default()), + (key2.pubkey(), AccountSharedData::default()), + ]; + + let context = TransactionContext::new(transaction_accounts, Rent::default(), 20, 20, 1); + + let result = TransactionAccountStateInfo::verify_changes( + &pre_rent_state, + &post_rent_state, + &context, + ); + assert!(result.is_ok()); + + let pre_rent_state = vec![TransactionAccountStateInfo { + rent_state: Some(RentState::Uninitialized), + }]; + let post_rent_state = vec![TransactionAccountStateInfo { + rent_state: Some(RentState::RentPaying { data_size: 2, lamports: 5 }), + }]; + + let transaction_accounts = vec![ + (key1.pubkey(), AccountSharedData::default()), + (key2.pubkey(), AccountSharedData::default()), + ]; + + let context = TransactionContext::new(transaction_accounts, Rent::default(), 20, 20, 1); + let result = TransactionAccountStateInfo::verify_changes( + &pre_rent_state, + &post_rent_state, + &context, + ); + assert_eq!( + result.err(), + Some(TransactionError::InsufficientFundsForRent { account_index: 0 }) + ); + } +} diff --git a/solana/svm/src/transaction_balances.rs b/solana/svm/src/transaction_balances.rs new file mode 100644 index 00000000..28a817c1 --- /dev/null +++ b/solana/svm/src/transaction_balances.rs @@ -0,0 +1,60 @@ +#[cfg(feature = "dev-context-only-utils")] +use qualifier_attr::field_qualifiers; +use solana_account::ReadableAccount; +use solana_transaction_context::transaction_accounts::KeyedAccountSharedData; + +// Use an internal alias so this stays tied to native lamport balances. +type TxNativeBalances = Vec; + +// Implemented for Option to keep call sites branch-free. +pub(crate) trait BalanceCollectionRoutines { + fn collect_pre_balances(&mut self, accounts: &[KeyedAccountSharedData]); + + fn collect_post_balances(&mut self, accounts: &[KeyedAccountSharedData]); +} + +/// Native account balances recorded before and after execution. +#[derive(Debug, Default, Clone)] +#[cfg_attr( + feature = "dev-context-only-utils", + field_qualifiers(native_pre(pub), native_post(pub)) +)] +pub struct BalanceCollector { + native_pre: TxNativeBalances, + native_post: TxNativeBalances, +} + +impl BalanceCollector { + /// Returns recorded pre- and post-execution lamport balances. + pub fn into_vecs(self) -> (TxNativeBalances, TxNativeBalances) { + (self.native_pre, self.native_post) + } + + fn collect_balances(&mut self, accounts: &[KeyedAccountSharedData]) -> TxNativeBalances { + accounts.iter().map(|a| a.1.lamports()).collect() + } +} + +impl BalanceCollectionRoutines for BalanceCollector { + fn collect_pre_balances(&mut self, accounts: &[KeyedAccountSharedData]) { + self.native_pre = self.collect_balances(accounts); + } + + fn collect_post_balances(&mut self, accounts: &[KeyedAccountSharedData]) { + self.native_post = self.collect_balances(accounts); + } +} + +impl BalanceCollectionRoutines for Option { + fn collect_pre_balances(&mut self, accounts: &[KeyedAccountSharedData]) { + if let Some(inner) = self { + inner.collect_pre_balances(accounts) + } + } + + fn collect_post_balances(&mut self, accounts: &[KeyedAccountSharedData]) { + if let Some(inner) = self { + inner.collect_post_balances(accounts) + } + } +} diff --git a/solana/svm/src/transaction_execution_result.rs b/solana/svm/src/transaction_execution_result.rs new file mode 100644 index 00000000..954c1f0d --- /dev/null +++ b/solana/svm/src/transaction_execution_result.rs @@ -0,0 +1,56 @@ +use { + crate::account_loader::LoadedTransaction, + solana_message::inner_instruction::InnerInstructionsList, + solana_transaction_context::transaction::TransactionReturnData, + solana_transaction_error::TransactionResult, std::sync::Arc, +}; + +/// Loaded-account statistics for one transaction. +#[derive(Debug, Default, Clone, PartialEq)] +pub struct TransactionLoadedAccountsStats { + /// Total loaded account data size charged to the transaction. + pub loaded_accounts_data_size: u32, + /// Number of loaded transaction accounts. + pub loaded_accounts_count: usize, +} + +/// A transaction that reached execution, including mutated accounts. +#[derive(Debug, Clone)] +pub struct ExecutedTransaction { + /// Loaded transaction state after execution. + pub loaded_transaction: LoadedTransaction, + /// Execution status and optional recording data. + pub execution_details: TransactionExecutionDetails, +} + +impl ExecutedTransaction { + /// Returns true when execution completed with `Ok(())`. + pub fn was_successful(&self) -> bool { + self.execution_details.was_successful() + } +} + +/// Status and recordings produced by transaction execution. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct TransactionExecutionDetails { + /// Final execution status. + pub status: TransactionResult<()>, + /// Program log messages when log recording is enabled. + pub log_messages: Option>>, + /// CPI inner instructions when CPI recording is enabled. + pub inner_instructions: Option, + /// Non-empty return data when return-data recording is enabled. + pub return_data: Option, + /// Compute units consumed by top-level instruction execution. + pub executed_units: u64, + /// The change in accounts data len for this transaction. + /// NOTE: This value is valid IFF `status` is `Ok`. + pub accounts_data_len_delta: i64, +} + +impl TransactionExecutionDetails { + /// Returns true when `status` is `Ok(())`. + pub fn was_successful(&self) -> bool { + self.status.is_ok() + } +} diff --git a/solana/svm/src/transaction_processing_callback.rs b/solana/svm/src/transaction_processing_callback.rs new file mode 100644 index 00000000..cf53bea1 --- /dev/null +++ b/solana/svm/src/transaction_processing_callback.rs @@ -0,0 +1 @@ +pub use solana_svm_callback::{AccountState, InvokeContextCallback, TransactionProcessingCallback}; diff --git a/solana/svm/src/transaction_processing_result.rs b/solana/svm/src/transaction_processing_result.rs new file mode 100644 index 00000000..6e1316cb --- /dev/null +++ b/solana/svm/src/transaction_processing_result.rs @@ -0,0 +1,46 @@ +use { + crate::transaction_execution_result::ExecutedTransaction, + solana_transaction_error::TransactionResult, +}; + +/// Result of loading and executing a transaction. +/// +/// `Err` means execution did not produce an `ExecutedTransaction`. `Ok` means +/// execution was attempted; inspect the contained execution status for program +/// success or failure. +pub type TransactionProcessingResult = TransactionResult>; + +/// Convenience methods for nested transaction processing results. +pub trait TransactionProcessingResultExtensions { + /// Returns true when the transaction reached execution. + fn was_processed(&self) -> bool; + /// Returns true when the transaction reached execution and succeeded. + fn was_processed_with_successful_result(&self) -> bool; + /// Returns the executed transaction when execution was attempted. + fn processed_transaction(&self) -> Option<&ExecutedTransaction>; + /// Collapses load and execution status into a single transaction result. + fn flattened_result(&self) -> TransactionResult<()>; +} + +impl TransactionProcessingResultExtensions for TransactionProcessingResult { + fn was_processed(&self) -> bool { + self.is_ok() + } + + fn was_processed_with_successful_result(&self) -> bool { + match self { + Ok(processed_tx) => processed_tx.was_successful(), + Err(_) => false, + } + } + + fn processed_transaction(&self) -> Option<&ExecutedTransaction> { + self.as_deref().ok() + } + + fn flattened_result(&self) -> TransactionResult<()> { + self.as_ref() + .map_err(|err| err.clone()) + .and_then(|processed_tx| processed_tx.execution_details.status.clone()) + } +} diff --git a/solana/svm/src/transaction_processor.rs b/solana/svm/src/transaction_processor.rs new file mode 100644 index 00000000..997233a5 --- /dev/null +++ b/solana/svm/src/transaction_processor.rs @@ -0,0 +1,915 @@ +#[cfg(test)] +use solana_svm_type_overrides::sync::RwLock; +use solana_transaction_context::transaction_accounts::KeyedAccountSharedData; + +#[cfg(feature = "dev-context-only-utils")] +use qualifier_attr::{field_qualifiers, qualifiers}; +use { + crate::{ + account_loader::{ + CheckedTransactionDetails, LoadedTransaction, TransactionLoadResult, + ValidatedTransactionDetails, load_transaction, + }, + message_processor::process_message, + program_loader::load_program, + transaction_account_state_info::TransactionAccountStateInfo, + transaction_balances::{BalanceCollectionRoutines, BalanceCollector}, + transaction_execution_result::{ExecutedTransaction, TransactionExecutionDetails}, + transaction_processing_result::TransactionProcessingResult, + }, + solana_account::{AccountSharedData, PROGRAM_OWNERS, ReadableAccount}, + solana_clock::Slot, + solana_hash::Hash, + solana_instruction::TRANSACTION_LEVEL_STACK_HEIGHT, + solana_message::{ + compiled_instruction::CompiledInstruction, + inner_instruction::{InnerInstruction, InnerInstructionsList}, + }, + solana_program_runtime::{ + execution_budget::SVMTransactionExecutionCost, + invoke_context::{EnvironmentConfig, InvokeContext}, + loaded_programs::{ProgramCache, ProgramCacheForTxBatch, ProgramRuntimeEnvironments}, + sysvar_cache::SysvarCache, + }, + solana_pubkey::Pubkey, + solana_rent::Rent, + solana_svm_callback::TransactionProcessingCallback, + solana_svm_feature_set::SVMFeatureSet, + solana_svm_log_collector::LogCollector, + solana_svm_transaction::svm_transaction::SVMTransaction, + solana_svm_type_overrides::sync::Arc, + solana_transaction_context::transaction::{ExecutionRecord, TransactionContext}, + solana_transaction_error::TransactionError, + std::{ + fmt::{Debug, Formatter}, + rc::Rc, + }, +}; + +/// Log messages emitted during a transaction. +pub type TransactionLogMessages = Vec; + +/// Result of loading and executing one sanitized transaction. +pub struct LoadAndExecuteSanitizedTransactionOutput { + /// Load or execution result. + /// + /// `Ok` means execution was attempted. The contained transaction can still + /// have a failed execution status. + pub processing_result: TransactionProcessingResult, + /// Native pre/post balances when balance recording is enabled. + pub balance_collector: Option, +} + +/// Controls which execution artifacts are retained in the result. +#[derive(Copy, Clone, Default)] +pub struct ExecutionRecordingConfig { + /// Record inner instructions produced by CPI. + pub enable_cpi_recording: bool, + /// Record program log messages. + pub enable_log_recording: bool, + /// Record non-empty return data. + pub enable_return_data_recording: bool, + /// Record native account balances before and after execution. + pub enable_transaction_balance_recording: bool, +} + +impl ExecutionRecordingConfig { + /// Creates a recording config with every flag set to the same value. + pub fn new_single_setting(option: bool) -> Self { + ExecutionRecordingConfig { + enable_return_data_recording: option, + enable_log_recording: option, + enable_cpi_recording: option, + enable_transaction_balance_recording: option, + } + } +} + +/// Transaction execution options. +#[derive(Default)] +pub struct TransactionProcessingConfig { + /// The maximum number of bytes that log messages can consume. + pub log_messages_bytes_limit: Option, + /// Recording capabilities for transaction execution. + pub recording_config: ExecutionRecordingConfig, +} + +/// Runtime inputs that are shared across a transaction execution. +#[derive(Default)] +pub struct TransactionProcessingEnvironment { + /// Blockhash exposed to programs through the invocation environment. + pub blockhash: Hash, + /// Lamports per signature associated with `blockhash`. + pub blockhash_lamports_per_signature: u64, + /// Retained for API compatibility; the current execution path does not use + /// stake weighting. + pub epoch_total_stake: u64, + /// Runtime feature set used during execution. + pub feature_set: SVMFeatureSet, + /// Runtime environments used for executing already deployed programs. + pub program_runtime_environments_for_execution: ProgramRuntimeEnvironments, + /// Rent calculator used for transaction-context construction and rent-state checks. + pub rent: Rent, +} + +#[cfg_attr( + feature = "dev-context-only-utils", + field_qualifiers(slot(pub), sysvar_cache(pub)) +)] +pub struct TransactionBatchProcessor { + /// Slot associated with this processor. + pub slot: Slot, + + /// Sysvars exposed to programs during execution. + sysvar_cache: SysvarCache, + + /// Shared cache of loaded programs. + pub program_cache: Arc, + + execution_cost: SVMTransactionExecutionCost, +} + +impl Debug for TransactionBatchProcessor { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TransactionBatchProcessor") + .field("slot", &self.slot) + .field("sysvar_cache", &self.sysvar_cache) + .field("program_cache", &self.program_cache) + .finish() + } +} + +impl Default for TransactionBatchProcessor { + fn default() -> Self { + Self { + slot: Slot::default(), + sysvar_cache: Default::default(), + program_cache: Arc::new(ProgramCache::default()), + execution_cost: SVMTransactionExecutionCost::default(), + } + } +} + +impl TransactionBatchProcessor { + /// Create a `TransactionBatchProcessor` using the supplied program cache. + /// + /// The processor preserves the cache contents and does not add builtins. + pub fn new_uninitialized(slot: Slot, cache: Arc) -> Self { + Self { + slot, + program_cache: cache, + ..Self::default() + } + } + + /// Create a new `TransactionBatchProcessor`. + /// + /// Runtime environments are supplied per execution through + /// [`TransactionProcessingEnvironment::program_runtime_environments_for_execution`]. + pub fn new(slot: Slot, cache: Arc) -> Self { + Self::new_uninitialized(slot, cache) + } + + /// Sets the base execution cost charged by this processor. + pub fn set_execution_cost(&mut self, cost: SVMTransactionExecutionCost) { + self.execution_cost = cost; + } + + /// Returns mutable access to cached sysvars for the current processor slot. + pub fn sysvar_cache_mut(&mut self) -> &mut SysvarCache { + &mut self.sysvar_cache + } + + /// Loads accounts, prepares programs, and executes one sanitized transaction. + pub fn load_and_execute_sanitized_transaction( + &self, + callbacks: &CB, + tx: &impl SVMTransaction, + details: CheckedTransactionDetails, + environment: &TransactionProcessingEnvironment, + config: &TransactionProcessingConfig, + ) -> LoadAndExecuteSanitizedTransactionOutput { + // Create the transaction balance collector if recording is enabled. + let mut balance_collector = config + .recording_config + .enable_transaction_balance_recording + .then(BalanceCollector::default); + + // Create the batch-local program cache. + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::new(self.slot); + + let details = ValidatedTransactionDetails { + compute_budget: details.compute_budget_and_limits.budget, + loaded_accounts_bytes_limit: details + .compute_budget_and_limits + .loaded_accounts_data_size_limit, + fee_details: details.compute_budget_and_limits.fee_details, + }; + let load_result = load_transaction(callbacks, tx, details); + + let processing_result = match load_result { + TransactionLoadResult::NotLoaded(err) => Err(err), + TransactionLoadResult::Loaded(loaded_transaction) => { + balance_collector.collect_pre_balances(&loaded_transaction.accounts); + self.replenish_program_cache( + &environment.program_runtime_environments_for_execution, + &mut program_cache_for_tx_batch, + &loaded_transaction.accounts, + ); + + let mut executed_tx = self.execute_loaded_transaction( + callbacks, + tx, + loaded_transaction, + &mut program_cache_for_tx_batch, + environment, + config, + ); + balance_collector.collect_post_balances(&executed_tx.loaded_transaction.accounts); + if executed_tx.access_is_valid(tx) { + let cache = program_cache_for_tx_batch.drain_modified_entries(); + self.program_cache.merge(&cache); + } + Ok(Box::new(executed_tx)) + } + }; + + LoadAndExecuteSanitizedTransactionOutput { + processing_result, + balance_collector, + } + } + + #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))] + fn replenish_program_cache( + &self, + environments: &ProgramRuntimeEnvironments, + program_cache_for_tx_batch: &mut ProgramCacheForTxBatch, + accounts: &[KeyedAccountSharedData], + ) { + for (pubkey, acc) in accounts { + if !(acc.executable() && PROGRAM_OWNERS.iter().any(|o| o == acc.owner())) { + continue; + } + let entry = if let Some(entry) = self.program_cache.get(pubkey) { + entry + } else { + let entry = load_program(environments, acc); + self.program_cache.assign_program(*pubkey, entry.clone()); + entry + }; + program_cache_for_tx_batch.replenish(*pubkey, entry); + } + } + + /// Executes a transaction using already loaded accounts. + #[allow(clippy::too_many_arguments)] + fn execute_loaded_transaction( + &self, + callback: &CB, + tx: &impl SVMTransaction, + mut loaded_transaction: LoadedTransaction, + program_cache_for_tx_batch: &mut ProgramCacheForTxBatch, + environment: &TransactionProcessingEnvironment, + config: &TransactionProcessingConfig, + ) -> ExecutedTransaction { + let transaction_accounts = std::mem::take(&mut loaded_transaction.accounts); + + // Ensure the length of accounts matches the expected length from tx.account_keys(). + // This is a sanity check in case that someone starts adding some additional accounts + // since this has been done before. See discussion in PR #4497 for details + debug_assert!(transaction_accounts.len() == tx.account_keys().len()); + + fn transaction_accounts_lamports_sum( + accounts: &[(Pubkey, AccountSharedData)], + ) -> Option { + accounts.iter().try_fold(0u128, |sum, (_, account)| { + sum.checked_add(u128::from(account.lamports())) + }) + } + + let lamports_before_tx = + transaction_accounts_lamports_sum(&transaction_accounts).unwrap_or(0); + + let compute_budget = loaded_transaction.compute_budget; + + let mut transaction_context = TransactionContext::new( + transaction_accounts, + environment.rent.clone(), + compute_budget.max_instruction_stack_depth, + compute_budget.max_instruction_trace_length, + tx.num_instructions(), + ); + + let pre_account_state_info = + TransactionAccountStateInfo::new(&transaction_context, tx, &environment.rent); + + let log_collector = if config.recording_config.enable_log_recording { + match config.log_messages_bytes_limit { + None => Some(LogCollector::new_ref()), + Some(log_messages_bytes_limit) => Some(LogCollector::new_ref_with_limit(Some( + log_messages_bytes_limit, + ))), + } + } else { + None + }; + + let mut executed_units = 0u64; + + let mut invoke_context = InvokeContext::new( + &mut transaction_context, + program_cache_for_tx_batch, + EnvironmentConfig::new( + environment.blockhash, + environment.blockhash_lamports_per_signature, + callback, + &environment.feature_set, + &environment.program_runtime_environments_for_execution, + &self.sysvar_cache, + ), + log_collector.clone(), + compute_budget, + self.execution_cost, + ); + + let process_result = process_message( + tx, + &loaded_transaction.program_indices, + &mut invoke_context, + &mut executed_units, + ); + + drop(invoke_context); + + let mut status = process_result.and_then(|info| { + let post_account_state_info = + TransactionAccountStateInfo::new(&transaction_context, tx, &environment.rent); + TransactionAccountStateInfo::verify_changes( + &pre_account_state_info, + &post_account_state_info, + &transaction_context, + ) + .map(|_| info) + }); + + let log_messages: Option = + log_collector.and_then(|log_collector| { + Rc::try_unwrap(log_collector) + .map(|log_collector| log_collector.into_inner().into_messages()) + .ok() + }); + + let (execution_record, inner_instructions) = Self::deconstruct_transaction( + transaction_context, + config.recording_config.enable_cpi_recording, + ); + + let ExecutionRecord { + accounts, + return_data, + accounts_resize_delta: accounts_data_len_delta, + .. + } = execution_record; + + if status.is_ok() + && transaction_accounts_lamports_sum(&accounts) + .filter(|lamports_after_tx| lamports_before_tx == *lamports_after_tx) + .is_none() + { + status = Err(TransactionError::UnbalancedTransaction); + } + let status = status.map(|_| ()); + + loaded_transaction.accounts = accounts; + + let return_data = if config.recording_config.enable_return_data_recording + && !return_data.data.is_empty() + { + Some(return_data) + } else { + None + }; + + ExecutedTransaction { + execution_details: TransactionExecutionDetails { + status, + log_messages: log_messages.map(Arc::new), + inner_instructions, + return_data, + executed_units, + accounts_data_len_delta, + }, + loaded_transaction, + } + } + + /// Extract an ExecutionRecord and an InnerInstructionsList from a TransactionContext + fn deconstruct_transaction( + mut transaction_context: TransactionContext, + record_inner_instructions: bool, + ) -> (ExecutionRecord, Option) { + let inner_ix = if record_inner_instructions { + debug_assert!( + transaction_context + .get_instruction_context_at_index_in_trace(0) + .map(|instruction_context| instruction_context.get_stack_height() + == TRANSACTION_LEVEL_STACK_HEIGHT) + .unwrap_or(true) + ); + + let (ix_trace, accounts, ix_data_trace) = transaction_context.take_instruction_trace(); + let mut outer_instructions = Vec::new(); + for ((ix_in_trace, ix_data), ix_accounts) in + ix_trace.into_iter().zip(ix_data_trace).zip(accounts) + { + let stack_height = ix_in_trace.nesting_level.saturating_add(1) as usize; + if stack_height == TRANSACTION_LEVEL_STACK_HEIGHT { + outer_instructions.push(Vec::new()); + } else if let Some(inner_instructions) = outer_instructions.last_mut() { + let stack_height = u8::try_from(stack_height).unwrap_or(u8::MAX); + inner_instructions.push(InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts( + ix_in_trace.program_account_index_in_tx as u8, + ix_data.into_owned(), + ix_accounts.iter().map(|acc| acc.index_in_transaction as u8).collect(), + ), + stack_height, + }); + } else { + debug_assert!(false); + } + } + + Some(outer_instructions) + } else { + None + }; + + let record: ExecutionRecord = transaction_context.into(); + + (record, inner_ix) + } + + pub fn fill_missing_sysvar_cache_entries( + &mut self, + callbacks: &CB, + ) { + self.sysvar_cache.fill_missing_entries(|pubkey, set_sysvar| { + if let Some((account, _slot)) = callbacks.get_account_shared_data(pubkey) { + set_sysvar(account.data()); + } + }); + } + + pub fn reset_sysvar_cache(&mut self) { + self.sysvar_cache.reset(); + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + #[allow(deprecated)] + use solana_sysvar::fees::Fees; + use { + super::*, + solana_account::{WritableAccount, create_account_shared_data_for_test}, + solana_clock::Clock, + solana_epoch_schedule::EpochSchedule, + solana_fee_calculator::FeeCalculator, + solana_fee_structure::FeeDetails, + solana_hash::Hash, + solana_message::{LegacyMessage, Message, MessageHeader, SanitizedMessage}, + solana_program_runtime::{ + execution_budget::SVMTransactionExecutionBudget, loaded_programs::ProgramCacheEntryType, + }, + solana_rent::Rent, + solana_sdk_ids::{bpf_loader, sysvar}, + solana_signature::Signature, + solana_svm_callback::{AccountState, InvokeContextCallback}, + solana_transaction::sanitized::SanitizedTransaction, + solana_transaction_context::transaction::TransactionContext, + std::collections::HashMap, + }; + + fn new_unchecked_sanitized_message(message: Message) -> SanitizedMessage { + SanitizedMessage::Legacy(LegacyMessage::new(message, &HashSet::new())) + } + + #[derive(Clone, Default)] + struct MockBankCallback { + account_shared_data: Arc>>, + #[allow(clippy::type_complexity)] + inspected_accounts: + Arc, /* is_writable */ bool)>>>>, + } + + impl InvokeContextCallback for MockBankCallback {} + + impl TransactionProcessingCallback for MockBankCallback { + fn get_account_shared_data(&self, pubkey: &Pubkey) -> Option<(AccountSharedData, Slot)> { + self.account_shared_data + .read() + .unwrap() + .get(pubkey) + .map(|account| (account.clone(), 0)) + } + + fn inspect_account( + &self, + address: &Pubkey, + account_state: AccountState, + is_writable: bool, + ) { + let account = match account_state { + AccountState::Dead => None, + AccountState::Alive(account) => Some(account.clone()), + }; + self.inspected_accounts + .write() + .unwrap() + .entry(*address) + .or_default() + .push((account, is_writable)); + } + } + + #[test] + fn test_inner_instructions_list_from_instruction_trace() { + let mut transaction_context = TransactionContext::new( + vec![( + Pubkey::new_unique(), + AccountSharedData::new(1, 1, &bpf_loader::ID), + )], + Rent::default(), + 4, + 11, + 4, + ); + + // To be uncommented when we reorder the instruction trace + // Four top level instructions + // for i in 0..4 { + // transaction_context + // .configure_instruction_at_index( + // i, + // 0, + // vec![], + // vec![u16::MAX; 256], + // Cow::Owned(vec![i as u8]), + // None, + // ) + // .unwrap(); + // } + + // Execute ix #0 + transaction_context + .configure_top_level_instruction_for_tests(0, vec![], vec![0]) + .unwrap(); + transaction_context.push().unwrap(); + // ix #0 does a CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![0, 0]).unwrap(); + transaction_context.push().unwrap(); + // Returning from everything + transaction_context.pop().unwrap(); + transaction_context.pop().unwrap(); + // Execute ix #1 + transaction_context + .configure_top_level_instruction_for_tests(0, vec![], vec![1]) + .unwrap(); + transaction_context.push().unwrap(); + transaction_context.pop().unwrap(); + // Execute ix #2 + transaction_context + .configure_top_level_instruction_for_tests(0, vec![], vec![2]) + .unwrap(); + transaction_context.push().unwrap(); + // ix #2 does a CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![2, 0]).unwrap(); + transaction_context.push().unwrap(); + // A nested CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![2, 1]).unwrap(); + transaction_context.push().unwrap(); + // Return from nested CPI + transaction_context.pop().unwrap(); + // Return from CPI + transaction_context.pop().unwrap(); + // ix #2 does another CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![2, 2]).unwrap(); + transaction_context.push().unwrap(); + // Return from everything related to ix #2 + transaction_context.pop().unwrap(); + transaction_context.pop().unwrap(); + // Execute ix #3 + transaction_context + .configure_top_level_instruction_for_tests(0, vec![], vec![3]) + .unwrap(); + transaction_context.push().unwrap(); + // ix #3 does a CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![3, 0]).unwrap(); + transaction_context.push().unwrap(); + // ix #3 does a nested CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![3, 1]).unwrap(); + transaction_context.push().unwrap(); + // ix #3 does a second nested CPI + transaction_context.configure_next_cpi_for_tests(0, vec![], vec![3, 2]).unwrap(); + transaction_context.push().unwrap(); + // Return from everything related to ix #3 + transaction_context.pop().unwrap(); + transaction_context.pop().unwrap(); + transaction_context.pop().unwrap(); + transaction_context.pop().unwrap(); + + let inner_instructions = + TransactionBatchProcessor::deconstruct_transaction(transaction_context, true) + .1 + .unwrap(); + + assert_eq!( + inner_instructions, + vec![ + vec![InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![0, 0], vec![]), + stack_height: 2, + }], + vec![], + vec![ + InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![2, 0], vec![]), + stack_height: 2, + }, + InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![2, 1], vec![]), + stack_height: 3, + }, + InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![2, 2], vec![]), + stack_height: 2, + }, + ], + vec![ + InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![3, 0], vec![]), + stack_height: 2, + }, + InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![3, 1], vec![]), + stack_height: 3, + }, + InnerInstruction { + instruction: CompiledInstruction::new_from_raw_parts(0, vec![3, 2], vec![]), + stack_height: 4, + }, + ] + ] + ); + } + + #[test] + fn test_execute_loaded_transaction_recordings() { + // Setting all the arguments correctly is too burdensome for testing + // execute_loaded_transaction separately.This function will be tested in an integration + // test with load_and_execute_sanitized_transactions + let message = Message { + account_keys: vec![Pubkey::new_from_array([0; 32])], + header: MessageHeader::default(), + instructions: vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }], + recent_blockhash: Hash::default(), + }; + + let sanitized_message = new_unchecked_sanitized_message(message); + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::default(); + let batch_processor = TransactionBatchProcessor::default(); + + let sanitized_transaction = SanitizedTransaction::new_for_tests( + sanitized_message, + vec![Signature::new_unique()], + false, + ); + + let loaded_transaction = LoadedTransaction { + accounts: vec![(Pubkey::new_unique(), AccountSharedData::default())], + program_indices: vec![0], + fee_details: FeeDetails::default(), + compute_budget: SVMTransactionExecutionBudget::default(), + loaded_accounts_data_size: 32, + }; + + let processing_environment = TransactionProcessingEnvironment::default(); + + let mut processing_config = TransactionProcessingConfig::default(); + processing_config.recording_config.enable_log_recording = true; + + let mock_bank = MockBankCallback::default(); + + let executed_tx = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + loaded_transaction.clone(), + &mut program_cache_for_tx_batch, + &processing_environment, + &processing_config, + ); + assert!(executed_tx.execution_details.log_messages.is_some()); + + processing_config.log_messages_bytes_limit = Some(2); + + let executed_tx = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + loaded_transaction.clone(), + &mut program_cache_for_tx_batch, + &processing_environment, + &processing_config, + ); + assert!(executed_tx.execution_details.log_messages.is_some()); + assert!(executed_tx.execution_details.inner_instructions.is_none()); + + processing_config.recording_config.enable_log_recording = false; + processing_config.recording_config.enable_cpi_recording = true; + processing_config.log_messages_bytes_limit = None; + + let executed_tx = batch_processor.execute_loaded_transaction( + &mock_bank, + &sanitized_transaction, + loaded_transaction, + &mut program_cache_for_tx_batch, + &processing_environment, + &processing_config, + ); + + assert!(executed_tx.execution_details.log_messages.is_none()); + assert!(executed_tx.execution_details.inner_instructions.is_some()); + } + + #[test] + fn test_replenish_program_cache() { + let batch_processor = TransactionBatchProcessor::default(); + let key = Pubkey::new_unique(); + + let mut account_data = AccountSharedData::default(); + account_data.set_owner(bpf_loader::id()); + account_data.set_executable(true); + let accounts = vec![(key, account_data)]; + + let mut program_cache_for_tx_batch = ProgramCacheForTxBatch::new(batch_processor.slot); + + batch_processor.replenish_program_cache( + &Default::default(), + &mut program_cache_for_tx_batch, + &accounts, + ); + + let program = program_cache_for_tx_batch.find(&key).unwrap(); + assert!(matches!( + program.program, + ProgramCacheEntryType::FailedVerification(_) + )); + assert!(batch_processor.program_cache.get(&key).is_some()); + } + + #[test] + #[allow(deprecated)] + fn test_sysvar_cache_initialization1() { + let mock_bank = MockBankCallback::default(); + + let clock = Clock { + slot: 1, + epoch_start_timestamp: 2, + epoch: 3, + leader_schedule_epoch: 4, + unix_timestamp: 5, + }; + let clock_account = create_account_shared_data_for_test(&clock); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::clock::id(), clock_account); + + let epoch_schedule = EpochSchedule::custom(64, 2, true); + let epoch_schedule_account = create_account_shared_data_for_test(&epoch_schedule); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::epoch_schedule::id(), epoch_schedule_account); + + let fees = Fees { + fee_calculator: FeeCalculator { lamports_per_signature: 123 }, + }; + let fees_account = create_account_shared_data_for_test(&fees); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::fees::id(), fees_account); + + let rent = Rent::default(); + let rent_account = create_account_shared_data_for_test(&rent); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::rent::id(), rent_account); + + let mut transaction_processor = TransactionBatchProcessor::default(); + transaction_processor.fill_missing_sysvar_cache_entries(&mock_bank); + + let sysvar_cache = &transaction_processor.sysvar_cache; + let cached_clock = sysvar_cache.get_clock(); + let cached_rent = sysvar_cache.get_rent(); + + assert_eq!( + cached_clock.expect("clock sysvar missing in cache"), + clock.into() + ); + assert_eq!( + cached_rent.expect("rent sysvar missing in cache"), + rent.into() + ); + assert!(sysvar_cache.get_slot_hashes().is_err()); + } + + #[test] + #[allow(deprecated)] + fn test_reset_and_fill_sysvar_cache() { + let mock_bank = MockBankCallback::default(); + + let clock = Clock { + slot: 1, + epoch_start_timestamp: 2, + epoch: 3, + leader_schedule_epoch: 4, + unix_timestamp: 5, + }; + let clock_account = create_account_shared_data_for_test(&clock); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::clock::id(), clock_account); + + let epoch_schedule = EpochSchedule::custom(64, 2, true); + let epoch_schedule_account = create_account_shared_data_for_test(&epoch_schedule); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::epoch_schedule::id(), epoch_schedule_account); + + let fees = Fees { + fee_calculator: FeeCalculator { lamports_per_signature: 123 }, + }; + let fees_account = create_account_shared_data_for_test(&fees); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::fees::id(), fees_account); + + let rent = Rent::default(); + let rent_account = create_account_shared_data_for_test(&rent); + mock_bank + .account_shared_data + .write() + .unwrap() + .insert(sysvar::rent::id(), rent_account); + + let mut transaction_processor = TransactionBatchProcessor::default(); + // Fill the sysvar cache + transaction_processor.fill_missing_sysvar_cache_entries(&mock_bank); + // Reset the sysvar cache + transaction_processor.reset_sysvar_cache(); + + { + let sysvar_cache = &transaction_processor.sysvar_cache; + // Test that sysvar cache is empty and none of the values are found + assert!(sysvar_cache.get_clock().is_err()); + assert!(sysvar_cache.get_rent().is_err()); + } + + // Refill the cache and test the values are available. + transaction_processor.fill_missing_sysvar_cache_entries(&mock_bank); + + let sysvar_cache = &transaction_processor.sysvar_cache; + let cached_clock = sysvar_cache.get_clock(); + let cached_rent = sysvar_cache.get_rent(); + + assert_eq!( + cached_clock.expect("clock sysvar missing in cache"), + clock.into() + ); + assert_eq!( + cached_rent.expect("rent sysvar missing in cache"), + rent.into() + ); + assert!(sysvar_cache.get_slot_hashes().is_err()); + } +} diff --git a/solana/svm/tests/example-programs/clock-sysvar/Cargo.toml b/solana/svm/tests/example-programs/clock-sysvar/Cargo.toml new file mode 100644 index 00000000..3dacc69b --- /dev/null +++ b/solana/svm/tests/example-programs/clock-sysvar/Cargo.toml @@ -0,0 +1,11 @@ +[package] +edition = "2021" +name = "clock-sysvar-program" +version = "4.0.0-rc.1" + +[dependencies] + +[lib] +crate-type = ["cdylib", "rlib"] + +[workspace] diff --git a/solana/svm/tests/example-programs/clock-sysvar/clock_sysvar_program.so b/solana/svm/tests/example-programs/clock-sysvar/clock_sysvar_program.so new file mode 100755 index 00000000..dee43a1e Binary files /dev/null and b/solana/svm/tests/example-programs/clock-sysvar/clock_sysvar_program.so differ diff --git a/solana/svm/tests/example-programs/clock-sysvar/src/lib.rs b/solana/svm/tests/example-programs/clock-sysvar/src/lib.rs new file mode 100644 index 00000000..b35d142b --- /dev/null +++ b/solana/svm/tests/example-programs/clock-sysvar/src/lib.rs @@ -0,0 +1,21 @@ +use { + solana_account_info::AccountInfo, + solana_program::program::set_return_data, + solana_program_entrypoint::entrypoint, + solana_program_error::ProgramResult, + solana_pubkey::Pubkey, + solana_sysvar::{clock::Clock, Sysvar}, +}; + +entrypoint!(process_instruction); + +fn process_instruction( + _program_id: &Pubkey, + _accounts: &[AccountInfo], + _instruction_data: &[u8], +) -> ProgramResult { + let time_now = Clock::get().unwrap().unix_timestamp; + let return_data = time_now.to_be_bytes(); + set_return_data(&return_data); + Ok(()) +} diff --git a/solana/svm/tests/example-programs/hello-solana/Cargo.toml b/solana/svm/tests/example-programs/hello-solana/Cargo.toml new file mode 100644 index 00000000..4f08a1b8 --- /dev/null +++ b/solana/svm/tests/example-programs/hello-solana/Cargo.toml @@ -0,0 +1,11 @@ +[package] +edition = "2021" +name = "hello-solana-program" +version = "4.0.0-rc.1" + +[dependencies] + +[lib] +crate-type = ["cdylib", "rlib"] + +[workspace] diff --git a/solana/svm/tests/example-programs/hello-solana/hello_solana_program.so b/solana/svm/tests/example-programs/hello-solana/hello_solana_program.so new file mode 100755 index 00000000..a9da4ff4 Binary files /dev/null and b/solana/svm/tests/example-programs/hello-solana/hello_solana_program.so differ diff --git a/solana/svm/tests/example-programs/hello-solana/src/lib.rs b/solana/svm/tests/example-programs/hello-solana/src/lib.rs new file mode 100644 index 00000000..3f6799c2 --- /dev/null +++ b/solana/svm/tests/example-programs/hello-solana/src/lib.rs @@ -0,0 +1,16 @@ +use { + solana_account_info::AccountInfo, solana_msg::msg, solana_program_entrypoint::entrypoint, + solana_program_error::ProgramResult, solana_pubkey::Pubkey, +}; + +entrypoint!(process_instruction); + +fn process_instruction( + _program_id: &Pubkey, + _accounts: &[AccountInfo], + _instruction_data: &[u8], +) -> ProgramResult { + msg!("Hello, Solana!"); + + Ok(()) +} diff --git a/solana/svm/tests/example-programs/simple-transfer/Cargo.toml b/solana/svm/tests/example-programs/simple-transfer/Cargo.toml new file mode 100644 index 00000000..8d93d9df --- /dev/null +++ b/solana/svm/tests/example-programs/simple-transfer/Cargo.toml @@ -0,0 +1,11 @@ +[package] +edition = "2021" +name = "simple-transfer-program" +version = "4.0.0-rc.1" + +[dependencies] + +[lib] +crate-type = ["cdylib", "rlib"] + +[workspace] diff --git a/solana/svm/tests/example-programs/simple-transfer/simple_transfer_program.so b/solana/svm/tests/example-programs/simple-transfer/simple_transfer_program.so new file mode 100755 index 00000000..5132b38c Binary files /dev/null and b/solana/svm/tests/example-programs/simple-transfer/simple_transfer_program.so differ diff --git a/solana/svm/tests/example-programs/simple-transfer/src/lib.rs b/solana/svm/tests/example-programs/simple-transfer/src/lib.rs new file mode 100644 index 00000000..1e922c2f --- /dev/null +++ b/solana/svm/tests/example-programs/simple-transfer/src/lib.rs @@ -0,0 +1,29 @@ +use { + solana_account_info::{next_account_info, AccountInfo}, + solana_program::program::invoke, + solana_program_entrypoint::entrypoint, + solana_program_error::ProgramResult, + solana_pubkey::Pubkey, + solana_system_interface::instruction as system_instruction, +}; + +entrypoint!(process_instruction); + +fn process_instruction( + _program_id: &Pubkey, + accounts: &[AccountInfo], + data: &[u8], +) -> ProgramResult { + let amount = u64::from_be_bytes(data[0..8].try_into().unwrap()); + let accounts_iter = &mut accounts.iter(); + let payer = next_account_info(accounts_iter)?; + let recipient = next_account_info(accounts_iter)?; + let system_program = next_account_info(accounts_iter)?; + + invoke( + &system_instruction::transfer(payer.key, recipient.key, amount), + &[payer.clone(), recipient.clone(), system_program.clone()], + )?; + + Ok(()) +} diff --git a/solana/svm/tests/example-programs/transfer-from-account/Cargo.toml b/solana/svm/tests/example-programs/transfer-from-account/Cargo.toml new file mode 100644 index 00000000..c431ebe9 --- /dev/null +++ b/solana/svm/tests/example-programs/transfer-from-account/Cargo.toml @@ -0,0 +1,11 @@ +[package] +edition = "2021" +name = "transfer-from-account" +version = "4.0.0-rc.1" + +[dependencies] + +[lib] +crate-type = ["cdylib", "rlib"] + +[workspace] diff --git a/solana/svm/tests/example-programs/transfer-from-account/src/lib.rs b/solana/svm/tests/example-programs/transfer-from-account/src/lib.rs new file mode 100644 index 00000000..4460873f --- /dev/null +++ b/solana/svm/tests/example-programs/transfer-from-account/src/lib.rs @@ -0,0 +1,31 @@ +use { + solana_account_info::{next_account_info, AccountInfo}, + solana_program::program::invoke, + solana_program_entrypoint::entrypoint, + solana_program_error::ProgramResult, + solana_pubkey::Pubkey, + solana_system_interface::instruction as system_instruction, +}; + +entrypoint!(process_instruction); + +fn process_instruction( + _program_id: &Pubkey, + accounts: &[AccountInfo], + _data: &[u8], +) -> ProgramResult { + let accounts_iter = &mut accounts.iter(); + let payer = next_account_info(accounts_iter)?; + let recipient = next_account_info(accounts_iter)?; + let data_account = next_account_info(accounts_iter)?; + let system_program = next_account_info(accounts_iter)?; + + let amount = u64::from_le_bytes(data_account.data.borrow()[0..8].try_into().unwrap()); + + invoke( + &system_instruction::transfer(payer.key, recipient.key, amount), + &[payer.clone(), recipient.clone(), system_program.clone()], + )?; + + Ok(()) +} diff --git a/solana/svm/tests/example-programs/transfer-from-account/transfer_from_account_program.so b/solana/svm/tests/example-programs/transfer-from-account/transfer_from_account_program.so new file mode 100755 index 00000000..a3ef926d Binary files /dev/null and b/solana/svm/tests/example-programs/transfer-from-account/transfer_from_account_program.so differ diff --git a/solana/svm/tests/example-programs/write-to-account/Cargo.toml b/solana/svm/tests/example-programs/write-to-account/Cargo.toml new file mode 100644 index 00000000..122f3215 --- /dev/null +++ b/solana/svm/tests/example-programs/write-to-account/Cargo.toml @@ -0,0 +1,11 @@ +[package] +edition = "2021" +name = "write-to-account" +version = "4.0.0-rc.1" + +[dependencies] + +[lib] +crate-type = ["cdylib", "rlib"] + +[workspace] diff --git a/solana/svm/tests/example-programs/write-to-account/src/lib.rs b/solana/svm/tests/example-programs/write-to-account/src/lib.rs new file mode 100644 index 00000000..0fde9fac --- /dev/null +++ b/solana/svm/tests/example-programs/write-to-account/src/lib.rs @@ -0,0 +1,62 @@ +use { + solana_account_info::{next_account_info, AccountInfo}, + solana_msg::msg, + solana_program_entrypoint::entrypoint, + solana_program_error::{ProgramError, ProgramResult}, + solana_pubkey::Pubkey, + solana_sdk_ids::incinerator, +}; + +entrypoint!(process_instruction); + +fn process_instruction( + _program_id: &Pubkey, + accounts: &[AccountInfo], + data: &[u8], +) -> ProgramResult { + let accounts_iter = &mut accounts.iter(); + let target_account_info = next_account_info(accounts_iter)?; + match data[0] { + // print account size + 0 => { + msg!( + "account size {}", + target_account_info.try_borrow_data()?.len() + ); + } + // set account data + 1 => { + let mut account_data = target_account_info.try_borrow_mut_data()?; + account_data[0] = 100; + } + // deallocate account + 2 => { + let incinerator_info = next_account_info(accounts_iter)?; + if !incinerator::check_id(incinerator_info.key) { + return Err(ProgramError::InvalidAccountData); + } + + let mut target_lamports = target_account_info.try_borrow_mut_lamports()?; + let mut incinerator_lamports = incinerator_info.try_borrow_mut_lamports()?; + + **incinerator_lamports = incinerator_lamports + .checked_add(**target_lamports) + .ok_or(ProgramError::ArithmeticOverflow)?; + + **target_lamports = target_lamports + .checked_sub(**target_lamports) + .ok_or(ProgramError::InsufficientFunds)?; + } + // reallocate account + 3 => { + let new_size = usize::from_le_bytes(data[1..9].try_into().unwrap()); + target_account_info.realloc(new_size, true)?; + } + // bad ixn + _ => { + return Err(ProgramError::InvalidArgument); + } + } + + Ok(()) +} diff --git a/solana/svm/tests/example-programs/write-to-account/write_to_account_program.so b/solana/svm/tests/example-programs/write-to-account/write_to_account_program.so new file mode 100755 index 00000000..e4364263 Binary files /dev/null and b/solana/svm/tests/example-programs/write-to-account/write_to_account_program.so differ diff --git a/solana/transaction-context/Cargo.toml b/solana/transaction-context/Cargo.toml new file mode 100644 index 00000000..5d2126b5 --- /dev/null +++ b/solana/transaction-context/Cargo.toml @@ -0,0 +1,48 @@ +[package] +name = "solana-transaction-context" + +authors = { workspace = true } +description = "Solana data shared between program runtime and built-in programs as well as SBF programs." +documentation = "https://docs.rs/solana-transaction-context" +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +version = "4.1.1" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg=docsrs"] +targets = ["x86_64-unknown-linux-gnu"] + +[features] +# No-op stub retained only so external (patched-in) crates that reference +# `solana-transaction-context/agave-unstable-api` still resolve; the lib is no +# longer gated on it. +agave-unstable-api = [] +bincode = ["dep:bincode", "serde", "solana-account/bincode"] +dev-context-only-utils = ["bincode"] +serde = ["serde/derive", "solana-pubkey/serde"] + +[dependencies] +solana-account = { workspace = true } +solana-instruction = { workspace = true, features = ["std"] } +solana-instructions-sysvar = { workspace = true } +solana-pubkey = { workspace = true } + +[target.'cfg(not(any(target_arch = "sbf", target_arch = "bpf")))'.dependencies] +bincode = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +solana-rent = { workspace = true } +solana-sbpf = { workspace = true } +solana-sdk-ids = { workspace = true } + +[dev-dependencies] +solana-account-info = { workspace = true } +solana-program-entrypoint = { workspace = true } +solana-system-interface = { workspace = true } +solana-transaction-context = { path = ".", features = ["dev-context-only-utils"] } +static_assertions = "1.1.0" + +[lints.rust] +unexpected_cfgs = "allow" diff --git a/solana/transaction-context/README.md b/solana/transaction-context/README.md new file mode 100644 index 00000000..e2936917 --- /dev/null +++ b/solana/transaction-context/README.md @@ -0,0 +1,15 @@ +# `solana-transaction-context` + +This Agave fork defines the account and instruction state used by one executing +transaction. Workspace `[patch.crates-io]` entries force the dependency graph to +use this copy. + +`TransactionAccounts` stores account cells behind runtime borrow counters. +`AccountRef` and `AccountRefMut` enforce those counters while VM access handlers +can resize and remap an account's directly mapped data. The context also tracks +touched accounts, resize and lamport deltas, return data, instruction state, and +execution limits. + +The direct-mapping and access-violation contracts are documented in +[`../README.md`](../README.md). All account references must be released before a +`TransactionContext` is deconstructed. diff --git a/solana/transaction-context/src/instruction.rs b/solana/transaction-context/src/instruction.rs new file mode 100644 index 00000000..35a4d3c8 --- /dev/null +++ b/solana/transaction-context/src/instruction.rs @@ -0,0 +1,275 @@ +use { + crate::{ + IndexOfAccount, + instruction_accounts::{BorrowedInstructionAccount, InstructionAccount}, + transaction::TransactionContext, + vm_addresses::{ + GUEST_INSTRUCTION_ACCOUNT_BASE_ADDRESS, GUEST_INSTRUCTION_DATA_BASE_ADDRESS, + GUEST_REGION_SIZE, + }, + vm_slice::VmSlice, + }, + solana_account::ReadableAccount, + solana_instruction::error::InstructionError, + solana_pubkey::Pubkey, + std::collections::HashSet, +}; + +/// Instruction shared between runtime and programs. +#[repr(C)] +#[derive(Debug)] +pub struct InstructionFrame { + /// Reserved field for alignment and potential future usage. + pub reserved: u16, + pub program_account_index_in_tx: u16, + pub nesting_level: u16, + /// This is the index of the parent instruction if this is a CPI and u16::MAX if this is a + /// top-level instruction + pub index_of_caller_instruction: u16, + pub instruction_accounts: VmSlice, + pub instruction_data: VmSlice, +} + +impl Default for InstructionFrame { + fn default() -> Self { + InstructionFrame { + nesting_level: 0, + program_account_index_in_tx: 0, + index_of_caller_instruction: u16::MAX, + // Using u64::MAX as the default pointer value, since it shall never be accessible. + instruction_accounts: VmSlice::new(0, 0), + instruction_data: VmSlice::new(0, 0), + reserved: 0, + } + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl InstructionFrame { + pub fn configure_vm_slices( + &mut self, + instruction_index: u64, + instruction_accounts_len: usize, + instruction_data_len: u64, + ) { + let common_offset = GUEST_REGION_SIZE.saturating_mul(instruction_index); + + // Instruction data slice + self.instruction_data = VmSlice::new( + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(common_offset), + instruction_data_len, + ); + + // Instruction accounts slice + self.instruction_accounts = VmSlice::new( + GUEST_INSTRUCTION_ACCOUNT_BASE_ADDRESS.saturating_add(common_offset), + instruction_accounts_len as u64, + ); + } +} + +/// View interface to read instructions. +#[derive(Debug)] +pub struct InstructionContext<'a, 'ix_data> { + pub(crate) transaction_context: &'a TransactionContext<'ix_data>, + // The rest of the fields are redundant shortcuts + pub(crate) index_in_trace: usize, + pub(crate) nesting_level: usize, + pub(crate) index_of_caller_instruction: usize, + pub(crate) program_account_index_in_tx: IndexOfAccount, + pub(crate) instruction_accounts: &'a [InstructionAccount], + pub(crate) dedup_map: &'a [u16], + pub(crate) instruction_data: &'ix_data [u8], +} + +impl<'a> InstructionContext<'a, '_> { + /// How many Instructions were on the trace before this one was pushed + pub fn get_index_in_trace(&self) -> usize { + self.index_in_trace + } + + /// Returns the index of the instruction that called into this one. + pub fn get_index_of_caller(&self) -> usize { + self.index_of_caller_instruction + } + + /// How many Instructions were on the stack after this one was pushed + /// + /// That is the number of nested parent Instructions plus one (itself). + pub fn get_stack_height(&self) -> usize { + self.nesting_level.saturating_add(1) + } + + /// Number of accounts in this Instruction (without program accounts) + pub fn get_number_of_instruction_accounts(&self) -> IndexOfAccount { + self.instruction_accounts.len() as IndexOfAccount + } + + /// Assert that enough accounts were supplied to this Instruction + pub fn check_number_of_instruction_accounts( + &self, + expected_at_least: IndexOfAccount, + ) -> Result<(), InstructionError> { + if self.get_number_of_instruction_accounts() < expected_at_least { + Err(InstructionError::MissingAccount) + } else { + Ok(()) + } + } + + /// Data parameter for the programs `process_instruction` handler + pub fn get_instruction_data(&self) -> &[u8] { + self.instruction_data + } + + /// Translates the given instruction wide program_account_index into a transaction wide index + pub fn get_index_of_program_account_in_transaction( + &self, + ) -> Result { + if self.program_account_index_in_tx == u16::MAX { + Err(InstructionError::MissingAccount) + } else { + Ok(self.program_account_index_in_tx) + } + } + + /// Translates the given instruction wide instruction_account_index into a transaction wide index + pub fn get_index_of_instruction_account_in_transaction( + &self, + instruction_account_index: IndexOfAccount, + ) -> Result { + Ok(self + .instruction_accounts + .get(instruction_account_index as usize) + .ok_or(InstructionError::MissingAccount)? + .index_in_transaction as IndexOfAccount) + } + + /// Get the index of account in instruction from the index in transaction + pub fn get_index_of_account_in_instruction( + &self, + index_in_transaction: IndexOfAccount, + ) -> Result { + self.dedup_map + .get(index_in_transaction as usize) + .and_then(|idx| { + if *idx as usize >= self.instruction_accounts.len() { + None + } else { + Some(*idx as IndexOfAccount) + } + }) + .ok_or(InstructionError::MissingAccount) + } + + /// Returns `Some(instruction_account_index)` if this is a duplicate + /// and `None` if it is the first account with this key + pub fn is_instruction_account_duplicate( + &self, + instruction_account_index: IndexOfAccount, + ) -> Result, InstructionError> { + let index_in_transaction = + self.get_index_of_instruction_account_in_transaction(instruction_account_index)?; + let first_instruction_account_index = + self.get_index_of_account_in_instruction(index_in_transaction)?; + + Ok( + if first_instruction_account_index == instruction_account_index { + None + } else { + Some(first_instruction_account_index) + }, + ) + } + + /// Gets the key of the last program account of this Instruction + pub fn get_program_key(&self) -> Result<&'a Pubkey, InstructionError> { + self.get_index_of_program_account_in_transaction() + .and_then(|index_in_transaction| { + self.transaction_context.get_key_of_account_at_index(index_in_transaction) + }) + } + + /// Get the owner of the program account of this instruction + pub fn get_program_owner(&self) -> Result { + self.get_index_of_program_account_in_transaction() + .and_then(|index_in_transaction| { + self.transaction_context.accounts.try_borrow(index_in_transaction) + }) + .map(|acc| *acc.owner()) + } + + /// Gets an instruction account of this Instruction + pub fn try_borrow_instruction_account( + &self, + index_in_instruction: IndexOfAccount, + ) -> Result, InstructionError> { + let instruction_account = *self + .instruction_accounts + .get(index_in_instruction as usize) + .ok_or(InstructionError::MissingAccount)?; + + let account = self + .transaction_context + .accounts + .try_borrow_mut(instruction_account.index_in_transaction)?; + + Ok(BorrowedInstructionAccount { + transaction_context: self.transaction_context, + instruction_account, + account, + index_in_transaction_of_instruction_program: self.program_account_index_in_tx, + }) + } + + /// Returns whether an instruction account is a signer + pub fn is_instruction_account_signer( + &self, + instruction_account_index: IndexOfAccount, + ) -> Result { + Ok(self + .instruction_accounts + .get(instruction_account_index as usize) + .ok_or(InstructionError::MissingAccount)? + .is_signer()) + } + + /// Returns whether an instruction account is writable + pub fn is_instruction_account_writable( + &self, + instruction_account_index: IndexOfAccount, + ) -> Result { + Ok(self + .instruction_accounts + .get(instruction_account_index as usize) + .ok_or(InstructionError::MissingAccount)? + .is_writable()) + } + + /// Calculates the set of all keys of signer instruction accounts in this Instruction + pub fn get_signers(&self) -> Result, InstructionError> { + let mut result = HashSet::new(); + for instruction_account in self.instruction_accounts.iter() { + if instruction_account.is_signer() { + result.insert( + *self + .transaction_context + .get_key_of_account_at_index(instruction_account.index_in_transaction)?, + ); + } + } + Ok(result) + } + + pub fn instruction_accounts(&self) -> &[InstructionAccount] { + self.instruction_accounts + } + + pub fn get_key_of_instruction_account( + &self, + index_in_instruction: IndexOfAccount, + ) -> Result<&'a Pubkey, InstructionError> { + self.get_index_of_instruction_account_in_transaction(index_in_instruction) + .and_then(|idx| self.transaction_context.get_key_of_account_at_index(idx)) + } +} diff --git a/solana/transaction-context/src/instruction_accounts.rs b/solana/transaction-context/src/instruction_accounts.rs new file mode 100644 index 00000000..4bfbac9b --- /dev/null +++ b/solana/transaction-context/src/instruction_accounts.rs @@ -0,0 +1,381 @@ +use { + crate::{ + IndexOfAccount, MAX_ACCOUNT_DATA_GROWTH_PER_INSTRUCTION, transaction::TransactionContext, + transaction_accounts::AccountRefMut, + }, + solana_account::{CoWAccount, ReadableAccount, WritableAccount}, + solana_instruction::error::InstructionError, + solana_pubkey::Pubkey, +}; + +/// Contains account meta data which varies between instruction. +/// +/// It also contains indices to other structures for faster lookup. +/// +/// This data structure is supposed to be shared with programs in ABIv2, so do not modify it +/// without consulting SIMD-0177. +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct InstructionAccount { + /// Points to the account and its key in the `TransactionContext` + pub index_in_transaction: IndexOfAccount, + /// Is this account supposed to sign + is_signer: u8, + /// Is this account allowed to become writable + is_writable: u8, +} + +impl InstructionAccount { + pub fn new( + index_in_transaction: IndexOfAccount, + is_signer: bool, + is_writable: bool, + ) -> InstructionAccount { + InstructionAccount { + index_in_transaction, + is_signer: is_signer as u8, + is_writable: is_writable as u8, + } + } + + pub fn is_signer(&self) -> bool { + self.is_signer != 0 + } + + pub fn is_writable(&self) -> bool { + self.is_writable != 0 + } + + pub fn set_is_signer(&mut self, value: bool) { + self.is_signer = value as u8; + } + + pub fn set_is_writable(&mut self, value: bool) { + self.is_writable = value as u8; + } +} + +/// Shared account borrowed from the TransactionContext and an InstructionContext. +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +#[derive(Debug)] +pub struct BorrowedInstructionAccount<'a, 'ix_data> { + pub(crate) transaction_context: &'a TransactionContext<'ix_data>, + pub(crate) account: AccountRefMut<'a>, + pub(crate) instruction_account: InstructionAccount, + pub(crate) index_in_transaction_of_instruction_program: IndexOfAccount, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl BorrowedInstructionAccount<'_, '_> { + /// Returns the index of this account (transaction wide) + #[inline] + pub fn get_index_in_transaction(&self) -> IndexOfAccount { + self.instruction_account.index_in_transaction + } + + /// Returns the public key of this account (transaction wide) + #[inline] + pub fn get_key(&self) -> &Pubkey { + self.transaction_context + .get_key_of_account_at_index(self.instruction_account.index_in_transaction) + .unwrap() + } + + /// Returns the owner of this account (transaction wide) + #[inline] + pub fn get_owner(&self) -> &Pubkey { + self.account.owner() + } + + /// Assignes the owner of this account (transaction wide) + pub fn set_owner(&mut self, pubkey: &[u8]) -> Result<(), InstructionError> { + // Only the owner can assign a new owner + if !self.is_owned_by_current_program() { + return Err(InstructionError::ModifiedProgramId); + } + // and only if the account is writable + if !self.is_writable() { + return Err(InstructionError::ModifiedProgramId); + } + // and only if the data is zero-initialized or empty + if !is_zeroed(self.get_data()) { + return Err(InstructionError::ModifiedProgramId); + } + // don't touch the account if the owner does not change + if self.get_owner().to_bytes() == pubkey { + return Ok(()); + } + self.touch()?; + self.account.copy_into_owner_from_slice(pubkey); + Ok(()) + } + + /// Returns the number of lamports of this account (transaction wide) + #[inline] + pub fn get_lamports(&self) -> u64 { + self.account.lamports() + } + + /// Overwrites the number of lamports of this account (transaction wide) + pub fn set_lamports(&mut self, lamports: u64) -> Result<(), InstructionError> { + // An account not owned by the program cannot have its balance decrease + if !self.is_owned_by_current_program() && lamports < self.get_lamports() { + return Err(InstructionError::ExternalAccountLamportSpend); + } + // The balance of read-only may not change + if !self.is_writable() { + return Err(InstructionError::ReadonlyLamportChange); + } + // don't touch the account if the lamports do not change + let old_lamports = self.get_lamports(); + if old_lamports == lamports { + return Ok(()); + } + + let lamports_balance = (lamports as i128).saturating_sub(old_lamports as i128); + self.transaction_context.accounts.add_lamports_delta(lamports_balance)?; + + self.touch()?; + self.account.set_lamports(lamports); + Ok(()) + } + + /// Adds lamports to this account (transaction wide) + pub fn checked_add_lamports(&mut self, lamports: u64) -> Result<(), InstructionError> { + self.set_lamports( + self.get_lamports() + .checked_add(lamports) + .ok_or(InstructionError::ArithmeticOverflow)?, + ) + } + + /// Subtracts lamports from this account (transaction wide) + pub fn checked_sub_lamports(&mut self, lamports: u64) -> Result<(), InstructionError> { + self.set_lamports( + self.get_lamports() + .checked_sub(lamports) + .ok_or(InstructionError::ArithmeticOverflow)?, + ) + } + + /// Returns a read-only slice of the account data (transaction wide) + #[inline] + pub fn get_data(&self) -> &[u8] { + self.account.data() + } + + /// Returns a writable slice of the account data (transaction wide) + pub fn get_data_mut(&mut self) -> Result<&mut [u8], InstructionError> { + self.can_data_be_changed()?; + self.touch()?; + self.make_data_mut(); + Ok(self.account.data_as_mut_slice()) + } + + /// Overwrites the account data and size (transaction wide). + /// + /// Call this when you have a slice of data you do not own and want to + /// replace the account data with it. + pub fn set_data_from_slice(&mut self, data: &[u8]) -> Result<(), InstructionError> { + self.can_data_be_resized(data.len())?; + self.touch()?; + self.update_accounts_resize_delta(data.len())?; + // Note that we intentionally don't call self.make_data_mut() here. make_data_mut() will + // allocate + memcpy the current data if self.account is shared. We don't need the memcpy + // here tho because account.set_data_from_slice(data) is going to replace the content + // anyway. + self.account.set_data_from_slice(data); + + Ok(()) + } + + /// Resizes the account data (transaction wide) + /// + /// Fills it with zeros at the end if is extended or truncates at the end otherwise. + pub fn set_data_length(&mut self, new_length: usize) -> Result<(), InstructionError> { + self.can_data_be_resized(new_length)?; + // don't touch the account if the length does not change + if self.get_data().len() == new_length { + return Ok(()); + } + self.touch()?; + self.update_accounts_resize_delta(new_length)?; + self.account.resize(new_length, 0); + Ok(()) + } + + /// Appends all elements in a slice to the account + pub fn extend_from_slice(&mut self, data: &[u8]) -> Result<(), InstructionError> { + let new_len = self.get_data().len().saturating_add(data.len()); + self.can_data_be_resized(new_len)?; + + if data.is_empty() { + return Ok(()); + } + + self.touch()?; + self.update_accounts_resize_delta(new_len)?; + // Even if extend_from_slice never reduces capacity, still realloc using + // make_data_mut() if necessary so that we grow the account of the full + // max realloc length in one go, avoiding smaller reallocations. + self.make_data_mut(); + self.account.extend_from_slice(data); + Ok(()) + } + + /// Returns whether account data must be mapped through the CoW handler. + /// + /// Owned shared buffers and borrowed account images both need first-write + /// translation before the VM can mutate them. + pub fn is_shared(&self) -> bool { + self.account.is_shared() || matches!(self.account.cow(), CoWAccount::Borrowed(_)) + } + + fn make_data_mut(&mut self) { + // Reserve the maximum per-instruction growth before mutating shared buffers or borrowed + // account images. Borrowed images with enough spare capacity can remain borrowed. + if self.is_shared() { + self.account.reserve(MAX_ACCOUNT_DATA_GROWTH_PER_INSTRUCTION); + } + } + + /// Deserializes the account data into a state + #[cfg(feature = "bincode")] + pub fn get_state(&self) -> Result { + bincode::deserialize(self.account.data()).map_err(|_| InstructionError::InvalidAccountData) + } + + /// Serializes a state into the account data + #[cfg(feature = "bincode")] + pub fn set_state(&mut self, state: &T) -> Result<(), InstructionError> { + let data = self.get_data_mut()?; + let serialized_size = + bincode::serialized_size(state).map_err(|_| InstructionError::GenericError)?; + if serialized_size > data.len() as u64 { + return Err(InstructionError::AccountDataTooSmall); + } + bincode::serialize_into(&mut *data, state).map_err(|_| InstructionError::GenericError)?; + Ok(()) + } + + // Returns whether or the lamports currently in the account is sufficient for rent exemption should the + // data be resized to the given size + pub fn is_rent_exempt_at_data_length(&self, data_length: usize) -> bool { + self.transaction_context.rent.is_exempt(self.get_lamports(), data_length) + } + + /// Returns whether this account is executable (transaction wide) + #[inline] + #[deprecated(since = "2.1.0", note = "Use `get_owner` instead")] + pub fn is_executable(&self) -> bool { + #[allow(deprecated)] + self.account.executable() + } + + /// Configures whether this account is executable (transaction wide) + pub fn set_executable(&mut self, is_executable: bool) -> Result<(), InstructionError> { + // To become executable an account must be rent exempt + if !self + .transaction_context + .rent + .is_exempt(self.get_lamports(), self.get_data().len()) + { + return Err(InstructionError::ExecutableAccountNotRentExempt); + } + // Only the owner can set the executable flag + if !self.is_owned_by_current_program() { + return Err(InstructionError::ExecutableModified); + } + // and only if the account is writable + if !self.is_writable() { + return Err(InstructionError::ExecutableModified); + } + // don't touch the account if the executable flag does not change + #[allow(deprecated)] + if self.is_executable() == is_executable { + return Ok(()); + } + self.touch()?; + self.account.set_executable(is_executable); + Ok(()) + } + + /// Returns the rent epoch of this account (transaction wide) + #[inline] + pub fn get_rent_epoch(&self) -> u64 { + self.account.rent_epoch() + } + + /// Returns whether this account is a signer (instruction wide) + pub fn is_signer(&self) -> bool { + self.instruction_account.is_signer() + } + + /// Returns whether this account is writable (instruction wide) + pub fn is_writable(&self) -> bool { + self.instruction_account.is_writable() + } + + /// Returns true if the owner of this account is the current `InstructionContext`s last program (instruction wide) + pub fn is_owned_by_current_program(&self) -> bool { + self.transaction_context + .get_key_of_account_at_index(self.index_in_transaction_of_instruction_program) + .map(|program_key| program_key == self.get_owner()) + .unwrap_or_default() + } + + /// Returns an error if the account data can not be mutated by the current program + pub fn can_data_be_changed(&self) -> Result<(), InstructionError> { + // and only if the account is writable + if !self.is_writable() { + return Err(InstructionError::ReadonlyDataModified); + } + // and only if we are the owner + if !self.is_owned_by_current_program() { + return Err(InstructionError::ExternalAccountDataModified); + } + Ok(()) + } + + /// Returns an error if the account data can not be resized to the given length + pub fn can_data_be_resized(&self, new_len: usize) -> Result<(), InstructionError> { + let old_len = self.get_data().len(); + if new_len != old_len { + use solana_account::AccountMode; + if !self.is_owned_by_current_program() { + // Only the owner can change the length of the data + return Err(InstructionError::AccountDataSizeChanged); + } else if self.account.is(AccountMode::Ephemeral) { + // Ephemeral accounts can only be resized with special builtin instruction + return Err(InstructionError::InvalidRealloc); + } + } + self.transaction_context.accounts.can_data_be_resized(old_len, new_len)?; + self.can_data_be_changed() + } + + fn touch(&self) -> Result<(), InstructionError> { + self.transaction_context + .accounts + .touch(self.instruction_account.index_in_transaction) + } + + fn update_accounts_resize_delta(&mut self, new_len: usize) -> Result<(), InstructionError> { + self.transaction_context + .accounts + .update_accounts_resize_delta(self.get_data().len(), new_len) + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +fn is_zeroed(buf: &[u8]) -> bool { + const ZEROS_LEN: usize = 1024; + const ZEROS: [u8; ZEROS_LEN] = [0; ZEROS_LEN]; + let mut chunks = buf.chunks_exact(ZEROS_LEN); + + #[allow(clippy::indexing_slicing)] + { + chunks.all(|chunk| chunk == &ZEROS[..]) + && chunks.remainder() == &ZEROS[..chunks.remainder().len()] + } +} diff --git a/solana/transaction-context/src/lib.rs b/solana/transaction-context/src/lib.rs new file mode 100644 index 00000000..35663868 --- /dev/null +++ b/solana/transaction-context/src/lib.rs @@ -0,0 +1,52 @@ +#![allow(clippy::disallowed_methods)] +#![deny(clippy::indexing_slicing)] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc = include_str!("../README.md")] + +pub mod instruction; +pub mod instruction_accounts; +pub mod transaction_accounts; +mod vm_addresses; +pub mod vm_slice; + +pub mod transaction; + +pub const MAX_ACCOUNTS_PER_TRANSACTION: usize = 256; +// This is one less than MAX_ACCOUNTS_PER_TRANSACTION because +// one index is used as NON_DUP_MARKER in ABI v0 and v1. +pub const MAX_ACCOUNTS_PER_INSTRUCTION: usize = 255; +pub const MAX_INSTRUCTION_DATA_LEN: usize = 10 * 1024; +pub const MAX_ACCOUNT_DATA_LEN: u64 = 10 * 1024 * 1024; +// Note: Direct account-region mapping lets programs grow accounts through the +// AccessViolationHandler, which might grow an account up to +// MAX_ACCOUNT_DATA_GROWTH_PER_INSTRUCTION at once. +pub const MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION: i64 = MAX_ACCOUNT_DATA_LEN as i64 * 2; +pub const MAX_ACCOUNT_DATA_GROWTH_PER_INSTRUCTION: usize = 10 * 1_024; +// Maximum cross-program invocation and instructions per transaction +pub const MAX_INSTRUCTION_TRACE_LENGTH: usize = 64; +/// Maximum cross-program invocations per transaction. +pub const MAX_CPI_TRACE_LENGTH: usize = 64; + +#[cfg(test)] +static_assertions::const_assert_eq!( + MAX_ACCOUNTS_PER_INSTRUCTION, + solana_program_entrypoint::NON_DUP_MARKER as usize, +); +#[cfg(test)] +static_assertions::const_assert_eq!( + MAX_ACCOUNT_DATA_LEN, + solana_system_interface::MAX_PERMITTED_DATA_LENGTH, +); +#[cfg(test)] +static_assertions::const_assert_eq!( + MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION, + solana_system_interface::MAX_PERMITTED_ACCOUNTS_DATA_ALLOCATIONS_PER_TRANSACTION, +); +#[cfg(test)] +static_assertions::const_assert_eq!( + MAX_ACCOUNT_DATA_GROWTH_PER_INSTRUCTION, + solana_account_info::MAX_PERMITTED_DATA_INCREASE, +); + +/// Index of an account inside of the transaction or an instruction. +pub type IndexOfAccount = u16; diff --git a/solana/transaction-context/src/transaction.rs b/solana/transaction-context/src/transaction.rs new file mode 100644 index 00000000..462099da --- /dev/null +++ b/solana/transaction-context/src/transaction.rs @@ -0,0 +1,1253 @@ +use { + crate::{ + IndexOfAccount, MAX_ACCOUNT_DATA_LEN, MAX_ACCOUNTS_PER_TRANSACTION, MAX_CPI_TRACE_LENGTH, + instruction::{InstructionContext, InstructionFrame}, + instruction_accounts::InstructionAccount, + transaction_accounts::{KeyedAccountSharedData, TransactionAccounts}, + vm_addresses::{ + GUEST_INSTRUCTION_DATA_BASE_ADDRESS, GUEST_REGION_SIZE, RETURN_DATA_SCRATCHPAD, + }, + vm_slice::VmSlice, + }, + solana_account::{AccountSharedData, ReadableAccount, WritableAccount}, + solana_instruction::error::InstructionError, + solana_instructions_sysvar as instructions, + solana_pubkey::Pubkey, + solana_rent::Rent, + solana_sbpf::memory_region::{AccessType, AccessViolationHandler, MemoryRegion}, + std::{borrow::Cow, cell::Cell, rc::Rc}, +}; + +/// Used only in fn `take_instruction_trace` for deconstructing TransactionContext +pub type InstructionTrace<'ix_data> = ( + Vec, + Vec>, + Vec>, +); + +/// This data structure is shared with programs in ABIv2, providing information about the +/// transaction metadata. +/// +/// Modifications without a feature gate and proper versioning might break programs. +#[repr(C)] +#[derive(Debug)] +struct TransactionFrame { + /// Pubkey of the last program to write to the return data scratchpad + return_data_pubkey: Pubkey, + return_data_scratchpad: VmSlice, + /// Scratchpad for programs to write CPI instruction data + cpi_scratchpad: VmSlice, + /// Index of current executing instruction + current_executing_instruction: u16, + /// Number of instructions in the instruction trace (including top level and CPIs) + total_number_of_instructions_in_trace: u16, + /// Number of CPIs in the instruction trace + number_of_cpis_in_trace: u16, + /// Number of transaction accounts + number_of_transaction_accounts: u16, +} + +/// Loaded transaction shared between runtime and programs. +/// +/// This context is valid for the entire duration of a transaction being processed. +#[derive(Debug)] +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct TransactionContext<'ix_data> { + pub(crate) accounts: Rc, + instruction_stack_capacity: usize, + instruction_trace_capacity: usize, + instruction_stack: Vec, + instruction_trace: Vec, + transaction_frame: TransactionFrame, + return_data_bytes: Vec, + next_top_level_instruction_index: usize, + #[cfg(not(target_os = "solana"))] + pub(crate) rent: Rent, + /// This is an account deduplication map that maps index_in_transaction to index_in_instruction + /// Usage: dedup_map[index_in_transaction] = index_in_instruction + /// Each entry in `deduplication_maps` represents the deduplication map for each instruction. + deduplication_maps: Vec>, + /// Each entry in `instruction_accounts` represents the array of accounts for each instruction. + instruction_accounts: Vec>, + /// Each entry in `instruction_data` represents the data for instruction at the corresponding + /// index. + instruction_data: Vec>, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl<'ix_data> TransactionContext<'ix_data> { + /// Constructs a new TransactionContext + pub fn new( + transaction_accounts: Vec, + rent: Rent, + instruction_stack_capacity: usize, + instruction_trace_capacity: usize, + number_of_top_level_instructions: usize, + ) -> Self { + let transaction_frame = TransactionFrame { + return_data_pubkey: Pubkey::default(), + return_data_scratchpad: VmSlice::new(RETURN_DATA_SCRATCHPAD, 0), + cpi_scratchpad: VmSlice::new(0, 0), + current_executing_instruction: 0, + total_number_of_instructions_in_trace: number_of_top_level_instructions as u16, + number_of_cpis_in_trace: 0, + number_of_transaction_accounts: transaction_accounts.len() as u16, + }; + + Self { + accounts: Rc::new(TransactionAccounts::new(transaction_accounts)), + instruction_stack_capacity, + instruction_trace_capacity, + instruction_stack: Vec::with_capacity(instruction_stack_capacity), + instruction_trace: vec![InstructionFrame::default()], + return_data_bytes: Vec::new(), + transaction_frame, + next_top_level_instruction_index: 0, + rent, + instruction_accounts: Vec::with_capacity(instruction_trace_capacity), + deduplication_maps: Vec::with_capacity(instruction_trace_capacity), + instruction_data: Vec::with_capacity(instruction_trace_capacity), + } + } + + /// Used in mock_process_instruction + pub fn deconstruct_without_keys(self) -> Result, InstructionError> { + if !self.instruction_stack.is_empty() { + return Err(InstructionError::CallDepth); + } + + let accounts = Rc::try_unwrap(self.accounts) + .expect("transaction_context.accounts has unexpected outstanding refs") + .deconstruct_into_account_shared_data(); + + Ok(accounts) + } + + pub fn accounts(&self) -> &Rc { + &self.accounts + } + + /// Returns the total number of accounts loaded in this Transaction + pub fn get_number_of_accounts(&self) -> IndexOfAccount { + self.accounts.len() as IndexOfAccount + } + + /// Searches for an account by its key + pub fn get_key_of_account_at_index( + &self, + index_in_transaction: IndexOfAccount, + ) -> Result<&Pubkey, InstructionError> { + self.accounts + .account_key(index_in_transaction) + .ok_or(InstructionError::MissingAccount) + } + + /// Searches for an account by its key + pub fn find_index_of_account(&self, pubkey: &Pubkey) -> Option { + self.accounts + .account_keys_iter() + .position(|key| key == pubkey) + .map(|index| index as IndexOfAccount) + } + + /// Gets the max length of the instruction trace + pub fn get_instruction_trace_capacity(&self) -> usize { + self.instruction_trace_capacity + } + + /// Returns the instruction trace length. + /// + /// Not counting the last empty instruction which is always pre-reserved for the next instruction. + pub fn get_instruction_trace_length(&self) -> usize { + self.instruction_trace.len().saturating_sub(1) + } + + /// Gets a view on an instruction by its index in the trace + pub fn get_instruction_context_at_index_in_trace( + &self, + index_in_trace: usize, + ) -> Result, InstructionError> { + let instruction = + self.instruction_trace.get(index_in_trace).ok_or(InstructionError::CallDepth)?; + + // These commands will return a default empty slice if we are retrieving an instruction + // that hasn't been configured yet. + let instruction_accounts = self + .instruction_accounts + .get(index_in_trace) + .map(|item| item.as_ref()) + .unwrap_or_default(); + let dedup_map = self + .deduplication_maps + .get(index_in_trace) + .map(|item| item.as_ref()) + .unwrap_or_default(); + let instruction_data = self + .instruction_data + .get(index_in_trace) + .map(|item| item.as_ref()) + .unwrap_or_default(); + Ok(InstructionContext { + transaction_context: self, + index_in_trace, + nesting_level: instruction.nesting_level as usize, + program_account_index_in_tx: instruction.program_account_index_in_tx as IndexOfAccount, + instruction_accounts, + dedup_map, + instruction_data, + index_of_caller_instruction: instruction.index_of_caller_instruction as usize, + }) + } + + /// Gets a view on the instruction by its nesting level in the stack + pub fn get_instruction_context_at_nesting_level( + &self, + nesting_level: usize, + ) -> Result, InstructionError> { + let index_in_trace = + *self.instruction_stack.get(nesting_level).ok_or(InstructionError::CallDepth)?; + let instruction_context = self.get_instruction_context_at_index_in_trace(index_in_trace)?; + debug_assert_eq!(instruction_context.nesting_level, nesting_level); + Ok(instruction_context) + } + + /// Gets the max height of the instruction stack + pub fn get_instruction_stack_capacity(&self) -> usize { + self.instruction_stack_capacity + } + + /// Gets instruction stack height, top-level instructions are height + /// `solana_instruction::TRANSACTION_LEVEL_STACK_HEIGHT` + pub fn get_instruction_stack_height(&self) -> usize { + self.instruction_stack.len() + } + + /// Returns the index in the instruction trace of the current executing instruction + pub fn get_current_instruction_index(&self) -> Result { + self.instruction_stack.last().copied().ok_or(InstructionError::CallDepth) + } + + /// Returns a view on the current instruction + pub fn get_current_instruction_context( + &self, + ) -> Result, InstructionError> { + let index_in_trace = self.get_current_instruction_index()?; + self.get_instruction_context_at_index_in_trace(index_in_trace) + } + + /// Returns a view on the next instruction. This function assumes it has already been + /// configured with the correct values in `prepare_next_instruction` or + /// `prepare_next_top_level_instruction` + pub fn get_next_instruction_context( + &self, + ) -> Result, InstructionError> { + let index_in_trace = + self.instruction_trace.len().checked_sub(1).ok_or(InstructionError::CallDepth)?; + self.get_instruction_context_at_index_in_trace(index_in_trace) + } + + /// Configures an instruction at a specific index in trace. + pub fn configure_instruction_at_index( + &mut self, + instruction_index: usize, + program_index: IndexOfAccount, + instruction_accounts: Vec, + deduplication_map: Vec, + instruction_data: Cow<'ix_data, [u8]>, + caller_index: Option, + ) -> Result<(), InstructionError> { + debug_assert_eq!(deduplication_map.len(), MAX_ACCOUNTS_PER_TRANSACTION); + + let instruction = self + .instruction_trace + .get_mut(instruction_index) + .ok_or(InstructionError::MaxInstructionTraceLengthExceeded)?; + + let total_number_of_instructions_in_trace = if let Some(caller_index) = caller_index { + instruction.index_of_caller_instruction = caller_index; + self.transaction_frame.total_number_of_instructions_in_trace.saturating_add(1) + } else { + self.transaction_frame.total_number_of_instructions_in_trace + }; + + self.transaction_frame.cpi_scratchpad = VmSlice::new( + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add( + GUEST_REGION_SIZE.saturating_mul(total_number_of_instructions_in_trace as u64), + ), + 0, + ); + + instruction.program_account_index_in_tx = program_index; + instruction.configure_vm_slices( + instruction_index as u64, + instruction_accounts.len(), + instruction_data.len() as u64, + ); + self.deduplication_maps.push(deduplication_map.into_boxed_slice()); + self.instruction_accounts.push(instruction_accounts.into_boxed_slice()); + self.instruction_data.push(instruction_data); + Ok(()) + } + + /// For tests only + fn deduplicate_accounts_for_tests(instruction_accounts: &[InstructionAccount]) -> Vec { + let mut dedup_map = vec![u16::MAX; MAX_ACCOUNTS_PER_TRANSACTION]; + for (idx, account) in instruction_accounts.iter().enumerate() { + let index_in_instruction = + dedup_map.get_mut(account.index_in_transaction as usize).unwrap(); + if *index_in_instruction == u16::MAX { + *index_in_instruction = idx as u16; + } + } + dedup_map + } + + /// A version of `configure_top_level_instruction` to help creating the deduplication map in tests + pub fn configure_top_level_instruction_for_tests( + &mut self, + program_index: IndexOfAccount, + instruction_accounts: Vec, + instruction_data: Vec, + ) -> Result<(), InstructionError> { + debug_assert!(instruction_accounts.len() <= u16::MAX as usize); + let dedup_map = Self::deduplicate_accounts_for_tests(&instruction_accounts); + + self.configure_instruction_at_index( + self.get_instruction_trace_length(), + program_index, + instruction_accounts, + dedup_map, + Cow::Owned(instruction_data), + None, + )?; + Ok(()) + } + + /// A helper function to facilitate creating a CPI in tests + pub fn configure_next_cpi_for_tests( + &mut self, + program_index: IndexOfAccount, + instruction_accounts: Vec, + instruction_data: Vec, + ) -> Result<(), InstructionError> { + debug_assert!(instruction_accounts.len() <= u16::MAX as usize); + let dedup_map = Self::deduplicate_accounts_for_tests(&instruction_accounts); + let caller_index = self.get_current_instruction_index()?; + let cpi_index = self.get_instruction_trace_length(); + self.configure_instruction_at_index( + cpi_index, + program_index, + instruction_accounts, + dedup_map, + Cow::Owned(instruction_data), + Some(caller_index as u16), + )?; + Ok(()) + } + + /// Pushes the next instruction + pub fn push(&mut self) -> Result<(), InstructionError> { + let nesting_level = self.get_instruction_stack_height(); + if !self.instruction_stack.is_empty() && self.accounts.get_lamports_delta() != 0 { + return Err(InstructionError::UnbalancedInstruction); + } + let index_in_trace = self.get_instruction_trace_length(); + if index_in_trace >= self.instruction_trace_capacity { + return Err(InstructionError::MaxInstructionTraceLengthExceeded); + } + + let is_cpi = !self.instruction_stack.is_empty(); + if is_cpi + && (self.transaction_frame.number_of_cpis_in_trace as usize >= MAX_CPI_TRACE_LENGTH + || self.transaction_frame.total_number_of_instructions_in_trace as usize + >= self.instruction_trace_capacity) + { + return Err(InstructionError::MaxInstructionTraceLengthExceeded); + } + + let instruction = self.instruction_trace.last_mut().ok_or(InstructionError::CallDepth)?; + instruction.nesting_level = nesting_level as u16; + + let current_top_level_instruction = if is_cpi { + self.transaction_frame.total_number_of_instructions_in_trace = + self.transaction_frame.total_number_of_instructions_in_trace.saturating_add(1); + self.transaction_frame.number_of_cpis_in_trace = + self.transaction_frame.number_of_cpis_in_trace.saturating_add(1); + self.next_top_level_instruction_index.saturating_sub(1) + } else { + let index = self.next_top_level_instruction_index; + self.next_top_level_instruction_index = + self.next_top_level_instruction_index.saturating_add(1); + index + }; + + self.instruction_trace.push(InstructionFrame::default()); + if nesting_level >= self.instruction_stack_capacity { + return Err(InstructionError::CallDepth); + } + self.transaction_frame.current_executing_instruction = index_in_trace as u16; + self.instruction_stack.push(index_in_trace); + if let Some(index_in_transaction) = self.find_index_of_account(&instructions::id()) { + let mut mut_account_ref = self.accounts.try_borrow_mut(index_in_transaction)?; + if mut_account_ref.owner() != &solana_sdk_ids::sysvar::id() { + return Err(InstructionError::InvalidAccountOwner); + } + instructions::store_current_index_checked( + mut_account_ref.data_as_mut_slice(), + current_top_level_instruction as u16, + )?; + } + Ok(()) + } + + /// Pops the current instruction + pub fn pop(&mut self) -> Result<(), InstructionError> { + if self.instruction_stack.is_empty() { + return Err(InstructionError::CallDepth); + } + // Verify (before we pop) that the total sum of all lamports in this instruction did not change + let detected_an_unbalanced_instruction = + self.get_current_instruction_context().and_then(|instruction_context| { + // Verify all executable accounts have no outstanding refs + self.accounts + .try_borrow_mut( + instruction_context.get_index_of_program_account_in_transaction()?, + ) + .map_err(|err| { + if err == InstructionError::AccountBorrowFailed { + InstructionError::AccountBorrowOutstanding + } else { + err + } + })?; + Ok(self.accounts.get_lamports_delta() != 0) + }); + // Always pop, even if we `detected_an_unbalanced_instruction` + self.instruction_stack.pop(); + if let Some(instr_idx) = self.instruction_stack.last() { + self.transaction_frame.current_executing_instruction = *instr_idx as u16; + } + if detected_an_unbalanced_instruction? { + Err(InstructionError::UnbalancedInstruction) + } else { + Ok(()) + } + } + + /// Gets the return data of the current instruction or any above + pub fn get_return_data(&self) -> (&Pubkey, &[u8]) { + ( + &self.transaction_frame.return_data_pubkey, + &self.return_data_bytes, + ) + } + + /// Set the return data of the current instruction + pub fn set_return_data( + &mut self, + program_id: Pubkey, + data: Vec, + ) -> Result<(), InstructionError> { + self.transaction_frame.return_data_pubkey = program_id; + // SAFETY: `return_data_scratchpad` is backed by `self.return_data_bytes` + // and `return_data_bytes` is being reset to `data` + // in the next statement. + unsafe { + self.transaction_frame.return_data_scratchpad.set_len(data.len() as u64); + } + self.return_data_bytes = data; + Ok(()) + } + + /// Returns a new account data write access handler + pub fn access_violation_handler(&self) -> AccessViolationHandler { + let accounts = Rc::clone(&self.accounts); + Box::new( + move |region: &mut MemoryRegion, + address_space_reserved_for_account: u64, + access_type: AccessType, + vm_addr: u64, + len: u64| { + use solana_account::AccountMode; + + if access_type == AccessType::Load { + return; + } + let Some(index_in_transaction) = region.access_violation_handler_payload else { + // This region is not a writable account. + return; + }; + let requested_length = + vm_addr.saturating_add(len).saturating_sub(region.vm_addr) as usize; + if requested_length > address_space_reserved_for_account as usize { + // Requested access goes further than the account region. + return; + } + + // The four calls below can't really fail. If they fail because of a bug, + // whatever is writing will trigger an EbpfError::AccessViolation like + // if the region was readonly, and the transaction will fail gracefully. + let Ok(mut account) = accounts.try_borrow_mut(index_in_transaction) else { + debug_assert!(false); + return; + }; + if accounts.touch(index_in_transaction).is_err() { + debug_assert!(false); + return; + } + + if requested_length > region.len as usize { + let old_len = account.data().len(); + let new_len = requested_length; + if new_len > MAX_ACCOUNT_DATA_LEN as usize + || accounts.can_data_be_resized(old_len, new_len).is_err() + || account.is(AccountMode::Ephemeral) + { + return; + } + if accounts.update_accounts_resize_delta(old_len, new_len).is_err() { + return; + } + account.resize(new_len, 0); + } + + let data = account.data_as_mut_slice(); + region.host_addr = data.as_mut_ptr() as u64; + region.len = data.len() as u64; + region.writable = true; + }, + ) + } + + /// Take ownership of the instruction trace + pub fn take_instruction_trace(&mut self) -> InstructionTrace<'_> { + // The last frame is a placeholder for the next instruction to be executed, so it + // is empty. + self.instruction_trace.pop(); + ( + std::mem::take(&mut self.instruction_trace), + std::mem::take(&mut self.instruction_accounts), + std::mem::take(&mut self.instruction_data), + ) + } + + /// An active instruction is either one that has already finished execution or that is + /// under execution (e.g. all nested CPIs are active). + /// For ABIv2 only. + pub fn number_of_active_instructions_in_trace(&self) -> usize { + self.next_top_level_instruction_index + .saturating_add(self.transaction_frame.number_of_cpis_in_trace as usize) + } + + /// Return next top level instruction to execute + pub fn next_top_level_instruction_index(&self) -> usize { + self.next_top_level_instruction_index + } + + /// Return number of CPIs in instruction trace + pub fn number_of_cpis_in_trace(&self) -> usize { + self.transaction_frame.number_of_cpis_in_trace as usize + } +} + +/// Return data at the end of a transaction +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TransactionReturnData { + pub program_id: Pubkey, + pub data: Vec, +} + +/// Everything that needs to be recorded from a TransactionContext after execution +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct ExecutionRecord { + pub accounts: Vec, + pub return_data: TransactionReturnData, + pub touched_account_count: u64, + pub accounts_resize_delta: i64, +} + +/// Used by the bank in the runtime to write back the processed accounts and recorded instructions +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl From> for ExecutionRecord { + fn from(context: TransactionContext) -> Self { + let (accounts, touched_flags, resize_delta) = Rc::try_unwrap(context.accounts) + .expect("transaction_context.accounts has unexpected outstanding refs") + .take(); + let touched_account_count = touched_flags.iter().fold(0usize, |accumulator, was_touched| { + accumulator.saturating_add(was_touched.get() as usize) + }) as u64; + + let return_data = TransactionReturnData { + program_id: context.transaction_frame.return_data_pubkey, + data: context.return_data_bytes, + }; + + Self { + accounts, + return_data, + touched_account_count, + accounts_resize_delta: Cell::into_inner(resize_delta), + } + } +} + +#[cfg(all(test, not(target_arch = "sbf"), not(target_arch = "bpf")))] +mod tests { + use super::*; + + /// Proves CPIs retain their 64-call ceiling when Magicblock enlarges the + /// total trace, and cannot consume slots reserved for later top-level + /// instructions. + #[test] + fn test_cpi_trace_limits() { + fn push_cpi(transaction_context: &mut TransactionContext<'_>) { + transaction_context + .configure_next_cpi_for_tests(0, Vec::new(), Vec::new()) + .unwrap(); + transaction_context.push().unwrap(); + transaction_context.pop().unwrap(); + } + + let mut transaction_context = TransactionContext::new( + vec![(Pubkey::new_unique(), AccountSharedData::default())], + Rent::default(), + 2, + 255, + 1, + ); + transaction_context + .configure_top_level_instruction_for_tests(0, Vec::new(), Vec::new()) + .unwrap(); + transaction_context.push().unwrap(); + + for _ in 0..MAX_CPI_TRACE_LENGTH { + push_cpi(&mut transaction_context); + } + + transaction_context + .configure_next_cpi_for_tests(0, Vec::new(), Vec::new()) + .unwrap(); + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 1 + MAX_CPI_TRACE_LENGTH as u16, + ); + assert_eq!( + transaction_context.push(), + Err(InstructionError::MaxInstructionTraceLengthExceeded), + ); + assert_eq!( + transaction_context.get_instruction_trace_length(), + 1 + MAX_CPI_TRACE_LENGTH, + ); + assert_eq!(transaction_context.get_instruction_stack_height(), 1); + assert_eq!(transaction_context.next_top_level_instruction_index(), 1); + assert_eq!( + transaction_context.number_of_cpis_in_trace(), + MAX_CPI_TRACE_LENGTH, + ); + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 1 + MAX_CPI_TRACE_LENGTH as u16, + ); + + let mut transaction_context = TransactionContext::new( + vec![(Pubkey::new_unique(), AccountSharedData::default())], + Rent::default(), + 2, + 4, + 2, + ); + transaction_context + .configure_top_level_instruction_for_tests(0, Vec::new(), Vec::new()) + .unwrap(); + transaction_context.push().unwrap(); + push_cpi(&mut transaction_context); + push_cpi(&mut transaction_context); + transaction_context + .configure_next_cpi_for_tests(0, Vec::new(), Vec::new()) + .unwrap(); + + assert_eq!( + transaction_context.push(), + Err(InstructionError::MaxInstructionTraceLengthExceeded), + ); + assert_eq!(transaction_context.get_instruction_trace_length(), 3); + assert_eq!(transaction_context.get_instruction_stack_height(), 1); + assert_eq!(transaction_context.next_top_level_instruction_index(), 1); + assert_eq!(transaction_context.number_of_cpis_in_trace(), 2); + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 4, + ); + } + + #[test] + fn test_instructions_sysvar_store_index_checked() { + let build_transaction_context = |account: AccountSharedData| { + TransactionContext::new( + vec![ + (Pubkey::new_unique(), AccountSharedData::default()), + (instructions::id(), account), + ], + Rent::default(), + /* max_instruction_stack_depth */ 2, + /* max_instruction_trace_length */ 2, + /* number_of_top_level_instructions */ 1, + ) + }; + + let correct_space = 2; + let rent_exempt_lamports = Rent::default().minimum_balance(correct_space); + + // First try it with the wrong owner. + let account = + AccountSharedData::new(rent_exempt_lamports, correct_space, &Pubkey::new_unique()); + assert_eq!( + build_transaction_context(account).push(), + Err(InstructionError::InvalidAccountOwner), + ); + + // Now with the wrong data length. + let account = + AccountSharedData::new(rent_exempt_lamports, 0, &solana_sdk_ids::sysvar::id()); + assert_eq!( + build_transaction_context(account).push(), + Err(InstructionError::AccountDataTooSmall), + ); + + // Finally provide the correct account setup. + let account = AccountSharedData::new( + rent_exempt_lamports, + correct_space, + &solana_sdk_ids::sysvar::id(), + ); + assert_eq!(build_transaction_context(account).push(), Ok(()),); + } + + #[test] + fn test_invalid_native_loader_index() { + let mut transaction_context = TransactionContext::new( + vec![( + Pubkey::new_unique(), + AccountSharedData::new(1, 1, &Pubkey::new_unique()), + )], + Rent::default(), + 20, + 20, + 1, + ); + + transaction_context + .configure_top_level_instruction_for_tests( + u16::MAX, + vec![InstructionAccount::new(0, false, false)], + vec![], + ) + .unwrap(); + let instruction_context = transaction_context.get_next_instruction_context().unwrap(); + + let result = instruction_context.get_index_of_program_account_in_transaction(); + assert_eq!(result, Err(InstructionError::MissingAccount)); + + let result = instruction_context.get_program_key(); + assert_eq!(result, Err(InstructionError::MissingAccount)); + + let result = instruction_context.get_program_owner(); + assert_eq!(result.err(), Some(InstructionError::MissingAccount)); + } + + #[test] + fn test_instruction_shared_items() { + let transaction_accounts = vec![(Pubkey::new_unique(), AccountSharedData::default()); 10]; + let mut transaction_context = + TransactionContext::new(transaction_accounts, Rent::default(), 20, 20, 3); + + let instruction_accounts_1 = + vec![InstructionAccount::new(0, false, true), InstructionAccount::new(3, true, false)]; + transaction_context + .configure_top_level_instruction_for_tests( + 1, + instruction_accounts_1.clone(), + vec![1, 2, 3, 4], + ) + .unwrap(); + transaction_context.push().unwrap(); + + let instruction_accounts_2 = vec![ + InstructionAccount::new(0, false, true), + InstructionAccount::new(3, true, false), + InstructionAccount::new(5, false, false), + ]; + transaction_context + .configure_top_level_instruction_for_tests( + 1, + instruction_accounts_2.clone(), + vec![5, 6, 7, 8, 9], + ) + .unwrap(); + transaction_context.push().unwrap(); + + let instruction_accounts_3 = vec![ + InstructionAccount::new(0, false, true), + InstructionAccount::new(3, true, false), + InstructionAccount::new(5, false, false), + InstructionAccount::new(3, false, false), + InstructionAccount::new(10, false, false), + ]; + transaction_context + .configure_top_level_instruction_for_tests( + 1, + instruction_accounts_3.clone(), + vec![10, 11], + ) + .unwrap(); + transaction_context.push().unwrap(); + + let first_ix_context = + transaction_context.get_instruction_context_at_index_in_trace(0).unwrap(); + assert_eq!( + instruction_accounts_1.as_slice(), + first_ix_context.instruction_accounts + ); + assert_eq!( + *first_ix_context.instruction_data, + **transaction_context.instruction_data.first().unwrap() + ); + for (idx_in_ix, acc) in instruction_accounts_1.iter().enumerate() { + assert_eq!( + *first_ix_context.dedup_map.get(acc.index_in_transaction as usize).unwrap(), + idx_in_ix as u16 + ); + } + + let second_ix_context = + transaction_context.get_instruction_context_at_index_in_trace(1).unwrap(); + assert_eq!( + instruction_accounts_2.as_slice(), + second_ix_context.instruction_accounts + ); + assert_eq!( + *second_ix_context.instruction_data, + **transaction_context.instruction_data.get(1).unwrap() + ); + for (idx_in_ix, acc) in instruction_accounts_2.iter().enumerate() { + assert_eq!( + *second_ix_context.dedup_map.get(acc.index_in_transaction as usize).unwrap(), + idx_in_ix as u16 + ); + } + + let third_ix_context = + transaction_context.get_instruction_context_at_index_in_trace(2).unwrap(); + assert_eq!( + instruction_accounts_3.as_slice(), + third_ix_context.instruction_accounts + ); + assert_eq!( + *third_ix_context.instruction_data, + **transaction_context.instruction_data.get(2).unwrap() + ); + for (idx_in_ix, acc) in instruction_accounts_3.iter().enumerate() { + if idx_in_ix == 3 { + assert_eq!( + *third_ix_context.dedup_map.get(acc.index_in_transaction as usize).unwrap(), + 1 + ); + } else { + assert_eq!( + *third_ix_context.dedup_map.get(acc.index_in_transaction as usize).unwrap(), + idx_in_ix as u16 + ); + } + } + } + + #[test] + fn test_number_of_instructions() { + let transaction_accounts = vec![(Pubkey::new_unique(), AccountSharedData::default()); 3]; + let mut transaction_context = + TransactionContext::new(transaction_accounts, Rent::default(), 20, 20, 2); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 0 + ); + + // Instruction #0 + transaction_context + .configure_instruction_at_index( + 0, + 0, + vec![InstructionAccount::new(1, false, false)], + vec![0; MAX_ACCOUNTS_PER_TRANSACTION], + Vec::new().into(), + None, + ) + .unwrap(); + + // Executing instruction #0 + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 0 + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 2 + ); + + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 0 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(2)) + ); + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.len(), + 0, + ); + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 1 + ); + + // Instruction #0 does a CPI. + transaction_context + .configure_next_cpi_for_tests( + 0, + vec![InstructionAccount::new(2, false, true)], + Vec::new(), + ) + .unwrap(); + + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 1, + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 3 + ); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 1 + ); + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 2 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(3)) + ); + + // A nested CPI + transaction_context + .configure_next_cpi_for_tests( + 0, + vec![InstructionAccount::new(2, false, true)], + Vec::new(), + ) + .unwrap(); + + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 2 + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 4 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(4)) + ); + + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 2 + ); + + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 3 + ); + // Return from nested CPI + transaction_context.pop().unwrap(); + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 3 + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 4 + ); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 2, + ); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 1 + ); + + // A second nested CPI + transaction_context + .configure_next_cpi_for_tests( + 0, + vec![InstructionAccount::new(2, false, true)], + Vec::new(), + ) + .unwrap(); + + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 3 + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 5 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(5)) + ); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 3 + ); + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 4 + ); + + // Return from second nested CPI + transaction_context.pop().unwrap(); + + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 1 + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 5 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(5)) + ); + + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 3 + ); + + // Return from first CPI + transaction_context.pop().unwrap(); + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 4 + ); + + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 0 + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 5 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(5)) + ); + + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 3, + ); + + // Let's go to Instruction #1 (top level) + transaction_context.pop().unwrap(); + + // Instruction #1 + transaction_context + .configure_top_level_instruction_for_tests( + 0, + vec![InstructionAccount::new(1, false, false)], + Vec::new(), + ) + .unwrap(); + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 4, + ); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 3 + ); + + // Instruction #1 will do a CPI. + transaction_context + .configure_next_cpi_for_tests( + 0, + vec![InstructionAccount::new(2, false, true)], + Vec::new(), + ) + .unwrap(); + + transaction_context.push().unwrap(); + + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 5, + ); + + assert_eq!( + transaction_context.transaction_frame.total_number_of_instructions_in_trace, + 6 + ); + + assert_eq!( + transaction_context.transaction_frame.cpi_scratchpad.ptr(), + GUEST_INSTRUCTION_DATA_BASE_ADDRESS.saturating_add(GUEST_REGION_SIZE.saturating_mul(6)) + ); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 4 + ); + assert_eq!( + transaction_context.number_of_active_instructions_in_trace(), + 6 + ); + + // Return from CPI + transaction_context.pop().unwrap(); + assert_eq!( + transaction_context.transaction_frame.number_of_cpis_in_trace, + 4 + ); + assert_eq!( + transaction_context.transaction_frame.current_executing_instruction, + 4, + ); + + transaction_context.pop().unwrap(); + } + + #[test] + fn test_get_current_instruction_index() { + let transaction_accounts = vec![(Pubkey::new_unique(), AccountSharedData::default()); 3]; + let mut transaction_context = + TransactionContext::new(transaction_accounts, Rent::default(), 20, 20, 2); + + // First top level instruction + transaction_context + .configure_instruction_at_index( + 0, + 1, + vec![ + InstructionAccount::new(0, false, false), + InstructionAccount::new(1, false, false), + ], + vec![u16::MAX; 256], + Cow::Owned(Vec::new()), + None, + ) + .unwrap(); + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.get_current_instruction_index().unwrap(), + 0 + ); + transaction_context.pop().unwrap(); + + // Second top-level instruction + transaction_context + .configure_instruction_at_index( + 1, + 1, + vec![ + InstructionAccount::new(0, false, false), + InstructionAccount::new(1, false, true), + ], + vec![u16::MAX; 256], + Cow::Owned(Vec::new()), + None, + ) + .unwrap(); + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.get_current_instruction_index().unwrap(), + 1 + ); + + // Simulating a CPI + transaction_context + .configure_next_cpi_for_tests( + 1, + vec![ + InstructionAccount::new(0, false, true), + InstructionAccount::new(1, false, false), + ], + Vec::new(), + ) + .unwrap(); + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.get_current_instruction_index().unwrap(), + 2 + ); + + // Yet another CPI + transaction_context + .configure_next_cpi_for_tests( + 1, + vec![ + InstructionAccount::new(0, false, true), + InstructionAccount::new(1, false, false), + ], + Vec::new(), + ) + .unwrap(); + transaction_context.push().unwrap(); + assert_eq!( + transaction_context.get_current_instruction_index().unwrap(), + 3 + ); + + // CPI return + transaction_context.pop().unwrap(); + assert_eq!( + transaction_context.get_current_instruction_index().unwrap(), + 2 + ); + + // CPI return 2 + transaction_context.pop().unwrap(); + assert_eq!( + transaction_context.get_current_instruction_index().unwrap(), + 1 + ); + } +} diff --git a/solana/transaction-context/src/transaction_accounts.rs b/solana/transaction-context/src/transaction_accounts.rs new file mode 100644 index 00000000..957dcf27 --- /dev/null +++ b/solana/transaction-context/src/transaction_accounts.rs @@ -0,0 +1,529 @@ +use { + crate::{IndexOfAccount, MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION, MAX_ACCOUNT_DATA_LEN}, + solana_account::AccountSharedData, + solana_instruction::error::InstructionError, + solana_pubkey::Pubkey, + std::{ + cell::{Cell, UnsafeCell}, + ops::{Deref, DerefMut}, + }, +}; + +#[derive(Debug, PartialEq)] +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct TransactionAccountView<'a> { + account: &'a AccountSharedData, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl Deref for TransactionAccountView<'_> { + type Target = AccountSharedData; + fn deref(&self) -> &Self::Target { + self.account + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl PartialEq for TransactionAccountView<'_> { + fn eq(&self, other: &AccountSharedData) -> bool { + self.account == other + } +} + +#[derive(Debug)] +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct TransactionAccountViewMut<'a> { + account: &'a mut AccountSharedData, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl TransactionAccountViewMut<'_> { + pub(crate) fn reserve(&mut self, additional: usize) { + self.account.cow_mut().reserve(additional); + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl Deref for TransactionAccountViewMut<'_> { + type Target = AccountSharedData; + fn deref(&self) -> &Self::Target { + self.account + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl DerefMut for TransactionAccountViewMut<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.account + } +} + +// +/// An account key and the matching account +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub type KeyedAccountSharedData = (Pubkey, AccountSharedData); +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub(crate) type DeconstructedTransactionAccounts = + (Vec, Box<[Cell]>, Cell); + +#[derive(Debug)] +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct TransactionAccounts { + accounts: Box<[UnsafeCell]>, + borrow_counters: Box<[BorrowCounter]>, + touched_flags: Box<[Cell]>, + resize_delta: Cell, + lamports_delta: Cell, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl TransactionAccounts { + pub(crate) fn new(accounts: Vec) -> TransactionAccounts { + let touched_flags = vec![Cell::new(false); accounts.len()].into_boxed_slice(); + let borrow_counters = vec![BorrowCounter::default(); accounts.len()].into_boxed_slice(); + let accounts = + accounts.into_iter().map(UnsafeCell::new).collect::>().into_boxed_slice(); + + TransactionAccounts { + accounts, + borrow_counters, + touched_flags, + resize_delta: Cell::new(0), + lamports_delta: Cell::new(0), + } + } + + pub(crate) fn len(&self) -> usize { + self.accounts.len() + } + + pub fn touch(&self, index: IndexOfAccount) -> Result<(), InstructionError> { + self.touched_flags + .get(index as usize) + .ok_or(InstructionError::MissingAccount)? + .set(true); + Ok(()) + } + + pub(crate) fn update_accounts_resize_delta( + &self, + old_len: usize, + new_len: usize, + ) -> Result<(), InstructionError> { + let accounts_resize_delta = self.resize_delta.get(); + self.resize_delta.set( + accounts_resize_delta.saturating_add((new_len as i64).saturating_sub(old_len as i64)), + ); + Ok(()) + } + + pub(crate) fn can_data_be_resized( + &self, + old_len: usize, + new_len: usize, + ) -> Result<(), InstructionError> { + // The new length can not exceed the maximum permitted length + if new_len > MAX_ACCOUNT_DATA_LEN as usize { + return Err(InstructionError::InvalidRealloc); + } + // The resize can not exceed the per-transaction maximum + let length_delta = (new_len as i64).saturating_sub(old_len as i64); + if self.resize_delta.get().saturating_add(length_delta) + > MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION + { + return Err(InstructionError::MaxAccountsDataAllocationsExceeded); + } + Ok(()) + } + + pub fn try_borrow_mut( + &self, + index: IndexOfAccount, + ) -> Result, InstructionError> { + let borrow_counter = self + .borrow_counters + .get(index as usize) + .ok_or(InstructionError::MissingAccount)?; + borrow_counter.try_borrow_mut()?; + + // SAFETY: The borrow counter guarantees this is the only mutable borrow of this account. + // The unwrap is safe because accounts.len() == borrow_counters.len(), so the missing + // account error should have been returned above. + let account = TransactionAccountViewMut { + account: unsafe { &mut (*self.accounts.get(index as usize).unwrap().get()).1 }, + }; + + Ok(AccountRefMut { account, borrow_counter }) + } + + pub fn try_borrow(&self, index: IndexOfAccount) -> Result, InstructionError> { + let borrow_counter = self + .borrow_counters + .get(index as usize) + .ok_or(InstructionError::MissingAccount)?; + borrow_counter.try_borrow()?; + + // SAFETY: The borrow counter guarantees there are no mutable borrow of this account. + // The unwrap is safe because accounts.len() == borrow_counters.len(), so the missing + // account error should have been returned above. + let keyed_account = unsafe { &*self.accounts.get(index as usize).unwrap().get() }; + + let account = TransactionAccountView { account: &keyed_account.1 }; + + Ok(AccountRef { account, borrow_counter }) + } + + pub(crate) fn add_lamports_delta(&self, balance: i128) -> Result<(), InstructionError> { + let delta = self.lamports_delta.get(); + self.lamports_delta + .set(delta.checked_add(balance).ok_or(InstructionError::ArithmeticOverflow)?); + Ok(()) + } + + pub(crate) fn get_lamports_delta(&self) -> i128 { + self.lamports_delta.get() + } + + fn drain_accounts(&mut self) -> Box<[UnsafeCell]> { + debug_assert_eq!(self.accounts.len(), self.borrow_counters.len()); + debug_assert_eq!(self.accounts.len(), self.touched_flags.len()); + std::mem::take(&mut self.accounts) + } + + fn deconstruct_into_keyed_account_shared_data(&mut self) -> Vec { + self.drain_accounts().into_iter().map(UnsafeCell::into_inner).collect() + } + + pub(crate) fn deconstruct_into_account_shared_data(&mut self) -> Vec { + self.drain_accounts().into_iter().map(|cell| cell.into_inner().1).collect() + } + + pub(crate) fn take(mut self) -> DeconstructedTransactionAccounts { + let shared_data = self.deconstruct_into_keyed_account_shared_data(); + (shared_data, self.touched_flags, self.resize_delta) + } + + pub fn resize_delta(&self) -> i64 { + self.resize_delta.get() + } + + pub(crate) fn account_key(&self, index: IndexOfAccount) -> Option<&Pubkey> { + // SAFETY: We never modify an account key, so returning a reference to it is safe. + unsafe { self.accounts.get(index as usize).map(|acc| &(*acc.get()).0) } + } + + pub(crate) fn account_keys_iter(&self) -> impl Iterator { + // SAFETY: We never modify account keys, so returning an immutable reference to them is safe. + unsafe { self.accounts.iter().map(|item| &(*item.get()).0) } + } +} + +#[derive(Default, Debug, Clone)] +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +struct BorrowCounter { + counter: Cell, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl BorrowCounter { + #[inline] + fn is_writing(&self) -> bool { + self.counter.get() < 0 + } + + #[inline] + fn is_reading(&self) -> bool { + self.counter.get() > 0 + } + + #[inline] + fn try_borrow(&self) -> Result<(), InstructionError> { + if self.is_writing() { + return Err(InstructionError::AccountBorrowFailed); + } + + if let Some(counter) = self.counter.get().checked_add(1) { + self.counter.set(counter); + return Ok(()); + } + + Err(InstructionError::AccountBorrowFailed) + } + + #[inline] + fn try_borrow_mut(&self) -> Result<(), InstructionError> { + if self.is_writing() || self.is_reading() { + return Err(InstructionError::AccountBorrowFailed); + } + + self.counter.set(self.counter.get().saturating_sub(1)); + + Ok(()) + } + + #[inline] + fn release_borrow(&self) { + self.counter.set(self.counter.get().saturating_sub(1)); + } + + #[inline] + fn release_borrow_mut(&self) { + self.counter.set(self.counter.get().saturating_add(1)); + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct AccountRef<'a> { + account: TransactionAccountView<'a>, + borrow_counter: &'a BorrowCounter, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl Drop for AccountRef<'_> { + fn drop(&mut self) { + self.borrow_counter.release_borrow(); + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl<'a> Deref for AccountRef<'a> { + type Target = TransactionAccountView<'a>; + fn deref(&self) -> &Self::Target { + &self.account + } +} + +#[derive(Debug)] +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +pub struct AccountRefMut<'a> { + account: TransactionAccountViewMut<'a>, + borrow_counter: &'a BorrowCounter, +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl Drop for AccountRefMut<'_> { + fn drop(&mut self) { + self.borrow_counter.release_borrow_mut(); + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl<'a> Deref for AccountRefMut<'a> { + type Target = TransactionAccountViewMut<'a>; + fn deref(&self) -> &Self::Target { + &self.account + } +} + +#[cfg(not(any(target_arch = "bpf", target_arch = "sbf")))] +impl DerefMut for AccountRefMut<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.account + } +} + +#[cfg(all(test, not(target_arch = "sbf"), not(target_arch = "bpf")))] +mod tests { + use { + crate::transaction_accounts::TransactionAccounts, + solana_account::{ + AccountBuilder, AccountFieldPatch, AccountMode, AccountSharedData, DirtyMarkers, + ReadableAccount, StateFlags, WritableAccount, + }, + solana_instruction::error::InstructionError, + solana_pubkey::Pubkey, + }; + + #[test] + fn test_missing_account() { + let accounts = vec![ + ( + Pubkey::new_unique(), + AccountSharedData::new(2, 1, &Pubkey::new_unique()), + ), + ( + Pubkey::new_unique(), + AccountSharedData::new(2, 1, &Pubkey::new_unique()), + ), + ]; + + let tx_accounts = TransactionAccounts::new(accounts); + + let res = tx_accounts.try_borrow(3); + assert_eq!(res.err(), Some(InstructionError::MissingAccount)); + + let res = tx_accounts.try_borrow_mut(3); + assert_eq!(res.err(), Some(InstructionError::MissingAccount)); + } + + #[test] + fn test_invalid_borrow() { + let accounts = vec![ + ( + Pubkey::new_unique(), + AccountSharedData::new(2, 1, &Pubkey::new_unique()), + ), + ( + Pubkey::new_unique(), + AccountSharedData::new(2, 1, &Pubkey::new_unique()), + ), + ]; + + let tx_accounts = TransactionAccounts::new(accounts); + + // Two immutable borrows are valid + { + let acc_1 = tx_accounts.try_borrow(0); + assert!(acc_1.is_ok()); + + let acc_2 = tx_accounts.try_borrow(1); + assert!(acc_2.is_ok()); + + let acc_1_new = tx_accounts.try_borrow(0); + assert!(acc_1_new.is_ok()); + + assert_eq!(acc_1.unwrap().account, acc_1_new.unwrap().account); + } + + // Two mutable borrows are invalid + { + let acc_1 = tx_accounts.try_borrow_mut(0); + assert!(acc_1.is_ok()); + + let acc_2 = tx_accounts.try_borrow_mut(1); + assert!(acc_2.is_ok()); + + let acc_1_new = tx_accounts.try_borrow_mut(0); + assert_eq!(acc_1_new.err(), Some(InstructionError::AccountBorrowFailed)); + } + + // Mutable after immutable must fail + { + let acc_1 = tx_accounts.try_borrow(0); + assert!(acc_1.is_ok()); + + let acc_2 = tx_accounts.try_borrow(1); + assert!(acc_2.is_ok()); + + let acc_1_new = tx_accounts.try_borrow_mut(0); + assert_eq!(acc_1_new.err(), Some(InstructionError::AccountBorrowFailed)); + } + + // Immutable after mutable must fail + { + let acc_1 = tx_accounts.try_borrow_mut(0); + assert!(acc_1.is_ok()); + + let acc_2 = tx_accounts.try_borrow_mut(1); + assert!(acc_2.is_ok()); + + let acc_1_new = tx_accounts.try_borrow(0); + assert_eq!(acc_1_new.err(), Some(InstructionError::AccountBorrowFailed)); + } + + // Different scopes are good + { + let acc_1 = tx_accounts.try_borrow_mut(0); + assert!(acc_1.is_ok()); + } + + { + let acc_1 = tx_accounts.try_borrow_mut(0); + assert!(acc_1.is_ok()); + } + } + + #[test] + fn too_many_borrows() { + let accounts = vec![ + ( + Pubkey::new_unique(), + AccountSharedData::new(2, 1, &Pubkey::new_unique()), + ), + ( + Pubkey::new_unique(), + AccountSharedData::new(2, 1, &Pubkey::new_unique()), + ), + ]; + + let tx_accounts = TransactionAccounts::new(accounts); + let mut borrows = Vec::new(); + for i in 0..129 { + let acc = tx_accounts.try_borrow(1); + if i < 127 { + assert!(acc.is_ok()); + borrows.push(acc.unwrap()); + } else { + assert_eq!(acc.err(), Some(InstructionError::AccountBorrowFailed)); + } + } + } + + #[test] + fn preserves_account_shared_data_on_deconstruct() { + let key = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let mut account = AccountBuilder::default() + .lamports(23) + .data(vec![1, 2, 3]) + .owner(owner) + .mode(AccountMode::ReadOnly) + .slot(41) + .executable(true) + .build::(); + + AccountFieldPatch::Mode(AccountMode::Ephemeral).apply(&mut account).unwrap(); + AccountFieldPatch::Slot(42).apply(&mut account).unwrap(); + account.set_flags(StateFlags::EXECUTABLE); + AccountFieldPatch::DataAt { offset: 0, data: vec![4, 5] } + .apply(&mut account) + .unwrap(); + + let expected_markers = account.markers().bits(); + let mut tx_accounts = TransactionAccounts::new(vec![(key, account)]); + let mut accounts = tx_accounts.deconstruct_into_account_shared_data(); + let account = accounts.pop().unwrap(); + + assert!(accounts.is_empty()); + assert!(account.is(AccountMode::Ephemeral)); + assert_eq!(account.slot(), 42); + assert!(account.executable()); + assert_eq!(account.data(), &[4, 5, 3]); + assert_eq!(account.markers().bits(), expected_markers); + } + + #[test] + fn mutable_view_updates_account_shared_data() { + let key = Pubkey::new_unique(); + let owner = Pubkey::new_unique(); + let new_owner = Pubkey::new_unique(); + let tx_accounts = + TransactionAccounts::new(vec![(key, AccountSharedData::new(7, 2, &owner))]); + + { + let mut account = tx_accounts.try_borrow_mut(0).unwrap(); + account.set_lamports(11); + account.set_owner(new_owner); + account.set_executable(true); + account.resize(4, 9); + assert_eq!(account.data(), &[0, 0, 9, 9]); + account.set_data_from_slice(&[1, 2, 3]); + account.extend_from_slice(&[4, 5]); + account.data_as_mut_slice()[0] = 8; + } + + let mut tx_accounts = tx_accounts; + let mut accounts = tx_accounts.deconstruct_into_account_shared_data(); + let account = accounts.pop().unwrap(); + + assert!(accounts.is_empty()); + assert_eq!(account.lamports(), 11); + assert_eq!(account.owner(), &new_owner); + assert!(account.executable()); + assert_eq!(account.data(), &[8, 2, 3, 4, 5]); + assert!(account.markers().contains(DirtyMarkers::LAMPORTS)); + assert!(account.markers().contains(DirtyMarkers::OWNER)); + assert!(account.markers().contains(DirtyMarkers::FLAGS)); + assert!(account.markers().contains(DirtyMarkers::DATA)); + } +} diff --git a/solana/transaction-context/src/vm_addresses.rs b/solana/transaction-context/src/vm_addresses.rs new file mode 100644 index 00000000..c2bb03f1 --- /dev/null +++ b/solana/transaction-context/src/vm_addresses.rs @@ -0,0 +1,4 @@ +pub(crate) const GUEST_REGION_SIZE: u64 = 1 << 32; +pub(crate) const RETURN_DATA_SCRATCHPAD: u64 = 7 * GUEST_REGION_SIZE; +pub(crate) const GUEST_INSTRUCTION_DATA_BASE_ADDRESS: u64 = 264 * GUEST_REGION_SIZE; +pub(crate) const GUEST_INSTRUCTION_ACCOUNT_BASE_ADDRESS: u64 = 328 * GUEST_REGION_SIZE; diff --git a/solana/transaction-context/src/vm_slice.rs b/solana/transaction-context/src/vm_slice.rs new file mode 100644 index 00000000..a98566c5 --- /dev/null +++ b/solana/transaction-context/src/vm_slice.rs @@ -0,0 +1,54 @@ +// The VmSlice class is used for cases when you need a slice that is stored in the BPF +// interpreter's virtual address space. Because this source code can be compiled with +// addresses of different bit depths, we cannot assume that the 64-bit BPF interpreter's +// pointer sizes can be mapped to physical pointer sizes. In particular, if you need a +// slice-of-slices in the virtual space, the inner slices will be different sizes in a +// 32-bit app build than in the 64-bit virtual space. Therefore instead of a slice-of-slices, +// you should implement a slice-of-VmSlices, which can then use VmSlice::translate() to +// map to the physical address. +// This class must consist only of 16 bytes: a u64 ptr and a u64 len, to match the 64-bit +// implementation of a slice in Rust. The PhantomData entry takes up 0 bytes. + +use std::marker::PhantomData; + +#[repr(C)] +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct VmSlice { + ptr: u64, + len: u64, + resource_type: PhantomData, +} + +impl VmSlice { + pub fn new(ptr: u64, len: u64) -> Self { + VmSlice { + ptr, + len, + resource_type: PhantomData, + } + } + + pub fn ptr(&self) -> u64 { + self.ptr + } + + pub fn len(&self) -> u64 { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } + + pub fn end(&self) -> u64 { + self.ptr().saturating_add(self.len().saturating_mul(size_of::() as u64)) + } + + /// # Safety + /// Set a new length for the mapped area. + /// This function is not safe to use if not coupled with the respective change in + /// the underlying vector. + pub unsafe fn set_len(&mut self, new_len: u64) { + self.len = new_len; + } +} diff --git a/solana/transaction-view/Cargo.toml b/solana/transaction-view/Cargo.toml new file mode 100644 index 00000000..cf632939 --- /dev/null +++ b/solana/transaction-view/Cargo.toml @@ -0,0 +1,50 @@ +[package] +authors = { workspace = true } +description = "Zero-copy parser and sanitizer for serialized Solana transactions" +documentation = "https://docs.rs/agave-transaction-view" +edition = "2024" +homepage = { workspace = true } +license = { workspace = true } +name = "agave-transaction-view" +readme = "README.md" +repository = { workspace = true } +version = "4.1.1" + +[features] +agave-unstable-api = [] +dev-context-only-utils = [] + +[dependencies] +solana-hash = { workspace = true } +solana-message = { workspace = true } +solana-packet = { workspace = true } +solana-program-runtime = { workspace = true } +solana-pubkey = { workspace = true } +solana-sdk-ids = { workspace = true } +solana-short-vec = { workspace = true } +solana-signature = { workspace = true } +solana-svm-transaction = { workspace = true } +solana-transaction = { workspace = true } +solana-transaction-context = { workspace = true } + +[dev-dependencies] +# See order-crates-for-publishing.py for using this unusual `path = "."` +agave-transaction-view = { path = ".", features = ["agave-unstable-api", "dev-context-only-utils"] } +bincode = { workspace = true } +criterion = { workspace = true } +solana-instruction = { workspace = true } +solana-keypair = { workspace = true } +solana-message = { workspace = true, features = ["serde"] } +solana-signature = { workspace = true, features = ["serde"] } +solana-signer = { workspace = true } +solana-system-interface = { workspace = true, features = ["wincode"] } +solana-transaction = { workspace = true, features = ["serde", "wincode"] } +wincode = { workspace = true } + +[[bench]] +harness = false +name = "bytes" + +[[bench]] +harness = false +name = "transaction_view" diff --git a/solana/transaction-view/README.md b/solana/transaction-view/README.md new file mode 100644 index 00000000..f8ca07fb --- /dev/null +++ b/solana/transaction-view/README.md @@ -0,0 +1,56 @@ +# `agave-transaction-view` + +This Agave fork parses and sanitizes serialized transactions without fully +deserializing them. Workspace `[patch.crates-io]` entries force the dependency +graph to use this copy. + +The view owns or borrows the original transaction bytes through +`TransactionData` and caches only framing metadata. Accessors and iterators then +read signatures, account keys, instructions, configuration, and address-table +metadata directly from the validated byte layout. + +## Supported formats + +- Legacy and v0 use the standard Solana wire layouts. +- V1 uses the Agave V1 layout with instruction headers, contiguous payloads, + transaction configuration, and trailing signatures. +- `Magicblock` is the Engine-private version `127`. It uses the V1 layout and a + distinct version prefix; it is not a client-facing Solana transaction version. + +For V1 and Magicblock transactions, `message_data()` covers the version byte +through the instruction payloads and excludes the trailing signatures. Engine +construction must sign exactly that range after writing the final version +prefix. + +## Limits and parsing invariants + +Legacy, v0, and V1 transactions may be at most `u16::MAX` bytes, inclusive. +Magicblock transactions may be at most 16 MiB and may use an instruction trace +length of 255. Legacy and v0 framing requires between 1 and 12 signatures; V1 +and Magicblock enforce the signature limit during sanitization. Other structural +limits, including account-index limits, are also enforced by sanitization. + +Compact-u16 values are parsed in their complete canonical one-, two-, or +three-byte form. Absolute offsets and transaction lengths are stored as `u32`. +Fallible parsing validates additions, multiplications, ranges, and conversion to +that representation before any unchecked iterator or typed-slice access. Code +using a sanitized view may rely on those validated frame boundaries. + +## Address lookup tables + +Address lookup tables are unsupported. A transaction containing any lookup +table entry fails sanitization with `TransactionViewError::AddressLookupMismatch`. +A v0 transaction with an empty lookup list remains valid and requires no loaded +addresses. + +## Maintenance constraints + +- Preserve standard Legacy and v0 wire compatibility for client-produced + transactions. +- Keep V1 and Magicblock framing synchronized; only the version, size, account, + and instruction-trace policies intentionally differ. +- Keep the Magicblock prefix, signed message range, and Engine transaction + composer synchronized. +- Treat the sanitizer as the authoritative boundary for rejecting address + lookup tables. +- Validate new frame offsets before exposing them through unchecked accessors. diff --git a/solana/transaction-view/benches/bytes.rs b/solana/transaction-view/benches/bytes.rs new file mode 100644 index 00000000..713f6f1a --- /dev/null +++ b/solana/transaction-view/benches/bytes.rs @@ -0,0 +1,67 @@ +use { + agave_transaction_view::bytes::read_compressed_u16, + bincode::{DefaultOptions, Options, serialize_into}, + criterion::{Criterion, Throughput, criterion_group, criterion_main}, + solana_packet::PACKET_DATA_SIZE, + solana_short_vec::{ShortU16, decode_shortu16_len}, + std::hint::black_box, +}; + +fn setup() -> Vec<(u16, usize, Vec)> { + let options = DefaultOptions::new().with_fixint_encoding(); // Ensure fixed-int encoding + + // Create a vector of all valid u16 values serialized into 16-byte buffers. + let mut values = Vec::with_capacity(PACKET_DATA_SIZE); + for value in 0..PACKET_DATA_SIZE as u16 { + let short_u16 = ShortU16(value); + let mut buffer = vec![0u8; 16]; + let serialized_len = + options.serialized_size(&short_u16).expect("Failed to get serialized size"); + serialize_into(&mut buffer[..], &short_u16).expect("Serialization failed"); + values.push((value, serialized_len as usize, buffer)); + } + + values +} + +fn bench_u16_parsing(c: &mut Criterion) { + let values_serialized_lengths_and_buffers = setup(); + let mut group = c.benchmark_group("compressed_u16_parsing"); + group.throughput(Throughput::Elements( + values_serialized_lengths_and_buffers.len() as u64, + )); + + // Benchmark the decode_shortu16_len function from `solana-sdk` + group.bench_function("short_u16_decode", |c| { + c.iter(|| { + decode_shortu16_len_iter(&values_serialized_lengths_and_buffers); + }) + }); + + // Benchmark `read_compressed_u16` + group.bench_function("read_compressed_u16", |c| { + c.iter(|| { + read_compressed_u16_iter(&values_serialized_lengths_and_buffers); + }) + }); +} + +fn decode_shortu16_len_iter(values_serialized_lengths_and_buffers: &[(u16, usize, Vec)]) { + for (value, serialized_len, buffer) in values_serialized_lengths_and_buffers.iter() { + let (read_value, bytes_read) = decode_shortu16_len(black_box(buffer)).unwrap(); + assert_eq!(read_value, *value as usize, "Value mismatch for: {value}"); + assert_eq!(bytes_read, *serialized_len, "Offset mismatch for: {value}"); + } +} + +fn read_compressed_u16_iter(values_serialized_lengths_and_buffers: &[(u16, usize, Vec)]) { + for (value, serialized_len, buffer) in values_serialized_lengths_and_buffers.iter() { + let mut offset = 0; + let read_value = read_compressed_u16(black_box(buffer), &mut offset).unwrap(); + assert_eq!(read_value, *value, "Value mismatch for: {value}"); + assert_eq!(offset, *serialized_len, "Offset mismatch for: {value}"); + } +} + +criterion_group!(benches, bench_u16_parsing); +criterion_main!(benches); diff --git a/solana/transaction-view/benches/transaction_view.rs b/solana/transaction-view/benches/transaction_view.rs new file mode 100644 index 00000000..6a94dc3e --- /dev/null +++ b/solana/transaction-view/benches/transaction_view.rs @@ -0,0 +1,230 @@ +use { + agave_transaction_view::transaction_view::TransactionView, + criterion::{ + BenchmarkGroup, Criterion, Throughput, criterion_group, criterion_main, + measurement::Measurement, + }, + solana_hash::Hash, + solana_instruction::Instruction, + solana_keypair::Keypair, + solana_message::{ + Message, MessageHeader, VersionedMessage, + v0::{self, MessageAddressTableLookup}, + }, + solana_pubkey::Pubkey, + solana_signer::Signer, + solana_system_interface::instruction as system_instruction, + solana_transaction::versioned::{ + VersionedTransaction, sanitized::SanitizedVersionedTransaction, + }, + std::hint::black_box, +}; + +const NUM_TRANSACTIONS: usize = 1024; + +fn serialize_transactions(transactions: Vec) -> Vec> { + transactions + .into_iter() + .map(|transaction| wincode::serialize(&transaction).unwrap()) + .collect() +} + +fn bench_transactions_parsing( + group: &mut BenchmarkGroup, + serialized_transactions: Vec>, +) { + // Legacy Transaction Parsing + group.bench_function("VersionedTransaction", |c| { + c.iter(|| { + for bytes in serialized_transactions.iter() { + let _ = wincode::deserialize::(black_box(bytes)).unwrap(); + } + }); + }); + + // Legacy Transaction Parsing and Sanitize checks + group.bench_function("SanitizedVersionedTransaction", |c| { + c.iter(|| { + for bytes in serialized_transactions.iter() { + let tx = wincode::deserialize::(black_box(bytes)).unwrap(); + let _ = SanitizedVersionedTransaction::try_new(tx).unwrap(); + } + }); + }); + + // New Transaction Parsing + group.bench_function("TransactionView", |c| { + c.iter(|| { + for bytes in serialized_transactions.iter() { + let _ = TransactionView::try_new_unsanitized(black_box(bytes.as_ref())).unwrap(); + } + }); + }); + + // New Transaction Parsing and Sanitize checks + group.bench_function("TransactionView (Sanitized)", |c| { + c.iter(|| { + for bytes in serialized_transactions.iter() { + let _ = + TransactionView::try_new_sanitized(black_box(bytes.as_ref()), true).unwrap(); + } + }); + }); +} + +fn minimum_sized_transactions() -> Vec { + (0..NUM_TRANSACTIONS) + .map(|_| { + let keypair = Keypair::new(); + VersionedTransaction::try_new( + VersionedMessage::Legacy(Message::new_with_blockhash( + &[], + Some(&keypair.pubkey()), + &Hash::default(), + )), + &[&keypair], + ) + .unwrap() + }) + .collect() +} + +fn simple_transfers() -> Vec { + (0..NUM_TRANSACTIONS) + .map(|_| { + let keypair = Keypair::new(); + VersionedTransaction::try_new( + VersionedMessage::Legacy(Message::new_with_blockhash( + &[system_instruction::transfer(&keypair.pubkey(), &Pubkey::new_unique(), 1)], + Some(&keypair.pubkey()), + &Hash::default(), + )), + &[&keypair], + ) + .unwrap() + }) + .collect() +} + +fn packed_transfers() -> Vec { + // Creating transfer instructions between same keys to maximize the number + // of transfers per transaction. We can fit up to 60 transfers. + const MAX_TRANSFERS_PER_TX: usize = 60; + + (0..NUM_TRANSACTIONS) + .map(|_| { + let keypair = Keypair::new(); + let to_pubkey = Pubkey::new_unique(); + let ixs = system_instruction::transfer_many( + &keypair.pubkey(), + &vec![(to_pubkey, 1); MAX_TRANSFERS_PER_TX], + ); + VersionedTransaction::try_new( + VersionedMessage::Legacy(Message::new(&ixs, Some(&keypair.pubkey()))), + &[&keypair], + ) + .unwrap() + }) + .collect() +} + +fn packed_noops() -> Vec { + // Creating noop instructions to maximize the number of instructions per + // transaction. We are allowed to fit up to 64 instructions per transaction. + const MAX_INSTRUCTIONS_PER_TRANSACTION: usize = 64; + + (0..NUM_TRANSACTIONS) + .map(|_| { + let keypair = Keypair::new(); + let program_id = Pubkey::new_unique(); + let ixs = (0..MAX_INSTRUCTIONS_PER_TRANSACTION) + .map(|_| Instruction::new_with_bytes(program_id, &[], vec![])); + VersionedTransaction::try_new( + VersionedMessage::Legacy(Message::new( + &ixs.collect::>(), + Some(&keypair.pubkey()), + )), + &[&keypair], + ) + .unwrap() + }) + .collect() +} + +fn packed_atls() -> Vec { + // Creating ATLs to maximize the number of ATLS per transaction. We can fit + // up to 31. + const MAX_ATLS_PER_TRANSACTION: usize = 31; + + (0..NUM_TRANSACTIONS) + .map(|_| { + let keypair = Keypair::new(); + VersionedTransaction::try_new( + VersionedMessage::V0(v0::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![keypair.pubkey()], + recent_blockhash: Hash::default(), + instructions: vec![], + address_table_lookups: Vec::from_iter((0..MAX_ATLS_PER_TRANSACTION).map( + |_| MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![0], + readonly_indexes: vec![], + }, + )), + }), + &[&keypair], + ) + .unwrap() + }) + .collect() +} + +fn bench_parse_min_sized_transactions(c: &mut Criterion) { + let serialized_transactions = serialize_transactions(minimum_sized_transactions()); + let mut group = c.benchmark_group("min sized transactions"); + group.throughput(Throughput::Elements(serialized_transactions.len() as u64)); + bench_transactions_parsing(&mut group, serialized_transactions); +} + +fn bench_parse_simple_transfers(c: &mut Criterion) { + let serialized_transactions = serialize_transactions(simple_transfers()); + let mut group = c.benchmark_group("simple transfers"); + group.throughput(Throughput::Elements(serialized_transactions.len() as u64)); + bench_transactions_parsing(&mut group, serialized_transactions); +} + +fn bench_parse_packed_transfers(c: &mut Criterion) { + let serialized_transactions = serialize_transactions(packed_transfers()); + let mut group = c.benchmark_group("packed transfers"); + group.throughput(Throughput::Elements(serialized_transactions.len() as u64)); + bench_transactions_parsing(&mut group, serialized_transactions); +} + +fn bench_parse_packed_noops(c: &mut Criterion) { + let serialized_transactions = serialize_transactions(packed_noops()); + let mut group = c.benchmark_group("packed noops"); + group.throughput(Throughput::Elements(serialized_transactions.len() as u64)); + bench_transactions_parsing(&mut group, serialized_transactions); +} + +fn bench_parse_packed_atls(c: &mut Criterion) { + let serialized_transactions = serialize_transactions(packed_atls()); + let mut group = c.benchmark_group("packed atls"); + group.throughput(Throughput::Elements(serialized_transactions.len() as u64)); + bench_transactions_parsing(&mut group, serialized_transactions); +} + +criterion_group!( + benches, + bench_parse_min_sized_transactions, + bench_parse_simple_transfers, + bench_parse_packed_transfers, + bench_parse_packed_noops, + bench_parse_packed_atls +); +criterion_main!(benches); diff --git a/solana/transaction-view/src/address_table_lookup_frame.rs b/solana/transaction-view/src/address_table_lookup_frame.rs new file mode 100644 index 00000000..e274bbd5 --- /dev/null +++ b/solana/transaction-view/src/address_table_lookup_frame.rs @@ -0,0 +1,314 @@ +use { + crate::{ + bytes::{ + advance_offset_for_array, advance_offset_for_type, check_remaining, read_byte, + read_compressed_u16, read_slice_data, read_type, try_u32_offset, + }, + result::{Result, TransactionViewError}, + }, + core::fmt::{Debug, Formatter}, + solana_hash::Hash, + solana_packet::PACKET_DATA_SIZE, + solana_pubkey::Pubkey, + solana_signature::Signature, + solana_svm_transaction::message_address_table_lookup::SVMMessageAddressTableLookup, +}; + +// Each ATL has at least a Pubkey, one byte for the number of write indexes, +// and one byte for the number of read indexes. Additionally, for validity +// the ATL must have at least one write or read index giving a minimum size +// of 35 bytes. +const MIN_SIZED_ATL: usize = { + core::mem::size_of::() // account key + + 1 // writable indexes length + + 1 // readonly indexes length + + 1 // single account (either write or read) +}; + +// A valid packet with ATLs has: +// 1. At least 1 signature +// 2. 1 message prefix byte +// 3. 3 bytes for the message header +// 4. 1 static account key +// 5. 1 recent blockhash +// 6. 1 byte for the number of instructions (0) +// 7. 1 byte for the number of ATLS +const MIN_SIZED_PACKET_WITH_ATLS: usize = { + 1 // signatures count + + core::mem::size_of::() // signature + + 1 // message prefix + + 3 // message header + + 1 // static account keys count + + core::mem::size_of::() // static account key + + core::mem::size_of::() // recent blockhash + + 1 // number of instructions + + 1 // number of ATLS +}; + +/// The maximum number of ATLS that can fit in a valid packet. +const MAX_ATLS_PER_PACKET: u8 = + ((PACKET_DATA_SIZE - MIN_SIZED_PACKET_WITH_ATLS) / MIN_SIZED_ATL) as u8; + +/// Contains metadata about the address table lookups in a transaction packet. +#[derive(Debug)] +pub(crate) struct AddressTableLookupFrame { + /// The number of address table lookups in the transaction. + pub(crate) num_address_table_lookups: u8, + /// The offset to the first address table lookup in the transaction. + pub(crate) offset: u32, + /// The total number of writable lookup accounts in the transaction. + pub(crate) total_writable_lookup_accounts: u16, + /// The total number of readonly lookup accounts in the transaction. + pub(crate) total_readonly_lookup_accounts: u16, +} + +impl AddressTableLookupFrame { + /// Get the number of address table lookups (ATL) and offset to the first. + /// The offset will be updated to point to the first byte after the last + /// ATL. + /// This function will parse each ATL to ensure the data is well-formed, + /// but will not cache data related to these ATLs. + #[inline(always)] + pub(crate) fn try_new(bytes: &[u8], offset: &mut usize) -> Result { + // Maximum number of ATLs should be represented by a single byte, + // thus the MSB should not be set. + const _: () = assert!(MAX_ATLS_PER_PACKET & 0b1000_0000 == 0); + let num_address_table_lookups = read_byte(bytes, offset)?; + if num_address_table_lookups > MAX_ATLS_PER_PACKET { + return Err(TransactionViewError::ParseError); + } + + // Check that the remaining bytes are enough to hold the ATLs. + check_remaining( + bytes, + *offset, + MIN_SIZED_ATL + .checked_mul(usize::from(num_address_table_lookups)) + .ok_or(TransactionViewError::ParseError)?, + )?; + + let address_table_lookups_offset = try_u32_offset(*offset)?; + + // Check that there is no chance of overflow when calculating the total + // number of writable and readonly lookup accounts using a u32. + const _: () = + assert!(u16::MAX as usize * MAX_ATLS_PER_PACKET as usize <= u32::MAX as usize); + let mut total_writable_lookup_accounts: u32 = 0; + let mut total_readonly_lookup_accounts: u32 = 0; + + // The ATLs do not have a fixed size. So we must iterate over + // each ATL to find the total size of the ATLs in the packet, + // and check for any malformed ATLs or buffer overflows. + for _index in 0..num_address_table_lookups { + // Each ATL has 3 pieces: + // 1. Address (Pubkey) + // 2. write indexes ([u8]) + // 3. read indexes ([u8]) + + // Advance offset for address of the lookup table. + advance_offset_for_type::(bytes, offset)?; + + // Read the number of write indexes, and then update the offset. + let num_write_accounts = read_compressed_u16(bytes, offset)?; + total_writable_lookup_accounts = total_writable_lookup_accounts + .checked_add(u32::from(num_write_accounts)) + .ok_or(TransactionViewError::ParseError)?; + advance_offset_for_array::(bytes, offset, num_write_accounts)?; + + // Read the number of read indexes, and then update the offset. + let num_read_accounts = read_compressed_u16(bytes, offset)?; + total_readonly_lookup_accounts = total_readonly_lookup_accounts + .checked_add(u32::from(num_read_accounts)) + .ok_or(TransactionViewError::ParseError)?; + advance_offset_for_array::(bytes, offset, num_read_accounts)?; + } + + Ok(Self { + num_address_table_lookups, + offset: address_table_lookups_offset, + total_writable_lookup_accounts: u16::try_from(total_writable_lookup_accounts) + .map_err(|_| TransactionViewError::SanitizeError)?, + total_readonly_lookup_accounts: u16::try_from(total_readonly_lookup_accounts) + .map_err(|_| TransactionViewError::SanitizeError)?, + }) + } +} + +#[derive(Clone)] +pub struct AddressTableLookupIterator<'a> { + pub(crate) bytes: &'a [u8], + pub(crate) offset: usize, + pub(crate) num_address_table_lookups: u8, + pub(crate) index: u8, +} + +impl<'a> Iterator for AddressTableLookupIterator<'a> { + type Item = SVMMessageAddressTableLookup<'a>; + + #[inline] + fn next(&mut self) -> Option { + if self.index < self.num_address_table_lookups { + self.index = self.index.wrapping_add(1); + + // Each ATL has 3 pieces: + // 1. Address (Pubkey) + // 2. write indexes ([u8]) + // 3. read indexes ([u8]) + + // Advance offset for address of the lookup table. + const _: () = assert!(core::mem::align_of::() == 1, "Pubkey alignment"); + // SAFETY: + // - The offset is checked to be valid in the slice. + // - The alignment of Pubkey is 1. + // - `Pubkey` is a byte array, it cannot be improperly initialized. + let account_key = unsafe { read_type::(self.bytes, &mut self.offset) }.ok()?; + + // Read the number of write indexes, and then update the offset. + let num_write_accounts = read_compressed_u16(self.bytes, &mut self.offset).ok()?; + + const _: () = assert!(core::mem::align_of::() == 1, "u8 alignment"); + // SAFETY: + // - The offset is checked to be valid in the byte slice. + // - The alignment of u8 is 1. + // - The slice length is checked to be valid. + // - `u8` cannot be improperly initialized. + let writable_indexes = + unsafe { read_slice_data::(self.bytes, &mut self.offset, num_write_accounts) } + .ok()?; + + // Read the number of read indexes, and then update the offset. + let num_read_accounts = read_compressed_u16(self.bytes, &mut self.offset).ok()?; + + const _: () = assert!(core::mem::align_of::() == 1, "u8 alignment"); + // SAFETY: + // - The offset is checked to be valid in the byte slice. + // - The alignment of u8 is 1. + // - The slice length is checked to be valid. + // - `u8` cannot be improperly initialized. + let readonly_indexes = + unsafe { read_slice_data::(self.bytes, &mut self.offset, num_read_accounts) } + .ok()?; + + Some(SVMMessageAddressTableLookup { + account_key, + writable_indexes, + readonly_indexes, + }) + } else { + None + } + } +} + +impl ExactSizeIterator for AddressTableLookupIterator<'_> { + fn len(&self) -> usize { + usize::from(self.num_address_table_lookups.wrapping_sub(self.index)) + } +} + +impl Debug for AddressTableLookupIterator<'_> { + fn fmt(&self, f: &mut Formatter) -> core::fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +#[cfg(test)] +mod tests { + use {super::*, solana_message::v0::MessageAddressTableLookup, solana_short_vec::ShortVec}; + + #[test] + fn test_zero_atls() { + let bytes = bincode::serialize(&ShortVec::(vec![])).unwrap(); + let mut offset = 0; + let frame = AddressTableLookupFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_address_table_lookups, 0); + assert_eq!(frame.offset, 1); + assert_eq!(offset, bytes.len()); + assert_eq!(frame.total_writable_lookup_accounts, 0); + assert_eq!(frame.total_readonly_lookup_accounts, 0); + } + + #[test] + fn test_length_too_high() { + let mut bytes = bincode::serialize(&ShortVec::(vec![])).unwrap(); + let mut offset = 0; + // modify the number of atls to be too high + bytes[0] = 5; + assert!(AddressTableLookupFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_single_atl() { + let bytes = bincode::serialize(&ShortVec::(vec![ + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![1, 2, 3], + readonly_indexes: vec![4, 5, 6], + }, + ])) + .unwrap(); + let mut offset = 0; + let frame = AddressTableLookupFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_address_table_lookups, 1); + assert_eq!(frame.offset, 1); + assert_eq!(offset, bytes.len()); + assert_eq!(frame.total_writable_lookup_accounts, 3); + assert_eq!(frame.total_readonly_lookup_accounts, 3); + } + + #[test] + fn test_multiple_atls() { + let bytes = bincode::serialize(&ShortVec::(vec![ + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![1, 2, 3], + readonly_indexes: vec![4, 5, 6], + }, + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![1, 2, 3], + readonly_indexes: vec![4, 5], + }, + ])) + .unwrap(); + let mut offset = 0; + let frame = AddressTableLookupFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_address_table_lookups, 2); + assert_eq!(frame.offset, 1); + assert_eq!(offset, bytes.len()); + assert_eq!(frame.total_writable_lookup_accounts, 6); + assert_eq!(frame.total_readonly_lookup_accounts, 5); + } + + #[test] + fn test_invalid_writable_indexes_vec() { + let mut bytes = bincode::serialize(&ShortVec(vec![MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![1, 2, 3], + readonly_indexes: vec![4, 5, 6], + }])) + .unwrap(); + + // modify the number of accounts to be too high + bytes[33] = 127; + + let mut offset = 0; + assert!(AddressTableLookupFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_invalid_readonly_indexes_vec() { + let mut bytes = bincode::serialize(&ShortVec(vec![MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![1, 2, 3], + readonly_indexes: vec![4, 5, 6], + }])) + .unwrap(); + + // modify the number of accounts to be too high + bytes[37] = 127; + + let mut offset = 0; + assert!(AddressTableLookupFrame::try_new(&bytes, &mut offset).is_err()); + } +} diff --git a/solana/transaction-view/src/bytes.rs b/solana/transaction-view/src/bytes.rs new file mode 100644 index 00000000..6a3eac49 --- /dev/null +++ b/solana/transaction-view/src/bytes.rs @@ -0,0 +1,343 @@ +use crate::result::{Result, TransactionViewError}; + +#[inline(always)] +pub(crate) fn try_u32_offset(offset: usize) -> Result { + u32::try_from(offset).map_err(|_| TransactionViewError::ParseError) +} + +/// Check that the buffer has at least `len` bytes remaining starting at +/// `offset`. Returns Err if the buffer is too short. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// * `num_bytes` - Number of bytes that must be remaining. +/// +#[inline(always)] +pub fn check_remaining(bytes: &[u8], offset: usize, num_bytes: usize) -> Result<()> { + let end = offset.checked_add(num_bytes).ok_or(TransactionViewError::ParseError)?; + (end <= bytes.len()).then_some(()).ok_or(TransactionViewError::ParseError) +} + +/// Check that the buffer has at least 1 byte remaining starting at `offset`. +/// Returns Err if the buffer is too short. +#[inline(always)] +pub fn read_byte(bytes: &[u8], offset: &mut usize) -> Result { + // Implicitly checks that the offset is within bounds, no need + // to call `check_remaining` explicitly here. + let value = bytes.get(*offset).copied().ok_or(TransactionViewError::ParseError)?; + *offset = offset.checked_add(1).ok_or(TransactionViewError::ParseError)?; + Ok(value) +} + +/// Read a byte and advance the offset without any bounds checks. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// +/// # Safety +/// 1. `bytes` must be a valid slice of bytes. +/// 2. `offset` must be a valid offset into `bytes`. +#[inline(always)] +pub unsafe fn unchecked_read_byte(bytes: &[u8], offset: &mut usize) -> u8 { + let value = unsafe { *bytes.get_unchecked(*offset) }; + *offset += 1; + value +} + +/// Read a compressed u16 from `bytes` starting at `offset`. +/// If the buffer is too short or the encoding is invalid, return Err. +/// `offset` is updated to point to the byte after the compressed u16. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// +/// Assumptions: +/// - The current offset is not greater than `bytes.len()`. +#[allow(dead_code)] +#[inline(always)] +pub fn read_compressed_u16(bytes: &[u8], offset: &mut usize) -> Result { + let mut result = 0u16; + let mut shift = 0u16; + + for i in 0..3 { + // Implicitly checks that the offset is within bounds, no need + // to call check_remaining explicitly here. + let index = offset.checked_add(i).ok_or(TransactionViewError::ParseError)?; + let byte = *bytes.get(index).ok_or(TransactionViewError::ParseError)?; + // non-minimal encoding or overflow + if (i > 0 && byte == 0) || (i == 2 && byte > 3) { + return Err(TransactionViewError::ParseError); + } + result |= ((byte & 0x7F) as u16) << shift; + shift += 7; + if byte & 0x80 == 0 { + *offset = index.checked_add(1).ok_or(TransactionViewError::ParseError)?; + return Ok(result); + } + } + + // if we reach here, it means that all 3 bytes were used + *offset = offset.checked_add(3).ok_or(TransactionViewError::ParseError)?; + Ok(result) +} + +/// Update the `offset` to point to the byte after an array of length `len` and +/// of type `T`. If the buffer is too short, return Err. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// * `num_elements` - Number of `T` elements in the array. +/// +/// Assumptions: +/// 1. The current offset is not greater than `bytes.len()`. +/// 2. The size of `T` is small enough such that a usize will not overflow if +/// given the maximum array size (u16::MAX). +#[inline(always)] +pub fn advance_offset_for_array( + bytes: &[u8], + offset: &mut usize, + num_elements: u16, +) -> Result<()> { + let array_len_bytes = usize::from(num_elements) + .checked_mul(core::mem::size_of::()) + .ok_or(TransactionViewError::ParseError)?; + check_remaining(bytes, *offset, array_len_bytes)?; + *offset = offset.checked_add(array_len_bytes).ok_or(TransactionViewError::ParseError)?; + Ok(()) +} + +/// Update the `offset` to point t the byte after the `T`. +/// If the buffer is too short, return Err. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// +/// Assumptions: +/// 1. The current offset is not greater than `bytes.len()`. +/// 2. The size of `T` is small enough such that a usize will not overflow. +#[inline(always)] +pub fn advance_offset_for_type(bytes: &[u8], offset: &mut usize) -> Result<()> { + let type_size = core::mem::size_of::(); + check_remaining(bytes, *offset, type_size)?; + *offset = offset.checked_add(type_size).ok_or(TransactionViewError::ParseError)?; + Ok(()) +} + +/// Return a reference to the next slice of `T` in the buffer, checking bounds +/// and advancing the offset. +/// If the buffer is too short, return Err. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// * `num_elements` - Number of `T` elements in the slice. +/// +/// # Safety +/// 1. `bytes` must be a valid slice of bytes. +/// 2. `offset` must be a valid offset into `bytes`. +/// 3. `bytes + offset` must be properly aligned for `T`. +/// 4. `T` slice must be validly initialized. +/// 5. The size of `T` is small enough such that a usize will not overflow if +/// given the maximum slice size (u16::MAX). +#[inline(always)] +pub unsafe fn read_slice_data<'a, T: Sized>( + bytes: &'a [u8], + offset: &mut usize, + num_elements: u16, +) -> Result<&'a [T]> { + let start = *offset; + advance_offset_for_array::(bytes, offset, num_elements)?; + let current_ptr = unsafe { bytes.as_ptr().add(start) }; + Ok(unsafe { core::slice::from_raw_parts(current_ptr as *const T, usize::from(num_elements)) }) +} + +/// Return a reference to the next slice of `T` in the buffer, +/// and advancing the offset. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// * `num_elements` - Number of `T` elements in the slice. +/// +/// # Safety +/// 1. `bytes` must be a valid slice of bytes. +/// 2. `offset` must be a valid offset into `bytes`. +/// 3. `bytes + offset` must be properly aligned for `T`. +/// 4. `T` slice must be validly initialized. +/// 5. The size of `T` is small enough such that a usize will not overflow if +/// given the maximum slice size (u16::MAX). +#[inline(always)] +pub unsafe fn unchecked_read_slice_data<'a, T: Sized>( + bytes: &'a [u8], + offset: &mut usize, + num_elements: u16, +) -> &'a [T] { + let current_ptr = unsafe { bytes.as_ptr().add(*offset) }; + let array_len_bytes = usize::from(num_elements) * core::mem::size_of::(); + *offset += array_len_bytes; + unsafe { core::slice::from_raw_parts(current_ptr as *const T, usize::from(num_elements)) } +} + +/// Return a reference to the next `T` in the buffer, checking bounds and +/// advancing the offset. +/// If the buffer is too short, return Err. +/// +/// * `bytes` - Slice of bytes to read from. +/// * `offset` - Current offset into `bytes`. +/// +/// # Safety +/// 1. `bytes` must be a valid slice of bytes. +/// 2. `offset` must be a valid offset into `bytes`. +/// 3. `bytes + offset` must be properly aligned for `T`. +/// 4. `T` must be validly initialized. +#[inline(always)] +pub unsafe fn read_type<'a, T: Sized>(bytes: &'a [u8], offset: &mut usize) -> Result<&'a T> { + let start = *offset; + advance_offset_for_type::(bytes, offset)?; + let current_ptr = unsafe { bytes.as_ptr().add(start) }; + Ok(unsafe { &*(current_ptr as *const T) }) +} + +/// Copy a `T` in the buffer without checking bounds or advancing offset. +/// +/// # Safety +/// 1. `bytes` must be a valid slice of bytes. +/// 2. `offset` must be a valid offset into `bytes`. +/// 4. `T` must be validly initialized. +#[inline(always)] +pub unsafe fn unchecked_copy_value(bytes: &[u8], offset: usize) -> T { + let current_ptr = unsafe { bytes.as_ptr().add(offset) }.cast::(); + unsafe { current_ptr.read_unaligned() } +} + +#[cfg(test)] +mod tests { + use { + super::*, + bincode::{DefaultOptions, Options, serialize_into}, + solana_short_vec::ShortU16, + }; + + #[test] + fn test_check_remaining() { + // Empty buffer checks + assert!(check_remaining(&[], 0, 0).is_ok()); + assert!(check_remaining(&[], 0, 1).is_err()); + + // Buffer with data checks + assert!(check_remaining(&[1, 2, 3], 0, 0).is_ok()); + assert!(check_remaining(&[1, 2, 3], 0, 1).is_ok()); + assert!(check_remaining(&[1, 2, 3], 0, 3).is_ok()); + assert!(check_remaining(&[1, 2, 3], 0, 4).is_err()); + + // Non-zero offset. + assert!(check_remaining(&[1, 2, 3], 1, 0).is_ok()); + assert!(check_remaining(&[1, 2, 3], 1, 1).is_ok()); + assert!(check_remaining(&[1, 2, 3], 1, 2).is_ok()); + assert!(check_remaining(&[1, 2, 3], 1, usize::MAX).is_err()); + } + + #[test] + fn test_read_byte() { + let bytes = [5, 6, 7]; + let mut offset = 0; + assert_eq!(read_byte(&bytes, &mut offset), Ok(5)); + assert_eq!(offset, 1); + assert_eq!(read_byte(&bytes, &mut offset), Ok(6)); + assert_eq!(offset, 2); + assert_eq!(read_byte(&bytes, &mut offset), Ok(7)); + assert_eq!(offset, 3); + assert!(read_byte(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_read_compressed_u16() { + let mut buffer = [0u8; 1024]; + let options = DefaultOptions::new().with_fixint_encoding(); // Ensure fixed-int encoding + + // Test all possible u16 values + for value in 0..=u16::MAX { + let mut offset; + let short_u16 = ShortU16(value); + + // Serialize the value into the buffer + serialize_into(&mut buffer[..], &short_u16).expect("Serialization failed"); + + // Use bincode's size calculation to determine the length of the serialized data + let serialized_len = + options.serialized_size(&short_u16).expect("Failed to get serialized size"); + + // Reset offset + offset = 0; + + // Read the value back using unchecked_read_u16_compressed + let read_value = read_compressed_u16(&buffer, &mut offset); + + // Assert that the read value matches the original value + assert_eq!(read_value, Ok(value), "Value mismatch for: {value}"); + + // Assert that the offset matches the serialized length + assert_eq!( + offset, serialized_len as usize, + "Offset mismatch for: {value}" + ); + } + + // Test bounds. + // All 0s => 0 + assert_eq!(Ok(0), read_compressed_u16(&[0; 3], &mut 0)); + // Overflow + assert!(read_compressed_u16(&[0xFF, 0xFF, 0x04], &mut 0).is_err()); + assert_eq!( + read_compressed_u16(&[0xFF, 0xFF, 0x03], &mut 0), + Ok(u16::MAX) + ); + + // overflow errors + assert!(read_compressed_u16(&[u8::MAX; 1], &mut 0).is_err()); + assert!(read_compressed_u16(&[u8::MAX; 2], &mut 0).is_err()); + + // Minimal encoding checks + assert!(read_compressed_u16(&[0x81, 0x80, 0x00], &mut 0).is_err()); + } + + #[test] + fn test_advance_offset_for_array() { + #[repr(C)] + struct MyStruct { + _a: u8, + _b: u8, + } + const _: () = assert!(core::mem::size_of::() == 2); + + // Test with a buffer that is too short + let bytes = [0u8; 1]; + let mut offset = 0; + assert!(advance_offset_for_array::(&bytes, &mut offset, 1).is_err()); + + // Test with a buffer that is long enough + let bytes = [0u8; 4]; + let mut offset = 0; + assert!(advance_offset_for_array::(&bytes, &mut offset, 2).is_ok()); + assert_eq!(offset, 4); + } + + #[test] + fn test_advance_offset_for_type() { + #[repr(C)] + struct MyStruct { + _a: u8, + _b: u8, + } + const _: () = assert!(core::mem::size_of::() == 2); + + // Test with a buffer that is too short + let bytes = [0u8; 1]; + let mut offset = 0; + assert!(advance_offset_for_type::(&bytes, &mut offset).is_err()); + + // Test with a buffer that is long enough + let bytes = [0u8; 4]; + let mut offset = 0; + assert!(advance_offset_for_type::(&bytes, &mut offset).is_ok()); + assert_eq!(offset, 2); + } +} diff --git a/solana/transaction-view/src/instructions_frame.rs b/solana/transaction-view/src/instructions_frame.rs new file mode 100644 index 00000000..dafe5ea6 --- /dev/null +++ b/solana/transaction-view/src/instructions_frame.rs @@ -0,0 +1,831 @@ +use { + crate::{ + bytes::{ + advance_offset_for_array, check_remaining, read_byte, read_compressed_u16, + try_u32_offset, unchecked_copy_value, unchecked_read_byte, unchecked_read_slice_data, + }, + result::{Result, TransactionViewError}, + }, + core::fmt::{Debug, Formatter}, + solana_svm_transaction::instruction::SVMInstruction, +}; + +/// Contains metadata about the instructions in a transaction packet. +#[derive(Debug)] +pub(crate) enum InstructionsFrame { + LegacyAndV0 { + /// The number of instructions in the transaction. + num_instructions: u16, + /// The offset to the first instruction in the transaction. + offset: u32, + frames: Vec, + }, + V1 { + num_instructions: u16, + headers_offset: u32, + payloads_offset: u32, + }, +} + +#[derive(Debug)] +pub struct LegacyAndV0InstructionFrame { + num_accounts: u16, + data_len: u16, + num_accounts_len: u8, // either 1 or 2 + data_len_len: u8, // either 1 or 2 +} + +#[allow(dead_code)] +#[repr(C)] +#[derive(Debug)] +struct V1InstructionHeader { + program_id_index: u8, + num_accounts: u8, + data_len: u16, +} + +impl InstructionsFrame { + /// Get the number of instructions and offset to the first instruction. + /// The offset will be updated to point to the first byte after the last + /// instruction. + /// This function will parse each individual instruction to ensure the + /// instruction data is well-formed, but will not cache data related to + /// these instructions. + #[inline(always)] + pub(crate) fn try_new_for_legacy_and_v0(bytes: &[u8], offset: &mut usize) -> Result { + // Read the number of instructions at the current offset. + // Each instruction needs at least 3 bytes, so do a sanity check here to + // ensure we have enough bytes to read the number of instructions. + let num_instructions = read_compressed_u16(bytes, offset)?; + let minimum_instructions_len = 3usize + .checked_mul(usize::from(num_instructions)) + .ok_or(TransactionViewError::ParseError)?; + check_remaining(bytes, *offset, minimum_instructions_len)?; + + let instructions_offset = try_u32_offset(*offset)?; + + // Pre-allocate buffer for frames. + let mut frames = Vec::with_capacity(usize::from(num_instructions)); + + // The instructions do not have a fixed size. So we must iterate over + // each instruction to find the total size of the instructions, + // and check for any malformed instructions or buffer overflows. + for _index in 0..num_instructions { + // Each instruction has 3 pieces: + // 1. Program ID index (u8) + // 2. Accounts indexes ([u8]) + // 3. Data ([u8]) + + // Read the program ID index. + let _program_id_index = read_byte(bytes, offset)?; + + // Read the number of account indexes, and then update the offset + // to skip over the account indexes. + let num_accounts_offset = *offset; + let num_accounts = read_compressed_u16(bytes, offset)?; + let num_accounts_len = u8::try_from( + offset + .checked_sub(num_accounts_offset) + .ok_or(TransactionViewError::ParseError)?, + ) + .map_err(|_| TransactionViewError::ParseError)?; + advance_offset_for_array::(bytes, offset, num_accounts)?; + + // Read the length of the data, and then update the offset to skip + // over the data. + let data_len_offset = *offset; + let data_len = read_compressed_u16(bytes, offset)?; + let data_len_len = u8::try_from( + offset.checked_sub(data_len_offset).ok_or(TransactionViewError::ParseError)?, + ) + .map_err(|_| TransactionViewError::ParseError)?; + advance_offset_for_array::(bytes, offset, data_len)?; + + frames.push(LegacyAndV0InstructionFrame { + num_accounts, + num_accounts_len, + data_len, + data_len_len, + }); + } + + Ok(Self::LegacyAndV0 { + num_instructions, + offset: instructions_offset, + frames, + }) + } + + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn try_new_for_v1( + bytes: &[u8], + offset: &mut usize, + num_instructions: u8, + ) -> Result { + let headers_offset = try_u32_offset(*offset)?; + let headers_len = core::mem::size_of::() + .checked_mul(usize::from(num_instructions)) + .ok_or(TransactionViewError::ParseError)?; + + check_remaining(bytes, *offset, headers_len)?; + + let mut header_offset = *offset; + *offset = offset.checked_add(headers_len).ok_or(TransactionViewError::ParseError)?; + + let payloads_offset = try_u32_offset(*offset)?; + + // Tx v1 stores all instruction payloads contiguously after the header block. + // We validate headers first, accumulate the total payload size across all + // instructions, and then do a single bounds check for the whole payload region + // instead of one bounds check per instruction. + let mut total_payload_len: usize = 0; + for _ in 0..num_instructions { + // SAFETY: we have already verified bytes contains enough space for `num_instruction` headers. + let header = unsafe { Self::read_v1_header(bytes, &mut header_offset) }; + + let payload_len = usize::from(header.num_accounts) + .checked_add(usize::from(header.data_len)) + .ok_or(TransactionViewError::ParseError)?; + + total_payload_len = total_payload_len + .checked_add(payload_len) + .ok_or(TransactionViewError::ParseError)?; + } + + check_remaining(bytes, *offset, total_payload_len)?; + *offset = offset.checked_add(total_payload_len).ok_or(TransactionViewError::ParseError)?; + + Ok(Self::V1 { + num_instructions: u16::from(num_instructions), + headers_offset, + payloads_offset, + }) + } + + /// # Safety + /// `bytes[*offset..*offset + size_of::()]` must be valid. + #[inline(always)] + unsafe fn read_v1_header(bytes: &[u8], offset: &mut usize) -> V1InstructionHeader { + let mut header: V1InstructionHeader = unsafe { unchecked_copy_value(bytes, *offset) }; + *offset += core::mem::size_of::(); + header.data_len = u16::from_le(header.data_len); + header + } + + #[inline(always)] + pub(crate) fn num_instructions(&self) -> u16 { + match self { + Self::LegacyAndV0 { num_instructions, .. } => *num_instructions, + Self::V1 { num_instructions, .. } => *num_instructions, + } + } + + #[inline(always)] + pub(crate) fn iter<'a>(&'a self, bytes: &'a [u8]) -> InstructionsIterator<'a> { + match self { + Self::LegacyAndV0 { num_instructions, offset, frames } => { + InstructionsIterator::LegacyAndV0 { + bytes, + offset: *offset as usize, + index: 0, + num_instructions: *num_instructions, + frames, + } + } + Self::V1 { + num_instructions, + headers_offset, + payloads_offset, + } => InstructionsIterator::V1 { + bytes, + index: 0, + num_instructions: *num_instructions, + headers_offset: *headers_offset as usize, + payloads_offset: *payloads_offset as usize, + }, + } + } +} + +#[derive(Clone)] +pub enum InstructionsIterator<'a> { + LegacyAndV0 { + bytes: &'a [u8], + offset: usize, + num_instructions: u16, + index: u16, + frames: &'a [LegacyAndV0InstructionFrame], + }, + V1 { + bytes: &'a [u8], + index: u16, + num_instructions: u16, + headers_offset: usize, + payloads_offset: usize, + }, +} + +impl<'a> Iterator for InstructionsIterator<'a> { + type Item = SVMInstruction<'a>; + + #[inline] + fn next(&mut self) -> Option { + match self { + Self::LegacyAndV0 { + bytes, + offset, + index, + num_instructions, + frames, + } => { + if *index >= *num_instructions { + return None; + } + + let LegacyAndV0InstructionFrame { + num_accounts, + num_accounts_len, + data_len, + data_len_len, + } = frames[usize::from(*index)]; + + *index = index.wrapping_add(1); + + Some(unsafe { + for_legacy_and_v0( + bytes, + offset, + num_accounts, + num_accounts_len, + data_len, + data_len_len, + ) + }) + } + Self::V1 { + bytes, + index, + num_instructions, + headers_offset, + payloads_offset, + } => { + if *index >= *num_instructions { + return None; + } + + let header = unsafe { InstructionsFrame::read_v1_header(bytes, headers_offset) }; + *index = index.wrapping_add(1); + + Some(unsafe { + for_v1( + bytes, + payloads_offset, + header.program_id_index, + u16::from(header.num_accounts), + header.data_len, + ) + }) + } + } + } +} + +/// Builds SNVInstruction from legacy/v0 pre-validated frame metadata. +/// +/// # Safety +/// The caller must ensure that: +/// - `offset` points to the beginning of a serialized legacy/v0 instruction +/// in `bytes`. +/// - `num_accounts_len` and `data_len_len` are the exact encoded lengths of the +/// compact-u16 account-count and data-length fields for that instruction. +/// - `num_accounts` and `data_len` exactly match the serialized instruction at +/// `offset`. +/// - The byte ranges implied by those values are fully in bounds of `bytes`. +/// +/// These invariants are expected to have been established by the initial +/// instruction frame parsing. Violating them may cause out-of-bounds unchecked +/// reads and undefined behavior. +#[inline(always)] +unsafe fn for_legacy_and_v0<'a>( + bytes: &'a [u8], + offset: &mut usize, + num_accounts: u16, + num_accounts_len: u8, + data_len: u16, + data_len_len: u8, +) -> SVMInstruction<'a> { + // Each instruction has 3 pieces: + // 1. Program ID index (u8) + // 2. Accounts indexes ([u8]) + // 3. Data ([u8]) + + // Read the program ID index. + // SAFETY: Offset and length checks have been done in the initial parsing. + let program_id_index = unsafe { unchecked_read_byte(bytes, offset) }; + + // Move offset to accounts offset - do not re-parse u16. + *offset += usize::from(num_accounts_len); + const _: () = assert!(core::mem::align_of::() == 1, "u8 alignment"); + // SAFETY: + // - The offset is checked to be valid in the byte slice. + // - The alignment of u8 is 1. + // - The slice length is checked to be valid. + // - `u8` cannot be improperly initialized. + // - Offset and length checks have been done in the initial parsing. + let accounts = unsafe { unchecked_read_slice_data::(bytes, offset, num_accounts) }; + + // Move offset to accounts offset - do not re-parse u16. + *offset += usize::from(data_len_len); + const _: () = assert!(core::mem::align_of::() == 1, "u8 alignment"); + // SAFETY: + // - The offset is checked to be valid in the byte slice. + // - The alignment of u8 is 1. + // - The slice length is checked to be valid. + // - `u8` cannot be improperly initialized. + // - Offset and length checks have been done in the initial parsing. + let data = unsafe { unchecked_read_slice_data::(bytes, offset, data_len) }; + + SVMInstruction { program_id_index, accounts, data } +} + +/// Builds SMVInstruction from v1 pre-validated frame metadata. +/// +/// # Safety +/// The caller must ensure that: +/// +/// - `payload_offset` points to the beginning of this instruction’s payload +/// (i.e. the first account index byte) within `bytes`. +/// - `num_accounts` and `data_len` exactly match the instruction header that +/// was previously parsed for this instruction. +/// - The byte range +/// `payload_offset .. payload_offset + num_accounts + data_len` +/// lies entirely within `bytes`. +/// - `bytes` has not been mutated since the initial parsing that produced +/// the instruction frames. +/// +/// These invariants are expected to have been established during the initial +/// tx-v1 instruction parsing phase, where header and payload bounds were +/// validated together. +/// +/// Violating any of these conditions may result in out-of-bounds unchecked +/// reads and thus undefined behavior. +#[inline(always)] +unsafe fn for_v1<'a>( + bytes: &'a [u8], + payloads_offset: &mut usize, + program_id_index: u8, + num_accounts: u16, + data_len: u16, +) -> SVMInstruction<'a> { + // SAFETY: + // - The offset is checked to be valid in the byte slice. + // - The alignment of u8 is 1. + // - The slice length is checked to be valid. + // - `u8` cannot be improperly initialized. + // - Offset and length checks have been done in the initial parsing. + let accounts = unsafe { unchecked_read_slice_data::(bytes, payloads_offset, num_accounts) }; + // SAFETY: + // - The offset is checked to be valid in the byte slice. + // - The alignment of u8 is 1. + // - The slice length is checked to be valid. + // - `u8` cannot be improperly initialized. + // - Offset and length checks have been done in the initial parsing. + let data = unsafe { unchecked_read_slice_data::(bytes, payloads_offset, data_len) }; + + SVMInstruction { program_id_index, accounts, data } +} + +impl ExactSizeIterator for InstructionsIterator<'_> { + fn len(&self) -> usize { + match self { + Self::LegacyAndV0 { num_instructions, index, .. } => { + usize::from(num_instructions.wrapping_sub(*index)) + } + Self::V1 { num_instructions, index, .. } => { + usize::from(num_instructions.wrapping_sub(*index)) + } + } + } +} + +impl Debug for InstructionsIterator<'_> { + fn fmt(&self, f: &mut Formatter) -> core::fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +#[cfg(test)] +mod tests { + use { + super::*, solana_message::compiled_instruction::CompiledInstruction, + solana_short_vec::ShortVec, + }; + + impl InstructionsFrame { + fn offset(&self) -> u32 { + match self { + Self::LegacyAndV0 { offset, .. } => *offset, + Self::V1 { headers_offset, .. } => *headers_offset, + } + } + } + + #[test] + fn test_zero_instructions() { + let bytes = bincode::serialize(&ShortVec(Vec::::new())).unwrap(); + let mut offset = 0; + let instructions_frame = + InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).unwrap(); + + assert_eq!(instructions_frame.num_instructions(), 0); + assert_eq!(instructions_frame.offset(), 1); + assert_eq!(offset, bytes.len()); + } + + #[test] + fn test_num_instructions_too_high() { + let mut bytes = bincode::serialize(&ShortVec(vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![], + data: vec![], + }])) + .unwrap(); + // modify the number of instructions to be too high + bytes[0] = 0x02; + let mut offset = 0; + assert!(InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_single_instruction() { + let bytes = bincode::serialize(&ShortVec(vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![1, 2, 3], + data: vec![4, 5, 6, 7, 8, 9, 10], + }])) + .unwrap(); + let mut offset = 0; + let instructions_frame = + InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).unwrap(); + assert_eq!(instructions_frame.num_instructions(), 1); + assert_eq!(instructions_frame.offset(), 1); + assert_eq!(offset, bytes.len()); + } + + #[test] + fn test_multiple_instructions() { + let bytes = bincode::serialize(&ShortVec(vec![ + CompiledInstruction { + program_id_index: 0, + accounts: vec![1, 2, 3], + data: vec![4, 5, 6, 7, 8, 9, 10], + }, + CompiledInstruction { + program_id_index: 1, + accounts: vec![4, 5, 6], + data: vec![7, 8, 9, 10, 11, 12, 13], + }, + ])) + .unwrap(); + let mut offset = 0; + let instructions_frame = + InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).unwrap(); + assert_eq!(instructions_frame.num_instructions(), 2); + assert_eq!(instructions_frame.offset(), 1); + assert_eq!(offset, bytes.len()); + } + + #[test] + fn test_invalid_instruction_accounts_vec() { + let mut bytes = bincode::serialize(&ShortVec(vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![1, 2, 3], + data: vec![4, 5, 6, 7, 8, 9, 10], + }])) + .unwrap(); + + // modify the number of accounts to be too high + bytes[2] = 127; + + let mut offset = 0; + assert!(InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_invalid_instruction_data_vec() { + let mut bytes = bincode::serialize(&ShortVec(vec![CompiledInstruction { + program_id_index: 0, + accounts: vec![1, 2, 3], + data: vec![4, 5, 6, 7, 8, 9, 10], + }])) + .unwrap(); + + // modify the number of data bytes to be too high + bytes[6] = 127; + + let mut offset = 0; + assert!(InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_txv1_instructions_iterator() { + let message = solana_message::v1::Message { + instructions: vec![ + CompiledInstruction { + program_id_index: 0, + accounts: vec![1, 2, 3], + data: vec![4, 5, 6, 7, 8, 9, 10], + }, + CompiledInstruction { + program_id_index: 10, + accounts: vec![11, 12], + data: vec![13, 14, 15, 16, 17, 18, 19, 20], + }, + ], + ..solana_message::v1::Message::default() + }; + + let serialized = solana_message::VersionedMessage::V1(message).serialize(); + + let mut offset = 42; // instruction headers start after the V1 prefix for no config, 0 addresses + let instructions_frame = + InstructionsFrame::try_new_for_v1(&serialized, &mut offset, 2).unwrap(); + + let mut iter = instructions_frame.iter(&serialized); + assert_eq!( + iter.next(), + Some(SVMInstruction { + program_id_index: 0, + accounts: &[1, 2, 3], + data: &[4, 5, 6, 7, 8, 9, 10] + }) + ); + assert_eq!( + iter.next(), + Some(SVMInstruction { + program_id_index: 10, + accounts: &[11, 12], + data: &[13, 14, 15, 16, 17, 18, 19, 20] + }) + ); + assert_eq!(iter.next(), None); + } + + fn short_u16_1(x: u8) -> Vec { + vec![x] + } + + // short_vec / compact-u16 encoding for 128..=16383 style values + fn short_u16_2(x: u16) -> Vec { + assert!(x >= 128); + vec![((x & 0x7f) as u8) | 0x80, (x >> 7) as u8] + } + + #[test] + fn test_try_new_legacy_single_instruction() { + // num_instructions = 1 + // instruction: + // program_id_index = 7 + // num_accounts = 2 + // accounts = [3, 4] + // data_len = 3 + // data = [9, 8, 7] + let bytes = vec![ + 1, // num_instructions + 7, // program_id_index + 2, // num_accounts + 3, 4, // account indexes + 3, // data_len + 9, 8, 7, // data + ]; + + let mut offset = 0; + let frame = InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).unwrap(); + + assert_eq!(offset, bytes.len()); + + match frame { + InstructionsFrame::LegacyAndV0 { num_instructions, offset, frames } => { + assert_eq!(num_instructions, 1); + assert_eq!(offset, 1); + assert_eq!(frames.len(), 1); + let ix = &frames[0]; + assert_eq!(ix.num_accounts, 2); + assert_eq!(ix.data_len, 3); + assert_eq!(ix.num_accounts_len, 1); + assert_eq!(ix.data_len_len, 1); + } + _ => panic!("expected legacy/v0 repr"), + } + } + + #[test] + fn test_try_new_legacy_two_byte_lengths() { + let num_accounts = 128u16; + let data_len = 130u16; + + let mut bytes = Vec::new(); + bytes.push(1); // num_instructions + bytes.push(42); // program_id_index + bytes.extend_from_slice(&short_u16_2(num_accounts)); + bytes.extend(std::iter::repeat_n(5u8, num_accounts as usize)); + bytes.extend_from_slice(&short_u16_2(data_len)); + bytes.extend(std::iter::repeat_n(9u8, data_len as usize)); + + let mut offset = 0; + let frame = InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).unwrap(); + + assert_eq!(offset, bytes.len()); + + match frame { + InstructionsFrame::LegacyAndV0 { num_instructions, offset, frames } => { + assert_eq!(num_instructions, 1); + assert_eq!(offset, 1); + assert_eq!(frames.len(), 1); + let ix = &frames[0]; + assert_eq!(ix.num_accounts, num_accounts); + assert_eq!(ix.data_len, data_len); + + assert_eq!(ix.num_accounts_len, 2); + assert_eq!(ix.data_len_len, 2); + } + _ => panic!("expected legacy/v0 repr"), + } + } + + #[test] + fn test_try_new_for_v1_single_instruction() { + // one v1 instruction + // header: + // program_id_index = 9 + // num_accounts = 2 + // data_len = 3 + // payload: + // accounts = [10, 11] + // data = [1, 2, 3] + let bytes = vec![ + 9, // program_id_index + 2, // num_accounts + 3, 0, // data_len (u16 LE) + 10, 11, // payload accounts + 1, 2, 3, // payload data + ]; + + let mut offset = 0; + let frame = InstructionsFrame::try_new_for_v1(&bytes, &mut offset, 1).unwrap(); + + assert_eq!(offset, bytes.len()); + + match frame { + InstructionsFrame::V1 { + num_instructions, + headers_offset, + payloads_offset, + } => { + assert_eq!(num_instructions, 1); + assert_eq!(headers_offset, 0); + assert_eq!(payloads_offset, 4); + let hdr = unsafe { InstructionsFrame::read_v1_header(&bytes, &mut 0) }; + assert_eq!(hdr.program_id_index, 9); + assert_eq!(hdr.num_accounts, 2); + assert_eq!(hdr.data_len, 3); + } + _ => panic!("expected v1 repr"), + } + } + + #[test] + fn test_try_new_for_v1_two_instructions() { + // headers: + // ix0: pid=1, accounts=2, data_len=1 + // ix1: pid=7, accounts=1, data_len=2 + // + // payloads: + // ix0: [20, 21] [99] + // ix1: [42] [5, 6] + let bytes = vec![ + // header 0 + 1, 2, 1, 0, // header 1 + 7, 1, 2, 0, // payload 0 + 20, 21, 99, // payload 1 + 42, 5, 6, + ]; + + let mut offset = 0; + let frame = InstructionsFrame::try_new_for_v1(&bytes, &mut offset, 2).unwrap(); + + assert_eq!(offset, bytes.len()); + match frame { + InstructionsFrame::V1 { + num_instructions, + headers_offset, + payloads_offset, + } => { + assert_eq!(num_instructions, 2); + assert_eq!(headers_offset, 0); + assert_eq!(payloads_offset, 8); + let hdr = unsafe { InstructionsFrame::read_v1_header(&bytes, &mut 0) }; + assert_eq!(hdr.program_id_index, 1); + assert_eq!(hdr.num_accounts, 2); + assert_eq!(hdr.data_len, 1); + let hdr = unsafe { InstructionsFrame::read_v1_header(&bytes, &mut 4) }; + assert_eq!(hdr.program_id_index, 7); + assert_eq!(hdr.num_accounts, 1); + assert_eq!(hdr.data_len, 2); + } + _ => panic!("expected v1 repr"), + } + } + + #[test] + fn test_try_new_for_v1_truncated_header_fails() { + // num_instructions = 1, but only 3 header bytes instead of 4 + let bytes = vec![ + 9, // program_id_index + 2, // num_accounts + 3, // incomplete data_len + ]; + + let mut offset = 0; + assert!(InstructionsFrame::try_new_for_v1(&bytes, &mut offset, 1).is_err()); + } + + #[test] + fn test_try_new_for_v1_truncated_payload_fails() { + // header says payload len = 2 + 3 = 5, but only 4 bytes provided + let bytes = vec![ + 9, 2, 3, 0, // header + 10, 11, 1, 2, // truncated payload + ]; + + let mut offset = 0; + assert!(InstructionsFrame::try_new_for_v1(&bytes, &mut offset, 1).is_err()); + } + + #[test] + fn test_try_new_legacy_truncated_payload_fails() { + // data_len says 3, only 2 bytes provided + let bytes = vec![ + 1, // num_instructions + 7, // program_id_index + 1, // num_accounts + 9, // account idx + 3, // data_len + 1, 2, // truncated data + ]; + + let mut offset = 0; + assert!(InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_try_new_for_v1_zero_instructions() { + let bytes = vec![]; + let mut offset = 0; + + let frame = InstructionsFrame::try_new_for_v1(&bytes, &mut offset, 0).unwrap(); + assert_eq!(offset, 0); + match frame { + InstructionsFrame::V1 { + num_instructions, + headers_offset, + payloads_offset, + } => { + assert_eq!(num_instructions, 0); + assert_eq!(headers_offset, 0); + assert_eq!(payloads_offset, 0); + } + _ => panic!("expected v1 repr"), + } + } + + #[test] + fn data_len_max_header_fails_parse() { + // header: pid=1, accounts=1, data_len=65535 + let bytes = vec![1, 1, 0xff, 0xff]; + let mut offset = 0; + assert_eq!( + InstructionsFrame::try_new_for_v1(&bytes, &mut offset, 1).err().unwrap(), + TransactionViewError::ParseError + ); + } + + #[test] + fn test_try_new_legacy_zero_instructions() { + let bytes = short_u16_1(0); + let mut offset = 0; + + let frame = InstructionsFrame::try_new_for_legacy_and_v0(&bytes, &mut offset).unwrap(); + assert_eq!(offset, 1); + + match frame { + InstructionsFrame::LegacyAndV0 { num_instructions, offset, frames } => { + assert_eq!(num_instructions, 0); + assert_eq!(offset, 1); + assert!(frames.is_empty()); + } + _ => panic!("expected legacy/v0 repr"), + } + } +} diff --git a/solana/transaction-view/src/lib.rs b/solana/transaction-view/src/lib.rs new file mode 100644 index 00000000..097a1782 --- /dev/null +++ b/solana/transaction-view/src/lib.rs @@ -0,0 +1,26 @@ +#![cfg(feature = "agave-unstable-api")] +#![doc = include_str!("../README.md")] +// Parsing helpers only need to be public for benchmarks. +#[cfg(feature = "dev-context-only-utils")] +pub mod bytes; +#[cfg(not(feature = "dev-context-only-utils"))] +mod bytes; + +mod address_table_lookup_frame; +mod instructions_frame; +mod message_header_frame; +pub mod resolved_transaction_view; +pub mod result; +mod sanitize; +mod signature_frame; +mod static_account_keys_frame; +mod transaction_config_frame; +pub mod transaction_data; +mod transaction_frame; +pub mod transaction_version; +pub mod transaction_view; + +pub use sanitize::{ + MAGICBLOCK_INSTRUCTION_TRACE_LENGTH, MAX_MAGICBLOCK_ACCOUNT_LOCKS, + MAX_MAGICBLOCK_TRANSACTION_SIZE, MAX_STANDARD_TRANSACTION_SIZE, +}; diff --git a/solana/transaction-view/src/message_header_frame.rs b/solana/transaction-view/src/message_header_frame.rs new file mode 100644 index 00000000..b37c2018 --- /dev/null +++ b/solana/transaction-view/src/message_header_frame.rs @@ -0,0 +1,110 @@ +use { + crate::{ + bytes::{read_byte, try_u32_offset}, + result::{Result, TransactionViewError}, + transaction_version::TransactionVersion, + }, + solana_message::MESSAGE_VERSION_PREFIX, +}; + +/// Metadata for accessing message header fields in a transaction view. +#[derive(Debug)] +pub(crate) struct MessageHeaderFrame { + /// The offset to the first byte of the message in the transaction packet. + pub(crate) offset: u32, + /// The version of the transaction. + pub(crate) version: TransactionVersion, + /// The number of signatures required for this message to be considered + /// valid. + pub(crate) num_required_signatures: u8, + /// The last `num_readonly_signed_accounts` of the signed keys are + /// read-only. + pub(crate) num_readonly_signed_accounts: u8, + /// The last `num_readonly_unsigned_accounts` of the unsigned keys are + /// read-only accounts. + pub(crate) num_readonly_unsigned_accounts: u8, +} + +impl MessageHeaderFrame { + #[inline(always)] + pub(crate) fn try_new(bytes: &[u8], offset: &mut usize) -> Result { + // Get the message offset. + let message_offset = try_u32_offset(*offset)?; + + // Read the message prefix byte if present. This byte is present in V0 + // transactions but not in legacy transactions. + // The message header begins immediately after the message prefix byte + // if present. + let message_prefix = read_byte(bytes, offset)?; + let (version, num_required_signatures) = if message_prefix & MESSAGE_VERSION_PREFIX != 0 { + let version = message_prefix & !MESSAGE_VERSION_PREFIX; + match version { + 0 => (TransactionVersion::V0, read_byte(bytes, offset)?), + _ => return Err(TransactionViewError::ParseError), + } + } else { + // Legacy transaction. The `message_prefix` that was just read is + // actually the number of required signatures. + (TransactionVersion::Legacy, message_prefix) + }; + + let num_readonly_signed_accounts = read_byte(bytes, offset)?; + let num_readonly_unsigned_accounts = read_byte(bytes, offset)?; + + Ok(Self { + offset: message_offset, + version, + num_required_signatures, + num_readonly_signed_accounts, + num_readonly_unsigned_accounts, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_invalid_version() { + let bytes = [0b1000_0001]; + let mut offset = 0; + assert!(MessageHeaderFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_legacy_transaction_missing_header_byte() { + let bytes = [5, 0]; + let mut offset = 0; + assert!(MessageHeaderFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_legacy_transaction_valid() { + let bytes = [5, 1, 2]; + let mut offset = 0; + let header = MessageHeaderFrame::try_new(&bytes, &mut offset).unwrap(); + assert!(matches!(header.version, TransactionVersion::Legacy)); + assert_eq!(header.num_required_signatures, 5); + assert_eq!(header.num_readonly_signed_accounts, 1); + assert_eq!(header.num_readonly_unsigned_accounts, 2); + } + + #[test] + fn test_v0_transaction_missing_header_byte() { + let bytes = [MESSAGE_VERSION_PREFIX, 5, 1]; + let mut offset = 0; + assert!(MessageHeaderFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_v0_transaction_valid() { + let bytes = [MESSAGE_VERSION_PREFIX, 5, 1, 2]; + let mut offset = 0; + let header = MessageHeaderFrame::try_new(&bytes, &mut offset).unwrap(); + assert!(matches!(header.version, TransactionVersion::V0)); + assert_eq!(header.num_required_signatures, 5); + assert_eq!(header.num_readonly_signed_accounts, 1); + assert_eq!(header.num_readonly_unsigned_accounts, 2); + } +} diff --git a/solana/transaction-view/src/resolved_transaction_view.rs b/solana/transaction-view/src/resolved_transaction_view.rs new file mode 100644 index 00000000..260f5ef7 --- /dev/null +++ b/solana/transaction-view/src/resolved_transaction_view.rs @@ -0,0 +1,334 @@ +use { + crate::{ + result::{Result, TransactionViewError}, + transaction_data::TransactionData, + transaction_view::TransactionView, + }, + core::{ + fmt::{Debug, Formatter}, + ops::Deref, + }, + solana_hash::Hash, + solana_message::{AccountKeys, v0::LoadedAddresses}, + solana_pubkey::Pubkey, + solana_sdk_ids::bpf_loader_upgradeable, + solana_signature::Signature, + solana_svm_transaction::{ + instruction::SVMInstruction, + message_address_table_lookup::SVMMessageAddressTableLookup, + svm_message::{SVMMessage, SVMStaticMessage}, + svm_transaction::SVMTransaction, + }, + std::collections::HashSet, +}; + +/// A parsed and sanitized transaction view with validated loaded-address state. +pub struct ResolvedTransactionView { + /// The parsed and sanitized transaction view. + view: TransactionView, + /// The resolved address lookups. + resolved_addresses: Option, + /// A cache for whether an address is writable. + // Sanitized transactions are guaranteed to have a maximum of 256 keys, + // because account indexing is done with a u8. + writable_cache: [bool; 256], +} + +impl Deref for ResolvedTransactionView { + type Target = TransactionView; + + fn deref(&self) -> &Self::Target { + &self.view + } +} + +impl ResolvedTransactionView { + /// Creates a resolved view after validating any supplied loaded addresses. + /// + /// Address lookup tables are rejected during sanitization, so loaded + /// addresses may only be absent or empty. + pub fn try_new( + view: TransactionView, + resolved_addresses: Option, + reserved_account_keys: &HashSet, + ) -> Result { + let resolved_addresses_ref = resolved_addresses.as_ref(); + + // Reject unexpected loaded addresses while retaining the upstream API. + if let Some(loaded_addresses) = resolved_addresses_ref { + if loaded_addresses.writable.len() != usize::from(view.total_writable_lookup_accounts()) + || loaded_addresses.readonly.len() + != usize::from(view.total_readonly_lookup_accounts()) + { + return Err(TransactionViewError::AddressLookupMismatch); + } + } else if view.total_writable_lookup_accounts() != 0 + || view.total_readonly_lookup_accounts() != 0 + { + return Err(TransactionViewError::AddressLookupMismatch); + } + + let writable_cache = + Self::cache_is_writable(&view, resolved_addresses_ref, reserved_account_keys); + Ok(Self { + view, + resolved_addresses, + writable_cache, + }) + } + + /// Helper function to check if an address is writable, + /// and cache the result. + /// This is done so we avoid recomputing the expensive checks each time we call + /// `is_writable` - since there is more to it than just checking index. + fn cache_is_writable( + view: &TransactionView, + resolved_addresses: Option<&LoadedAddresses>, + reserved_account_keys: &HashSet, + ) -> [bool; 256] { + // Build account keys so that we can iterate over and check if + // an address is writable. + let account_keys = AccountKeys::new(view.static_account_keys(), resolved_addresses); + + let mut is_writable_cache = [false; 256]; + let num_static_account_keys = usize::from(view.num_static_account_keys()); + let num_writable_lookup_accounts = usize::from(view.total_writable_lookup_accounts()); + let num_signed_accounts = usize::from(view.num_required_signatures()); + let num_writable_unsigned_static_accounts = + usize::from(view.num_writable_unsigned_static_accounts()); + let num_writable_signed_static_accounts = + usize::from(view.num_writable_signed_static_accounts()); + + for (index, key) in account_keys.iter().enumerate() { + let is_requested_write = { + // If the account is a resolved address, check if it is writable. + if index >= num_static_account_keys { + let loaded_address_index = index.wrapping_sub(num_static_account_keys); + loaded_address_index < num_writable_lookup_accounts + } else if index >= num_signed_accounts { + let unsigned_account_index = index.wrapping_sub(num_signed_accounts); + unsigned_account_index < num_writable_unsigned_static_accounts + } else { + index < num_writable_signed_static_accounts + } + }; + + // If the key is reserved it cannot be writable. + is_writable_cache[index] = is_requested_write && !reserved_account_keys.contains(key); + } + + // If a program account is locked, it cannot be writable unless the + // upgradable loader is present. + // However, checking for the upgradable loader is somewhat expensive, so + // we only do it if we find a writable program id. + let mut is_upgradable_loader_present = None; + for ix in view.instructions_iter() { + let program_id_index = usize::from(ix.program_id_index); + if is_writable_cache[program_id_index] + && !*is_upgradable_loader_present.get_or_insert_with(|| { + for key in account_keys.iter() { + if key == &bpf_loader_upgradeable::ID { + return true; + } + } + false + }) + { + is_writable_cache[program_id_index] = false; + } + } + + is_writable_cache + } + + pub fn loaded_addresses(&self) -> Option<&LoadedAddresses> { + self.resolved_addresses.as_ref() + } + + pub fn into_view(self) -> TransactionView { + self.view + } +} + +impl SVMStaticMessage for ResolvedTransactionView { + fn version(&self) -> solana_transaction::versioned::TransactionVersion { + self.view.version().into() + } + + fn num_transaction_signatures(&self) -> u64 { + u64::from(self.view.num_required_signatures()) + } + + fn num_write_locks(&self) -> u64 { + self.view.num_requested_write_locks() + } + + fn recent_blockhash(&self) -> &Hash { + self.view.recent_blockhash() + } + + fn num_instructions(&self) -> usize { + usize::from(self.view.num_instructions()) + } + + fn instructions_iter(&self) -> impl Iterator> { + self.view.instructions_iter() + } + + fn program_instructions_iter( + &self, + ) -> impl Iterator< + Item = ( + &solana_pubkey::Pubkey, + solana_svm_transaction::instruction::SVMInstruction<'_>, + ), + > + Clone { + self.view.program_instructions_iter() + } + + fn static_account_keys(&self) -> &[Pubkey] { + self.view.static_account_keys() + } + + fn fee_payer(&self) -> &Pubkey { + &self.view.static_account_keys()[0] + } + + fn num_lookup_tables(&self) -> usize { + usize::from(self.view.num_address_table_lookups()) + } + + fn message_address_table_lookups( + &self, + ) -> impl Iterator> { + self.view.address_table_lookup_iter() + } +} + +impl SVMMessage for ResolvedTransactionView { + fn account_keys(&self) -> AccountKeys<'_> { + AccountKeys::new( + self.view.static_account_keys(), + self.resolved_addresses.as_ref(), + ) + } + + fn is_writable(&self, index: usize) -> bool { + self.writable_cache.get(index).copied().unwrap_or(false) + } + + fn is_signer(&self, index: usize) -> bool { + index < usize::from(self.view.num_required_signatures()) + } + + fn is_invoked(&self, key_index: usize) -> bool { + let Ok(index) = u8::try_from(key_index) else { + return false; + }; + self.view.instructions_iter().any(|ix| ix.program_id_index == index) + } +} + +impl SVMTransaction for ResolvedTransactionView { + fn signature(&self) -> &Signature { + &self.view.signatures()[0] + } + + fn signatures(&self) -> &[Signature] { + self.view.signatures() + } +} + +impl Debug for ResolvedTransactionView { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ResolvedTransactionView").field("view", &self.view).finish() + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::transaction_view::SanitizedTransactionView, + solana_message::{ + MessageHeader, VersionedMessage, + v0::{self, MessageAddressTableLookup}, + }, + solana_signature::Signature, + solana_transaction::versioned::VersionedTransaction, + }; + + fn v0_transaction( + address_table_lookups: Vec, + ) -> VersionedTransaction { + VersionedTransaction { + signatures: vec![Signature::default()], + message: VersionedMessage::V0(v0::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + instructions: vec![], + account_keys: vec![Pubkey::new_unique(), Pubkey::new_unique()], + address_table_lookups, + recent_blockhash: Hash::default(), + }), + } + } + + #[test] + fn test_address_lookup_tables_are_rejected() { + let lookups = [ + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![0], + readonly_indexes: vec![], + }, + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![], + readonly_indexes: vec![0], + }, + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![0], + readonly_indexes: vec![1], + }, + ]; + + for lookup in lookups { + let transaction = v0_transaction(vec![lookup]); + let bytes = wincode::serialize(&transaction).unwrap(); + let result = SanitizedTransactionView::try_new_sanitized(bytes.as_ref(), true); + assert!(matches!( + result, + Err(TransactionViewError::AddressLookupMismatch) + )); + } + } + + #[test] + fn test_v0_without_lookups_needs_no_loaded_addresses() { + let bytes = wincode::serialize(&v0_transaction(vec![])).unwrap(); + let view = SanitizedTransactionView::try_new_sanitized(bytes.as_ref(), true).unwrap(); + let resolved = ResolvedTransactionView::try_new(view, None, &HashSet::default()).unwrap(); + assert!(resolved.loaded_addresses().is_none()); + } + + #[test] + fn test_unexpected_loaded_addresses() { + let loaded_addresses = LoadedAddresses { + writable: vec![Pubkey::new_unique()], + readonly: vec![], + }; + let bytes = wincode::serialize(&v0_transaction(vec![])).unwrap(); + let view = SanitizedTransactionView::try_new_sanitized(bytes.as_ref(), true).unwrap(); + let result = + ResolvedTransactionView::try_new(view, Some(loaded_addresses), &HashSet::default()); + assert!(matches!( + result, + Err(TransactionViewError::AddressLookupMismatch) + )); + } +} diff --git a/solana/transaction-view/src/result.rs b/solana/transaction-view/src/result.rs new file mode 100644 index 00000000..028a7f11 --- /dev/null +++ b/solana/transaction-view/src/result.rs @@ -0,0 +1,9 @@ +#[derive(Debug, PartialEq, Eq)] +#[repr(u8)] // repr(u8) is used to ensure that the enum is represented as a single byte in memory. +pub enum TransactionViewError { + ParseError, + SanitizeError, + AddressLookupMismatch, +} + +pub type Result = core::result::Result; diff --git a/solana/transaction-view/src/sanitize.rs b/solana/transaction-view/src/sanitize.rs new file mode 100644 index 00000000..46ca368a --- /dev/null +++ b/solana/transaction-view/src/sanitize.rs @@ -0,0 +1,1020 @@ +use { + crate::{ + result::{Result, TransactionViewError}, + signature_frame::MAX_SIGNATURES_PER_PACKET, + transaction_data::TransactionData, + transaction_version::TransactionVersion, + transaction_view::UnsanitizedTransactionView, + }, + solana_program_runtime::execution_budget::{MAX_HEAP_FRAME_BYTES, MIN_HEAP_FRAME_BYTES}, +}; + +/// Maximum instruction trace length for an Engine-private transaction. +pub const MAGICBLOCK_INSTRUCTION_TRACE_LENGTH: usize = 255; +/// Maximum serialized size accepted for standard transaction versions. +pub const MAX_STANDARD_TRANSACTION_SIZE: usize = u16::MAX as usize; +/// Maximum serialized size accepted for an Engine-private transaction. +pub const MAX_MAGICBLOCK_TRANSACTION_SIZE: usize = 16 * 1024 * 1024; + +/// Maximum account count encodable by an Engine-private transaction. +pub const MAX_MAGICBLOCK_ACCOUNT_LOCKS: usize = u8::MAX as usize; + +pub(crate) fn sanitize( + view: &UnsanitizedTransactionView, + enable_instruction_accounts_limit: bool, +) -> Result<()> { + sanitize_transaction_size(view)?; + sanitize_message_header(view)?; + sanitize_config(view)?; + sanitize_signatures(view)?; + sanitize_account_access(view)?; + sanitize_instructions(view, enable_instruction_accounts_limit)?; + sanitize_address_table_lookups(view) +} + +/// Transaction size constraints are version-specific. +fn sanitize_transaction_size( + view: &UnsanitizedTransactionView, +) -> Result<()> { + let max_transaction_size = match view.version() { + TransactionVersion::Legacy | TransactionVersion::V0 | TransactionVersion::V1 => { + MAX_STANDARD_TRANSACTION_SIZE + } + TransactionVersion::Magicblock => MAX_MAGICBLOCK_TRANSACTION_SIZE, + }; + + if view.data().len() > max_transaction_size { + return Err(TransactionViewError::SanitizeError); + } + Ok(()) +} + +/// message header constraints: +/// * num_required_signatures >= 1 +/// * num_readonly_signed_accounts < num_required_signatures (fee payer must be writable) +/// * num_readonly_unsigned_accounts <= (num_addresses - num_required_signatures) +fn sanitize_message_header(view: &UnsanitizedTransactionView) -> Result<()> { + if view.num_required_signatures() < 1 { + return Err(TransactionViewError::SanitizeError); + } + + if view.num_readonly_signed_static_accounts() >= view.num_required_signatures() { + return Err(TransactionViewError::SanitizeError); + } + + // Check there is no overlap of signing area and readonly non-signing area. + // We have already checked that `num_required_signatures` is less than or equal to `num_static_account_keys`, + // so it is safe to use wrapping arithmetic. + if view.num_readonly_unsigned_static_accounts() + > view.num_static_account_keys().wrapping_sub(view.num_required_signatures()) + { + return Err(TransactionViewError::SanitizeError); + } + + Ok(()) +} + +/// Config Constraints: +/// * heap_size must be multiples of 1024, if specified +fn sanitize_config(view: &UnsanitizedTransactionView) -> Result<()> { + #[allow(clippy::collapsible_if)] + if let Some(requested_heap_bytes) = + view.transaction_config().and_then(|config| config.requested_heap_size()) + { + if !(MIN_HEAP_FRAME_BYTES..=MAX_HEAP_FRAME_BYTES).contains(&requested_heap_bytes) + || !requested_heap_bytes.is_multiple_of(1024) + { + return Err(TransactionViewError::SanitizeError); + } + } + + Ok(()) +} + +/// Sigantures Constraint: +/// * Number of signatures must equal: num_required_signatures +/// * Max signatures <= 12 +fn sanitize_signatures(view: &UnsanitizedTransactionView) -> Result<()> { + // Check the required number of signatures matches the number of signatures. + if view.num_signatures() != view.num_required_signatures() { + return Err(TransactionViewError::SanitizeError); + } + + if view.num_signatures() > MAX_SIGNATURES_PER_PACKET { + return Err(TransactionViewError::SanitizeError); + } + + // Each signature is associated with a unique static public key. + // Check that there are at least as many static account keys as signatures. + if view.num_static_account_keys() < view.num_signatures() { + return Err(TransactionViewError::SanitizeError); + } + + Ok(()) +} + +/// Accounts (aka Addresses) Constraints: +/// * for v1: 1 <= NumAddresses <= 64 +/// * for Magicblock: 1 <= NumAddresses <= 255 +/// * legacy/v0 uses current limits of: num_accounts <= 256 (u8 bound) +/// * No duplicate addresses +fn sanitize_account_access(view: &UnsanitizedTransactionView) -> Result<()> { + let addresses_limit = match view.version() { + TransactionVersion::Legacy | TransactionVersion::V0 => 256, + TransactionVersion::V1 => 64, + TransactionVersion::Magicblock => MAX_MAGICBLOCK_ACCOUNT_LOCKS as u16, + }; + + if total_number_of_accounts(view) > addresses_limit { + return Err(TransactionViewError::SanitizeError); + } + + // No duplicated accounts + // Note: This check is performed downstream in `validate_account_locks()`. + // It is skipped here to avoid redundant work on the hot path. + + Ok(()) +} + +/// Instructions Constraints +/// * NumInstructions <= 64 +/// * Per instruction: +/// * 0 < program_id_index < MaxProgramIdIndex +/// * all account indices < MaxAccountIndex +fn sanitize_instructions( + view: &UnsanitizedTransactionView, + enable_instruction_accounts_limit: bool, +) -> Result<()> { + let instructions_limit = match view.version() { + TransactionVersion::Magicblock => MAGICBLOCK_INSTRUCTION_TRACE_LENGTH, + TransactionVersion::V0 | TransactionVersion::V1 | TransactionVersion::Legacy => { + solana_transaction_context::MAX_INSTRUCTION_TRACE_LENGTH + } + }; + let num_instructions = usize::from(view.num_instructions()); + // Standard transactions retain the SIMD-160 top-level instruction limit. + if num_instructions > instructions_limit { + return Err(TransactionViewError::SanitizeError); + } + + // already verified there is at least one static account. + let max_program_id_index = view.num_static_account_keys().wrapping_sub(1); + // Magicblock and V1 are bounded by their encoded address counts; standard + // formats may use all 256 u8 account indices. + let max_account_index = total_number_of_accounts(view).wrapping_sub(1) as u8; + + for instruction in view.instructions_iter() { + // Check that program indexes are static account keys. + if instruction.program_id_index > max_program_id_index { + return Err(TransactionViewError::SanitizeError); + } + + // Check that the program index is not the fee-payer. + if instruction.program_id_index == 0 { + return Err(TransactionViewError::SanitizeError); + } + + // Check that all account indexes are valid. + for account_index in instruction.accounts.iter().copied() { + if account_index > max_account_index { + return Err(TransactionViewError::SanitizeError); + } + } + + if enable_instruction_accounts_limit + && instruction.accounts.len() > solana_transaction_context::MAX_ACCOUNTS_PER_INSTRUCTION + { + return Err(TransactionViewError::SanitizeError); + } + } + + Ok(()) +} + +fn sanitize_address_table_lookups( + view: &UnsanitizedTransactionView, +) -> Result<()> { + if view.num_address_table_lookups() != 0 { + // Preserve the resolution-specific error at the earlier ingress + // boundary where lookup tables are now rejected. + return Err(TransactionViewError::AddressLookupMismatch); + } + Ok(()) +} + +fn total_number_of_accounts(view: &UnsanitizedTransactionView) -> u16 { + u16::from(view.num_static_account_keys()) + .saturating_add(view.total_writable_lookup_accounts()) + .saturating_add(view.total_readonly_lookup_accounts()) +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::transaction_view::TransactionView, + solana_hash::Hash, + solana_message::{ + Message, MessageHeader, VersionedMessage, + compiled_instruction::CompiledInstruction, + v0::{self, MessageAddressTableLookup}, + v1::{self, TransactionConfig}, + }, + solana_pubkey::Pubkey, + solana_signature::Signature, + solana_system_interface::instruction as system_instruction, + solana_transaction::versioned::VersionedTransaction, + }; + + fn create_legacy_transaction( + num_signatures: u8, + header: MessageHeader, + account_keys: Vec, + instructions: Vec, + ) -> VersionedTransaction { + VersionedTransaction { + signatures: vec![Signature::default(); num_signatures as usize], + message: VersionedMessage::Legacy(Message { + header, + account_keys, + recent_blockhash: Hash::default(), + instructions, + }), + } + } + + fn create_v0_transaction( + num_signatures: u8, + header: MessageHeader, + account_keys: Vec, + instructions: Vec, + address_table_lookups: Vec, + ) -> VersionedTransaction { + VersionedTransaction { + signatures: vec![Signature::default(); num_signatures as usize], + message: VersionedMessage::V0(v0::Message { + header, + account_keys, + recent_blockhash: Hash::default(), + instructions, + address_table_lookups, + }), + } + } + + fn create_v1_transaction( + num_signatures: u8, + header: MessageHeader, + account_keys: Vec, + instructions: Vec, + config: TransactionConfig, + ) -> VersionedTransaction { + VersionedTransaction { + signatures: vec![Signature::default(); num_signatures as usize], + message: VersionedMessage::V1(v1::Message { + header, + account_keys, + lifetime_specifier: Hash::default(), + instructions, + config, + }), + } + } + + fn multiple_transfers() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::Legacy(Message::new( + &[ + system_instruction::transfer(&payer, &Pubkey::new_unique(), 1), + system_instruction::transfer(&payer, &Pubkey::new_unique(), 1), + ], + Some(&payer), + )), + } + } + + #[test] + fn test_sanitize_multiple_transfers() { + let transaction = multiple_transfers(); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(view.sanitize(true).is_ok()); + } + + #[test] + fn test_sanitize_standard_transaction_size_boundaries() { + let account_keys = vec![Pubkey::new_unique(), Pubkey::new_unique()]; + let header = MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }; + let instruction = CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: Vec::new(), + }; + let transactions = [ + create_legacy_transaction(1, header, account_keys.clone(), vec![instruction.clone()]), + create_v0_transaction( + 1, + header, + account_keys.clone(), + vec![instruction.clone()], + vec![], + ), + create_v1_transaction( + 1, + header, + account_keys, + vec![instruction], + TransactionConfig::empty(), + ), + ]; + + for mut transaction in transactions { + resize_transaction(&mut transaction, MAX_STANDARD_TRANSACTION_SIZE); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(view.sanitize(true).is_ok()); + + resize_transaction(&mut transaction, MAX_STANDARD_TRANSACTION_SIZE + 1); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_transaction_size(&view), + Err(TransactionViewError::SanitizeError) + ); + } + } + + fn resize_transaction(transaction: &mut VersionedTransaction, target: usize) { + const SAMPLE_DATA_LEN: usize = 65_000; + + instruction_data(transaction).resize(SAMPLE_DATA_LEN, 0); + let overhead = wincode::serialize(&*transaction).unwrap().len() - SAMPLE_DATA_LEN; + instruction_data(transaction).resize(target - overhead, 0); + assert_eq!(wincode::serialize(&*transaction).unwrap().len(), target); + } + + fn instruction_data(transaction: &mut VersionedTransaction) -> &mut Vec { + match &mut transaction.message { + VersionedMessage::Legacy(message) => &mut message.instructions[0].data, + VersionedMessage::V0(message) => &mut message.instructions[0].data, + VersionedMessage::V1(message) => &mut message.instructions[0].data, + } + } + + #[test] + fn test_sanitize_signatures() { + // Too few signatures. + { + let transaction = create_legacy_transaction( + 1, + MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..3).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_signatures(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Too many signatures. + { + let transaction = create_legacy_transaction( + 2, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..3).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_signatures(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Not enough static accounts. + { + let transaction = create_legacy_transaction( + 2, + MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..1).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_signatures(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // More than 12 signatures. + { + let transaction = create_legacy_transaction( + 13, + MessageHeader { + num_required_signatures: 13, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..13).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()); + // SignatureFrame validates number of signatures, it throw ParseError if + // it is less than 12 + assert!(matches!(view, Err(TransactionViewError::ParseError))); + } + + { + let transaction = create_v1_transaction( + 13, + MessageHeader { + num_required_signatures: 13, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..13).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_signatures(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Not enough static accounts. + { + let transaction = create_legacy_transaction( + 2, + MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..1).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_signatures(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Not enough static accounts - with look up accounts + { + let transaction = create_v0_transaction( + 2, + MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..1).map(|_| Pubkey::new_unique()).collect(), + vec![], + vec![MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![0, 1, 2, 3, 4, 5], + readonly_indexes: vec![6, 7, 8], + }], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_signatures(&view), + Err(TransactionViewError::SanitizeError) + ); + } + } + + #[test] + fn test_sanitize_account_access() { + // num_required_signatures must be >= 1. + { + let transaction = create_legacy_transaction( + 0, + MessageHeader { + num_required_signatures: 0, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + vec![Pubkey::new_unique()], + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()); + // SignatureFrame validates number of signatures, it throw ParseError if + // it is less than 1 + assert!(matches!(view, Err(TransactionViewError::ParseError))); + } + { + let transaction = create_v1_transaction( + 0, + MessageHeader { + num_required_signatures: 0, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + vec![Pubkey::new_unique()], + vec![], + TransactionConfig::empty(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_message_header(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Overlap of signing and readonly non-signing accounts. + { + let transaction = create_legacy_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 2, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_message_header(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Not enough writable accounts. + { + let transaction = create_legacy_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 1, + num_readonly_unsigned_accounts: 0, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_message_header(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Too many accounts in legacy/v0 + { + let transaction = create_v0_transaction( + 2, + MessageHeader { + num_required_signatures: 2, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + (0..1).map(|_| Pubkey::new_unique()).collect(), + vec![], + vec![ + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: (0..100).collect(), + readonly_indexes: (100..200).collect(), + }, + MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: (100..200).collect(), + readonly_indexes: (0..100).collect(), + }, + ], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_account_access(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // V1: too many static accounts. + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 63, + }, + (0..65).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_account_access(&view), + Err(TransactionViewError::SanitizeError) + ); + } + } + + #[test] + fn test_sanitize_instructions() { + let num_signatures = 1; + let header = MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }; + let account_keys = vec![Pubkey::new_unique(), Pubkey::new_unique(), Pubkey::new_unique()]; + let valid_instructions = vec![ + CompiledInstruction { + program_id_index: 1, + accounts: vec![0, 1], + data: vec![1, 2, 3], + }, + CompiledInstruction { + program_id_index: 2, + accounts: vec![1, 0], + data: vec![3, 2, 1, 4], + }, + ]; + let atls = vec![MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![0, 1], + readonly_indexes: vec![2], + }]; + + // Verify that the unmodified transaction(s) are valid/sanitized. + { + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + valid_instructions.clone(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(sanitize_instructions(&view, true).is_ok()); + + let transaction = create_v0_transaction( + num_signatures, + header, + account_keys.clone(), + valid_instructions.clone(), + atls.clone(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(sanitize_instructions(&view, true).is_ok()); + } + + for instruction_index in 0..valid_instructions.len() { + // Invalid program index. + { + let mut instructions = valid_instructions.clone(); + instructions[instruction_index].program_id_index = account_keys.len() as u8; + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + instructions, + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + + // Invalid program index with lookups. + { + let mut instructions = valid_instructions.clone(); + instructions[instruction_index].program_id_index = account_keys.len() as u8; + let transaction = create_v0_transaction( + num_signatures, + header, + account_keys.clone(), + instructions, + atls.clone(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + + // Program index is fee-payer. + { + let mut instructions = valid_instructions.clone(); + instructions[instruction_index].program_id_index = 0; + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + instructions, + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + + // Invalid account index. + { + let mut instructions = valid_instructions.clone(); + instructions[instruction_index].accounts.push(account_keys.len() as u8); + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + instructions, + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + + // Invalid account index with v0. + { + let num_lookup_accounts = + atls[0].writable_indexes.len() + atls[0].readonly_indexes.len(); + let total_accounts = (account_keys.len() + num_lookup_accounts) as u8; + let mut instructions = valid_instructions.clone(); + instructions[instruction_index].accounts.push(total_accounts); + let transaction = create_v0_transaction( + num_signatures, + header, + account_keys.clone(), + instructions, + atls.clone(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + } + + // SIMD-0160, too many instructions are invalid + { + let too_many_instructions: Vec<_> = + valid_instructions.iter().cycle().take(65).cloned().collect(); + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + too_many_instructions.clone(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + + let transaction = create_v0_transaction( + num_signatures, + header, + account_keys.clone(), + too_many_instructions.clone(), + atls.clone(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + + // SIMD-406: Limit instruction accounts to 255 + { + let mut accounts: Vec = vec![0; 254]; + accounts.push(1); + accounts.push(2); + let instr = CompiledInstruction::new_from_raw_parts(2, Vec::new(), accounts); + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + vec![instr], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_instructions(&view, true), + Err(TransactionViewError::SanitizeError) + ); + } + + // SIMD-406: Limit instruction accounts to 255 + { + let mut accounts: Vec = vec![0; 254]; + accounts.push(1); + let instr = CompiledInstruction::new_from_raw_parts(2, Vec::new(), accounts); + let transaction = create_legacy_transaction( + num_signatures, + header, + account_keys.clone(), + vec![instr], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + // Exactly 255 accounts must pass sanitization. + assert!(sanitize_instructions(&view, true).is_ok()); + } + } + + #[test] + fn test_sanitize_address_table_lookups() { + let payer = Pubkey::new_unique(); + let header = MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }; + let transaction = create_v0_transaction(1, header, vec![payer], vec![], vec![]); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(sanitize_address_table_lookups(&view).is_ok()); + + let transaction = create_v0_transaction( + 1, + header, + vec![payer], + vec![], + vec![MessageAddressTableLookup { + account_key: Pubkey::new_unique(), + writable_indexes: vec![0], + readonly_indexes: vec![], + }], + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_address_table_lookups(&view), + Err(TransactionViewError::AddressLookupMismatch) + ); + } + + #[test] + fn test_sanitize_config() { + // Valid min heap size. + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty().with_heap_size(MIN_HEAP_FRAME_BYTES), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(sanitize_config(&view).is_ok()); + } + + // Valid max heap size. + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty().with_heap_size(MAX_HEAP_FRAME_BYTES), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(sanitize_config(&view).is_ok()); + } + + // Heap size below min. + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty().with_heap_size(MIN_HEAP_FRAME_BYTES - 1), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_config(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Heap size above max. + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty().with_heap_size(MAX_HEAP_FRAME_BYTES + 1), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_config(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Heap size not multiple of 1024. + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty().with_heap_size(MIN_HEAP_FRAME_BYTES + 1), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert_eq!( + sanitize_config(&view), + Err(TransactionViewError::SanitizeError) + ); + } + + // Config is not set, default is OK + { + let transaction = create_v1_transaction( + 1, + MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + (0..2).map(|_| Pubkey::new_unique()).collect(), + vec![], + TransactionConfig::empty(), + ); + let data = wincode::serialize(&transaction).unwrap(); + let view = TransactionView::try_new_unsanitized(data.as_ref()).unwrap(); + assert!(sanitize_config(&view).is_ok()); + } + } +} diff --git a/solana/transaction-view/src/signature_frame.rs b/solana/transaction-view/src/signature_frame.rs new file mode 100644 index 00000000..ac893a69 --- /dev/null +++ b/solana/transaction-view/src/signature_frame.rs @@ -0,0 +1,112 @@ +use { + crate::{ + bytes::{advance_offset_for_array, read_byte, try_u32_offset}, + result::{Result, TransactionViewError}, + }, + solana_packet::PACKET_DATA_SIZE, + solana_pubkey::Pubkey, + solana_signature::Signature, +}; + +// The packet has a maximum length of 1232 bytes. +// Each signature must be paired with a unique static pubkey, so each +// signature really requires 96 bytes. This means the maximum number of +// signatures in a **valid** transaction packet is 12. +// In our u16 encoding scheme, 12 would be encoded as a single byte. +// Rather than using the u16 decoding, we can simply read the byte and +// verify that the MSB is not set. +pub(crate) const MAX_SIGNATURES_PER_PACKET: u8 = + (PACKET_DATA_SIZE / (core::mem::size_of::() + core::mem::size_of::())) as u8; + +/// Metadata for accessing transaction-level signatures in a transaction view. +#[derive(Debug)] +pub(crate) struct SignatureFrame { + /// The number of signatures in the transaction. + pub(crate) num_signatures: u8, + /// Offset to the first signature in the transaction packet. + pub(crate) offset: u32, +} + +impl SignatureFrame { + /// Get the number of signatures and the offset to the first signature in + /// the transaction packet, starting at the given `offset`. + #[inline(always)] + pub(crate) fn try_new(bytes: &[u8], offset: &mut usize) -> Result { + // Maximum number of signatures should be represented by a single byte, + // thus the MSB should not be set. + const _: () = assert!(MAX_SIGNATURES_PER_PACKET & 0b1000_0000 == 0); + + let num_signatures = read_byte(bytes, offset)?; + if num_signatures == 0 || num_signatures > MAX_SIGNATURES_PER_PACKET { + return Err(TransactionViewError::ParseError); + } + + let signature_offset = try_u32_offset(*offset)?; + advance_offset_for_array::(bytes, offset, u16::from(num_signatures))?; + + Ok(Self { + num_signatures, + offset: signature_offset, + }) + } +} + +#[cfg(test)] +mod tests { + use {super::*, solana_short_vec::ShortVec}; + + #[test] + fn test_zero_signatures() { + let bytes = bincode::serialize(&ShortVec(Vec::::new())).unwrap(); + let mut offset = 0; + assert!(SignatureFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_one_signature() { + let bytes = bincode::serialize(&ShortVec(vec![Signature::default()])).unwrap(); + let mut offset = 0; + let frame = SignatureFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_signatures, 1); + assert_eq!(frame.offset, 1); + assert_eq!(offset, 1 + core::mem::size_of::()); + } + + #[test] + fn test_max_signatures() { + let signatures = vec![Signature::default(); usize::from(MAX_SIGNATURES_PER_PACKET)]; + let bytes = bincode::serialize(&ShortVec(signatures)).unwrap(); + let mut offset = 0; + let frame = SignatureFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_signatures, 12); + assert_eq!(frame.offset, 1); + assert_eq!(offset, 1 + 12 * core::mem::size_of::()); + } + + #[test] + fn test_non_zero_offset() { + let mut bytes = bincode::serialize(&ShortVec(vec![Signature::default()])).unwrap(); + bytes.insert(0, 0); // Insert a byte at the beginning of the packet. + let mut offset = 1; // Start at the second byte. + let frame = SignatureFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_signatures, 1); + assert_eq!(frame.offset, 2); + assert_eq!(offset, 2 + core::mem::size_of::()); + } + + #[test] + fn test_too_many_signatures() { + let signatures = vec![Signature::default(); usize::from(MAX_SIGNATURES_PER_PACKET) + 1]; + let bytes = bincode::serialize(&ShortVec(signatures)).unwrap(); + let mut offset = 0; + assert!(SignatureFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_u16_max_signatures() { + let signatures = vec![Signature::default(); u16::MAX as usize]; + let bytes = bincode::serialize(&ShortVec(signatures)).unwrap(); + let mut offset = 0; + assert!(SignatureFrame::try_new(&bytes, &mut offset).is_err()); + } +} diff --git a/solana/transaction-view/src/static_account_keys_frame.rs b/solana/transaction-view/src/static_account_keys_frame.rs new file mode 100644 index 00000000..2d394811 --- /dev/null +++ b/solana/transaction-view/src/static_account_keys_frame.rs @@ -0,0 +1,99 @@ +use { + crate::{ + bytes::{advance_offset_for_array, read_byte, try_u32_offset}, + result::{Result, TransactionViewError}, + }, + solana_packet::PACKET_DATA_SIZE, + solana_pubkey::Pubkey, +}; + +// A legacy/v0 packet has a maximum length of 1232 bytes. +// This means the maximum number of 32 byte keys is 38. +// 38 as an min-sized encoded u16 is 1 byte. +// We can simply read this byte, if it's >38 we can return None. +const LEGACY_OR_V0_MAX_STATIC_ACCOUNTS_PER_PACKET: u8 = + (PACKET_DATA_SIZE / core::mem::size_of::()) as u8; + +/// Contains metadata about the static account keys in a transaction packet. +#[derive(Debug, Default)] +pub(crate) struct StaticAccountKeysFrame { + /// The number of static accounts in the transaction. + pub(crate) num_static_accounts: u8, + /// The offset to the first static account in the transaction. + pub(crate) offset: u32, +} + +impl StaticAccountKeysFrame { + #[inline(always)] + pub(crate) fn try_new(bytes: &[u8], offset: &mut usize) -> Result { + // Max size must not have the MSB set so that it is size 1. + const _: () = assert!(LEGACY_OR_V0_MAX_STATIC_ACCOUNTS_PER_PACKET & 0b1000_0000 == 0); + + let num_static_accounts = read_byte(bytes, offset)?; + if num_static_accounts == 0 + || num_static_accounts > LEGACY_OR_V0_MAX_STATIC_ACCOUNTS_PER_PACKET + { + return Err(TransactionViewError::ParseError); + } + + let static_accounts_offset = try_u32_offset(*offset)?; + // Update offset for array of static accounts. + advance_offset_for_array::(bytes, offset, u16::from(num_static_accounts))?; + + Ok(Self { + num_static_accounts, + offset: static_accounts_offset, + }) + } +} + +#[cfg(test)] +mod tests { + use {super::*, solana_short_vec::ShortVec}; + + #[test] + fn test_zero_accounts() { + let bytes = bincode::serialize(&ShortVec(Vec::::new())).unwrap(); + let mut offset = 0; + assert!(StaticAccountKeysFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_one_account() { + let bytes = bincode::serialize(&ShortVec(vec![Pubkey::default()])).unwrap(); + let mut offset = 0; + let frame = StaticAccountKeysFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_static_accounts, 1); + assert_eq!(frame.offset, 1); + assert_eq!(offset, 1 + core::mem::size_of::()); + } + + #[test] + fn test_max_accounts() { + let signatures = + vec![Pubkey::default(); usize::from(LEGACY_OR_V0_MAX_STATIC_ACCOUNTS_PER_PACKET)]; + let bytes = bincode::serialize(&ShortVec(signatures)).unwrap(); + let mut offset = 0; + let frame = StaticAccountKeysFrame::try_new(&bytes, &mut offset).unwrap(); + assert_eq!(frame.num_static_accounts, 38); + assert_eq!(frame.offset, 1); + assert_eq!(offset, 1 + 38 * core::mem::size_of::()); + } + + #[test] + fn test_too_many_accounts() { + let signatures = + vec![Pubkey::default(); usize::from(LEGACY_OR_V0_MAX_STATIC_ACCOUNTS_PER_PACKET) + 1]; + let bytes = bincode::serialize(&ShortVec(signatures)).unwrap(); + let mut offset = 0; + assert!(StaticAccountKeysFrame::try_new(&bytes, &mut offset).is_err()); + } + + #[test] + fn test_u16_max_accounts() { + let signatures = vec![Pubkey::default(); u16::MAX as usize]; + let bytes = bincode::serialize(&ShortVec(signatures)).unwrap(); + let mut offset = 0; + assert!(StaticAccountKeysFrame::try_new(&bytes, &mut offset).is_err()); + } +} diff --git a/solana/transaction-view/src/transaction_config_frame.rs b/solana/transaction-view/src/transaction_config_frame.rs new file mode 100644 index 00000000..c2a3392d --- /dev/null +++ b/solana/transaction-view/src/transaction_config_frame.rs @@ -0,0 +1,451 @@ +use crate::{ + bytes::{advance_offset_for_array, try_u32_offset, unchecked_copy_value}, + result::{Result, TransactionViewError}, +}; + +/// Metadata for accessing the tx-v1 transaction config section. +/// +/// This frame is a permanent part of `TransactionFrame`, but it is only +/// applicable to tx-v1. For legacy and v0 transactions, use +/// `TransactionConfigFrame::not_applicable()`. +/// +/// Layout, per SIMD-0385: +/// TransactionConfigMask (u32 LE) +/// ... +/// ConfigValues [[u8; 4]] // len = popcount(mask) +/// +/// Notes: +/// - `mask_offset == 0` is reserved to mean "not applicable" (legacy/v0). +/// - Parsed tx-v1 config frames should always have `mask_offset != 0`. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct TransactionConfigFrame { + /// Offset of the 4-byte TransactionConfigMask. + /// + /// `0` means "not applicable" (legacy/v0). + pub(crate) mask_offset: u32, + + /// Decoded TransactionConfigMask. + pub(crate) mask: u32, + + /// Offset of the first ConfigValues word. + /// + /// `0` means "not applicable" (legacy/v0) + pub(crate) values_offset: u32, + + /// Number of 4-byte words in ConfigValues. + pub(crate) num_values: u8, +} + +#[allow(dead_code)] +impl TransactionConfigFrame { + pub(crate) const MASK_SIZE: usize = core::mem::size_of::(); + pub(crate) const CONFIG_VALUE_SIZE: usize = core::mem::size_of::(); + + /// Sentinel for legacy / v0 transactions. + #[inline(always)] + pub(crate) const fn not_applicable() -> Self { + Self { + mask_offset: 0, + mask: 0, + values_offset: 0, + num_values: 0, + } + } + + /// Returns true if this frame represents a tx-v1 transaction config. + #[inline(always)] + pub(crate) const fn is_present(&self) -> bool { + self.mask_offset != 0 + } + + /// Config Mask has been successfully parsed before advancing to `ConfigValues` + /// region; Now can try to create TransactionConfigFrame by parsing values. + #[inline(always)] + pub(crate) fn try_new( + bytes: &[u8], + mask_offset: usize, + mask: u32, + offset: &mut usize, + ) -> Result { + assert!(mask_offset > 0, "txv1 mask offset must be greater than 0"); + + Self::sanitize_mask(mask)?; + let num_values = mask.count_ones() as u8; + let mask_offset = try_u32_offset(mask_offset)?; + let values_offset = try_u32_offset(*offset)?; + + // advance offset + advance_offset_for_array::(bytes, offset, num_values as u16)?; + + Ok(Self { + mask_offset, + mask, + values_offset, + num_values, + }) + } + + /// Validate mask semantics. + /// + /// Check unknown / reserved bits are not used; And + /// Bits 0 and 1 together encode one logical 8-byte priority-fee field, + /// so they must either both be set or both be clear. + #[inline(always)] + fn sanitize_mask(mask: u32) -> Result<()> { + const ALLOWED_TRANSACTION_CONFIG_MASK: u32 = 0b1_1111; + + // Reject unknown / reserved bits + if mask & !ALLOWED_TRANSACTION_CONFIG_MASK != 0 { + return Err(TransactionViewError::SanitizeError); + } + + // priority fee uses first 2 bits + let bit0 = Self::has_bit(mask, 0); + let bit1 = Self::has_bit(mask, 1); + if bit0 ^ bit1 { + return Err(TransactionViewError::SanitizeError); + } + + Ok(()) + } + + #[inline(always)] + fn has_bit(mask: u32, bit: u8) -> bool { + bit < 32 && ((mask >> bit) & 1) != 0 + } + + /// Return the packed word index for a given set bit. Eg: counts + /// bits set below `bit`. + /// + /// Example: + /// mask = 0b0001_1100 + /// bit 2 -> 0 + /// bit 3 -> 1 + /// bit 4 -> 2 + #[inline(always)] + pub(crate) fn word_index_for_bit(&self, bit: u8) -> Option { + if !self.is_present() || !Self::has_bit(self.mask, bit) { + return None; + } + + let mask_before_bit = (1u32 << bit).wrapping_sub(1); + Some((self.mask & mask_before_bit).count_ones() as u8) + } + + #[inline(always)] + fn word_offset(&self, bit: u8) -> Option { + let word_index = usize::from(self.word_index_for_bit(bit)?); + (self.values_offset as usize).checked_add(word_index.checked_mul(Self::CONFIG_VALUE_SIZE)?) + } +} + +#[derive(Debug, Clone, Copy)] +pub struct TransactionConfigView<'a> { + pub(crate) transaction_config_frame: &'a TransactionConfigFrame, + pub(crate) bytes: &'a [u8], +} + +impl<'a> TransactionConfigView<'a> { + #[inline(always)] + pub fn priority_fee_lamports(&self) -> Option { + // bit 0 and 1 have been sanitized to be in same state, + self.transaction_config_frame.word_offset(0).map(|offset| { + // SAFETY: + // - The offsets are checked to be valid in the byte slice. + // - u64 is valid for any bytes + u64::from_le(unsafe { unchecked_copy_value(self.bytes, offset) }) + }) + } + + #[inline(always)] + pub fn compute_unit_limit(&self) -> Option { + self.transaction_config_frame.word_offset(2).map(|offset| { + // SAFETY: + // - The offsets are checked to be valid in the byte slice. + // - u32 is valid for any bytes + u32::from_le(unsafe { unchecked_copy_value(self.bytes, offset) }) + }) + } + + #[inline(always)] + pub fn loaded_accounts_data_size_limit(&self) -> Option { + self.transaction_config_frame.word_offset(3).map(|offset| { + // SAFETY: + // - The offsets are checked to be valid in the byte slice. + // - u32 is valid for any bytes + u32::from_le(unsafe { unchecked_copy_value(self.bytes, offset) }) + }) + } + + #[inline(always)] + pub fn requested_heap_size(&self) -> Option { + self.transaction_config_frame.word_offset(4).map(|offset| { + // SAFETY: + // - The offsets are checked to be valid in the byte slice. + // - u32 is valid for any bytes + u32::from_le(unsafe { unchecked_copy_value(self.bytes, offset) }) + }) + } + + #[inline(always)] + pub fn mask(&self) -> u32 { + self.transaction_config_frame.mask + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn u32le(x: u32) -> [u8; 4] { + x.to_le_bytes() + } + + fn u64_words_le(x: u64) -> ([u8; 4], [u8; 4]) { + let bytes = x.to_le_bytes(); + ( + [bytes[0], bytes[1], bytes[2], bytes[3]], + [bytes[4], bytes[5], bytes[6], bytes[7]], + ) + } + + #[test] + fn test_not_applicable_defaults() { + let frame = TransactionConfigFrame::not_applicable(); + + assert!(!frame.is_present()); + assert_eq!(TransactionConfigFrame::sanitize_mask(frame.mask), Ok(())); + } + + #[test] + fn test_try_new_zero_mask_is_present() { + let mask = 0u32; + let bytes = mask.to_le_bytes(); + let mut buf = vec![0u8; 5]; + let mask_offset = 5; + buf.extend_from_slice(&bytes); + + let mut offset = buf.len(); + let frame = TransactionConfigFrame::try_new(&buf, mask_offset, mask, &mut offset).unwrap(); + + assert!(frame.is_present()); + assert_eq!(frame.mask_offset, 5); + assert_eq!(frame.mask, 0); + assert_eq!(frame.num_values, 0); + assert_eq!(offset, 9); + } + + #[test] + fn test_try_new_invalid_priority_fee_half_set_low_bit() { + let mask = 0b00001u32; + let bytes = mask.to_le_bytes(); + let mut buf = vec![0u8; 5]; + let mask_offset = 5; + buf.extend_from_slice(&bytes); + let mut offset = 5; + + assert_eq!( + TransactionConfigFrame::try_new(&buf, mask_offset, mask, &mut offset), + Err(TransactionViewError::SanitizeError) + ); + } + + #[test] + fn test_try_new_invalid_priority_fee_half_set_high_bit() { + let mask = 0b00010u32; + let bytes = mask.to_le_bytes(); + let mut buf = vec![0u8; 5]; + let mask_offset = 5; + buf.extend_from_slice(&bytes); + let mut offset = 5; + + assert_eq!( + TransactionConfigFrame::try_new(&buf, mask_offset, mask, &mut offset), + Err(TransactionViewError::SanitizeError) + ); + } + + #[test] + fn test_try_new_invalid_config_values() { + // bits 0,1,2 => 3 words => 12 bytes needed + let mask = 0b00111u32; + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[7u8; 3]); + let mask_offset = bytes.len(); + bytes.extend_from_slice(&mask.to_le_bytes()); + + let mut offset = bytes.len(); + assert_eq!( + TransactionConfigFrame::try_new(&bytes, mask_offset, mask, &mut offset), + Err(TransactionViewError::ParseError) + ); + } + + #[test] + fn test_read_defaults_when_bits_unset() { + let mask = 0u32; + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[1u8; 2]); + let mask_offset = bytes.len(); + bytes.extend_from_slice(&mask.to_le_bytes()); + + let mut offset = bytes.len(); + let frame = + TransactionConfigFrame::try_new(&bytes, mask_offset, mask, &mut offset).unwrap(); + let view = TransactionConfigView { + transaction_config_frame: &frame, + bytes: &bytes, + }; + + assert!(view.priority_fee_lamports().is_none()); + assert!(view.compute_unit_limit().is_none()); + assert!(view.loaded_accounts_data_size_limit().is_none()); + assert!(view.requested_heap_size().is_none()); + } + + #[test] + fn test_unknown_bits_rejected() { + // Single unknown bit (bit 5) + assert_eq!( + TransactionConfigFrame::sanitize_mask(0b10_0000), + Err(TransactionViewError::SanitizeError) + ); + // Multiple unknown bits + assert_eq!( + TransactionConfigFrame::sanitize_mask(0b1111_1111), + Err(TransactionViewError::SanitizeError) + ); + // High bits set + assert_eq!( + TransactionConfigFrame::sanitize_mask(1 << 31), + Err(TransactionViewError::SanitizeError) + ); + // Unknown bits mixed with valid bits + assert_eq!( + TransactionConfigFrame::sanitize_mask(0b1_1111 | (1 << 16)), + Err(TransactionViewError::SanitizeError) + ); + } + + #[test] + fn test_priority_fee_only() { + let mask = 0b00011u32; + let fee = 123_456_789u64; + let (lo, hi) = u64_words_le(fee); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[9u8; 4]); + let mask_offset = bytes.len(); + bytes.extend_from_slice(&mask.to_le_bytes()); + let values_offset = bytes.len(); + bytes.extend_from_slice(&lo); + bytes.extend_from_slice(&hi); + + let mut offset = values_offset; + let frame = TransactionConfigFrame::try_new(&bytes, mask_offset, mask, &mut offset) + .inspect(|_| assert_eq!(offset, bytes.len())) + .unwrap(); + assert!(frame.is_present()); + assert_eq!(frame.num_values, 2); + + let view = TransactionConfigView { + transaction_config_frame: &frame, + bytes: &bytes, + }; + assert_eq!(view.priority_fee_lamports().unwrap(), fee); + assert!(view.compute_unit_limit().is_none()); + assert!(view.loaded_accounts_data_size_limit().is_none()); + assert!(view.requested_heap_size().is_none()); + } + + #[test] + fn test_all_initial_fields_present() { + // bits 0,1,2,3,4 => priority fee + cu + loaded data size + heap size + let mask = 0b1_1111u32; + let fee = 99u64; + let cu = 1_400_000u32; + let loaded = 64_000u32; + let heap = 64 * 1024u32; + + let (fee_lo, fee_hi) = u64_words_le(fee); + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[9u8; 7]); + let mask_offset = bytes.len(); + bytes.extend_from_slice(&mask.to_le_bytes()); + let values_offset = bytes.len(); + bytes.extend_from_slice(&fee_lo); + bytes.extend_from_slice(&fee_hi); + bytes.extend_from_slice(&u32le(cu)); + bytes.extend_from_slice(&u32le(loaded)); + bytes.extend_from_slice(&u32le(heap)); + + let mut offset = values_offset; + let frame = TransactionConfigFrame::try_new(&bytes, mask_offset, mask, &mut offset) + .inspect(|_| assert_eq!(offset, bytes.len())) + .unwrap(); + assert!(frame.is_present()); + assert_eq!(frame.num_values, 5); + + let view = TransactionConfigView { + transaction_config_frame: &frame, + bytes: &bytes, + }; + assert_eq!(view.priority_fee_lamports().unwrap(), fee); + assert_eq!(view.compute_unit_limit().unwrap(), cu); + assert_eq!(view.loaded_accounts_data_size_limit().unwrap(), loaded); + assert_eq!(view.requested_heap_size().unwrap(), heap); + } + + #[test] + fn test_sparse_bits_word_indexing() { + // bits 2 and 4 only + let mask = 0b10100u32; + let cu = 777u32; + let heap = 48 * 1024u32; + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[1u8; 3]); + let mask_offset = bytes.len(); + bytes.extend_from_slice(&mask.to_le_bytes()); + let values_offset = bytes.len(); + bytes.extend_from_slice(&u32le(cu)); // bit 2 -> word 0 + bytes.extend_from_slice(&u32le(heap)); // bit 4 -> word 1 + + let mut offset = values_offset; + let frame = TransactionConfigFrame::try_new(&bytes, mask_offset, mask, &mut offset) + .inspect(|_| assert_eq!(offset, bytes.len())) + .unwrap(); + assert_eq!(frame.word_index_for_bit(2), Some(0)); + assert_eq!(frame.word_index_for_bit(4), Some(1)); + assert_eq!(frame.word_index_for_bit(3), None); + + let view = TransactionConfigView { + transaction_config_frame: &frame, + bytes: &bytes, + }; + assert!(view.priority_fee_lamports().is_none()); + assert_eq!(view.compute_unit_limit().unwrap(), cu); + assert!(view.loaded_accounts_data_size_limit().is_none()); + assert_eq!(view.requested_heap_size().unwrap(), heap); + } + + #[test] + fn test_truncated_priority_fee_values() { + let mask = 0b00011u32; + + let mut bytes = Vec::new(); + bytes.extend_from_slice(&[5u8; 2]); + let mask_offset = bytes.len(); + bytes.extend_from_slice(&mask.to_le_bytes()); + let values_offset = bytes.len(); + bytes.extend_from_slice(&[1, 2, 3, 4]); // only one word present + + let mut offset = values_offset; + assert_eq!( + TransactionConfigFrame::try_new(&bytes, mask_offset, mask, &mut offset), + Err(TransactionViewError::ParseError) + ); + } +} diff --git a/solana/transaction-view/src/transaction_data.rs b/solana/transaction-view/src/transaction_data.rs new file mode 100644 index 00000000..323c0856 --- /dev/null +++ b/solana/transaction-view/src/transaction_data.rs @@ -0,0 +1,19 @@ +/// Trait for accessing transaction data from an abstract byte container. +pub trait TransactionData { + /// Returns a reference to the serialized transaction data. + fn data(&self) -> &[u8]; +} + +impl TransactionData for &[u8] { + #[inline] + fn data(&self) -> &[u8] { + self + } +} + +impl TransactionData for std::sync::Arc> { + #[inline] + fn data(&self) -> &[u8] { + self.as_ref() + } +} diff --git a/solana/transaction-view/src/transaction_frame.rs b/solana/transaction-view/src/transaction_frame.rs new file mode 100644 index 00000000..219c3568 --- /dev/null +++ b/solana/transaction-view/src/transaction_frame.rs @@ -0,0 +1,980 @@ +use { + crate::{ + address_table_lookup_frame::{AddressTableLookupFrame, AddressTableLookupIterator}, + bytes::{ + advance_offset_for_array, advance_offset_for_type, check_remaining, try_u32_offset, + unchecked_copy_value, unchecked_read_byte, + }, + instructions_frame::{InstructionsFrame, InstructionsIterator}, + message_header_frame::MessageHeaderFrame, + result::{Result, TransactionViewError}, + signature_frame::SignatureFrame, + static_account_keys_frame::StaticAccountKeysFrame, + transaction_config_frame::TransactionConfigFrame, + transaction_version::{MAGICBLOCK_PREFIX, TransactionVersion}, + }, + solana_hash::Hash, + solana_pubkey::Pubkey, + solana_signature::Signature, +}; + +#[derive(Debug)] +pub(crate) struct TransactionFrame { + /// Signature framing data. + signature: SignatureFrame, + /// Message header framing data. + message_header: MessageHeaderFrame, + /// Static account keys framing data. + static_account_keys: StaticAccountKeysFrame, + /// Recent blockhash offset. + recent_blockhash_offset: u32, + /// Instructions framing data. + instructions: InstructionsFrame, + /// Address table lookup framing data. + address_table_lookup: AddressTableLookupFrame, + /// Transaction config framing data + transaction_config_frame: TransactionConfigFrame, + /// The data length in bytes + data_len: u32, +} + +impl TransactionFrame { + /// Parse a serialized transaction and verify basic structure. + /// The `bytes` parameter must have no trailing data. + pub(crate) fn try_new(bytes: &[u8]) -> Result { + try_u32_offset(bytes.len())?; + if Self::is_legacy_or_v0(bytes)? { + Self::try_new_as_legacy_or_v0(bytes) + } else { + Self::try_new_as_v1(bytes) + } + } + + fn try_new_as_legacy_or_v0(bytes: &[u8]) -> Result { + let mut offset = 0; + let signature = SignatureFrame::try_new(bytes, &mut offset)?; + let message_header = MessageHeaderFrame::try_new(bytes, &mut offset)?; + let static_account_keys = StaticAccountKeysFrame::try_new(bytes, &mut offset)?; + + // The recent blockhash is the first account key after the static + // account keys. The recent blockhash is always present in a valid + // transaction and has a fixed size of 32 bytes. + let recent_blockhash_offset = try_u32_offset(offset)?; + advance_offset_for_type::(bytes, &mut offset)?; + + let instructions = InstructionsFrame::try_new_for_legacy_and_v0(bytes, &mut offset)?; + let address_table_lookup = match message_header.version { + TransactionVersion::Legacy => AddressTableLookupFrame { + num_address_table_lookups: 0, + offset: 0, + total_writable_lookup_accounts: 0, + total_readonly_lookup_accounts: 0, + }, + TransactionVersion::V0 => AddressTableLookupFrame::try_new(bytes, &mut offset)?, + TransactionVersion::V1 | TransactionVersion::Magicblock => { + unreachable!("unexpected variant") + } + }; + + // Verify that the entire transaction was parsed. + if offset != bytes.len() { + return Err(TransactionViewError::ParseError); + } + + Ok(Self { + signature, + message_header, + static_account_keys, + recent_blockhash_offset, + instructions, + address_table_lookup, + transaction_config_frame: TransactionConfigFrame::not_applicable(), + data_len: try_u32_offset(offset)?, + }) + } + + fn try_new_as_v1(bytes: &[u8]) -> Result { + let mut offset: usize = 0; + + // Fixed-size txv1 prefix up through NumAddresses: + // VersionByte (u8) + // LegacyHeader (u8, u8, u8) + // TransactionConfigMask (u32) + // LifetimeSpecifier ([u8; 32]) + // NumInstructions (u8) + // NumAddresses (u8) + const FIXED_V1_PREFIX_LEN: usize = 1 + 3 + 4 + size_of::() + 1 + 1; + + check_remaining(bytes, offset, FIXED_V1_PREFIX_LEN)?; + + // SAFETY: have checked bytes have enough space for preifx all the way up to + // NumAddresses. + + // message offset would be the first byte of txv1 packet, which is version byte + let message_offset = try_u32_offset(offset)?; + // Version Byte + let version = unsafe { unchecked_read_byte(bytes, &mut offset) }; + let version = match version { + solana_message::v1::V1_PREFIX => TransactionVersion::V1, + MAGICBLOCK_PREFIX => TransactionVersion::Magicblock, + _ => return Err(TransactionViewError::ParseError), + }; + // Legacy Header + let num_required_signatures = unsafe { unchecked_read_byte(bytes, &mut offset) }; + let num_readonly_signed_accounts = unsafe { unchecked_read_byte(bytes, &mut offset) }; + let num_readonly_unsigned_accounts = unsafe { unchecked_read_byte(bytes, &mut offset) }; + // Transaction Config Bit Mask + let transaction_config_mask_offset = offset; + let transaction_config_mask: u32 = unsafe { unchecked_copy_value(bytes, offset) }; + offset = offset.checked_add(size_of::()).ok_or(TransactionViewError::ParseError)?; + // Lifetime specifier + let recent_blockhash_offset = try_u32_offset(offset)?; + offset = offset.checked_add(size_of::()).ok_or(TransactionViewError::ParseError)?; + // Num instructions and addresses + let num_instructions = unsafe { unchecked_read_byte(bytes, &mut offset) }; + let num_addresses = unsafe { unchecked_read_byte(bytes, &mut offset) }; + + // addresses + let addresses_offset = try_u32_offset(offset)?; + advance_offset_for_array::(bytes, &mut offset, u16::from(num_addresses))?; + // config value slots: one 4-byte slot per set bit in mask + let transaction_config_frame = TransactionConfigFrame::try_new( + bytes, + transaction_config_mask_offset, + transaction_config_mask, + &mut offset, + )?; + // instruction headers and payloads + let instructions = InstructionsFrame::try_new_for_v1(bytes, &mut offset, num_instructions)?; + // signatures + let signatures_offset = try_u32_offset(offset)?; + advance_offset_for_array::( + bytes, + &mut offset, + u16::from(num_required_signatures), + )?; + // Verify that the entire transaction was parsed. + if offset != bytes.len() { + return Err(TransactionViewError::ParseError); + } + + let frame = Self { + signature: SignatureFrame { + num_signatures: num_required_signatures, + offset: signatures_offset, + }, + message_header: MessageHeaderFrame { + offset: message_offset, + version, + num_required_signatures, + num_readonly_signed_accounts, + num_readonly_unsigned_accounts, + }, + static_account_keys: StaticAccountKeysFrame { + num_static_accounts: num_addresses, // always static accounts in txv1 + offset: addresses_offset, + }, + recent_blockhash_offset, + instructions, + // Don't have ATL in txv1 + address_table_lookup: AddressTableLookupFrame { + num_address_table_lookups: 0, + offset: 0, + total_writable_lookup_accounts: 0, + total_readonly_lookup_accounts: 0, + }, + transaction_config_frame, + data_len: try_u32_offset(offset)?, + }; + + Ok(frame) + } + + fn is_legacy_or_v0(bytes: &[u8]) -> Result { + let first_byte = *bytes.first().ok_or(TransactionViewError::ParseError)?; + + // In wire format: + // - Legacy/v0 transactions start with signatures (compact-u16 count). + // The retained signature-count limit keeps the first byte below 128. + // - v1 transactions start with a version byte with MSB = 1. + Ok((first_byte & solana_message::MESSAGE_VERSION_PREFIX) == 0) + } + + /// Return the number of signatures in the transaction. + #[inline] + pub(crate) fn num_signatures(&self) -> u8 { + self.signature.num_signatures + } + + /// Return the version of the transaction. + #[inline] + pub(crate) fn version(&self) -> TransactionVersion { + self.message_header.version + } + + /// Return the number of required signatures in the transaction. + #[inline] + pub(crate) fn num_required_signatures(&self) -> u8 { + self.message_header.num_required_signatures + } + + /// Return the number of readonly signed static accounts in the transaction. + #[inline] + pub(crate) fn num_readonly_signed_static_accounts(&self) -> u8 { + self.message_header.num_readonly_signed_accounts + } + + /// Return the number of readonly unsigned static accounts in the transaction. + #[inline] + pub(crate) fn num_readonly_unsigned_static_accounts(&self) -> u8 { + self.message_header.num_readonly_unsigned_accounts + } + + /// Return the number of static account keys in the transaction. + #[inline] + pub(crate) fn num_static_account_keys(&self) -> u8 { + self.static_account_keys.num_static_accounts + } + + /// Return the number of instructions in the transaction. + #[inline] + pub(crate) fn num_instructions(&self) -> u16 { + self.instructions.num_instructions() + } + + /// Return the number of address table lookups in the transaction. + #[inline] + pub(crate) fn num_address_table_lookups(&self) -> u8 { + self.address_table_lookup.num_address_table_lookups + } + + /// Return the number of writable lookup accounts in the transaction. + #[inline] + pub(crate) fn total_writable_lookup_accounts(&self) -> u16 { + self.address_table_lookup.total_writable_lookup_accounts + } + + /// Return the number of readonly lookup accounts in the transaction. + #[inline] + pub(crate) fn total_readonly_lookup_accounts(&self) -> u16 { + self.address_table_lookup.total_readonly_lookup_accounts + } + + /// Return the range to the message as [begin, end] + #[inline] + pub(crate) fn message_range(&self) -> (u32, u32) { + let end = match self.version() { + TransactionVersion::V1 | TransactionVersion::Magicblock => self.signature.offset, + _ => self.data_len, + }; + (self.message_header.offset, end) + } + + /// Return transaction_config_frame + #[inline] + pub(crate) fn transaction_config_frame(&self) -> &TransactionConfigFrame { + &self.transaction_config_frame + } +} + +// Separate implementation for `unsafe` accessor methods. +impl TransactionFrame { + /// Return the slice of signatures in the transaction. + /// # Safety + /// - This function must be called with the same `bytes` slice that was + /// used to create the `TransactionFrame` instance. + #[inline] + pub(crate) unsafe fn signatures<'a>(&self, bytes: &'a [u8]) -> &'a [Signature] { + // Verify at compile time there are no alignment constraints. + const _: () = assert!(align_of::() == 1, "Signature alignment"); + // The length of the slice is not greater than isize::MAX. + const _: () = assert!(u8::MAX as usize * size_of::() <= isize::MAX as usize); + + // SAFETY: + // - If this `TransactionFrame` was created from `bytes`: + // - the pointer is valid for the range and is properly aligned. + // - `num_signatures` has been verified against the bounds if + // `TransactionFrame` was created successfully. + // - `Signature` are just byte arrays; there is no possibility the + // `Signature` are not initialized properly. + // - The lifetime of the returned slice is the same as the input + // `bytes`. This means it will not be mutated or deallocated while + // holding the slice. + // - The length does not overflow `isize`. + let start = self.signature.offset as usize; + let end = start + usize::from(self.signature.num_signatures) * size_of::(); + let signature_bytes = &bytes[start..end]; + unsafe { + core::slice::from_raw_parts( + signature_bytes.as_ptr() as *const Signature, + usize::from(self.signature.num_signatures), + ) + } + } + + /// Return the slice of static account keys in the transaction. + /// + /// # Safety + /// - This function must be called with the same `bytes` slice that was + /// used to create the `TransactionFrame` instance. + #[inline] + pub(crate) unsafe fn static_account_keys<'a>(&self, bytes: &'a [u8]) -> &'a [Pubkey] { + // Verify at compile time there are no alignment constraints. + const _: () = assert!(align_of::() == 1, "Pubkey alignment"); + // The length of the slice is not greater than isize::MAX. + const _: () = assert!(u8::MAX as usize * size_of::() <= isize::MAX as usize); + + // SAFETY: + // - If this `TransactionFrame` was created from `bytes`: + // - the pointer is valid for the range and is properly aligned. + // - `num_static_accounts` has been verified against the bounds if + // `TransactionFrame` was created successfully. + // - `Pubkey` are just byte arrays; there is no possibility the + // `Pubkey` are not initialized properly. + // - The lifetime of the returned slice is the same as the input + // `bytes`. This means it will not be mutated or deallocated while + // holding the slice. + // - The length does not overflow `isize`. + let start = self.static_account_keys.offset as usize; + let end = + start + usize::from(self.static_account_keys.num_static_accounts) * size_of::(); + let account_bytes = &bytes[start..end]; + unsafe { + core::slice::from_raw_parts( + account_bytes.as_ptr() as *const Pubkey, + usize::from(self.static_account_keys.num_static_accounts), + ) + } + } + + /// Return the recent blockhash in the transaction. + /// # Safety + /// - This function must be called with the same `bytes` slice that was + /// used to create the `TransactionFrame` instance. + #[inline] + pub(crate) unsafe fn recent_blockhash<'a>(&self, bytes: &'a [u8]) -> &'a Hash { + // Verify at compile time there are no alignment constraints. + const _: () = assert!(align_of::() == 1, "Hash alignment"); + + // SAFETY: + // - The pointer is correctly aligned (no alignment constraints). + // - `Hash` is just a byte array; there is no possibility the `Hash` + // is not initialized properly. + // - Aliasing rules are respected because the lifetime of the returned + // reference is the same as the input/source `bytes`. + let start = self.recent_blockhash_offset as usize; + let hash_bytes = &bytes[start..start + size_of::()]; + unsafe { &*(hash_bytes.as_ptr() as *const Hash) } + } + + /// Return an iterator over the instructions in the transaction. + /// # Safety + /// - This function must be called with the same `bytes` slice that was + /// used to create the `TransactionFrame` instance. + #[inline] + pub(crate) unsafe fn instructions_iter<'a>( + &'a self, + bytes: &'a [u8], + ) -> InstructionsIterator<'a> { + self.instructions.iter(bytes) + } + + /// Return an iterator over the address table lookups in the transaction. + /// # Safety + /// - This function must be called with the same `bytes` slice that was + /// used to create the `TransactionFrame` instance. + #[inline] + pub(crate) unsafe fn address_table_lookup_iter<'a>( + &self, + bytes: &'a [u8], + ) -> AddressTableLookupIterator<'a> { + AddressTableLookupIterator { + bytes, + offset: self.address_table_lookup.offset as usize, + num_address_table_lookups: self.address_table_lookup.num_address_table_lookups, + index: 0, + } + } +} + +#[cfg(test)] +impl TransactionFrame { + pub(crate) fn message_offset(&self) -> u32 { + self.message_header.offset + } + + pub(crate) fn signatures_offset(&self) -> u32 { + self.signature.offset + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + solana_message::{ + AddressLookupTableAccount, Message, MessageHeader, VersionedMessage, + compiled_instruction::CompiledInstruction, v0, v1, + }, + solana_pubkey::Pubkey, + solana_signature::Signature, + solana_system_interface::instruction::{self as system_instruction, SystemInstruction}, + solana_transaction::versioned::VersionedTransaction, + }; + + fn verify_transaction_view_frame(tx: &VersionedTransaction) { + let bytes = wincode::serialize(tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + assert_eq!(frame.signature.num_signatures, tx.signatures.len() as u8); + assert_eq!(frame.signature.offset as usize, 1); + + assert_eq!( + frame.message_header.num_required_signatures, + tx.message.header().num_required_signatures + ); + assert_eq!( + frame.message_header.num_readonly_signed_accounts, + tx.message.header().num_readonly_signed_accounts + ); + assert_eq!( + frame.message_header.num_readonly_unsigned_accounts, + tx.message.header().num_readonly_unsigned_accounts + ); + + assert_eq!( + frame.static_account_keys.num_static_accounts, + tx.message.static_account_keys().len() as u8 + ); + assert_eq!( + frame.instructions.num_instructions(), + tx.message.instructions().len() as u16 + ); + assert_eq!( + frame.address_table_lookup.num_address_table_lookups, + tx.message.address_table_lookups().map(|x| x.len() as u8).unwrap_or(0) + ); + } + + fn minimally_sized_transaction() -> VersionedTransaction { + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::Legacy(Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + account_keys: vec![Pubkey::default()], + recent_blockhash: Hash::default(), + instructions: vec![], + }), + } + } + + fn simple_transfer() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::Legacy(Message::new( + &[system_instruction::transfer(&payer, &Pubkey::new_unique(), 1)], + Some(&payer), + )), + } + } + + fn simple_transfer_v0() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::V0( + v0::Message::try_compile( + &payer, + &[system_instruction::transfer(&payer, &Pubkey::new_unique(), 1)], + &[], + Hash::default(), + ) + .unwrap(), + ), + } + } + + fn multiple_transfers() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::Legacy(Message::new( + &[ + system_instruction::transfer(&payer, &Pubkey::new_unique(), 1), + system_instruction::transfer(&payer, &Pubkey::new_unique(), 1), + ], + Some(&payer), + )), + } + } + + fn v0_with_single_lookup() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + let to = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::V0( + v0::Message::try_compile( + &payer, + &[system_instruction::transfer(&payer, &to, 1)], + &[AddressLookupTableAccount { + key: Pubkey::new_unique(), + addresses: vec![to], + }], + Hash::default(), + ) + .unwrap(), + ), + } + } + + fn v0_with_multiple_lookups() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + let to1 = Pubkey::new_unique(); + let to2 = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::V0( + v0::Message::try_compile( + &payer, + &[ + system_instruction::transfer(&payer, &to1, 1), + system_instruction::transfer(&payer, &to2, 1), + ], + &[ + AddressLookupTableAccount { + key: Pubkey::new_unique(), + addresses: vec![to1], + }, + AddressLookupTableAccount { + key: Pubkey::new_unique(), + addresses: vec![to2], + }, + ], + Hash::default(), + ) + .unwrap(), + ), + } + } + + fn simple_v1_transaction() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + let program = Pubkey::new_unique(); + let other = Pubkey::new_unique(); + + VersionedTransaction { + signatures: vec![Signature::default()], + message: VersionedMessage::V1(v1::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 1, + }, + config: v1::TransactionConfig { + priority_fee: Some(123), + compute_unit_limit: Some(456), + loaded_accounts_data_size_limit: Some(789), + heap_size: Some(1024), + }, + lifetime_specifier: Hash::default(), + account_keys: vec![payer, other, program], + instructions: vec![ + CompiledInstruction { + program_id_index: 2, + accounts: vec![0, 1], + data: vec![10, 11, 12], + }, + CompiledInstruction { + program_id_index: 2, + accounts: vec![], + data: vec![99], + }, + ], + }), + } + } + + #[test] + fn test_minimal_sized_transaction() { + verify_transaction_view_frame(&minimally_sized_transaction()); + } + + #[test] + fn test_simple_transfer() { + verify_transaction_view_frame(&simple_transfer()); + } + + #[test] + fn test_simple_transfer_v0() { + verify_transaction_view_frame(&simple_transfer_v0()); + } + + #[test] + fn test_v0_with_lookup() { + verify_transaction_view_frame(&v0_with_single_lookup()); + } + + #[test] + fn test_trailing_byte() { + let tx = simple_transfer(); + let mut bytes = wincode::serialize(&tx).unwrap(); + bytes.push(0); + assert!(TransactionFrame::try_new(&bytes).is_err()); + } + + #[test] + fn test_insufficient_bytes() { + let tx = simple_transfer(); + let bytes = wincode::serialize(&tx).unwrap(); + assert!(TransactionFrame::try_new(&bytes[..bytes.len().wrapping_sub(1)]).is_err()); + } + + #[test] + fn test_signature_overflow() { + let tx = simple_transfer(); + let mut bytes = wincode::serialize(&tx).unwrap(); + // Set the number of signatures to u16::MAX + bytes[0] = 0xff; + bytes[1] = 0xff; + bytes[2] = 0xff; + assert!(TransactionFrame::try_new(&bytes).is_err()); + } + + #[test] + fn test_account_key_overflow() { + let tx = simple_transfer(); + let mut bytes = wincode::serialize(&tx).unwrap(); + // Set the number of accounts to u16::MAX + let offset = 1 + size_of::() + 3; + bytes[offset] = 0xff; + bytes[offset + 1] = 0xff; + bytes[offset + 2] = 0xff; + assert!(TransactionFrame::try_new(&bytes).is_err()); + } + + #[test] + fn test_instructions_overflow() { + let tx = simple_transfer(); + let mut bytes = wincode::serialize(&tx).unwrap(); + // Set the number of instructions to u16::MAX + let offset = + 1 + size_of::() + 3 + 1 + 3 * size_of::() + size_of::(); + bytes[offset] = 0xff; + bytes[offset + 1] = 0xff; + bytes[offset + 2] = 0xff; + assert!(TransactionFrame::try_new(&bytes).is_err()); + } + + #[test] + fn test_alt_overflow() { + let tx = simple_transfer_v0(); + let ix_bytes = tx.message.instructions()[0].data.len(); + let mut bytes = wincode::serialize(&tx).unwrap(); + // Set the number of instructions to u16::MAX + let offset = 1 // byte for num signatures + + size_of::() // signature + + 1 // version byte + + 3 // message header + + 1 // byte for num account keys + + 3 * size_of::() // account keys + + size_of::() // recent blockhash + + 1 // byte for num instructions + + 1 // program index + + 1 // byte for num accounts + + 2 // bytes for account index + + 1 // byte for data length + + ix_bytes; + bytes[offset] = 0x01; + assert!(TransactionFrame::try_new(&bytes).is_err()); + } + + #[test] + fn test_basic_accessors() { + let tx = simple_transfer(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + assert_eq!(frame.num_signatures(), 1); + assert!(matches!(frame.version(), TransactionVersion::Legacy)); + assert_eq!(frame.num_required_signatures(), 1); + assert_eq!(frame.num_readonly_signed_static_accounts(), 0); + assert_eq!(frame.num_readonly_unsigned_static_accounts(), 1); + assert_eq!(frame.num_static_account_keys(), 3); + assert_eq!(frame.num_instructions(), 1); + assert_eq!(frame.num_address_table_lookups(), 0); + + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let signatures = frame.signatures(&bytes); + assert_eq!(signatures, &tx.signatures); + + let static_account_keys = frame.static_account_keys(&bytes); + assert_eq!(static_account_keys, tx.message.static_account_keys()); + + let recent_blockhash = frame.recent_blockhash(&bytes); + assert_eq!(recent_blockhash, tx.message.recent_blockhash()); + } + } + + #[test] + fn test_instructions_iter_empty() { + let tx = minimally_sized_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let mut iter = frame.instructions_iter(&bytes); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_instructions_iter_single() { + let tx = simple_transfer(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let mut iter = frame.instructions_iter(&bytes); + let ix = iter.next().unwrap(); + assert_eq!(ix.program_id_index, 2); + assert_eq!(ix.accounts, &[0, 1]); + assert_eq!( + ix.data, + &wincode::serialize(&SystemInstruction::Transfer { lamports: 1 }).unwrap() + ); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_instructions_iter_multiple() { + let tx = multiple_transfers(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let mut iter = frame.instructions_iter(&bytes); + let ix = iter.next().unwrap(); + assert_eq!(ix.program_id_index, 3); + assert_eq!(ix.accounts, &[0, 1]); + assert_eq!( + ix.data, + &wincode::serialize(&SystemInstruction::Transfer { lamports: 1 }).unwrap() + ); + let ix = iter.next().unwrap(); + assert_eq!(ix.program_id_index, 3); + assert_eq!(ix.accounts, &[0, 2]); + assert_eq!( + ix.data, + &wincode::serialize(&SystemInstruction::Transfer { lamports: 1 }).unwrap() + ); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_address_table_lookup_iter_empty() { + let tx = simple_transfer(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let mut iter = frame.address_table_lookup_iter(&bytes); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_address_table_lookup_iter_single() { + let tx = v0_with_single_lookup(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + let atls_actual = tx.message.address_table_lookups().unwrap(); + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let mut iter = frame.address_table_lookup_iter(&bytes); + let lookup = iter.next().unwrap(); + assert_eq!(lookup.account_key, &atls_actual[0].account_key); + assert_eq!(lookup.writable_indexes, atls_actual[0].writable_indexes); + assert_eq!(lookup.readonly_indexes, atls_actual[0].readonly_indexes); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_address_table_lookup_iter_multiple() { + let tx = v0_with_multiple_lookups(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + let atls_actual = tx.message.address_table_lookups().unwrap(); + // SAFETY: `bytes` is the same slice used to create `frame`. + unsafe { + let mut iter = frame.address_table_lookup_iter(&bytes); + + let lookup = iter.next().unwrap(); + assert_eq!(lookup.account_key, &atls_actual[0].account_key); + assert_eq!(lookup.writable_indexes, atls_actual[0].writable_indexes); + assert_eq!(lookup.readonly_indexes, atls_actual[0].readonly_indexes); + + let lookup = iter.next().unwrap(); + assert_eq!(lookup.account_key, &atls_actual[1].account_key); + assert_eq!(lookup.writable_indexes, atls_actual[1].writable_indexes); + assert_eq!(lookup.readonly_indexes, atls_actual[1].readonly_indexes); + + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_v1_transaction_frame_parses() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + assert!(matches!(frame.version(), TransactionVersion::V1)); + assert_eq!(frame.num_signatures(), 1); + assert_eq!(frame.num_required_signatures(), 1); + assert_eq!(frame.num_readonly_signed_static_accounts(), 0); + assert_eq!(frame.num_readonly_unsigned_static_accounts(), 1); + assert_eq!(frame.num_static_account_keys(), 3); + assert_eq!(frame.num_instructions(), 2); + + // txv1 should not have ALTs + assert_eq!(frame.num_address_table_lookups(), 0); + assert_eq!(frame.total_writable_lookup_accounts(), 0); + assert_eq!(frame.total_readonly_lookup_accounts(), 0); + + // new v1-only frame metadata + assert!(frame.signatures_offset() > frame.message_offset()); + } + + #[test] + fn test_magicblock_frame_preserves_large_offsets() { + let mut transaction = simple_v1_transaction(); + let VersionedMessage::V1(message) = &mut transaction.message else { + unreachable!(); + }; + message.config = v1::TransactionConfig::empty(); + message.instructions[0].data = vec![1; 40_000]; + message.instructions[1].data = vec![2; 40_000]; + + let mut bytes = wincode::serialize(&transaction).unwrap(); + bytes[0] = MAGICBLOCK_PREFIX; + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + assert!(matches!(frame.version(), TransactionVersion::Magicblock)); + assert_eq!( + solana_transaction::versioned::TransactionVersion::from(frame.version()), + solana_transaction::versioned::TransactionVersion::Number(127) + ); + assert!(frame.signatures_offset() > u32::from(u16::MAX)); + assert_eq!(frame.message_range(), (0, frame.signatures_offset())); + + let instructions: Vec<_> = unsafe { frame.instructions_iter(&bytes) }.collect(); + assert_eq!(instructions[0].data, vec![1; 40_000]); + assert_eq!(instructions[1].data, vec![2; 40_000]); + let signatures = unsafe { frame.signatures(&bytes) }; + assert_eq!(signatures, transaction.signatures); + } + + #[test] + fn test_v1_is_not_legacy_or_v0() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + + assert!(!TransactionFrame::is_legacy_or_v0(&bytes).unwrap()); + } + + #[test] + fn test_legacy_is_legacy_or_v0() { + let payer = Pubkey::new_unique(); + let tx = VersionedTransaction { + signatures: vec![Signature::default()], + message: VersionedMessage::Legacy(solana_message::Message::new(&[], Some(&payer))), + }; + let bytes = wincode::serialize(&tx).unwrap(); + + assert!(TransactionFrame::is_legacy_or_v0(&bytes).unwrap()); + } + + #[test] + fn test_is_legacy_or_v0_empty_bytes() { + assert!(matches!( + TransactionFrame::is_legacy_or_v0(&[]), + Err(TransactionViewError::ParseError), + )); + } + + #[test] + fn test_v1_rejects_unknown_version() { + let tx = simple_v1_transaction(); + let mut bytes = wincode::serialize(&tx).unwrap(); + + // First byte is version-tagged for versioned messages. + // Flip underlying version to an unsupported value. + bytes[0] = solana_message::MESSAGE_VERSION_PREFIX | 2; + + assert!(matches!( + TransactionFrame::try_new(&bytes), + Err(TransactionViewError::ParseError), + )); + } + + #[test] + fn test_v1_rejects_trailing_byte() { + let tx = simple_v1_transaction(); + let mut bytes = wincode::serialize(&tx).unwrap(); + bytes.push(0); + + assert!(matches!( + TransactionFrame::try_new(&bytes), + Err(TransactionViewError::ParseError), + )); + } + + #[test] + fn test_v1_rejects_truncated_bytes() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + + assert!(matches!( + TransactionFrame::try_new(&bytes[..bytes.len() - 1]), + Err(TransactionViewError::ParseError), + )); + } + + #[test] + fn test_v1_instruction_iteration() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + let frame = TransactionFrame::try_new(&bytes).unwrap(); + + let mut iter = unsafe { frame.instructions_iter(&bytes) }; + + let ix0 = iter.next().unwrap(); + assert_eq!(ix0.program_id_index, 2); + assert_eq!(ix0.accounts, &[0, 1]); + assert_eq!(ix0.data, &[10, 11, 12]); + + let ix1 = iter.next().unwrap(); + assert_eq!(ix1.program_id_index, 2); + assert_eq!(ix1.accounts, &[] as &[u8]); + assert_eq!(ix1.data, &[99]); + + assert!(iter.next().is_none()); + } +} diff --git a/solana/transaction-view/src/transaction_version.rs b/solana/transaction-view/src/transaction_version.rs new file mode 100644 index 00000000..36f6af82 --- /dev/null +++ b/solana/transaction-view/src/transaction_version.rs @@ -0,0 +1,26 @@ +/// Engine-private transaction version. +pub const MAGICBLOCK_VERSION: u8 = 127; +/// Engine-private versioned transaction prefix. +pub const MAGICBLOCK_PREFIX: u8 = solana_message::MESSAGE_VERSION_PREFIX | MAGICBLOCK_VERSION; + +/// A byte that represents the version of the transaction. +#[derive(Copy, Clone, Debug, Default)] +#[repr(u8)] +pub enum TransactionVersion { + #[default] + Legacy = u8::MAX, + V0 = 0, + V1 = 1, + Magicblock = MAGICBLOCK_VERSION, +} + +impl From for solana_transaction::versioned::TransactionVersion { + fn from(version: TransactionVersion) -> Self { + match version { + TransactionVersion::Legacy => Self::LEGACY, + TransactionVersion::V0 => Self::Number(0), + TransactionVersion::V1 => Self::Number(1), + TransactionVersion::Magicblock => Self::Number(MAGICBLOCK_VERSION), + } + } +} diff --git a/solana/transaction-view/src/transaction_view.rs b/solana/transaction-view/src/transaction_view.rs new file mode 100644 index 00000000..7496fdfb --- /dev/null +++ b/solana/transaction-view/src/transaction_view.rs @@ -0,0 +1,510 @@ +use { + crate::{ + address_table_lookup_frame::AddressTableLookupIterator, + instructions_frame::InstructionsIterator, result::Result, sanitize::sanitize, + transaction_config_frame::TransactionConfigView, transaction_data::TransactionData, + transaction_frame::TransactionFrame, transaction_version::TransactionVersion, + }, + core::fmt::{Debug, Formatter}, + solana_hash::Hash, + solana_pubkey::Pubkey, + solana_signature::Signature, + solana_svm_transaction::{ + instruction::SVMInstruction, message_address_table_lookup::SVMMessageAddressTableLookup, + svm_message::SVMStaticMessage, + }, +}; + +// alias for convenience +pub type UnsanitizedTransactionView = TransactionView; +pub type SanitizedTransactionView = TransactionView; + +/// A view into a serialized transaction. +/// +/// This struct provides access to the transaction data without +/// deserializing it. This is done by parsing and caching metadata +/// about the layout of the serialized transaction. +/// The owned `data` is abstracted through the `TransactionData` trait, +/// so that different containers for the serialized transaction can be used. +pub struct TransactionView { + data: D, + frame: TransactionFrame, +} + +impl TransactionView { + /// Creates a new `TransactionView` without running sanitization checks. + pub fn try_new_unsanitized(data: D) -> Result { + let frame = TransactionFrame::try_new(data.data())?; + Ok(Self { data, frame }) + } + + /// Sanitizes the transaction view, returning a sanitized view on success. + pub fn sanitize( + self, + enable_instruction_accounts_limit: bool, + ) -> Result> { + sanitize(&self, enable_instruction_accounts_limit)?; + Ok(SanitizedTransactionView { + data: self.data, + frame: self.frame, + }) + } +} + +impl TransactionView { + /// Creates a new `TransactionView`, running sanitization checks. + pub fn try_new_sanitized(data: D, enable_instruction_accounts_limit: bool) -> Result { + let unsanitized_view = TransactionView::try_new_unsanitized(data)?; + unsanitized_view.sanitize(enable_instruction_accounts_limit) + } +} + +impl TransactionView { + /// Return the number of signatures in the transaction. + #[inline] + pub fn num_signatures(&self) -> u8 { + self.frame.num_signatures() + } + + /// Return the version of the transaction. + #[inline] + pub fn version(&self) -> TransactionVersion { + self.frame.version() + } + + /// Return the number of required signatures in the transaction. + #[inline] + pub fn num_required_signatures(&self) -> u8 { + self.frame.num_required_signatures() + } + + /// Return the number of readonly signed static accounts in the transaction. + #[inline] + pub fn num_readonly_signed_static_accounts(&self) -> u8 { + self.frame.num_readonly_signed_static_accounts() + } + + /// Return the number of readonly unsigned static accounts in the transaction. + #[inline] + pub fn num_readonly_unsigned_static_accounts(&self) -> u8 { + self.frame.num_readonly_unsigned_static_accounts() + } + + /// Return the number of static account keys in the transaction. + #[inline] + pub fn num_static_account_keys(&self) -> u8 { + self.frame.num_static_account_keys() + } + + /// Return the number of instructions in the transaction. + #[inline] + pub fn num_instructions(&self) -> u16 { + self.frame.num_instructions() + } + + /// Return the number of address table lookups in the transaction. + #[inline] + pub fn num_address_table_lookups(&self) -> u8 { + self.frame.num_address_table_lookups() + } + + /// Return the number of writable lookup accounts in the transaction. + #[inline] + pub fn total_writable_lookup_accounts(&self) -> u16 { + self.frame.total_writable_lookup_accounts() + } + + /// Return the number of readonly lookup accounts in the transaction. + #[inline] + pub fn total_readonly_lookup_accounts(&self) -> u16 { + self.frame.total_readonly_lookup_accounts() + } + + /// Return the slice of signatures in the transaction. + #[inline] + pub fn signatures(&self) -> &[Signature] { + let data = self.data(); + // SAFETY: `frame` was created from `data`. + unsafe { self.frame.signatures(data) } + } + + /// Return the slice of static account keys in the transaction. + #[inline] + pub fn static_account_keys(&self) -> &[Pubkey] { + let data = self.data(); + // SAFETY: `frame` was created from `data`. + unsafe { self.frame.static_account_keys(data) } + } + + /// Return the recent blockhash in the transaction. + #[inline] + pub fn recent_blockhash(&self) -> &Hash { + let data = self.data(); + // SAFETY: `frame` was created from `data`. + unsafe { self.frame.recent_blockhash(data) } + } + + /// Return an iterator over the instructions in the transaction. + #[inline] + pub fn instructions_iter(&self) -> InstructionsIterator<'_> { + let data = self.data(); + // SAFETY: `frame` was created from `data`. + unsafe { self.frame.instructions_iter(data) } + } + + /// Return an iterator over the address table lookups in the transaction. + #[inline] + pub fn address_table_lookup_iter(&self) -> AddressTableLookupIterator<'_> { + let data = self.data(); + // SAFETY: `frame` was created from `data`. + unsafe { self.frame.address_table_lookup_iter(data) } + } + + /// Return Some(TransactionConfigView) for V1, None for legacy/V0 + #[inline] + pub fn transaction_config(&self) -> Option> { + let transaction_config_frame = self.frame.transaction_config_frame(); + transaction_config_frame.is_present().then_some(TransactionConfigView { + transaction_config_frame, + bytes: self.data(), + }) + } + + /// Return the full serialized transaction data. + #[inline] + pub fn data(&self) -> &[u8] { + self.data.data() + } + + /// Return the serialized **message** data. + /// This does not include the signatures. + #[inline] + pub fn message_data(&self) -> &[u8] { + let (start, end) = self.frame.message_range(); + &self.data()[start as usize..end as usize] + } + + #[inline] + pub fn inner_data(&self) -> &D { + &self.data + } + + #[inline] + pub fn into_inner_data(self) -> D { + self.data + } +} + +// Implementation that relies on sanitization checks having been run. +impl TransactionView { + /// Return an iterator over the instructions paired with their program ids. + pub fn program_instructions_iter( + &self, + ) -> impl Iterator)> + Clone { + self.instructions_iter().map(|ix| { + let program_id_index = usize::from(ix.program_id_index); + let program_id = &self.static_account_keys()[program_id_index]; + (program_id, ix) + }) + } + + /// Return the number of unsigned static account keys. + #[inline] + pub(crate) fn num_static_unsigned_static_accounts(&self) -> u8 { + self.num_static_account_keys().wrapping_sub(self.num_required_signatures()) + } + + /// Return the number of writable unsigned static accounts. + #[inline] + pub(crate) fn num_writable_unsigned_static_accounts(&self) -> u8 { + self.num_static_unsigned_static_accounts() + .wrapping_sub(self.num_readonly_unsigned_static_accounts()) + } + + /// Return the number of writable unsigned static accounts. + #[inline] + pub(crate) fn num_writable_signed_static_accounts(&self) -> u8 { + self.num_required_signatures() + .wrapping_sub(self.num_readonly_signed_static_accounts()) + } + + /// Return the total number of accounts in the transactions. + #[inline] + pub fn total_num_accounts(&self) -> u16 { + u16::from(self.num_static_account_keys()) + .wrapping_add(self.total_writable_lookup_accounts()) + .wrapping_add(self.total_readonly_lookup_accounts()) + } + + /// Return the number of requested writable keys. + #[inline] + pub fn num_requested_write_locks(&self) -> u64 { + u64::from( + u16::from( + (self.num_static_account_keys()) + .wrapping_sub(self.num_readonly_signed_static_accounts()) + .wrapping_sub(self.num_readonly_unsigned_static_accounts()), + ) + .wrapping_add(self.total_writable_lookup_accounts()), + ) + } +} + +// Manual implementation of `Debug` - avoids bound on `D`. +// Prints nicely formatted struct-ish fields even for the iterator fields. +impl Debug for TransactionView { + fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result { + f.debug_struct("TransactionView") + .field("frame", &self.frame) + .field("signatures", &self.signatures()) + .field("static_account_keys", &self.static_account_keys()) + .field("recent_blockhash", &self.recent_blockhash()) + .field("instructions", &self.instructions_iter()) + .field("address_table_lookups", &self.address_table_lookup_iter()) + .finish() + } +} + +impl SVMStaticMessage for TransactionView { + fn version(&self) -> solana_transaction::versioned::TransactionVersion { + self.version().into() + } + + fn num_transaction_signatures(&self) -> u64 { + self.num_required_signatures() as u64 + } + + fn num_write_locks(&self) -> u64 { + self.num_requested_write_locks() + } + + fn recent_blockhash(&self) -> &Hash { + self.recent_blockhash() + } + + fn num_instructions(&self) -> usize { + self.num_instructions() as usize + } + + fn instructions_iter(&self) -> impl Iterator> { + self.instructions_iter() + } + + fn program_instructions_iter( + &self, + ) -> impl Iterator)> + Clone { + self.program_instructions_iter() + } + + fn static_account_keys(&self) -> &[Pubkey] { + self.static_account_keys() + } + + fn fee_payer(&self) -> &Pubkey { + &self.static_account_keys()[0] + } + + fn num_lookup_tables(&self) -> usize { + self.num_address_table_lookups() as usize + } + + fn message_address_table_lookups( + &self, + ) -> impl Iterator> { + self.address_table_lookup_iter() + } +} + +impl SVMStaticMessage for &TransactionView { + fn version(&self) -> solana_transaction::versioned::TransactionVersion { + as SVMStaticMessage>::version(self) + } + + fn num_transaction_signatures(&self) -> u64 { + as SVMStaticMessage>::num_transaction_signatures(self) + } + + fn num_write_locks(&self) -> u64 { + as SVMStaticMessage>::num_write_locks(self) + } + + fn recent_blockhash(&self) -> &Hash { + as SVMStaticMessage>::recent_blockhash(self) + } + + fn num_instructions(&self) -> usize { + as SVMStaticMessage>::num_instructions(self) + } + + fn instructions_iter(&self) -> impl Iterator> { + as SVMStaticMessage>::instructions_iter(self) + } + + fn program_instructions_iter( + &self, + ) -> impl Iterator)> + Clone { + as SVMStaticMessage>::program_instructions_iter(self) + } + + fn static_account_keys(&self) -> &[Pubkey] { + as SVMStaticMessage>::static_account_keys(self) + } + + fn fee_payer(&self) -> &Pubkey { + as SVMStaticMessage>::fee_payer(self) + } + + fn num_lookup_tables(&self) -> usize { + as SVMStaticMessage>::num_lookup_tables(self) + } + + fn message_address_table_lookups( + &self, + ) -> impl Iterator> { + as SVMStaticMessage>::message_address_table_lookups(self) + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + solana_message::{ + Message, MessageHeader, VersionedMessage, compiled_instruction::CompiledInstruction, v1, + }, + solana_pubkey::Pubkey, + solana_signature::Signature, + solana_system_interface::instruction as system_instruction, + solana_transaction::versioned::VersionedTransaction, + }; + + fn verify_transaction_view_frame(tx: &VersionedTransaction) { + let bytes = wincode::serialize(tx).unwrap(); + let view = TransactionView::try_new_unsanitized(bytes.as_ref()).unwrap(); + + assert_eq!(view.num_signatures(), tx.signatures.len() as u8); + + assert_eq!( + view.num_required_signatures(), + tx.message.header().num_required_signatures + ); + assert_eq!( + view.num_readonly_signed_static_accounts(), + tx.message.header().num_readonly_signed_accounts + ); + assert_eq!( + view.num_readonly_unsigned_static_accounts(), + tx.message.header().num_readonly_unsigned_accounts + ); + + assert_eq!( + view.num_static_account_keys(), + tx.message.static_account_keys().len() as u8 + ); + assert_eq!( + view.num_instructions(), + tx.message.instructions().len() as u16 + ); + assert_eq!( + view.num_address_table_lookups(), + tx.message.address_table_lookups().map(|x| x.len() as u8).unwrap_or(0) + ); + + assert!(view.transaction_config().is_none()); + } + + fn multiple_transfers() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + VersionedTransaction { + signatures: vec![Signature::default()], // 1 signature to be valid. + message: VersionedMessage::Legacy(Message::new( + &[ + system_instruction::transfer(&payer, &Pubkey::new_unique(), 1), + system_instruction::transfer(&payer, &Pubkey::new_unique(), 1), + ], + Some(&payer), + )), + } + } + + #[test] + fn test_multiple_transfers() { + verify_transaction_view_frame(&multiple_transfers()); + } + + fn simple_v1_transaction() -> VersionedTransaction { + let payer = Pubkey::new_unique(); + let program = Pubkey::new_unique(); + + VersionedTransaction { + signatures: vec![Signature::default()], + message: VersionedMessage::V1(v1::Message { + header: MessageHeader { + num_required_signatures: 1, + num_readonly_signed_accounts: 0, + num_readonly_unsigned_accounts: 0, + }, + config: v1::TransactionConfig { + priority_fee: Some(111), + compute_unit_limit: Some(222), + loaded_accounts_data_size_limit: Some(333), + heap_size: Some(1024), + }, + lifetime_specifier: Hash::default(), + account_keys: vec![payer, program], + instructions: vec![CompiledInstruction { + program_id_index: 1, + accounts: vec![0], + data: vec![1, 2, 3, 4], + }], + }), + } + } + + #[test] + fn test_v1_transaction_config_present() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + let view = TransactionView::try_new_unsanitized(bytes.as_ref()).unwrap(); + + assert!(matches!(view.version(), TransactionVersion::V1)); + + let config = view.transaction_config().expect("v1 should have config"); + assert_eq!(config.priority_fee_lamports().unwrap(), 111); + assert_eq!(config.compute_unit_limit().unwrap(), 222); + assert_eq!(config.loaded_accounts_data_size_limit().unwrap(), 333); + assert_eq!(config.requested_heap_size().unwrap(), 1024); + } + + #[test] + fn test_v1_message_data_excludes_signatures() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + let view = TransactionView::try_new_unsanitized(bytes.as_ref()).unwrap(); + + let message_data = view.message_data(); + + // For v1, message_data should stop before the signatures region. + assert!(message_data.len() < bytes.len()); + + let full_message = + &bytes[view.frame.message_offset() as usize..view.frame.signatures_offset() as usize]; + assert_eq!(message_data, full_message); + } + + #[test] + fn test_v1_signatures_accessible() { + let tx = simple_v1_transaction(); + let bytes = wincode::serialize(&tx).unwrap(); + let view = TransactionView::try_new_unsanitized(bytes.as_ref()).unwrap(); + + assert_eq!(view.signatures().len(), 1); + assert_eq!(view.static_account_keys().len(), 2); + + let instructions: Vec<_> = view.instructions_iter().collect(); + assert_eq!(instructions.len(), 1); + assert_eq!(instructions[0].program_id_index, 1); + assert_eq!(instructions[0].accounts, &[0]); + assert_eq!(instructions[0].data, &[1, 2, 3, 4]); + } +} diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index 8b137891..00000000 --- a/src/lib.rs +++ /dev/null @@ -1 +0,0 @@ -