From 318a4e9d480c5d8513909b47d2734cea850a91e5 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 15:44:18 +0700 Subject: [PATCH 1/3] fix(drive)!: cursor pagination over multi-branch query levels dropped sibling-branch documents For a document query with a single In (or range) clause, leftover index properties, and startAt/startAfter, the path-query lowering baked the cursor document's per-level start keys into the default subquery applied to every sibling branch of the multi-branch level. Branches ordered after the cursor's branch silently dropped all values below the cursor key, and branches ordered before it wrongly included values above it. The lowering also ignored a descending orderBy on leftover index properties for cursorless queries, returning within-branch results in index (ascending) order. From v1 of the new DriveDocumentQueryMethodVersions.non_primary_key_path_query method version (protocol v14, unreleased): - branches ordered before the cursor's branch are trimmed from the outer query; - branches ordered after it get an unfiltered default subquery; - the cursor's own branch is refined with a conditional subquery at its branch key (the design the existing block comment already described); - the cursorless lowering derives each level's direction from orderBy instead of the index property's own direction. v0 behavior is preserved for released protocol versions, since the lowering is shared by the prover and verifier and is part of the consensus query contract. Co-Authored-By: Claude Fable 5 --- .../non_primary_key_path_query/v1/mod.rs | 615 +++++++++++++++++- packages/rs-drive/tests/query_tests.rs | 278 ++++++++ .../drive_document_method_versions/v4.rs | 9 +- 3 files changed, 891 insertions(+), 11 deletions(-) diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs index 4557cf0bdc..2d7cda0e7c 100644 --- a/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs +++ b/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs @@ -1,19 +1,28 @@ //! v1 of the non-primary-key path-query lowering (protocol version 14): //! multiple `In` clauses on consecutive index properties lower to -//! multi-level key-set path queries. Single-`In` shapes route through -//! the v0 lowering unchanged. Conservative v1 restrictions (cursor -//! rejection, the cross-product cap, index conformity) live here, as -//! does the order-by-aware left-over recursion the multi-`In` path -//! uses in place of the v0 helper. +//! multi-level key-set path queries, and two defects of the v0 lowering +//! are fixed. First, cursor pagination over a multi-branch level (a +//! single `In` or range clause with left-over index properties) no +//! longer bakes the cursor's per-level start keys into the default +//! subquery applied to every sibling branch — branches before the +//! cursor's are trimmed, branches after it are unfiltered, and only the +//! cursor's own branch is refined by a conditional subquery. Second, +//! every cursorless left-over level takes its direction from `order_by` +//! (falling back to the index property's own) where v0 always used the +//! index property's. Conservative v1 restrictions for the multi-`In` +//! shape (cursor rejection, the cross-product cap, index conformity) +//! also live here. +use crate::error::drive::DriveError; use crate::error::query::QuerySyntaxError; use crate::error::Error; use crate::query::conditions::WhereClause; use crate::query::ordering::OrderClause; -use crate::query::{defaults, DriveDocumentQuery}; +use crate::query::{defaults, DriveDocumentQuery, StartAtDocument}; use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; use dpp::data_contract::document_type::{Index, IndexProperty}; +use dpp::document::document_methods::DocumentMethodsV0; use dpp::document::Document; use dpp::version::PlatformVersion; use dpp::ProtocolError; @@ -87,7 +96,8 @@ impl<'a> DriveDocumentQuery<'a> { #[cfg(any(feature = "server", feature = "verify"))] /// v1 of the non-primary-key path query lowering (protocol version 14): - /// accepts multiple `In` clauses. Single-`In` shapes lower exactly as v0. + /// accepts multiple `In` clauses, and lowers at-most-one-`In` shapes with + /// sibling-branch-correct cursors and order-by-aware directions. pub(in crate::query) fn get_non_primary_key_path_query_v1( &self, document_type_path: Vec>, @@ -101,7 +111,7 @@ impl<'a> DriveDocumentQuery<'a> { platform_version, ) } else { - self.get_non_primary_key_path_query_v0( + self.get_non_primary_key_single_in_path_query( document_type_path, starts_at_document, platform_version, @@ -109,6 +119,595 @@ impl<'a> DriveDocumentQuery<'a> { } } + #[cfg(any(feature = "server", feature = "verify"))] + /// v1 counterpart of [`Self::recursive_create_query`]: identical cursor + /// threading, but recursion goes through the v1 insert helper so every + /// cursorless left-over level takes its direction from `order_by`. + pub(in crate::query) fn recursive_create_query_ordered( + left_over_index_properties: &[&IndexProperty], + unique: bool, + starts_at_document: Option<&StartAtDocument>, //for key level, included + indexed_property: &IndexProperty, + order_by: &IndexMap, + platform_version: &PlatformVersion, + ) -> Result, Error> { + match left_over_index_properties.split_first() { + None => Ok(None), + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + let mut inner_query = Self::inner_query_from_starts_at( + starts_at_document, + indexed_property, + left_to_right, + platform_version, + )?; + Self::recursive_insert_on_query_ordered_with_cursor( + &mut inner_query, + left_over, + unique, + starts_at_document, + left_to_right, + order_by, + platform_version, + )?; + Ok(Some(inner_query)) + } + } + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// v1 counterpart of [`Self::recursive_insert_on_query`]: identical + /// cursor threading, but a cursorless level takes its direction from + /// `order_by` (falling back to the index property's own) where the v0 + /// helper always used the index property's. + #[allow(clippy::too_many_arguments)] + pub(in crate::query) fn recursive_insert_on_query_ordered_with_cursor( + query: &mut Query, + left_over_index_properties: &[&IndexProperty], + unique: bool, + starts_at_document: Option<&StartAtDocument>, //for key level, included + default_left_to_right: bool, + order_by: &IndexMap, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match left_over_index_properties.split_first() { + None => { + match unique { + true => { + query.set_subquery_key(vec![0]); + + // In the case things are NULL we allow to have multiple values + let inner_query = Self::inner_query_from_starts_at_for_id( + starts_at_document, + true, //for ids we always go left to right + ); + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + false => { + query.set_subquery_key(vec![0]); + // we just get all by document id order ascending + let full_query = + Self::inner_query_from_starts_at_for_id(None, default_left_to_right); + query.set_subquery(full_query); + + let inner_query = Self::inner_query_from_starts_at_for_id( + starts_at_document, + default_left_to_right, + ); + + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + } + Ok(()) + } + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + if let Some(start_at_document_inner) = starts_at_document { + let StartAtDocument { + document, + document_type, + included, + } = start_at_document_inner; + let start_at_key = document + .get_raw_for_document_type( + first.name.as_str(), + *document_type, + None, + platform_version, + ) + .ok() + .flatten(); + + // We should always include if we have left_over + let non_conditional_included = + !left_over.is_empty() || *included || start_at_key.is_none(); + + let mut non_conditional_query = Self::inner_query_starts_from_key( + start_at_key.clone(), + left_to_right, + non_conditional_included, + ); + + Self::recursive_insert_on_query_ordered_with_cursor( + &mut non_conditional_query, + left_over, + unique, + None, + left_to_right, + order_by, + platform_version, + )?; + + Self::recursive_conditional_insert_on_query_ordered( + &mut non_conditional_query, + start_at_key, + left_over, + unique, + start_at_document_inner, + left_to_right, + order_by, + platform_version, + )?; + + query.set_subquery(non_conditional_query); + } else { + let mut inner_query = Query::new_with_direction(left_to_right); + inner_query.insert_all(); + Self::recursive_insert_on_query_ordered_with_cursor( + &mut inner_query, + left_over, + unique, + None, + left_to_right, + order_by, + platform_version, + )?; + query.set_subquery(inner_query); + } + query.set_subquery_key(first.name.as_bytes().to_vec()); + Ok(()) + } + } + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// v1 counterpart of [`Self::recursive_conditional_insert_on_query`]: + /// identical conditional refinement along the cursor's path, but the + /// cursorless sub-levels it creates go through the v1 insert helper so + /// they take their direction from `order_by`. + #[allow(clippy::too_many_arguments)] + pub(in crate::query) fn recursive_conditional_insert_on_query_ordered( + query: &mut Query, + conditional_value: Option>, + left_over_index_properties: &[&IndexProperty], + unique: bool, + starts_at_document: &StartAtDocument, + default_left_to_right: bool, + order_by: &IndexMap, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match left_over_index_properties.split_first() { + None => { + match unique { + true => { + // In the case things are NULL we allow to have multiple values + let inner_query = Self::inner_query_from_starts_at_for_id( + Some(starts_at_document), + true, //for ids we always go left to right + ); + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + false => { + let inner_query = Self::inner_query_from_starts_at_for_id( + Some(starts_at_document), + default_left_to_right, + ); + + query.add_conditional_subquery( + QueryItem::Key(conditional_value.unwrap_or_default()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + } + } + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + let StartAtDocument { + document, + document_type, + .. + } = starts_at_document; + + let lower_start_at_key = document + .get_raw_for_document_type( + first.name.as_str(), + *document_type, + None, + platform_version, + ) + .ok() + .flatten(); + + // We include it if we are not unique, + // or if we are unique but the value is empty + let non_conditional_included = !unique || lower_start_at_key.is_none(); + + let mut non_conditional_query = Self::inner_query_starts_from_key( + lower_start_at_key.clone(), + left_to_right, + non_conditional_included, + ); + + Self::recursive_insert_on_query_ordered_with_cursor( + &mut non_conditional_query, + left_over, + unique, + None, + left_to_right, + order_by, + platform_version, + )?; + + Self::recursive_conditional_insert_on_query_ordered( + &mut non_conditional_query, + lower_start_at_key, + left_over, + unique, + starts_at_document, + left_to_right, + order_by, + platform_version, + )?; + + query.add_conditional_subquery( + QueryItem::Key(conditional_value.unwrap_or_default()), + Some(vec![first.name.as_bytes().to_vec()]), + Some(non_conditional_query), + ); + } + } + Ok(()) + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// v1 lowering for queries with at most one `In` clause. Differs from + /// [`Self::get_non_primary_key_path_query_v0`] in two ways: cursor + /// pagination over a multi-branch level (an `In` or range last clause + /// with left-over index properties) trims the branches ordered before + /// the cursor's branch, gives the branches ordered after it an + /// unfiltered default subquery, and refines only the cursor's own + /// branch with a conditional subquery — where v0 baked the cursor's + /// per-level start keys into the default subquery applied to every + /// branch, silently dropping later branches' values below the cursor + /// and including earlier branches' values above it. And every + /// cursorless left-over level takes its direction from `order_by` + /// instead of always the index property's own. + pub(in crate::query) fn get_non_primary_key_single_in_path_query( + &self, + document_type_path: Vec>, + starts_at_document: Option<(Document, bool)>, + platform_version: &PlatformVersion, + ) -> Result { + if self.internal_clauses.in_clauses.len() > 1 { + return Err(Error::Query(QuerySyntaxError::MultipleInClauses( + "There should only be one in clause", + ))); + } + let index = self.find_best_index(platform_version)?; + let ordered_clauses: Vec<&WhereClause> = index + .properties + .iter() + .filter_map(|field| self.internal_clauses.equal_clauses.get(field.name.as_str())) + .collect(); + let (last_clause, last_clause_is_range, subquery_clause) = + match self.internal_clauses.in_clauses.first() { + None => match &self.internal_clauses.range_clause { + None => (ordered_clauses.last().copied(), false, None), + Some(where_clause) => (Some(where_clause), true, None), + }, + Some(in_clause) => match &self.internal_clauses.range_clause { + None => (Some(in_clause), true, None), + Some(range_clause) => { + // Same clause ordering rule as v0 — the outer path + // query must operate on the field that appears earlier + // in the chosen index. See issue #2409. + let position_of = |field: &str| -> Option { + index + .properties + .iter() + .position(|p| p.name.as_str() == field) + }; + let in_pos = position_of(in_clause.field.as_str()); + let range_pos = position_of(range_clause.field.as_str()); + match (in_pos, range_pos) { + (Some(i), Some(r)) if i > r => { + (Some(range_clause), true, Some(in_clause)) + } + _ => (Some(in_clause), true, Some(range_clause)), + } + } + }, + }; + + // We need to get the terminal indexes unused by clauses. + let left_over_index_properties = index + .properties + .iter() + .filter(|field| { + !(self + .internal_clauses + .equal_clauses + .contains_key(field.name.as_str()) + || (last_clause.is_some() && last_clause.unwrap().field == field.name) + || (subquery_clause.is_some() && subquery_clause.unwrap().field == field.name)) + }) + .collect::>(); + + let intermediate_values = index + .properties + .iter() + .filter_map(|field| { + match self.internal_clauses.equal_clauses.get(field.name.as_str()) { + None => None, + Some(where_clause) => { + if !last_clause_is_range + && last_clause.is_some() + && last_clause.unwrap().field == field.name + { + //there is no need to give an intermediate value as the last clause is an equality + None + } else { + Some(self.document_type.serialize_value_for_key( + field.name.as_str(), + &where_clause.value, + platform_version, + )) + } + } + } + }) + .collect::>, ProtocolError>>() + .map_err(Error::from)?; + + let final_query = match last_clause { + None => { + // There is no last_clause which means we are using an index most likely because of an order_by, however we have no + // clauses, in this case we should use the first value of the index. + let first_index = index.properties.first().ok_or(Error::Drive( + DriveError::CorruptedContractIndexes("index must have properties".to_string()), + ))?; // Index must have properties + Self::recursive_create_query_ordered( + left_over_index_properties.as_slice(), + index.unique, + starts_at_document + .map(|(document, included)| StartAtDocument { + document, + document_type: self.document_type, + included, + }) + .as_ref(), + first_index, + &self.order_by, + platform_version, + )? + .expect("Index must have left over properties if no last clause") + } + Some(where_clause) => { + let left_to_right = if where_clause.operator.is_range() { + let order_clause: &OrderClause = self + .order_by + .get(where_clause.field.as_str()) + .ok_or(Error::Query(QuerySyntaxError::MissingOrderByForRange( + "query must have an orderBy field for each range element", + )))?; + + order_clause.ascending + } else { + true + }; + + // Cursor pagination over a multi-branch level: the level fans + // out (an `In` or range last clause), left-over properties + // hang under each branch, and a cursor document is present. + let sibling_aware_cursor_lowering = last_clause_is_range + && subquery_clause.is_none() + && !left_over_index_properties.is_empty() + && starts_at_document.is_some(); + + let starts_at_document_with_branch_included = if sibling_aware_cursor_lowering { + starts_at_document + .as_ref() + .map(|(document, _)| (document.clone(), true)) + } else { + None + }; + + // We should set the starts at document to be included for the query if there are + // left over index properties. + + let query_starts_at_document = if left_over_index_properties.is_empty() { + &starts_at_document + } else if sibling_aware_cursor_lowering { + // The cursor's branch always stays included at this level, + // trimming only the branches ordered before it; whether + // the cursor document itself is included is decided by the + // conditional subquery below. + &starts_at_document_with_branch_included + } else { + &None + }; + + let mut query = where_clause.to_path_query( + self.document_type, + query_starts_at_document, + left_to_right, + platform_version, + )?; + + match subquery_clause { + None => { + if sibling_aware_cursor_lowering { + let (document, included) = starts_at_document + .as_ref() + .expect("starts_at_document was checked above"); + + let (first, deeper_left_over) = left_over_index_properties + .split_first() + .expect("left_over_index_properties was checked above"); + let first_left_to_right = self + .order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + // Branches ordered after the cursor's take everything. + let mut default_subquery = + Query::new_with_direction(first_left_to_right); + default_subquery.insert_all(); + Self::recursive_insert_on_query_ordered_with_cursor( + &mut default_subquery, + deeper_left_over, + index.unique, + None, + first_left_to_right, + &self.order_by, + platform_version, + )?; + query.set_subquery(default_subquery); + query.set_subquery_key(first.name.as_bytes().to_vec()); + + // The cursor's branch continues from the cursor. + let start_at_key = document + .get_raw_for_document_type( + where_clause.field.as_str(), + self.document_type, + None, + platform_version, + ) + .ok() + .flatten(); + Self::recursive_conditional_insert_on_query_ordered( + &mut query, + start_at_key, + left_over_index_properties.as_slice(), + index.unique, + &StartAtDocument { + document: document.clone(), + document_type: self.document_type, + included: *included, + }, + left_to_right, + &self.order_by, + platform_version, + )?; + } else { + Self::recursive_insert_on_query_ordered_with_cursor( + &mut query, + left_over_index_properties.as_slice(), + index.unique, + starts_at_document + .map(|(document, included)| StartAtDocument { + document, + document_type: self.document_type, + included, + }) + .as_ref(), + left_to_right, + &self.order_by, + platform_version, + )?; + } + } + Some(subquery_where_clause) => { + let order_clause: &OrderClause = self + .order_by + .get(subquery_where_clause.field.as_str()) + .ok_or(Error::Query(QuerySyntaxError::MissingOrderByForRange( + "query must have an orderBy field for each range element", + )))?; + let mut subquery = subquery_where_clause.to_path_query( + self.document_type, + &starts_at_document, + order_clause.ascending, + platform_version, + )?; + Self::recursive_insert_on_query_ordered_with_cursor( + &mut subquery, + left_over_index_properties.as_slice(), + index.unique, + starts_at_document + .map(|(document, included)| StartAtDocument { + document, + document_type: self.document_type, + included, + }) + .as_ref(), + left_to_right, + &self.order_by, + platform_version, + )?; + let subindex = subquery_where_clause.field.as_bytes().to_vec(); + query.set_subquery_key(subindex); + query.set_subquery(subquery); + } + }; + + query + } + }; + + let (intermediate_indexes, last_indexes) = + index.properties.split_at(intermediate_values.len()); + + // Now we should construct the path + let last_index = last_indexes.first().ok_or(Error::Query( + QuerySyntaxError::QueryOnDocumentTypeWithNoIndexes( + "document query has no index with fields", + ), + ))?; + + let mut path = document_type_path; + + for (intermediate_index, intermediate_value) in + intermediate_indexes.iter().zip(intermediate_values.iter()) + { + path.push(intermediate_index.name.as_bytes().to_vec()); + path.push(intermediate_value.as_slice().to_vec()); + } + + path.push(last_index.name.as_bytes().to_vec()); + + Ok(PathQuery::new( + path, + SizedQuery::new(final_query, self.limit, self.offset), + )) + } + #[cfg(any(feature = "server", feature = "verify"))] /// Lowers a query with multiple `In` clauses into a path query whose /// levels carry one key set per `In` clause, in index property order, diff --git a/packages/rs-drive/tests/query_tests.rs b/packages/rs-drive/tests/query_tests.rs index bdcd8ab557..fc601fd30f 100644 --- a/packages/rs-drive/tests/query_tests.rs +++ b/packages/rs-drive/tests/query_tests.rs @@ -293,6 +293,81 @@ pub fn setup_family_tests( (drive, contract) } +#[cfg(feature = "server")] +/// Inserts the test "family" contract and adds the given people to it, so +/// tests that pin an exact document order (e.g. cursor-based pagination) +/// can use explicit ids and names. +fn setup_family_tests_with_people( + people: &[Person], + platform_version: &PlatformVersion, +) -> (Drive, DataContract) { + let drive_config = DriveConfig::default(); + + let drive = setup_drive(Some(drive_config)); + + let db_transaction = drive.grove.start_transaction(); + + // Create contracts tree + let mut batch = GroveDbOpBatch::new(); + + add_init_contracts_structure_operations(&mut batch); + + drive + .grove_apply_batch(batch, false, Some(&db_transaction), &platform_version.drive) + .expect("expected to create contracts tree successfully"); + + // setup code + let contract = test_helpers::setup_contract( + &drive, + "tests/supporting_files/contract/family/family-contract.json", + None, + None, + None::, + Some(&db_transaction), + Some(platform_version), + ); + + for person in people { + let value = serde_json::to_value(person).expect("serialized person"); + let document_cbor = cbor_serializer::serializable_value_to_cbor(&value, Some(0)) + .expect("expected to serialize to cbor"); + let document = Document::from_cbor(document_cbor.as_slice(), None, None, platform_version) + .expect("document should be properly deserialized"); + + let document_type = contract + .document_type_for_name("person") + .expect("expected to get document type"); + + let storage_flags = Some(Cow::Owned(StorageFlags::SingleEpoch(0))); + + drive + .add_document_for_contract( + DocumentAndContractInfo { + owned_document_info: OwnedDocumentInfo { + document_info: DocumentRefInfo((&document, storage_flags)), + owner_id: None, + }, + contract: &contract, + document_type, + }, + true, + BlockInfo::genesis(), + true, + Some(&db_transaction), + platform_version, + None, + ) + .expect("document should be inserted"); + } + drive + .grove + .commit_transaction(db_transaction) + .unwrap() + .expect("transaction should be committed"); + + (drive, contract) +} + #[cfg(feature = "server")] /// Inserts the test "family" contract and adds `count` documents containing randomly named people to it. pub fn setup_countable_family_tests( @@ -4356,6 +4431,209 @@ mod tests { assert_eq!(results, proof_results); } + #[cfg(feature = "server")] + #[test] + fn test_family_single_in_clause_with_cursor_keeps_sibling_branches_intact() { + let platform_version = PlatformVersion::latest(); + + // Every branch has last names on both sides of the cursors' last + // names, so wrongly applying the cursor's lastName filter to a + // sibling branch changes the result set. + let people: Vec = [ + (1u8, "Adam", "Kline"), + (2, "Adam", "Moore"), + (3, "Adam", "Sinclair"), + (4, "Ben", "Barber"), + (5, "Ben", "Nichols"), + (6, "Cara", "Abbott"), + (7, "Cara", "Zane"), + ] + .into_iter() + .map(|(seed, first_name, last_name)| Person { + id: vec![seed; 32], + owner_id: vec![seed.wrapping_add(100); 32], + first_name: first_name.to_string(), + middle_name: format!("{}middle", first_name), + last_name: last_name.to_string(), + age: seed + 20, + }) + .collect(); + + let (drive, contract) = setup_family_tests_with_people(&people, platform_version); + let person_document_type = contract + .document_type_for_name("person") + .expect("contract should have a person document type"); + + let root_hash = drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("there is always a root hash"); + + let full_names = |results: &Vec>| -> Vec<(String, String)> { + results + .iter() + .map(|result| { + let document = Document::from_bytes( + result.as_slice(), + person_document_type, + platform_version, + ) + .expect("we should be able to deserialize the document"); + let first_name = document + .get("firstName") + .expect("we should be able to get the first name") + .as_text() + .expect("the first name should be a string") + .to_string(); + let last_name = document + .get("lastName") + .expect("we should be able to get the last name") + .as_text() + .expect("the last name should be a string") + .to_string(); + (first_name, last_name) + }) + .collect() + }; + + // Brute force over the inserted people: the full (firstName, + // lastName) ordering. + let ordered_names = |ascending: bool| -> Vec<(String, String)> { + let mut ordered: Vec<(String, String)> = people + .iter() + .map(|person| (person.first_name.clone(), person.last_name.clone())) + .collect(); + ordered.sort(); + if !ascending { + ordered.reverse(); + } + ordered + }; + + // Everything strictly after the cursor person. + let expected_after = |cursor_id: u8, ascending: bool| -> Vec<(String, String)> { + let mut ordered = ordered_names(ascending); + let cursor = people + .iter() + .find(|person| person.id == vec![cursor_id; 32]) + .expect("cursor person should exist"); + let cursor_position = ordered + .iter() + .position(|name| name == &(cursor.first_name.clone(), cursor.last_name.clone())) + .expect("cursor person should be in the ordering"); + ordered.split_off(cursor_position + 1) + }; + + let assert_query_returns = + |query_value: &serde_json::Value, expected: Vec<(String, String)>| { + let where_cbor = cbor_serializer::serializable_value_to_cbor(query_value, None) + .expect("expected to serialize to cbor"); + let query = DriveDocumentQuery::from_cbor( + where_cbor.as_slice(), + &contract, + person_document_type, + &drive.config, + platform_version, + ) + .expect("query should be built"); + let (results, _, _) = query + .execute_raw_results_no_proof(&drive, None, None, platform_version) + .expect("query should be executed"); + + assert_eq!(full_names(&results), expected); + + let (proof_root_hash, proof_results, _) = query + .execute_with_proof_only_get_elements(&drive, None, None, platform_version) + .expect("we should be able to a proof"); + assert_eq!(root_hash, proof_root_hash); + assert_eq!(results, proof_results); + }; + + let encoded_id = |seed: u8| bs58::encode(vec![seed; 32]).into_string(); + + // Page-one (no cursor) queries: the cursor pages below must agree + // with these on ordering. + assert_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "limit": 100, + "orderBy": [ + ["firstName", "asc"], + ["lastName", "asc"] + ] + }), + ordered_names(true), + ); + + assert_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "limit": 100, + "orderBy": [ + ["firstName", "desc"], + ["lastName", "desc"] + ] + }), + ordered_names(false), + ); + + // Cursor in the first branch: both later branches must come back + // whole. + assert_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "startAfter": encoded_id(2), // Adam Moore + "limit": 100, + "orderBy": [ + ["firstName", "asc"], + ["lastName", "asc"] + ] + }), + expected_after(2, true), + ); + + // Cursor in a middle branch: the earlier branch must return + // nothing, the later branch must come back whole. + assert_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "startAfter": encoded_id(5), // Ben Nichols + "limit": 100, + "orderBy": [ + ["firstName", "asc"], + ["lastName", "asc"] + ] + }), + expected_after(5, true), + ); + + // Descending: the branch ordered after the cursor's branch must + // come back whole. + assert_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "startAfter": encoded_id(4), // Ben Barber + "limit": 100, + "orderBy": [ + ["firstName", "desc"], + ["lastName", "desc"] + ] + }), + expected_after(4, false), + ); + } + #[cfg(feature = "server")] #[test] fn test_family_sql_query() { diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs index b57236bac3..172be6080b 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs @@ -7,9 +7,12 @@ use crate::version::drive_versions::drive_document_method_versions::{ /// V4 is protocol version 14's document-method table. It hosts three /// independent changes that all gate at v14 (ranked aggregates, the -/// shared-prefix aggregate index fix, and multi-`In` document queries -/// via `query.non_primary_key_path_query: 1` — v13 and earlier keep -/// the v0 lowering, which rejects more than one `In` clause). +/// shared-prefix aggregate index fix, and the reworked non-primary-key +/// query lowering via `query.non_primary_key_path_query: 1` — multiple +/// `In` clauses, sibling-branch-correct cursor pagination over +/// multi-branch levels, and order-by-aware left-over directions; v13 +/// and earlier keep the v0 lowering, which rejects more than one `In` +/// clause and bakes the cursor's start keys into every sibling branch). /// /// ## 1. Contract-level ranked aggregates /// From 0e697f9160fc61b5724a22cae763443d425b7932 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 16:04:11 +0700 Subject: [PATCH 2/3] test(drive): freeze v0 cursor lowering outputs at protocol v13 Protocol versions <= 13 are on chain, so the pre-v14 (defective) cursor lowering must replay byte-for-byte. Pin its outputs for the same scenarios the v14 fix corrects, on both the execution and proof paths, and assert protocol v13 selects method version 0. Co-Authored-By: Claude Fable 5 --- packages/rs-drive/tests/query_tests.rs | 185 +++++++++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/packages/rs-drive/tests/query_tests.rs b/packages/rs-drive/tests/query_tests.rs index fc601fd30f..92ba675e11 100644 --- a/packages/rs-drive/tests/query_tests.rs +++ b/packages/rs-drive/tests/query_tests.rs @@ -4634,6 +4634,191 @@ mod tests { ); } + #[cfg(feature = "server")] + #[test] + fn test_family_single_in_clause_with_cursor_v0_lowering_frozen_at_protocol_v13() { + // Freeze test for the pre-v14 lowering: protocol versions <= 13 are + // on chain, so their (defective) cursor lowering must replay + // byte-for-byte. The expected values below are the v0 outputs the + // sibling-branch fix corrects at v14 — see + // test_family_single_in_clause_with_cursor_keeps_sibling_branches_intact + // for the correct shapes. Never edit these expectations. + let platform_version = PlatformVersion::latest(); + let protocol_v13 = PlatformVersion::get(13).expect("protocol version 13 exists"); + assert_eq!( + protocol_v13 + .drive + .methods + .document + .query + .non_primary_key_path_query, + 0, + "protocol v13 must keep the v0 non-primary-key path query lowering" + ); + + let people: Vec = [ + (1u8, "Adam", "Kline"), + (2, "Adam", "Moore"), + (3, "Adam", "Sinclair"), + (4, "Ben", "Barber"), + (5, "Ben", "Nichols"), + (6, "Cara", "Abbott"), + (7, "Cara", "Zane"), + ] + .into_iter() + .map(|(seed, first_name, last_name)| Person { + id: vec![seed; 32], + owner_id: vec![seed.wrapping_add(100); 32], + first_name: first_name.to_string(), + middle_name: format!("{}middle", first_name), + last_name: last_name.to_string(), + age: seed + 20, + }) + .collect(); + + let (drive, contract) = setup_family_tests_with_people(&people, platform_version); + let person_document_type = contract + .document_type_for_name("person") + .expect("contract should have a person document type"); + + let root_hash = drive + .grove + .root_hash(None, &platform_version.drive.grove_version) + .unwrap() + .expect("there is always a root hash"); + + let assert_v13_query_returns = + |query_value: &serde_json::Value, expected: Vec<(&str, &str)>| { + let where_cbor = cbor_serializer::serializable_value_to_cbor(query_value, None) + .expect("expected to serialize to cbor"); + let query = DriveDocumentQuery::from_cbor( + where_cbor.as_slice(), + &contract, + person_document_type, + &drive.config, + protocol_v13, + ) + .expect("query should be built"); + let (results, _, _) = query + .execute_raw_results_no_proof(&drive, None, None, protocol_v13) + .expect("query should be executed"); + + let names: Vec<(String, String)> = results + .iter() + .map(|result| { + let document = Document::from_bytes( + result.as_slice(), + person_document_type, + protocol_v13, + ) + .expect("we should be able to deserialize the document"); + let first_name = document + .get("firstName") + .expect("we should be able to get the first name") + .as_text() + .expect("the first name should be a string") + .to_string(); + let last_name = document + .get("lastName") + .expect("we should be able to get the last name") + .as_text() + .expect("the last name should be a string") + .to_string(); + (first_name, last_name) + }) + .collect(); + let expected: Vec<(String, String)> = expected + .into_iter() + .map(|(first, last)| (first.to_string(), last.to_string())) + .collect(); + + assert_eq!(names, expected); + + let (proof_root_hash, proof_results, _) = query + .execute_with_proof_only_get_elements(&drive, None, None, protocol_v13) + .expect("we should be able to a proof"); + assert_eq!(root_hash, proof_root_hash); + assert_eq!(results, proof_results); + }; + + let encoded_id = |seed: u8| bs58::encode(vec![seed; 32]).into_string(); + + // v0 bakes the cursor's lastName ("Moore") into every branch's + // default subquery, dropping (Ben, Barber) and (Cara, Abbott). + assert_v13_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "startAfter": encoded_id(2), // Adam Moore + "limit": 100, + "orderBy": [ + ["firstName", "asc"], + ["lastName", "asc"] + ] + }), + vec![("Adam", "Sinclair"), ("Ben", "Nichols"), ("Cara", "Zane")], + ); + + // Cursor in a middle branch: v0 wrongly includes (Adam, Sinclair), + // which is ordered before the cursor, and drops (Cara, Abbott). + assert_v13_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "startAfter": encoded_id(5), // Ben Nichols + "limit": 100, + "orderBy": [ + ["firstName", "asc"], + ["lastName", "asc"] + ] + }), + vec![("Adam", "Sinclair"), ("Cara", "Zane")], + ); + + // Descending cursor: v0 keeps only earlier-branch values below the + // cursor's lastName and drops the entire Adam branch. + assert_v13_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "startAfter": encoded_id(4), // Ben Barber + "limit": 100, + "orderBy": [ + ["firstName", "desc"], + ["lastName", "desc"] + ] + }), + vec![("Cara", "Abbott")], + ); + + // Cursorless descending page: v0 iterates branches descending but + // each branch's lastNames ascending (index direction). + assert_v13_query_returns( + &json!({ + "where": [ + ["firstName", "in", ["Adam", "Ben", "Cara"]], + ], + "limit": 100, + "orderBy": [ + ["firstName", "desc"], + ["lastName", "desc"] + ] + }), + vec![ + ("Cara", "Abbott"), + ("Cara", "Zane"), + ("Ben", "Barber"), + ("Ben", "Nichols"), + ("Adam", "Kline"), + ("Adam", "Moore"), + ("Adam", "Sinclair"), + ], + ); + } + #[cfg(feature = "server")] #[test] fn test_family_sql_query() { From 5882f9f69e0c3c5ab27e103a91a9eae5360735d2 Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Thu, 13 Aug 2026 17:53:08 +0700 Subject: [PATCH 3/3] refactor(drive): version the shape lowerings behind their own method-version slots The v1 non-primary-key lowering routed by shape to two unversioned worker methods. Give each shape lowering the standard versioned-method structure: get_non_primary_key_single_in_path_query and get_non_primary_key_multiple_in_path_query dispatch through new DriveDocumentQueryMethodVersions slots (0 in every table) to frozen v0 modules holding the constructions and their recursion helpers, so a future change to one shape bumps that slot alone instead of copying the whole v1 lowering. Unknown slot values error with UnknownVersionMismatch. No behavior change. Co-Authored-By: Claude Fable 5 --- .../query/non_primary_key_path_query/mod.rs | 7 +- .../multiple_in_path_query/mod.rs | 47 + .../multiple_in_path_query/v0/mod.rs | 372 +++++++ .../single_in_path_query/mod.rs | 45 + .../single_in_path_query/v0/mod.rs | 619 +++++++++++ .../non_primary_key_path_query/v1/mod.rs | 977 +----------------- .../drive_document_method_versions/mod.rs | 13 + .../drive_document_method_versions/v1.rs | 2 + .../drive_document_method_versions/v2.rs | 2 + .../drive_document_method_versions/v3.rs | 2 + .../drive_document_method_versions/v4.rs | 2 + 11 files changed, 1123 insertions(+), 965 deletions(-) create mode 100644 packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/mod.rs create mode 100644 packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/v0/mod.rs create mode 100644 packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/mod.rs create mode 100644 packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/v0/mod.rs diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/mod.rs index 3987fd1714..78461fb131 100644 --- a/packages/rs-drive/src/query/non_primary_key_path_query/mod.rs +++ b/packages/rs-drive/src/query/non_primary_key_path_query/mod.rs @@ -2,7 +2,12 @@ //! over a secondary index into a grovedb path query. Dispatch lives on //! `DriveDocumentQuery::get_non_primary_key_path_query` in the parent //! module; the per-version implementations live here so already-live -//! behavior is isolated from later edits. +//! behavior is isolated from later edits. From v1 on, the lowering only +//! routes by query shape — the shape lowerings themselves are versioned +//! methods in the `single_in_path_query` and `multiple_in_path_query` +//! submodules. +mod multiple_in_path_query; +mod single_in_path_query; mod v0; mod v1; diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/mod.rs new file mode 100644 index 0000000000..36da50ebeb --- /dev/null +++ b/packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/mod.rs @@ -0,0 +1,47 @@ +//! Versioned lowering for document queries with multiple +//! non-primary-key `In` clauses on consecutive index properties, +//! dispatched by +//! `DriveDocumentQueryMethodVersions.non_primary_key_multiple_in_path_query`. +//! Only reachable through the v1+ non-primary-key lowering: the v0 +//! lowering rejects these shapes before selection. + +mod v0; + +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::query::DriveDocumentQuery; +use dpp::document::Document; +use dpp::version::PlatformVersion; +use grovedb::PathQuery; + +impl<'a> DriveDocumentQuery<'a> { + #[cfg(any(feature = "server", feature = "verify"))] + /// Lowers a query with multiple `In` clauses into a path query whose + /// levels carry one key set per `In` clause. + pub(in crate::query) fn get_non_primary_key_multiple_in_path_query( + &self, + document_type_path: Vec>, + starts_at_document: Option<(Document, bool)>, + platform_version: &PlatformVersion, + ) -> Result { + match platform_version + .drive + .methods + .document + .query + .non_primary_key_multiple_in_path_query + { + 0 => self.get_non_primary_key_multiple_in_path_query_v0( + document_type_path, + starts_at_document, + platform_version, + ), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "DriveDocumentQuery::get_non_primary_key_multiple_in_path_query" + .to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/v0/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/v0/mod.rs new file mode 100644 index 0000000000..0cf2bb6720 --- /dev/null +++ b/packages/rs-drive/src/query/non_primary_key_path_query/multiple_in_path_query/v0/mod.rs @@ -0,0 +1,372 @@ +//! v0 of the multiple-`In` lowering (protocol version 14, reached +//! through v1 of the non-primary-key lowering): the `In` clauses lower +//! to multi-level key-set path queries, in index property order, +//! followed by an optional range level and the usual left-over / +//! terminal levels. Conservative v0 restrictions (cursor rejection, the +//! cross-product cap, index conformity) live here, as does the +//! order-by-aware left-over recursion this shape uses. Frozen: changes +//! to already-live behavior belong in a new version module. + +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use crate::query::conditions::WhereClause; +use crate::query::ordering::OrderClause; +use crate::query::{defaults, DriveDocumentQuery}; +use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; +use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; +use dpp::data_contract::document_type::{Index, IndexProperty}; +use dpp::document::Document; +use dpp::version::PlatformVersion; +use dpp::ProtocolError; +use grovedb::{PathQuery, Query, QueryItem, SizedQuery}; +use indexmap::IndexMap; + +impl<'a> DriveDocumentQuery<'a> { + #[cfg(any(feature = "server", feature = "verify"))] + /// Counterpart of [`Self::recursive_insert_on_query`] for this + /// lowering: no cursor support (cursors are rejected by the shape + /// preflight), and each left-over level takes its direction from + /// `order_by` — falling back to the index property's — instead of + /// always using the index property's like the v0 helper does. + pub(in crate::query) fn recursive_insert_on_query_ordered( + query: &mut Query, + left_over_index_properties: &[&IndexProperty], + unique: bool, + default_left_to_right: bool, + order_by: &IndexMap, + ) { + match left_over_index_properties.split_first() { + None => match unique { + true => { + query.set_subquery_key(vec![0]); + + // In the case things are NULL we allow to have multiple values + let inner_query = Self::inner_query_from_starts_at_for_id( + None, true, //for ids we always go left to right + ); + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + false => { + query.set_subquery_key(vec![0]); + // we just get all by document id order ascending + let full_query = + Self::inner_query_from_starts_at_for_id(None, default_left_to_right); + query.set_subquery(full_query); + + let inner_query = + Self::inner_query_from_starts_at_for_id(None, default_left_to_right); + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + }, + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + let mut inner_query = Query::new_with_direction(left_to_right); + inner_query.insert_all(); + Self::recursive_insert_on_query_ordered( + &mut inner_query, + left_over, + unique, + left_to_right, + order_by, + ); + query.set_subquery(inner_query); + query.set_subquery_key(first.name.as_bytes().to_vec()); + } + } + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// v0 lowering for queries with multiple `In` clauses: a path query + /// whose levels carry one key set per `In` clause, in index property + /// order, followed by an optional range level and the usual + /// left-over / terminal levels. + pub(in crate::query) fn get_non_primary_key_multiple_in_path_query_v0( + &self, + document_type_path: Vec>, + starts_at_document: Option<(Document, bool)>, + platform_version: &PlatformVersion, + ) -> Result { + // Conservative v0: the cross-branch cursor machinery is not wired + // for key-set branching at more than one level, so reject cursors + // instead of shipping silently wrong pagination. + if starts_at_document.is_some() || self.start_at.is_some() { + return Err(Error::Query(QuerySyntaxError::Unsupported( + "startAt/startAfter is not supported with multiple in clauses".to_string(), + ))); + } + + let (index, ordered_in_clauses, equality_len) = + self.find_best_index_for_multiple_in_clauses()?; + + // Bound the branch enumeration: the product of the in list sizes is + // the number of index subtrees the query opens. + let mut cross_product: usize = 1; + for in_clause in &ordered_in_clauses { + let in_values = in_clause.in_values().into_data_with_error()??; + cross_product = cross_product.saturating_mul(in_values.len()); + } + if cross_product > defaults::MAX_IN_CROSS_PRODUCT_SIZE { + return Err(Error::Query(QuerySyntaxError::InvalidInClause(format!( + "the product of in clause list sizes must be at most {}, got {}", + defaults::MAX_IN_CROSS_PRODUCT_SIZE, + cross_product + )))); + } + + let left_over_index_properties = index + .properties + .iter() + .filter(|field| { + !(self + .internal_clauses + .equal_clauses + .contains_key(field.name.as_str()) + || ordered_in_clauses + .iter() + .any(|in_clause| in_clause.field == field.name) + || self + .internal_clauses + .range_clause + .as_ref() + .is_some_and(|range_clause| range_clause.field == field.name)) + }) + .collect::>(); + + // Every level that fans out (each in clause, and the range clause) + // needs an explicit ordering, like any range-class clause. + let direction_for = |field: &str| -> Result { + let order_clause: &OrderClause = self.order_by.get(field).ok_or(Error::Query( + QuerySyntaxError::MissingOrderByForRange( + "query must have an orderBy field for each range element", + ), + ))?; + Ok(order_clause.ascending) + }; + + // Build the query bottom-up. The deepest clause level is the range + // clause when present, otherwise the last in clause; the left-over + // index properties and the terminal document level hang under it. + let (mut child_field, mut child_query, deepest_left_to_right) = + match &self.internal_clauses.range_clause { + Some(range_clause) => { + let left_to_right = direction_for(range_clause.field.as_str())?; + let query = range_clause.to_path_query( + self.document_type, + &None, + left_to_right, + platform_version, + )?; + (range_clause.field.clone(), query, left_to_right) + } + None => { + let deepest_in_clause = + *ordered_in_clauses.last().expect("more than one in clause"); + let left_to_right = direction_for(deepest_in_clause.field.as_str())?; + let query = deepest_in_clause.to_path_query( + self.document_type, + &None, + left_to_right, + platform_version, + )?; + (deepest_in_clause.field.clone(), query, left_to_right) + } + }; + Self::recursive_insert_on_query_ordered( + &mut child_query, + left_over_index_properties.as_slice(), + index.unique, + deepest_left_to_right, + &self.order_by, + ); + + // Wrap the remaining in levels around it, deepest first. When a + // range clause is present every in clause wraps; otherwise the + // deepest in clause is already the leaf built above. + let wrapped_in_clauses = if self.internal_clauses.range_clause.is_some() { + ordered_in_clauses.as_slice() + } else { + &ordered_in_clauses[..ordered_in_clauses.len() - 1] + }; + for in_clause in wrapped_in_clauses.iter().rev() { + let left_to_right = direction_for(in_clause.field.as_str())?; + let mut query = in_clause.to_path_query( + self.document_type, + &None, + left_to_right, + platform_version, + )?; + query.set_subquery_key(child_field.as_bytes().to_vec()); + query.set_subquery(child_query); + child_field = in_clause.field.clone(); + child_query = query; + } + + let intermediate_values = index.properties[..equality_len] + .iter() + .map(|field| { + let where_clause = self + .internal_clauses + .equal_clauses + .get(field.name.as_str()) + .expect("equality prefix was validated during index selection"); + self.document_type.serialize_value_for_key( + field.name.as_str(), + &where_clause.value, + platform_version, + ) + }) + .collect::>, ProtocolError>>() + .map_err(Error::from)?; + + let mut path = document_type_path; + for (intermediate_index, intermediate_value) in index.properties[..equality_len] + .iter() + .zip(intermediate_values.iter()) + { + path.push(intermediate_index.name.as_bytes().to_vec()); + path.push(intermediate_value.as_slice().to_vec()); + } + path.push(child_field.as_bytes().to_vec()); + + Ok(PathQuery::new( + path, + SizedQuery::new(child_query, self.limit, self.offset), + )) + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// Finds the best index for a query with more than one `In` clause, and + /// returns it together with the query's `In` clauses ordered by their + /// position in that index and the length of the equality prefix. + /// + /// A candidate index conforms when its properties decompose, left to + /// right, into: the equality-clause fields (exactly covering positions + /// `0..E`), then every `In` field on consecutive positions `E..E+K`, + /// then — when a range clause is present — the range field. The usual + /// tail rule from [`Index::matches`] still applies with the deepest `In` + /// field playing the role of the in field, as do the order-by continuity + /// rules and [`defaults::MAX_INDEX_DIFFERENCE`]. + pub(in crate::query) fn find_best_index_for_multiple_in_clauses( + &self, + ) -> Result<(&Index, Vec<&WhereClause>, usize), Error> { + let equal_clauses = &self.internal_clauses.equal_clauses; + let in_clauses = &self.internal_clauses.in_clauses; + let range_field = self + .internal_clauses + .range_clause + .as_ref() + .map(|range_clause| range_clause.field.as_str()); + + let mut fields = equal_clauses + .keys() + .map(|s| s.as_str()) + .collect::>(); + if let Some(range_field) = range_field { + fields.push(range_field); + } + fields.extend(in_clauses.iter().map(|in_clause| in_clause.field.as_str())); + + let order_by_keys: Vec<&str> = self + .order_by + .keys() + .map(|key: &String| { + let str = key.as_str(); + if !fields.contains(&str) { + fields.push(str); + } + str + }) + .collect(); + + let equality_len = equal_clauses.len(); + let mut best: Option<(&Index, Vec<&WhereClause>, u16)> = None; + for index in self.document_type.indexes().values() { + let mut positioned: Vec<(usize, &WhereClause)> = Vec::with_capacity(in_clauses.len()); + for in_clause in in_clauses { + match index + .properties + .iter() + .position(|property| property.name == in_clause.field) + { + Some(position) => positioned.push((position, in_clause)), + None => break, + } + } + if positioned.len() != in_clauses.len() { + continue; + } + positioned.sort_by_key(|(position, _)| *position); + + // The equality clauses must exactly cover the index prefix + if index.properties.len() < equality_len + || !index.properties[..equality_len] + .iter() + .all(|property| equal_clauses.contains_key(property.name.as_str())) + { + continue; + } + // The in clauses must sit on consecutive properties right after it + let consecutive_after_prefix = positioned + .iter() + .enumerate() + .all(|(i, (position, _))| *position == equality_len + i); + if !consecutive_after_prefix { + continue; + } + // A range clause must sit immediately after the in block + if let Some(range_field) = range_field { + match index.properties.get(equality_len + positioned.len()) { + Some(property) if property.name == range_field => {} + _ => continue, + } + } + + let deepest_in_field = positioned + .last() + .expect("more than one in clause") + .1 + .field + .as_str(); + let Some(difference) = index.matches(&fields, Some(deepest_in_field), &order_by_keys) + else { + continue; + }; + let ordered = positioned + .into_iter() + .map(|(_, in_clause)| in_clause) + .collect::>(); + if difference == 0 { + return Ok((index, ordered, equality_len)); + } + match &best { + Some((_, _, best_difference)) if *best_difference <= difference => {} + _ => best = Some((index, ordered, difference)), + } + } + + let (index, ordered, difference) = best.ok_or(Error::Query( + QuerySyntaxError::WhereClauseOnNonIndexedProperty(format!( + "query with multiple in clauses must be for valid indexes with the in clauses on \ + consecutive index properties after the equality clauses, valid indexes are: {:?}", + self.document_type.indexes() + )), + ))?; + if difference > defaults::MAX_INDEX_DIFFERENCE { + return Err(Error::Query(QuerySyntaxError::QueryTooFarFromIndex( + "query must better match an existing index", + ))); + } + Ok((index, ordered, equality_len)) + } +} diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/mod.rs new file mode 100644 index 0000000000..9641286cc1 --- /dev/null +++ b/packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/mod.rs @@ -0,0 +1,45 @@ +//! Versioned lowering for document queries with at most one +//! non-primary-key `In` clause, dispatched by +//! `DriveDocumentQueryMethodVersions.non_primary_key_single_in_path_query`. +//! Only reachable through the v1+ non-primary-key lowering: the v0 +//! lowering predates this method and carries its own frozen single-`In` +//! construction. + +mod v0; + +use crate::error::drive::DriveError; +use crate::error::Error; +use crate::query::DriveDocumentQuery; +use dpp::document::Document; +use dpp::version::PlatformVersion; +use grovedb::PathQuery; + +impl<'a> DriveDocumentQuery<'a> { + #[cfg(any(feature = "server", feature = "verify"))] + /// Lowers a query with at most one `In` clause into a path query. + pub(in crate::query) fn get_non_primary_key_single_in_path_query( + &self, + document_type_path: Vec>, + starts_at_document: Option<(Document, bool)>, + platform_version: &PlatformVersion, + ) -> Result { + match platform_version + .drive + .methods + .document + .query + .non_primary_key_single_in_path_query + { + 0 => self.get_non_primary_key_single_in_path_query_v0( + document_type_path, + starts_at_document, + platform_version, + ), + version => Err(Error::Drive(DriveError::UnknownVersionMismatch { + method: "DriveDocumentQuery::get_non_primary_key_single_in_path_query".to_string(), + known_versions: vec![0], + received: version, + })), + } + } +} diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/v0/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/v0/mod.rs new file mode 100644 index 0000000000..891b87b791 --- /dev/null +++ b/packages/rs-drive/src/query/non_primary_key_path_query/single_in_path_query/v0/mod.rs @@ -0,0 +1,619 @@ +//! v0 of the at-most-one-`In` lowering (protocol version 14, reached +//! through v1 of the non-primary-key lowering). Differs from the frozen +//! pre-v14 construction in `non_primary_key_path_query::v0` in two +//! ways: cursor pagination over a multi-branch level is sibling-branch +//! correct, and every cursorless left-over level takes its direction +//! from `order_by`. Frozen: changes to already-live behavior belong in +//! a new version module. + +use crate::error::drive::DriveError; +use crate::error::query::QuerySyntaxError; +use crate::error::Error; +use crate::query::conditions::WhereClause; +use crate::query::ordering::OrderClause; +use crate::query::{DriveDocumentQuery, StartAtDocument}; +use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; +use dpp::data_contract::document_type::IndexProperty; +use dpp::document::document_methods::DocumentMethodsV0; +use dpp::document::Document; +use dpp::version::PlatformVersion; +use dpp::ProtocolError; +use grovedb::{PathQuery, Query, QueryItem, SizedQuery}; +use indexmap::IndexMap; + +impl<'a> DriveDocumentQuery<'a> { + #[cfg(any(feature = "server", feature = "verify"))] + /// Counterpart of [`Self::recursive_create_query`] for this + /// lowering: identical cursor threading, but recursion goes through + /// [`Self::recursive_insert_on_query_ordered_with_cursor`] so every + /// cursorless left-over level takes its direction from `order_by`. + pub(in crate::query) fn recursive_create_query_ordered( + left_over_index_properties: &[&IndexProperty], + unique: bool, + starts_at_document: Option<&StartAtDocument>, //for key level, included + indexed_property: &IndexProperty, + order_by: &IndexMap, + platform_version: &PlatformVersion, + ) -> Result, Error> { + match left_over_index_properties.split_first() { + None => Ok(None), + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + let mut inner_query = Self::inner_query_from_starts_at( + starts_at_document, + indexed_property, + left_to_right, + platform_version, + )?; + Self::recursive_insert_on_query_ordered_with_cursor( + &mut inner_query, + left_over, + unique, + starts_at_document, + left_to_right, + order_by, + platform_version, + )?; + Ok(Some(inner_query)) + } + } + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// Counterpart of [`Self::recursive_insert_on_query`] for this + /// lowering: identical cursor threading, but a cursorless level + /// takes its direction from `order_by` (falling back to the index + /// property's own) where the v0 helper always used the index + /// property's. + #[allow(clippy::too_many_arguments)] + pub(in crate::query) fn recursive_insert_on_query_ordered_with_cursor( + query: &mut Query, + left_over_index_properties: &[&IndexProperty], + unique: bool, + starts_at_document: Option<&StartAtDocument>, //for key level, included + default_left_to_right: bool, + order_by: &IndexMap, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match left_over_index_properties.split_first() { + None => { + match unique { + true => { + query.set_subquery_key(vec![0]); + + // In the case things are NULL we allow to have multiple values + let inner_query = Self::inner_query_from_starts_at_for_id( + starts_at_document, + true, //for ids we always go left to right + ); + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + false => { + query.set_subquery_key(vec![0]); + // we just get all by document id order ascending + let full_query = + Self::inner_query_from_starts_at_for_id(None, default_left_to_right); + query.set_subquery(full_query); + + let inner_query = Self::inner_query_from_starts_at_for_id( + starts_at_document, + default_left_to_right, + ); + + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + } + Ok(()) + } + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + if let Some(start_at_document_inner) = starts_at_document { + let StartAtDocument { + document, + document_type, + included, + } = start_at_document_inner; + let start_at_key = document + .get_raw_for_document_type( + first.name.as_str(), + *document_type, + None, + platform_version, + ) + .ok() + .flatten(); + + // We should always include if we have left_over + let non_conditional_included = + !left_over.is_empty() || *included || start_at_key.is_none(); + + let mut non_conditional_query = Self::inner_query_starts_from_key( + start_at_key.clone(), + left_to_right, + non_conditional_included, + ); + + Self::recursive_insert_on_query_ordered_with_cursor( + &mut non_conditional_query, + left_over, + unique, + None, + left_to_right, + order_by, + platform_version, + )?; + + Self::recursive_conditional_insert_on_query_ordered( + &mut non_conditional_query, + start_at_key, + left_over, + unique, + start_at_document_inner, + left_to_right, + order_by, + platform_version, + )?; + + query.set_subquery(non_conditional_query); + } else { + let mut inner_query = Query::new_with_direction(left_to_right); + inner_query.insert_all(); + Self::recursive_insert_on_query_ordered_with_cursor( + &mut inner_query, + left_over, + unique, + None, + left_to_right, + order_by, + platform_version, + )?; + query.set_subquery(inner_query); + } + query.set_subquery_key(first.name.as_bytes().to_vec()); + Ok(()) + } + } + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// Counterpart of [`Self::recursive_conditional_insert_on_query`] + /// for this lowering: identical conditional refinement along the + /// cursor's path, but the cursorless sub-levels it creates go + /// through [`Self::recursive_insert_on_query_ordered_with_cursor`] + /// so they take their direction from `order_by`. + #[allow(clippy::too_many_arguments)] + pub(in crate::query) fn recursive_conditional_insert_on_query_ordered( + query: &mut Query, + conditional_value: Option>, + left_over_index_properties: &[&IndexProperty], + unique: bool, + starts_at_document: &StartAtDocument, + default_left_to_right: bool, + order_by: &IndexMap, + platform_version: &PlatformVersion, + ) -> Result<(), Error> { + match left_over_index_properties.split_first() { + None => { + match unique { + true => { + // In the case things are NULL we allow to have multiple values + let inner_query = Self::inner_query_from_starts_at_for_id( + Some(starts_at_document), + true, //for ids we always go left to right + ); + query.add_conditional_subquery( + QueryItem::Key(b"".to_vec()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + false => { + let inner_query = Self::inner_query_from_starts_at_for_id( + Some(starts_at_document), + default_left_to_right, + ); + + query.add_conditional_subquery( + QueryItem::Key(conditional_value.unwrap_or_default()), + Some(vec![vec![0]]), + Some(inner_query), + ); + } + } + } + Some((first, left_over)) => { + let left_to_right = order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + let StartAtDocument { + document, + document_type, + .. + } = starts_at_document; + + let lower_start_at_key = document + .get_raw_for_document_type( + first.name.as_str(), + *document_type, + None, + platform_version, + ) + .ok() + .flatten(); + + // We include it if we are not unique, + // or if we are unique but the value is empty + let non_conditional_included = !unique || lower_start_at_key.is_none(); + + let mut non_conditional_query = Self::inner_query_starts_from_key( + lower_start_at_key.clone(), + left_to_right, + non_conditional_included, + ); + + Self::recursive_insert_on_query_ordered_with_cursor( + &mut non_conditional_query, + left_over, + unique, + None, + left_to_right, + order_by, + platform_version, + )?; + + Self::recursive_conditional_insert_on_query_ordered( + &mut non_conditional_query, + lower_start_at_key, + left_over, + unique, + starts_at_document, + left_to_right, + order_by, + platform_version, + )?; + + query.add_conditional_subquery( + QueryItem::Key(conditional_value.unwrap_or_default()), + Some(vec![first.name.as_bytes().to_vec()]), + Some(non_conditional_query), + ); + } + } + Ok(()) + } + + #[cfg(any(feature = "server", feature = "verify"))] + /// v0 lowering for queries with at most one `In` clause. Differs + /// from the pre-v14 construction + /// ([`Self::get_non_primary_key_path_query_v0`]) in two ways: cursor + /// pagination over a multi-branch level (an `In` or range last + /// clause with left-over index properties) trims the branches + /// ordered before the cursor's branch, gives the branches ordered + /// after it an unfiltered default subquery, and refines only the + /// cursor's own branch with a conditional subquery — where the + /// pre-v14 construction baked the cursor's per-level start keys into + /// the default subquery applied to every branch, silently dropping + /// later branches' values below the cursor and including earlier + /// branches' values above it. And every cursorless left-over level + /// takes its direction from `order_by` instead of always the index + /// property's own. + pub(in crate::query) fn get_non_primary_key_single_in_path_query_v0( + &self, + document_type_path: Vec>, + starts_at_document: Option<(Document, bool)>, + platform_version: &PlatformVersion, + ) -> Result { + if self.internal_clauses.in_clauses.len() > 1 { + return Err(Error::Query(QuerySyntaxError::MultipleInClauses( + "There should only be one in clause", + ))); + } + let index = self.find_best_index(platform_version)?; + let ordered_clauses: Vec<&WhereClause> = index + .properties + .iter() + .filter_map(|field| self.internal_clauses.equal_clauses.get(field.name.as_str())) + .collect(); + let (last_clause, last_clause_is_range, subquery_clause) = + match self.internal_clauses.in_clauses.first() { + None => match &self.internal_clauses.range_clause { + None => (ordered_clauses.last().copied(), false, None), + Some(where_clause) => (Some(where_clause), true, None), + }, + Some(in_clause) => match &self.internal_clauses.range_clause { + None => (Some(in_clause), true, None), + Some(range_clause) => { + // Same clause ordering rule as the pre-v14 + // construction — the outer path query must operate on + // the field that appears earlier in the chosen index. + // See issue #2409. + let position_of = |field: &str| -> Option { + index + .properties + .iter() + .position(|p| p.name.as_str() == field) + }; + let in_pos = position_of(in_clause.field.as_str()); + let range_pos = position_of(range_clause.field.as_str()); + match (in_pos, range_pos) { + (Some(i), Some(r)) if i > r => { + (Some(range_clause), true, Some(in_clause)) + } + _ => (Some(in_clause), true, Some(range_clause)), + } + } + }, + }; + + // We need to get the terminal indexes unused by clauses. + let left_over_index_properties = index + .properties + .iter() + .filter(|field| { + !(self + .internal_clauses + .equal_clauses + .contains_key(field.name.as_str()) + || (last_clause.is_some() && last_clause.unwrap().field == field.name) + || (subquery_clause.is_some() && subquery_clause.unwrap().field == field.name)) + }) + .collect::>(); + + let intermediate_values = index + .properties + .iter() + .filter_map(|field| { + match self.internal_clauses.equal_clauses.get(field.name.as_str()) { + None => None, + Some(where_clause) => { + if !last_clause_is_range + && last_clause.is_some() + && last_clause.unwrap().field == field.name + { + //there is no need to give an intermediate value as the last clause is an equality + None + } else { + Some(self.document_type.serialize_value_for_key( + field.name.as_str(), + &where_clause.value, + platform_version, + )) + } + } + } + }) + .collect::>, ProtocolError>>() + .map_err(Error::from)?; + + let final_query = match last_clause { + None => { + // There is no last_clause which means we are using an index most likely because of an order_by, however we have no + // clauses, in this case we should use the first value of the index. + let first_index = index.properties.first().ok_or(Error::Drive( + DriveError::CorruptedContractIndexes("index must have properties".to_string()), + ))?; // Index must have properties + Self::recursive_create_query_ordered( + left_over_index_properties.as_slice(), + index.unique, + starts_at_document + .map(|(document, included)| StartAtDocument { + document, + document_type: self.document_type, + included, + }) + .as_ref(), + first_index, + &self.order_by, + platform_version, + )? + .expect("Index must have left over properties if no last clause") + } + Some(where_clause) => { + let left_to_right = if where_clause.operator.is_range() { + let order_clause: &OrderClause = self + .order_by + .get(where_clause.field.as_str()) + .ok_or(Error::Query(QuerySyntaxError::MissingOrderByForRange( + "query must have an orderBy field for each range element", + )))?; + + order_clause.ascending + } else { + true + }; + + // Cursor pagination over a multi-branch level: the level fans + // out (an `In` or range last clause), left-over properties + // hang under each branch, and a cursor document is present. + let sibling_aware_cursor_lowering = last_clause_is_range + && subquery_clause.is_none() + && !left_over_index_properties.is_empty() + && starts_at_document.is_some(); + + let starts_at_document_with_branch_included = if sibling_aware_cursor_lowering { + starts_at_document + .as_ref() + .map(|(document, _)| (document.clone(), true)) + } else { + None + }; + + // We should set the starts at document to be included for the query if there are + // left over index properties. + + let query_starts_at_document = if left_over_index_properties.is_empty() { + &starts_at_document + } else if sibling_aware_cursor_lowering { + // The cursor's branch always stays included at this level, + // trimming only the branches ordered before it; whether + // the cursor document itself is included is decided by the + // conditional subquery below. + &starts_at_document_with_branch_included + } else { + &None + }; + + let mut query = where_clause.to_path_query( + self.document_type, + query_starts_at_document, + left_to_right, + platform_version, + )?; + + match subquery_clause { + None => { + if sibling_aware_cursor_lowering { + let (document, included) = starts_at_document + .as_ref() + .expect("starts_at_document was checked above"); + + let (first, deeper_left_over) = left_over_index_properties + .split_first() + .expect("left_over_index_properties was checked above"); + let first_left_to_right = self + .order_by + .get(first.name.as_str()) + .map(|order_clause| order_clause.ascending) + .unwrap_or(first.ascending); + + // Branches ordered after the cursor's take everything. + let mut default_subquery = + Query::new_with_direction(first_left_to_right); + default_subquery.insert_all(); + Self::recursive_insert_on_query_ordered_with_cursor( + &mut default_subquery, + deeper_left_over, + index.unique, + None, + first_left_to_right, + &self.order_by, + platform_version, + )?; + query.set_subquery(default_subquery); + query.set_subquery_key(first.name.as_bytes().to_vec()); + + // The cursor's branch continues from the cursor. + let start_at_key = document + .get_raw_for_document_type( + where_clause.field.as_str(), + self.document_type, + None, + platform_version, + ) + .ok() + .flatten(); + Self::recursive_conditional_insert_on_query_ordered( + &mut query, + start_at_key, + left_over_index_properties.as_slice(), + index.unique, + &StartAtDocument { + document: document.clone(), + document_type: self.document_type, + included: *included, + }, + left_to_right, + &self.order_by, + platform_version, + )?; + } else { + Self::recursive_insert_on_query_ordered_with_cursor( + &mut query, + left_over_index_properties.as_slice(), + index.unique, + starts_at_document + .map(|(document, included)| StartAtDocument { + document, + document_type: self.document_type, + included, + }) + .as_ref(), + left_to_right, + &self.order_by, + platform_version, + )?; + } + } + Some(subquery_where_clause) => { + let order_clause: &OrderClause = self + .order_by + .get(subquery_where_clause.field.as_str()) + .ok_or(Error::Query(QuerySyntaxError::MissingOrderByForRange( + "query must have an orderBy field for each range element", + )))?; + let mut subquery = subquery_where_clause.to_path_query( + self.document_type, + &starts_at_document, + order_clause.ascending, + platform_version, + )?; + Self::recursive_insert_on_query_ordered_with_cursor( + &mut subquery, + left_over_index_properties.as_slice(), + index.unique, + starts_at_document + .map(|(document, included)| StartAtDocument { + document, + document_type: self.document_type, + included, + }) + .as_ref(), + left_to_right, + &self.order_by, + platform_version, + )?; + let subindex = subquery_where_clause.field.as_bytes().to_vec(); + query.set_subquery_key(subindex); + query.set_subquery(subquery); + } + }; + + query + } + }; + + let (intermediate_indexes, last_indexes) = + index.properties.split_at(intermediate_values.len()); + + // Now we should construct the path + let last_index = last_indexes.first().ok_or(Error::Query( + QuerySyntaxError::QueryOnDocumentTypeWithNoIndexes( + "document query has no index with fields", + ), + ))?; + + let mut path = document_type_path; + + for (intermediate_index, intermediate_value) in + intermediate_indexes.iter().zip(intermediate_values.iter()) + { + path.push(intermediate_index.name.as_bytes().to_vec()); + path.push(intermediate_value.as_slice().to_vec()); + } + + path.push(last_index.name.as_bytes().to_vec()); + + Ok(PathQuery::new( + path, + SizedQuery::new(final_query, self.limit, self.offset), + )) + } +} diff --git a/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs b/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs index 2d7cda0e7c..7d9d75a948 100644 --- a/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs +++ b/packages/rs-drive/src/query/non_primary_key_path_query/v1/mod.rs @@ -1,103 +1,25 @@ //! v1 of the non-primary-key path-query lowering (protocol version 14): -//! multiple `In` clauses on consecutive index properties lower to -//! multi-level key-set path queries, and two defects of the v0 lowering -//! are fixed. First, cursor pagination over a multi-branch level (a -//! single `In` or range clause with left-over index properties) no -//! longer bakes the cursor's per-level start keys into the default -//! subquery applied to every sibling branch — branches before the -//! cursor's are trimmed, branches after it are unfiltered, and only the -//! cursor's own branch is refined by a conditional subquery. Second, -//! every cursorless left-over level takes its direction from `order_by` -//! (falling back to the index property's own) where v0 always used the -//! index property's. Conservative v1 restrictions for the multi-`In` -//! shape (cursor rejection, the cross-product cap, index conformity) -//! also live here. +//! routes by query shape to the versioned shape lowerings — multiple +//! `In` clauses on consecutive index properties go to +//! [`DriveDocumentQuery::get_non_primary_key_multiple_in_path_query`], +//! everything else to +//! [`DriveDocumentQuery::get_non_primary_key_single_in_path_query`]. +//! Relative to v0 this accepts multiple `In` clauses, makes cursor +//! pagination over a multi-branch level sibling-branch correct, and +//! derives cursorless left-over directions from `order_by`; the actual +//! constructions live in the shape lowerings' version modules. -use crate::error::drive::DriveError; -use crate::error::query::QuerySyntaxError; use crate::error::Error; -use crate::query::conditions::WhereClause; -use crate::query::ordering::OrderClause; -use crate::query::{defaults, DriveDocumentQuery, StartAtDocument}; -use dpp::data_contract::document_type::accessors::DocumentTypeV0Getters; -use dpp::data_contract::document_type::methods::DocumentTypeV0Methods; -use dpp::data_contract::document_type::{Index, IndexProperty}; -use dpp::document::document_methods::DocumentMethodsV0; +use crate::query::DriveDocumentQuery; use dpp::document::Document; use dpp::version::PlatformVersion; -use dpp::ProtocolError; -use grovedb::{PathQuery, Query, QueryItem, SizedQuery}; -use indexmap::IndexMap; +use grovedb::PathQuery; impl<'a> DriveDocumentQuery<'a> { - #[cfg(any(feature = "server", feature = "verify"))] - /// v1 counterpart of [`Self::recursive_insert_on_query`] for the - /// multi-`In` lowering: no cursor support (cursors are rejected by the - /// shape preflight), and each left-over level takes its direction from - /// `order_by` — falling back to the index property's — instead of - /// always using the index property's like the v0 helper does. - pub(in crate::query) fn recursive_insert_on_query_ordered( - query: &mut Query, - left_over_index_properties: &[&IndexProperty], - unique: bool, - default_left_to_right: bool, - order_by: &IndexMap, - ) { - match left_over_index_properties.split_first() { - None => match unique { - true => { - query.set_subquery_key(vec![0]); - - // In the case things are NULL we allow to have multiple values - let inner_query = Self::inner_query_from_starts_at_for_id( - None, true, //for ids we always go left to right - ); - query.add_conditional_subquery( - QueryItem::Key(b"".to_vec()), - Some(vec![vec![0]]), - Some(inner_query), - ); - } - false => { - query.set_subquery_key(vec![0]); - // we just get all by document id order ascending - let full_query = - Self::inner_query_from_starts_at_for_id(None, default_left_to_right); - query.set_subquery(full_query); - - let inner_query = - Self::inner_query_from_starts_at_for_id(None, default_left_to_right); - query.add_conditional_subquery( - QueryItem::Key(b"".to_vec()), - Some(vec![vec![0]]), - Some(inner_query), - ); - } - }, - Some((first, left_over)) => { - let left_to_right = order_by - .get(first.name.as_str()) - .map(|order_clause| order_clause.ascending) - .unwrap_or(first.ascending); - let mut inner_query = Query::new_with_direction(left_to_right); - inner_query.insert_all(); - Self::recursive_insert_on_query_ordered( - &mut inner_query, - left_over, - unique, - left_to_right, - order_by, - ); - query.set_subquery(inner_query); - query.set_subquery_key(first.name.as_bytes().to_vec()); - } - } - } - #[cfg(any(feature = "server", feature = "verify"))] /// v1 of the non-primary-key path query lowering (protocol version 14): - /// accepts multiple `In` clauses, and lowers at-most-one-`In` shapes with - /// sibling-branch-correct cursors and order-by-aware directions. + /// routes by shape to the versioned single-`In` / multiple-`In` + /// lowerings. pub(in crate::query) fn get_non_primary_key_path_query_v1( &self, document_type_path: Vec>, @@ -118,877 +40,4 @@ impl<'a> DriveDocumentQuery<'a> { ) } } - - #[cfg(any(feature = "server", feature = "verify"))] - /// v1 counterpart of [`Self::recursive_create_query`]: identical cursor - /// threading, but recursion goes through the v1 insert helper so every - /// cursorless left-over level takes its direction from `order_by`. - pub(in crate::query) fn recursive_create_query_ordered( - left_over_index_properties: &[&IndexProperty], - unique: bool, - starts_at_document: Option<&StartAtDocument>, //for key level, included - indexed_property: &IndexProperty, - order_by: &IndexMap, - platform_version: &PlatformVersion, - ) -> Result, Error> { - match left_over_index_properties.split_first() { - None => Ok(None), - Some((first, left_over)) => { - let left_to_right = order_by - .get(first.name.as_str()) - .map(|order_clause| order_clause.ascending) - .unwrap_or(first.ascending); - - let mut inner_query = Self::inner_query_from_starts_at( - starts_at_document, - indexed_property, - left_to_right, - platform_version, - )?; - Self::recursive_insert_on_query_ordered_with_cursor( - &mut inner_query, - left_over, - unique, - starts_at_document, - left_to_right, - order_by, - platform_version, - )?; - Ok(Some(inner_query)) - } - } - } - - #[cfg(any(feature = "server", feature = "verify"))] - /// v1 counterpart of [`Self::recursive_insert_on_query`]: identical - /// cursor threading, but a cursorless level takes its direction from - /// `order_by` (falling back to the index property's own) where the v0 - /// helper always used the index property's. - #[allow(clippy::too_many_arguments)] - pub(in crate::query) fn recursive_insert_on_query_ordered_with_cursor( - query: &mut Query, - left_over_index_properties: &[&IndexProperty], - unique: bool, - starts_at_document: Option<&StartAtDocument>, //for key level, included - default_left_to_right: bool, - order_by: &IndexMap, - platform_version: &PlatformVersion, - ) -> Result<(), Error> { - match left_over_index_properties.split_first() { - None => { - match unique { - true => { - query.set_subquery_key(vec![0]); - - // In the case things are NULL we allow to have multiple values - let inner_query = Self::inner_query_from_starts_at_for_id( - starts_at_document, - true, //for ids we always go left to right - ); - query.add_conditional_subquery( - QueryItem::Key(b"".to_vec()), - Some(vec![vec![0]]), - Some(inner_query), - ); - } - false => { - query.set_subquery_key(vec![0]); - // we just get all by document id order ascending - let full_query = - Self::inner_query_from_starts_at_for_id(None, default_left_to_right); - query.set_subquery(full_query); - - let inner_query = Self::inner_query_from_starts_at_for_id( - starts_at_document, - default_left_to_right, - ); - - query.add_conditional_subquery( - QueryItem::Key(b"".to_vec()), - Some(vec![vec![0]]), - Some(inner_query), - ); - } - } - Ok(()) - } - Some((first, left_over)) => { - let left_to_right = order_by - .get(first.name.as_str()) - .map(|order_clause| order_clause.ascending) - .unwrap_or(first.ascending); - - if let Some(start_at_document_inner) = starts_at_document { - let StartAtDocument { - document, - document_type, - included, - } = start_at_document_inner; - let start_at_key = document - .get_raw_for_document_type( - first.name.as_str(), - *document_type, - None, - platform_version, - ) - .ok() - .flatten(); - - // We should always include if we have left_over - let non_conditional_included = - !left_over.is_empty() || *included || start_at_key.is_none(); - - let mut non_conditional_query = Self::inner_query_starts_from_key( - start_at_key.clone(), - left_to_right, - non_conditional_included, - ); - - Self::recursive_insert_on_query_ordered_with_cursor( - &mut non_conditional_query, - left_over, - unique, - None, - left_to_right, - order_by, - platform_version, - )?; - - Self::recursive_conditional_insert_on_query_ordered( - &mut non_conditional_query, - start_at_key, - left_over, - unique, - start_at_document_inner, - left_to_right, - order_by, - platform_version, - )?; - - query.set_subquery(non_conditional_query); - } else { - let mut inner_query = Query::new_with_direction(left_to_right); - inner_query.insert_all(); - Self::recursive_insert_on_query_ordered_with_cursor( - &mut inner_query, - left_over, - unique, - None, - left_to_right, - order_by, - platform_version, - )?; - query.set_subquery(inner_query); - } - query.set_subquery_key(first.name.as_bytes().to_vec()); - Ok(()) - } - } - } - - #[cfg(any(feature = "server", feature = "verify"))] - /// v1 counterpart of [`Self::recursive_conditional_insert_on_query`]: - /// identical conditional refinement along the cursor's path, but the - /// cursorless sub-levels it creates go through the v1 insert helper so - /// they take their direction from `order_by`. - #[allow(clippy::too_many_arguments)] - pub(in crate::query) fn recursive_conditional_insert_on_query_ordered( - query: &mut Query, - conditional_value: Option>, - left_over_index_properties: &[&IndexProperty], - unique: bool, - starts_at_document: &StartAtDocument, - default_left_to_right: bool, - order_by: &IndexMap, - platform_version: &PlatformVersion, - ) -> Result<(), Error> { - match left_over_index_properties.split_first() { - None => { - match unique { - true => { - // In the case things are NULL we allow to have multiple values - let inner_query = Self::inner_query_from_starts_at_for_id( - Some(starts_at_document), - true, //for ids we always go left to right - ); - query.add_conditional_subquery( - QueryItem::Key(b"".to_vec()), - Some(vec![vec![0]]), - Some(inner_query), - ); - } - false => { - let inner_query = Self::inner_query_from_starts_at_for_id( - Some(starts_at_document), - default_left_to_right, - ); - - query.add_conditional_subquery( - QueryItem::Key(conditional_value.unwrap_or_default()), - Some(vec![vec![0]]), - Some(inner_query), - ); - } - } - } - Some((first, left_over)) => { - let left_to_right = order_by - .get(first.name.as_str()) - .map(|order_clause| order_clause.ascending) - .unwrap_or(first.ascending); - - let StartAtDocument { - document, - document_type, - .. - } = starts_at_document; - - let lower_start_at_key = document - .get_raw_for_document_type( - first.name.as_str(), - *document_type, - None, - platform_version, - ) - .ok() - .flatten(); - - // We include it if we are not unique, - // or if we are unique but the value is empty - let non_conditional_included = !unique || lower_start_at_key.is_none(); - - let mut non_conditional_query = Self::inner_query_starts_from_key( - lower_start_at_key.clone(), - left_to_right, - non_conditional_included, - ); - - Self::recursive_insert_on_query_ordered_with_cursor( - &mut non_conditional_query, - left_over, - unique, - None, - left_to_right, - order_by, - platform_version, - )?; - - Self::recursive_conditional_insert_on_query_ordered( - &mut non_conditional_query, - lower_start_at_key, - left_over, - unique, - starts_at_document, - left_to_right, - order_by, - platform_version, - )?; - - query.add_conditional_subquery( - QueryItem::Key(conditional_value.unwrap_or_default()), - Some(vec![first.name.as_bytes().to_vec()]), - Some(non_conditional_query), - ); - } - } - Ok(()) - } - - #[cfg(any(feature = "server", feature = "verify"))] - /// v1 lowering for queries with at most one `In` clause. Differs from - /// [`Self::get_non_primary_key_path_query_v0`] in two ways: cursor - /// pagination over a multi-branch level (an `In` or range last clause - /// with left-over index properties) trims the branches ordered before - /// the cursor's branch, gives the branches ordered after it an - /// unfiltered default subquery, and refines only the cursor's own - /// branch with a conditional subquery — where v0 baked the cursor's - /// per-level start keys into the default subquery applied to every - /// branch, silently dropping later branches' values below the cursor - /// and including earlier branches' values above it. And every - /// cursorless left-over level takes its direction from `order_by` - /// instead of always the index property's own. - pub(in crate::query) fn get_non_primary_key_single_in_path_query( - &self, - document_type_path: Vec>, - starts_at_document: Option<(Document, bool)>, - platform_version: &PlatformVersion, - ) -> Result { - if self.internal_clauses.in_clauses.len() > 1 { - return Err(Error::Query(QuerySyntaxError::MultipleInClauses( - "There should only be one in clause", - ))); - } - let index = self.find_best_index(platform_version)?; - let ordered_clauses: Vec<&WhereClause> = index - .properties - .iter() - .filter_map(|field| self.internal_clauses.equal_clauses.get(field.name.as_str())) - .collect(); - let (last_clause, last_clause_is_range, subquery_clause) = - match self.internal_clauses.in_clauses.first() { - None => match &self.internal_clauses.range_clause { - None => (ordered_clauses.last().copied(), false, None), - Some(where_clause) => (Some(where_clause), true, None), - }, - Some(in_clause) => match &self.internal_clauses.range_clause { - None => (Some(in_clause), true, None), - Some(range_clause) => { - // Same clause ordering rule as v0 — the outer path - // query must operate on the field that appears earlier - // in the chosen index. See issue #2409. - let position_of = |field: &str| -> Option { - index - .properties - .iter() - .position(|p| p.name.as_str() == field) - }; - let in_pos = position_of(in_clause.field.as_str()); - let range_pos = position_of(range_clause.field.as_str()); - match (in_pos, range_pos) { - (Some(i), Some(r)) if i > r => { - (Some(range_clause), true, Some(in_clause)) - } - _ => (Some(in_clause), true, Some(range_clause)), - } - } - }, - }; - - // We need to get the terminal indexes unused by clauses. - let left_over_index_properties = index - .properties - .iter() - .filter(|field| { - !(self - .internal_clauses - .equal_clauses - .contains_key(field.name.as_str()) - || (last_clause.is_some() && last_clause.unwrap().field == field.name) - || (subquery_clause.is_some() && subquery_clause.unwrap().field == field.name)) - }) - .collect::>(); - - let intermediate_values = index - .properties - .iter() - .filter_map(|field| { - match self.internal_clauses.equal_clauses.get(field.name.as_str()) { - None => None, - Some(where_clause) => { - if !last_clause_is_range - && last_clause.is_some() - && last_clause.unwrap().field == field.name - { - //there is no need to give an intermediate value as the last clause is an equality - None - } else { - Some(self.document_type.serialize_value_for_key( - field.name.as_str(), - &where_clause.value, - platform_version, - )) - } - } - } - }) - .collect::>, ProtocolError>>() - .map_err(Error::from)?; - - let final_query = match last_clause { - None => { - // There is no last_clause which means we are using an index most likely because of an order_by, however we have no - // clauses, in this case we should use the first value of the index. - let first_index = index.properties.first().ok_or(Error::Drive( - DriveError::CorruptedContractIndexes("index must have properties".to_string()), - ))?; // Index must have properties - Self::recursive_create_query_ordered( - left_over_index_properties.as_slice(), - index.unique, - starts_at_document - .map(|(document, included)| StartAtDocument { - document, - document_type: self.document_type, - included, - }) - .as_ref(), - first_index, - &self.order_by, - platform_version, - )? - .expect("Index must have left over properties if no last clause") - } - Some(where_clause) => { - let left_to_right = if where_clause.operator.is_range() { - let order_clause: &OrderClause = self - .order_by - .get(where_clause.field.as_str()) - .ok_or(Error::Query(QuerySyntaxError::MissingOrderByForRange( - "query must have an orderBy field for each range element", - )))?; - - order_clause.ascending - } else { - true - }; - - // Cursor pagination over a multi-branch level: the level fans - // out (an `In` or range last clause), left-over properties - // hang under each branch, and a cursor document is present. - let sibling_aware_cursor_lowering = last_clause_is_range - && subquery_clause.is_none() - && !left_over_index_properties.is_empty() - && starts_at_document.is_some(); - - let starts_at_document_with_branch_included = if sibling_aware_cursor_lowering { - starts_at_document - .as_ref() - .map(|(document, _)| (document.clone(), true)) - } else { - None - }; - - // We should set the starts at document to be included for the query if there are - // left over index properties. - - let query_starts_at_document = if left_over_index_properties.is_empty() { - &starts_at_document - } else if sibling_aware_cursor_lowering { - // The cursor's branch always stays included at this level, - // trimming only the branches ordered before it; whether - // the cursor document itself is included is decided by the - // conditional subquery below. - &starts_at_document_with_branch_included - } else { - &None - }; - - let mut query = where_clause.to_path_query( - self.document_type, - query_starts_at_document, - left_to_right, - platform_version, - )?; - - match subquery_clause { - None => { - if sibling_aware_cursor_lowering { - let (document, included) = starts_at_document - .as_ref() - .expect("starts_at_document was checked above"); - - let (first, deeper_left_over) = left_over_index_properties - .split_first() - .expect("left_over_index_properties was checked above"); - let first_left_to_right = self - .order_by - .get(first.name.as_str()) - .map(|order_clause| order_clause.ascending) - .unwrap_or(first.ascending); - - // Branches ordered after the cursor's take everything. - let mut default_subquery = - Query::new_with_direction(first_left_to_right); - default_subquery.insert_all(); - Self::recursive_insert_on_query_ordered_with_cursor( - &mut default_subquery, - deeper_left_over, - index.unique, - None, - first_left_to_right, - &self.order_by, - platform_version, - )?; - query.set_subquery(default_subquery); - query.set_subquery_key(first.name.as_bytes().to_vec()); - - // The cursor's branch continues from the cursor. - let start_at_key = document - .get_raw_for_document_type( - where_clause.field.as_str(), - self.document_type, - None, - platform_version, - ) - .ok() - .flatten(); - Self::recursive_conditional_insert_on_query_ordered( - &mut query, - start_at_key, - left_over_index_properties.as_slice(), - index.unique, - &StartAtDocument { - document: document.clone(), - document_type: self.document_type, - included: *included, - }, - left_to_right, - &self.order_by, - platform_version, - )?; - } else { - Self::recursive_insert_on_query_ordered_with_cursor( - &mut query, - left_over_index_properties.as_slice(), - index.unique, - starts_at_document - .map(|(document, included)| StartAtDocument { - document, - document_type: self.document_type, - included, - }) - .as_ref(), - left_to_right, - &self.order_by, - platform_version, - )?; - } - } - Some(subquery_where_clause) => { - let order_clause: &OrderClause = self - .order_by - .get(subquery_where_clause.field.as_str()) - .ok_or(Error::Query(QuerySyntaxError::MissingOrderByForRange( - "query must have an orderBy field for each range element", - )))?; - let mut subquery = subquery_where_clause.to_path_query( - self.document_type, - &starts_at_document, - order_clause.ascending, - platform_version, - )?; - Self::recursive_insert_on_query_ordered_with_cursor( - &mut subquery, - left_over_index_properties.as_slice(), - index.unique, - starts_at_document - .map(|(document, included)| StartAtDocument { - document, - document_type: self.document_type, - included, - }) - .as_ref(), - left_to_right, - &self.order_by, - platform_version, - )?; - let subindex = subquery_where_clause.field.as_bytes().to_vec(); - query.set_subquery_key(subindex); - query.set_subquery(subquery); - } - }; - - query - } - }; - - let (intermediate_indexes, last_indexes) = - index.properties.split_at(intermediate_values.len()); - - // Now we should construct the path - let last_index = last_indexes.first().ok_or(Error::Query( - QuerySyntaxError::QueryOnDocumentTypeWithNoIndexes( - "document query has no index with fields", - ), - ))?; - - let mut path = document_type_path; - - for (intermediate_index, intermediate_value) in - intermediate_indexes.iter().zip(intermediate_values.iter()) - { - path.push(intermediate_index.name.as_bytes().to_vec()); - path.push(intermediate_value.as_slice().to_vec()); - } - - path.push(last_index.name.as_bytes().to_vec()); - - Ok(PathQuery::new( - path, - SizedQuery::new(final_query, self.limit, self.offset), - )) - } - - #[cfg(any(feature = "server", feature = "verify"))] - /// Lowers a query with multiple `In` clauses into a path query whose - /// levels carry one key set per `In` clause, in index property order, - /// followed by an optional range level and the usual left-over / - /// terminal levels. Only reachable through the v1 lowering. - pub(in crate::query) fn get_non_primary_key_multiple_in_path_query( - &self, - document_type_path: Vec>, - starts_at_document: Option<(Document, bool)>, - platform_version: &PlatformVersion, - ) -> Result { - // Conservative v1: the cross-branch cursor machinery is not wired - // for key-set branching at more than one level, so reject cursors - // instead of shipping silently wrong pagination. - if starts_at_document.is_some() || self.start_at.is_some() { - return Err(Error::Query(QuerySyntaxError::Unsupported( - "startAt/startAfter is not supported with multiple in clauses".to_string(), - ))); - } - - let (index, ordered_in_clauses, equality_len) = - self.find_best_index_for_multiple_in_clauses()?; - - // Bound the branch enumeration: the product of the in list sizes is - // the number of index subtrees the query opens. - let mut cross_product: usize = 1; - for in_clause in &ordered_in_clauses { - let in_values = in_clause.in_values().into_data_with_error()??; - cross_product = cross_product.saturating_mul(in_values.len()); - } - if cross_product > defaults::MAX_IN_CROSS_PRODUCT_SIZE { - return Err(Error::Query(QuerySyntaxError::InvalidInClause(format!( - "the product of in clause list sizes must be at most {}, got {}", - defaults::MAX_IN_CROSS_PRODUCT_SIZE, - cross_product - )))); - } - - let left_over_index_properties = index - .properties - .iter() - .filter(|field| { - !(self - .internal_clauses - .equal_clauses - .contains_key(field.name.as_str()) - || ordered_in_clauses - .iter() - .any(|in_clause| in_clause.field == field.name) - || self - .internal_clauses - .range_clause - .as_ref() - .is_some_and(|range_clause| range_clause.field == field.name)) - }) - .collect::>(); - - // Every level that fans out (each in clause, and the range clause) - // needs an explicit ordering, like any range-class clause. - let direction_for = |field: &str| -> Result { - let order_clause: &OrderClause = self.order_by.get(field).ok_or(Error::Query( - QuerySyntaxError::MissingOrderByForRange( - "query must have an orderBy field for each range element", - ), - ))?; - Ok(order_clause.ascending) - }; - - // Build the query bottom-up. The deepest clause level is the range - // clause when present, otherwise the last in clause; the left-over - // index properties and the terminal document level hang under it. - let (mut child_field, mut child_query, deepest_left_to_right) = - match &self.internal_clauses.range_clause { - Some(range_clause) => { - let left_to_right = direction_for(range_clause.field.as_str())?; - let query = range_clause.to_path_query( - self.document_type, - &None, - left_to_right, - platform_version, - )?; - (range_clause.field.clone(), query, left_to_right) - } - None => { - let deepest_in_clause = - *ordered_in_clauses.last().expect("more than one in clause"); - let left_to_right = direction_for(deepest_in_clause.field.as_str())?; - let query = deepest_in_clause.to_path_query( - self.document_type, - &None, - left_to_right, - platform_version, - )?; - (deepest_in_clause.field.clone(), query, left_to_right) - } - }; - Self::recursive_insert_on_query_ordered( - &mut child_query, - left_over_index_properties.as_slice(), - index.unique, - deepest_left_to_right, - &self.order_by, - ); - - // Wrap the remaining in levels around it, deepest first. When a - // range clause is present every in clause wraps; otherwise the - // deepest in clause is already the leaf built above. - let wrapped_in_clauses = if self.internal_clauses.range_clause.is_some() { - ordered_in_clauses.as_slice() - } else { - &ordered_in_clauses[..ordered_in_clauses.len() - 1] - }; - for in_clause in wrapped_in_clauses.iter().rev() { - let left_to_right = direction_for(in_clause.field.as_str())?; - let mut query = in_clause.to_path_query( - self.document_type, - &None, - left_to_right, - platform_version, - )?; - query.set_subquery_key(child_field.as_bytes().to_vec()); - query.set_subquery(child_query); - child_field = in_clause.field.clone(); - child_query = query; - } - - let intermediate_values = index.properties[..equality_len] - .iter() - .map(|field| { - let where_clause = self - .internal_clauses - .equal_clauses - .get(field.name.as_str()) - .expect("equality prefix was validated during index selection"); - self.document_type.serialize_value_for_key( - field.name.as_str(), - &where_clause.value, - platform_version, - ) - }) - .collect::>, ProtocolError>>() - .map_err(Error::from)?; - - let mut path = document_type_path; - for (intermediate_index, intermediate_value) in index.properties[..equality_len] - .iter() - .zip(intermediate_values.iter()) - { - path.push(intermediate_index.name.as_bytes().to_vec()); - path.push(intermediate_value.as_slice().to_vec()); - } - path.push(child_field.as_bytes().to_vec()); - - Ok(PathQuery::new( - path, - SizedQuery::new(child_query, self.limit, self.offset), - )) - } - - #[cfg(any(feature = "server", feature = "verify"))] - /// Finds the best index for a query with more than one `In` clause, and - /// returns it together with the query's `In` clauses ordered by their - /// position in that index and the length of the equality prefix. - /// - /// A candidate index conforms when its properties decompose, left to - /// right, into: the equality-clause fields (exactly covering positions - /// `0..E`), then every `In` field on consecutive positions `E..E+K`, - /// then — when a range clause is present — the range field. The usual - /// tail rule from [`Index::matches`] still applies with the deepest `In` - /// field playing the role of the in field, as do the order-by continuity - /// rules and [`defaults::MAX_INDEX_DIFFERENCE`]. - pub(in crate::query) fn find_best_index_for_multiple_in_clauses( - &self, - ) -> Result<(&Index, Vec<&WhereClause>, usize), Error> { - let equal_clauses = &self.internal_clauses.equal_clauses; - let in_clauses = &self.internal_clauses.in_clauses; - let range_field = self - .internal_clauses - .range_clause - .as_ref() - .map(|range_clause| range_clause.field.as_str()); - - let mut fields = equal_clauses - .keys() - .map(|s| s.as_str()) - .collect::>(); - if let Some(range_field) = range_field { - fields.push(range_field); - } - fields.extend(in_clauses.iter().map(|in_clause| in_clause.field.as_str())); - - let order_by_keys: Vec<&str> = self - .order_by - .keys() - .map(|key: &String| { - let str = key.as_str(); - if !fields.contains(&str) { - fields.push(str); - } - str - }) - .collect(); - - let equality_len = equal_clauses.len(); - let mut best: Option<(&Index, Vec<&WhereClause>, u16)> = None; - for index in self.document_type.indexes().values() { - let mut positioned: Vec<(usize, &WhereClause)> = Vec::with_capacity(in_clauses.len()); - for in_clause in in_clauses { - match index - .properties - .iter() - .position(|property| property.name == in_clause.field) - { - Some(position) => positioned.push((position, in_clause)), - None => break, - } - } - if positioned.len() != in_clauses.len() { - continue; - } - positioned.sort_by_key(|(position, _)| *position); - - // The equality clauses must exactly cover the index prefix - if index.properties.len() < equality_len - || !index.properties[..equality_len] - .iter() - .all(|property| equal_clauses.contains_key(property.name.as_str())) - { - continue; - } - // The in clauses must sit on consecutive properties right after it - let consecutive_after_prefix = positioned - .iter() - .enumerate() - .all(|(i, (position, _))| *position == equality_len + i); - if !consecutive_after_prefix { - continue; - } - // A range clause must sit immediately after the in block - if let Some(range_field) = range_field { - match index.properties.get(equality_len + positioned.len()) { - Some(property) if property.name == range_field => {} - _ => continue, - } - } - - let deepest_in_field = positioned - .last() - .expect("more than one in clause") - .1 - .field - .as_str(); - let Some(difference) = index.matches(&fields, Some(deepest_in_field), &order_by_keys) - else { - continue; - }; - let ordered = positioned - .into_iter() - .map(|(_, in_clause)| in_clause) - .collect::>(); - if difference == 0 { - return Ok((index, ordered, equality_len)); - } - match &best { - Some((_, _, best_difference)) if *best_difference <= difference => {} - _ => best = Some((index, ordered, difference)), - } - } - - let (index, ordered, difference) = best.ok_or(Error::Query( - QuerySyntaxError::WhereClauseOnNonIndexedProperty(format!( - "query with multiple in clauses must be for valid indexes with the in clauses on \ - consecutive index properties after the equality clauses, valid indexes are: {:?}", - self.document_type.indexes() - )), - ))?; - if difference > defaults::MAX_INDEX_DIFFERENCE { - return Err(Error::Query(QuerySyntaxError::QueryTooFarFromIndex( - "query must better match an existing index", - ))); - } - Ok((index, ordered, equality_len)) - } } diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs index e728061577..9c503513cd 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/mod.rs @@ -48,6 +48,19 @@ pub struct DriveDocumentQueryMethodVersions { /// version 14) accepts multiple `In` clauses on consecutive index /// properties, lowering them to multi-level key-set path queries. pub non_primary_key_path_query: FeatureVersion, + /// Lowering for query shapes with at most one non-primary-key `In` + /// clause. Same consensus rationale as + /// `non_primary_key_path_query`, which routes those shapes here + /// from its v1 on. Present in every version table so the slot + /// exists for older protocol versions; unreachable while + /// `non_primary_key_path_query` is 0, whose v0 lowering carries its + /// own frozen single-`In` construction. + pub non_primary_key_single_in_path_query: FeatureVersion, + /// Lowering for query shapes with multiple non-primary-key `In` + /// clauses on consecutive index properties, producing multi-level + /// key-set path queries. Same consensus rationale and reachability + /// as `non_primary_key_single_in_path_query`. + pub non_primary_key_multiple_in_path_query: FeatureVersion, /// Grouping of a query's raw where clauses into equality / range / /// in buckets (`WhereClause::group_clauses`). Versioned because the /// error surface for rejected shapes is part of the query contract: diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs index 004078759d..a49fbdea9c 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v1.rs @@ -19,6 +19,8 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V1: DriveDocumentMethodVersions = detect_sum_mode: 0, detect_ranked_mode: 0, non_primary_key_path_query: 0, + non_primary_key_single_in_path_query: 0, + non_primary_key_multiple_in_path_query: 0, where_clause_grouping: 0, }, delete: DriveDocumentDeleteMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs index f1cfbe10f4..a00c60f532 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v2.rs @@ -21,6 +21,8 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V2: DriveDocumentMethodVersions = detect_sum_mode: 0, detect_ranked_mode: 0, non_primary_key_path_query: 0, + non_primary_key_single_in_path_query: 0, + non_primary_key_multiple_in_path_query: 0, where_clause_grouping: 0, }, delete: DriveDocumentDeleteMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs index 31966e7f68..6cb890d609 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v3.rs @@ -31,6 +31,8 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V3: DriveDocumentMethodVersions = detect_sum_mode: 0, detect_ranked_mode: 0, non_primary_key_path_query: 0, + non_primary_key_single_in_path_query: 0, + non_primary_key_multiple_in_path_query: 0, where_clause_grouping: 0, }, delete: DriveDocumentDeleteMethodVersions { diff --git a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs index 172be6080b..912dad4aaa 100644 --- a/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs +++ b/packages/rs-platform-version/src/version/drive_versions/drive_document_method_versions/v4.rs @@ -78,6 +78,8 @@ pub const DRIVE_DOCUMENT_METHOD_VERSIONS_V4: DriveDocumentMethodVersions = detect_sum_mode: 0, detect_ranked_mode: 0, non_primary_key_path_query: 1, + non_primary_key_single_in_path_query: 0, + non_primary_key_multiple_in_path_query: 0, where_clause_grouping: 1, }, delete: DriveDocumentDeleteMethodVersions {