From cf9aba65d1ecae48b44beed088f2c3d7244be353 Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Mon, 17 Aug 2026 12:47:43 -0700 Subject: [PATCH] Add search tutorial --- docs/tutorials/python/search.md | 371 ++++++++++++ .../python/tutorial_scripts/search.py | 539 ++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 911 insertions(+) create mode 100644 docs/tutorials/python/search.md create mode 100644 docs/tutorials/python/tutorial_scripts/search.py diff --git a/docs/tutorials/python/search.md b/docs/tutorials/python/search.md new file mode 100644 index 000000000..8b01d1071 --- /dev/null +++ b/docs/tutorials/python/search.md @@ -0,0 +1,371 @@ +# Search Indexes + +A [SearchIndex][synapseclient.models.SearchIndex] is a Synapse entity whose content is +defined by a Synapse SQL query. Synapse builds an OpenSearch index from the rows that +query returns, which gives you full-text search, relevance ranking, faceting, and +autocomplete over a table or view. + +This is a different way of asking questions than a +[Table](table.md) or a [Materialized View](materializedview.md). A table is queried with +Synapse SQL and answers "which rows match these exact conditions?". A search index is +queried with the +[OpenSearch Query DSL](https://docs.opensearch.org/latest/query-dsl/) and answers +"which rows are most relevant to this text?" — matching word stems, ignoring +punctuation and case, ranking the best matches first, and counting how many rows fall +into each category. It is what you would put behind a search box. + +This tutorial will walk you through creating a search index and querying it with the +Synapse Python client. + +## Tutorial Purpose +In this tutorial, you will: + +1. Log in, get your project, and create a table to index +2. Create a SearchIndex and wait for it to build +3. Run a full-text search +4. Highlight where the match happened +5. Combine scored clauses with unscored filters, and sort the results +6. Count facets with aggregations +7. Power a type-ahead box with autocomplete +8. Page through results +9. Tune matching with synonyms and analyzers + +## Prerequisites +* This tutorial assumes that you have a Synapse project. +* Pandas must also be installed as shown in the [installation documentation](../installation.md). +* Creating a SearchIndex may be restricted on some Synapse stacks. If `store()` fails + with a 403, your account is not permitted to create search indexes there. + +## 1. Log in, get your project, and create a table to index + +A search index is always defined over an existing table-like entity, so we first create +a small table of study summaries to search over. + +You will want to replace `"My uniquely named project about Alzheimer's Disease"` with +the name of your project. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:setup" +``` + +The steps below use two small helpers — one to print the rows a query matched, and one +to wait out the index build. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:helpers" +``` + +## 2. Create a SearchIndex and wait for it to build + +The `defining_sql` decides which rows and columns are indexed. Unlike a Materialized +View, it must reference exactly one table-like entity — JOIN and UNION across several +entities are not supported. If you need to search across several tables, build a +[Materialized View](materializedview.md) first and index that. + +Storing the entity returns as soon as Synapse has accepted it, but the OpenSearch index +behind it is built in the background. Until the build finishes, queries against the +index either raise an error or report zero hits, which is why we poll with +`wait_for_index`. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:create_index" +``` + +
+ Creating the index should look like: +``` +Created SearchIndex with ID: syn68123456 +Waiting for the search index to build... +Waiting for the search index to build... +Index syn68123456 is queryable with 6 rows +``` +
+ +**Note**: The index tracks its source. When rows in the underlying table change, the +index is updated in the background — you do not need to re-store the SearchIndex. + +## 3. Run a full-text search + +A [`match`](https://docs.opensearch.org/latest/query-dsl/full-text/match/) clause is the +workhorse of full-text search: the text you pass is analyzed the same way the column was +analyzed, so `"alzheimer"` matches `"Alzheimer's disease"`. Every clause kind Synapse +accepts is listed on [Query][synapseclient.models.search_dsl.Query]. + +By default a hit carries every indexed column. `source` narrows that down, and +`response_parts` asks for extras beyond the hits themselves — here the total hit count +and the columns each hit carries. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:full_text_search" +``` + +
+ The results of your searches should look like: +``` +Abstracts mentioning Alzheimer's: +columns: ['study_name', 'diagnosis'] +total_hits=3, returned=3 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + +Anything mentioning tau: +total_hits=1, returned=1 + ROW_ID=5 {'study_name': 'MCI Plasma Biomarkers'} +``` +
+ +Hits come back ranked by relevance, and each one carries its score on +[`hit.score`][synapseclient.models.SearchHit] along with the `row_id` and `row_version` +of the source row. + +## 4. Highlight where the match happened + +A result list is much easier to read when it shows the matching text in context. +`highlight` returns short fragments of the matched columns with the matching terms +wrapped in `` tags. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:columns_and_highlights" +``` + +
+ The result of your highlighted search should look like: +``` +Studies that sequenced something: + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'assay': 'rnaSeq'} + abstract: ['Bulk RNA sequencing across four brain regions in a cohort'] + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'assay': 'wholeGenomeSeq'} + abstract: ['Whole genome sequencing of temporal cortex samples from'] + ROW_ID=4 {'study_name': 'Healthy Aging Single Cell Atlas', 'assay': 'snrnaSeq'} + abstract: ['Single nucleus RNA sequencing of hippocampus from'] +``` +
+ +**Note**: Highlighting, like relevance scoring, depends on the column being indexed as +analyzed text. Step 9 covers how to control that with a +[SearchConfiguration][synapseclient.models.SearchConfiguration]. + +## 5. Combine scored clauses with unscored filters, and sort the results + +A [`bool`](https://docs.opensearch.org/latest/query-dsl/compound/bool/) clause is how +you build a real search request out of several conditions: + +* `must` clauses have to match and **do** contribute to the relevance score +* `filter` and `must_not` clauses have to match (or not match) but **do not** affect + the score — use these for hard constraints like a numeric cutoff +* `should` clauses boost the rows that match them without excluding the rows that don't + +Passing `sort` replaces relevance ranking with an ordering of your choosing. Only column +and `_score` sorts are accepted. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:filters_and_sorting" +``` + +
+ The result of your filtered search should look like: +``` +Sequencing studies with at least 200 participants, largest first: +total_hits=2, returned=2 + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} +``` +
+ +## 6. Count facets with aggregations + +Aggregations answer "how many rows are there of each kind?" — the counts you see next to +the checkboxes in a faceted search UI. A +[`terms`](https://docs.opensearch.org/latest/aggregations/bucket/terms/) aggregation +produces one bucket per distinct value of a column; metric aggregations like `avg` and +`stats` summarize a numeric column. Results come back on `aggregation_results` as the +raw OpenSearch response, with field references rewritten back to your column names. + +`post_filter` is what keeps a facet list usable: it narrows the hits *after* the +aggregations have been computed, so selecting one diagnosis does not make the other +diagnosis counts disappear. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:aggregations" +``` + +
+ The result of your faceted search should look like: +``` +Hits after the post filter: +total_hits=3, returned=3 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} + +Facet counts across all studies: +{ + "by_diagnosis": { + "doc_count_error_upper_bound": 0, + "sum_other_doc_count": 0, + "buckets": [ + { + "key": "Alzheimer's Disease", + "doc_count": 3 + }, + { + "key": "Cognitively Normal", + "doc_count": 1 + }, + { + "key": "Mild Cognitive Impairment", + "doc_count": 1 + }, + { + "key": "Parkinson's Disease", + "doc_count": 1 + } + ] + }, + "mean_cohort_size": { + "value": 261.6666666666667 + } +} +``` +
+ +## 7. Power a type-ahead box with autocomplete + +[`autocomplete()`][synapseclient.models.SearchIndex.autocomplete] is a separate, +synchronous endpoint meant for search-as-you-type: it returns its hits directly instead +of going through the asynchronous job service, so it is fast enough to call on every +keystroke. In exchange, it only accepts prefix-style clauses — `prefix`, +`match_phrase_prefix`, or `match_bool_prefix` — and returns at most 8 hits. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:autocomplete" +``` + +
+ The result of your autocomplete request should look like: +``` +Suggestions for 'Mayo Cl': + ['Mayo Clinic Whole Genome'] +``` +
+ +## 8. Page through results + +A query returns at most 100 hits at a time (25 by default). `from_` and `size` walk +through the result set the way page numbers do. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:pagination" +``` + +
+ The result of paging through your index should look like: +``` +Page starting at offset 0: +total_hits=6, returned=2 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'participant_count': '400'} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'participant_count': '350'} +Page starting at offset 2: +total_hits=6, returned=2 + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'participant_count': '300'} + ROW_ID=5 {'study_name': 'MCI Plasma Biomarkers', 'participant_count': '220'} +Page starting at offset 4: +total_hits=6, returned=2 + ROW_ID=6 {'study_name': 'Parkinson Comparative Cohort', 'participant_count': '180'} + ROW_ID=4 {'study_name': 'Healthy Aging Single Cell Atlas', 'participant_count': '120'} +``` +
+ +**Note**: Offset paging gets expensive deep into a large result set. For that case each +response carries a `next_search_after` cursor — pass it back unchanged as +`SearchQuery(search_after=...)` on the following request and leave `from_` unset. + +## 9. Tune matching with synonyms and analyzers + +Everything above relies on how each column was analyzed when the index was built: how +text is split into tokens, which tokens are dropped, and how they are normalized. Four +org-scoped resources let you control that: + +* [SynonymSet][synapseclient.models.SynonymSet] — terms that should be treated as + equivalent, so someone searching `AD` finds abstracts that say + "Alzheimer's disease" +* [TextAnalyzer][synapseclient.models.TextAnalyzer] — a named OpenSearch analyzer: a + tokenizer plus a chain of token filters, which may reference a SynonymSet +* [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] — a reusable + bundle assigning specific analyzers to specific columns +* [SearchConfiguration][synapseclient.models.SearchConfiguration] — bundles a default + analyzer with any column overrides; this is what a SearchIndex actually points at + +Each resource belongs to an [Organization][synapseclient.models.Organization] and is +referenced from another resource by its qualified name, +`{organization_name}-{name}`, written as `{"$ref": "my.org-my_analyzer"}`. + +!!! warning "Restricted and permanent" + Creating and updating these resources is restricted to Sage Bionetworks employees, + and the REST API has no delete endpoint for any of them. Once created, a + SynonymSet, TextAnalyzer, ColumnAnalyzerOverride, or SearchConfiguration cannot be + removed, and its owning Organization can no longer be deleted either. Choose names + deliberately. + +Note where the synonym filter goes below. The analyzer declares both a `default` chain, +used when rows are indexed, and a `default_search` chain, used when a query is analyzed. +Putting the synonyms only in `default_search` expands the incoming query instead of +storing every synonym for every row. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:search_configuration" +``` + +A SearchIndex resolves its configuration when the index is built, so set it up front. +Either point the index straight at a configuration with `search_configuration_id`, or +bind a configuration to the parent folder or project — an index with no +`search_configuration_id` of its own walks up the entity hierarchy and uses the first +[SearchConfigBinding][synapseclient.models.SearchConfigBinding] it finds, falling back +to the platform defaults. + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py:apply_search_configuration" +``` + +
+ Searching the abbreviation against the new index should look like: +``` +Created SearchIndex syn68123457 using config 4321 +Index syn68123457 is queryable with 6 rows +Bound configuration 4321 to syn12345678 +Abstracts matching the abbreviation 'AD': +total_hits=3, returned=3 + ROW_ID=1 {'study_name': 'ROSMAP Cortex Proteomics', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=2 {'study_name': 'MSBB RNA Sequencing', 'diagnosis': "Alzheimer's Disease"} + ROW_ID=3 {'study_name': 'Mayo Clinic Whole Genome', 'diagnosis': "Alzheimer's Disease"} +``` +
+ +## Source Code for this Tutorial + +
+ Click to show me + +```python +--8<-- "docs/tutorials/python/tutorial_scripts/search.py" +``` +
+ +## References +- [SearchIndex][synapseclient.models.SearchIndex] +- [SearchQuery][synapseclient.models.SearchQuery] +- [SearchQueryPart][synapseclient.models.SearchQueryPart] +- [SearchIndexQuery][synapseclient.models.SearchIndexQuery] +- [SearchHit][synapseclient.models.SearchHit] +- [Query][synapseclient.models.search_dsl.Query] +- [SearchConfiguration][synapseclient.models.SearchConfiguration] +- [TextAnalyzer][synapseclient.models.TextAnalyzer] +- [SynonymSet][synapseclient.models.SynonymSet] +- [ColumnAnalyzerOverride][synapseclient.models.ColumnAnalyzerOverride] +- [SearchConfigBinding][synapseclient.models.SearchConfigBinding] +- [Organization][synapseclient.models.Organization] +- [Table][synapseclient.models.Table] +- [syn.login][synapseclient.Synapse.login] +- [OpenSearch query DSL](https://docs.opensearch.org/latest/query-dsl/) +- [OpenSearch aggregations](https://docs.opensearch.org/latest/aggregations/) diff --git a/docs/tutorials/python/tutorial_scripts/search.py b/docs/tutorials/python/tutorial_scripts/search.py new file mode 100644 index 000000000..fc017ed21 --- /dev/null +++ b/docs/tutorials/python/tutorial_scripts/search.py @@ -0,0 +1,539 @@ +"""Here is where you'll find the code for the SearchIndex tutorial.""" + +# --8<-- [start:setup] +import json +import time + +import pandas as pd + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseError +from synapseclient.models import ( + Column, + ColumnType, + Project, + SearchIndex, + SearchIndexQuery, + SearchQuery, + SearchQueryPart, + Table, +) +from synapseclient.models.search_dsl import ( + Aggregation, + AvgAggregation, + BoolQuery, + Highlight, + HighlightField, + MatchBoolPrefixFieldOptions, + MatchFieldOptions, + MatchPhraseFieldOptions, + MultiMatchQuery, + Query, + RangeFieldOptions, + SourceFilter, + TermsAggregation, +) + +# Initialize Synapse client +syn = Synapse() +syn.login() + +# Get the project where we want to create the search index +project = Project(name="My uniquely named project about Alzheimer's Disease").get() +project_id = project.id +print(f"Got project with ID: {project_id}") + +# Create the table that will be indexed +table = Table( + name="Study Summaries", + parent_id=project_id, + columns=[ + Column(name="study_name", column_type=ColumnType.STRING), + Column(name="abstract", column_type=ColumnType.LARGETEXT), + Column(name="diagnosis", column_type=ColumnType.STRING), + Column(name="assay", column_type=ColumnType.STRING), + Column(name="participant_count", column_type=ColumnType.INTEGER), + ], +).store() +print(f"Created table with ID: {table.id}") + +# Add the rows we are going to search over +studies = pd.DataFrame( + [ + { + "study_name": "ROSMAP Cortex Proteomics", + "abstract": "Quantitative proteomics of dorsolateral prefrontal cortex " + "from donors with Alzheimer's disease and cognitively normal controls.", + "diagnosis": "Alzheimer's Disease", + "assay": "TMT quantitation", + "participant_count": 400, + }, + { + "study_name": "MSBB RNA Sequencing", + "abstract": "Bulk RNA sequencing across four brain regions in a cohort " + "spanning the full range of Alzheimer's disease neuropathology.", + "diagnosis": "Alzheimer's Disease", + "assay": "rnaSeq", + "participant_count": 300, + }, + { + "study_name": "Mayo Clinic Whole Genome", + "abstract": "Whole genome sequencing of temporal cortex samples from " + "donors with Alzheimer's disease, progressive supranuclear palsy, " + "and controls.", + "diagnosis": "Alzheimer's Disease", + "assay": "wholeGenomeSeq", + "participant_count": 350, + }, + { + "study_name": "Healthy Aging Single Cell Atlas", + "abstract": "Single nucleus RNA sequencing of hippocampus from " + "cognitively normal aged donors, establishing a baseline atlas.", + "diagnosis": "Cognitively Normal", + "assay": "snrnaSeq", + "participant_count": 120, + }, + { + "study_name": "MCI Plasma Biomarkers", + "abstract": "Plasma biomarker panel measuring phosphorylated tau and " + "neurofilament light chain in mild cognitive impairment.", + "diagnosis": "Mild Cognitive Impairment", + "assay": "immunoassay", + "participant_count": 220, + }, + { + "study_name": "Parkinson Comparative Cohort", + "abstract": "Comparative transcriptomic profiling of substantia nigra " + "in Parkinson disease versus age-matched controls.", + "diagnosis": "Parkinson's Disease", + "assay": "rnaSeq", + "participant_count": 180, + }, + ] +) +table.upsert_rows(values=studies, primary_keys=["study_name"]) +print(f"Stored {len(studies)} rows in {table.id}") +# --8<-- [end:setup] + + +# --8<-- [start:helpers] +def print_hits(results: SearchIndexQuery) -> None: + """Print the rows a query matched, one line per hit.""" + print(f"total_hits={results.total_hits}, returned={len(results.hits)}") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") + + +def wait_for_index(index: SearchIndex, timeout: int = 600) -> None: + """Wait until the search index has finished building. + + Building the OpenSearch index behind a SearchIndex happens in the background + after `store()` returns. Until that build completes, a query against the + index either raises an error or reports zero hits. + """ + deadline = time.time() + timeout + while time.time() < deadline: + try: + results = index.query( + search_query=SearchQuery(query=Query(match_all={}), size=1), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + if results.total_hits: + print(f"Index {index.id} is queryable with {results.total_hits} rows") + return + except SynapseError: + pass # The index has not been created yet + print("Waiting for the search index to build...") + time.sleep(10) + raise TimeoutError(f"{index.id} did not finish building within {timeout} seconds") + + +# --8<-- [end:helpers] + + +# --8<-- [start:create_index] +def create_search_index() -> SearchIndex: + """ + Example: Create a SearchIndex over a single table and wait for it to build. + """ + index = SearchIndex( + name="Study Summaries Search Index", + description="Full text search over the study summary table", + parent_id=project_id, + # The defining SQL must reference exactly one table-like entity + defining_sql=f"SELECT * FROM {table.id}", + ) + index = index.store() + print(f"Created SearchIndex with ID: {index.id}") + + wait_for_index(index) + return index + + +# --8<-- [end:create_index] + + +# --8<-- [start:full_text_search] +def search_free_text(index: SearchIndex) -> None: + """ + Example: Find every study whose abstract mentions Alzheimer's disease, then + search across several columns at once. + """ + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="alzheimer")}), + # Every indexed column comes back on each hit unless a source filter + # narrows them down, and the abstracts are long + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS, SearchQueryPart.SELECT_COLUMNS], + ) + print("Abstracts mentioning Alzheimer's:") + print(f"columns: {[column.name for column in results.select_columns]}") + print_hits(results) + + # A multi_match clause runs the same text across several columns, so the + # person searching does not need to know which column holds the term. + results = index.query( + search_query=SearchQuery( + query=Query( + multi_match=MultiMatchQuery( + query="tau", + # ^2 boosts a match in the study name over one in the abstract + fields=["study_name^2", "abstract"], + ) + ), + source=SourceFilter(includes=["study_name"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("\nAnything mentioning tau:") + print_hits(results) + + +# --8<-- [end:full_text_search] + + +# --8<-- [start:columns_and_highlights] +def search_with_highlighting(index: SearchIndex) -> None: + """ + Example: Ask for a snippet of the matching text alongside each hit, so a + result list can show why the row matched. + """ + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="sequencing")}), + source=SourceFilter(includes=["study_name", "assay"]), + highlight=Highlight( + fields={"abstract": HighlightField(number_of_fragments=1)} + ), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Studies that sequenced something:") + for hit in results.hits: + fields = {field.name: field.value for field in hit.fields} + print(f" ROW_ID={hit.row_id} {fields}") + for highlight in hit.highlights: + print(f" {highlight.name}: {highlight.snippets}") + + +# --8<-- [end:columns_and_highlights] + + +# --8<-- [start:filters_and_sorting] +def search_with_filters(index: SearchIndex) -> None: + """ + Example: Combine a scored clause with unscored filters using a bool query, + then order the results by a numeric column instead of by relevance. + """ + results = index.query( + search_query=SearchQuery( + query=Query( + bool=BoolQuery( + # Scored: how well the abstract matches drives relevance + must=[ + Query(match={"abstract": MatchFieldOptions(query="sequencing")}) + ], + # Unscored: a hard cutoff on cohort size + filter=[ + Query(range={"participant_count": RangeFieldOptions(gte=200)}) + ], + # Unscored: drop a diagnosis we are not interested in + must_not=[ + Query( + match_phrase={ + "diagnosis": MatchPhraseFieldOptions( + query="Parkinson's Disease" + ) + } + ) + ], + ) + ), + source=SourceFilter(includes=["study_name", "participant_count"]), + sort=[{"participant_count": "desc"}], + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Sequencing studies with at least 200 participants, largest first:") + print_hits(results) + + +# --8<-- [end:filters_and_sorting] + + +# --8<-- [start:aggregations] +def facet_the_results(index: SearchIndex) -> None: + """ + Example: Count how many studies fall under each diagnosis and average their + cohort sizes, while the hit list itself shows only one diagnosis. + """ + results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + aggregations={ + "by_diagnosis": Aggregation( + terms=TermsAggregation(field="diagnosis", size=10) + ), + "mean_cohort_size": Aggregation( + avg=AvgAggregation(field="participant_count") + ), + }, + # post_filter narrows the hits but not the aggregations, so the facet + # counts still show every option a person could pick next + post_filter=Query( + match_phrase={ + "diagnosis": MatchPhraseFieldOptions(query="Alzheimer's Disease") + } + ), + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Hits after the post filter:") + print_hits(results) + print("\nFacet counts across all studies:") + print(json.dumps(results.aggregation_results, indent=2)) + + +# --8<-- [end:aggregations] + + +# --8<-- [start:autocomplete] +def autocomplete_study_names(index: SearchIndex) -> None: + """ + Example: Back a type-ahead box with the autocomplete endpoint, which returns + its results directly instead of running as an asynchronous job. + """ + hits = index.autocomplete( + query=Query( + match_bool_prefix={ + "study_name": MatchBoolPrefixFieldOptions(query="Mayo Cl") + } + ), + source=SourceFilter(includes=["study_name"]), + ) + print("Suggestions for 'Mayo Cl':") + for hit in hits: + print(f" {[field.value for field in hit.fields]}") + + +# --8<-- [end:autocomplete] + + +# --8<-- [start:pagination] +def page_through_results(index: SearchIndex) -> None: + """ + Example: Walk every row in the index two hits at a time. + """ + page_size = 2 + offset = 0 + while True: + results = index.query( + search_query=SearchQuery( + query=Query(match_all={}), + source=SourceFilter(includes=["study_name", "participant_count"]), + sort=[{"participant_count": "desc"}], + from_=offset, + size=page_size, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print(f"Page starting at offset {offset}:") + print_hits(results) + + offset += page_size + if offset >= results.total_hits: + break + + +# --8<-- [end:pagination] + + +# --8<-- [start:search_configuration] +def create_search_configuration() -> str: + """ + Example: Teach the index that "AD" means "Alzheimer's disease" by building a + SynonymSet, wrapping it in a TextAnalyzer, and bundling that analyzer into a + SearchConfiguration. + + These resources belong to an Organization, and creating them is restricted + to Sage Bionetworks employees. None of them can be deleted once created. + """ + from synapseclient.models import ( + ColumnAnalyzerOverride, + ColumnAnalyzerOverrideEntry, + Organization, + SearchConfiguration, + SynonymSet, + TextAnalyzer, + ) + + organization_name = "my.uniquely.named.organization" + organization = Organization(name=organization_name).store() + print(f"Using organization: {organization.id} ({organization.name})") + + # Comma-separated entries are interchangeable in both directions; entries + # written with "=>" expand the left side to the right side only. + synonyms = SynonymSet( + organization_name=organization_name, + name="ad_synonyms", + description="Abbreviations used across Alzheimer's disease studies", + definition={ + "type": "synonym_graph", + "synonyms": [ + "rna sequencing, rna-seq, rnaseq", + "ad => alzheimer's disease, alzheimers disease", + "mci => mild cognitive impairment", + ], + }, + ).store() + print(f"Created SynonymSet: {synonyms.id} ({synonyms.qualified_name})") + + # The synonym filter is applied in `default_search` only, so synonyms expand + # the incoming query rather than bloating the stored index. + analyzer = TextAnalyzer( + organization_name=organization_name, + name="ad_synonym_analyzer", + description="English analyzer that expands AD abbreviations at search time", + settings={ + "filter": { + "english_stop": {"type": "stop", "stopwords": "_english_"}, + "english_stemmer": {"type": "stemmer", "language": "english"}, + # A $ref resolves to the SynonymSet by its qualified name + "ad_synonyms": {"$ref": synonyms.qualified_name}, + }, + "analyzer": { + "default": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase", "english_stop", "english_stemmer"], + }, + "default_search": { + "type": "custom", + "tokenizer": "standard", + "filter": [ + "lowercase", + "ad_synonyms", + "english_stop", + "english_stemmer", + ], + }, + }, + }, + ).store() + print(f"Created TextAnalyzer: {analyzer.id} ({analyzer.qualified_name})") + + # Columns not named here fall back to the configuration's default analyzer + overrides = ColumnAnalyzerOverride( + organization_name=organization_name, + name="study_column_overrides", + description="Treat the diagnosis column as a single exact value", + overrides=[ + ColumnAnalyzerOverrideEntry( + column_name="diagnosis", + analyzer={"analyzer": {"default": {"type": "keyword"}}}, + ), + ], + ).store() + print(f"Created ColumnAnalyzerOverride: {overrides.id}") + + configuration = SearchConfiguration( + organization_name=organization_name, + name="study_search_config", + description="Analyzer settings for the study summary search index", + default_analyzer={"$ref": analyzer.qualified_name}, + column_analyzer_overrides=[{"$ref": overrides.qualified_name}], + ).store() + print(f"Created SearchConfiguration: {configuration.id}") + return configuration.id + + +# --8<-- [end:search_configuration] + + +# --8<-- [start:apply_search_configuration] +def create_index_with_configuration(search_configuration_id: str) -> SearchIndex: + """ + Example: Build an index that uses a specific SearchConfiguration, and bind + the same configuration to the project so later indexes inherit it. + """ + from synapseclient.models import SearchConfigBinding + + index = SearchIndex( + name="Study Summaries Search Index With Synonyms", + parent_id=project_id, + defining_sql=f"SELECT * FROM {table.id}", + search_configuration_id=search_configuration_id, + ).store() + print(f"Created SearchIndex {index.id} using config {search_configuration_id}") + wait_for_index(index) + + # Any index created under this project without its own + # search_configuration_id now inherits this configuration + binding = SearchConfigBinding( + object_id=project_id, + search_configuration_id=search_configuration_id, + ).store() + print(f"Bound configuration {binding.search_configuration_id} to {project_id}") + + # "AD" now matches the abstracts that spell out "Alzheimer's disease" + results = index.query( + search_query=SearchQuery( + query=Query(match={"abstract": MatchFieldOptions(query="AD")}), + source=SourceFilter(includes=["study_name", "diagnosis"]), + size=10, + ), + response_parts=[SearchQueryPart.TOTAL_HITS], + ) + print("Abstracts matching the abbreviation 'AD':") + print_hits(results) + return index + + +# --8<-- [end:apply_search_configuration] + + +def main(): + index = create_search_index() + search_free_text(index) + search_with_highlighting(index) + search_with_filters(index) + facet_the_results(index) + autocomplete_study_names(index) + page_through_results(index) + + # Requires an Organization you can write to + # search_configuration_id = create_search_configuration() + # create_index_with_configuration(search_configuration_id) + + +if __name__ == "__main__": + main() diff --git a/mkdocs.yml b/mkdocs.yml index 60aab2d64..d759fa3b8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -39,6 +39,7 @@ nav: - Dataset: tutorials/python/dataset.md - Dataset Collection: tutorials/python/dataset_collection.md - Materialized View: tutorials/python/materializedview.md + - Search Index: tutorials/python/search.md - Submission View: tutorials/python/submissionview.md - Sharing Settings: tutorials/python/sharing_settings.md - Wiki: tutorials/python/wiki.md