diff --git a/pkg/cmd/search.go b/pkg/cmd/search.go index b6c21bb..b42216f 100644 --- a/pkg/cmd/search.go +++ b/pkg/cmd/search.go @@ -20,7 +20,10 @@ var ( maxPagesFlag int ) -const defaultMaxPages = 100 +const ( + defaultMaxPages = 100 + paginationRestrictorPath = "serpapi_pagination.next" +) var searchCmd = &cobra.Command{ Use: "search [PARAMS...]", @@ -74,6 +77,11 @@ func runSearch(cmd *cobra.Command, args []string) error { maxPages = maxPagesFlag } + paginationRestrictor := withPaginationRestrictor(paramsMap["json_restrictor"]) + if paginationRestrictor != "" { + paramsMap["json_restrictor"] = paginationRestrictor + } + client := api.New(apiKey) currentParams := paramsMap var accumulated map[string]any @@ -114,7 +122,7 @@ func runSearch(cmd *cobra.Command, args []string) error { break } - nextParams, err := parseNextParams(nextURL) + nextParams, err := parseNextParams(nextURL, paginationRestrictor) if err != nil { return err } @@ -142,6 +150,15 @@ func runSearch(cmd *cobra.Command, args []string) error { return handleOutput(json.RawMessage(bytes.TrimRight(buf.Bytes(), "\n"))) } +// withPaginationRestrictor adds the pagination field needed internally while +// preserving the caller's server-side field projection. +func withPaginationRestrictor(restrictor string) string { + if restrictor == "" { + return "" + } + return restrictor + "," + paginationRestrictorPath +} + // extractNextURL pulls the next pagination URL from a search result. func extractNextURL(result map[string]any) string { pag, ok := result["serpapi_pagination"] @@ -175,8 +192,9 @@ func mergeArrayFields(dst, src map[string]any) { } } -// parseNextParams extracts query parameters from a full URL. -func parseNextParams(nextURL string) (map[string]string, error) { +// parseNextParams extracts query parameters from a full URL and reapplies the +// internal restrictor because pagination URLs may omit response-shaping params. +func parseNextParams(nextURL, jsonRestrictor string) (map[string]string, error) { parsed, err := url.Parse(nextURL) if err != nil { return nil, &clierrors.NetworkError{Message: "Invalid pagination URL: " + err.Error()} @@ -187,6 +205,9 @@ func parseNextParams(nextURL string) (map[string]string, error) { result[k] = v[0] } } + if jsonRestrictor != "" { + result["json_restrictor"] = jsonRestrictor + } return result, nil } diff --git a/pkg/cmd/search_test.go b/pkg/cmd/search_test.go index c7459e5..258539d 100644 --- a/pkg/cmd/search_test.go +++ b/pkg/cmd/search_test.go @@ -20,8 +20,24 @@ func TestCanonicalParamsKeyIsOrderIndependent(t *testing.T) { } } +func TestWithPaginationRestrictor(t *testing.T) { + if got := withPaginationRestrictor(""); got != "" { + t.Errorf("expected empty restrictor, got %q", got) + } + + const fields = "organic_results[].{title,link}" + want := fields + ",serpapi_pagination.next" + if got := withPaginationRestrictor(fields); got != want { + t.Errorf("expected %q, got %q", want, got) + } +} + func TestParseNextParams(t *testing.T) { - params, err := parseNextParams("https://serpapi.com/search.json?q=coffee&start=10&engine=google") + const restrictor = "organic_results[].{title,link},serpapi_pagination.next" + params, err := parseNextParams( + "https://serpapi.com/search.json?q=coffee&start=10&engine=google", + restrictor, + ) if err != nil { t.Fatal(err) } @@ -34,4 +50,7 @@ func TestParseNextParams(t *testing.T) { if params["engine"] != "google" { t.Errorf("expected engine=google, got %s", params["engine"]) } + if params["json_restrictor"] != restrictor { + t.Errorf("expected json_restrictor=%q, got %q", restrictor, params["json_restrictor"]) + } } diff --git a/tests/e2e_test.go b/tests/e2e_test.go index bb99c68..ac8d910 100644 --- a/tests/e2e_test.go +++ b/tests/e2e_test.go @@ -161,6 +161,62 @@ func TestSearchAllPages(t *testing.T) { } } +func TestSearchAllPagesWithFields(t *testing.T) { + key := requireKey(t) + cmd := exec.Command( + binaryPath, + "--api-key", key, + "search", + "--fields", "organic_results[].{title,link},search_parameters.start", + "engine=google", + "q=coffee", + "--all-pages", + "--max-pages", "2", + ) + out, err := cmd.Output() + if err != nil { + t.Fatalf("search all pages with fields failed: %v", err) + } + + var result map[string]any + if err := json.Unmarshal(out, &result); err != nil { + t.Fatalf("failed to parse search output: %v", err) + } + for field := range result { + if field != "organic_results" && field != "search_parameters" { + t.Errorf("unexpected top-level field %q", field) + } + } + + searchParameters, ok := result["search_parameters"].(map[string]any) + if !ok { + t.Fatal("expected search_parameters object from the final page") + } + start, ok := searchParameters["start"].(float64) + if !ok || start <= 0 { + t.Fatalf("expected a positive final-page start offset, got %v", searchParameters["start"]) + } + + organicResults, ok := result["organic_results"].([]any) + if !ok { + t.Fatal("expected organic_results array") + } + if len(organicResults) < 2 { + t.Fatalf("expected results from multiple pages, got %d", len(organicResults)) + } + for i, item := range organicResults { + resultItem, ok := item.(map[string]any) + if !ok { + t.Fatalf("organic result %d is not an object", i) + } + for field := range resultItem { + if field != "title" && field != "link" { + t.Errorf("organic result %d contains unexpected field %q", i, field) + } + } + } +} + func TestInvalidAPIKey(t *testing.T) { cmd := exec.Command(binaryPath, "--api-key", "invalid", "search", "engine=google", "q=test") err := cmd.Run()