fix: pagination fix for `--fields and `--all-pages combination

pkg/cmd/search.go

@@ -20,7 +20,10 @@ var (
 	maxPagesFlag int
 )
 
-const defaultMaxPages = 100
+const (
+	defaultMaxPages          = 100
+	paginationRestrictorPath = "serpapi_pagination.next"
+)
 
 var searchCmd = &cobra.Command{
 	Use:   "search [PARAMS...]",
@@ -74,6 +77,11 @@ func runSearch(cmd *cobra.Command, args []string) error {
 		maxPages = maxPagesFlag
 	}
 
+	paginationRestrictor := withPaginationRestrictor(paramsMap["json_restrictor"])
+	if paginationRestrictor != "" {
+		paramsMap["json_restrictor"] = paginationRestrictor
+	}
+
 	client := api.New(apiKey)
 	currentParams := paramsMap
 	var accumulated map[string]any
@@ -114,7 +122,7 @@ func runSearch(cmd *cobra.Command, args []string) error {
 			break
 		}
 
-		nextParams, err := parseNextParams(nextURL)
+		nextParams, err := parseNextParams(nextURL, paginationRestrictor)
 		if err != nil {
 			return err
 		}
@@ -142,6 +150,15 @@ func runSearch(cmd *cobra.Command, args []string) error {
 	return handleOutput(json.RawMessage(bytes.TrimRight(buf.Bytes(), "\n")))
 }
 
+// withPaginationRestrictor adds the pagination field needed internally while
+// preserving the caller's server-side field projection.
+func withPaginationRestrictor(restrictor string) string {
+	if restrictor == "" {
+		return ""
+	}
+	return restrictor + "," + paginationRestrictorPath
+}
+
 // extractNextURL pulls the next pagination URL from a search result.
 func extractNextURL(result map[string]any) string {
 	pag, ok := result["serpapi_pagination"]
@@ -175,8 +192,9 @@ func mergeArrayFields(dst, src map[string]any) {
 	}
 }
 
-// parseNextParams extracts query parameters from a full URL.
-func parseNextParams(nextURL string) (map[string]string, error) {
+// parseNextParams extracts query parameters from a full URL and reapplies the
+// internal restrictor because pagination URLs may omit response-shaping params.
+func parseNextParams(nextURL, jsonRestrictor string) (map[string]string, error) {
 	parsed, err := url.Parse(nextURL)
 	if err != nil {
 		return nil, &clierrors.NetworkError{Message: "Invalid pagination URL: " + err.Error()}
@@ -187,6 +205,9 @@ func parseNextParams(nextURL string) (map[string]string, error) {
 			result[k] = v[0]
 		}
 	}
+	if jsonRestrictor != "" {
+		result["json_restrictor"] = jsonRestrictor
+	}
 	return result, nil
 }
 

pkg/cmd/search_test.go

@@ -20,8 +20,24 @@ func TestCanonicalParamsKeyIsOrderIndependent(t *testing.T) {
 	}
 }
 
+func TestWithPaginationRestrictor(t *testing.T) {
+	if got := withPaginationRestrictor(""); got != "" {
+		t.Errorf("expected empty restrictor, got %q", got)
+	}
+
+	const fields = "organic_results[].{title,link}"
+	want := fields + ",serpapi_pagination.next"
+	if got := withPaginationRestrictor(fields); got != want {
+		t.Errorf("expected %q, got %q", want, got)
+	}
+}
+
 func TestParseNextParams(t *testing.T) {
-	params, err := parseNextParams("https://serpapi.com/search.json?q=coffee&start=10&engine=google")
+	const restrictor = "organic_results[].{title,link},serpapi_pagination.next"
+	params, err := parseNextParams(
+		"https://serpapi.com/search.json?q=coffee&start=10&engine=google",
+		restrictor,
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -34,4 +50,7 @@ func TestParseNextParams(t *testing.T) {
 	if params["engine"] != "google" {
 		t.Errorf("expected engine=google, got %s", params["engine"])
 	}
+	if params["json_restrictor"] != restrictor {
+		t.Errorf("expected json_restrictor=%q, got %q", restrictor, params["json_restrictor"])
+	}
 }

tests/e2e_test.go

@@ -161,6 +161,62 @@ func TestSearchAllPages(t *testing.T) {
 	}
 }
 
+func TestSearchAllPagesWithFields(t *testing.T) {
+	key := requireKey(t)
+	cmd := exec.Command(
+		binaryPath,
+		"--api-key", key,
+		"search",
+		"--fields", "organic_results[].{title,link},search_parameters.start",
+		"engine=google",
+		"q=coffee",
+		"--all-pages",
+		"--max-pages", "2",
+	)
+	out, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("search all pages with fields failed: %v", err)
+	}
+
+	var result map[string]any
+	if err := json.Unmarshal(out, &result); err != nil {
+		t.Fatalf("failed to parse search output: %v", err)
+	}
+	for field := range result {
+		if field != "organic_results" && field != "search_parameters" {
+			t.Errorf("unexpected top-level field %q", field)
+		}
+	}
+
+	searchParameters, ok := result["search_parameters"].(map[string]any)
+	if !ok {
+		t.Fatal("expected search_parameters object from the final page")
+	}
+	start, ok := searchParameters["start"].(float64)
+	if !ok || start <= 0 {
+		t.Fatalf("expected a positive final-page start offset, got %v", searchParameters["start"])
+	}
+
+	organicResults, ok := result["organic_results"].([]any)
+	if !ok {
+		t.Fatal("expected organic_results array")
+	}
+	if len(organicResults) < 2 {
+		t.Fatalf("expected results from multiple pages, got %d", len(organicResults))
+	}
+	for i, item := range organicResults {
+		resultItem, ok := item.(map[string]any)
+		if !ok {
+			t.Fatalf("organic result %d is not an object", i)
+		}
+		for field := range resultItem {
+			if field != "title" && field != "link" {
+				t.Errorf("organic result %d contains unexpected field %q", i, field)
+			}
+		}
+	}
+}
+
 func TestInvalidAPIKey(t *testing.T) {
 	cmd := exec.Command(binaryPath, "--api-key", "invalid", "search", "engine=google", "q=test")
 	err := cmd.Run()