From d106c0093cfd1972db116abe25066fb96212947b Mon Sep 17 00:00:00 2001 From: artemo--brd Date: Thu, 30 Jul 2026 18:56:22 +0300 Subject: [PATCH] chore: fix ruff/black lint drift, pin explicit ruff select - ruff --fix: sort/organize imports across src/ and tests/ (3560 auto-fixes, all mechanical import reordering, no semantic changes) - pyproject.toml: pin tool.ruff.lint select to [E4, E7, E9, F, I] explicitly. Without an explicit select, an unpinned 'ruff>=0.1.0' silently picks up its current default rule set, which now includes plugin categories (datetimez, simplify, etc.) never intended by this repo's ignore list (which only ever suppressed E402/F841). This made 'ruff check' in CI fail with 3573 findings unrelated to any recent change. - black: reformat 32 files that had drifted from the pinned black==26.5.1 formatting (also pre-existing, reproduced identically on a clean main checkout before this branch's changes). Verified: ruff check and black --check both clean; full unit suite (321 tests) passes unchanged. --- pyproject.toml | 4 + src/brightdata/__init__.py | 86 +++--- src/brightdata/browser/service.py | 4 +- src/brightdata/cli_credentials.py | 3 +- src/brightdata/client.py | 101 +++---- src/brightdata/core/engine.py | 68 ++--- src/brightdata/core/zone_manager.py | 28 +- src/brightdata/crawler/__init__.py | 2 +- src/brightdata/crawler/models.py | 12 +- src/brightdata/crawler/service.py | 18 +- src/brightdata/datasets/__init__.py | 264 +++++++++--------- src/brightdata/datasets/agoda/properties.py | 4 +- src/brightdata/datasets/airbnb/properties.py | 4 +- src/brightdata/datasets/amazon/__init__.py | 12 +- .../datasets/amazon/best_sellers.py | 4 +- src/brightdata/datasets/amazon/products.py | 4 +- .../datasets/amazon/products_global.py | 4 +- .../datasets/amazon/products_search.py | 4 +- src/brightdata/datasets/amazon/reviews.py | 4 +- src/brightdata/datasets/amazon/sellers.py | 4 +- src/brightdata/datasets/amazon/walmart.py | 4 +- .../datasets/american_eagle/products.py | 4 +- .../datasets/apple_appstore/__init__.py | 2 +- .../datasets/apple_appstore/reviews.py | 4 +- .../datasets/apple_appstore/store.py | 4 +- .../datasets/ashley_furniture/products.py | 4 +- src/brightdata/datasets/asos/products.py | 4 +- src/brightdata/datasets/autozone/products.py | 4 +- .../datasets/balenciaga/products.py | 4 +- src/brightdata/datasets/base.py | 14 +- src/brightdata/datasets/bbc/news.py | 4 +- src/brightdata/datasets/berluti/products.py | 4 +- src/brightdata/datasets/bestbuy/products.py | 4 +- src/brightdata/datasets/bh/products.py | 4 +- src/brightdata/datasets/bluesky/posts.py | 4 +- .../datasets/bluesky/top_profiles.py | 4 +- src/brightdata/datasets/booking/__init__.py | 4 +- .../datasets/booking/hotel_listings.py | 4 +- .../datasets/booking/listings_search.py | 4 +- .../datasets/bottegaveneta/products.py | 4 +- src/brightdata/datasets/carsales/listings.py | 4 +- src/brightdata/datasets/carters/products.py | 4 +- src/brightdata/datasets/celine/products.py | 4 +- src/brightdata/datasets/chanel/products.py | 4 +- src/brightdata/datasets/chileautos/cars.py | 4 +- src/brightdata/datasets/client.py | 4 +- src/brightdata/datasets/client.pyi | 182 +----------- src/brightdata/datasets/cnn/news.py | 4 +- .../datasets/companies_enriched/companies.py | 30 +- src/brightdata/datasets/costco/products.py | 4 +- .../datasets/crateandbarrel/products.py | 4 +- .../datasets/creative_commons/__init__.py | 2 +- .../datasets/creative_commons/images.py | 4 +- .../datasets/creative_commons/models_3d.py | 4 +- .../datasets/crunchbase/companies.py | 4 +- src/brightdata/datasets/delvaux/products.py | 4 +- src/brightdata/datasets/digikey/products.py | 4 +- src/brightdata/datasets/dior/products.py | 4 +- src/brightdata/datasets/ebay/products.py | 4 +- .../datasets/employees_enriched/employees.py | 28 +- src/brightdata/datasets/etsy/products.py | 4 +- src/brightdata/datasets/facebook/__init__.py | 24 +- src/brightdata/datasets/facebook/comments.py | 4 +- .../datasets/facebook/company_reviews.py | 4 +- src/brightdata/datasets/facebook/events.py | 4 +- .../datasets/facebook/group_posts.py | 4 +- .../datasets/facebook/marketplace.py | 4 +- .../datasets/facebook/pages_posts.py | 4 +- .../datasets/facebook/pages_profiles.py | 4 +- .../datasets/facebook/posts_by_url.py | 4 +- src/brightdata/datasets/facebook/profiles.py | 4 +- src/brightdata/datasets/facebook/reels.py | 4 +- src/brightdata/datasets/fanatics/products.py | 4 +- src/brightdata/datasets/fendi/products.py | 4 +- src/brightdata/datasets/g2/products.py | 32 ++- src/brightdata/datasets/g2/reviews.py | 26 +- .../datasets/github/repositories.py | 4 +- src/brightdata/datasets/glassdoor/__init__.py | 4 +- .../datasets/glassdoor/companies.py | 34 +-- src/brightdata/datasets/glassdoor/jobs.py | 28 +- src/brightdata/datasets/glassdoor/reviews.py | 30 +- src/brightdata/datasets/goodreads/books.py | 4 +- .../datasets/google_maps/__init__.py | 4 +- .../datasets/google_maps/full_info.py | 4 +- .../datasets/google_maps/reviews.py | 26 +- src/brightdata/datasets/google_news/news.py | 4 +- .../datasets/google_play/__init__.py | 4 +- .../datasets/google_play/reviews.py | 4 +- src/brightdata/datasets/google_play/store.py | 4 +- .../datasets/google_shopping/products.py | 4 +- .../datasets/google_shopping/search_us.py | 4 +- src/brightdata/datasets/hermes/products.py | 4 +- src/brightdata/datasets/hm/products.py | 4 +- src/brightdata/datasets/homedepot/__init__.py | 2 +- .../datasets/homedepot/products_ca.py | 4 +- .../datasets/homedepot/products_us.py | 4 +- src/brightdata/datasets/ikea/products.py | 4 +- src/brightdata/datasets/imdb/movies.py | 4 +- src/brightdata/datasets/indeed/companies.py | 30 +- src/brightdata/datasets/indeed/jobs.py | 4 +- .../datasets/infocasas/properties.py | 4 +- .../datasets/inmuebles24/properties.py | 4 +- src/brightdata/datasets/instagram/__init__.py | 8 +- src/brightdata/datasets/instagram/comments.py | 4 +- src/brightdata/datasets/instagram/posts.py | 4 +- src/brightdata/datasets/instagram/profiles.py | 4 +- src/brightdata/datasets/instagram/reels.py | 4 +- src/brightdata/datasets/kroger/products.py | 4 +- src/brightdata/datasets/lawyers/us_lawyers.py | 30 +- src/brightdata/datasets/lazada/__init__.py | 4 +- src/brightdata/datasets/lazada/products.py | 4 +- .../datasets/lazada/products_search.py | 4 +- src/brightdata/datasets/lazada/reviews.py | 4 +- src/brightdata/datasets/lazboy/products.py | 4 +- src/brightdata/datasets/lego/products.py | 4 +- src/brightdata/datasets/linkedin/__init__.py | 4 +- .../datasets/linkedin/company_profiles.py | 4 +- .../datasets/linkedin/job_listings.py | 26 +- .../datasets/linkedin/people_profiles.py | 4 +- src/brightdata/datasets/linkedin/posts.py | 4 +- .../linkedin/profiles_job_listings.py | 4 +- src/brightdata/datasets/llbean/products.py | 4 +- src/brightdata/datasets/loewe/products.py | 4 +- src/brightdata/datasets/lowes/products.py | 4 +- src/brightdata/datasets/macys/products.py | 4 +- src/brightdata/datasets/mango/products.py | 4 +- src/brightdata/datasets/manta/businesses.py | 30 +- .../datasets/massimo_dutti/products.py | 4 +- .../datasets/mattressfirm/products.py | 4 +- .../datasets/mediamarkt/products.py | 4 +- .../datasets/mercadolivre/products.py | 4 +- .../datasets/metrocuadrado/properties.py | 4 +- .../datasets/microcenter/products.py | 4 +- src/brightdata/datasets/models.py | 22 +- src/brightdata/datasets/montblanc/products.py | 4 +- src/brightdata/datasets/mouser/products.py | 4 +- src/brightdata/datasets/moynat/products.py | 4 +- src/brightdata/datasets/mybobs/products.py | 4 +- src/brightdata/datasets/myntra/products.py | 4 +- src/brightdata/datasets/naver/products.py | 4 +- src/brightdata/datasets/nba/players_stats.py | 4 +- src/brightdata/datasets/olx/ads.py | 4 +- src/brightdata/datasets/otodom/properties.py | 4 +- src/brightdata/datasets/owler/companies.py | 30 +- src/brightdata/datasets/ozon/products.py | 4 +- src/brightdata/datasets/pinterest/posts.py | 4 +- src/brightdata/datasets/pinterest/profiles.py | 4 +- .../datasets/pitchbook/companies.py | 28 +- src/brightdata/datasets/prada/products.py | 4 +- .../datasets/properati/properties.py | 4 +- src/brightdata/datasets/quora/posts.py | 4 +- .../datasets/raymourflanigan/products.py | 4 +- .../datasets/real_estate/australia.py | 4 +- .../realtor/international_properties.py | 4 +- src/brightdata/datasets/reddit/__init__.py | 4 +- src/brightdata/datasets/reddit/comments.py | 4 +- src/brightdata/datasets/reddit/posts.py | 4 +- src/brightdata/datasets/rona/products.py | 4 +- src/brightdata/datasets/sephora/products.py | 4 +- src/brightdata/datasets/shein/products.py | 4 +- src/brightdata/datasets/shopee/products.py | 4 +- .../datasets/sleepnumber/products.py | 4 +- src/brightdata/datasets/slintel/companies.py | 26 +- src/brightdata/datasets/snapchat/posts.py | 4 +- src/brightdata/datasets/tiktok/__init__.py | 4 +- src/brightdata/datasets/tiktok/comments.py | 4 +- src/brightdata/datasets/tiktok/posts.py | 4 +- src/brightdata/datasets/tiktok/profiles.py | 4 +- src/brightdata/datasets/tiktok/shop.py | 4 +- src/brightdata/datasets/toctoc/properties.py | 4 +- src/brightdata/datasets/tokopedia/products.py | 4 +- src/brightdata/datasets/toysrus/products.py | 4 +- src/brightdata/datasets/trustpilot/reviews.py | 28 +- .../datasets/trustradius/reviews.py | 30 +- src/brightdata/datasets/utils.py | 25 +- .../datasets/ventureradar/companies.py | 28 +- src/brightdata/datasets/vimeo/videos.py | 4 +- src/brightdata/datasets/walmart/products.py | 4 +- src/brightdata/datasets/walmart/sellers.py | 4 +- src/brightdata/datasets/wayfair/products.py | 4 +- src/brightdata/datasets/webmotors/vehicles.py | 4 +- src/brightdata/datasets/wikipedia/articles.py | 4 +- .../datasets/wildberries/products.py | 4 +- .../datasets/world_population/countries.py | 4 +- .../datasets/world_zipcodes/zipcodes.py | 4 +- src/brightdata/datasets/x_twitter/posts.py | 4 +- src/brightdata/datasets/x_twitter/profiles.py | 4 +- src/brightdata/datasets/xing/profiles.py | 26 +- .../datasets/yahoo_finance/businesses.py | 4 +- src/brightdata/datasets/yapo/ads.py | 4 +- src/brightdata/datasets/yelp/businesses.py | 30 +- src/brightdata/datasets/yelp/reviews.py | 26 +- src/brightdata/datasets/youtube/__init__.py | 4 +- src/brightdata/datasets/youtube/comments.py | 4 +- src/brightdata/datasets/youtube/profiles.py | 4 +- src/brightdata/datasets/youtube/videos.py | 4 +- src/brightdata/datasets/ysl/products.py | 4 +- src/brightdata/datasets/zalando/products.py | 4 +- src/brightdata/datasets/zara/__init__.py | 4 +- src/brightdata/datasets/zara/home_products.py | 4 +- src/brightdata/datasets/zara/products.py | 4 +- src/brightdata/datasets/zillow/__init__.py | 4 +- .../datasets/zillow/price_history.py | 4 +- src/brightdata/datasets/zillow/properties.py | 4 +- .../datasets/zonaprop/properties.py | 4 +- src/brightdata/datasets/zoominfo/companies.py | 30 +- src/brightdata/datasets/zoopla/properties.py | 4 +- src/brightdata/discover/__init__.py | 4 +- src/brightdata/discover/models.py | 20 +- src/brightdata/discover/service.py | 48 ++-- src/brightdata/exceptions/__init__.py | 16 +- src/brightdata/exceptions/errors.py | 12 - src/brightdata/models.py | 78 +++--- src/brightdata/payloads.py | 115 ++++---- src/brightdata/scraper_studio/__init__.py | 4 +- src/brightdata/scraper_studio/client.py | 10 +- src/brightdata/scraper_studio/models.py | 18 +- src/brightdata/scraper_studio/service.py | 16 +- src/brightdata/scrapers/__init__.py | 22 +- src/brightdata/scrapers/amazon/schemas.py | 174 ++++++------ src/brightdata/scrapers/amazon/scraper.py | 76 ++--- src/brightdata/scrapers/amazon/search.py | 58 ++-- src/brightdata/scrapers/api_client.py | 14 +- src/brightdata/scrapers/base.py | 52 ++-- src/brightdata/scrapers/chatgpt/scraper.py | 66 +++-- src/brightdata/scrapers/chatgpt/search.py | 30 +- src/brightdata/scrapers/digikey/scraper.py | 32 +-- src/brightdata/scrapers/facebook/scraper.py | 170 +++++------ src/brightdata/scrapers/instagram/scraper.py | 66 ++--- src/brightdata/scrapers/instagram/search.py | 78 +++--- src/brightdata/scrapers/job.py | 14 +- src/brightdata/scrapers/linkedin/scraper.py | 64 ++--- src/brightdata/scrapers/linkedin/search.py | 98 +++---- src/brightdata/scrapers/perplexity/scraper.py | 58 ++-- src/brightdata/scrapers/pinterest/scraper.py | 40 +-- src/brightdata/scrapers/pinterest/search.py | 58 ++-- src/brightdata/scrapers/reddit/scraper.py | 100 +++---- src/brightdata/scrapers/registry.py | 12 +- src/brightdata/scrapers/tiktok/scraper.py | 108 ++++--- src/brightdata/scrapers/tiktok/search.py | 82 +++--- src/brightdata/scrapers/workflow.py | 23 +- src/brightdata/scrapers/x/scraper.py | 144 +++++----- src/brightdata/scrapers/youtube/scraper.py | 88 +++--- src/brightdata/scrapers/youtube/search.py | 144 +++++----- src/brightdata/serp/__init__.py | 8 +- src/brightdata/serp/base.py | 59 ++-- src/brightdata/serp/bing.py | 7 +- src/brightdata/serp/data_normalizer.py | 2 +- src/brightdata/serp/google.py | 7 +- src/brightdata/serp/service.py | 52 ++-- src/brightdata/serp/url_builder.py | 13 +- src/brightdata/serp/yandex.py | 7 +- src/brightdata/sync_client.py | 38 +-- src/brightdata/types.py | 25 +- src/brightdata/utils/function_detection.py | 3 +- src/brightdata/utils/location.py | 5 +- src/brightdata/utils/polling.py | 11 +- src/brightdata/utils/retry.py | 6 +- src/brightdata/utils/ssl_helpers.py | 4 +- src/brightdata/utils/url.py | 6 +- src/brightdata/utils/validation.py | 4 +- src/brightdata/web_unlocker/async_client.py | 11 +- src/brightdata/web_unlocker/base.py | 2 +- src/brightdata/web_unlocker/service.py | 62 ++-- tests/conftest.py | 6 +- tests/unit/test_async_unblocker.py | 7 +- tests/unit/test_cli_credentials.py | 2 +- tests/unit/test_client.py | 13 +- tests/unit/test_colorless_service_verbs.py | 12 +- tests/unit/test_models.py | 4 +- tests/unit/test_payloads.py | 20 +- tests/unit/test_retry.py | 3 +- tests/unit/test_scraper_core.py | 5 +- tests/unit/test_ssl_helpers.py | 3 +- tests/unit/test_sync_client_coverage.py | 14 +- tests/unit/test_x_scraper.py | 5 +- tests/unit/test_zone_manager.py | 6 +- 277 files changed, 2328 insertions(+), 2487 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0583e5b..037ad9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,10 @@ exclude = [ ] [tool.ruff.lint] +# Pyflakes + pycodestyle errors + isort (import sorting). Explicit, so newer +# ruff releases that broaden the "no select" default (e.g. datetimez, simplify, +# bugbear) don't silently start failing CI with unrelated findings. +select = ["E4", "E7", "E9", "F", "I"] ignore = [ "E402", # Module level import not at top of file (common in notebooks/scripts) "F841", # Local variable assigned but never used (common in tests) diff --git a/src/brightdata/__init__.py b/src/brightdata/__init__.py index 83ae8c0..0f5f7c7 100644 --- a/src/brightdata/__init__.py +++ b/src/brightdata/__init__.py @@ -1,6 +1,6 @@ """Bright Data Python SDK - Modern async-first SDK for Bright Data APIs.""" -from importlib.metadata import version, PackageNotFoundError +from importlib.metadata import PackageNotFoundError, version try: __version__ = version("brightdata-sdk") @@ -9,79 +9,79 @@ __version__ = "0.0.0.dev" # Export main client (async) +from .browser.service import BrowserService from .client import BrightDataClient +from .core.zone_manager import ZoneManager -# Export sync client adapter -from .sync_client import SyncBrightDataClient +# Export Discover API models +from .discover.models import DiscoverJob, DiscoverResult, DiscoverSnapshot + +# Export exceptions +from .exceptions import ( + APIError, + AuthenticationError, + BrightDataError, + NetworkError, + SSLError, + ValidationError, + ZoneError, +) # Export result models from .models import ( BaseResult, - ScrapeResult, - SearchResult, CrawlResult, Result, + ScrapeResult, + SearchResult, ) -# Export job model for manual trigger/poll/fetch -from .scrapers.job import ScrapeJob - -# Export Discover API models -from .discover.models import DiscoverResult, DiscoverJob, DiscoverSnapshot - # Export payload models (dataclasses) from .payloads import ( - # Base - BasePayload, - URLPayload, # Amazon AmazonProductPayload, AmazonReviewPayload, AmazonSellerPayload, - # LinkedIn - LinkedInProfilePayload, - LinkedInJobPayload, - LinkedInCompanyPayload, - LinkedInPostPayload, - LinkedInProfileSearchPayload, - LinkedInJobSearchPayload, - LinkedInPostSearchPayload, + # Base + BasePayload, # ChatGPT ChatGPTPromptPayload, + FacebookCommentsPayload, + FacebookPostPayload, + FacebookPostsGroupPayload, # Facebook FacebookPostsProfilePayload, - FacebookPostsGroupPayload, - FacebookPostPayload, - FacebookCommentsPayload, FacebookReelsPayload, + InstagramCommentPayload, + InstagramPostPayload, + InstagramPostsDiscoverPayload, # Instagram InstagramProfilePayload, - InstagramPostPayload, - InstagramCommentPayload, InstagramReelPayload, - InstagramPostsDiscoverPayload, InstagramReelsDiscoverPayload, -) - -# Export exceptions -from .exceptions import ( - BrightDataError, - ValidationError, - AuthenticationError, - APIError, - ZoneError, - NetworkError, - SSLError, + LinkedInCompanyPayload, + LinkedInJobPayload, + LinkedInJobSearchPayload, + LinkedInPostPayload, + LinkedInPostSearchPayload, + # LinkedIn + LinkedInProfilePayload, + LinkedInProfileSearchPayload, + URLPayload, ) # Export Scraper Studio models -from .scraper_studio.models import ScraperStudioJob, JobStatus +from .scraper_studio.models import JobStatus, ScraperStudioJob +from .scraper_studio.service import ScraperStudioService + +# Export job model for manual trigger/poll/fetch +from .scrapers.job import ScrapeJob + +# Export sync client adapter +from .sync_client import SyncBrightDataClient # Export services for advanced usage from .web_unlocker.service import WebUnlockerService -from .scraper_studio.service import ScraperStudioService -from .browser.service import BrowserService -from .core.zone_manager import ZoneManager __all__ = [ "__version__", diff --git a/src/brightdata/browser/service.py b/src/brightdata/browser/service.py index 97adf97..a7bd7ec 100644 --- a/src/brightdata/browser/service.py +++ b/src/brightdata/browser/service.py @@ -1,7 +1,5 @@ """Browser API service — builds CDP WebSocket URLs for Playwright/Puppeteer.""" -from typing import Optional - class BrowserService: """ @@ -43,7 +41,7 @@ def __init__( self._host = host self._port = port - def get_connect_url(self, country: Optional[str] = None) -> str: + def get_connect_url(self, country: str | None = None) -> str: """ Return the CDP WebSocket URL for connecting to a remote browser. diff --git a/src/brightdata/cli_credentials.py b/src/brightdata/cli_credentials.py index 8c9a847..10dbab0 100644 --- a/src/brightdata/cli_credentials.py +++ b/src/brightdata/cli_credentials.py @@ -18,7 +18,6 @@ import os import sys from pathlib import Path -from typing import Optional def _cli_credentials_path() -> Path: @@ -32,7 +31,7 @@ def _cli_credentials_path() -> Path: return base / "brightdata-cli" / "credentials.json" -def read_cli_credentials() -> Optional[str]: +def read_cli_credentials() -> str | None: """ Return the API key stored by `brightdata login`, or None if unavailable. diff --git a/src/brightdata/client.py b/src/brightdata/client.py index 3f3c1a8..bd4437b 100644 --- a/src/brightdata/client.py +++ b/src/brightdata/client.py @@ -8,11 +8,11 @@ - Follow principle of least surprise - common patterns from other SDKs """ -import os import asyncio +import os import warnings -from typing import Optional, Dict, Any, Union, List from datetime import datetime, timezone +from typing import Any try: from dotenv import load_dotenv @@ -21,22 +21,23 @@ except ImportError: pass +from http import HTTPStatus + +from .browser.service import BrowserService +from .cli_credentials import read_cli_credentials from .core.engine import AsyncEngine from .core.zone_manager import ZoneManager -from .web_unlocker.service import WebUnlockerService -from .scrapers.service import ScrapeService -from .serp.service import SearchService from .crawler.service import CrawlerService -from .scraper_studio.service import ScraperStudioService -from .browser.service import BrowserService -from .discover.service import DiscoverService -from .discover.models import DiscoverResult, DiscoverJob from .datasets import DatasetsClient +from .discover.models import DiscoverJob, DiscoverResult +from .discover.service import DiscoverService +from .exceptions import APIError, AuthenticationError, ValidationError from .models import ScrapeResult +from .scraper_studio.service import ScraperStudioService +from .scrapers.service import ScrapeService +from .serp.service import SearchService from .types import AccountInfo -from .cli_credentials import read_cli_credentials -from http import HTTPStatus -from .exceptions import ValidationError, AuthenticationError, APIError +from .web_unlocker.service import WebUnlockerService class BrightDataClient: @@ -75,20 +76,20 @@ class BrightDataClient: def __init__( self, - token: Optional[str] = None, + token: str | None = None, timeout: int = DEFAULT_TIMEOUT, - web_unlocker_zone: Optional[str] = None, - serp_zone: Optional[str] = None, - browser_username: Optional[str] = None, - browser_password: Optional[str] = None, - browser_host: Optional[str] = None, - browser_port: Optional[int] = None, + web_unlocker_zone: str | None = None, + serp_zone: str | None = None, + browser_username: str | None = None, + browser_password: str | None = None, + browser_host: str | None = None, + browser_port: int | None = None, auto_create_zones: bool = True, validate_token: bool = False, - rate_limit: Optional[float] = None, + rate_limit: float | None = None, rate_period: float = 1.0, ssl_verify: bool = True, - ssl_ca_cert: Optional[str] = None, + ssl_ca_cert: str | None = None, ): """ Initialize Bright Data client. @@ -153,17 +154,17 @@ def __init__( auth_source=self.auth_source, ) - self._scrape_service: Optional[ScrapeService] = None - self._search_service: Optional[SearchService] = None - self._crawler_service: Optional[CrawlerService] = None - self._web_unlocker_service: Optional[WebUnlockerService] = None - self._datasets_client: Optional[DatasetsClient] = None - self._scraper_studio_service: Optional[ScraperStudioService] = None - self._browser_service: Optional[BrowserService] = None - self._discover_service: Optional[DiscoverService] = None - self._zone_manager: Optional[ZoneManager] = None + self._scrape_service: ScrapeService | None = None + self._search_service: SearchService | None = None + self._crawler_service: CrawlerService | None = None + self._web_unlocker_service: WebUnlockerService | None = None + self._datasets_client: DatasetsClient | None = None + self._scraper_studio_service: ScraperStudioService | None = None + self._browser_service: BrowserService | None = None + self._discover_service: DiscoverService | None = None + self._zone_manager: ZoneManager | None = None self._is_connected = False - self._account_info: Optional[Dict[str, Any]] = None + self._account_info: dict[str, Any] | None = None self._zones_ensured = False # Store for validation during __aenter__ @@ -182,7 +183,7 @@ def _ensure_initialized(self) -> None: "Use: async with BrightDataClient() as client: ..." ) - def _load_token(self, token: Optional[str]) -> tuple: + def _load_token(self, token: str | None) -> tuple: """ Resolve the API token and record where it came from. @@ -542,9 +543,9 @@ async def get_account_info(self, refresh: bool = False) -> AccountInfo: except (AuthenticationError, APIError): raise except Exception as e: - raise APIError(f"Unexpected error getting account info: {str(e)}") + raise APIError(f"Unexpected error getting account info: {e!s}") - async def list_zones(self) -> List[Dict[str, Any]]: + async def list_zones(self) -> list[dict[str, Any]]: """ List all active zones in your Bright Data account. @@ -597,16 +598,16 @@ async def delete_zone(self, zone_name: str) -> None: async def scrape_url( self, - url: Union[str, List[str]], - zone: Optional[str] = None, + url: str | list[str], + zone: str | None = None, country: str = "", response_format: str = "raw", method: str = "GET", - timeout: Optional[int] = None, + timeout: int | None = None, mode: str = "sync", poll_interval: int = 2, poll_timeout: int = 30, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Direct scraping method (flat API). @@ -644,13 +645,13 @@ async def scrape_url( async def discover( self, query: str, - intent: Optional[str] = None, + intent: str | None = None, include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, + country: str | None = None, + city: str | None = None, + language: str | None = None, + filter_keywords: list[str] | None = None, + num_results: int | None = None, format: str = "json", timeout: int = 60, poll_interval: int = 2, @@ -708,13 +709,13 @@ async def discover( async def discover_trigger( self, query: str, - intent: Optional[str] = None, + intent: str | None = None, include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, + country: str | None = None, + city: str | None = None, + language: str | None = None, + filter_keywords: list[str] | None = None, + num_results: int | None = None, format: str = "json", ) -> DiscoverJob: """ diff --git a/src/brightdata/core/engine.py b/src/brightdata/core/engine.py index 32490fa..97c8d2b 100644 --- a/src/brightdata/core/engine.py +++ b/src/brightdata/core/engine.py @@ -1,14 +1,16 @@ """Async HTTP engine for Bright Data API operations.""" import asyncio -import aiohttp import ssl import warnings -from typing import Optional, Dict, Any +from http import HTTPStatus +from typing import Any + +import aiohttp + from .. import __version__ from ..exceptions import AuthenticationError, NetworkError, SSLError -from http import HTTPStatus -from ..utils.ssl_helpers import is_ssl_certificate_error, get_ssl_error_message +from ..utils.ssl_helpers import get_ssl_error_message, is_ssl_certificate_error # Rate limiting support try: @@ -48,11 +50,11 @@ def __init__( self, bearer_token: str, timeout: int = 30, - rate_limit: Optional[float] = None, + rate_limit: float | None = None, rate_period: float = 1.0, ssl_verify: bool = True, - ssl_ca_cert: Optional[str] = None, - auth_source: Optional[str] = None, + ssl_ca_cert: str | None = None, + auth_source: str | None = None, ): """ Initialize async engine. @@ -74,7 +76,7 @@ def __init__( """ self.bearer_token = bearer_token self.timeout = aiohttp.ClientTimeout(total=timeout) - self._session: Optional[aiohttp.ClientSession] = None + self._session: aiohttp.ClientSession | None = None self._ssl_verify = ssl_verify self._ssl_ca_cert = ssl_ca_cert self._auth_source = auth_source @@ -85,7 +87,7 @@ def __init__( self._rate_limit = rate_limit self._rate_period = rate_period - self._rate_limiter: Optional[AsyncLimiter] = None + self._rate_limiter: AsyncLimiter | None = None async def __aenter__(self): """Context manager entry - idempotent (safe to call multiple times).""" @@ -179,9 +181,9 @@ def request( self, method: str, endpoint: str, - json_data: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + json_data: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ): """ Make an async HTTP request. @@ -226,9 +228,9 @@ def request( def post( self, endpoint: str, - json_data: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + json_data: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ): """Make POST request. Returns context manager.""" return self.request("POST", endpoint, json_data=json_data, params=params, headers=headers) @@ -236,8 +238,8 @@ def post( def get( self, endpoint: str, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ): """Make GET request. Returns context manager.""" return self.request("GET", endpoint, params=params, headers=headers) @@ -245,9 +247,9 @@ def get( def delete( self, endpoint: str, - json_data: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, + json_data: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, ): """Make DELETE request. Returns context manager.""" return self.request("DELETE", endpoint, json_data=json_data, params=params, headers=headers) @@ -255,10 +257,10 @@ def delete( def post_to_url( self, url: str, - json_data: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, - timeout: Optional[aiohttp.ClientTimeout] = None, + json_data: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + timeout: aiohttp.ClientTimeout | None = None, ): """ Make POST request to arbitrary URL. @@ -304,9 +306,9 @@ def post_to_url( def get_from_url( self, url: str, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, - timeout: Optional[aiohttp.ClientTimeout] = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + timeout: aiohttp.ClientTimeout | None = None, ): """ Make GET request to arbitrary URL. @@ -351,11 +353,11 @@ def _make_request( self, method: str, url: str, - json_data: Optional[Dict[str, Any]] = None, - params: Optional[Dict[str, Any]] = None, - headers: Optional[Dict[str, str]] = None, - timeout: Optional[aiohttp.ClientTimeout] = None, - rate_limiter: Optional[Any] = None, + json_data: dict[str, Any] | None = None, + params: dict[str, Any] | None = None, + headers: dict[str, str] | None = None, + timeout: aiohttp.ClientTimeout | None = None, + rate_limiter: Any | None = None, ): """ Internal method to make HTTP request with error handling. @@ -436,7 +438,7 @@ async def __aenter__(self): error_message = get_ssl_error_message(e) raise SSLError(error_message) from e # Other network errors - raise NetworkError(f"Network error: {str(e)}") from e + raise NetworkError(f"Network error: {e!s}") from e async def __aexit__(self, exc_type, exc_val, exc_tb): if self._response: diff --git a/src/brightdata/core/zone_manager.py b/src/brightdata/core/zone_manager.py index b0a7b6f..f514cf4 100644 --- a/src/brightdata/core/zone_manager.py +++ b/src/brightdata/core/zone_manager.py @@ -5,10 +5,12 @@ import asyncio import logging -import aiohttp -from typing import List, Dict, Any, Optional, Tuple from http import HTTPStatus -from ..exceptions.errors import ZoneError, APIError, AuthenticationError +from typing import Any + +import aiohttp + +from ..exceptions.errors import APIError, AuthenticationError, ZoneError logger = logging.getLogger(__name__) @@ -33,7 +35,7 @@ def __init__(self, engine): async def ensure_required_zones( self, web_unlocker_zone: str, - serp_zone: Optional[str] = None, + serp_zone: str | None = None, skip_verification: bool = False, ) -> None: """ @@ -54,7 +56,7 @@ async def ensure_required_zones( zone_names = {zone.get("name") for zone in zones} logger.info(f"Found {len(zones)} existing zones") - zones_to_create: List[Tuple[str, str]] = [] + zones_to_create: list[tuple[str, str]] = [] # Check web unlocker zone if web_unlocker_zone not in zone_names: @@ -114,9 +116,9 @@ async def ensure_required_zones( raise except Exception as e: logger.error(f"Unexpected error while ensuring zones exist: {e}") - raise ZoneError(f"Unexpected error during zone creation: {str(e)}") + raise ZoneError(f"Unexpected error during zone creation: {e!s}") - async def _get_zones(self) -> List[Dict[str, Any]]: + async def _get_zones(self) -> list[dict[str, Any]]: """ Get list of all active zones. @@ -163,7 +165,7 @@ async def _get_zones(self) -> List[Dict[str, Any]]: ) await asyncio.sleep(retry_delay * (1.5**attempt)) continue - raise ZoneError(f"Failed to get zones: {str(e)}") + raise ZoneError(f"Failed to get zones: {e!s}") raise ZoneError("Failed to get zones after all retry attempts") @@ -275,11 +277,11 @@ async def _create_zone(self, zone_name: str, zone_type: str) -> None: ) await asyncio.sleep(retry_delay * (1.5**attempt)) continue - raise ZoneError(f"Failed to create zone '{zone_name}': {str(e)}") + raise ZoneError(f"Failed to create zone '{zone_name}': {e!s}") raise ZoneError(f"Failed to create zone '{zone_name}' after all retry attempts") - async def _verify_zones_created(self, zone_names: List[str]) -> None: + async def _verify_zones_created(self, zone_names: list[str]) -> None: """ Verify that zones were successfully created by checking the zones list. @@ -335,7 +337,7 @@ async def _verify_zones_created(self, zone_names: List[str]) -> None: logger.warning(f"Zone verification attempt {attempt + 1} failed, retrying...") await asyncio.sleep(base_delay * (1.5**attempt)) - async def list_zones(self) -> List[Dict[str, Any]]: + async def list_zones(self) -> list[dict[str, Any]]: """ List all active zones in your Bright Data account. @@ -357,7 +359,7 @@ async def list_zones(self) -> List[Dict[str, Any]]: raise except Exception as e: logger.error(f"Unexpected error listing zones: {e}") - raise ZoneError(f"Unexpected error while listing zones: {str(e)}") + raise ZoneError(f"Unexpected error while listing zones: {e!s}") async def delete_zone(self, zone_name: str) -> None: """ @@ -438,6 +440,6 @@ async def delete_zone(self, zone_name: str) -> None: ) await asyncio.sleep(retry_delay * (1.5**attempt)) continue - raise ZoneError(f"Failed to delete zone '{zone_name}': {str(e)}") + raise ZoneError(f"Failed to delete zone '{zone_name}': {e!s}") raise ZoneError(f"Failed to delete zone '{zone_name}' after all retry attempts") diff --git a/src/brightdata/crawler/__init__.py b/src/brightdata/crawler/__init__.py index 7b5018a..5aaf1c1 100644 --- a/src/brightdata/crawler/__init__.py +++ b/src/brightdata/crawler/__init__.py @@ -3,4 +3,4 @@ from .models import CrawlJob, CrawlResult from .service import CrawlerService -__all__ = ["CrawlerService", "CrawlResult", "CrawlJob"] +__all__ = ["CrawlJob", "CrawlResult", "CrawlerService"] diff --git a/src/brightdata/crawler/models.py b/src/brightdata/crawler/models.py index 54d72e2..3e6c48b 100644 --- a/src/brightdata/crawler/models.py +++ b/src/brightdata/crawler/models.py @@ -2,7 +2,7 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any @dataclass @@ -16,12 +16,12 @@ class CrawlResult: """ success: bool - data: List[Dict[str, Any]] = field(default_factory=list) + data: list[dict[str, Any]] = field(default_factory=list) page_count: int = 0 - snapshot_id: Optional[str] = None # None on sync path, set on async path - trigger_sent_at: Optional[datetime] = None - data_fetched_at: Optional[datetime] = None - error: Optional[str] = None + snapshot_id: str | None = None # None on sync path, set on async path + trigger_sent_at: datetime | None = None + data_fetched_at: datetime | None = None + error: str | None = None def __repr__(self) -> str: sid = f" snapshot_id={self.snapshot_id}" if self.snapshot_id else "" diff --git a/src/brightdata/crawler/service.py b/src/brightdata/crawler/service.py index 50d1f44..3e10bc0 100644 --- a/src/brightdata/crawler/service.py +++ b/src/brightdata/crawler/service.py @@ -19,12 +19,12 @@ import json from datetime import datetime, timezone from http import HTTPStatus -from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any -from .models import CrawlJob, CrawlResult from ..exceptions import APIError, ValidationError from ..utils.function_detection import get_caller_function_name from ..utils.validation import validate_url, validate_url_list +from .models import CrawlJob, CrawlResult if TYPE_CHECKING: from ..client import BrightDataClient @@ -68,7 +68,7 @@ def __init__(self, client: "BrightDataClient"): async def crawl( self, - urls: Union[str, List[str]], + urls: str | list[str], include_errors: bool = True, ) -> CrawlResult: """ @@ -106,7 +106,7 @@ async def crawl( async def trigger( self, - urls: Union[str, List[str]], + urls: str | list[str], include_errors: bool = True, ) -> CrawlJob: """ @@ -256,9 +256,9 @@ async def download( async def _scrape_sync( self, - urls: List[str], + urls: list[str], include_errors: bool, - sdk_function: Optional[str], + sdk_function: str | None, ) -> CrawlResult: """POST /scrape and return the inline result wrapped in CrawlResult.""" trigger_sent_at = datetime.now(timezone.utc) @@ -350,7 +350,7 @@ async def _fetch_snapshot( ) @staticmethod - def _parse_records(text: str) -> List[Dict[str, Any]]: + def _parse_records(text: str) -> list[dict[str, Any]]: """ Parse the response body into a list of crawl records. @@ -365,7 +365,7 @@ def _parse_records(text: str) -> List[Dict[str, Any]]: try: parsed = json.loads(text) except json.JSONDecodeError: - records: List[Dict[str, Any]] = [] + records: list[dict[str, Any]] = [] for line in text.splitlines(): line = line.strip() if not line: @@ -387,7 +387,7 @@ def _parse_records(text: str) -> List[Dict[str, Any]]: return [] @staticmethod - def _normalize_urls(urls: Union[str, List[str]]) -> List[str]: + def _normalize_urls(urls: str | list[str]) -> list[str]: """Validate and normalize input to a non-empty list of URLs.""" if isinstance(urls, str): validate_url(urls) diff --git a/src/brightdata/datasets/__init__.py b/src/brightdata/datasets/__init__.py index af5e6c2..1c66abf 100644 --- a/src/brightdata/datasets/__init__.py +++ b/src/brightdata/datasets/__init__.py @@ -4,163 +4,163 @@ Access pre-collected datasets and filter records. """ -from .client import DatasetsClient -from .base import BaseDataset, DatasetError -from .models import DatasetInfo, DatasetField, DatasetMetadata, SnapshotStatus -from .utils import export, export_json, export_jsonl, export_csv - -# Platform-specific datasets -from .linkedin import ( - LinkedInPeopleProfiles, - LinkedInCompanyProfiles, - LinkedInJobListings, - LinkedInPosts, - LinkedInProfilesJobListings, -) +from .agoda import AgodaProperties +from .airbnb import AirbnbProperties from .amazon import ( + AmazonBestSellers, AmazonProducts, + AmazonProductsGlobal, + AmazonProductsSearch, AmazonReviews, AmazonSellersInfo, - AmazonBestSellers, - AmazonProductsSearch, - AmazonProductsGlobal, AmazonWalmart, ) -from .crunchbase import CrunchbaseCompanies -from .imdb import IMDBMovies -from .nba import NBAPlayersStats -from .goodreads import GoodreadsBooks -from .world_population import WorldPopulation -from .companies_enriched import CompaniesEnriched -from .employees_enriched import EmployeesEnriched -from .glassdoor import GlassdoorCompanies, GlassdoorReviews, GlassdoorJobs -from .google_maps import GoogleMapsReviews, GoogleMapsFullInfo -from .yelp import YelpBusinesses, YelpReviews -from .zoominfo import ZoomInfoCompanies -from .pitchbook import PitchBookCompanies -from .g2 import G2Products, G2Reviews -from .trustpilot import TrustpilotReviews -from .indeed import IndeedCompanies, IndeedJobs -from .xing import XingProfiles -from .slintel import SlintelCompanies -from .owler import OwlerCompanies -from .lawyers import USLawyers -from .manta import MantaBusinesses -from .ventureradar import VentureRadarCompanies -from .trustradius import TrustRadiusReviews -from .instagram import InstagramProfiles, InstagramPosts, InstagramComments, InstagramReels -from .tiktok import TikTokProfiles, TikTokComments, TikTokPosts, TikTokShop -from .real_estate import AustraliaRealEstate -from .walmart import WalmartProducts, WalmartSellersInfo -from .mediamarkt import MediamarktProducts -from .fendi import FendiProducts -from .zalando import ZalandoProducts -from .sephora import SephoraProducts -from .zara import ZaraProducts, ZaraHomeProducts -from .mango import MangoProducts -from .massimo_dutti import MassimoDuttiProducts -from .otodom import OtodomPoland -from .webmotors import WebmotorsBrasil -from .airbnb import AirbnbProperties -from .asos import AsosProducts -from .chanel import ChanelProducts -from .ashley_furniture import AshleyFurnitureProducts -from .fanatics import FanaticsProducts -from .carters import CartersProducts from .american_eagle import AmericanEagleProducts -from .ikea import IkeaProducts -from .hm import HMProducts -from .lego import LegoProducts -from .mattressfirm import MattressfirmProducts -from .crateandbarrel import CrateAndBarrelProducts -from .llbean import LLBeanProducts -from .shein import SheinProducts -from .toysrus import ToysRUsProducts -from .mybobs import MybobsProducts -from .sleepnumber import SleepNumberProducts -from .raymourflanigan import RaymourFlaniganProducts -from .inmuebles24 import Inmuebles24Mexico -from .mouser import MouserProducts -from .zillow import ZillowProperties, ZillowPriceHistory -from .zonaprop import ZonapropArgentina -from .metrocuadrado import MetrocuadradoProperties -from .chileautos import ChileautosChile -from .infocasas import InfocasasUruguay -from .lazboy import LaZBoyProducts -from .properati import ProperatiProperties -from .yapo import YapoChile -from .toctoc import ToctocProperties -from .dior import DiorProducts +from .apple_appstore import AppleAppStore, AppleAppStoreReviews +from .ashley_furniture import AshleyFurnitureProducts +from .asos import AsosProducts +from .autozone import AutozoneProducts from .balenciaga import BalenciagaProducts +from .base import BaseDataset, DatasetError +from .bbc import BBCNews +from .berluti import BerlutiProducts +from .bestbuy import BestBuyProducts +from .bh import BHProducts +from .bluesky import BlueskyPosts, BlueskyTopProfiles +from .booking import BookingHotelListings, BookingListingsSearch from .bottegaveneta import BottegaVenetaProducts -from .olx import OLXBrazil +from .carsales import CarsalesListings +from .carters import CartersProducts from .celine import CelineProducts -from .loewe import LoeweProducts -from .berluti import BerlutiProducts -from .moynat import MoynatProducts -from .hermes import HermesProducts +from .chanel import ChanelProducts +from .chileautos import ChileautosChile +from .client import DatasetsClient +from .cnn import CNNNews +from .companies_enriched import CompaniesEnriched +from .costco import CostcoProducts +from .crateandbarrel import CrateAndBarrelProducts +from .creative_commons import CreativeCommons3DModels, CreativeCommonsImages +from .crunchbase import CrunchbaseCompanies from .delvaux import DelvauxProducts -from .prada import PradaProducts -from .montblanc import MontblancProducts -from .ysl import YSLProducts -from .world_zipcodes import WorldZipcodes -from .pinterest import PinterestPosts, PinterestProfiles -from .shopee import ShopeeProducts -from .lazada import LazadaProducts, LazadaReviews, LazadaProductsSearch -from .youtube import YouTubeProfiles, YouTubeVideos, YouTubeComments from .digikey import DigikeyProducts +from .dior import DiorProducts +from .ebay import EbayProducts +from .employees_enriched import EmployeesEnriched +from .etsy import EtsyProducts from .facebook import ( - FacebookPagesPosts, FacebookComments, - FacebookPostsByUrl, - FacebookReels, - FacebookMarketplace, FacebookCompanyReviews, FacebookEvents, - FacebookProfiles, - FacebookPagesProfiles, FacebookGroupPosts, + FacebookMarketplace, + FacebookPagesPosts, + FacebookPagesProfiles, + FacebookPostsByUrl, + FacebookProfiles, + FacebookReels, ) -from .x_twitter import XTwitterPosts, XTwitterProfiles -from .reddit import RedditPosts, RedditComments -from .bluesky import BlueskyPosts, BlueskyTopProfiles -from .snapchat import SnapchatPosts -from .quora import QuoraPosts -from .vimeo import VimeoVideos -from .google_news import GoogleNews -from .wikipedia import WikipediaArticles -from .bbc import BBCNews -from .cnn import CNNNews +from .fanatics import FanaticsProducts +from .fendi import FendiProducts +from .g2 import G2Products, G2Reviews from .github import GithubRepositories -from .creative_commons import CreativeCommonsImages, CreativeCommons3DModels -from .google_play import GooglePlayStore, GooglePlayReviews -from .apple_appstore import AppleAppStore, AppleAppStoreReviews -from .ebay import EbayProducts -from .etsy import EtsyProducts -from .wayfair import WayfairProducts -from .bestbuy import BestBuyProducts -from .myntra import MyntraProducts -from .ozon import OzonProducts -from .wildberries import WildberriesProducts -from .tokopedia import TokopediaProducts +from .glassdoor import GlassdoorCompanies, GlassdoorJobs, GlassdoorReviews +from .goodreads import GoodreadsBooks +from .google_maps import GoogleMapsFullInfo, GoogleMapsReviews +from .google_news import GoogleNews +from .google_play import GooglePlayReviews, GooglePlayStore from .google_shopping import GoogleShoppingProducts, GoogleShoppingSearchUS -from .mercadolivre import MercadolivreProducts -from .naver import NaverProducts -from .homedepot import HomeDepotUSProducts, HomeDepotCAProducts -from .lowes import LowesProducts -from .rona import RonaProducts +from .hermes import HermesProducts +from .hm import HMProducts +from .homedepot import HomeDepotCAProducts, HomeDepotUSProducts +from .ikea import IkeaProducts +from .imdb import IMDBMovies +from .indeed import IndeedCompanies, IndeedJobs +from .infocasas import InfocasasUruguay +from .inmuebles24 import Inmuebles24Mexico +from .instagram import InstagramComments, InstagramPosts, InstagramProfiles, InstagramReels from .kroger import KrogerProducts +from .lawyers import USLawyers +from .lazada import LazadaProducts, LazadaProductsSearch, LazadaReviews +from .lazboy import LaZBoyProducts +from .lego import LegoProducts + +# Platform-specific datasets +from .linkedin import ( + LinkedInCompanyProfiles, + LinkedInJobListings, + LinkedInPeopleProfiles, + LinkedInPosts, + LinkedInProfilesJobListings, +) +from .llbean import LLBeanProducts +from .loewe import LoeweProducts +from .lowes import LowesProducts from .macys import MacysProducts -from .costco import CostcoProducts -from .bh import BHProducts +from .mango import MangoProducts +from .manta import MantaBusinesses +from .massimo_dutti import MassimoDuttiProducts +from .mattressfirm import MattressfirmProducts +from .mediamarkt import MediamarktProducts +from .mercadolivre import MercadolivreProducts +from .metrocuadrado import MetrocuadradoProperties from .microcenter import MicroCenterProducts -from .autozone import AutozoneProducts -from .zoopla import ZooplaProperties -from .booking import BookingListingsSearch, BookingHotelListings +from .models import DatasetField, DatasetInfo, DatasetMetadata, SnapshotStatus +from .montblanc import MontblancProducts +from .mouser import MouserProducts +from .moynat import MoynatProducts +from .mybobs import MybobsProducts +from .myntra import MyntraProducts +from .naver import NaverProducts +from .nba import NBAPlayersStats +from .olx import OLXBrazil +from .otodom import OtodomPoland +from .owler import OwlerCompanies +from .ozon import OzonProducts +from .pinterest import PinterestPosts, PinterestProfiles +from .pitchbook import PitchBookCompanies +from .prada import PradaProducts +from .properati import ProperatiProperties +from .quora import QuoraPosts +from .raymourflanigan import RaymourFlaniganProducts +from .real_estate import AustraliaRealEstate from .realtor import RealtorInternationalProperties -from .agoda import AgodaProperties -from .carsales import CarsalesListings +from .reddit import RedditComments, RedditPosts +from .rona import RonaProducts +from .sephora import SephoraProducts +from .shein import SheinProducts +from .shopee import ShopeeProducts +from .sleepnumber import SleepNumberProducts +from .slintel import SlintelCompanies +from .snapchat import SnapchatPosts +from .tiktok import TikTokComments, TikTokPosts, TikTokProfiles, TikTokShop +from .toctoc import ToctocProperties +from .tokopedia import TokopediaProducts +from .toysrus import ToysRUsProducts +from .trustpilot import TrustpilotReviews +from .trustradius import TrustRadiusReviews +from .utils import export, export_csv, export_json, export_jsonl +from .ventureradar import VentureRadarCompanies +from .vimeo import VimeoVideos +from .walmart import WalmartProducts, WalmartSellersInfo +from .wayfair import WayfairProducts +from .webmotors import WebmotorsBrasil +from .wikipedia import WikipediaArticles +from .wildberries import WildberriesProducts +from .world_population import WorldPopulation +from .world_zipcodes import WorldZipcodes +from .x_twitter import XTwitterPosts, XTwitterProfiles +from .xing import XingProfiles from .yahoo_finance import YahooFinanceBusinesses +from .yapo import YapoChile +from .yelp import YelpBusinesses, YelpReviews +from .youtube import YouTubeComments, YouTubeProfiles, YouTubeVideos +from .ysl import YSLProducts +from .zalando import ZalandoProducts +from .zara import ZaraHomeProducts, ZaraProducts +from .zillow import ZillowPriceHistory, ZillowProperties +from .zonaprop import ZonapropArgentina +from .zoominfo import ZoomInfoCompanies +from .zoopla import ZooplaProperties __all__ = [ # Client diff --git a/src/brightdata/datasets/agoda/properties.py b/src/brightdata/datasets/agoda/properties.py index 5d66e37..1b58f56 100644 --- a/src/brightdata/datasets/agoda/properties.py +++ b/src/brightdata/datasets/agoda/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AgodaProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/airbnb/properties.py b/src/brightdata/datasets/airbnb/properties.py index a03a561..9ee5d33 100644 --- a/src/brightdata/datasets/airbnb/properties.py +++ b/src/brightdata/datasets/airbnb/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AirbnbProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/amazon/__init__.py b/src/brightdata/datasets/amazon/__init__.py index 54efa19..095d338 100644 --- a/src/brightdata/datasets/amazon/__init__.py +++ b/src/brightdata/datasets/amazon/__init__.py @@ -1,19 +1,19 @@ """Amazon datasets.""" +from .best_sellers import AmazonBestSellers from .products import AmazonProducts +from .products_global import AmazonProductsGlobal +from .products_search import AmazonProductsSearch from .reviews import AmazonReviews from .sellers import AmazonSellersInfo -from .best_sellers import AmazonBestSellers -from .products_search import AmazonProductsSearch -from .products_global import AmazonProductsGlobal from .walmart import AmazonWalmart __all__ = [ + "AmazonBestSellers", "AmazonProducts", + "AmazonProductsGlobal", + "AmazonProductsSearch", "AmazonReviews", "AmazonSellersInfo", - "AmazonBestSellers", - "AmazonProductsSearch", - "AmazonProductsGlobal", "AmazonWalmart", ] diff --git a/src/brightdata/datasets/amazon/best_sellers.py b/src/brightdata/datasets/amazon/best_sellers.py index 7840898..3f41bf5 100644 --- a/src/brightdata/datasets/amazon/best_sellers.py +++ b/src/brightdata/datasets/amazon/best_sellers.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmazonBestSellers(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/amazon/products.py b/src/brightdata/datasets/amazon/products.py index 058bddc..9fef02f 100644 --- a/src/brightdata/datasets/amazon/products.py +++ b/src/brightdata/datasets/amazon/products.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -35,7 +35,7 @@ class AmazonProducts(BaseDataset): # All available fields with metadata # Format: field_name -> {"type": str, "description": str} - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { # Core product identification "title": { "type": "text", diff --git a/src/brightdata/datasets/amazon/products_global.py b/src/brightdata/datasets/amazon/products_global.py index ad889d6..764ccea 100644 --- a/src/brightdata/datasets/amazon/products_global.py +++ b/src/brightdata/datasets/amazon/products_global.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmazonProductsGlobal(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/amazon/products_search.py b/src/brightdata/datasets/amazon/products_search.py index fa3be9f..72784ca 100644 --- a/src/brightdata/datasets/amazon/products_search.py +++ b/src/brightdata/datasets/amazon/products_search.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmazonProductsSearch(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/amazon/reviews.py b/src/brightdata/datasets/amazon/reviews.py index 5d7ea24..3b53257 100644 --- a/src/brightdata/datasets/amazon/reviews.py +++ b/src/brightdata/datasets/amazon/reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmazonReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/amazon/sellers.py b/src/brightdata/datasets/amazon/sellers.py index 7e44c68..7e3bd9e 100644 --- a/src/brightdata/datasets/amazon/sellers.py +++ b/src/brightdata/datasets/amazon/sellers.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmazonSellersInfo(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/amazon/walmart.py b/src/brightdata/datasets/amazon/walmart.py index ddfee5b..0b6c682 100644 --- a/src/brightdata/datasets/amazon/walmart.py +++ b/src/brightdata/datasets/amazon/walmart.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmazonWalmart(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/american_eagle/products.py b/src/brightdata/datasets/american_eagle/products.py index 15808ff..ae5ceda 100644 --- a/src/brightdata/datasets/american_eagle/products.py +++ b/src/brightdata/datasets/american_eagle/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AmericanEagleProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/apple_appstore/__init__.py b/src/brightdata/datasets/apple_appstore/__init__.py index 804a553..dce6b93 100644 --- a/src/brightdata/datasets/apple_appstore/__init__.py +++ b/src/brightdata/datasets/apple_appstore/__init__.py @@ -1,6 +1,6 @@ """Apple App Store datasets.""" -from .store import AppleAppStore from .reviews import AppleAppStoreReviews +from .store import AppleAppStore __all__ = ["AppleAppStore", "AppleAppStoreReviews"] diff --git a/src/brightdata/datasets/apple_appstore/reviews.py b/src/brightdata/datasets/apple_appstore/reviews.py index 0373901..d4ccf22 100644 --- a/src/brightdata/datasets/apple_appstore/reviews.py +++ b/src/brightdata/datasets/apple_appstore/reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AppleAppStoreReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/apple_appstore/store.py b/src/brightdata/datasets/apple_appstore/store.py index 653a961..fbe295c 100644 --- a/src/brightdata/datasets/apple_appstore/store.py +++ b/src/brightdata/datasets/apple_appstore/store.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AppleAppStore(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/ashley_furniture/products.py b/src/brightdata/datasets/ashley_furniture/products.py index b944567..3efa107 100644 --- a/src/brightdata/datasets/ashley_furniture/products.py +++ b/src/brightdata/datasets/ashley_furniture/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AshleyFurnitureProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/asos/products.py b/src/brightdata/datasets/asos/products.py index 05c6ae0..2f67499 100644 --- a/src/brightdata/datasets/asos/products.py +++ b/src/brightdata/datasets/asos/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AsosProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/autozone/products.py b/src/brightdata/datasets/autozone/products.py index 87ed189..8f8da7a 100644 --- a/src/brightdata/datasets/autozone/products.py +++ b/src/brightdata/datasets/autozone/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class AutozoneProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/balenciaga/products.py b/src/brightdata/datasets/balenciaga/products.py index 4224d5d..93f42e1 100644 --- a/src/brightdata/datasets/balenciaga/products.py +++ b/src/brightdata/datasets/balenciaga/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BalenciagaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/base.py b/src/brightdata/datasets/base.py index 1a913f2..67b3d60 100644 --- a/src/brightdata/datasets/base.py +++ b/src/brightdata/datasets/base.py @@ -4,7 +4,7 @@ import asyncio import time -from typing import Dict, List, Any, Optional, Literal, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Literal from .models import DatasetMetadata, SnapshotStatus @@ -15,8 +15,6 @@ class DatasetError(Exception): """Error related to dataset operations.""" - pass - class BaseDataset: """ @@ -33,7 +31,7 @@ class BaseDataset: def __init__(self, engine: "AsyncEngine"): self._engine = engine - self._metadata: Optional[DatasetMetadata] = None + self._metadata: DatasetMetadata | None = None @property def dataset_id(self) -> str: @@ -63,8 +61,8 @@ async def get_metadata(self) -> DatasetMetadata: async def __call__( self, - filter: Dict[str, Any], - records_limit: Optional[int] = None, + filter: dict[str, Any], + records_limit: int | None = None, ) -> str: """ Filter dataset records and create a snapshot. @@ -88,7 +86,7 @@ async def __call__( Returns: snapshot_id (str) - use with download() to get data """ - payload: Dict[str, Any] = { + payload: dict[str, Any] = { "dataset_id": self.DATASET_ID, "filter": filter, } @@ -154,7 +152,7 @@ async def download( format: Literal["json", "jsonl", "csv"] = "jsonl", timeout: int = 300, poll_interval: int = 5, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: """ Download snapshot data. diff --git a/src/brightdata/datasets/bbc/news.py b/src/brightdata/datasets/bbc/news.py index 0ea7527..154cd65 100644 --- a/src/brightdata/datasets/bbc/news.py +++ b/src/brightdata/datasets/bbc/news.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BBCNews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/berluti/products.py b/src/brightdata/datasets/berluti/products.py index daee0f7..b7e96cf 100644 --- a/src/brightdata/datasets/berluti/products.py +++ b/src/brightdata/datasets/berluti/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BerlutiProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/bestbuy/products.py b/src/brightdata/datasets/bestbuy/products.py index ffe30bc..759a44e 100644 --- a/src/brightdata/datasets/bestbuy/products.py +++ b/src/brightdata/datasets/bestbuy/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BestBuyProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/bh/products.py b/src/brightdata/datasets/bh/products.py index 7effc5b..06107d7 100644 --- a/src/brightdata/datasets/bh/products.py +++ b/src/brightdata/datasets/bh/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BHProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/bluesky/posts.py b/src/brightdata/datasets/bluesky/posts.py index a4454b9..ebf2f8f 100644 --- a/src/brightdata/datasets/bluesky/posts.py +++ b/src/brightdata/datasets/bluesky/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BlueskyPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/bluesky/top_profiles.py b/src/brightdata/datasets/bluesky/top_profiles.py index 3b8dbe9..5836172 100644 --- a/src/brightdata/datasets/bluesky/top_profiles.py +++ b/src/brightdata/datasets/bluesky/top_profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BlueskyTopProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/booking/__init__.py b/src/brightdata/datasets/booking/__init__.py index e003247..f9f53f5 100644 --- a/src/brightdata/datasets/booking/__init__.py +++ b/src/brightdata/datasets/booking/__init__.py @@ -1,6 +1,6 @@ """Booking.com datasets.""" -from .listings_search import BookingListingsSearch from .hotel_listings import BookingHotelListings +from .listings_search import BookingListingsSearch -__all__ = ["BookingListingsSearch", "BookingHotelListings"] +__all__ = ["BookingHotelListings", "BookingListingsSearch"] diff --git a/src/brightdata/datasets/booking/hotel_listings.py b/src/brightdata/datasets/booking/hotel_listings.py index b00260c..e723525 100644 --- a/src/brightdata/datasets/booking/hotel_listings.py +++ b/src/brightdata/datasets/booking/hotel_listings.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BookingHotelListings(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/booking/listings_search.py b/src/brightdata/datasets/booking/listings_search.py index cbc4125..f22449c 100644 --- a/src/brightdata/datasets/booking/listings_search.py +++ b/src/brightdata/datasets/booking/listings_search.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BookingListingsSearch(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/bottegaveneta/products.py b/src/brightdata/datasets/bottegaveneta/products.py index 66b1282..e6f60ec 100644 --- a/src/brightdata/datasets/bottegaveneta/products.py +++ b/src/brightdata/datasets/bottegaveneta/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class BottegaVenetaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/carsales/listings.py b/src/brightdata/datasets/carsales/listings.py index 95fbba9..431898b 100644 --- a/src/brightdata/datasets/carsales/listings.py +++ b/src/brightdata/datasets/carsales/listings.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CarsalesListings(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/carters/products.py b/src/brightdata/datasets/carters/products.py index 4741307..1eec8a5 100644 --- a/src/brightdata/datasets/carters/products.py +++ b/src/brightdata/datasets/carters/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CartersProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/celine/products.py b/src/brightdata/datasets/celine/products.py index a65109c..e543c71 100644 --- a/src/brightdata/datasets/celine/products.py +++ b/src/brightdata/datasets/celine/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CelineProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/chanel/products.py b/src/brightdata/datasets/chanel/products.py index 5684ff2..fa8f877 100644 --- a/src/brightdata/datasets/chanel/products.py +++ b/src/brightdata/datasets/chanel/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ChanelProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/chileautos/cars.py b/src/brightdata/datasets/chileautos/cars.py index 1cc3749..775ae71 100644 --- a/src/brightdata/datasets/chileautos/cars.py +++ b/src/brightdata/datasets/chileautos/cars.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ChileautosChile(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/client.py b/src/brightdata/datasets/client.py index 72df47b..8c7bb43 100644 --- a/src/brightdata/datasets/client.py +++ b/src/brightdata/datasets/client.py @@ -6,7 +6,7 @@ """ import importlib -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING from .models import DatasetInfo @@ -262,7 +262,7 @@ def __getattr__(self, name: str): f"Available datasets: {', '.join(sorted(_DATASET_REGISTRY))}" ) - async def list(self) -> List[DatasetInfo]: + async def list(self) -> list[DatasetInfo]: """ List all available datasets. diff --git a/src/brightdata/datasets/client.pyi b/src/brightdata/datasets/client.pyi index 4ff743d..dfc82bd 100644 --- a/src/brightdata/datasets/client.pyi +++ b/src/brightdata/datasets/client.pyi @@ -1,9 +1,8 @@ """Type stub for DatasetsClient — provides IDE autocomplete.""" -from typing import List -from .models import DatasetInfo -from ..core.engine import AsyncEngine +import builtins +from ..core.engine import AsyncEngine from .agoda import AgodaProperties from .airbnb import AirbnbProperties from .amazon import ( @@ -143,6 +142,7 @@ from .mediamarkt import MediamarktProducts from .mercadolivre import MercadolivreProducts from .metrocuadrado import MetrocuadradoProperties from .microcenter import MicroCenterProducts +from .models import DatasetInfo from .montblanc import MontblancProducts from .mouser import MouserProducts from .moynat import MoynatProducts @@ -233,873 +233,699 @@ class DatasetsClient: BASE_URL: str def __init__(self, engine: AsyncEngine) -> None: ... - async def list(self) -> List[DatasetInfo]: ... + async def list(self) -> builtins.list[DatasetInfo]: ... @property def linkedin_profiles(self) -> LinkedInPeopleProfiles: """LinkedIn People Profiles dataset (620M+ records).""" - ... @property def linkedin_companies(self) -> LinkedInCompanyProfiles: """LinkedIn Company Profiles dataset.""" - ... @property def linkedin_job_listings(self) -> LinkedInJobListings: """LinkedIn Profiles Jobs Listings dataset.""" - ... @property def amazon_products(self) -> AmazonProducts: """Amazon Products dataset.""" - ... @property def amazon_reviews(self) -> AmazonReviews: """Amazon Reviews dataset.""" - ... @property def crunchbase_companies(self) -> CrunchbaseCompanies: """Crunchbase Companies dataset (2.3M+ records).""" - ... @property def imdb_movies(self) -> IMDBMovies: """IMDB Movies dataset (867K+ records).""" - ... @property def nba_players_stats(self) -> NBAPlayersStats: """NBA Players Stats dataset (17K+ records).""" - ... @property def goodreads_books(self) -> GoodreadsBooks: """Goodreads Books dataset.""" - ... @property def world_population(self) -> WorldPopulation: """World Population dataset.""" - ... @property def companies_enriched(self) -> CompaniesEnriched: """Companies Enriched dataset - multi-source company information.""" - ... @property def employees_enriched(self) -> EmployeesEnriched: """Employees Business Enriched dataset - LinkedIn profiles with company data.""" - ... @property def glassdoor_companies(self) -> GlassdoorCompanies: """Glassdoor Companies Overview dataset - ratings, reviews, and company details.""" - ... @property def glassdoor_reviews(self) -> GlassdoorReviews: """Glassdoor Companies Reviews dataset - employee reviews and ratings.""" - ... @property def glassdoor_jobs(self) -> GlassdoorJobs: """Glassdoor Job Listings dataset - job postings with company data.""" - ... @property def google_maps_reviews(self) -> GoogleMapsReviews: """Google Maps Reviews dataset - place reviews and ratings.""" - ... @property def yelp_businesses(self) -> YelpBusinesses: """Yelp Businesses Overview dataset - business listings and ratings.""" - ... @property def yelp_reviews(self) -> YelpReviews: """Yelp Business Reviews dataset - individual business reviews.""" - ... @property def zoominfo_companies(self) -> ZoomInfoCompanies: """ZoomInfo Companies dataset - company data with financials and contacts.""" - ... @property def pitchbook_companies(self) -> PitchBookCompanies: """PitchBook Companies dataset - PE/VC company data with deals.""" - ... @property def g2_products(self) -> G2Products: """G2 Software Product Overview dataset - software ratings and reviews.""" - ... @property def g2_reviews(self) -> G2Reviews: """G2 Software Product Reviews dataset - individual product reviews.""" - ... @property def trustpilot_reviews(self) -> TrustpilotReviews: """Trustpilot Business Reviews dataset - company reviews and ratings.""" - ... @property def indeed_companies(self) -> IndeedCompanies: """Indeed Companies Info dataset - company profiles with jobs and reviews.""" - ... @property def xing_profiles(self) -> XingProfiles: """Xing Social Network Profiles dataset - professional profiles.""" - ... @property def slintel_companies(self) -> SlintelCompanies: """Slintel 6sense Company Information dataset - technographics and company data.""" - ... @property def owler_companies(self) -> OwlerCompanies: """Owler Companies Information dataset - competitive intelligence and metrics.""" - ... @property def us_lawyers(self) -> USLawyers: """US Lawyers Directory dataset - lawyer profiles and practice areas.""" - ... @property def manta_businesses(self) -> MantaBusinesses: """Manta Businesses dataset - business listings with revenue and employees.""" - ... @property def ventureradar_companies(self) -> VentureRadarCompanies: """VentureRadar Company Information dataset - startup intelligence.""" - ... @property def trustradius_reviews(self) -> TrustRadiusReviews: """TrustRadius Product Reviews dataset - software product reviews.""" - ... @property def instagram_profiles(self) -> InstagramProfiles: """Instagram Profiles dataset - user profiles and engagement.""" - ... @property def tiktok_profiles(self) -> TikTokProfiles: """TikTok Profiles dataset - user profiles and engagement.""" - ... @property def australia_real_estate(self) -> AustraliaRealEstate: """Australia Real Estate Properties dataset.""" - ... @property def indeed_jobs(self) -> IndeedJobs: """Indeed Job Listings dataset.""" - ... @property def walmart_products(self) -> WalmartProducts: """Walmart Products dataset.""" - ... @property def mediamarkt_products(self) -> MediamarktProducts: """Mediamarkt.de Products dataset.""" - ... @property def fendi_products(self) -> FendiProducts: """Fendi Products dataset.""" - ... @property def zalando_products(self) -> ZalandoProducts: """Zalando Products dataset.""" - ... @property def sephora_products(self) -> SephoraProducts: """Sephora Products dataset.""" - ... @property def zara_products(self) -> ZaraProducts: """Zara Products dataset.""" - ... @property def zara_home_products(self) -> ZaraHomeProducts: """Zara Home Products dataset.""" - ... @property def mango_products(self) -> MangoProducts: """Mango Products dataset.""" - ... @property def massimo_dutti_products(self) -> MassimoDuttiProducts: """Massimo Dutti Products dataset.""" - ... @property def otodom_poland(self) -> OtodomPoland: """Otodom Poland real estate dataset.""" - ... @property def webmotors_brasil(self) -> WebmotorsBrasil: """Webmotors Brasil vehicle listings dataset.""" - ... @property def airbnb_properties(self) -> AirbnbProperties: """Airbnb Properties dataset.""" - ... @property def asos_products(self) -> AsosProducts: """Asos Products dataset.""" - ... @property def chanel_products(self) -> ChanelProducts: """Chanel Products dataset.""" - ... @property def ashley_furniture_products(self) -> AshleyFurnitureProducts: """Ashley Furniture Products dataset.""" - ... @property def fanatics_products(self) -> FanaticsProducts: """Fanatics Products dataset.""" - ... @property def carters_products(self) -> CartersProducts: """Carters Products dataset.""" - ... @property def american_eagle_products(self) -> AmericanEagleProducts: """American Eagle Products dataset.""" - ... @property def ikea_products(self) -> IkeaProducts: """Ikea Products dataset.""" - ... @property def hm_products(self) -> HMProducts: """H&M Products dataset.""" - ... @property def lego_products(self) -> LegoProducts: """Lego Products dataset.""" - ... @property def mattressfirm_products(self) -> MattressfirmProducts: """Mattressfirm Products dataset.""" - ... @property def crateandbarrel_products(self) -> CrateAndBarrelProducts: """Crate and Barrel Products dataset.""" - ... @property def llbean_products(self) -> LLBeanProducts: """L.L. Bean Products dataset.""" - ... @property def shein_products(self) -> SheinProducts: """Shein Products dataset.""" - ... @property def toysrus_products(self) -> ToysRUsProducts: """Toys R Us Products dataset.""" - ... @property def mybobs_products(self) -> MybobsProducts: """Mybobs Products dataset.""" - ... @property def sleepnumber_products(self) -> SleepNumberProducts: """Sleep Number Products dataset.""" - ... @property def raymourflanigan_products(self) -> RaymourFlaniganProducts: """Raymour and Flanigan Products dataset.""" - ... @property def inmuebles24_mexico(self) -> Inmuebles24Mexico: """Inmuebles24 Mexico real estate dataset.""" - ... @property def mouser_products(self) -> MouserProducts: """Mouser Products dataset.""" - ... @property def zillow_properties(self) -> ZillowProperties: """Zillow Properties dataset.""" - ... @property def zonaprop_argentina(self) -> ZonapropArgentina: """Zonaprop Argentina real estate dataset.""" - ... @property def metrocuadrado_properties(self) -> MetrocuadradoProperties: """Metrocuadrado Properties dataset.""" - ... @property def chileautos_chile(self) -> ChileautosChile: """Chileautos Chile car listings dataset.""" - ... @property def infocasas_uruguay(self) -> InfocasasUruguay: """Infocasas Uruguay real estate dataset.""" - ... @property def lazboy_products(self) -> LaZBoyProducts: """La-Z-Boy Products dataset.""" - ... @property def properati_properties(self) -> ProperatiProperties: """Properati Properties dataset.""" - ... @property def yapo_chile(self) -> YapoChile: """Yapo Chile marketplace ads dataset.""" - ... @property def toctoc_properties(self) -> ToctocProperties: """Toctoc Properties dataset.""" - ... @property def dior_products(self) -> DiorProducts: """Dior Products dataset.""" - ... @property def balenciaga_products(self) -> BalenciagaProducts: """Balenciaga Products dataset.""" - ... @property def bottegaveneta_products(self) -> BottegaVenetaProducts: """Bottega Veneta Products dataset.""" - ... @property def olx_brazil(self) -> OLXBrazil: """OLX Brazil marketplace ads dataset.""" - ... @property def celine_products(self) -> CelineProducts: """Celine Products dataset.""" - ... @property def loewe_products(self) -> LoeweProducts: """Loewe Products dataset.""" - ... @property def berluti_products(self) -> BerlutiProducts: """Berluti Products dataset.""" - ... @property def moynat_products(self) -> MoynatProducts: """Moynat Products dataset.""" - ... @property def hermes_products(self) -> HermesProducts: """Hermes Products dataset.""" - ... @property def delvaux_products(self) -> DelvauxProducts: """Delvaux Products dataset.""" - ... @property def prada_products(self) -> PradaProducts: """Prada Products dataset.""" - ... @property def montblanc_products(self) -> MontblancProducts: """Montblanc Products dataset.""" - ... @property def ysl_products(self) -> YSLProducts: """YSL Products dataset.""" - ... @property def amazon_sellers_info(self) -> AmazonSellersInfo: """Amazon Sellers Info dataset.""" - ... @property def world_zipcodes(self) -> WorldZipcodes: """World Zipcodes dataset.""" - ... @property def pinterest_posts(self) -> PinterestPosts: """Pinterest Posts dataset.""" - ... @property def pinterest_profiles(self) -> PinterestProfiles: """Pinterest Profiles dataset.""" - ... @property def shopee_products(self) -> ShopeeProducts: """Shopee Products dataset.""" - ... @property def lazada_products(self) -> LazadaProducts: """Lazada Products dataset.""" - ... @property def instagram_posts(self) -> InstagramPosts: """Instagram Posts dataset.""" - ... @property def youtube_profiles(self) -> YouTubeProfiles: """YouTube Profiles dataset.""" - ... @property def youtube_videos(self) -> YouTubeVideos: """YouTube Videos dataset.""" - ... @property def youtube_comments(self) -> YouTubeComments: """YouTube Comments dataset.""" - ... @property def digikey_products(self) -> DigikeyProducts: """Digikey Products dataset.""" - ... @property def facebook_pages_posts(self) -> FacebookPagesPosts: """Facebook Pages Posts dataset.""" - ... @property def facebook_comments(self) -> FacebookComments: """Facebook Comments dataset.""" - ... @property def facebook_posts_by_url(self) -> FacebookPostsByUrl: """Facebook Posts by URL dataset.""" - ... @property def facebook_reels(self) -> FacebookReels: """Facebook Reels dataset.""" - ... @property def facebook_marketplace(self) -> FacebookMarketplace: """Facebook Marketplace dataset.""" - ... @property def facebook_company_reviews(self) -> FacebookCompanyReviews: """Facebook Company Reviews dataset.""" - ... @property def facebook_events(self) -> FacebookEvents: """Facebook Events dataset.""" - ... @property def facebook_profiles(self) -> FacebookProfiles: """Facebook Profiles dataset.""" - ... @property def facebook_pages_profiles(self) -> FacebookPagesProfiles: """Facebook Pages and Profiles dataset.""" - ... @property def facebook_group_posts(self) -> FacebookGroupPosts: """Facebook Group Posts dataset.""" - ... @property def tiktok_comments(self) -> TikTokComments: """TikTok Comments dataset.""" - ... @property def tiktok_posts(self) -> TikTokPosts: """TikTok Posts dataset.""" - ... @property def tiktok_shop(self) -> TikTokShop: """TikTok Shop dataset.""" - ... @property def instagram_comments(self) -> InstagramComments: """Instagram Comments dataset.""" - ... @property def instagram_reels(self) -> InstagramReels: """Instagram Reels dataset.""" - ... @property def linkedin_posts(self) -> LinkedInPosts: """LinkedIn Posts dataset.""" - ... @property def linkedin_profiles_job_listings(self) -> LinkedInProfilesJobListings: """LinkedIn Profiles Job Listings dataset.""" - ... @property def x_twitter_posts(self) -> XTwitterPosts: """X (Twitter) Posts dataset.""" - ... @property def x_twitter_profiles(self) -> XTwitterProfiles: """X (Twitter) Profiles dataset.""" - ... @property def reddit_posts(self) -> RedditPosts: """Reddit Posts dataset.""" - ... @property def reddit_comments(self) -> RedditComments: """Reddit Comments dataset.""" - ... @property def bluesky_posts(self) -> BlueskyPosts: """Bluesky Posts dataset.""" - ... @property def bluesky_top_profiles(self) -> BlueskyTopProfiles: """Top 500 Bluesky Profiles dataset.""" - ... @property def snapchat_posts(self) -> SnapchatPosts: """Snapchat Posts dataset.""" - ... @property def quora_posts(self) -> QuoraPosts: """Quora Posts dataset.""" - ... @property def vimeo_videos(self) -> VimeoVideos: """Vimeo Videos dataset.""" - ... @property def google_news(self) -> GoogleNews: """Google News dataset.""" - ... @property def wikipedia_articles(self) -> WikipediaArticles: """Wikipedia Articles dataset.""" - ... @property def bbc_news(self) -> BBCNews: """BBC News dataset.""" - ... @property def cnn_news(self) -> CNNNews: """CNN News dataset.""" - ... @property def github_repositories(self) -> GithubRepositories: """GitHub Repositories dataset.""" - ... @property def creative_commons_images(self) -> CreativeCommonsImages: """Creative Commons Images dataset.""" - ... @property def creative_commons_3d_models(self) -> CreativeCommons3DModels: """Creative Commons 3D Models dataset.""" - ... @property def google_play_store(self) -> GooglePlayStore: """Google Play Store dataset.""" - ... @property def google_play_reviews(self) -> GooglePlayReviews: """Google Play Store Reviews dataset.""" - ... @property def apple_app_store(self) -> AppleAppStore: """Apple App Store dataset.""" - ... @property def apple_app_store_reviews(self) -> AppleAppStoreReviews: """Apple App Store Reviews dataset.""" - ... @property def amazon_best_sellers(self) -> AmazonBestSellers: """Amazon Best Sellers dataset.""" - ... @property def amazon_products_search(self) -> AmazonProductsSearch: """Amazon Products Search dataset.""" - ... @property def amazon_products_global(self) -> AmazonProductsGlobal: """Amazon Products Global dataset.""" - ... @property def amazon_walmart(self) -> AmazonWalmart: """Amazon Walmart dataset.""" - ... @property def walmart_sellers_info(self) -> WalmartSellersInfo: """Walmart Sellers Info dataset.""" - ... @property def ebay_products(self) -> EbayProducts: """eBay Products dataset.""" - ... @property def etsy_products(self) -> EtsyProducts: """Etsy Products dataset.""" - ... @property def wayfair_products(self) -> WayfairProducts: """Wayfair Products dataset.""" - ... @property def bestbuy_products(self) -> BestBuyProducts: """Best Buy Products dataset.""" - ... @property def myntra_products(self) -> MyntraProducts: """Myntra Products dataset.""" - ... @property def ozon_products(self) -> OzonProducts: """Ozon.ru Products dataset.""" - ... @property def wildberries_products(self) -> WildberriesProducts: """Wildberries.ru Products dataset.""" - ... @property def tokopedia_products(self) -> TokopediaProducts: """Tokopedia Products dataset.""" - ... @property def google_shopping_products(self) -> GoogleShoppingProducts: """Google Shopping Products dataset.""" - ... @property def google_shopping_search_us(self) -> GoogleShoppingSearchUS: """Google Shopping Search US dataset.""" - ... @property def mercadolivre_products(self) -> MercadolivreProducts: """MercadoLivre Products dataset.""" - ... @property def naver_products(self) -> NaverProducts: """Naver Products dataset.""" - ... @property def lazada_reviews(self) -> LazadaReviews: """Lazada Reviews dataset.""" - ... @property def lazada_products_search(self) -> LazadaProductsSearch: """Lazada Products Search dataset.""" - ... @property def homedepot_us_products(self) -> HomeDepotUSProducts: """Home Depot US Products dataset.""" - ... @property def homedepot_ca_products(self) -> HomeDepotCAProducts: """Home Depot Canada Products dataset.""" - ... @property def lowes_products(self) -> LowesProducts: """Lowes Products dataset.""" - ... @property def rona_products(self) -> RonaProducts: """Rona.ca Products dataset.""" - ... @property def kroger_products(self) -> KrogerProducts: """Kroger Products dataset.""" - ... @property def macys_products(self) -> MacysProducts: """Macys Products dataset.""" - ... @property def costco_products(self) -> CostcoProducts: """Costco Products dataset.""" - ... @property def bh_products(self) -> BHProducts: """B&H Products dataset.""" - ... @property def microcenter_products(self) -> MicroCenterProducts: """Micro Center Products dataset.""" - ... @property def autozone_products(self) -> AutozoneProducts: """AutoZone Products dataset.""" - ... @property def zillow_price_history(self) -> ZillowPriceHistory: """Zillow Price History dataset.""" - ... @property def zoopla_properties(self) -> ZooplaProperties: """Zoopla Properties dataset.""" - ... @property def booking_listings_search(self) -> BookingListingsSearch: """Booking.com Listings Search dataset.""" - ... @property def booking_hotel_listings(self) -> BookingHotelListings: """Booking.com Hotel Listings dataset.""" - ... @property def realtor_international_properties(self) -> RealtorInternationalProperties: """Realtor International Properties dataset.""" - ... @property def agoda_properties(self) -> AgodaProperties: """Agoda Properties dataset.""" - ... @property def carsales_listings(self) -> CarsalesListings: """Carsales Car Listings dataset.""" - ... @property def yahoo_finance_businesses(self) -> YahooFinanceBusinesses: """Yahoo Finance Businesses dataset.""" - ... @property def google_maps_full_info(self) -> GoogleMapsFullInfo: """Google Maps Full Info dataset.""" - ... diff --git a/src/brightdata/datasets/cnn/news.py b/src/brightdata/datasets/cnn/news.py index 880388e..2058435 100644 --- a/src/brightdata/datasets/cnn/news.py +++ b/src/brightdata/datasets/cnn/news.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CNNNews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/companies_enriched/companies.py b/src/brightdata/datasets/companies_enriched/companies.py index 4649da4..75bce4b 100644 --- a/src/brightdata/datasets/companies_enriched/companies.py +++ b/src/brightdata/datasets/companies_enriched/companies.py @@ -13,7 +13,7 @@ Use get_metadata() to discover all 336+ available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -73,9 +73,9 @@ class CompaniesEnriched(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_source: Optional[Dict[str, List[str]]] = None + self._fields_by_source: dict[str, list[str]] | None = None - async def get_fields_by_source(self, source: str, include_inactive: bool = False) -> List[str]: + async def get_fields_by_source(self, source: str, include_inactive: bool = False) -> list[str]: """ Get field names from a specific data source. @@ -107,7 +107,7 @@ async def get_fields_by_source(self, source: str, include_inactive: bool = False return sorted(fields) - async def get_all_sources(self) -> Dict[str, List[str]]: + async def get_all_sources(self) -> dict[str, list[str]]: """ Get all fields grouped by data source. @@ -123,7 +123,7 @@ async def get_all_sources(self) -> Dict[str, List[str]]: return self._fields_by_source metadata = await self.get_metadata() - result: Dict[str, List[str]] = {source: [] for source in SOURCES.values()} + result: dict[str, list[str]] = {source: [] for source in SOURCES.values()} result["other"] = [] # Fields without recognized suffix for name, field_info in metadata.fields.items(): @@ -147,7 +147,7 @@ async def get_all_sources(self) -> Dict[str, List[str]]: self._fields_by_source = result return result - async def get_common_fields(self) -> Dict[str, Dict[str, str]]: + async def get_common_fields(self) -> dict[str, dict[str, str]]: """ Get common field types across sources. @@ -232,7 +232,7 @@ async def get_common_fields(self) -> Dict[str, Dict[str, str]]: metadata = await self.get_metadata() available_fields = set(metadata.fields.keys()) - result: Dict[str, Dict[str, str]] = {} + result: dict[str, dict[str, str]] = {} for concept, field_names in common_patterns.items(): result[concept] = {} for field_name in field_names: @@ -245,7 +245,7 @@ async def get_common_fields(self) -> Dict[str, Dict[str, str]]: return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """ Search for fields containing a keyword. @@ -264,14 +264,16 @@ async def search_fields(self, keyword: str) -> List[str]: matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) - def _get_suffix_for_source(self, source: str) -> Optional[str]: + def _get_suffix_for_source(self, source: str) -> str | None: """Get the field suffix for a source name.""" source_lower = source.lower() for suffix, name in SOURCES.items(): @@ -280,7 +282,7 @@ def _get_suffix_for_source(self, source: str) -> Optional[str]: return None @staticmethod - def get_source_for_field(field_name: str) -> Optional[str]: + def get_source_for_field(field_name: str) -> str | None: """ Get the data source for a field name. @@ -296,7 +298,7 @@ def get_source_for_field(field_name: str) -> Optional[str]: return None @classmethod - def list_sources(cls) -> List[str]: + def list_sources(cls) -> list[str]: """ List all available data sources. diff --git a/src/brightdata/datasets/costco/products.py b/src/brightdata/datasets/costco/products.py index a0e1163..7545538 100644 --- a/src/brightdata/datasets/costco/products.py +++ b/src/brightdata/datasets/costco/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CostcoProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/crateandbarrel/products.py b/src/brightdata/datasets/crateandbarrel/products.py index 824b553..55bf2d0 100644 --- a/src/brightdata/datasets/crateandbarrel/products.py +++ b/src/brightdata/datasets/crateandbarrel/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CrateAndBarrelProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/creative_commons/__init__.py b/src/brightdata/datasets/creative_commons/__init__.py index 9ca4091..4dad389 100644 --- a/src/brightdata/datasets/creative_commons/__init__.py +++ b/src/brightdata/datasets/creative_commons/__init__.py @@ -3,4 +3,4 @@ from .images import CreativeCommonsImages from .models_3d import CreativeCommons3DModels -__all__ = ["CreativeCommonsImages", "CreativeCommons3DModels"] +__all__ = ["CreativeCommons3DModels", "CreativeCommonsImages"] diff --git a/src/brightdata/datasets/creative_commons/images.py b/src/brightdata/datasets/creative_commons/images.py index ab56ff7..525ce9c 100644 --- a/src/brightdata/datasets/creative_commons/images.py +++ b/src/brightdata/datasets/creative_commons/images.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CreativeCommonsImages(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/creative_commons/models_3d.py b/src/brightdata/datasets/creative_commons/models_3d.py index 6bcd676..8e27e57 100644 --- a/src/brightdata/datasets/creative_commons/models_3d.py +++ b/src/brightdata/datasets/creative_commons/models_3d.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class CreativeCommons3DModels(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/crunchbase/companies.py b/src/brightdata/datasets/crunchbase/companies.py index 616d158..ac9760e 100644 --- a/src/brightdata/datasets/crunchbase/companies.py +++ b/src/brightdata/datasets/crunchbase/companies.py @@ -7,7 +7,7 @@ See FIELDS dict for all filterable fields with descriptions and fill rates. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -36,7 +36,7 @@ class CrunchbaseCompanies(BaseDataset): # All available fields with metadata # Format: field_name -> {"type": str, "description": str, "fill_rate": float} - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { # Core identification "name": { "type": "text", diff --git a/src/brightdata/datasets/delvaux/products.py b/src/brightdata/datasets/delvaux/products.py index d43daee..3a5da83 100644 --- a/src/brightdata/datasets/delvaux/products.py +++ b/src/brightdata/datasets/delvaux/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class DelvauxProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/digikey/products.py b/src/brightdata/datasets/digikey/products.py index fc30ba0..513d7f6 100644 --- a/src/brightdata/datasets/digikey/products.py +++ b/src/brightdata/datasets/digikey/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class DigikeyProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/dior/products.py b/src/brightdata/datasets/dior/products.py index b638f2d..8188e4c 100644 --- a/src/brightdata/datasets/dior/products.py +++ b/src/brightdata/datasets/dior/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class DiorProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/ebay/products.py b/src/brightdata/datasets/ebay/products.py index cef3397..347398a 100644 --- a/src/brightdata/datasets/ebay/products.py +++ b/src/brightdata/datasets/ebay/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class EbayProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/employees_enriched/employees.py b/src/brightdata/datasets/employees_enriched/employees.py index da533ef..ce4de3d 100644 --- a/src/brightdata/datasets/employees_enriched/employees.py +++ b/src/brightdata/datasets/employees_enriched/employees.py @@ -8,7 +8,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -114,10 +114,10 @@ class EmployeesEnriched(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_profile_fields() -> List[str]: + def get_profile_fields() -> list[str]: """ Get all profile-related field names. @@ -131,7 +131,7 @@ def get_profile_fields() -> List[str]: return PROFILE_FIELDS.copy() @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """ Get all company-related field names. @@ -144,7 +144,7 @@ def get_company_fields() -> List[str]: """ return COMPANY_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """ Get all fields grouped by category. @@ -160,7 +160,7 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "profile": [], "company": [], "other": [], @@ -187,7 +187,7 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """ Search for fields containing a keyword. @@ -206,14 +206,16 @@ async def search_fields(self, keyword: str) -> List[str]: matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) - async def get_experience_fields(self) -> List[str]: + async def get_experience_fields(self) -> list[str]: """ Get fields related to work experience. @@ -222,7 +224,7 @@ async def get_experience_fields(self) -> List[str]: """ return await self.search_fields("experience") - async def get_education_fields(self) -> List[str]: + async def get_education_fields(self) -> list[str]: """ Get fields related to education. @@ -232,7 +234,7 @@ async def get_education_fields(self) -> List[str]: return await self.search_fields("education") @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """ Get fields that can be used as unique identifiers. diff --git a/src/brightdata/datasets/etsy/products.py b/src/brightdata/datasets/etsy/products.py index d085431..c71267a 100644 --- a/src/brightdata/datasets/etsy/products.py +++ b/src/brightdata/datasets/etsy/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class EtsyProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/__init__.py b/src/brightdata/datasets/facebook/__init__.py index 2a07c8a..0e1ff95 100644 --- a/src/brightdata/datasets/facebook/__init__.py +++ b/src/brightdata/datasets/facebook/__init__.py @@ -1,25 +1,25 @@ """Facebook datasets.""" -from .pages_posts import FacebookPagesPosts from .comments import FacebookComments -from .posts_by_url import FacebookPostsByUrl -from .reels import FacebookReels -from .marketplace import FacebookMarketplace from .company_reviews import FacebookCompanyReviews from .events import FacebookEvents -from .profiles import FacebookProfiles -from .pages_profiles import FacebookPagesProfiles from .group_posts import FacebookGroupPosts +from .marketplace import FacebookMarketplace +from .pages_posts import FacebookPagesPosts +from .pages_profiles import FacebookPagesProfiles +from .posts_by_url import FacebookPostsByUrl +from .profiles import FacebookProfiles +from .reels import FacebookReels __all__ = [ - "FacebookPagesPosts", "FacebookComments", - "FacebookPostsByUrl", - "FacebookReels", - "FacebookMarketplace", "FacebookCompanyReviews", "FacebookEvents", - "FacebookProfiles", - "FacebookPagesProfiles", "FacebookGroupPosts", + "FacebookMarketplace", + "FacebookPagesPosts", + "FacebookPagesProfiles", + "FacebookPostsByUrl", + "FacebookProfiles", + "FacebookReels", ] diff --git a/src/brightdata/datasets/facebook/comments.py b/src/brightdata/datasets/facebook/comments.py index f7ec84a..14b5796 100644 --- a/src/brightdata/datasets/facebook/comments.py +++ b/src/brightdata/datasets/facebook/comments.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookComments(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/company_reviews.py b/src/brightdata/datasets/facebook/company_reviews.py index b9f0fee..71282bc 100644 --- a/src/brightdata/datasets/facebook/company_reviews.py +++ b/src/brightdata/datasets/facebook/company_reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookCompanyReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/events.py b/src/brightdata/datasets/facebook/events.py index 9b4ad79..64713a9 100644 --- a/src/brightdata/datasets/facebook/events.py +++ b/src/brightdata/datasets/facebook/events.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookEvents(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/group_posts.py b/src/brightdata/datasets/facebook/group_posts.py index 6f93e61..9c03fad 100644 --- a/src/brightdata/datasets/facebook/group_posts.py +++ b/src/brightdata/datasets/facebook/group_posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookGroupPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/marketplace.py b/src/brightdata/datasets/facebook/marketplace.py index f44faad..5fafe8a 100644 --- a/src/brightdata/datasets/facebook/marketplace.py +++ b/src/brightdata/datasets/facebook/marketplace.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookMarketplace(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/pages_posts.py b/src/brightdata/datasets/facebook/pages_posts.py index f00a353..40adcea 100644 --- a/src/brightdata/datasets/facebook/pages_posts.py +++ b/src/brightdata/datasets/facebook/pages_posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookPagesPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/pages_profiles.py b/src/brightdata/datasets/facebook/pages_profiles.py index 42637c4..2ab2312 100644 --- a/src/brightdata/datasets/facebook/pages_profiles.py +++ b/src/brightdata/datasets/facebook/pages_profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookPagesProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/posts_by_url.py b/src/brightdata/datasets/facebook/posts_by_url.py index 52fdd19..f70ea4f 100644 --- a/src/brightdata/datasets/facebook/posts_by_url.py +++ b/src/brightdata/datasets/facebook/posts_by_url.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookPostsByUrl(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/profiles.py b/src/brightdata/datasets/facebook/profiles.py index ef3e5a4..0e61987 100644 --- a/src/brightdata/datasets/facebook/profiles.py +++ b/src/brightdata/datasets/facebook/profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/facebook/reels.py b/src/brightdata/datasets/facebook/reels.py index cae1240..01e402d 100644 --- a/src/brightdata/datasets/facebook/reels.py +++ b/src/brightdata/datasets/facebook/reels.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FacebookReels(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/fanatics/products.py b/src/brightdata/datasets/fanatics/products.py index fa11422..55fdaeb 100644 --- a/src/brightdata/datasets/fanatics/products.py +++ b/src/brightdata/datasets/fanatics/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FanaticsProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/fendi/products.py b/src/brightdata/datasets/fendi/products.py index 853782d..dcc1e01 100644 --- a/src/brightdata/datasets/fendi/products.py +++ b/src/brightdata/datasets/fendi/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class FendiProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/g2/products.py b/src/brightdata/datasets/g2/products.py index ae0df2b..d06d2a9 100644 --- a/src/brightdata/datasets/g2/products.py +++ b/src/brightdata/datasets/g2/products.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -111,45 +111,45 @@ class G2Products(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_product_fields() -> List[str]: + def get_product_fields() -> list[str]: """Get product identity field names.""" return PRODUCT_FIELDS.copy() @staticmethod - def get_seller_fields() -> List[str]: + def get_seller_fields() -> list[str]: """Get seller/vendor field names.""" return SELLER_FIELDS.copy() @staticmethod - def get_rating_fields() -> List[str]: + def get_rating_fields() -> list[str]: """Get rating and review field names.""" return RATING_FIELDS.copy() @staticmethod - def get_competitive_fields() -> List[str]: + def get_competitive_fields() -> list[str]: """Get competitive analysis field names.""" return COMPETITIVE_FIELDS.copy() @staticmethod - def get_content_fields() -> List[str]: + def get_content_fields() -> list[str]: """Get content field names (pricing, media).""" return CONTENT_FIELDS.copy() @staticmethod - def get_category_fields() -> List[str]: + def get_category_fields() -> list[str]: """Get category and classification field names.""" return CATEGORY_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "product": [], "seller": [], "rating": [], @@ -191,22 +191,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "product_id", diff --git a/src/brightdata/datasets/g2/reviews.py b/src/brightdata/datasets/g2/reviews.py index def0053..8a4ec07 100644 --- a/src/brightdata/datasets/g2/reviews.py +++ b/src/brightdata/datasets/g2/reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -74,30 +74,30 @@ class G2Reviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """Get review content field names.""" return REVIEW_FIELDS.copy() @staticmethod - def get_author_fields() -> List[str]: + def get_author_fields() -> list[str]: """Get author-related field names.""" return AUTHOR_FIELDS.copy() @staticmethod - def get_product_fields() -> List[str]: + def get_product_fields() -> list[str]: """Get product-related field names.""" return PRODUCT_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "review": [], "author": [], "product": [], @@ -127,22 +127,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "review_id", diff --git a/src/brightdata/datasets/github/repositories.py b/src/brightdata/datasets/github/repositories.py index 87c0fb7..9f26ef7 100644 --- a/src/brightdata/datasets/github/repositories.py +++ b/src/brightdata/datasets/github/repositories.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GithubRepositories(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/glassdoor/__init__.py b/src/brightdata/datasets/glassdoor/__init__.py index aadf858..5d48a4c 100644 --- a/src/brightdata/datasets/glassdoor/__init__.py +++ b/src/brightdata/datasets/glassdoor/__init__.py @@ -1,7 +1,7 @@ """Glassdoor datasets.""" from .companies import GlassdoorCompanies -from .reviews import GlassdoorReviews from .jobs import GlassdoorJobs +from .reviews import GlassdoorReviews -__all__ = ["GlassdoorCompanies", "GlassdoorReviews", "GlassdoorJobs"] +__all__ = ["GlassdoorCompanies", "GlassdoorJobs", "GlassdoorReviews"] diff --git a/src/brightdata/datasets/glassdoor/companies.py b/src/brightdata/datasets/glassdoor/companies.py index b0eece2..848490d 100644 --- a/src/brightdata/datasets/glassdoor/companies.py +++ b/src/brightdata/datasets/glassdoor/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -118,10 +118,10 @@ class GlassdoorCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_details_fields() -> List[str]: + def get_details_fields() -> list[str]: """ Get company details field names. @@ -135,7 +135,7 @@ def get_details_fields() -> List[str]: return DETAILS_FIELDS.copy() @staticmethod - def get_ratings_fields() -> List[str]: + def get_ratings_fields() -> list[str]: """ Get all ratings-related field names. @@ -149,7 +149,7 @@ def get_ratings_fields() -> List[str]: return RATINGS_FIELDS.copy() @staticmethod - def get_url_fields() -> List[str]: + def get_url_fields() -> list[str]: """ Get all URL field names. @@ -163,7 +163,7 @@ def get_url_fields() -> List[str]: return URL_FIELDS.copy() @staticmethod - def get_count_fields() -> List[str]: + def get_count_fields() -> list[str]: """ Get all count-related field names. @@ -177,7 +177,7 @@ def get_count_fields() -> List[str]: return COUNT_FIELDS.copy() @staticmethod - def get_interview_fields() -> List[str]: + def get_interview_fields() -> list[str]: """ Get interview-related field names. @@ -190,7 +190,7 @@ def get_interview_fields() -> List[str]: """ return INTERVIEW_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """ Get all fields grouped by category. @@ -206,7 +206,7 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "details": [], "ratings": [], "urls": [], @@ -241,7 +241,7 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """ Search for fields containing a keyword. @@ -260,14 +260,16 @@ async def search_fields(self, keyword: str) -> List[str]: matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) - async def get_diversity_fields(self) -> List[str]: + async def get_diversity_fields(self) -> list[str]: """ Get diversity and inclusion related fields. @@ -276,7 +278,7 @@ async def get_diversity_fields(self) -> List[str]: """ return await self.search_fields("diversity") - async def get_ceo_fields(self) -> List[str]: + async def get_ceo_fields(self) -> list[str]: """ Get CEO-related fields. @@ -286,7 +288,7 @@ async def get_ceo_fields(self) -> List[str]: return await self.search_fields("ceo") @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """ Get fields that can be used as unique identifiers. diff --git a/src/brightdata/datasets/glassdoor/jobs.py b/src/brightdata/datasets/glassdoor/jobs.py index cb77d5e..5c2efcc 100644 --- a/src/brightdata/datasets/glassdoor/jobs.py +++ b/src/brightdata/datasets/glassdoor/jobs.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -93,35 +93,35 @@ class GlassdoorJobs(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_job_fields() -> List[str]: + def get_job_fields() -> list[str]: """Get job-related field names.""" return JOB_FIELDS.copy() @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_rating_fields() -> List[str]: + def get_rating_fields() -> list[str]: """Get rating field names.""" return RATING_FIELDS.copy() @staticmethod - def get_pay_fields() -> List[str]: + def get_pay_fields() -> list[str]: """Get pay-related field names.""" return PAY_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "job": [], "company": [], "rating": [], @@ -155,22 +155,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "job_posting_id", diff --git a/src/brightdata/datasets/glassdoor/reviews.py b/src/brightdata/datasets/glassdoor/reviews.py index 3990797..e43eb86 100644 --- a/src/brightdata/datasets/glassdoor/reviews.py +++ b/src/brightdata/datasets/glassdoor/reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -102,40 +102,40 @@ class GlassdoorReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """Get review content field names.""" return REVIEW_FIELDS.copy() @staticmethod - def get_employee_fields() -> List[str]: + def get_employee_fields() -> list[str]: """Get employee-related field names.""" return EMPLOYEE_FIELDS.copy() @staticmethod - def get_rating_fields() -> List[str]: + def get_rating_fields() -> list[str]: """Get all rating field names.""" return RATING_FIELDS.copy() @staticmethod - def get_flag_fields() -> List[str]: + def get_flag_fields() -> list[str]: """Get flag field names.""" return FLAG_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "review": [], "employee": [], @@ -173,22 +173,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "review_id", diff --git a/src/brightdata/datasets/goodreads/books.py b/src/brightdata/datasets/goodreads/books.py index abcc4e1..474f525 100644 --- a/src/brightdata/datasets/goodreads/books.py +++ b/src/brightdata/datasets/goodreads/books.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -34,7 +34,7 @@ class GoodreadsBooks(BaseDataset): NAME = "goodreads_books" # All available fields with metadata - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { # Book identification "id": { "type": "text", diff --git a/src/brightdata/datasets/google_maps/__init__.py b/src/brightdata/datasets/google_maps/__init__.py index 1453afb..758c800 100644 --- a/src/brightdata/datasets/google_maps/__init__.py +++ b/src/brightdata/datasets/google_maps/__init__.py @@ -1,9 +1,9 @@ """Google Maps datasets.""" -from .reviews import GoogleMapsReviews from .full_info import GoogleMapsFullInfo +from .reviews import GoogleMapsReviews __all__ = [ - "GoogleMapsReviews", "GoogleMapsFullInfo", + "GoogleMapsReviews", ] diff --git a/src/brightdata/datasets/google_maps/full_info.py b/src/brightdata/datasets/google_maps/full_info.py index 773558e..28c8703 100644 --- a/src/brightdata/datasets/google_maps/full_info.py +++ b/src/brightdata/datasets/google_maps/full_info.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GoogleMapsFullInfo(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/google_maps/reviews.py b/src/brightdata/datasets/google_maps/reviews.py index ec8cb8b..0fe20aa 100644 --- a/src/brightdata/datasets/google_maps/reviews.py +++ b/src/brightdata/datasets/google_maps/reviews.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -87,10 +87,10 @@ class GoogleMapsReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_place_fields() -> List[str]: + def get_place_fields() -> list[str]: """ Get place-related field names. @@ -100,7 +100,7 @@ def get_place_fields() -> List[str]: return PLACE_FIELDS.copy() @staticmethod - def get_reviewer_fields() -> List[str]: + def get_reviewer_fields() -> list[str]: """ Get reviewer-related field names. @@ -110,7 +110,7 @@ def get_reviewer_fields() -> List[str]: return REVIEWER_FIELDS.copy() @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """ Get review-related field names. @@ -119,7 +119,7 @@ def get_review_fields() -> List[str]: """ return REVIEW_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """ Get all fields grouped by category. @@ -130,7 +130,7 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "place": [], "reviewer": [], "review": [], @@ -160,7 +160,7 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """ Search for fields containing a keyword. @@ -175,15 +175,17 @@ async def search_fields(self, keyword: str) -> List[str]: matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """ Get fields that can be used as unique identifiers. diff --git a/src/brightdata/datasets/google_news/news.py b/src/brightdata/datasets/google_news/news.py index 1caffba..632404a 100644 --- a/src/brightdata/datasets/google_news/news.py +++ b/src/brightdata/datasets/google_news/news.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GoogleNews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/google_play/__init__.py b/src/brightdata/datasets/google_play/__init__.py index ee96916..c5c3bf0 100644 --- a/src/brightdata/datasets/google_play/__init__.py +++ b/src/brightdata/datasets/google_play/__init__.py @@ -1,6 +1,6 @@ """Google Play datasets.""" -from .store import GooglePlayStore from .reviews import GooglePlayReviews +from .store import GooglePlayStore -__all__ = ["GooglePlayStore", "GooglePlayReviews"] +__all__ = ["GooglePlayReviews", "GooglePlayStore"] diff --git a/src/brightdata/datasets/google_play/reviews.py b/src/brightdata/datasets/google_play/reviews.py index cdcd0c1..e0a1995 100644 --- a/src/brightdata/datasets/google_play/reviews.py +++ b/src/brightdata/datasets/google_play/reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GooglePlayReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/google_play/store.py b/src/brightdata/datasets/google_play/store.py index a4b7fc3..027a3fb 100644 --- a/src/brightdata/datasets/google_play/store.py +++ b/src/brightdata/datasets/google_play/store.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GooglePlayStore(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/google_shopping/products.py b/src/brightdata/datasets/google_shopping/products.py index 82a1440..39e5ed3 100644 --- a/src/brightdata/datasets/google_shopping/products.py +++ b/src/brightdata/datasets/google_shopping/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GoogleShoppingProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/google_shopping/search_us.py b/src/brightdata/datasets/google_shopping/search_us.py index ad01086..c0a2483 100644 --- a/src/brightdata/datasets/google_shopping/search_us.py +++ b/src/brightdata/datasets/google_shopping/search_us.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class GoogleShoppingSearchUS(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/hermes/products.py b/src/brightdata/datasets/hermes/products.py index ef2a134..bb55920 100644 --- a/src/brightdata/datasets/hermes/products.py +++ b/src/brightdata/datasets/hermes/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class HermesProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/hm/products.py b/src/brightdata/datasets/hm/products.py index d528d82..2fa694b 100644 --- a/src/brightdata/datasets/hm/products.py +++ b/src/brightdata/datasets/hm/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class HMProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/homedepot/__init__.py b/src/brightdata/datasets/homedepot/__init__.py index d7fa1fc..37a0d69 100644 --- a/src/brightdata/datasets/homedepot/__init__.py +++ b/src/brightdata/datasets/homedepot/__init__.py @@ -3,4 +3,4 @@ from .products_ca import HomeDepotCAProducts from .products_us import HomeDepotUSProducts -__all__ = ["HomeDepotUSProducts", "HomeDepotCAProducts"] +__all__ = ["HomeDepotCAProducts", "HomeDepotUSProducts"] diff --git a/src/brightdata/datasets/homedepot/products_ca.py b/src/brightdata/datasets/homedepot/products_ca.py index 4d6b96a..917def3 100644 --- a/src/brightdata/datasets/homedepot/products_ca.py +++ b/src/brightdata/datasets/homedepot/products_ca.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class HomeDepotCAProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/homedepot/products_us.py b/src/brightdata/datasets/homedepot/products_us.py index 6a19cb2..11483af 100644 --- a/src/brightdata/datasets/homedepot/products_us.py +++ b/src/brightdata/datasets/homedepot/products_us.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class HomeDepotUSProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/ikea/products.py b/src/brightdata/datasets/ikea/products.py index be19a90..f470629 100644 --- a/src/brightdata/datasets/ikea/products.py +++ b/src/brightdata/datasets/ikea/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class IkeaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/imdb/movies.py b/src/brightdata/datasets/imdb/movies.py index 21aee7a..1b67401 100644 --- a/src/brightdata/datasets/imdb/movies.py +++ b/src/brightdata/datasets/imdb/movies.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -34,7 +34,7 @@ class IMDBMovies(BaseDataset): NAME = "imdb_movies" # All available fields with metadata - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { # Core identification "id": { "type": "text", diff --git a/src/brightdata/datasets/indeed/companies.py b/src/brightdata/datasets/indeed/companies.py index 13b9a3f..2ecded8 100644 --- a/src/brightdata/datasets/indeed/companies.py +++ b/src/brightdata/datasets/indeed/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -100,40 +100,40 @@ class IndeedCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_work_culture_fields() -> List[str]: + def get_work_culture_fields() -> list[str]: """Get work culture field names.""" return WORK_CULTURE_FIELDS.copy() @staticmethod - def get_reviews_fields() -> List[str]: + def get_reviews_fields() -> list[str]: """Get reviews-related field names.""" return REVIEWS_FIELDS.copy() @staticmethod - def get_salaries_fields() -> List[str]: + def get_salaries_fields() -> list[str]: """Get salaries-related field names.""" return SALARIES_FIELDS.copy() @staticmethod - def get_jobs_fields() -> List[str]: + def get_jobs_fields() -> list[str]: """Get jobs-related field names.""" return JOBS_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "work_culture": [], "reviews": [], @@ -174,22 +174,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "company_id", diff --git a/src/brightdata/datasets/indeed/jobs.py b/src/brightdata/datasets/indeed/jobs.py index 702bb41..6e79609 100644 --- a/src/brightdata/datasets/indeed/jobs.py +++ b/src/brightdata/datasets/indeed/jobs.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -36,4 +36,4 @@ class IndeedJobs(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/infocasas/properties.py b/src/brightdata/datasets/infocasas/properties.py index 8118959..4878eed 100644 --- a/src/brightdata/datasets/infocasas/properties.py +++ b/src/brightdata/datasets/infocasas/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class InfocasasUruguay(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/inmuebles24/properties.py b/src/brightdata/datasets/inmuebles24/properties.py index 055b23a..ec9a176 100644 --- a/src/brightdata/datasets/inmuebles24/properties.py +++ b/src/brightdata/datasets/inmuebles24/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class Inmuebles24Mexico(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/instagram/__init__.py b/src/brightdata/datasets/instagram/__init__.py index b8504e9..b7544d2 100644 --- a/src/brightdata/datasets/instagram/__init__.py +++ b/src/brightdata/datasets/instagram/__init__.py @@ -1,13 +1,13 @@ """Instagram datasets.""" -from .profiles import InstagramProfiles -from .posts import InstagramPosts from .comments import InstagramComments +from .posts import InstagramPosts +from .profiles import InstagramProfiles from .reels import InstagramReels __all__ = [ - "InstagramProfiles", - "InstagramPosts", "InstagramComments", + "InstagramPosts", + "InstagramProfiles", "InstagramReels", ] diff --git a/src/brightdata/datasets/instagram/comments.py b/src/brightdata/datasets/instagram/comments.py index 37517aa..d58bebf 100644 --- a/src/brightdata/datasets/instagram/comments.py +++ b/src/brightdata/datasets/instagram/comments.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class InstagramComments(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/instagram/posts.py b/src/brightdata/datasets/instagram/posts.py index 290afef..6d1f2ad 100644 --- a/src/brightdata/datasets/instagram/posts.py +++ b/src/brightdata/datasets/instagram/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class InstagramPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/instagram/profiles.py b/src/brightdata/datasets/instagram/profiles.py index c3ea260..ea4b55e 100644 --- a/src/brightdata/datasets/instagram/profiles.py +++ b/src/brightdata/datasets/instagram/profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -36,4 +36,4 @@ class InstagramProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/instagram/reels.py b/src/brightdata/datasets/instagram/reels.py index f064ccc..599a5be 100644 --- a/src/brightdata/datasets/instagram/reels.py +++ b/src/brightdata/datasets/instagram/reels.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class InstagramReels(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/kroger/products.py b/src/brightdata/datasets/kroger/products.py index 986e712..3064c05 100644 --- a/src/brightdata/datasets/kroger/products.py +++ b/src/brightdata/datasets/kroger/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class KrogerProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/lawyers/us_lawyers.py b/src/brightdata/datasets/lawyers/us_lawyers.py index ccde4bf..7eed811 100644 --- a/src/brightdata/datasets/lawyers/us_lawyers.py +++ b/src/brightdata/datasets/lawyers/us_lawyers.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -119,40 +119,40 @@ class USLawyers(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_profile_fields() -> List[str]: + def get_profile_fields() -> list[str]: """Get profile-related field names.""" return PROFILE_FIELDS.copy() @staticmethod - def get_education_fields() -> List[str]: + def get_education_fields() -> list[str]: """Get education-related field names.""" return EDUCATION_FIELDS.copy() @staticmethod - def get_practice_fields() -> List[str]: + def get_practice_fields() -> list[str]: """Get practice-related field names.""" return PRACTICE_FIELDS.copy() @staticmethod - def get_contact_fields() -> List[str]: + def get_contact_fields() -> list[str]: """Get contact-related field names.""" return CONTACT_FIELDS.copy() @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """Get review-related field names.""" return REVIEW_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "profile": [], "education": [], "practice": [], @@ -194,22 +194,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "isln", diff --git a/src/brightdata/datasets/lazada/__init__.py b/src/brightdata/datasets/lazada/__init__.py index 897bb2d..a1a03f7 100644 --- a/src/brightdata/datasets/lazada/__init__.py +++ b/src/brightdata/datasets/lazada/__init__.py @@ -1,11 +1,11 @@ """Lazada datasets.""" from .products import LazadaProducts -from .reviews import LazadaReviews from .products_search import LazadaProductsSearch +from .reviews import LazadaReviews __all__ = [ "LazadaProducts", - "LazadaReviews", "LazadaProductsSearch", + "LazadaReviews", ] diff --git a/src/brightdata/datasets/lazada/products.py b/src/brightdata/datasets/lazada/products.py index fa17377..f094979 100644 --- a/src/brightdata/datasets/lazada/products.py +++ b/src/brightdata/datasets/lazada/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LazadaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/lazada/products_search.py b/src/brightdata/datasets/lazada/products_search.py index 69d23b8..b8b7a0c 100644 --- a/src/brightdata/datasets/lazada/products_search.py +++ b/src/brightdata/datasets/lazada/products_search.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LazadaProductsSearch(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/lazada/reviews.py b/src/brightdata/datasets/lazada/reviews.py index 14af421..e9ac757 100644 --- a/src/brightdata/datasets/lazada/reviews.py +++ b/src/brightdata/datasets/lazada/reviews.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LazadaReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/lazboy/products.py b/src/brightdata/datasets/lazboy/products.py index 784921a..d9d483b 100644 --- a/src/brightdata/datasets/lazboy/products.py +++ b/src/brightdata/datasets/lazboy/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LaZBoyProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/lego/products.py b/src/brightdata/datasets/lego/products.py index 30a5ffc..46e6649 100644 --- a/src/brightdata/datasets/lego/products.py +++ b/src/brightdata/datasets/lego/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LegoProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/linkedin/__init__.py b/src/brightdata/datasets/linkedin/__init__.py index b123902..ca24d62 100644 --- a/src/brightdata/datasets/linkedin/__init__.py +++ b/src/brightdata/datasets/linkedin/__init__.py @@ -1,15 +1,15 @@ """LinkedIn datasets.""" -from .people_profiles import LinkedInPeopleProfiles from .company_profiles import LinkedInCompanyProfiles from .job_listings import LinkedInJobListings +from .people_profiles import LinkedInPeopleProfiles from .posts import LinkedInPosts from .profiles_job_listings import LinkedInProfilesJobListings __all__ = [ - "LinkedInPeopleProfiles", "LinkedInCompanyProfiles", "LinkedInJobListings", + "LinkedInPeopleProfiles", "LinkedInPosts", "LinkedInProfilesJobListings", ] diff --git a/src/brightdata/datasets/linkedin/company_profiles.py b/src/brightdata/datasets/linkedin/company_profiles.py index daefbd7..f8c5a22 100644 --- a/src/brightdata/datasets/linkedin/company_profiles.py +++ b/src/brightdata/datasets/linkedin/company_profiles.py @@ -7,7 +7,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -36,7 +36,7 @@ class LinkedInCompanyProfiles(BaseDataset): # All available fields with metadata # Format: field_name -> {"type": str, "description": str} - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { "id": { "type": "text", "description": "Unique identifier for the company profile (URL slug)", diff --git a/src/brightdata/datasets/linkedin/job_listings.py b/src/brightdata/datasets/linkedin/job_listings.py index fcde0d0..754e4a8 100644 --- a/src/brightdata/datasets/linkedin/job_listings.py +++ b/src/brightdata/datasets/linkedin/job_listings.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -66,30 +66,30 @@ class LinkedInJobListings(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_profile_fields() -> List[str]: + def get_profile_fields() -> list[str]: """Get profile-related field names.""" return PROFILE_FIELDS.copy() @staticmethod - def get_career_fields() -> List[str]: + def get_career_fields() -> list[str]: """Get career-related field names.""" return CAREER_FIELDS.copy() @staticmethod - def get_job_fields() -> List[str]: + def get_job_fields() -> list[str]: """Get job listing field names.""" return JOB_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "profile": [], "career": [], "jobs": [], @@ -119,22 +119,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "url", diff --git a/src/brightdata/datasets/linkedin/people_profiles.py b/src/brightdata/datasets/linkedin/people_profiles.py index 34bc07f..196aa46 100644 --- a/src/brightdata/datasets/linkedin/people_profiles.py +++ b/src/brightdata/datasets/linkedin/people_profiles.py @@ -7,7 +7,7 @@ See FIELDS dict for all filterable fields with descriptions and fill rates. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -36,7 +36,7 @@ class LinkedInPeopleProfiles(BaseDataset): # All available fields with metadata # Format: field_name -> {"type": str, "description": str, "fill_rate": float} - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { "id": { "type": "text", "description": "A unique identifier for the person's LinkedIn profile", diff --git a/src/brightdata/datasets/linkedin/posts.py b/src/brightdata/datasets/linkedin/posts.py index 32acf94..10fc1b3 100644 --- a/src/brightdata/datasets/linkedin/posts.py +++ b/src/brightdata/datasets/linkedin/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LinkedInPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/linkedin/profiles_job_listings.py b/src/brightdata/datasets/linkedin/profiles_job_listings.py index ff6356d..e291cbc 100644 --- a/src/brightdata/datasets/linkedin/profiles_job_listings.py +++ b/src/brightdata/datasets/linkedin/profiles_job_listings.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LinkedInProfilesJobListings(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/llbean/products.py b/src/brightdata/datasets/llbean/products.py index e123a6c..1705c3d 100644 --- a/src/brightdata/datasets/llbean/products.py +++ b/src/brightdata/datasets/llbean/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LLBeanProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/loewe/products.py b/src/brightdata/datasets/loewe/products.py index 39f3ecc..8648e88 100644 --- a/src/brightdata/datasets/loewe/products.py +++ b/src/brightdata/datasets/loewe/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LoeweProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/lowes/products.py b/src/brightdata/datasets/lowes/products.py index 108b278..d97cea7 100644 --- a/src/brightdata/datasets/lowes/products.py +++ b/src/brightdata/datasets/lowes/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class LowesProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/macys/products.py b/src/brightdata/datasets/macys/products.py index de3ed15..b5d0a79 100644 --- a/src/brightdata/datasets/macys/products.py +++ b/src/brightdata/datasets/macys/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MacysProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/mango/products.py b/src/brightdata/datasets/mango/products.py index 43138b4..a23f9d1 100644 --- a/src/brightdata/datasets/mango/products.py +++ b/src/brightdata/datasets/mango/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MangoProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/manta/businesses.py b/src/brightdata/datasets/manta/businesses.py index 861df55..687e3ad 100644 --- a/src/brightdata/datasets/manta/businesses.py +++ b/src/brightdata/datasets/manta/businesses.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -95,40 +95,40 @@ class MantaBusinesses(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_location_fields() -> List[str]: + def get_location_fields() -> list[str]: """Get location-related field names.""" return LOCATION_FIELDS.copy() @staticmethod - def get_contact_fields() -> List[str]: + def get_contact_fields() -> list[str]: """Get contact-related field names.""" return CONTACT_FIELDS.copy() @staticmethod - def get_operating_fields() -> List[str]: + def get_operating_fields() -> list[str]: """Get operating hours field names.""" return OPERATING_FIELDS.copy() @staticmethod - def get_metrics_fields() -> List[str]: + def get_metrics_fields() -> list[str]: """Get business metrics field names.""" return METRICS_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "location": [], "contact": [], @@ -166,22 +166,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "company_id", diff --git a/src/brightdata/datasets/massimo_dutti/products.py b/src/brightdata/datasets/massimo_dutti/products.py index 4839123..d0650fa 100644 --- a/src/brightdata/datasets/massimo_dutti/products.py +++ b/src/brightdata/datasets/massimo_dutti/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MassimoDuttiProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/mattressfirm/products.py b/src/brightdata/datasets/mattressfirm/products.py index 7429663..0c2d947 100644 --- a/src/brightdata/datasets/mattressfirm/products.py +++ b/src/brightdata/datasets/mattressfirm/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MattressfirmProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/mediamarkt/products.py b/src/brightdata/datasets/mediamarkt/products.py index 539b19e..f3a7366 100644 --- a/src/brightdata/datasets/mediamarkt/products.py +++ b/src/brightdata/datasets/mediamarkt/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -32,4 +32,4 @@ class MediamarktProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/mercadolivre/products.py b/src/brightdata/datasets/mercadolivre/products.py index 67188cc..4eedbb2 100644 --- a/src/brightdata/datasets/mercadolivre/products.py +++ b/src/brightdata/datasets/mercadolivre/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MercadolivreProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/metrocuadrado/properties.py b/src/brightdata/datasets/metrocuadrado/properties.py index 80e1406..763ebb7 100644 --- a/src/brightdata/datasets/metrocuadrado/properties.py +++ b/src/brightdata/datasets/metrocuadrado/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MetrocuadradoProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/microcenter/products.py b/src/brightdata/datasets/microcenter/products.py index 117cd5b..bb34525 100644 --- a/src/brightdata/datasets/microcenter/products.py +++ b/src/brightdata/datasets/microcenter/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MicroCenterProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/models.py b/src/brightdata/datasets/models.py index fdc56d0..1c1279e 100644 --- a/src/brightdata/datasets/models.py +++ b/src/brightdata/datasets/models.py @@ -3,7 +3,7 @@ """ from dataclasses import dataclass, field -from typing import Dict, Optional, Any, Literal +from typing import Any, Literal @dataclass @@ -22,7 +22,7 @@ class DatasetField: type: str # "text", "number", "url", "array", "object", "boolean" active: bool = True required: bool = False - description: Optional[str] = None + description: str | None = None @dataclass @@ -30,10 +30,10 @@ class DatasetMetadata: """Dataset metadata returned by get_metadata().""" id: str - fields: Dict[str, DatasetField] = field(default_factory=dict) + fields: dict[str, DatasetField] = field(default_factory=dict) @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "DatasetMetadata": + def from_dict(cls, data: dict[str, Any]) -> "DatasetMetadata": """Create from API response.""" fields = {} for name, field_data in data.get("fields", {}).items(): @@ -53,15 +53,15 @@ class SnapshotStatus: id: str status: Literal["scheduled", "building", "ready", "failed"] - dataset_id: Optional[str] = None - dataset_size: Optional[int] = None # records in snapshot - file_size: Optional[int] = None # bytes - cost: Optional[float] = None - error: Optional[str] = None - raw: Dict[str, Any] = field(default_factory=dict) # full API response — never lose the reason + dataset_id: str | None = None + dataset_size: int | None = None # records in snapshot + file_size: int | None = None # bytes + cost: float | None = None + error: str | None = None + raw: dict[str, Any] = field(default_factory=dict) # full API response — never lose the reason @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "SnapshotStatus": + def from_dict(cls, data: dict[str, Any]) -> "SnapshotStatus": """Create from API response.""" return cls( id=data.get("id", data.get("snapshot_id", "")), diff --git a/src/brightdata/datasets/montblanc/products.py b/src/brightdata/datasets/montblanc/products.py index e5c3d87..5f8cf20 100644 --- a/src/brightdata/datasets/montblanc/products.py +++ b/src/brightdata/datasets/montblanc/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MontblancProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/mouser/products.py b/src/brightdata/datasets/mouser/products.py index 4d07998..225a109 100644 --- a/src/brightdata/datasets/mouser/products.py +++ b/src/brightdata/datasets/mouser/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MouserProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/moynat/products.py b/src/brightdata/datasets/moynat/products.py index eade7d4..c92c660 100644 --- a/src/brightdata/datasets/moynat/products.py +++ b/src/brightdata/datasets/moynat/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MoynatProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/mybobs/products.py b/src/brightdata/datasets/mybobs/products.py index 8d6075a..9aae107 100644 --- a/src/brightdata/datasets/mybobs/products.py +++ b/src/brightdata/datasets/mybobs/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MybobsProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/myntra/products.py b/src/brightdata/datasets/myntra/products.py index f252198..e35faa7 100644 --- a/src/brightdata/datasets/myntra/products.py +++ b/src/brightdata/datasets/myntra/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class MyntraProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/naver/products.py b/src/brightdata/datasets/naver/products.py index 8e0e4eb..1d0be4f 100644 --- a/src/brightdata/datasets/naver/products.py +++ b/src/brightdata/datasets/naver/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class NaverProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/nba/players_stats.py b/src/brightdata/datasets/nba/players_stats.py index 19b6cf7..abd05c1 100644 --- a/src/brightdata/datasets/nba/players_stats.py +++ b/src/brightdata/datasets/nba/players_stats.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -34,7 +34,7 @@ class NBAPlayersStats(BaseDataset): NAME = "nba_players_stats" # All available fields with metadata - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { # Player identification "url": { "type": "url", diff --git a/src/brightdata/datasets/olx/ads.py b/src/brightdata/datasets/olx/ads.py index ea33529..a173122 100644 --- a/src/brightdata/datasets/olx/ads.py +++ b/src/brightdata/datasets/olx/ads.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class OLXBrazil(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/otodom/properties.py b/src/brightdata/datasets/otodom/properties.py index 5f0104a..57cfe00 100644 --- a/src/brightdata/datasets/otodom/properties.py +++ b/src/brightdata/datasets/otodom/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class OtodomPoland(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/owler/companies.py b/src/brightdata/datasets/owler/companies.py index 76cd6a6..e5a6dcd 100644 --- a/src/brightdata/datasets/owler/companies.py +++ b/src/brightdata/datasets/owler/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -128,40 +128,40 @@ class OwlerCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_employee_revenue_fields() -> List[str]: + def get_employee_revenue_fields() -> list[str]: """Get employee and revenue field names.""" return EMPLOYEE_REVENUE_FIELDS.copy() @staticmethod - def get_location_fields() -> List[str]: + def get_location_fields() -> list[str]: """Get location-related field names.""" return LOCATION_FIELDS.copy() @staticmethod - def get_funding_fields() -> List[str]: + def get_funding_fields() -> list[str]: """Get funding-related field names.""" return FUNDING_FIELDS.copy() @staticmethod - def get_leadership_fields() -> List[str]: + def get_leadership_fields() -> list[str]: """Get leadership-related field names.""" return LEADERSHIP_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "employee_revenue": [], "location": [], @@ -207,22 +207,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "companyId", diff --git a/src/brightdata/datasets/ozon/products.py b/src/brightdata/datasets/ozon/products.py index d3a86e5..3c7e979 100644 --- a/src/brightdata/datasets/ozon/products.py +++ b/src/brightdata/datasets/ozon/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class OzonProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/pinterest/posts.py b/src/brightdata/datasets/pinterest/posts.py index 43666d5..c72c971 100644 --- a/src/brightdata/datasets/pinterest/posts.py +++ b/src/brightdata/datasets/pinterest/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class PinterestPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/pinterest/profiles.py b/src/brightdata/datasets/pinterest/profiles.py index a94c9e6..d0e0c60 100644 --- a/src/brightdata/datasets/pinterest/profiles.py +++ b/src/brightdata/datasets/pinterest/profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class PinterestProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/pitchbook/companies.py b/src/brightdata/datasets/pitchbook/companies.py index 0aae293..aabd32e 100644 --- a/src/brightdata/datasets/pitchbook/companies.py +++ b/src/brightdata/datasets/pitchbook/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -86,35 +86,35 @@ class PitchBookCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company identity field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_deal_fields() -> List[str]: + def get_deal_fields() -> list[str]: """Get deal-related field names.""" return DEAL_FIELDS.copy() @staticmethod - def get_investment_fields() -> List[str]: + def get_investment_fields() -> list[str]: """Get investment field names.""" return INVESTMENT_FIELDS.copy() @staticmethod - def get_ip_fields() -> List[str]: + def get_ip_fields() -> list[str]: """Get intellectual property field names.""" return IP_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "deals": [], "investments": [], @@ -148,22 +148,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "id", diff --git a/src/brightdata/datasets/prada/products.py b/src/brightdata/datasets/prada/products.py index b6c0698..c94aa28 100644 --- a/src/brightdata/datasets/prada/products.py +++ b/src/brightdata/datasets/prada/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class PradaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/properati/properties.py b/src/brightdata/datasets/properati/properties.py index dd59812..6403b2c 100644 --- a/src/brightdata/datasets/properati/properties.py +++ b/src/brightdata/datasets/properati/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ProperatiProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/quora/posts.py b/src/brightdata/datasets/quora/posts.py index 1462a4a..ddfdfa0 100644 --- a/src/brightdata/datasets/quora/posts.py +++ b/src/brightdata/datasets/quora/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class QuoraPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/raymourflanigan/products.py b/src/brightdata/datasets/raymourflanigan/products.py index 0390e72..883ef35 100644 --- a/src/brightdata/datasets/raymourflanigan/products.py +++ b/src/brightdata/datasets/raymourflanigan/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class RaymourFlaniganProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/real_estate/australia.py b/src/brightdata/datasets/real_estate/australia.py index 42c7bb9..ba641f0 100644 --- a/src/brightdata/datasets/real_estate/australia.py +++ b/src/brightdata/datasets/real_estate/australia.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -36,4 +36,4 @@ class AustraliaRealEstate(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/realtor/international_properties.py b/src/brightdata/datasets/realtor/international_properties.py index f8dde86..402dd14 100644 --- a/src/brightdata/datasets/realtor/international_properties.py +++ b/src/brightdata/datasets/realtor/international_properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class RealtorInternationalProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/reddit/__init__.py b/src/brightdata/datasets/reddit/__init__.py index 2c5cab3..f1e7b0d 100644 --- a/src/brightdata/datasets/reddit/__init__.py +++ b/src/brightdata/datasets/reddit/__init__.py @@ -1,6 +1,6 @@ """Reddit datasets.""" -from .posts import RedditPosts from .comments import RedditComments +from .posts import RedditPosts -__all__ = ["RedditPosts", "RedditComments"] +__all__ = ["RedditComments", "RedditPosts"] diff --git a/src/brightdata/datasets/reddit/comments.py b/src/brightdata/datasets/reddit/comments.py index 3b2342f..2582f67 100644 --- a/src/brightdata/datasets/reddit/comments.py +++ b/src/brightdata/datasets/reddit/comments.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class RedditComments(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/reddit/posts.py b/src/brightdata/datasets/reddit/posts.py index c9283a4..52d5226 100644 --- a/src/brightdata/datasets/reddit/posts.py +++ b/src/brightdata/datasets/reddit/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class RedditPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/rona/products.py b/src/brightdata/datasets/rona/products.py index a0ee538..e0075d5 100644 --- a/src/brightdata/datasets/rona/products.py +++ b/src/brightdata/datasets/rona/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class RonaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/sephora/products.py b/src/brightdata/datasets/sephora/products.py index 61410d4..821ca29 100644 --- a/src/brightdata/datasets/sephora/products.py +++ b/src/brightdata/datasets/sephora/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class SephoraProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/shein/products.py b/src/brightdata/datasets/shein/products.py index 800f822..d730aac 100644 --- a/src/brightdata/datasets/shein/products.py +++ b/src/brightdata/datasets/shein/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class SheinProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/shopee/products.py b/src/brightdata/datasets/shopee/products.py index 6b32b66..3499167 100644 --- a/src/brightdata/datasets/shopee/products.py +++ b/src/brightdata/datasets/shopee/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ShopeeProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/sleepnumber/products.py b/src/brightdata/datasets/sleepnumber/products.py index 8584d57..7f1fb3e 100644 --- a/src/brightdata/datasets/sleepnumber/products.py +++ b/src/brightdata/datasets/sleepnumber/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class SleepNumberProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/slintel/companies.py b/src/brightdata/datasets/slintel/companies.py index 63ebbe1..75e61ef 100644 --- a/src/brightdata/datasets/slintel/companies.py +++ b/src/brightdata/datasets/slintel/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -75,30 +75,30 @@ class SlintelCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_tech_fields() -> List[str]: + def get_tech_fields() -> list[str]: """Get technology-related field names.""" return TECH_FIELDS.copy() @staticmethod - def get_social_fields() -> List[str]: + def get_social_fields() -> list[str]: """Get social and news field names.""" return SOCIAL_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "tech": [], "social": [], @@ -128,22 +128,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "id", diff --git a/src/brightdata/datasets/snapchat/posts.py b/src/brightdata/datasets/snapchat/posts.py index 10bc728..bb26861 100644 --- a/src/brightdata/datasets/snapchat/posts.py +++ b/src/brightdata/datasets/snapchat/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class SnapchatPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/tiktok/__init__.py b/src/brightdata/datasets/tiktok/__init__.py index b7b2190..df3dca1 100644 --- a/src/brightdata/datasets/tiktok/__init__.py +++ b/src/brightdata/datasets/tiktok/__init__.py @@ -1,13 +1,13 @@ """TikTok datasets.""" -from .profiles import TikTokProfiles from .comments import TikTokComments from .posts import TikTokPosts +from .profiles import TikTokProfiles from .shop import TikTokShop __all__ = [ - "TikTokProfiles", "TikTokComments", "TikTokPosts", + "TikTokProfiles", "TikTokShop", ] diff --git a/src/brightdata/datasets/tiktok/comments.py b/src/brightdata/datasets/tiktok/comments.py index 1fa22a4..9afbe39 100644 --- a/src/brightdata/datasets/tiktok/comments.py +++ b/src/brightdata/datasets/tiktok/comments.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class TikTokComments(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/tiktok/posts.py b/src/brightdata/datasets/tiktok/posts.py index 8b9fd7b..4d6b58d 100644 --- a/src/brightdata/datasets/tiktok/posts.py +++ b/src/brightdata/datasets/tiktok/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class TikTokPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/tiktok/profiles.py b/src/brightdata/datasets/tiktok/profiles.py index e94e657..4c2a943 100644 --- a/src/brightdata/datasets/tiktok/profiles.py +++ b/src/brightdata/datasets/tiktok/profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -36,4 +36,4 @@ class TikTokProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/tiktok/shop.py b/src/brightdata/datasets/tiktok/shop.py index 0cf46f6..0daa0dc 100644 --- a/src/brightdata/datasets/tiktok/shop.py +++ b/src/brightdata/datasets/tiktok/shop.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class TikTokShop(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/toctoc/properties.py b/src/brightdata/datasets/toctoc/properties.py index 4eeb34c..bda26e2 100644 --- a/src/brightdata/datasets/toctoc/properties.py +++ b/src/brightdata/datasets/toctoc/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ToctocProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/tokopedia/products.py b/src/brightdata/datasets/tokopedia/products.py index bcd33b5..d9e78c7 100644 --- a/src/brightdata/datasets/tokopedia/products.py +++ b/src/brightdata/datasets/tokopedia/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class TokopediaProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/toysrus/products.py b/src/brightdata/datasets/toysrus/products.py index f0ad329..68a16ca 100644 --- a/src/brightdata/datasets/toysrus/products.py +++ b/src/brightdata/datasets/toysrus/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ToysRUsProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/trustpilot/reviews.py b/src/brightdata/datasets/trustpilot/reviews.py index 25672d1..aa8f194 100644 --- a/src/brightdata/datasets/trustpilot/reviews.py +++ b/src/brightdata/datasets/trustpilot/reviews.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -98,35 +98,35 @@ class TrustpilotReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_rating_distribution_fields() -> List[str]: + def get_rating_distribution_fields() -> list[str]: """Get rating distribution field names.""" return RATING_DISTRIBUTION_FIELDS.copy() @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """Get review content field names.""" return REVIEW_FIELDS.copy() @staticmethod - def get_reviewer_fields() -> List[str]: + def get_reviewer_fields() -> list[str]: """Get reviewer-related field names.""" return REVIEWER_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "rating_distribution": [], "review": [], @@ -160,22 +160,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "review_id", diff --git a/src/brightdata/datasets/trustradius/reviews.py b/src/brightdata/datasets/trustradius/reviews.py index ed3f18e..1e59dd8 100644 --- a/src/brightdata/datasets/trustradius/reviews.py +++ b/src/brightdata/datasets/trustradius/reviews.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -115,40 +115,40 @@ class TrustRadiusReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_product_fields() -> List[str]: + def get_product_fields() -> list[str]: """Get product-related field names.""" return PRODUCT_FIELDS.copy() @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """Get review-related field names.""" return REVIEW_FIELDS.copy() @staticmethod - def get_author_fields() -> List[str]: + def get_author_fields() -> list[str]: """Get author-related field names.""" return AUTHOR_FIELDS.copy() @staticmethod - def get_content_fields() -> List[str]: + def get_content_fields() -> list[str]: """Get content-related field names.""" return CONTENT_FIELDS.copy() @staticmethod - def get_product_details_fields() -> List[str]: + def get_product_details_fields() -> list[str]: """Get product details field names.""" return PRODUCT_DETAILS_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "product": [], "review": [], "author": [], @@ -186,22 +186,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "review_id", diff --git a/src/brightdata/datasets/utils.py b/src/brightdata/datasets/utils.py index c5b70a5..b418e6b 100644 --- a/src/brightdata/datasets/utils.py +++ b/src/brightdata/datasets/utils.py @@ -2,15 +2,15 @@ Dataset utilities - helpers for exporting and processing dataset results. """ -import json import csv +import json from pathlib import Path -from typing import List, Dict, Any, Union, Optional +from typing import Any def export_json( - data: List[Dict[str, Any]], - filepath: Union[str, Path], + data: list[dict[str, Any]], + filepath: str | Path, indent: int = 2, ) -> Path: """ @@ -31,8 +31,8 @@ def export_json( def export_jsonl( - data: List[Dict[str, Any]], - filepath: Union[str, Path], + data: list[dict[str, Any]], + filepath: str | Path, ) -> Path: """ Export dataset results to JSONL (newline-delimited JSON) file. @@ -46,15 +46,14 @@ def export_jsonl( """ filepath = Path(filepath) with open(filepath, "w", encoding="utf-8") as f: - for record in data: - f.write(json.dumps(record, default=str, ensure_ascii=False) + "\n") + f.writelines(json.dumps(record, default=str, ensure_ascii=False) + "\n" for record in data) return filepath def export_csv( - data: List[Dict[str, Any]], - filepath: Union[str, Path], - fields: Optional[List[str]] = None, + data: list[dict[str, Any]], + filepath: str | Path, + fields: list[str] | None = None, flatten_nested: bool = True, ) -> Path: """ @@ -101,8 +100,8 @@ def export_csv( def export( - data: List[Dict[str, Any]], - filepath: Union[str, Path], + data: list[dict[str, Any]], + filepath: str | Path, **kwargs, ) -> Path: """ diff --git a/src/brightdata/datasets/ventureradar/companies.py b/src/brightdata/datasets/ventureradar/companies.py index d70f5ef..dbd7571 100644 --- a/src/brightdata/datasets/ventureradar/companies.py +++ b/src/brightdata/datasets/ventureradar/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -87,35 +87,35 @@ class VentureRadarCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company-related field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_scores_fields() -> List[str]: + def get_scores_fields() -> list[str]: """Get scores and metrics field names.""" return SCORES_FIELDS.copy() @staticmethod - def get_contact_fields() -> List[str]: + def get_contact_fields() -> list[str]: """Get contact and social field names.""" return CONTACT_FIELDS.copy() @staticmethod - def get_intelligence_fields() -> List[str]: + def get_intelligence_fields() -> list[str]: """Get business intelligence field names.""" return INTELLIGENCE_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "scores": [], "contact": [], @@ -149,22 +149,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "company_id", diff --git a/src/brightdata/datasets/vimeo/videos.py b/src/brightdata/datasets/vimeo/videos.py index 6c226f1..ec41e78 100644 --- a/src/brightdata/datasets/vimeo/videos.py +++ b/src/brightdata/datasets/vimeo/videos.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class VimeoVideos(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/walmart/products.py b/src/brightdata/datasets/walmart/products.py index c6d8f57..2e1f9e3 100644 --- a/src/brightdata/datasets/walmart/products.py +++ b/src/brightdata/datasets/walmart/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -36,4 +36,4 @@ class WalmartProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/walmart/sellers.py b/src/brightdata/datasets/walmart/sellers.py index 3dbb556..c07412c 100644 --- a/src/brightdata/datasets/walmart/sellers.py +++ b/src/brightdata/datasets/walmart/sellers.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class WalmartSellersInfo(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/wayfair/products.py b/src/brightdata/datasets/wayfair/products.py index edd1df1..f3c94ac 100644 --- a/src/brightdata/datasets/wayfair/products.py +++ b/src/brightdata/datasets/wayfair/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class WayfairProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/webmotors/vehicles.py b/src/brightdata/datasets/webmotors/vehicles.py index 3b72aa4..0c424fe 100644 --- a/src/brightdata/datasets/webmotors/vehicles.py +++ b/src/brightdata/datasets/webmotors/vehicles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class WebmotorsBrasil(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/wikipedia/articles.py b/src/brightdata/datasets/wikipedia/articles.py index 607f1b3..b78a507 100644 --- a/src/brightdata/datasets/wikipedia/articles.py +++ b/src/brightdata/datasets/wikipedia/articles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class WikipediaArticles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/wildberries/products.py b/src/brightdata/datasets/wildberries/products.py index 52f13d8..e8df20a 100644 --- a/src/brightdata/datasets/wildberries/products.py +++ b/src/brightdata/datasets/wildberries/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class WildberriesProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/world_population/countries.py b/src/brightdata/datasets/world_population/countries.py index a406b23..3e80f82 100644 --- a/src/brightdata/datasets/world_population/countries.py +++ b/src/brightdata/datasets/world_population/countries.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any from ..base import BaseDataset @@ -34,7 +34,7 @@ class WorldPopulation(BaseDataset): NAME = "world_population" # All available fields with metadata - FIELDS: Dict[str, Dict[str, Any]] = { + FIELDS: dict[str, dict[str, Any]] = { # Country identification "url": { "type": "url", diff --git a/src/brightdata/datasets/world_zipcodes/zipcodes.py b/src/brightdata/datasets/world_zipcodes/zipcodes.py index 81e4a5e..ce7292f 100644 --- a/src/brightdata/datasets/world_zipcodes/zipcodes.py +++ b/src/brightdata/datasets/world_zipcodes/zipcodes.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class WorldZipcodes(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/x_twitter/posts.py b/src/brightdata/datasets/x_twitter/posts.py index 4cc05d3..d300607 100644 --- a/src/brightdata/datasets/x_twitter/posts.py +++ b/src/brightdata/datasets/x_twitter/posts.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class XTwitterPosts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/x_twitter/profiles.py b/src/brightdata/datasets/x_twitter/profiles.py index c5e4f2b..a63cead 100644 --- a/src/brightdata/datasets/x_twitter/profiles.py +++ b/src/brightdata/datasets/x_twitter/profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class XTwitterProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/xing/profiles.py b/src/brightdata/datasets/xing/profiles.py index 04ed868..f84d0b1 100644 --- a/src/brightdata/datasets/xing/profiles.py +++ b/src/brightdata/datasets/xing/profiles.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -77,30 +77,30 @@ class XingProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_profile_fields() -> List[str]: + def get_profile_fields() -> list[str]: """Get profile-related field names.""" return PROFILE_FIELDS.copy() @staticmethod - def get_experience_fields() -> List[str]: + def get_experience_fields() -> list[str]: """Get experience and education field names.""" return EXPERIENCE_FIELDS.copy() @staticmethod - def get_social_fields() -> List[str]: + def get_social_fields() -> list[str]: """Get social networking field names.""" return SOCIAL_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "profile": [], "experience": [], "social": [], @@ -130,22 +130,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "account_id", diff --git a/src/brightdata/datasets/yahoo_finance/businesses.py b/src/brightdata/datasets/yahoo_finance/businesses.py index 2aef7f7..3d96c9a 100644 --- a/src/brightdata/datasets/yahoo_finance/businesses.py +++ b/src/brightdata/datasets/yahoo_finance/businesses.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class YahooFinanceBusinesses(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/yapo/ads.py b/src/brightdata/datasets/yapo/ads.py index b666fd8..7db7b11 100644 --- a/src/brightdata/datasets/yapo/ads.py +++ b/src/brightdata/datasets/yapo/ads.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class YapoChile(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/yelp/businesses.py b/src/brightdata/datasets/yelp/businesses.py index d350fb7..19d38aa 100644 --- a/src/brightdata/datasets/yelp/businesses.py +++ b/src/brightdata/datasets/yelp/businesses.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -92,40 +92,40 @@ class YelpBusinesses(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_business_fields() -> List[str]: + def get_business_fields() -> list[str]: """Get business identity field names.""" return BUSINESS_FIELDS.copy() @staticmethod - def get_location_fields() -> List[str]: + def get_location_fields() -> list[str]: """Get location-related field names.""" return LOCATION_FIELDS.copy() @staticmethod - def get_contact_fields() -> List[str]: + def get_contact_fields() -> list[str]: """Get contact information field names.""" return CONTACT_FIELDS.copy() @staticmethod - def get_rating_fields() -> List[str]: + def get_rating_fields() -> list[str]: """Get rating-related field names.""" return RATING_FIELDS.copy() @staticmethod - def get_content_fields() -> List[str]: + def get_content_fields() -> list[str]: """Get content field names.""" return CONTENT_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "business": [], "location": [], "contact": [], @@ -163,22 +163,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "business_id", diff --git a/src/brightdata/datasets/yelp/reviews.py b/src/brightdata/datasets/yelp/reviews.py index f8355c4..75faa40 100644 --- a/src/brightdata/datasets/yelp/reviews.py +++ b/src/brightdata/datasets/yelp/reviews.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -74,30 +74,30 @@ class YelpReviews(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_business_fields() -> List[str]: + def get_business_fields() -> list[str]: """Get business-related field names.""" return BUSINESS_FIELDS.copy() @staticmethod - def get_review_fields() -> List[str]: + def get_review_fields() -> list[str]: """Get review content field names.""" return REVIEW_FIELDS.copy() @staticmethod - def get_reviewer_fields() -> List[str]: + def get_reviewer_fields() -> list[str]: """Get reviewer-related field names.""" return REVIEWER_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "business": [], "review": [], "reviewer": [], @@ -127,22 +127,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "review_id", diff --git a/src/brightdata/datasets/youtube/__init__.py b/src/brightdata/datasets/youtube/__init__.py index 29d8144..2d9a372 100644 --- a/src/brightdata/datasets/youtube/__init__.py +++ b/src/brightdata/datasets/youtube/__init__.py @@ -1,7 +1,7 @@ """YouTube datasets.""" +from .comments import YouTubeComments from .profiles import YouTubeProfiles from .videos import YouTubeVideos -from .comments import YouTubeComments -__all__ = ["YouTubeProfiles", "YouTubeVideos", "YouTubeComments"] +__all__ = ["YouTubeComments", "YouTubeProfiles", "YouTubeVideos"] diff --git a/src/brightdata/datasets/youtube/comments.py b/src/brightdata/datasets/youtube/comments.py index 28005c2..0ecb15b 100644 --- a/src/brightdata/datasets/youtube/comments.py +++ b/src/brightdata/datasets/youtube/comments.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class YouTubeComments(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/youtube/profiles.py b/src/brightdata/datasets/youtube/profiles.py index 164a98b..e82379b 100644 --- a/src/brightdata/datasets/youtube/profiles.py +++ b/src/brightdata/datasets/youtube/profiles.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class YouTubeProfiles(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/youtube/videos.py b/src/brightdata/datasets/youtube/videos.py index 5e9ec56..c990feb 100644 --- a/src/brightdata/datasets/youtube/videos.py +++ b/src/brightdata/datasets/youtube/videos.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class YouTubeVideos(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/ysl/products.py b/src/brightdata/datasets/ysl/products.py index c7f4b73..a46aa46 100644 --- a/src/brightdata/datasets/ysl/products.py +++ b/src/brightdata/datasets/ysl/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class YSLProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zalando/products.py b/src/brightdata/datasets/zalando/products.py index a214632..6c8acc6 100644 --- a/src/brightdata/datasets/zalando/products.py +++ b/src/brightdata/datasets/zalando/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZalandoProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zara/__init__.py b/src/brightdata/datasets/zara/__init__.py index 504dd15..d2bdbab 100644 --- a/src/brightdata/datasets/zara/__init__.py +++ b/src/brightdata/datasets/zara/__init__.py @@ -1,6 +1,6 @@ """Zara datasets.""" -from .products import ZaraProducts from .home_products import ZaraHomeProducts +from .products import ZaraProducts -__all__ = ["ZaraProducts", "ZaraHomeProducts"] +__all__ = ["ZaraHomeProducts", "ZaraProducts"] diff --git a/src/brightdata/datasets/zara/home_products.py b/src/brightdata/datasets/zara/home_products.py index 28d3c19..37c2a31 100644 --- a/src/brightdata/datasets/zara/home_products.py +++ b/src/brightdata/datasets/zara/home_products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZaraHomeProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zara/products.py b/src/brightdata/datasets/zara/products.py index 978c782..c93c977 100644 --- a/src/brightdata/datasets/zara/products.py +++ b/src/brightdata/datasets/zara/products.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZaraProducts(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zillow/__init__.py b/src/brightdata/datasets/zillow/__init__.py index 23b0254..c62f600 100644 --- a/src/brightdata/datasets/zillow/__init__.py +++ b/src/brightdata/datasets/zillow/__init__.py @@ -1,9 +1,9 @@ """Zillow datasets.""" -from .properties import ZillowProperties from .price_history import ZillowPriceHistory +from .properties import ZillowProperties __all__ = [ - "ZillowProperties", "ZillowPriceHistory", + "ZillowProperties", ] diff --git a/src/brightdata/datasets/zillow/price_history.py b/src/brightdata/datasets/zillow/price_history.py index e0e31a3..c80a26d 100644 --- a/src/brightdata/datasets/zillow/price_history.py +++ b/src/brightdata/datasets/zillow/price_history.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZillowPriceHistory(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zillow/properties.py b/src/brightdata/datasets/zillow/properties.py index 7107ded..5cb0e7d 100644 --- a/src/brightdata/datasets/zillow/properties.py +++ b/src/brightdata/datasets/zillow/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZillowProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zonaprop/properties.py b/src/brightdata/datasets/zonaprop/properties.py index bae3a73..14efc20 100644 --- a/src/brightdata/datasets/zonaprop/properties.py +++ b/src/brightdata/datasets/zonaprop/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZonapropArgentina(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/datasets/zoominfo/companies.py b/src/brightdata/datasets/zoominfo/companies.py index b23c72a..5594b57 100644 --- a/src/brightdata/datasets/zoominfo/companies.py +++ b/src/brightdata/datasets/zoominfo/companies.py @@ -7,7 +7,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -107,40 +107,40 @@ class ZoomInfoCompanies(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None @staticmethod - def get_company_fields() -> List[str]: + def get_company_fields() -> list[str]: """Get company identity field names.""" return COMPANY_FIELDS.copy() @staticmethod - def get_financial_fields() -> List[str]: + def get_financial_fields() -> list[str]: """Get financial-related field names.""" return FINANCIAL_FIELDS.copy() @staticmethod - def get_employee_fields() -> List[str]: + def get_employee_fields() -> list[str]: """Get employee count field names.""" return EMPLOYEE_FIELDS.copy() @staticmethod - def get_leadership_fields() -> List[str]: + def get_leadership_fields() -> list[str]: """Get leadership and contact field names.""" return LEADERSHIP_FIELDS.copy() @staticmethod - def get_tech_fields() -> List[str]: + def get_tech_fields() -> list[str]: """Get technology-related field names.""" return TECH_FIELDS.copy() - async def get_fields_by_category(self) -> Dict[str, List[str]]: + async def get_fields_by_category(self) -> dict[str, list[str]]: """Get all fields grouped by category.""" if self._fields_by_category is not None: return self._fields_by_category metadata = await self.get_metadata() - result: Dict[str, List[str]] = { + result: dict[str, list[str]] = { "company": [], "financial": [], "employees": [], @@ -178,22 +178,24 @@ async def get_fields_by_category(self) -> Dict[str, List[str]]: self._fields_by_category = result return result - async def search_fields(self, keyword: str) -> List[str]: + async def search_fields(self, keyword: str) -> list[str]: """Search for fields containing a keyword.""" metadata = await self.get_metadata() keyword_lower = keyword.lower() matches = [] for name, field_info in metadata.fields.items(): - if keyword_lower in name.lower(): - matches.append(name) - elif field_info.description and keyword_lower in field_info.description.lower(): + if ( + keyword_lower in name.lower() + or field_info.description + and keyword_lower in field_info.description.lower() + ): matches.append(name) return sorted(matches) @staticmethod - def get_identifier_fields() -> List[str]: + def get_identifier_fields() -> list[str]: """Get fields that can be used as unique identifiers.""" return [ "id", diff --git a/src/brightdata/datasets/zoopla/properties.py b/src/brightdata/datasets/zoopla/properties.py index 5cd299e..f818581 100644 --- a/src/brightdata/datasets/zoopla/properties.py +++ b/src/brightdata/datasets/zoopla/properties.py @@ -6,7 +6,7 @@ Use get_metadata() to discover all available fields dynamically. """ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from ..base import BaseDataset @@ -22,4 +22,4 @@ class ZooplaProperties(BaseDataset): def __init__(self, engine: "AsyncEngine"): super().__init__(engine) - self._fields_by_category: Optional[Dict[str, List[str]]] = None + self._fields_by_category: dict[str, list[str]] | None = None diff --git a/src/brightdata/discover/__init__.py b/src/brightdata/discover/__init__.py index afd4abf..d4de22b 100644 --- a/src/brightdata/discover/__init__.py +++ b/src/brightdata/discover/__init__.py @@ -1,6 +1,6 @@ """Discover API — AI-powered web search with relevance ranking.""" +from .models import DiscoverJob, DiscoverResult from .service import DiscoverService -from .models import DiscoverResult, DiscoverJob -__all__ = ["DiscoverService", "DiscoverResult", "DiscoverJob"] +__all__ = ["DiscoverJob", "DiscoverResult", "DiscoverService"] diff --git a/src/brightdata/discover/models.py b/src/brightdata/discover/models.py index 70a2731..d0b1ebb 100644 --- a/src/brightdata/discover/models.py +++ b/src/brightdata/discover/models.py @@ -5,7 +5,7 @@ import asyncio import time from dataclasses import dataclass, field -from typing import Any, Optional, List, Dict, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from ..models import BaseResult @@ -31,11 +31,11 @@ class DiscoverResult(BaseResult): """ query: str = "" - intent: Optional[str] = None - data: Optional[List[Dict[str, Any]]] = None - duration_seconds: Optional[float] = None - total_results: Optional[int] = None - task_id: Optional[str] = None + intent: str | None = None + data: list[dict[str, Any]] | None = None + duration_seconds: float | None = None + total_results: int | None = None + task_id: str | None = None def __repr__(self) -> str: """String representation with query info.""" @@ -63,7 +63,7 @@ class DiscoverSnapshot: task_id: str query: str = "" - intent: Optional[str] = None + intent: str | None = None def __repr__(self) -> str: return f"" @@ -87,9 +87,9 @@ class DiscoverJob: """ task_id: str - _service: "DiscoverService" = field(repr=False) + _service: DiscoverService = field(repr=False) query: str = "" - intent: Optional[str] = None + intent: str | None = None def __repr__(self) -> str: return f"" @@ -138,7 +138,7 @@ async def wait(self, timeout: int = 60, poll_interval: int = 2) -> str: await asyncio.sleep(poll_interval) - async def fetch(self) -> List[Dict[str, Any]]: + async def fetch(self) -> list[dict[str, Any]]: """ Fetch results. Call after wait() returns 'done'. diff --git a/src/brightdata/discover/service.py b/src/brightdata/discover/service.py index 714409d..3caae6f 100644 --- a/src/brightdata/discover/service.py +++ b/src/brightdata/discover/service.py @@ -7,12 +7,12 @@ import asyncio import time -from typing import Optional, List, Dict, Any from datetime import datetime, timezone +from typing import Any from ..core.engine import AsyncEngine from ..exceptions import APIError -from .models import DiscoverResult, DiscoverJob +from .models import DiscoverJob, DiscoverResult class DiscoverService: @@ -39,13 +39,13 @@ def __init__(self, engine: AsyncEngine): async def search( self, query: str, - intent: Optional[str] = None, + intent: str | None = None, include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, + country: str | None = None, + city: str | None = None, + language: str | None = None, + filter_keywords: list[str] | None = None, + num_results: int | None = None, format: str = "json", timeout: int = 60, poll_interval: int = 2, @@ -110,13 +110,13 @@ async def search( async def trigger( self, query: str, - intent: Optional[str] = None, + intent: str | None = None, include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, + country: str | None = None, + city: str | None = None, + language: str | None = None, + filter_keywords: list[str] | None = None, + num_results: int | None = None, format: str = "json", ) -> DiscoverJob: """ @@ -173,7 +173,7 @@ async def status(self, task_id: str) -> str: response_data = await self._poll_once(task_id) return response_data.get("status", "processing") - async def fetch(self, task_id: str) -> List[Dict[str, Any]]: + async def fetch(self, task_id: str) -> list[dict[str, Any]]: """Fetch a discover task's results by task_id. Call after status == 'done'.""" response_data = await self._poll_once(task_id) return response_data.get("results", []) @@ -219,17 +219,17 @@ async def to_result( async def _trigger( self, query: str, - intent: Optional[str] = None, + intent: str | None = None, include_content: bool = False, - country: Optional[str] = None, - city: Optional[str] = None, - language: Optional[str] = None, - filter_keywords: Optional[List[str]] = None, - num_results: Optional[int] = None, + country: str | None = None, + city: str | None = None, + language: str | None = None, + filter_keywords: list[str] | None = None, + num_results: int | None = None, format: str = "json", ) -> str: """POST /discover, return task_id.""" - payload: Dict[str, Any] = {"query": query} + payload: dict[str, Any] = {"query": query} if intent: payload["intent"] = intent @@ -262,7 +262,7 @@ async def _trigger( raise APIError("No task_id in discover response") return task_id - async def _poll_once(self, task_id: str) -> Dict[str, Any]: + async def _poll_once(self, task_id: str) -> dict[str, Any]: """GET /discover?task_id=, return response data.""" async with self._engine.get("/discover", params={"task_id": task_id}) as response: if response.status >= 400: @@ -275,7 +275,7 @@ async def _poll_once(self, task_id: str) -> Dict[str, Any]: async def _poll_until_done( self, task_id: str, timeout: int, poll_interval: int - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """Poll GET /discover?task_id= until done or timeout.""" start = time.time() diff --git a/src/brightdata/exceptions/__init__.py b/src/brightdata/exceptions/__init__.py index 9974381..324e031 100644 --- a/src/brightdata/exceptions/__init__.py +++ b/src/brightdata/exceptions/__init__.py @@ -1,23 +1,23 @@ """Exception classes for Bright Data SDK.""" from .errors import ( - BrightDataError, - ValidationError, - AuthenticationError, APIError, + AuthenticationError, + BrightDataError, DataNotReadyError, - ZoneError, NetworkError, SSLError, + ValidationError, + ZoneError, ) __all__ = [ - "BrightDataError", - "ValidationError", - "AuthenticationError", "APIError", + "AuthenticationError", + "BrightDataError", "DataNotReadyError", - "ZoneError", "NetworkError", "SSLError", + "ValidationError", + "ZoneError", ] diff --git a/src/brightdata/exceptions/errors.py b/src/brightdata/exceptions/errors.py index fc476d3..85c7633 100644 --- a/src/brightdata/exceptions/errors.py +++ b/src/brightdata/exceptions/errors.py @@ -14,14 +14,10 @@ def __init__(self, message: str, *args, **kwargs): class ValidationError(BrightDataError): """Input validation failed.""" - pass - class AuthenticationError(BrightDataError): """Authentication or authorization failed.""" - pass - class APIError(BrightDataError): """API request failed.""" @@ -42,20 +38,14 @@ def __init__( class DataNotReadyError(BrightDataError): """Data is not ready yet (HTTP 202). Should retry.""" - pass - class ZoneError(BrightDataError): """Zone operation failed.""" - pass - class NetworkError(BrightDataError): """Network connectivity issue.""" - pass - class SSLError(BrightDataError): """ @@ -63,5 +53,3 @@ class SSLError(BrightDataError): Common on macOS where Python doesn't have access to system certificates. """ - - pass diff --git a/src/brightdata/models.py b/src/brightdata/models.py index d2ebee1..e2ea37e 100644 --- a/src/brightdata/models.py +++ b/src/brightdata/models.py @@ -2,11 +2,11 @@ from __future__ import annotations -from dataclasses import dataclass, field, asdict -from datetime import datetime -from typing import Any, Optional, List, Dict, Union, Literal import json +from dataclasses import asdict, dataclass, field +from datetime import datetime from pathlib import Path +from typing import Any, Literal, Optional, Union StatusType = Literal["ready", "error", "timeout", "in_progress"] PlatformType = Optional[Literal["linkedin", "amazon", "chatgpt", "instagram", "facebook"]] @@ -30,17 +30,17 @@ class BaseResult: """ success: bool - cost: Optional[float] = None - error: Optional[str] = None - trigger_sent_at: Optional[datetime] = None - data_fetched_at: Optional[datetime] = None + cost: float | None = None + error: str | None = None + trigger_sent_at: datetime | None = None + data_fetched_at: datetime | None = None def __post_init__(self) -> None: """Validate data after initialization.""" if self.cost is not None and self.cost < 0: raise ValueError(f"Cost must be non-negative, got {self.cost}") - def elapsed_ms(self) -> Optional[float]: + def elapsed_ms(self) -> float | None: """ Calculate total elapsed time in milliseconds. @@ -52,7 +52,7 @@ def elapsed_ms(self) -> Optional[float]: return delta.total_seconds() * 1000 return None - def get_timing_breakdown(self) -> Dict[str, Optional[Union[float, str]]]: + def get_timing_breakdown(self) -> dict[str, float | str | None]: """ Get detailed timing breakdown for debugging and optimization. @@ -68,7 +68,7 @@ def get_timing_breakdown(self) -> Dict[str, Optional[Union[float, str]]]: "data_fetched_at": self.data_fetched_at.isoformat() if self.data_fetched_at else None, } - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Convert result to dictionary for serialization. @@ -85,7 +85,7 @@ def to_dict(self) -> Dict[str, Any]: result[key] = [v.isoformat() if isinstance(v, datetime) else v for v in value] return result - def to_json(self, indent: Optional[int] = None) -> str: + def to_json(self, indent: int | None = None) -> str: """ Serialize result to JSON string. @@ -100,7 +100,7 @@ def to_json(self, indent: Optional[int] = None) -> str: """ return json.dumps(self.to_dict(), indent=indent, default=str) - def save_to_file(self, filepath: Union[str, Path], format: str = "json") -> None: + def save_to_file(self, filepath: str | Path, format: str = "json") -> None: """ Save result data to file. @@ -159,16 +159,16 @@ class ScrapeResult(BaseResult): url: str = "" status: StatusType = "ready" - data: Optional[Any] = None - snapshot_id: Optional[str] = None + data: Any | None = None + snapshot_id: str | None = None platform: PlatformType = None - method: Optional[str] = None - root_domain: Optional[str] = None - snapshot_id_received_at: Optional[datetime] = None - snapshot_polled_at: List[datetime] = field(default_factory=list) - html_char_size: Optional[int] = None - row_count: Optional[int] = None - field_count: Optional[int] = None + method: str | None = None + root_domain: str | None = None + snapshot_id_received_at: datetime | None = None + snapshot_polled_at: list[datetime] = field(default_factory=list) + html_char_size: int | None = None + row_count: int | None = None + field_count: int | None = None def __post_init__(self) -> None: """Validate ScrapeResult-specific fields.""" @@ -184,7 +184,7 @@ def __post_init__(self) -> None: if self.field_count is not None and self.field_count < 0: raise ValueError(f"field_count must be non-negative, got {self.field_count}") - def get_timing_breakdown(self) -> Dict[str, Optional[Union[float, str, int]]]: + def get_timing_breakdown(self) -> dict[str, float | str | int | None]: """ Get detailed timing breakdown including polling information. @@ -243,17 +243,17 @@ class SearchResult(BaseResult): results_per_page: Number of results per page. """ - query: Dict[str, Any] = field(default_factory=dict) - data: Optional[List[Dict[str, Any]]] = None - total_found: Optional[int] = None + query: dict[str, Any] = field(default_factory=dict) + data: list[dict[str, Any]] | None = None + total_found: int | None = None search_engine: SearchEngineType = None - country: Optional[str] = None - page: Optional[int] = None - results_per_page: Optional[int] = None + country: str | None = None + page: int | None = None + results_per_page: int | None = None # Populated when the underlying API returns raw HTML instead of (or # alongside) structured results — e.g. Yandex, which has no parser, or # Google/Bing when the zone is misconfigured. - raw_html: Optional[str] = None + raw_html: str | None = None def __post_init__(self) -> None: """Validate SearchResult-specific fields.""" @@ -293,15 +293,15 @@ class CrawlResult(BaseResult): crawl_completed_at: Timestamp when crawl completed. """ - domain: Optional[str] = None - pages: List[Dict[str, Any]] = field(default_factory=list) - total_pages: Optional[int] = None - depth: Optional[int] = None - start_url: Optional[str] = None - filter_pattern: Optional[str] = None - exclude_pattern: Optional[str] = None - crawl_started_at: Optional[datetime] = None - crawl_completed_at: Optional[datetime] = None + domain: str | None = None + pages: list[dict[str, Any]] = field(default_factory=list) + total_pages: int | None = None + depth: int | None = None + start_url: str | None = None + filter_pattern: str | None = None + exclude_pattern: str | None = None + crawl_started_at: datetime | None = None + crawl_completed_at: datetime | None = None def __post_init__(self) -> None: """Validate CrawlResult-specific fields.""" @@ -311,7 +311,7 @@ def __post_init__(self) -> None: if self.depth is not None and self.depth < 0: raise ValueError(f"depth must be non-negative, got {self.depth}") - def get_timing_breakdown(self) -> Dict[str, Optional[Union[float, str]]]: + def get_timing_breakdown(self) -> dict[str, float | str | None]: """ Get detailed timing breakdown including crawl duration. diff --git a/src/brightdata/payloads.py b/src/brightdata/payloads.py index e16932f..234c2b1 100644 --- a/src/brightdata/payloads.py +++ b/src/brightdata/payloads.py @@ -14,9 +14,9 @@ from __future__ import annotations -from dataclasses import dataclass, field, asdict -from typing import Optional, List, Dict, Any import re +from dataclasses import asdict, dataclass, field +from typing import Any from urllib.parse import urlparse # ============================================================================ @@ -28,7 +28,7 @@ class BasePayload: """Base class for all payloads with common validation.""" - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Convert payload to dictionary for API calls. @@ -48,7 +48,6 @@ def validate(self) -> None: Raises: ValueError: If validation fails. """ - pass @dataclass @@ -105,8 +104,8 @@ class AmazonProductPayload(URLPayload): >>> print(payload.asin) # "B0CRMZHDG8" """ - reviews_count: Optional[int] = None - images_count: Optional[int] = None + reviews_count: int | None = None + images_count: int | None = None def __post_init__(self): """Validate Amazon-specific fields.""" @@ -122,7 +121,7 @@ def __post_init__(self): raise ValueError(f"images_count must be non-negative, got {self.images_count}") @property - def asin(self) -> Optional[str]: + def asin(self) -> str | None: """Extract ASIN (Amazon Standard Identification Number) from URL.""" match = re.search(r"/dp/([A-Z0-9]{10})", self.url) return match.group(1) if match else None @@ -153,9 +152,9 @@ class AmazonReviewPayload(URLPayload): ... ) """ - pastDays: Optional[int] = None - keyWord: Optional[str] = None - numOfReviews: Optional[int] = None + pastDays: int | None = None + keyWord: str | None = None + numOfReviews: int | None = None def __post_init__(self): """Validate Amazon review fields.""" @@ -308,11 +307,11 @@ class LinkedInProfileSearchPayload(BasePayload): """ firstName: str - lastName: Optional[str] = None - title: Optional[str] = None - company: Optional[str] = None - location: Optional[str] = None - max_results: Optional[int] = None + lastName: str | None = None + title: str | None = None + company: str | None = None + location: str | None = None + max_results: int | None = None def __post_init__(self): """Validate profile search fields.""" @@ -354,16 +353,16 @@ class LinkedInJobSearchPayload(BasePayload): ... ) """ - url: Optional[str] = None - keyword: Optional[str] = None - location: Optional[str] = None - country: Optional[str] = None - timeRange: Optional[str] = None - jobType: Optional[str] = None - experienceLevel: Optional[str] = None - remote: Optional[bool] = None - company: Optional[str] = None - locationRadius: Optional[str] = None + url: str | None = None + keyword: str | None = None + location: str | None = None + country: str | None = None + timeRange: str | None = None + jobType: str | None = None + experienceLevel: str | None = None + remote: bool | None = None + company: str | None = None + locationRadius: str | None = None def __post_init__(self): """Validate job search fields.""" @@ -406,8 +405,8 @@ class LinkedInPostSearchPayload(URLPayload): ... ) """ - start_date: Optional[str] = None - end_date: Optional[str] = None + start_date: str | None = None + end_date: str | None = None def __post_init__(self): """Validate post search fields.""" @@ -452,7 +451,7 @@ class ChatGPTPromptPayload(BasePayload): prompt: str country: str = "US" web_search: bool = False - additional_prompt: Optional[str] = None + additional_prompt: str | None = None def __post_init__(self): """Validate ChatGPT prompt fields.""" @@ -502,10 +501,10 @@ class FacebookPostsProfilePayload(URLPayload): ... ) """ - num_of_posts: Optional[int] = None - posts_to_not_include: Optional[List[str]] = field(default_factory=list) - start_date: Optional[str] = None - end_date: Optional[str] = None + num_of_posts: int | None = None + posts_to_not_include: list[str] | None = field(default_factory=list) + start_date: str | None = None + end_date: str | None = None def __post_init__(self): """Validate Facebook posts payload.""" @@ -545,10 +544,10 @@ class FacebookPostsGroupPayload(URLPayload): ... ) """ - num_of_posts: Optional[int] = None - posts_to_not_include: Optional[List[str]] = field(default_factory=list) - start_date: Optional[str] = None - end_date: Optional[str] = None + num_of_posts: int | None = None + posts_to_not_include: list[str] | None = field(default_factory=list) + start_date: str | None = None + end_date: str | None = None def __post_init__(self): """Validate Facebook group payload.""" @@ -605,10 +604,10 @@ class FacebookCommentsPayload(URLPayload): ... ) """ - num_of_comments: Optional[int] = None - comments_to_not_include: Optional[List[str]] = field(default_factory=list) - start_date: Optional[str] = None - end_date: Optional[str] = None + num_of_comments: int | None = None + comments_to_not_include: list[str] | None = field(default_factory=list) + start_date: str | None = None + end_date: str | None = None def __post_init__(self): """Validate Facebook comments payload.""" @@ -640,10 +639,10 @@ class FacebookReelsPayload(URLPayload): ... ) """ - num_of_posts: Optional[int] = None - posts_to_not_include: Optional[List[str]] = field(default_factory=list) - start_date: Optional[str] = None - end_date: Optional[str] = None + num_of_posts: int | None = None + posts_to_not_include: list[str] | None = field(default_factory=list) + start_date: str | None = None + end_date: str | None = None def __post_init__(self): """Validate Facebook reels payload.""" @@ -780,11 +779,11 @@ class InstagramPostsDiscoverPayload(URLPayload): ... ) """ - num_of_posts: Optional[int] = None - posts_to_not_include: Optional[List[str]] = field(default_factory=list) - start_date: Optional[str] = None - end_date: Optional[str] = None - post_type: Optional[str] = None + num_of_posts: int | None = None + posts_to_not_include: list[str] | None = field(default_factory=list) + start_date: str | None = None + end_date: str | None = None + post_type: str | None = None def __post_init__(self): """Validate Instagram posts discovery payload.""" @@ -816,10 +815,10 @@ class InstagramReelsDiscoverPayload(URLPayload): ... ) """ - num_of_posts: Optional[int] = None - posts_to_not_include: Optional[List[str]] = field(default_factory=list) - start_date: Optional[str] = None - end_date: Optional[str] = None + num_of_posts: int | None = None + posts_to_not_include: list[str] | None = field(default_factory=list) + start_date: str | None = None + end_date: str | None = None def __post_init__(self): """Validate Instagram reels discovery payload.""" @@ -858,11 +857,11 @@ class DatasetTriggerPayload(BasePayload): ... ) """ - url: Optional[str] = None - keyword: Optional[str] = None - location: Optional[str] = None - country: Optional[str] = None - max_results: Optional[int] = None + url: str | None = None + keyword: str | None = None + location: str | None = None + country: str | None = None + max_results: int | None = None def __post_init__(self): """Validate dataset trigger fields.""" diff --git a/src/brightdata/scraper_studio/__init__.py b/src/brightdata/scraper_studio/__init__.py index bedfc5a..ba1397f 100644 --- a/src/brightdata/scraper_studio/__init__.py +++ b/src/brightdata/scraper_studio/__init__.py @@ -1,6 +1,6 @@ """Scraper Studio - trigger and fetch results from user-created custom scrapers.""" -from .models import ScraperStudioJob, JobStatus +from .models import JobStatus, ScraperStudioJob from .service import ScraperStudioService -__all__ = ["ScraperStudioJob", "JobStatus", "ScraperStudioService"] +__all__ = ["JobStatus", "ScraperStudioJob", "ScraperStudioService"] diff --git a/src/brightdata/scraper_studio/client.py b/src/brightdata/scraper_studio/client.py index 34d27a4..3264ce9 100644 --- a/src/brightdata/scraper_studio/client.py +++ b/src/brightdata/scraper_studio/client.py @@ -9,10 +9,10 @@ Follows the same pattern as DatasetAPIClient and AsyncUnblockerClient. """ -from typing import Dict, List, Any +from http import HTTPStatus +from typing import Any from ..core.engine import AsyncEngine -from http import HTTPStatus from ..exceptions import APIError, DataNotReadyError BASE_URL = "https://api.brightdata.com" @@ -43,7 +43,7 @@ def __init__(self, engine: AsyncEngine): async def trigger_immediate( self, collector: str, - input: Dict[str, Any], + input: dict[str, Any], ) -> str: """ Trigger a real-time async scrape. @@ -78,7 +78,7 @@ async def trigger_immediate( async def fetch_immediate_result( self, response_id: str, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: """ Fetch results from a real-time async scrape. @@ -110,7 +110,7 @@ async def fetch_immediate_result( async def get_status( self, job_id: str, - ) -> Dict[str, Any]: + ) -> dict[str, Any]: """ Get job status/log. diff --git a/src/brightdata/scraper_studio/models.py b/src/brightdata/scraper_studio/models.py index 2f4ff33..dff6c63 100644 --- a/src/brightdata/scraper_studio/models.py +++ b/src/brightdata/scraper_studio/models.py @@ -5,7 +5,7 @@ import asyncio import time from dataclasses import dataclass -from typing import Dict, List, Any, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any from ..exceptions import DataNotReadyError @@ -41,13 +41,13 @@ class JobStatus: fails: int = 0 success_rate: float = 0.0 created: str = "" - started: Optional[str] = None - finished: Optional[str] = None - job_time: Optional[int] = None - queue_time: Optional[int] = None + started: str | None = None + finished: str | None = None + job_time: int | None = None + queue_time: int | None = None @classmethod - def from_api_response(cls, data: Dict[str, Any]) -> "JobStatus": + def from_api_response(cls, data: dict[str, Any]) -> "JobStatus": """ Create from API response. @@ -98,12 +98,12 @@ def __init__( ): self.response_id = response_id self._api_client = api_client - self._cached_data: Optional[List[Dict[str, Any]]] = None + self._cached_data: list[dict[str, Any]] | None = None def __repr__(self) -> str: return f"" - async def fetch(self) -> List[Dict[str, Any]]: + async def fetch(self) -> list[dict[str, Any]]: """ Fetch results via GET /dca/get_result. @@ -120,7 +120,7 @@ async def wait_and_fetch( self, timeout: int = 300, poll_interval: int = 10, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: """ Poll fetch() until data arrives or timeout. diff --git a/src/brightdata/scraper_studio/service.py b/src/brightdata/scraper_studio/service.py index 27b0e3c..1d686c4 100644 --- a/src/brightdata/scraper_studio/service.py +++ b/src/brightdata/scraper_studio/service.py @@ -7,11 +7,11 @@ All methods are async-only. For sync usage, use SyncBrightDataClient. """ -from typing import Dict, List, Any, Union, TYPE_CHECKING +from typing import TYPE_CHECKING, Any -from .client import ScraperStudioAPIClient -from .models import ScraperStudioJob, JobStatus from ..constants import SCRAPER_STUDIO_DEFAULT_TIMEOUT, SCRAPER_STUDIO_POLL_INTERVAL +from .client import ScraperStudioAPIClient +from .models import JobStatus, ScraperStudioJob if TYPE_CHECKING: from ..client import BrightDataClient @@ -45,10 +45,10 @@ def __init__(self, client: "BrightDataClient"): async def run( self, collector: str, - input: Union[Dict[str, Any], List[Dict[str, Any]]], + input: dict[str, Any] | list[dict[str, Any]], timeout: int = SCRAPER_STUDIO_DEFAULT_TIMEOUT, poll_interval: int = SCRAPER_STUDIO_POLL_INTERVAL, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: """ Trigger a scrape and wait for results. @@ -72,7 +72,7 @@ async def run( # Normalize list input to individual triggers if isinstance(input, list): # Trigger each input separately and collect results - all_data: List[Dict[str, Any]] = [] + all_data: list[dict[str, Any]] = [] for single_input in input: response_id = await self._api.trigger_immediate(collector, single_input) job = ScraperStudioJob(response_id=response_id, api_client=self._api) @@ -87,7 +87,7 @@ async def run( async def trigger( self, collector: str, - input: Dict[str, Any], + input: dict[str, Any], ) -> ScraperStudioJob: """ Trigger a scrape and return a job object for manual control. @@ -130,7 +130,7 @@ async def status( async def fetch( self, response_id: str, - ) -> List[Dict[str, Any]]: + ) -> list[dict[str, Any]]: """ Fetch results of a completed scrape. diff --git a/src/brightdata/scrapers/__init__.py b/src/brightdata/scrapers/__init__.py index b624471..12a007b 100644 --- a/src/brightdata/scrapers/__init__.py +++ b/src/brightdata/scrapers/__init__.py @@ -1,8 +1,8 @@ """Specialized platform scrapers.""" from .base import BaseWebScraper -from .registry import register, get_scraper_for, get_registered_platforms, is_platform_supported from .job import ScrapeJob +from .registry import get_registered_platforms, get_scraper_for, is_platform_supported, register # Import scrapers to trigger registration try: @@ -77,24 +77,24 @@ __all__ = [ - "BaseWebScraper", - "ScrapeJob", - "register", - "get_scraper_for", - "get_registered_platforms", - "is_platform_supported", "AmazonScraper", - "LinkedInScraper", + "BaseWebScraper", "ChatGPTScraper", + "DigiKeyScraper", "FacebookScraper", "InstagramScraper", "InstagramSearchScraper", + "LinkedInScraper", "PerplexityScraper", + "RedditScraper", + "ScrapeJob", "TikTokScraper", "TikTokSearchScraper", + "XScraper", "YouTubeScraper", "YouTubeSearchScraper", - "DigiKeyScraper", - "RedditScraper", - "XScraper", + "get_registered_platforms", + "get_scraper_for", + "is_platform_supported", + "register", ] diff --git a/src/brightdata/scrapers/amazon/schemas.py b/src/brightdata/scrapers/amazon/schemas.py index 2b7c821..ab3db08 100644 --- a/src/brightdata/scrapers/amazon/schemas.py +++ b/src/brightdata/scrapers/amazon/schemas.py @@ -6,41 +6,41 @@ """ from dataclasses import dataclass -from typing import Optional, List, Dict, Any +from typing import Any @dataclass class SubcategoryRank: """Amazon subcategory ranking info.""" - subcategory_name: Optional[str] = None - subcategory_rank: Optional[int] = None + subcategory_name: str | None = None + subcategory_rank: int | None = None @dataclass class ProductDetail: """Product detail key-value pair.""" - type: Optional[str] = None - value: Optional[str] = None + type: str | None = None + value: str | None = None @dataclass class OtherSellerPrice: """Price from other sellers.""" - price: Optional[float] = None - price_per_unit: Optional[float] = None - unit: Optional[str] = None - seller_name: Optional[str] = None - seller_url: Optional[str] = None + price: float | None = None + price_per_unit: float | None = None + unit: str | None = None + seller_name: str | None = None + seller_url: str | None = None @dataclass class CustomersSay: """Customer sentiment keywords.""" - keywords: Optional[Dict[str, Any]] = None + keywords: dict[str, Any] | None = None @dataclass @@ -60,109 +60,109 @@ class AmazonProductResult: """ # Basic product info - title: Optional[str] = None - brand: Optional[str] = None - description: Optional[str] = None - manufacturer: Optional[str] = None - department: Optional[str] = None - model_number: Optional[str] = None + title: str | None = None + brand: str | None = None + description: str | None = None + manufacturer: str | None = None + department: str | None = None + model_number: str | None = None # Identifiers - asin: Optional[str] = None - parent_asin: Optional[str] = None - upc: Optional[str] = None + asin: str | None = None + parent_asin: str | None = None + upc: str | None = None # URLs - url: Optional[str] = None - domain: Optional[str] = None - image_url: Optional[str] = None - image: Optional[str] = None - seller_url: Optional[str] = None - store_url: Optional[str] = None + url: str | None = None + domain: str | None = None + image_url: str | None = None + image: str | None = None + seller_url: str | None = None + store_url: str | None = None # Pricing - currency: Optional[str] = None - final_price_high: Optional[float] = None - prices_breakdown: Optional[List[Dict[str, Any]]] = None - other_sellers_prices: Optional[List[Dict[str, Any]]] = None - coupon: Optional[str] = None - coupon_description: Optional[str] = None + currency: str | None = None + final_price_high: float | None = None + prices_breakdown: list[dict[str, Any]] | None = None + other_sellers_prices: list[dict[str, Any]] | None = None + coupon: str | None = None + coupon_description: str | None = None # Ratings and reviews - rating: Optional[float] = None - reviews_count: Optional[int] = None - top_review: Optional[str] = None - customer_says: Optional[str] = None - customers_say: Optional[Dict[str, Any]] = None - answered_questions: Optional[int] = None + rating: float | None = None + reviews_count: int | None = None + top_review: str | None = None + customer_says: str | None = None + customers_say: dict[str, Any] | None = None + answered_questions: int | None = None # Seller info - seller_name: Optional[str] = None - seller_id: Optional[str] = None - number_of_sellers: Optional[int] = None - ships_from: Optional[str] = None - buybox_seller_rating: Optional[float] = None - inactive_buy_box: Optional[bool] = None + seller_name: str | None = None + seller_id: str | None = None + number_of_sellers: int | None = None + ships_from: str | None = None + buybox_seller_rating: float | None = None + inactive_buy_box: bool | None = None # Categories and rankings - categories: Optional[List[str]] = None - root_bs_category: Optional[str] = None - bs_category: Optional[str] = None - root_bs_rank: Optional[int] = None - bs_rank: Optional[int] = None - subcategory_rank: Optional[List[Dict[str, Any]]] = None + categories: list[str] | None = None + root_bs_category: str | None = None + bs_category: str | None = None + root_bs_rank: int | None = None + bs_rank: int | None = None + subcategory_rank: list[dict[str, Any]] | None = None # Product details - features: Optional[List[str]] = None - product_details: Optional[List[Dict[str, Any]]] = None - product_description: Optional[List[Dict[str, Any]]] = None - product_dimensions: Optional[str] = None - item_weight: Optional[str] = None - country_of_origin: Optional[str] = None - date_first_available: Optional[str] = None - language: Optional[str] = None + features: list[str] | None = None + product_details: list[dict[str, Any]] | None = None + product_description: list[dict[str, Any]] | None = None + product_dimensions: str | None = None + item_weight: str | None = None + country_of_origin: str | None = None + date_first_available: str | None = None + language: str | None = None # Media - images: Optional[List[str]] = None - images_count: Optional[int] = None - video: Optional[bool] = None - videos: Optional[List[str]] = None - video_count: Optional[int] = None - downloadable_videos: Optional[List[str]] = None + images: list[str] | None = None + images_count: int | None = None + video: bool | None = None + videos: list[str] | None = None + video_count: int | None = None + downloadable_videos: list[str] | None = None # Availability and badges - is_available: Optional[bool] = None - max_quantity_available: Optional[int] = None - amazon_choice: Optional[bool] = None - amazon_prime: Optional[bool] = None - badge: Optional[str] = None - all_badges: Optional[List[str]] = None - premium_brand: Optional[bool] = None - climate_pledge_friendly: Optional[bool] = None + is_available: bool | None = None + max_quantity_available: int | None = None + amazon_choice: bool | None = None + amazon_prime: bool | None = None + badge: str | None = None + all_badges: list[str] | None = None + premium_brand: bool | None = None + climate_pledge_friendly: bool | None = None # Additional content - plus_content: Optional[bool] = None - from_the_brand: Optional[List[str]] = None - editorial_reviews: Optional[str] = None - about_the_author: Optional[str] = None - sustainability_features: Optional[str] = None - return_policy: Optional[str] = None - variations_values: Optional[Dict[str, Any]] = None + plus_content: bool | None = None + from_the_brand: list[str] | None = None + editorial_reviews: str | None = None + about_the_author: str | None = None + sustainability_features: str | None = None + return_policy: str | None = None + variations_values: dict[str, Any] | None = None # Location - zipcode: Optional[str] = None - city: Optional[str] = None + zipcode: str | None = None + city: str | None = None # Sponsored/advertising - sponsored: Optional[bool] = None - sponsered: Optional[bool] = None # Note: typo exists in API response + sponsored: bool | None = None + sponsered: bool | None = None # Note: typo exists in API response # Metadata - timestamp: Optional[str] = None - input: Optional[Dict[str, Any]] = None + timestamp: str | None = None + input: dict[str, Any] | None = None @classmethod - def from_dict(cls, data: Dict[str, Any]) -> "AmazonProductResult": + def from_dict(cls, data: dict[str, Any]) -> "AmazonProductResult": """ Create AmazonProductResult from dictionary. @@ -183,7 +183,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "AmazonProductResult": return cls(**filtered_data) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """ Convert to dictionary, excluding None values. diff --git a/src/brightdata/scrapers/amazon/scraper.py b/src/brightdata/scrapers/amazon/scraper.py index 91d7d49..6e6a3dc 100644 --- a/src/brightdata/scrapers/amazon/scraper.py +++ b/src/brightdata/scrapers/amazon/scraper.py @@ -14,15 +14,15 @@ """ import asyncio -from typing import Union, List, Optional, Any +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("amazon") @@ -60,9 +60,9 @@ class AmazonScraper(BaseWebScraper): async def products( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape Amazon products from URLs. @@ -92,9 +92,9 @@ async def products( def products_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape Amazon products from URLs (sync version). @@ -113,7 +113,7 @@ async def _run(): async def products_trigger( self, - url: Union[str, List[str]], + url: str | list[str], ) -> ScrapeJob: """ Trigger Amazon products scrape (manual control). @@ -140,7 +140,7 @@ async def products_trigger( urls=url, sdk_function=sdk_function or "products_trigger" ) - def products_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def products_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger Amazon products scrape (sync version).""" return asyncio.run(self.products_trigger(url)) @@ -188,12 +188,12 @@ def products_fetch_sync(self, snapshot_id: str) -> Any: async def reviews( self, - url: Union[str, List[str]], - pastDays: Optional[int] = None, - keyWord: Optional[str] = None, - numOfReviews: Optional[int] = None, + url: str | list[str], + pastDays: int | None = None, + keyWord: str | None = None, + numOfReviews: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape Amazon product reviews from URLs. @@ -276,12 +276,12 @@ async def reviews( def reviews_sync( self, - url: Union[str, List[str]], - pastDays: Optional[int] = None, - keyWord: Optional[str] = None, - numOfReviews: Optional[int] = None, + url: str | list[str], + pastDays: int | None = None, + keyWord: str | None = None, + numOfReviews: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape Amazon product reviews from URLs (sync version). @@ -300,10 +300,10 @@ async def _run(): async def reviews_trigger( self, - url: Union[str, List[str]], - pastDays: Optional[int] = None, - keyWord: Optional[str] = None, - numOfReviews: Optional[int] = None, + url: str | list[str], + pastDays: int | None = None, + keyWord: str | None = None, + numOfReviews: int | None = None, ) -> ScrapeJob: """ Trigger Amazon reviews scrape (manual control). @@ -333,10 +333,10 @@ async def reviews_trigger( def reviews_trigger_sync( self, - url: Union[str, List[str]], - pastDays: Optional[int] = None, - keyWord: Optional[str] = None, - numOfReviews: Optional[int] = None, + url: str | list[str], + pastDays: int | None = None, + keyWord: str | None = None, + numOfReviews: int | None = None, ) -> ScrapeJob: """Trigger Amazon reviews scrape (sync version).""" return asyncio.run(self.reviews_trigger(url, pastDays, keyWord, numOfReviews)) @@ -363,9 +363,9 @@ def reviews_fetch_sync(self, snapshot_id: str) -> Any: async def sellers( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape Amazon seller information from URLs. @@ -395,9 +395,9 @@ async def sellers( def sellers_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape Amazon seller information from URLs (sync version). @@ -416,7 +416,7 @@ async def _run(): async def sellers_trigger( self, - url: Union[str, List[str]], + url: str | list[str], ) -> ScrapeJob: """ Trigger Amazon sellers scrape (manual control). @@ -441,7 +441,7 @@ async def sellers_trigger( sdk_function=sdk_function or "sellers_trigger", ) - def sellers_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def sellers_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger Amazon sellers scrape (sync version).""" return asyncio.run(self.sellers_trigger(url)) @@ -467,10 +467,10 @@ def sellers_fetch_sync(self, snapshot_id: str) -> Any: async def _scrape_urls( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, timeout: int, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape URLs using standard async workflow (trigger/poll/fetch). diff --git a/src/brightdata/scrapers/amazon/search.py b/src/brightdata/scrapers/amazon/search.py index 74b2a13..e99776e 100644 --- a/src/brightdata/scrapers/amazon/search.py +++ b/src/brightdata/scrapers/amazon/search.py @@ -9,12 +9,12 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any -from ...models import ScrapeResult +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...exceptions import ValidationError +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD from ..base import ScraperCore @@ -50,14 +50,14 @@ class AmazonSearchScraper(ScraperCore): async def products( self, - keyword: Optional[Union[str, List[str]]] = None, - url: Optional[Union[str, List[str]]] = None, - category: Optional[Union[str, List[str]]] = None, - min_price: Optional[Union[int, List[int]]] = None, - max_price: Optional[Union[int, List[int]]] = None, - condition: Optional[Union[str, List[str]]] = None, - prime_eligible: Optional[bool] = None, - country: Optional[Union[str, List[str]]] = None, + keyword: str | list[str] | None = None, + url: str | list[str] | None = None, + category: str | list[str] | None = None, + min_price: int | list[int] | None = None, + max_price: int | list[int] | None = None, + condition: str | list[str] | None = None, + prime_eligible: bool | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -154,14 +154,14 @@ async def products( def products_sync( self, - keyword: Optional[Union[str, List[str]]] = None, - url: Optional[Union[str, List[str]]] = None, - category: Optional[Union[str, List[str]]] = None, - min_price: Optional[Union[int, List[int]]] = None, - max_price: Optional[Union[int, List[int]]] = None, - condition: Optional[Union[str, List[str]]] = None, - prime_eligible: Optional[bool] = None, - country: Optional[Union[str, List[str]]] = None, + keyword: str | list[str] | None = None, + url: str | list[str] | None = None, + category: str | list[str] | None = None, + min_price: int | list[int] | None = None, + max_price: int | list[int] | None = None, + condition: str | list[str] | None = None, + prime_eligible: bool | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -191,8 +191,8 @@ async def _run(): # ============================================================================ def _normalize_param( - self, param: Optional[Union[str, int, List[str], List[int]]], target_length: int - ) -> Optional[List]: + self, param: str | int | list[str] | list[int] | None, target_length: int + ) -> list | None: """ Normalize parameter to list. @@ -214,13 +214,13 @@ def _normalize_param( def _build_amazon_search_url( self, - keyword: Optional[str] = None, - category: Optional[str] = None, - min_price: Optional[int] = None, - max_price: Optional[int] = None, - condition: Optional[str] = None, - prime_eligible: Optional[bool] = None, - country: Optional[str] = None, + keyword: str | None = None, + category: str | None = None, + min_price: int | None = None, + max_price: int | None = None, + condition: str | None = None, + prime_eligible: bool | None = None, + country: str | None = None, ) -> str: """ Build Amazon search URL from parameters. @@ -315,7 +315,7 @@ def _build_amazon_search_url( async def _execute_search( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, timeout: int, ) -> ScrapeResult: diff --git a/src/brightdata/scrapers/api_client.py b/src/brightdata/scrapers/api_client.py index 2443517..a0e40ea 100644 --- a/src/brightdata/scrapers/api_client.py +++ b/src/brightdata/scrapers/api_client.py @@ -7,10 +7,10 @@ - Fetching snapshot results """ -from typing import List, Dict, Any, Optional +from http import HTTPStatus +from typing import Any from ..core.engine import AsyncEngine -from http import HTTPStatus from ..exceptions import APIError, DataNotReadyError @@ -41,13 +41,13 @@ def __init__(self, engine: AsyncEngine): async def trigger( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, include_errors: bool = True, - sdk_function: Optional[str] = None, - extra_params: Optional[Dict[str, str]] = None, - limit_per_input: Optional[int] = None, - ) -> Optional[str]: + sdk_function: str | None = None, + extra_params: dict[str, str] | None = None, + limit_per_input: int | None = None, + ) -> str | None: """ Trigger dataset collection and get snapshot_id. diff --git a/src/brightdata/scrapers/base.py b/src/brightdata/scrapers/base.py index c76b86f..04e5d4c 100644 --- a/src/brightdata/scrapers/base.py +++ b/src/brightdata/scrapers/base.py @@ -10,26 +10,26 @@ """ import asyncio +import concurrent.futures import os import time -import concurrent.futures from abc import ABC from datetime import datetime, timezone -from typing import List, Dict, Any, Optional, Union +from typing import Any -from ..core.engine import AsyncEngine -from ..models import ScrapeResult -from ..exceptions import ValidationError, APIError -from ..utils.validation import validate_url, validate_url_list -from ..utils.function_detection import get_caller_function_name from ..constants import ( - DEFAULT_POLL_INTERVAL, - DEFAULT_MIN_POLL_TIMEOUT, DEFAULT_COST_PER_RECORD, + DEFAULT_MIN_POLL_TIMEOUT, + DEFAULT_POLL_INTERVAL, ) +from ..core.engine import AsyncEngine +from ..exceptions import APIError, ValidationError +from ..models import ScrapeResult +from ..utils.function_detection import get_caller_function_name +from ..utils.validation import validate_url, validate_url_list from .api_client import DatasetAPIClient -from .workflow import WorkflowExecutor from .job import ScrapeJob +from .workflow import WorkflowExecutor class ScraperCore: @@ -50,7 +50,7 @@ class ScraperCore: MIN_POLL_TIMEOUT: int = DEFAULT_MIN_POLL_TIMEOUT COST_PER_RECORD: float = DEFAULT_COST_PER_RECORD - def __init__(self, bearer_token: Optional[str] = None, engine: Optional[AsyncEngine] = None): + def __init__(self, bearer_token: str | None = None, engine: AsyncEngine | None = None): """ Initialize scraper core. @@ -131,7 +131,7 @@ class BaseWebScraper(ScraperCore, ABC): DATASET_ID: str = "" - def __init__(self, bearer_token: Optional[str] = None, engine: Optional[AsyncEngine] = None): + def __init__(self, bearer_token: str | None = None, engine: AsyncEngine | None = None): """ Initialize platform scraper. @@ -153,12 +153,12 @@ def __init__(self, bearer_token: Optional[str] = None, engine: Optional[AsyncEng async def scrape_async( self, - urls: Union[str, List[str]], + urls: str | list[str], include_errors: bool = True, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, **kwargs, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape one or more URLs asynchronously. @@ -238,9 +238,7 @@ async def scrape_async( return result - def scrape( - self, urls: Union[str, List[str]], **kwargs - ) -> Union[ScrapeResult, List[ScrapeResult]]: + def scrape(self, urls: str | list[str], **kwargs) -> ScrapeResult | list[ScrapeResult]: """ Scrape URLs synchronously. @@ -275,7 +273,7 @@ def normalize_result(self, data: Any) -> Any: """ return data - def _build_scrape_payload(self, urls: List[str], **kwargs) -> List[Dict[str, Any]]: + def _build_scrape_payload(self, urls: list[str], **kwargs) -> list[dict[str, Any]]: """ Build payload for scrape operation. @@ -302,9 +300,9 @@ def _build_scrape_payload(self, urls: List[str], **kwargs) -> List[Dict[str, Any async def _trigger_scrape_async( self, - urls: Union[str, List[str]], - dataset_id: Optional[str] = None, - sdk_function: Optional[str] = None, + urls: str | list[str], + dataset_id: str | None = None, + sdk_function: str | None = None, **kwargs, ) -> ScrapeJob: """ @@ -359,9 +357,9 @@ async def _trigger_scrape_async( def _trigger_scrape( self, - urls: Union[str, List[str]], - dataset_id: Optional[str] = None, - sdk_function: Optional[str] = None, + urls: str | list[str], + dataset_id: str | None = None, + sdk_function: str | None = None, **kwargs, ) -> ScrapeJob: """Trigger scrape job (internal sync wrapper).""" @@ -437,7 +435,7 @@ async def status(self, snapshot_id: str) -> str: async def wait( self, snapshot_id: str, - timeout: Optional[int] = None, + timeout: int | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, verbose: bool = False, ) -> str: @@ -495,7 +493,7 @@ async def fetch(self, snapshot_id: str, format: str = "json") -> Any: async def to_result( self, snapshot_id: str, - timeout: Optional[int] = None, + timeout: int | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, ) -> ScrapeResult: """ diff --git a/src/brightdata/scrapers/chatgpt/scraper.py b/src/brightdata/scrapers/chatgpt/scraper.py index d98cce4..692a74d 100644 --- a/src/brightdata/scrapers/chatgpt/scraper.py +++ b/src/brightdata/scrapers/chatgpt/scraper.py @@ -8,15 +8,15 @@ """ import asyncio -from typing import List, Any, Optional, Union +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_CHATGPT, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_LONG +from ...exceptions import ValidationError from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_LONG, COST_PER_RECORD_CHATGPT -from ...exceptions import ValidationError +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("chatgpt") @@ -63,9 +63,9 @@ async def prompt( prompt: str, country: str = "us", web_search: bool = False, - additional_prompt: Optional[str] = None, + additional_prompt: str | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, ) -> ScrapeResult: """ Send single prompt to ChatGPT (async). @@ -125,9 +125,9 @@ def prompt_sync( prompt: str, country: str = "us", web_search: bool = False, - additional_prompt: Optional[str] = None, + additional_prompt: str | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, ) -> ScrapeResult: """ Send prompt to ChatGPT (sync). @@ -160,7 +160,7 @@ async def prompt_trigger( prompt: str, country: str = "us", web_search: bool = False, - additional_prompt: Optional[str] = None, + additional_prompt: str | None = None, ) -> "ScrapeJob": """Trigger ChatGPT prompt (async - manual control).""" from ..job import ScrapeJob @@ -196,7 +196,7 @@ def prompt_trigger_sync( prompt: str, country: str = "us", web_search: bool = False, - additional_prompt: Optional[str] = None, + additional_prompt: str | None = None, ) -> "ScrapeJob": """Trigger ChatGPT prompt (sync wrapper).""" @@ -238,12 +238,12 @@ async def _run(): async def prompts( self, - prompts: List[str], - countries: Optional[List[str]] = None, - web_searches: Optional[List[bool]] = None, - additional_prompts: Optional[List[str]] = None, + prompts: list[str], + countries: list[str] | None = None, + web_searches: list[bool] | None = None, + additional_prompts: list[str] | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, ) -> ScrapeResult: """ Send multiple prompts to ChatGPT in batch (async). @@ -305,12 +305,12 @@ async def prompts( def prompts_sync( self, - prompts: List[str], - countries: Optional[List[str]] = None, - web_searches: Optional[List[bool]] = None, - additional_prompts: Optional[List[str]] = None, + prompts: list[str], + countries: list[str] | None = None, + web_searches: list[bool] | None = None, + additional_prompts: list[str] | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, ) -> ScrapeResult: """ Send multiple prompts (sync). @@ -337,10 +337,10 @@ async def _run(): async def prompts_trigger( self, - prompts: List[str], - countries: Optional[List[str]] = None, - web_searches: Optional[List[bool]] = None, - additional_prompts: Optional[List[str]] = None, + prompts: list[str], + countries: list[str] | None = None, + web_searches: list[bool] | None = None, + additional_prompts: list[str] | None = None, ) -> "ScrapeJob": """Trigger ChatGPT batch prompts (async - manual control).""" from ..job import ScrapeJob @@ -373,10 +373,10 @@ async def prompts_trigger( def prompts_trigger_sync( self, - prompts: List[str], - countries: Optional[List[str]] = None, - web_searches: Optional[List[bool]] = None, - additional_prompts: Optional[List[str]] = None, + prompts: list[str], + countries: list[str] | None = None, + web_searches: list[bool] | None = None, + additional_prompts: list[str] | None = None, ) -> "ScrapeJob": """Trigger ChatGPT batch prompts (sync wrapper).""" @@ -418,9 +418,7 @@ async def _run(): # SCRAPE OVERRIDE (ChatGPT doesn't use URL-based scraping) # ============================================================================ - async def scrape( - self, urls: Union[str, List[str]], **kwargs - ) -> Union[ScrapeResult, List[ScrapeResult]]: + async def scrape(self, urls: str | list[str], **kwargs) -> ScrapeResult | list[ScrapeResult]: """ ChatGPT doesn't support URL-based scraping. @@ -431,7 +429,7 @@ async def scrape( "Use prompt() or prompts() methods instead." ) - def scrape_sync(self, urls: Union[str, List[str]], **kwargs): + def scrape_sync(self, urls: str | list[str], **kwargs): """ChatGPT doesn't support URL-based scraping.""" raise NotImplementedError( "ChatGPT scraper doesn't support URL-based scraping. " diff --git a/src/brightdata/scrapers/chatgpt/search.py b/src/brightdata/scrapers/chatgpt/search.py index a214085..66f02c0 100644 --- a/src/brightdata/scrapers/chatgpt/search.py +++ b/src/brightdata/scrapers/chatgpt/search.py @@ -10,12 +10,12 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any -from ...models import ScrapeResult +from ...constants import COST_PER_RECORD_CHATGPT, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT from ...exceptions import ValidationError +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_CHATGPT from ..base import ScraperCore @@ -61,10 +61,10 @@ class ChatGPTSearchService(ScraperCore): async def prompt( self, - prompt: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, - secondaryPrompt: Optional[Union[str, List[str]]] = None, - webSearch: Optional[Union[bool, List[bool]]] = None, + prompt: str | list[str], + country: str | list[str] | None = None, + secondaryPrompt: str | list[str] | None = None, + webSearch: bool | list[bool] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -120,7 +120,7 @@ async def prompt( # Build payload (URL fixed to https://chatgpt.com per spec) payload = [] for i in range(batch_size): - item: Dict[str, Any] = { + item: dict[str, Any] = { "url": "https://chatgpt.com", # Fixed URL per API spec "prompt": prompts[i], "country": countries[i].upper() if countries[i] else "US", @@ -139,10 +139,10 @@ async def prompt( def prompt_sync( self, - prompt: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, - secondaryPrompt: Optional[Union[str, List[str]]] = None, - webSearch: Optional[Union[bool, List[bool]]] = None, + prompt: str | list[str], + country: str | list[str] | None = None, + secondaryPrompt: str | list[str] | None = None, + webSearch: bool | list[bool] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -174,8 +174,8 @@ async def _run(): # ============================================================================ def _normalize_param( - self, param: Optional[Union[Any, List[Any]]], target_length: int, default_value: Any = None - ) -> List[Any]: + self, param: Any | list[Any] | None, target_length: int, default_value: Any = None + ) -> list[Any]: """ Normalize parameter to list of specified length. @@ -206,7 +206,7 @@ def _normalize_param( async def _execute_async_mode( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], timeout: int, ) -> ScrapeResult: """Execute using standard async workflow (/trigger endpoint with polling).""" diff --git a/src/brightdata/scrapers/digikey/scraper.py b/src/brightdata/scrapers/digikey/scraper.py index 3ffeb98..5eda688 100644 --- a/src/brightdata/scrapers/digikey/scraper.py +++ b/src/brightdata/scrapers/digikey/scraper.py @@ -13,15 +13,15 @@ """ import asyncio -from typing import List, Any, Union +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("digikey") @@ -60,9 +60,9 @@ class DigiKeyScraper(BaseWebScraper): async def products( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect DigiKey product data by URL (async). @@ -106,9 +106,9 @@ async def products( def products_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect DigiKey product data by URL (sync).""" async def _run(): @@ -119,7 +119,7 @@ async def _run(): # --- Products Trigger/Status/Fetch --- - async def products_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def products_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger DigiKey products collection (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [{"url": u} for u in url_list] @@ -132,7 +132,7 @@ async def products_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: cost_per_record=self.COST_PER_RECORD, ) - def products_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def products_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger DigiKey products collection (sync).""" return asyncio.run(self.products_trigger(url)) @@ -158,7 +158,7 @@ def products_fetch_sync(self, snapshot_id: str) -> Any: async def discover_by_category( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -201,7 +201,7 @@ async def discover_by_category( def discover_by_category_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Discover DigiKey products by category URL (sync).""" @@ -214,7 +214,7 @@ async def _run(): # --- Discover Trigger/Status/Fetch --- - async def discover_by_category_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def discover_by_category_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger DigiKey category discovery (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [{"category_url": u} for u in url_list] @@ -231,7 +231,7 @@ async def discover_by_category_trigger(self, url: Union[str, List[str]]) -> Scra cost_per_record=self.COST_PER_RECORD, ) - def discover_by_category_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def discover_by_category_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger DigiKey category discovery (sync).""" return asyncio.run(self.discover_by_category_trigger(url)) diff --git a/src/brightdata/scrapers/facebook/scraper.py b/src/brightdata/scrapers/facebook/scraper.py index 6473547..6b1070c 100644 --- a/src/brightdata/scrapers/facebook/scraper.py +++ b/src/brightdata/scrapers/facebook/scraper.py @@ -19,15 +19,15 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_FACEBOOK, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, COST_PER_RECORD_FACEBOOK +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("facebook") @@ -76,13 +76,13 @@ class FacebookScraper(BaseWebScraper): async def posts_by_profile( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect posts from Facebook profile URL (async). @@ -127,13 +127,13 @@ async def posts_by_profile( def posts_by_profile_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect posts from Facebook profile URL (sync wrapper).""" async def _run(): @@ -148,11 +148,11 @@ async def _run(): async def posts_by_profile_trigger( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, ) -> "ScrapeJob": """Trigger Facebook posts by profile scrape (async - manual control).""" from ..job import ScrapeJob @@ -184,7 +184,7 @@ async def posts_by_profile_trigger( cost_per_record=self.COST_PER_RECORD, ) - def posts_by_profile_trigger_sync(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + def posts_by_profile_trigger_sync(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook posts by profile scrape (sync wrapper).""" return asyncio.run(self.posts_by_profile_trigger(url, **kwargs)) @@ -210,13 +210,13 @@ def posts_by_profile_fetch_sync(self, snapshot_id: str) -> Any: async def posts_by_group( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect posts from Facebook group URL (async). @@ -259,13 +259,13 @@ async def posts_by_group( def posts_by_group_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect posts from Facebook group URL (sync wrapper).""" async def _run(): @@ -278,7 +278,7 @@ async def _run(): # --- Trigger Interface (Manual Control) --- - async def posts_by_group_trigger(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + async def posts_by_group_trigger(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook posts by group scrape (async - manual control).""" from ..job import ScrapeJob @@ -297,7 +297,7 @@ async def posts_by_group_trigger(self, url: Union[str, List[str]], **kwargs) -> cost_per_record=self.COST_PER_RECORD, ) - def posts_by_group_trigger_sync(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + def posts_by_group_trigger_sync(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook posts by group scrape (sync wrapper).""" return asyncio.run(self.posts_by_group_trigger(url, **kwargs)) @@ -323,9 +323,9 @@ def posts_by_group_fetch_sync(self, snapshot_id: str) -> Any: async def posts_by_url( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect detailed data from specific Facebook post URLs (async). @@ -359,9 +359,9 @@ async def posts_by_url( def posts_by_url_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect detailed data from specific Facebook post URLs (sync wrapper).""" async def _run(): @@ -372,7 +372,7 @@ async def _run(): # --- Trigger Interface (Manual Control) --- - async def posts_by_url_trigger(self, url: Union[str, List[str]]) -> "ScrapeJob": + async def posts_by_url_trigger(self, url: str | list[str]) -> "ScrapeJob": """Trigger Facebook posts by URL scrape (async - manual control).""" sdk_function = get_caller_function_name() @@ -382,7 +382,7 @@ async def posts_by_url_trigger(self, url: Union[str, List[str]]) -> "ScrapeJob": sdk_function=sdk_function or "posts_by_url_trigger", ) - def posts_by_url_trigger_sync(self, url: Union[str, List[str]]) -> "ScrapeJob": + def posts_by_url_trigger_sync(self, url: str | list[str]) -> "ScrapeJob": """Trigger Facebook posts by URL scrape (sync wrapper).""" return asyncio.run(self.posts_by_url_trigger(url)) @@ -408,13 +408,13 @@ def posts_by_url_fetch_sync(self, snapshot_id: str) -> Any: async def comments( self, - url: Union[str, List[str]], - num_of_comments: Optional[int] = None, - comments_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_comments: int | None = None, + comments_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect comments from Facebook post URL (async). @@ -459,13 +459,13 @@ async def comments( def comments_sync( self, - url: Union[str, List[str]], - num_of_comments: Optional[int] = None, - comments_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_comments: int | None = None, + comments_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect comments from Facebook post URL (sync wrapper).""" async def _run(): @@ -478,7 +478,7 @@ async def _run(): # --- Trigger Interface (Manual Control) --- - async def comments_trigger(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + async def comments_trigger(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook comments scrape (async - manual control).""" from ..job import ScrapeJob @@ -497,7 +497,7 @@ async def comments_trigger(self, url: Union[str, List[str]], **kwargs) -> "Scrap cost_per_record=self.COST_PER_RECORD, ) - def comments_trigger_sync(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + def comments_trigger_sync(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook comments scrape (sync wrapper).""" return asyncio.run(self.comments_trigger(url, **kwargs)) @@ -523,13 +523,13 @@ def comments_fetch_sync(self, snapshot_id: str) -> Any: async def reels( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect reels from Facebook profile URL (async). @@ -572,13 +572,13 @@ async def reels( def reels_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect reels from Facebook profile URL (sync wrapper).""" async def _run(): @@ -591,7 +591,7 @@ async def _run(): # --- Trigger Interface (Manual Control) --- - async def reels_trigger(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + async def reels_trigger(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook reels scrape (async - manual control).""" from ..job import ScrapeJob @@ -610,7 +610,7 @@ async def reels_trigger(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJo cost_per_record=self.COST_PER_RECORD, ) - def reels_trigger_sync(self, url: Union[str, List[str]], **kwargs) -> "ScrapeJob": + def reels_trigger_sync(self, url: str | list[str], **kwargs) -> "ScrapeJob": """Trigger Facebook reels scrape (sync wrapper).""" return asyncio.run(self.reels_trigger(url, **kwargs)) @@ -636,11 +636,11 @@ def reels_fetch_sync(self, snapshot_id: str) -> Any: async def _scrape_urls( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, timeout: int, - sdk_function: Optional[str] = None, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + sdk_function: str | None = None, + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape URLs using standard async workflow (trigger/poll/fetch). @@ -700,17 +700,17 @@ async def _scrape_urls( async def _scrape_with_params( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, - num_of_posts: Optional[int] = None, - num_of_comments: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - comments_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + num_of_posts: int | None = None, + num_of_comments: int | None = None, + posts_to_not_include: list[str] | None = None, + comments_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - sdk_function: Optional[str] = None, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + sdk_function: str | None = None, + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape URLs with additional parameters using standard async workflow. @@ -733,7 +733,7 @@ async def _scrape_with_params( payload = [] for u in url_list: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if num_of_posts is not None: item["num_of_posts"] = num_of_posts diff --git a/src/brightdata/scrapers/instagram/scraper.py b/src/brightdata/scrapers/instagram/scraper.py index b7c45df..c81b73b 100644 --- a/src/brightdata/scrapers/instagram/scraper.py +++ b/src/brightdata/scrapers/instagram/scraper.py @@ -9,19 +9,19 @@ """ import asyncio -from typing import List, Any, Union +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_INSTAGRAM, - DEFAULT_TIMEOUT_SHORT, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_SHORT, ) -from ...utils.validation import validate_url, validate_url_list +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("instagram") @@ -55,10 +55,10 @@ class InstagramScraper(BaseWebScraper): async def _scrape_urls( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, timeout: int, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Internal method to scrape URLs with specified dataset. @@ -132,9 +132,9 @@ async def _scrape_urls( async def profiles( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Extract profile data from Instagram profile URLs. @@ -158,9 +158,9 @@ async def profiles( def profiles_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Synchronous version of profiles(). See profiles() for documentation.""" async def _run(): @@ -169,7 +169,7 @@ async def _run(): return asyncio.run(_run()) - async def profiles_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def profiles_trigger(self, url: str | list[str]) -> ScrapeJob: """ Trigger profile extraction job without waiting for results. @@ -192,7 +192,7 @@ async def profiles_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: sdk_function=sdk_function, ) - def profiles_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def profiles_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Synchronous version of profiles_trigger().""" async def _run(): @@ -233,9 +233,9 @@ async def _run(): async def posts( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Extract post data from Instagram post URLs. @@ -279,9 +279,9 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Synchronous version of posts().""" async def _run(): @@ -290,7 +290,7 @@ async def _run(): return asyncio.run(_run()) - async def posts_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def posts_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger post extraction job without waiting for results.""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( @@ -299,7 +299,7 @@ async def posts_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: sdk_function=sdk_function, ) - def posts_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def posts_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Synchronous version of posts_trigger().""" async def _run(): @@ -340,9 +340,9 @@ async def _run(): async def reels( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Extract reel data from Instagram reel URLs. @@ -390,9 +390,9 @@ async def reels( def reels_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Synchronous version of reels().""" async def _run(): @@ -401,7 +401,7 @@ async def _run(): return asyncio.run(_run()) - async def reels_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def reels_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger reel extraction job without waiting for results.""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( @@ -410,7 +410,7 @@ async def reels_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: sdk_function=sdk_function, ) - def reels_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def reels_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Synchronous version of reels_trigger().""" async def _run(): @@ -451,9 +451,9 @@ async def _run(): async def comments( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Extract comments from Instagram post or reel URLs. @@ -492,9 +492,9 @@ async def comments( def comments_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Synchronous version of comments().""" async def _run(): @@ -503,7 +503,7 @@ async def _run(): return asyncio.run(_run()) - async def comments_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def comments_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger comment extraction job without waiting for results.""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( @@ -512,7 +512,7 @@ async def comments_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: sdk_function=sdk_function, ) - def comments_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def comments_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Synchronous version of comments_trigger().""" async def _run(): diff --git a/src/brightdata/scrapers/instagram/search.py b/src/brightdata/scrapers/instagram/search.py index 8f5a882..2caa731 100644 --- a/src/brightdata/scrapers/instagram/search.py +++ b/src/brightdata/scrapers/instagram/search.py @@ -8,17 +8,17 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_INSTAGRAM, - DEFAULT_TIMEOUT_SHORT, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_SHORT, ) -from ...utils.validation import validate_url_list, validate_instagram_date +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ...utils.validation import validate_instagram_date, validate_url_list +from ..base import ScraperCore class InstagramSearchScraper(ScraperCore): @@ -57,7 +57,7 @@ class InstagramSearchScraper(ScraperCore): async def _execute_discovery( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, discover_by: str, timeout: int, @@ -101,7 +101,7 @@ async def _execute_discovery( async def profiles( self, - user_name: Union[str, List[str]], + user_name: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -159,7 +159,7 @@ async def profiles( def profiles_sync( self, - user_name: Union[str, List[str]], + user_name: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """Synchronous version of profiles().""" @@ -176,12 +176,12 @@ async def _run(): async def posts( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - post_type: Optional[str] = None, - posts_to_not_include: Optional[List[str]] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, + post_type: str | None = None, + posts_to_not_include: list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -238,7 +238,7 @@ async def posts( # Build payload - omit None values (don't send empty strings) payload = [] for u in urls: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if num_of_posts is not None: item["num_of_posts"] = num_of_posts @@ -262,12 +262,12 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - post_type: Optional[str] = None, - posts_to_not_include: Optional[List[str]] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, + post_type: str | None = None, + posts_to_not_include: list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """Synchronous version of posts().""" @@ -292,10 +292,10 @@ async def _run(): async def reels( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -346,7 +346,7 @@ async def reels( # Build payload payload = [] for u in urls: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if num_of_posts is not None: item["num_of_posts"] = num_of_posts if start_date: @@ -364,10 +364,10 @@ async def reels( def reels_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """Synchronous version of reels().""" @@ -384,10 +384,10 @@ async def _run(): async def reels_all( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -439,7 +439,7 @@ async def reels_all( # Build payload payload = [] for u in urls: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if num_of_posts is not None: item["num_of_posts"] = num_of_posts if start_date: @@ -458,10 +458,10 @@ async def reels_all( def reels_all_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """Synchronous version of reels_all().""" diff --git a/src/brightdata/scrapers/job.py b/src/brightdata/scrapers/job.py index a4d079f..4ac8f61 100644 --- a/src/brightdata/scrapers/job.py +++ b/src/brightdata/scrapers/job.py @@ -9,12 +9,12 @@ import asyncio import time -from typing import Optional, Any from datetime import datetime, timezone +from typing import Any -from ..models import ScrapeResult -from ..exceptions import APIError from ..constants import DEFAULT_POLL_INTERVAL +from ..exceptions import APIError +from ..models import ScrapeResult from .api_client import DatasetAPIClient @@ -46,9 +46,9 @@ def __init__( self, snapshot_id: str, api_client: DatasetAPIClient, - platform_name: Optional[str] = None, + platform_name: str | None = None, cost_per_record: float = 0.001, - triggered_at: Optional[datetime] = None, + triggered_at: datetime | None = None, ): """ Initialize scrape job. @@ -65,8 +65,8 @@ def __init__( self.platform_name = platform_name self.cost_per_record = cost_per_record self.triggered_at = triggered_at or datetime.now(timezone.utc) - self._cached_status: Optional[str] = None - self._cached_data: Optional[Any] = None + self._cached_status: str | None = None + self._cached_data: Any | None = None def __repr__(self) -> str: """String representation.""" diff --git a/src/brightdata/scrapers/linkedin/scraper.py b/src/brightdata/scrapers/linkedin/scraper.py index 3327e67..cf3eedd 100644 --- a/src/brightdata/scrapers/linkedin/scraper.py +++ b/src/brightdata/scrapers/linkedin/scraper.py @@ -23,15 +23,15 @@ """ import asyncio -from typing import Union, List, Any +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_LINKEDIN, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_LINKEDIN +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("linkedin") @@ -77,9 +77,9 @@ class LinkedInScraper(BaseWebScraper): async def posts( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape LinkedIn posts from URLs. @@ -108,9 +108,9 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape LinkedIn posts from URLs (sync version). @@ -127,14 +127,14 @@ async def _run(): # POSTS TRIGGER/STATUS/FETCH (Manual Control) # ============================================================================ - async def posts_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def posts_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn posts scrape (manual control).""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( urls=url, dataset_id=self.DATASET_ID_POSTS, sdk_function=sdk_function or "posts_trigger" ) - def posts_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def posts_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn posts scrape (sync version).""" return asyncio.run(self.posts_trigger(url)) @@ -160,9 +160,9 @@ def posts_fetch_sync(self, snapshot_id: str) -> Any: async def jobs( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape LinkedIn jobs from URLs. @@ -190,9 +190,9 @@ async def jobs( def jobs_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Scrape LinkedIn jobs from URLs (sync version).""" async def _run(): @@ -205,14 +205,14 @@ async def _run(): # JOBS TRIGGER/STATUS/FETCH (Manual Control) # ============================================================================ - async def jobs_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def jobs_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn jobs scrape (manual control).""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( urls=url, dataset_id=self.DATASET_ID_JOBS, sdk_function=sdk_function or "jobs_trigger" ) - def jobs_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def jobs_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn jobs scrape (sync version).""" return asyncio.run(self.jobs_trigger(url)) @@ -238,9 +238,9 @@ def jobs_fetch_sync(self, snapshot_id: str) -> Any: async def profiles( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape LinkedIn profiles from URLs. @@ -268,9 +268,9 @@ async def profiles( def profiles_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Scrape LinkedIn profiles from URLs (sync version).""" async def _run(): @@ -283,14 +283,14 @@ async def _run(): # PROFILES TRIGGER/STATUS/FETCH (Manual Control) # ============================================================================ - async def profiles_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def profiles_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn profiles scrape (manual control).""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( urls=url, sdk_function=sdk_function or "profiles_trigger" ) - def profiles_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def profiles_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn profiles scrape (sync version).""" return asyncio.run(self.profiles_trigger(url)) @@ -316,9 +316,9 @@ def profiles_fetch_sync(self, snapshot_id: str) -> Any: async def companies( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape LinkedIn companies from URLs. @@ -348,9 +348,9 @@ async def companies( def companies_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_SHORT, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Scrape LinkedIn companies from URLs (sync version).""" async def _run(): @@ -363,7 +363,7 @@ async def _run(): # COMPANIES TRIGGER/STATUS/FETCH (Manual Control) # ============================================================================ - async def companies_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def companies_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn companies scrape (manual control).""" sdk_function = get_caller_function_name() return await self._trigger_scrape_async( @@ -372,7 +372,7 @@ async def companies_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: sdk_function=sdk_function or "companies_trigger", ) - def companies_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def companies_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger LinkedIn companies scrape (sync version).""" return asyncio.run(self.companies_trigger(url)) @@ -398,10 +398,10 @@ def companies_fetch_sync(self, snapshot_id: str) -> Any: async def _scrape_urls( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, timeout: int, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape URLs using standard async workflow (trigger/poll/fetch). diff --git a/src/brightdata/scrapers/linkedin/search.py b/src/brightdata/scrapers/linkedin/search.py index f72261e..bdf6f42 100644 --- a/src/brightdata/scrapers/linkedin/search.py +++ b/src/brightdata/scrapers/linkedin/search.py @@ -11,12 +11,12 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any -from ...models import ScrapeResult +from ...constants import COST_PER_RECORD_LINKEDIN, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT from ...exceptions import ValidationError +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_LINKEDIN from ..base import ScraperCore @@ -66,9 +66,9 @@ class LinkedInSearchScraper(ScraperCore): async def posts( self, - url: Union[str, List[str]], - start_date: Optional[Union[str, List[str]]] = None, - end_date: Optional[Union[str, List[str]]] = None, + url: str | list[str], + start_date: str | list[str] | None = None, + end_date: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -98,7 +98,7 @@ async def posts( # Build payload - API requires "url" field, not "profile_url" payload = [] for i, profile_url in enumerate(profile_urls): - item: Dict[str, Any] = {"url": profile_url} + item: dict[str, Any] = {"url": profile_url} if start_dates and i < len(start_dates): item["start_date"] = start_dates[i] @@ -117,9 +117,9 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], - start_date: Optional[Union[str, List[str]]] = None, - end_date: Optional[Union[str, List[str]]] = None, + url: str | list[str], + start_date: str | list[str] | None = None, + end_date: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -140,8 +140,8 @@ async def _run(): async def profiles( self, - first_name: Union[str, List[str]], - last_name: Optional[Union[str, List[str]]] = None, + first_name: str | list[str], + last_name: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -168,7 +168,7 @@ async def profiles( # Build payload - API requires url + first_name + last_name payload = [] for i, fname in enumerate(first_names): - item: Dict[str, Any] = { + item: dict[str, Any] = { "url": "https://www.linkedin.com", "first_name": fname, } @@ -184,8 +184,8 @@ async def profiles( def profiles_sync( self, - first_name: Union[str, List[str]], - last_name: Optional[Union[str, List[str]]] = None, + first_name: str | list[str], + last_name: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -206,16 +206,16 @@ async def _run(): async def jobs( self, - url: Optional[Union[str, List[str]]] = None, - location: Optional[Union[str, List[str]]] = None, - keyword: Optional[Union[str, List[str]]] = None, - country: Optional[Union[str, List[str]]] = None, - timeRange: Optional[Union[str, List[str]]] = None, - jobType: Optional[Union[str, List[str]]] = None, - experienceLevel: Optional[Union[str, List[str]]] = None, - remote: Optional[bool] = None, - company: Optional[Union[str, List[str]]] = None, - locationRadius: Optional[Union[str, List[str]]] = None, + url: str | list[str] | None = None, + location: str | list[str] | None = None, + keyword: str | list[str] | None = None, + country: str | list[str] | None = None, + timeRange: str | list[str] | None = None, + jobType: str | list[str] | None = None, + experienceLevel: str | list[str] | None = None, + remote: bool | None = None, + company: str | list[str] | None = None, + locationRadius: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -285,7 +285,7 @@ async def jobs( else: # Use discovery dataset with keyword/location parameters use_discovery = True - item: Dict[str, Any] = {} + item: dict[str, Any] = {} if keywords and i < len(keywords): item["keyword"] = keywords[i] @@ -321,16 +321,16 @@ async def jobs( def jobs_sync( self, - url: Optional[Union[str, List[str]]] = None, - location: Optional[Union[str, List[str]]] = None, - keyword: Optional[Union[str, List[str]]] = None, - country: Optional[Union[str, List[str]]] = None, - timeRange: Optional[Union[str, List[str]]] = None, - jobType: Optional[Union[str, List[str]]] = None, - experienceLevel: Optional[Union[str, List[str]]] = None, - remote: Optional[bool] = None, - company: Optional[Union[str, List[str]]] = None, - locationRadius: Optional[Union[str, List[str]]] = None, + url: str | list[str] | None = None, + location: str | list[str] | None = None, + keyword: str | list[str] | None = None, + country: str | list[str] | None = None, + timeRange: str | list[str] | None = None, + jobType: str | list[str] | None = None, + experienceLevel: str | list[str] | None = None, + remote: bool | None = None, + company: str | list[str] | None = None, + locationRadius: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_SHORT, ) -> ScrapeResult: """ @@ -362,8 +362,8 @@ async def _run(): # ============================================================================ def _normalize_param( - self, param: Optional[Union[str, List[str]]], target_length: int - ) -> Optional[List[str]]: + self, param: str | list[str] | None, target_length: int + ) -> list[str] | None: """ Normalize parameter to list. @@ -385,15 +385,15 @@ def _normalize_param( def _build_linkedin_jobs_search_url( self, - keyword: Optional[str] = None, - location: Optional[str] = None, - country: Optional[str] = None, - time_range: Optional[str] = None, - job_type: Optional[str] = None, - experience_level: Optional[str] = None, - remote: Optional[bool] = None, - company: Optional[str] = None, - location_radius: Optional[str] = None, + keyword: str | None = None, + location: str | None = None, + country: str | None = None, + time_range: str | None = None, + job_type: str | None = None, + experience_level: str | None = None, + remote: bool | None = None, + company: str | None = None, + location_radius: str | None = None, ) -> str: """ Build LinkedIn job search URL from parameters. @@ -498,10 +498,10 @@ def _build_linkedin_jobs_search_url( async def _execute_search( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, timeout: int, - extra_params: Optional[Dict[str, str]] = None, + extra_params: dict[str, str] | None = None, ) -> ScrapeResult: """ Execute search operation via trigger/poll/fetch. diff --git a/src/brightdata/scrapers/perplexity/scraper.py b/src/brightdata/scrapers/perplexity/scraper.py index 58fc9fa..2321b8b 100644 --- a/src/brightdata/scrapers/perplexity/scraper.py +++ b/src/brightdata/scrapers/perplexity/scraper.py @@ -13,15 +13,15 @@ """ import asyncio -from typing import List, Any, Optional, Union +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_PERPLEXITY, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT +from ...exceptions import ValidationError from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_PERPLEXITY -from ...exceptions import ValidationError +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("perplexity") @@ -75,12 +75,12 @@ class PerplexityScraper(BaseWebScraper): async def search( self, - prompt: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, - index: Optional[Union[int, List[int]]] = None, - export_markdown_file: Optional[Union[bool, List[bool]]] = None, + prompt: str | list[str], + country: str | list[str] | None = None, + index: int | list[int] | None = None, + export_markdown_file: bool | list[bool] | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, ) -> ScrapeResult: """ Search Perplexity AI with prompt(s) (async). @@ -168,12 +168,12 @@ async def search( def search_sync( self, - prompt: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, - index: Optional[Union[int, List[int]]] = None, - export_markdown_file: Optional[Union[bool, List[bool]]] = None, + prompt: str | list[str], + country: str | list[str] | None = None, + index: int | list[int] | None = None, + export_markdown_file: bool | list[bool] | None = None, poll_interval: int = DEFAULT_POLL_INTERVAL, - poll_timeout: Optional[int] = None, + poll_timeout: int | None = None, ) -> ScrapeResult: """ Search Perplexity AI with prompt(s) (sync). @@ -206,10 +206,10 @@ async def _run(): async def search_trigger( self, - prompt: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, - index: Optional[Union[int, List[int]]] = None, - export_markdown_file: Optional[Union[bool, List[bool]]] = None, + prompt: str | list[str], + country: str | list[str] | None = None, + index: int | list[int] | None = None, + export_markdown_file: bool | list[bool] | None = None, ) -> ScrapeJob: """ Trigger Perplexity search (async - manual control). @@ -276,10 +276,10 @@ async def search_trigger( def search_trigger_sync( self, - prompt: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, - index: Optional[Union[int, List[int]]] = None, - export_markdown_file: Optional[Union[bool, List[bool]]] = None, + prompt: str | list[str], + country: str | list[str] | None = None, + index: int | list[int] | None = None, + export_markdown_file: bool | list[bool] | None = None, ) -> ScrapeJob: """Trigger Perplexity search (sync wrapper).""" @@ -342,8 +342,8 @@ async def _run(): # ============================================================================ async def scrape_async( - self, urls: Union[str, List[str]], **kwargs - ) -> Union[ScrapeResult, List[ScrapeResult]]: + self, urls: str | list[str], **kwargs + ) -> ScrapeResult | list[ScrapeResult]: """ Perplexity doesn't support URL-based scraping. @@ -354,7 +354,7 @@ async def scrape_async( "Use search() or search_sync() methods instead." ) - def scrape(self, urls: Union[str, List[str]], **kwargs): + def scrape(self, urls: str | list[str], **kwargs): """Perplexity doesn't support URL-based scraping.""" raise NotImplementedError( "Perplexity scraper doesn't support URL-based scraping. " @@ -367,10 +367,10 @@ def scrape(self, urls: Union[str, List[str]], **kwargs): def _normalize_param( self, - param: Optional[Union[Any, List[Any]]], + param: Any | list[Any] | None, target_length: int, default_value: Any = None, - ) -> List[Any]: + ) -> list[Any]: """ Normalize parameter to list of specified length. diff --git a/src/brightdata/scrapers/pinterest/scraper.py b/src/brightdata/scrapers/pinterest/scraper.py index 388efab..6c35d9a 100644 --- a/src/brightdata/scrapers/pinterest/scraper.py +++ b/src/brightdata/scrapers/pinterest/scraper.py @@ -15,15 +15,15 @@ """ import asyncio -from typing import List, Any, Union +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("pinterest") @@ -66,9 +66,9 @@ class PinterestScraper(BaseWebScraper): async def posts( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect Pinterest post (pin) data by URL (async). @@ -98,9 +98,9 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect Pinterest post data by URL (sync).""" async def _run(): @@ -113,7 +113,7 @@ async def _run(): async def posts_trigger( self, - url: Union[str, List[str]], + url: str | list[str], ) -> ScrapeJob: """Trigger Pinterest posts collection (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -129,7 +129,7 @@ async def posts_trigger( cost_per_record=self.COST_PER_RECORD, ) - def posts_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def posts_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger Pinterest posts collection (sync).""" return asyncio.run(self.posts_trigger(url)) @@ -155,9 +155,9 @@ def posts_fetch_sync(self, snapshot_id: str) -> Any: async def profiles( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect Pinterest profile data by URL (async). @@ -187,9 +187,9 @@ async def profiles( def profiles_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect Pinterest profile data by URL (sync).""" async def _run(): @@ -202,7 +202,7 @@ async def _run(): async def profiles_trigger( self, - url: Union[str, List[str]], + url: str | list[str], ) -> ScrapeJob: """Trigger Pinterest profiles collection (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -218,7 +218,7 @@ async def profiles_trigger( cost_per_record=self.COST_PER_RECORD, ) - def profiles_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def profiles_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger Pinterest profiles collection (sync).""" return asyncio.run(self.profiles_trigger(url)) @@ -244,10 +244,10 @@ def profiles_fetch_sync(self, snapshot_id: str) -> Any: async def _scrape_urls( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, timeout: int, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Scrape URLs using standard async workflow.""" is_single = isinstance(url, str) url_list = [url] if is_single else url diff --git a/src/brightdata/scrapers/pinterest/search.py b/src/brightdata/scrapers/pinterest/search.py index c116833..873a9a8 100644 --- a/src/brightdata/scrapers/pinterest/search.py +++ b/src/brightdata/scrapers/pinterest/search.py @@ -16,16 +16,16 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( DEFAULT_COST_PER_RECORD, - DEFAULT_TIMEOUT_MEDIUM, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_MEDIUM, ) +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ..base import ScraperCore class PinterestSearchScraper(ScraperCore): @@ -72,7 +72,7 @@ class PinterestSearchScraper(ScraperCore): async def _execute_discovery( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, discover_by: str, timeout: int, @@ -114,11 +114,11 @@ async def _execute_discovery( async def posts_by_keyword( self, - keyword: Union[str, List[str]], - videos_only: Optional[bool] = None, - new_posts: Optional[bool] = None, - top_posts: Optional[bool] = None, - food: Optional[bool] = None, + keyword: str | list[str], + videos_only: bool | None = None, + new_posts: bool | None = None, + top_posts: bool | None = None, + food: bool | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -147,7 +147,7 @@ async def posts_by_keyword( payload = [] for kw in keywords: - item: Dict[str, Any] = {"keyword": kw} + item: dict[str, Any] = {"keyword": kw} if videos_only is not None: item["videos_only"] = videos_only if new_posts is not None: @@ -167,11 +167,11 @@ async def posts_by_keyword( def posts_by_keyword_sync( self, - keyword: Union[str, List[str]], - videos_only: Optional[bool] = None, - new_posts: Optional[bool] = None, - top_posts: Optional[bool] = None, - food: Optional[bool] = None, + keyword: str | list[str], + videos_only: bool | None = None, + new_posts: bool | None = None, + top_posts: bool | None = None, + food: bool | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of posts_by_keyword().""" @@ -190,11 +190,11 @@ async def _run(): async def posts_by_profile( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -222,7 +222,7 @@ async def posts_by_profile( payload = [] for u in urls: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if num_of_posts is not None: item["num_of_posts"] = num_of_posts if posts_to_not_include: @@ -242,11 +242,11 @@ async def posts_by_profile( def posts_by_profile_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of posts_by_profile().""" @@ -265,7 +265,7 @@ async def _run(): async def profiles( self, - keyword: Union[str, List[str]], + keyword: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -296,7 +296,7 @@ async def profiles( def profiles_sync( self, - keyword: Union[str, List[str]], + keyword: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of profiles().""" diff --git a/src/brightdata/scrapers/reddit/scraper.py b/src/brightdata/scrapers/reddit/scraper.py index 3351cf6..310c56c 100644 --- a/src/brightdata/scrapers/reddit/scraper.py +++ b/src/brightdata/scrapers/reddit/scraper.py @@ -19,15 +19,15 @@ """ import asyncio -from typing import List, Any, Optional, Union, Dict +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("reddit") @@ -83,9 +83,9 @@ class RedditScraper(BaseWebScraper): async def posts( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect Reddit post data by URL (async). @@ -129,9 +129,9 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect Reddit post data by URL (sync).""" async def _run(): @@ -142,7 +142,7 @@ async def _run(): # --- Posts Trigger/Status/Fetch --- - async def posts_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def posts_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger Reddit posts collection (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [{"url": u} for u in url_list] @@ -157,7 +157,7 @@ async def posts_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: cost_per_record=self.COST_PER_RECORD, ) - def posts_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def posts_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger Reddit posts collection (sync).""" return asyncio.run(self.posts_trigger(url)) @@ -183,10 +183,10 @@ def posts_fetch_sync(self, snapshot_id: str) -> Any: async def posts_by_keyword( self, - keyword: Union[str, List[str]], - date: Optional[Union[str, List[str]]] = None, - num_of_posts: Optional[Union[int, List[int]]] = None, - sort_by: Optional[Union[str, List[str]]] = None, + keyword: str | list[str], + date: str | list[str] | None = None, + num_of_posts: int | list[int] | None = None, + sort_by: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -219,7 +219,7 @@ async def posts_by_keyword( payload = [] for i in range(batch_size): - item: Dict[str, Any] = {"keyword": keywords[i]} + item: dict[str, Any] = {"keyword": keywords[i]} if dates[i] is not None: item["date"] = dates[i] if nums[i] is not None: @@ -243,10 +243,10 @@ async def posts_by_keyword( def posts_by_keyword_sync( self, - keyword: Union[str, List[str]], - date: Optional[Union[str, List[str]]] = None, - num_of_posts: Optional[Union[int, List[int]]] = None, - sort_by: Optional[Union[str, List[str]]] = None, + keyword: str | list[str], + date: str | list[str] | None = None, + num_of_posts: int | list[int] | None = None, + sort_by: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Discover Reddit posts by keyword search (sync).""" @@ -263,9 +263,9 @@ async def _run(): async def posts_by_subreddit( self, - url: Union[str, List[str]], - sort_by: Optional[Union[str, List[str]]] = None, - sort_by_time: Optional[Union[str, List[str]]] = None, + url: str | list[str], + sort_by: str | list[str] | None = None, + sort_by_time: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -300,7 +300,7 @@ async def posts_by_subreddit( payload = [] for i in range(batch_size): - item: Dict[str, Any] = {"url": urls[i]} + item: dict[str, Any] = {"url": urls[i]} if sorts[i] is not None: item["sort_by"] = sorts[i] if sort_times[i] is not None: @@ -322,9 +322,9 @@ async def posts_by_subreddit( def posts_by_subreddit_sync( self, - url: Union[str, List[str]], - sort_by: Optional[Union[str, List[str]]] = None, - sort_by_time: Optional[Union[str, List[str]]] = None, + url: str | list[str], + sort_by: str | list[str] | None = None, + sort_by_time: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Discover Reddit posts from subreddit URL (sync).""" @@ -341,12 +341,12 @@ async def _run(): async def comments( self, - url: Union[str, List[str]], - days_back: Optional[int] = None, - load_all_replies: Optional[bool] = None, - comment_limit: Optional[int] = None, + url: str | list[str], + days_back: int | None = None, + load_all_replies: bool | None = None, + comment_limit: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect Reddit comments by URL (async). @@ -378,7 +378,7 @@ async def comments( payload = [] for u in url_list: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if days_back is not None: item["days_back"] = days_back if load_all_replies is not None: @@ -405,12 +405,12 @@ async def comments( def comments_sync( self, - url: Union[str, List[str]], - days_back: Optional[int] = None, - load_all_replies: Optional[bool] = None, - comment_limit: Optional[int] = None, + url: str | list[str], + days_back: int | None = None, + load_all_replies: bool | None = None, + comment_limit: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect Reddit comments by URL (sync).""" async def _run(): @@ -423,17 +423,17 @@ async def _run(): async def comments_trigger( self, - url: Union[str, List[str]], - days_back: Optional[int] = None, - load_all_replies: Optional[bool] = None, - comment_limit: Optional[int] = None, + url: str | list[str], + days_back: int | None = None, + load_all_replies: bool | None = None, + comment_limit: int | None = None, ) -> ScrapeJob: """Trigger Reddit comments collection (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [] for u in url_list: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if days_back is not None: item["days_back"] = days_back if load_all_replies is not None: @@ -454,10 +454,10 @@ async def comments_trigger( def comments_trigger_sync( self, - url: Union[str, List[str]], - days_back: Optional[int] = None, - load_all_replies: Optional[bool] = None, - comment_limit: Optional[int] = None, + url: str | list[str], + days_back: int | None = None, + load_all_replies: bool | None = None, + comment_limit: int | None = None, ) -> ScrapeJob: """Trigger Reddit comments collection (sync).""" return asyncio.run(self.comments_trigger(url, days_back, load_all_replies, comment_limit)) @@ -484,10 +484,10 @@ def comments_fetch_sync(self, snapshot_id: str) -> Any: def _normalize_param( self, - param: Optional[Union[Any, List[Any]]], + param: Any | list[Any] | None, target_length: int, default_value: Any = None, - ) -> List[Any]: + ) -> list[Any]: """Normalize parameter to list of specified length.""" if param is None: return [default_value] * target_length diff --git a/src/brightdata/scrapers/registry.py b/src/brightdata/scrapers/registry.py index 9ba05bc..93a2b0a 100644 --- a/src/brightdata/scrapers/registry.py +++ b/src/brightdata/scrapers/registry.py @@ -12,7 +12,7 @@ import logging import pkgutil from functools import lru_cache -from typing import Dict, Type, Optional, List + import tldextract # Configure logger for registry operations @@ -20,7 +20,7 @@ # Global registry mapping domain → scraper class -_SCRAPER_REGISTRY: Dict[str, Type] = {} +_SCRAPER_REGISTRY: dict[str, type] = {} def register(domain: str): @@ -51,7 +51,7 @@ def register(domain: str): >>> # Returns AmazonScraper class """ - def decorator(cls: Type) -> Type: + def decorator(cls: type) -> type: _SCRAPER_REGISTRY[domain.lower()] = cls return cls @@ -94,7 +94,7 @@ def _import_all_scrapers(): ) -def get_scraper_for(url: str) -> Optional[Type]: +def get_scraper_for(url: str) -> type | None: """ Get scraper class for a URL based on domain. @@ -133,7 +133,7 @@ def get_scraper_for(url: str) -> Optional[Type]: return _SCRAPER_REGISTRY.get(domain) -def get_registered_platforms() -> List[str]: +def get_registered_platforms() -> list[str]: """ Get list of all registered platform domains. @@ -169,7 +169,7 @@ def is_platform_supported(url: str) -> bool: # For backward compatibility and explicit access -def get_registry() -> Dict[str, Type]: +def get_registry() -> dict[str, type]: """ Get the complete scraper registry. diff --git a/src/brightdata/scrapers/tiktok/scraper.py b/src/brightdata/scrapers/tiktok/scraper.py index f250bf4..439adb5 100644 --- a/src/brightdata/scrapers/tiktok/scraper.py +++ b/src/brightdata/scrapers/tiktok/scraper.py @@ -22,15 +22,15 @@ """ import asyncio -from typing import List, Any, Optional, Union, Dict +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_TIKTOK, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, COST_PER_RECORD_TIKTOK +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("tiktok") @@ -84,10 +84,10 @@ class TikTokScraper(BaseWebScraper): async def profiles( self, - url: Union[str, List[str]], - country: Optional[str] = None, + url: str | list[str], + country: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect TikTok profile data by URL (async). @@ -119,10 +119,10 @@ async def profiles( def profiles_sync( self, - url: Union[str, List[str]], - country: Optional[str] = None, + url: str | list[str], + country: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect TikTok profile data by URL (sync).""" async def _run(): @@ -135,8 +135,8 @@ async def _run(): async def profiles_trigger( self, - url: Union[str, List[str]], - country: Optional[str] = None, + url: str | list[str], + country: str | None = None, ) -> ScrapeJob: """Trigger TikTok profiles collection (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -152,9 +152,7 @@ async def profiles_trigger( cost_per_record=self.COST_PER_RECORD, ) - def profiles_trigger_sync( - self, url: Union[str, List[str]], country: Optional[str] = None - ) -> ScrapeJob: + def profiles_trigger_sync(self, url: str | list[str], country: str | None = None) -> ScrapeJob: """Trigger TikTok profiles collection (sync).""" return asyncio.run(self.profiles_trigger(url, country)) @@ -180,10 +178,10 @@ def profiles_fetch_sync(self, snapshot_id: str) -> Any: async def posts( self, - url: Union[str, List[str]], - country: Optional[str] = None, + url: str | list[str], + country: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect TikTok post data by URL (async). @@ -214,10 +212,10 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], - country: Optional[str] = None, + url: str | list[str], + country: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect TikTok post data by URL (sync).""" async def _run(): @@ -228,9 +226,7 @@ async def _run(): # --- Posts Trigger/Status/Fetch --- - async def posts_trigger( - self, url: Union[str, List[str]], country: Optional[str] = None - ) -> ScrapeJob: + async def posts_trigger(self, url: str | list[str], country: str | None = None) -> ScrapeJob: """Trigger TikTok posts collection (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [{"url": u, "country": country or ""} for u in url_list] @@ -245,9 +241,7 @@ async def posts_trigger( cost_per_record=self.COST_PER_RECORD, ) - def posts_trigger_sync( - self, url: Union[str, List[str]], country: Optional[str] = None - ) -> ScrapeJob: + def posts_trigger_sync(self, url: str | list[str], country: str | None = None) -> ScrapeJob: """Trigger TikTok posts collection (sync).""" return asyncio.run(self.posts_trigger(url, country)) @@ -273,9 +267,9 @@ def posts_fetch_sync(self, snapshot_id: str) -> Any: async def comments( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect TikTok comments from video URL(s) (async). @@ -318,9 +312,9 @@ async def comments( def comments_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect TikTok comments (sync).""" async def _run(): @@ -331,7 +325,7 @@ async def _run(): # --- Comments Trigger/Status/Fetch --- - async def comments_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def comments_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger TikTok comments collection (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [{"url": u} for u in url_list] @@ -346,7 +340,7 @@ async def comments_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: cost_per_record=self.COST_PER_RECORD, ) - def comments_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def comments_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger TikTok comments collection (sync).""" return asyncio.run(self.comments_trigger(url)) @@ -372,9 +366,9 @@ def comments_fetch_sync(self, snapshot_id: str) -> Any: async def posts_by_profile_fast( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect TikTok posts from profile using Fast API (async). @@ -405,9 +399,9 @@ async def posts_by_profile_fast( def posts_by_profile_fast_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect TikTok posts from profile using Fast API (sync).""" async def _run(): @@ -422,9 +416,9 @@ async def _run(): async def posts_by_url_fast( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect TikTok posts from various URLs using Fast API (async). @@ -474,9 +468,9 @@ async def posts_by_url_fast( def posts_by_url_fast_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect TikTok posts by URL using Fast API (sync).""" async def _run(): @@ -491,10 +485,10 @@ async def _run(): async def posts_by_search_url_fast( self, - url: Union[str, List[str]], - num_of_posts: Optional[Union[int, List[int]]] = None, - country: Optional[Union[str, List[str]]] = None, - start_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | list[int] | None = None, + country: str | list[str] | None = None, + start_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -523,7 +517,7 @@ async def posts_by_search_url_fast( payload = [] for i in range(batch_size): - item: Dict[str, Any] = {"url": urls[i], "country": countries[i]} + item: dict[str, Any] = {"url": urls[i], "country": countries[i]} if nums[i] is not None: item["num_of_posts"] = nums[i] if start_date: @@ -544,10 +538,10 @@ async def posts_by_search_url_fast( def posts_by_search_url_fast_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[Union[int, List[int]]] = None, - country: Optional[Union[str, List[str]]] = None, - start_date: Optional[str] = None, + url: str | list[str], + num_of_posts: int | list[int] | None = None, + country: str | list[str] | None = None, + start_date: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Collect TikTok posts from search URL using Fast API (sync).""" @@ -566,11 +560,11 @@ async def _run(): async def _scrape_urls( self, - url: Union[str, List[str]], + url: str | list[str], dataset_id: str, timeout: int, - country: Optional[str] = None, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + country: str | None = None, + ) -> ScrapeResult | list[ScrapeResult]: """Scrape URLs using standard async workflow.""" is_single = isinstance(url, str) url_list = [url] if is_single else url @@ -621,10 +615,10 @@ async def _scrape_urls( def _normalize_param( self, - param: Optional[Union[Any, List[Any]]], + param: Any | list[Any] | None, target_length: int, default_value: Any = None, - ) -> List[Any]: + ) -> list[Any]: """Normalize parameter to list of specified length.""" if param is None: return [default_value] * target_length diff --git a/src/brightdata/scrapers/tiktok/search.py b/src/brightdata/scrapers/tiktok/search.py index b7eede2..24bc1fe 100644 --- a/src/brightdata/scrapers/tiktok/search.py +++ b/src/brightdata/scrapers/tiktok/search.py @@ -16,16 +16,16 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_TIKTOK, - DEFAULT_TIMEOUT_MEDIUM, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_MEDIUM, ) +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ..base import ScraperCore class TikTokSearchScraper(ScraperCore): @@ -74,7 +74,7 @@ class TikTokSearchScraper(ScraperCore): async def _execute_discovery( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, discover_by: str, timeout: int, @@ -112,10 +112,10 @@ async def _execute_discovery( def _normalize_param( self, - param: Optional[Union[Any, List[Any]]], + param: Any | list[Any] | None, target_length: int, default_value: Any = None, - ) -> List[Any]: + ) -> list[Any]: """Normalize parameter to list of specified length.""" if param is None: return [default_value] * target_length @@ -137,8 +137,8 @@ def _normalize_param( async def profiles( self, - search_url: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, + search_url: str | list[str], + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -176,8 +176,8 @@ async def profiles( def profiles_sync( self, - search_url: Union[str, List[str]], - country: Optional[Union[str, List[str]]] = None, + search_url: str | list[str], + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of profiles().""" @@ -194,10 +194,10 @@ async def _run(): async def posts_by_keyword( self, - keyword: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - country: Optional[Union[str, List[str]]] = None, + keyword: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -229,7 +229,7 @@ async def posts_by_keyword( payload = [] for i in range(batch_size): - item: Dict[str, Any] = { + item: dict[str, Any] = { "search_keyword": keywords[i], "country": countries[i], } @@ -248,10 +248,10 @@ async def posts_by_keyword( def posts_by_keyword_sync( self, - keyword: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - country: Optional[Union[str, List[str]]] = None, + keyword: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of posts_by_keyword().""" @@ -270,15 +270,15 @@ async def _run(): async def posts_by_profile( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - what_to_collect: Optional[str] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - post_type: Optional[str] = None, - country: Optional[str] = None, - sort_by: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + what_to_collect: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + post_type: str | None = None, + country: str | None = None, + sort_by: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -310,7 +310,7 @@ async def posts_by_profile( payload = [] for u in urls: - item: Dict[str, Any] = {"url": u} + item: dict[str, Any] = {"url": u} if num_of_posts is not None: item["num_of_posts"] = num_of_posts if posts_to_not_include: @@ -338,15 +338,15 @@ async def posts_by_profile( def posts_by_profile_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - what_to_collect: Optional[str] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - post_type: Optional[str] = None, - country: Optional[str] = None, - sort_by: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + what_to_collect: str | None = None, + start_date: str | None = None, + end_date: str | None = None, + post_type: str | None = None, + country: str | None = None, + sort_by: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of posts_by_profile().""" @@ -374,7 +374,7 @@ async def _run(): async def posts_by_url( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -406,7 +406,7 @@ async def posts_by_url( def posts_by_url_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of posts_by_url().""" diff --git a/src/brightdata/scrapers/workflow.py b/src/brightdata/scrapers/workflow.py index 02d0952..8fcc115 100644 --- a/src/brightdata/scrapers/workflow.py +++ b/src/brightdata/scrapers/workflow.py @@ -7,12 +7,13 @@ 3. Fetch results when ready """ -from typing import List, Dict, Any, Optional, Callable +from collections.abc import Callable from datetime import datetime, timezone +from typing import Any -from ..models import ScrapeResult +from ..constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT from ..exceptions import APIError -from ..constants import DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT, DEFAULT_COST_PER_RECORD +from ..models import ScrapeResult from ..utils.polling import poll_until_ready from .api_client import DatasetAPIClient @@ -28,7 +29,7 @@ class WorkflowExecutor: def __init__( self, api_client: DatasetAPIClient, - platform_name: Optional[str] = None, + platform_name: str | None = None, cost_per_record: float = DEFAULT_COST_PER_RECORD, ): """ @@ -45,15 +46,15 @@ def __init__( async def execute( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, poll_interval: int = DEFAULT_POLL_INTERVAL, poll_timeout: int = DEFAULT_POLL_TIMEOUT, include_errors: bool = True, - normalize_func: Optional[Callable[[Any], Any]] = None, - sdk_function: Optional[str] = None, - extra_params: Optional[Dict[str, str]] = None, - limit_per_input: Optional[int] = None, + normalize_func: Callable[[Any], Any] | None = None, + sdk_function: str | None = None, + extra_params: dict[str, str] | None = None, + limit_per_input: int | None = None, ) -> ScrapeResult: """ Execute complete trigger/poll/fetch workflow. @@ -88,7 +89,7 @@ async def execute( success=False, url="", status="error", - error=f"Trigger failed: {str(e)}", + error=f"Trigger failed: {e!s}", platform=self.platform_name, method="web_scraper", trigger_sent_at=trigger_sent_at, @@ -127,7 +128,7 @@ async def _poll_and_fetch( poll_timeout: int, trigger_sent_at: datetime, snapshot_id_received_at: datetime, - normalize_func: Optional[Callable[[Any], Any]] = None, + normalize_func: Callable[[Any], Any] | None = None, ) -> ScrapeResult: """ Poll snapshot until ready, then fetch results. diff --git a/src/brightdata/scrapers/x/scraper.py b/src/brightdata/scrapers/x/scraper.py index c25d82a..da0fa95 100644 --- a/src/brightdata/scrapers/x/scraper.py +++ b/src/brightdata/scrapers/x/scraper.py @@ -19,15 +19,15 @@ """ import asyncio -from typing import List, Any, Union, Optional +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("x") @@ -79,10 +79,10 @@ class XScraper(BaseWebScraper): async def posts( self, - url: Union[str, List[str]], - limit_per_input: Optional[int] = None, + url: str | list[str], + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect X posts by post URL (async). @@ -122,10 +122,10 @@ async def posts( def posts_sync( self, - url: Union[str, List[str]], - limit_per_input: Optional[int] = None, + url: str | list[str], + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect X posts by post URL (sync).""" async def _run(): @@ -138,8 +138,8 @@ async def _run(): async def posts_trigger( self, - url: Union[str, List[str]], - limit_per_input: Optional[int] = None, + url: str | list[str], + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X posts collection by URL (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -159,8 +159,8 @@ async def posts_trigger( def posts_trigger_sync( self, - url: Union[str, List[str]], - limit_per_input: Optional[int] = None, + url: str | list[str], + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X posts collection by URL (sync).""" return asyncio.run(self.posts_trigger(url, limit_per_input=limit_per_input)) @@ -187,10 +187,10 @@ def posts_fetch_sync(self, snapshot_id: str) -> Any: async def posts_by_profile( self, - url: Union[str, List[str]], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -231,10 +231,10 @@ async def posts_by_profile( def posts_by_profile_sync( self, - url: Union[str, List[str]], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Discover X posts from a profile URL (sync).""" @@ -255,10 +255,10 @@ async def _run(): async def posts_by_profile_trigger( self, - url: Union[str, List[str]], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X posts discovery by profile URL (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -281,10 +281,10 @@ async def posts_by_profile_trigger( def posts_by_profile_trigger_sync( self, - url: Union[str, List[str]], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X posts discovery by profile URL (sync).""" return asyncio.run( @@ -315,10 +315,10 @@ def posts_by_profile_fetch_sync(self, snapshot_id: str) -> Any: async def posts_by_profiles_array( self, - urls: List[str], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + urls: list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -356,10 +356,10 @@ async def posts_by_profiles_array( def posts_by_profiles_array_sync( self, - urls: List[str], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + urls: list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Discover X posts from several profile URLs in one request (sync).""" @@ -380,10 +380,10 @@ async def _run(): async def posts_by_profiles_array_trigger( self, - urls: List[str], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + urls: list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X posts discovery by profiles array (manual control).""" validate_url_list(urls) @@ -404,10 +404,10 @@ async def posts_by_profiles_array_trigger( def posts_by_profiles_array_trigger_sync( self, - urls: List[str], - start_date: Optional[str] = None, - end_date: Optional[str] = None, - limit_per_input: Optional[int] = None, + urls: list[str], + start_date: str | None = None, + end_date: str | None = None, + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X posts discovery by profiles array (sync).""" return asyncio.run( @@ -438,11 +438,11 @@ def posts_by_profiles_array_fetch_sync(self, snapshot_id: str) -> Any: async def profiles( self, - url: Union[str, List[str]], - max_number_of_posts: Optional[int] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + max_number_of_posts: int | None = None, + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect X profile data by profile URL (async). @@ -488,11 +488,11 @@ async def profiles( def profiles_sync( self, - url: Union[str, List[str]], - max_number_of_posts: Optional[int] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + max_number_of_posts: int | None = None, + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect X profile data by profile URL (sync).""" async def _run(): @@ -510,9 +510,9 @@ async def _run(): async def profiles_trigger( self, - url: Union[str, List[str]], - max_number_of_posts: Optional[int] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + max_number_of_posts: int | None = None, + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X profiles collection by URL (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -537,9 +537,9 @@ async def profiles_trigger( def profiles_trigger_sync( self, - url: Union[str, List[str]], - max_number_of_posts: Optional[int] = None, - limit_per_input: Optional[int] = None, + url: str | list[str], + max_number_of_posts: int | None = None, + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X profiles collection by URL (sync).""" return asyncio.run( @@ -570,10 +570,10 @@ def profiles_fetch_sync(self, snapshot_id: str) -> Any: async def profiles_by_username( self, - user_name: Union[str, List[str]], - limit_per_input: Optional[int] = None, + user_name: str | list[str], + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Discover an X profile by handle / user name (async). @@ -608,10 +608,10 @@ async def profiles_by_username( def profiles_by_username_sync( self, - user_name: Union[str, List[str]], - limit_per_input: Optional[int] = None, + user_name: str | list[str], + limit_per_input: int | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Discover an X profile by handle / user name (sync).""" async def _run(): @@ -626,8 +626,8 @@ async def _run(): async def profiles_by_username_trigger( self, - user_name: Union[str, List[str]], - limit_per_input: Optional[int] = None, + user_name: str | list[str], + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X profile discovery by user name (manual control).""" name_list = [user_name] if isinstance(user_name, str) else user_name @@ -648,8 +648,8 @@ async def profiles_by_username_trigger( def profiles_by_username_trigger_sync( self, - user_name: Union[str, List[str]], - limit_per_input: Optional[int] = None, + user_name: str | list[str], + limit_per_input: int | None = None, ) -> ScrapeJob: """Trigger X profile discovery by user name (sync).""" return asyncio.run( diff --git a/src/brightdata/scrapers/youtube/scraper.py b/src/brightdata/scrapers/youtube/scraper.py index fc13a03..c3e9602 100644 --- a/src/brightdata/scrapers/youtube/scraper.py +++ b/src/brightdata/scrapers/youtube/scraper.py @@ -18,15 +18,15 @@ """ import asyncio -from typing import List, Any, Optional, Union +from typing import Any -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_YOUTUBE, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, COST_PER_RECORD_YOUTUBE +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("youtube") @@ -77,11 +77,11 @@ class YouTubeScraper(BaseWebScraper): async def videos( self, - url: Union[str, List[str]], - country: Optional[str] = None, - transcription_language: Optional[str] = None, + url: str | list[str], + country: str | None = None, + transcription_language: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect YouTube video data by URL (async). @@ -135,11 +135,11 @@ async def videos( def videos_sync( self, - url: Union[str, List[str]], - country: Optional[str] = None, - transcription_language: Optional[str] = None, + url: str | list[str], + country: str | None = None, + transcription_language: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect YouTube video data by URL (sync).""" async def _run(): @@ -152,9 +152,9 @@ async def _run(): async def videos_trigger( self, - url: Union[str, List[str]], - country: Optional[str] = None, - transcription_language: Optional[str] = None, + url: str | list[str], + country: str | None = None, + transcription_language: str | None = None, ) -> ScrapeJob: """Trigger YouTube videos collection (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -178,9 +178,9 @@ async def videos_trigger( def videos_trigger_sync( self, - url: Union[str, List[str]], - country: Optional[str] = None, - transcription_language: Optional[str] = None, + url: str | list[str], + country: str | None = None, + transcription_language: str | None = None, ) -> ScrapeJob: """Trigger YouTube videos collection (sync).""" return asyncio.run(self.videos_trigger(url, country, transcription_language)) @@ -207,9 +207,9 @@ def videos_fetch_sync(self, snapshot_id: str) -> Any: async def channels( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect YouTube channel/profile data by URL (async). @@ -253,9 +253,9 @@ async def channels( def channels_sync( self, - url: Union[str, List[str]], + url: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect YouTube channel data by URL (sync).""" async def _run(): @@ -266,7 +266,7 @@ async def _run(): # --- Channels Trigger/Status/Fetch --- - async def channels_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: + async def channels_trigger(self, url: str | list[str]) -> ScrapeJob: """Trigger YouTube channels collection (manual control).""" url_list = [url] if isinstance(url, str) else url payload = [{"url": u} for u in url_list] @@ -281,7 +281,7 @@ async def channels_trigger(self, url: Union[str, List[str]]) -> ScrapeJob: cost_per_record=self.COST_PER_RECORD, ) - def channels_trigger_sync(self, url: Union[str, List[str]]) -> ScrapeJob: + def channels_trigger_sync(self, url: str | list[str]) -> ScrapeJob: """Trigger YouTube channels collection (sync).""" return asyncio.run(self.channels_trigger(url)) @@ -307,12 +307,12 @@ def channels_fetch_sync(self, snapshot_id: str) -> Any: async def comments( self, - url: Union[str, List[str]], - num_of_comments: Optional[int] = None, - load_replies: Optional[bool] = None, - sort_by: Optional[str] = None, + url: str | list[str], + num_of_comments: int | None = None, + load_replies: bool | None = None, + sort_by: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Collect YouTube comments from video URL(s) (async). @@ -370,12 +370,12 @@ async def comments( def comments_sync( self, - url: Union[str, List[str]], - num_of_comments: Optional[int] = None, - load_replies: Optional[bool] = None, - sort_by: Optional[str] = None, + url: str | list[str], + num_of_comments: int | None = None, + load_replies: bool | None = None, + sort_by: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """Collect YouTube comments (sync).""" async def _run(): @@ -388,10 +388,10 @@ async def _run(): async def comments_trigger( self, - url: Union[str, List[str]], - num_of_comments: Optional[int] = None, - load_replies: Optional[bool] = None, - sort_by: Optional[str] = None, + url: str | list[str], + num_of_comments: int | None = None, + load_replies: bool | None = None, + sort_by: str | None = None, ) -> ScrapeJob: """Trigger YouTube comments collection (manual control).""" url_list = [url] if isinstance(url, str) else url @@ -419,10 +419,10 @@ async def comments_trigger( def comments_trigger_sync( self, - url: Union[str, List[str]], - num_of_comments: Optional[int] = None, - load_replies: Optional[bool] = None, - sort_by: Optional[str] = None, + url: str | list[str], + num_of_comments: int | None = None, + load_replies: bool | None = None, + sort_by: str | None = None, ) -> ScrapeJob: """Trigger YouTube comments collection (sync).""" return asyncio.run(self.comments_trigger(url, num_of_comments, load_replies, sort_by)) diff --git a/src/brightdata/scrapers/youtube/search.py b/src/brightdata/scrapers/youtube/search.py index 4869770..08456b9 100644 --- a/src/brightdata/scrapers/youtube/search.py +++ b/src/brightdata/scrapers/youtube/search.py @@ -19,16 +19,16 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_YOUTUBE, - DEFAULT_TIMEOUT_MEDIUM, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_MEDIUM, ) +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ..base import ScraperCore class YouTubeSearchScraper(ScraperCore): @@ -78,7 +78,7 @@ class YouTubeSearchScraper(ScraperCore): async def _execute_discovery( self, - payload: List[Dict[str, Any]], + payload: list[dict[str, Any]], dataset_id: str, discover_by: str, timeout: int, @@ -105,10 +105,10 @@ async def _execute_discovery( def _normalize_param( self, - param: Optional[Union[Any, List[Any]]], + param: Any | list[Any] | None, target_length: int, default_value: Any = None, - ) -> List[Any]: + ) -> list[Any]: """Normalize parameter to list of specified length.""" if param is None: return [default_value] * target_length @@ -130,9 +130,9 @@ def _normalize_param( async def videos_by_explore( self, - url: Union[str, List[str]], - all_tabs: Optional[bool] = None, - country: Optional[Union[str, List[str]]] = None, + url: str | list[str], + all_tabs: bool | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -161,7 +161,7 @@ async def videos_by_explore( payload = [] for i in range(batch_size): # API expects empty strings for optional fields - item: Dict[str, Any] = { + item: dict[str, Any] = { "url": urls[i], "country": countries[i], } @@ -178,9 +178,9 @@ async def videos_by_explore( def videos_by_explore_sync( self, - url: Union[str, List[str]], - all_tabs: Optional[bool] = None, - country: Optional[Union[str, List[str]]] = None, + url: str | list[str], + all_tabs: bool | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of videos_by_explore().""" @@ -197,12 +197,12 @@ async def _run(): async def videos_by_hashtag( self, - hashtag: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - country: Optional[Union[str, List[str]]] = None, + hashtag: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -235,7 +235,7 @@ async def videos_by_hashtag( payload = [] for i in range(batch_size): # API expects empty strings for optional fields - item: Dict[str, Any] = { + item: dict[str, Any] = { "hashtag": hashtags[i], "start_date": start_date or "", "end_date": end_date or "", @@ -256,12 +256,12 @@ async def videos_by_hashtag( def videos_by_hashtag_sync( self, - hashtag: Union[str, List[str]], - num_of_posts: Optional[int] = None, - posts_to_not_include: Optional[List[str]] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - country: Optional[Union[str, List[str]]] = None, + hashtag: str | list[str], + num_of_posts: int | None = None, + posts_to_not_include: list[str] | None = None, + start_date: str | None = None, + end_date: str | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of videos_by_hashtag().""" @@ -286,11 +286,11 @@ async def _run(): async def videos_by_keyword( self, - keyword: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - country: Optional[Union[str, List[str]]] = None, + keyword: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -323,7 +323,7 @@ async def videos_by_keyword( payload = [] for i in range(batch_size): # API expects empty strings for optional fields - item: Dict[str, Any] = { + item: dict[str, Any] = { "keyword": keywords[i], "start_date": start_date or "", "end_date": end_date or "", @@ -342,11 +342,11 @@ async def videos_by_keyword( def videos_by_keyword_sync( self, - keyword: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - country: Optional[Union[str, List[str]]] = None, + keyword: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of videos_by_keyword().""" @@ -365,13 +365,13 @@ async def _run(): async def videos_by_search_filters( self, - keyword_search: Union[str, List[str]], - upload_date: Optional[str] = None, - video_type: Optional[str] = None, - duration: Optional[str] = None, - features: Optional[str] = None, - sort_by: Optional[str] = None, - country: Optional[Union[str, List[str]]] = None, + keyword_search: str | list[str], + upload_date: str | None = None, + video_type: str | None = None, + duration: str | None = None, + features: str | None = None, + sort_by: str | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -409,7 +409,7 @@ async def videos_by_search_filters( payload = [] for i in range(batch_size): # API expects empty strings for optional fields - item: Dict[str, Any] = { + item: dict[str, Any] = { "keyword_search": keywords[i], "upload_date": upload_date or "", "type": video_type or "", @@ -429,13 +429,13 @@ async def videos_by_search_filters( def videos_by_search_filters_sync( self, - keyword_search: Union[str, List[str]], - upload_date: Optional[str] = None, - video_type: Optional[str] = None, - duration: Optional[str] = None, - features: Optional[str] = None, - sort_by: Optional[str] = None, - country: Optional[Union[str, List[str]]] = None, + keyword_search: str | list[str], + upload_date: str | None = None, + video_type: str | None = None, + duration: str | None = None, + features: str | None = None, + sort_by: str | None = None, + country: str | list[str] | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of videos_by_search_filters().""" @@ -461,14 +461,14 @@ async def _run(): async def videos_by_channel( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - order_by: Optional[str] = None, - time_period: Optional[str] = None, - country: Optional[str] = None, - transcription_language: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, + order_by: str | None = None, + time_period: str | None = None, + country: str | None = None, + transcription_language: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -503,7 +503,7 @@ async def videos_by_channel( payload = [] for u in urls: # API expects empty strings for optional fields - item: Dict[str, Any] = { + item: dict[str, Any] = { "url": u, "start_date": start_date or "", "end_date": end_date or "", @@ -525,14 +525,14 @@ async def videos_by_channel( def videos_by_channel_sync( self, - url: Union[str, List[str]], - num_of_posts: Optional[int] = None, - start_date: Optional[str] = None, - end_date: Optional[str] = None, - order_by: Optional[str] = None, - time_period: Optional[str] = None, - country: Optional[str] = None, - transcription_language: Optional[str] = None, + url: str | list[str], + num_of_posts: int | None = None, + start_date: str | None = None, + end_date: str | None = None, + order_by: str | None = None, + time_period: str | None = None, + country: str | None = None, + transcription_language: str | None = None, timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of videos_by_channel().""" @@ -559,7 +559,7 @@ async def _run(): async def channels_by_keyword( self, - keyword: Union[str, List[str]], + keyword: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """ @@ -592,7 +592,7 @@ async def channels_by_keyword( def channels_by_keyword_sync( self, - keyword: Union[str, List[str]], + keyword: str | list[str], timeout: int = DEFAULT_TIMEOUT_MEDIUM, ) -> ScrapeResult: """Synchronous version of channels_by_keyword().""" diff --git a/src/brightdata/serp/__init__.py b/src/brightdata/serp/__init__.py index 1ee43bc..43bc090 100644 --- a/src/brightdata/serp/__init__.py +++ b/src/brightdata/serp/__init__.py @@ -1,15 +1,15 @@ """SERP API services.""" from .base import BaseSERPService -from .google import GoogleSERPService from .bing import BingSERPService -from .yandex import YandexSERPService +from .google import GoogleSERPService from .service import SearchService +from .yandex import YandexSERPService __all__ = [ "BaseSERPService", - "GoogleSERPService", "BingSERPService", - "YandexSERPService", + "GoogleSERPService", "SearchService", + "YandexSERPService", ] diff --git a/src/brightdata/serp/base.py b/src/brightdata/serp/base.py index 3102d2a..de8fdd6 100644 --- a/src/brightdata/serp/base.py +++ b/src/brightdata/serp/base.py @@ -1,24 +1,25 @@ """Base SERP service with separated responsibilities.""" import asyncio -import aiohttp import json import re import time import warnings -from typing import Union, List, Optional, Dict, Any, Tuple from datetime import datetime, timezone +from http import HTTPStatus +from typing import Any + +import aiohttp -from .url_builder import BaseURLBuilder -from .data_normalizer import BaseDataNormalizer from ..core.engine import AsyncEngine -from ..models import SearchResult -from http import HTTPStatus from ..exceptions import ValidationError -from ..utils.validation import validate_zone_name -from ..utils.retry import retry_with_backoff +from ..models import SearchResult from ..utils.function_detection import get_caller_function_name +from ..utils.retry import retry_with_backoff +from ..utils.validation import validate_zone_name from ..web_unlocker.async_client import AsyncUnblockerClient +from .data_normalizer import BaseDataNormalizer +from .url_builder import BaseURLBuilder class BaseSERPService: @@ -39,14 +40,14 @@ class BaseSERPService: # opts in to engine-specific server-side parsing via the payload field # instead of (or in addition to) the URL-level brd_json=1 flag. Leave None # for engines that use brd_json=1 (Google) or have no parser (Yandex). - DATA_FORMAT: Optional[str] = None + DATA_FORMAT: str | None = None def __init__( self, engine: AsyncEngine, url_builder: BaseURLBuilder, data_normalizer: BaseDataNormalizer, - timeout: Optional[int] = None, + timeout: int | None = None, max_retries: int = 3, ): """ @@ -70,9 +71,9 @@ def __init__( async def search( self, - query: Union[str, List[str]], + query: str | list[str], zone: str, - location: Optional[str] = None, + location: str | None = None, language: str = "en", device: str = "desktop", num_results: int = 10, @@ -80,7 +81,7 @@ async def search( poll_interval: int = 2, poll_timeout: int = 30, **kwargs, - ) -> Union[SearchResult, List[SearchResult]]: + ) -> SearchResult | list[SearchResult]: """ Perform search asynchronously. @@ -178,7 +179,7 @@ async def _search_single_async( self, query: str, zone: str, - location: Optional[str], + location: str | None, language: str, device: str, num_results: int, @@ -245,7 +246,7 @@ async def _execute_serp_request( search_url: str, zone: str, trigger_sent_at: datetime, - ) -> Tuple[Dict[str, Any], datetime, Optional[str]]: + ) -> tuple[dict[str, Any], datetime, str | None]: """ Execute a single SERP request and parse response. @@ -333,13 +334,13 @@ async def _make_request(): try: return await retry_with_backoff(_make_request, max_retries=self.max_retries) except Exception as e: - return ({}, datetime.now(timezone.utc), f"Request error: {str(e)}") + return ({}, datetime.now(timezone.utc), f"Request error: {e!s}") async def _search_with_pagination( self, query: str, zone: str, - location: Optional[str], + location: str | None, language: str, device: str, num_results: int, @@ -353,7 +354,7 @@ async def _search_with_pagination( trigger_sent_at = datetime.now(timezone.utc) pagination_start_time = time.time() - all_results: List[Dict[str, Any]] = [] + all_results: list[dict[str, Any]] = [] pages_fetched = 0 current_start = 0 google_total_results = None @@ -447,14 +448,14 @@ async def _search_with_pagination( async def _search_multiple_async( self, - queries: List[str], + queries: list[str], zone: str, - location: Optional[str], + location: str | None, language: str, device: str, num_results: int, **kwargs, - ) -> List[SearchResult]: + ) -> list[SearchResult]: """Execute multiple search queries concurrently.""" tasks = [ self._search_single_async( @@ -478,7 +479,7 @@ async def _search_multiple_async( SearchResult( success=False, query={"q": queries[i]}, - error=f"Exception: {str(result)}", + error=f"Exception: {result!s}", search_engine=self.SEARCH_ENGINE, trigger_sent_at=datetime.now(timezone.utc), data_fetched_at=datetime.now(timezone.utc), @@ -493,7 +494,7 @@ async def _search_single_async_unblocker( self, query: str, zone: str, - location: Optional[str], + location: str | None, language: str, device: str, num_results: int, @@ -586,7 +587,7 @@ async def _search_single_async_unblocker( return SearchResult( success=False, query={"q": query}, - error=f"Failed to fetch results: {str(e)}", + error=f"Failed to fetch results: {e!s}", search_engine=self.SEARCH_ENGINE, trigger_sent_at=trigger_sent_at, data_fetched_at=data_fetched_at, @@ -607,16 +608,16 @@ async def _search_single_async_unblocker( async def _search_multiple_async_unblocker( self, - queries: List[str], + queries: list[str], zone: str, - location: Optional[str], + location: str | None, language: str, device: str, num_results: int, poll_interval: int, poll_timeout: int, **kwargs, - ) -> List[SearchResult]: + ) -> list[SearchResult]: """ Execute multiple searches using async unblocker. @@ -649,7 +650,7 @@ async def _search_multiple_async_unblocker( SearchResult( success=False, query={"q": queries[i]}, - error=f"Exception: {str(result)}", + error=f"Exception: {result!s}", search_engine=self.SEARCH_ENGINE, trigger_sent_at=datetime.now(timezone.utc), data_fetched_at=datetime.now(timezone.utc), @@ -660,7 +661,7 @@ async def _search_multiple_async_unblocker( return processed_results - def _validate_queries(self, queries: List[str]) -> None: + def _validate_queries(self, queries: list[str]) -> None: """Validate search queries.""" if not queries: raise ValidationError("Query list cannot be empty") diff --git a/src/brightdata/serp/bing.py b/src/brightdata/serp/bing.py index 64862e9..88e067d 100644 --- a/src/brightdata/serp/bing.py +++ b/src/brightdata/serp/bing.py @@ -1,10 +1,9 @@ """Bing SERP service.""" -from typing import Optional +from ..core.engine import AsyncEngine from .base import BaseSERPService -from .url_builder import BingURLBuilder from .data_normalizer import BingDataNormalizer -from ..core.engine import AsyncEngine +from .url_builder import BingURLBuilder class BingSERPService(BaseSERPService): @@ -16,7 +15,7 @@ class BingSERPService(BaseSERPService): def __init__( self, engine: AsyncEngine, - timeout: Optional[int] = None, + timeout: int | None = None, max_retries: int = 3, ): """Initialize Bing SERP service.""" diff --git a/src/brightdata/serp/data_normalizer.py b/src/brightdata/serp/data_normalizer.py index 3d3b31b..31a73f1 100644 --- a/src/brightdata/serp/data_normalizer.py +++ b/src/brightdata/serp/data_normalizer.py @@ -3,6 +3,7 @@ import warnings from abc import ABC, abstractmethod from typing import Any + from ..types import NormalizedSERPData @@ -12,7 +13,6 @@ class BaseDataNormalizer(ABC): @abstractmethod def normalize(self, data: Any) -> NormalizedSERPData: """Normalize SERP data to consistent format.""" - pass class GoogleDataNormalizer(BaseDataNormalizer): diff --git a/src/brightdata/serp/google.py b/src/brightdata/serp/google.py index 3bf662a..a8c64a9 100644 --- a/src/brightdata/serp/google.py +++ b/src/brightdata/serp/google.py @@ -1,10 +1,9 @@ """Google SERP service.""" -from typing import Optional +from ..core.engine import AsyncEngine from .base import BaseSERPService -from .url_builder import GoogleURLBuilder from .data_normalizer import GoogleDataNormalizer -from ..core.engine import AsyncEngine +from .url_builder import GoogleURLBuilder class GoogleSERPService(BaseSERPService): @@ -25,7 +24,7 @@ class GoogleSERPService(BaseSERPService): def __init__( self, engine: AsyncEngine, - timeout: Optional[int] = None, + timeout: int | None = None, max_retries: int = 3, ): """Initialize Google SERP service.""" diff --git a/src/brightdata/serp/service.py b/src/brightdata/serp/service.py index 8aff7d7..a324af7 100644 --- a/src/brightdata/serp/service.py +++ b/src/brightdata/serp/service.py @@ -6,21 +6,21 @@ All methods are async-only. For sync usage, use SyncBrightDataClient. """ -from typing import Optional, Union, List, TYPE_CHECKING +from typing import TYPE_CHECKING from ..models import SearchResult if TYPE_CHECKING: from ..client import BrightDataClient - from .google import GoogleSERPService - from .bing import BingSERPService - from .yandex import YandexSERPService from ..scrapers.amazon.search import AmazonSearchScraper - from ..scrapers.linkedin.search import LinkedInSearchScraper from ..scrapers.chatgpt.search import ChatGPTSearchService from ..scrapers.instagram.search import InstagramSearchScraper + from ..scrapers.linkedin.search import LinkedInSearchScraper from ..scrapers.tiktok.search import TikTokSearchScraper from ..scrapers.youtube.search import YouTubeSearchScraper + from .bing import BingSERPService + from .google import GoogleSERPService + from .yandex import YandexSERPService class SearchService: @@ -45,27 +45,27 @@ class SearchService: def __init__(self, client: "BrightDataClient"): """Initialize search service with client reference.""" self._client = client - self._google_service: Optional["GoogleSERPService"] = None - self._bing_service: Optional["BingSERPService"] = None - self._yandex_service: Optional["YandexSERPService"] = None - self._amazon_search: Optional["AmazonSearchScraper"] = None - self._linkedin_search: Optional["LinkedInSearchScraper"] = None - self._chatgpt_search: Optional["ChatGPTSearchService"] = None - self._instagram_search: Optional["InstagramSearchScraper"] = None - self._tiktok_search: Optional["TikTokSearchScraper"] = None - self._youtube_search: Optional["YouTubeSearchScraper"] = None + self._google_service: GoogleSERPService | None = None + self._bing_service: BingSERPService | None = None + self._yandex_service: YandexSERPService | None = None + self._amazon_search: AmazonSearchScraper | None = None + self._linkedin_search: LinkedInSearchScraper | None = None + self._chatgpt_search: ChatGPTSearchService | None = None + self._instagram_search: InstagramSearchScraper | None = None + self._tiktok_search: TikTokSearchScraper | None = None + self._youtube_search: YouTubeSearchScraper | None = None self._pinterest_search = None async def google( self, - query: Union[str, List[str]], - location: Optional[str] = None, + query: str | list[str], + location: str | None = None, language: str = "en", device: str = "desktop", num_results: int = 10, - zone: Optional[str] = None, + zone: str | None = None, **kwargs, - ) -> Union[SearchResult, List[SearchResult]]: + ) -> SearchResult | list[SearchResult]: """ Search Google asynchronously. @@ -110,13 +110,13 @@ async def google( async def bing( self, - query: Union[str, List[str]], - location: Optional[str] = None, + query: str | list[str], + location: str | None = None, language: str = "en", num_results: int = 10, - zone: Optional[str] = None, + zone: str | None = None, **kwargs, - ) -> Union[SearchResult, List[SearchResult]]: + ) -> SearchResult | list[SearchResult]: """Search Bing asynchronously.""" from .bing import BingSERPService @@ -138,13 +138,13 @@ async def bing( async def yandex( self, - query: Union[str, List[str]], - location: Optional[str] = None, + query: str | list[str], + location: str | None = None, language: str = "ru", num_results: int = 10, - zone: Optional[str] = None, + zone: str | None = None, **kwargs, - ) -> Union[SearchResult, List[SearchResult]]: + ) -> SearchResult | list[SearchResult]: """Search Yandex asynchronously.""" from .yandex import YandexSERPService diff --git a/src/brightdata/serp/url_builder.py b/src/brightdata/serp/url_builder.py index c66af13..f80fe87 100644 --- a/src/brightdata/serp/url_builder.py +++ b/src/brightdata/serp/url_builder.py @@ -1,9 +1,9 @@ """URL builder for SERP search engines.""" from abc import ABC, abstractmethod -from typing import Optional from urllib.parse import quote_plus -from ..utils.location import LocationService, LocationFormat + +from ..utils.location import LocationFormat, LocationService class BaseURLBuilder(ABC): @@ -13,14 +13,13 @@ class BaseURLBuilder(ABC): def build( self, query: str, - location: Optional[str] = None, + location: str | None = None, language: str = "en", device: str = "desktop", num_results: int = 10, **kwargs, ) -> str: """Build search URL.""" - pass class GoogleURLBuilder(BaseURLBuilder): @@ -29,7 +28,7 @@ class GoogleURLBuilder(BaseURLBuilder): def build( self, query: str, - location: Optional[str] = None, + location: str | None = None, language: str = "en", device: str = "desktop", num_results: int = 10, @@ -75,7 +74,7 @@ class BingURLBuilder(BaseURLBuilder): def build( self, query: str, - location: Optional[str] = None, + location: str | None = None, language: str = "en", device: str = "desktop", num_results: int = 10, @@ -106,7 +105,7 @@ class YandexURLBuilder(BaseURLBuilder): def build( self, query: str, - location: Optional[str] = None, + location: str | None = None, language: str = "en", device: str = "desktop", num_results: int = 10, diff --git a/src/brightdata/serp/yandex.py b/src/brightdata/serp/yandex.py index 6173876..266cc5f 100644 --- a/src/brightdata/serp/yandex.py +++ b/src/brightdata/serp/yandex.py @@ -1,10 +1,9 @@ """Yandex SERP service.""" -from typing import Optional +from ..core.engine import AsyncEngine from .base import BaseSERPService -from .url_builder import YandexURLBuilder from .data_normalizer import YandexDataNormalizer -from ..core.engine import AsyncEngine +from .url_builder import YandexURLBuilder class YandexSERPService(BaseSERPService): @@ -15,7 +14,7 @@ class YandexSERPService(BaseSERPService): def __init__( self, engine: AsyncEngine, - timeout: Optional[int] = None, + timeout: int | None = None, max_retries: int = 3, ): """Initialize Yandex SERP service.""" diff --git a/src/brightdata/sync_client.py b/src/brightdata/sync_client.py index 8a6995a..c22253a 100644 --- a/src/brightdata/sync_client.py +++ b/src/brightdata/sync_client.py @@ -6,14 +6,14 @@ import asyncio import logging -from typing import Optional, List, Dict, Any +from typing import Any logger = logging.getLogger(__name__) -from .client import BrightDataClient from .browser.service import BrowserService -from .models import ScrapeResult, SearchResult +from .client import BrightDataClient from .discover.models import DiscoverResult, DiscoverSnapshot +from .models import ScrapeResult, SearchResult from .types import AccountInfo @@ -35,20 +35,20 @@ class SyncBrightDataClient: def __init__( self, - token: Optional[str] = None, + token: str | None = None, timeout: int = 30, - web_unlocker_zone: Optional[str] = None, - serp_zone: Optional[str] = None, - browser_username: Optional[str] = None, - browser_password: Optional[str] = None, - browser_host: Optional[str] = None, - browser_port: Optional[int] = None, + web_unlocker_zone: str | None = None, + serp_zone: str | None = None, + browser_username: str | None = None, + browser_password: str | None = None, + browser_host: str | None = None, + browser_port: int | None = None, auto_create_zones: bool = True, validate_token: bool = False, - rate_limit: Optional[float] = None, + rate_limit: float | None = None, rate_period: float = 1.0, ssl_verify: bool = True, - ssl_ca_cert: Optional[str] = None, + ssl_ca_cert: str | None = None, ): """ Initialize sync client. @@ -102,12 +102,12 @@ def __init__( ssl_ca_cert=ssl_ca_cert, ) self._validate_token = validate_token - self._loop: Optional[asyncio.AbstractEventLoop] = None - self._scrape: Optional["SyncScrapeService"] = None - self._search: Optional["SyncSearchService"] = None - self._crawler: Optional["SyncCrawlerService"] = None - self._scraper_studio: Optional["SyncScraperStudioService"] = None - self._datasets: Optional["SyncDatasetsClient"] = None + self._loop: asyncio.AbstractEventLoop | None = None + self._scrape: SyncScrapeService | None = None + self._search: SyncSearchService | None = None + self._crawler: SyncCrawlerService | None = None + self._scraper_studio: SyncScraperStudioService | None = None + self._datasets: SyncDatasetsClient | None = None def __enter__(self): """Initialize persistent event loop and async client.""" @@ -173,7 +173,7 @@ def _run(self, coro): # Utility Methods # ======================================== - def list_zones(self) -> List[Dict[str, Any]]: + def list_zones(self) -> list[dict[str, Any]]: """List all active zones.""" return self._run(self._async_client.list_zones()) diff --git a/src/brightdata/types.py b/src/brightdata/types.py index 43a7807..d18a4bd 100644 --- a/src/brightdata/types.py +++ b/src/brightdata/types.py @@ -4,7 +4,8 @@ This module provides type definitions for API responses used internally. """ -from typing import TypedDict, Optional, List, Dict, Any +from typing import Any, TypedDict + from typing_extensions import NotRequired @@ -14,15 +15,15 @@ class ZoneInfo(TypedDict, total=False): name: str zone: NotRequired[str] status: NotRequired[str] - plan: NotRequired[Dict[str, Any]] + plan: NotRequired[dict[str, Any]] created: NotRequired[str] class AccountInfo(TypedDict): """Account information returned by get_account_info().""" - customer_id: Optional[str] - zones: List[ZoneInfo] + customer_id: str | None + zones: list[ZoneInfo] zone_count: int token_valid: bool retrieved_at: str @@ -57,22 +58,22 @@ class SERPKnowledgePanel(TypedDict, total=False): class NormalizedSERPData(TypedDict, total=False): """Normalized SERP data structure.""" - results: List[SERPOrganicResult] + results: list[SERPOrganicResult] total_results: NotRequired[int] featured_snippet: NotRequired[SERPFeaturedSnippet] knowledge_panel: NotRequired[SERPKnowledgePanel] - people_also_ask: NotRequired[List[Dict[str, str]]] - related_searches: NotRequired[List[str]] - ads: NotRequired[List[Dict[str, Any]]] - search_info: NotRequired[Dict[str, Any]] + people_also_ask: NotRequired[list[dict[str, str]]] + related_searches: NotRequired[list[str]] + ads: NotRequired[list[dict[str, Any]]] + search_info: NotRequired[dict[str, Any]] raw_html: NotRequired[str] __all__ = [ - "ZoneInfo", "AccountInfo", - "SERPOrganicResult", + "NormalizedSERPData", "SERPFeaturedSnippet", "SERPKnowledgePanel", - "NormalizedSERPData", + "SERPOrganicResult", + "ZoneInfo", ] diff --git a/src/brightdata/utils/function_detection.py b/src/brightdata/utils/function_detection.py index 5386d5e..379672f 100644 --- a/src/brightdata/utils/function_detection.py +++ b/src/brightdata/utils/function_detection.py @@ -6,10 +6,9 @@ """ import inspect -from typing import Optional -def get_caller_function_name(skip_frames: int = 1) -> Optional[str]: +def get_caller_function_name(skip_frames: int = 1) -> str | None: """ Get the name of the calling function. diff --git a/src/brightdata/utils/location.py b/src/brightdata/utils/location.py index 97b03d6..a21d752 100644 --- a/src/brightdata/utils/location.py +++ b/src/brightdata/utils/location.py @@ -1,6 +1,5 @@ """Location parsing utilities for SERP services.""" -from typing import Dict from enum import Enum @@ -16,7 +15,7 @@ class LocationService: """Unified location parsing service for all SERP engines.""" # Common country mappings - COUNTRY_MAP: Dict[str, str] = { + COUNTRY_MAP: dict[str, str] = { "united states": "us", "usa": "us", "united kingdom": "gb", @@ -49,7 +48,7 @@ class LocationService: } # Yandex-specific numeric region IDs - YANDEX_REGION_MAP: Dict[str, str] = { + YANDEX_REGION_MAP: dict[str, str] = { "russia": "225", "ukraine": "187", "belarus": "149", diff --git a/src/brightdata/utils/polling.py b/src/brightdata/utils/polling.py index 94c4678..39f0d45 100644 --- a/src/brightdata/utils/polling.py +++ b/src/brightdata/utils/polling.py @@ -11,12 +11,13 @@ from __future__ import annotations import asyncio -from typing import Any, List, Callable, Awaitable +from collections.abc import Awaitable, Callable from datetime import datetime, timezone +from typing import Any -from ..models import ScrapeResult from ..constants import DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT from ..exceptions import DataNotReadyError +from ..models import ScrapeResult async def poll_until_ready( @@ -71,7 +72,7 @@ async def poll_until_ready( ... ) """ start_time = datetime.now(timezone.utc) - snapshot_polled_at: List[datetime] = [] + snapshot_polled_at: list[datetime] = [] # Use provided timestamps or create new ones trigger_sent = trigger_sent_at or start_time @@ -107,7 +108,7 @@ async def poll_until_ready( success=False, url="", status="error", - error=f"Failed to get status: {str(e)}", + error=f"Failed to get status: {e!s}", snapshot_id=snapshot_id, platform=platform, method=method or "web_scraper", @@ -134,7 +135,7 @@ async def poll_until_ready( success=False, url="", status="error", - error=f"Failed to fetch results: {str(e)}", + error=f"Failed to fetch results: {e!s}", snapshot_id=snapshot_id, platform=platform, method=method or "web_scraper", diff --git a/src/brightdata/utils/retry.py b/src/brightdata/utils/retry.py index 6cb91d6..2c84726 100644 --- a/src/brightdata/utils/retry.py +++ b/src/brightdata/utils/retry.py @@ -1,7 +1,9 @@ """Retry logic with exponential backoff.""" import asyncio -from typing import Callable, Awaitable, TypeVar, Optional, List, Type +from collections.abc import Awaitable, Callable +from typing import TypeVar + from ..exceptions import APIError, NetworkError T = TypeVar("T") @@ -13,7 +15,7 @@ async def retry_with_backoff( initial_delay: float = 1.0, max_delay: float = 60.0, backoff_factor: float = 2.0, - retryable_exceptions: Optional[List[Type[Exception]]] = None, + retryable_exceptions: list[type[Exception]] | None = None, ) -> T: """ Retry function with exponential backoff. diff --git a/src/brightdata/utils/ssl_helpers.py b/src/brightdata/utils/ssl_helpers.py index a709651..04cf6d0 100644 --- a/src/brightdata/utils/ssl_helpers.py +++ b/src/brightdata/utils/ssl_helpers.py @@ -5,8 +5,8 @@ particularly common on macOS systems. """ -import sys import ssl +import sys try: import aiohttp @@ -125,4 +125,4 @@ def get_ssl_error_message(error: Exception) -> str: https://github.com/brightdata/brightdata-python-sdk/blob/main/docs/troubleshooting.md#ssl-certificate-errors """ - return base_message + fix_instructions + f"\n\nOriginal error: {str(error)}" + return base_message + fix_instructions + f"\n\nOriginal error: {error!s}" diff --git a/src/brightdata/utils/url.py b/src/brightdata/utils/url.py index 5127b32..a564dfd 100644 --- a/src/brightdata/utils/url.py +++ b/src/brightdata/utils/url.py @@ -1,10 +1,9 @@ """URL utilities.""" from urllib.parse import urlparse -from typing import Optional -def extract_root_domain(url: str) -> Optional[str]: +def extract_root_domain(url: str) -> str | None: """ Extract root domain from URL. @@ -21,8 +20,7 @@ def extract_root_domain(url: str) -> Optional[str]: if ":" in netloc: netloc = netloc.split(":")[0] - if netloc.startswith("www."): - netloc = netloc[4:] + netloc = netloc.removeprefix("www.") return netloc if netloc else None except Exception: diff --git a/src/brightdata/utils/validation.py b/src/brightdata/utils/validation.py index c38a52c..07d7ec6 100644 --- a/src/brightdata/utils/validation.py +++ b/src/brightdata/utils/validation.py @@ -2,7 +2,7 @@ import re from urllib.parse import urlparse -from typing import List + from ..exceptions import ValidationError @@ -31,7 +31,7 @@ def validate_url(url: str) -> None: raise ValidationError(f"Invalid URL format: {url}") from e -def validate_url_list(urls: List[str]) -> None: +def validate_url_list(urls: list[str]) -> None: """ Validate list of URLs. diff --git a/src/brightdata/web_unlocker/async_client.py b/src/brightdata/web_unlocker/async_client.py index 3b6d295..a601c9d 100644 --- a/src/brightdata/web_unlocker/async_client.py +++ b/src/brightdata/web_unlocker/async_client.py @@ -19,7 +19,8 @@ - See devdocs/web_unlocker_async_inspection.md for details """ -from typing import Optional, Any +from typing import Any + from ..core.engine import AsyncEngine from ..exceptions import APIError @@ -69,9 +70,9 @@ async def trigger( self, zone: str, url: str, - customer: Optional[str] = None, + customer: str | None = None, **kwargs, # Additional params like country, format, etc. - ) -> Optional[str]: + ) -> str | None: """ Trigger async unblocker request. @@ -112,7 +113,7 @@ async def trigger( response_id = response.headers.get("x-response-id") return response_id - async def get_status(self, zone: str, response_id: str, customer: Optional[str] = None) -> str: + async def get_status(self, zone: str, response_id: str, customer: str | None = None) -> str: """ Check if response is ready. @@ -157,7 +158,7 @@ async def fetch_result( zone: str, response_id: str, response_format: str = "json", - customer: Optional[str] = None, + customer: str | None = None, ) -> Any: """ Fetch results when ready. diff --git a/src/brightdata/web_unlocker/base.py b/src/brightdata/web_unlocker/base.py index 31cc7be..c8040dd 100644 --- a/src/brightdata/web_unlocker/base.py +++ b/src/brightdata/web_unlocker/base.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import Any + from ..core.engine import AsyncEngine @@ -30,4 +31,3 @@ async def _execute_async(self, *args: Any, **kwargs: Any) -> Any: This method should be implemented by subclasses to perform the actual async API operation. """ - pass diff --git a/src/brightdata/web_unlocker/service.py b/src/brightdata/web_unlocker/service.py index db2feca..6e3a68f 100644 --- a/src/brightdata/web_unlocker/service.py +++ b/src/brightdata/web_unlocker/service.py @@ -3,26 +3,26 @@ All methods are async-only. For sync usage, use SyncBrightDataClient. """ -from typing import Union, List, Optional, Dict, Any -from datetime import datetime, timezone import asyncio +from datetime import datetime, timezone +from http import HTTPStatus +from typing import Any -from .base import BaseAPI -from .async_client import AsyncUnblockerClient +from ..exceptions import APIError, ValidationError from ..models import ScrapeResult +from ..utils.function_detection import get_caller_function_name +from ..utils.url import extract_root_domain from ..utils.validation import ( + validate_country_code, + validate_http_method, + validate_response_format, + validate_timeout, validate_url, validate_url_list, validate_zone_name, - validate_country_code, - validate_timeout, - validate_response_format, - validate_http_method, ) -from ..utils.url import extract_root_domain -from ..utils.function_detection import get_caller_function_name -from http import HTTPStatus -from ..exceptions import ValidationError, APIError +from .async_client import AsyncUnblockerClient +from .base import BaseAPI class WebUnlockerService(BaseAPI): @@ -58,16 +58,16 @@ async def _execute_async(self, *args: Any, **kwargs: Any) -> Any: async def scrape_async( self, - url: Union[str, List[str]], + url: str | list[str], zone: str, country: str = "", response_format: str = "raw", method: str = "GET", - timeout: Optional[int] = None, + timeout: int | None = None, mode: str = "sync", poll_interval: int = 2, poll_timeout: int = 180, - ) -> Union[ScrapeResult, List[ScrapeResult]]: + ) -> ScrapeResult | list[ScrapeResult]: """ Scrape URL(s) asynchronously using Web Unlocker API. @@ -161,12 +161,12 @@ async def _scrape_single_async( country: str, response_format: str, method: str, - timeout: Optional[int], + timeout: int | None, ) -> ScrapeResult: """Scrape a single URL.""" trigger_sent_at = datetime.now(timezone.utc) - payload: Dict[str, Any] = { + payload: dict[str, Any] = { "zone": zone, "url": url, "format": response_format, @@ -192,7 +192,7 @@ async def _scrape_single_async( try: data = await response.json() except (ValueError, TypeError) as e: - raise APIError(f"Failed to parse JSON response: {str(e)}") + raise APIError(f"Failed to parse JSON response: {e!s}") else: data = await response.text() @@ -233,7 +233,7 @@ async def _scrape_single_async( success=False, url=url, status="error", - error=f"Unexpected error: {str(e)}", + error=f"Unexpected error: {e!s}", method="web_unlocker", trigger_sent_at=trigger_sent_at, data_fetched_at=data_fetched_at, @@ -241,13 +241,13 @@ async def _scrape_single_async( async def _scrape_multiple_async( self, - urls: List[str], + urls: list[str], zone: str, country: str, response_format: str, method: str, - timeout: Optional[int], - ) -> List[ScrapeResult]: + timeout: int | None, + ) -> list[ScrapeResult]: """Scrape multiple URLs concurrently.""" tasks = [ self._scrape_single_async( @@ -263,7 +263,7 @@ async def _scrape_multiple_async( results = await asyncio.gather(*tasks, return_exceptions=True) - processed_results: List[ScrapeResult] = [] + processed_results: list[ScrapeResult] = [] for i, result in enumerate(results): if isinstance(result, Exception): processed_results.append( @@ -271,7 +271,7 @@ async def _scrape_multiple_async( success=False, url=urls[i], status="error", - error=f"Exception: {str(result)}", + error=f"Exception: {result!s}", trigger_sent_at=datetime.now(timezone.utc), data_fetched_at=datetime.now(timezone.utc), ) @@ -315,7 +315,7 @@ async def _scrape_single_async_unblocker( success=False, url=url, status="error", - error=f"Failed to trigger async request: {str(e)}", + error=f"Failed to trigger async request: {e!s}", method="web_unlocker", trigger_sent_at=trigger_sent_at, data_fetched_at=datetime.now(timezone.utc), @@ -358,7 +358,7 @@ async def _scrape_single_async_unblocker( success=False, url=url, status="error", - error=f"Failed to check status: {str(e)}", + error=f"Failed to check status: {e!s}", method="web_unlocker", trigger_sent_at=trigger_sent_at, data_fetched_at=datetime.now(timezone.utc), @@ -393,7 +393,7 @@ async def _scrape_single_async_unblocker( success=False, url=url, status="error", - error=f"Failed to fetch results: {str(e)}", + error=f"Failed to fetch results: {e!s}", method="web_unlocker", trigger_sent_at=trigger_sent_at, data_fetched_at=data_fetched_at, @@ -415,14 +415,14 @@ async def _scrape_single_async_unblocker( async def _scrape_multiple_async_unblocker( self, - urls: List[str], + urls: list[str], zone: str, country: str, response_format: str, method: str, poll_interval: int, poll_timeout: int, - ) -> List[ScrapeResult]: + ) -> list[ScrapeResult]: """Execute multiple scrapes using async unblocker.""" tasks = [ self._scrape_single_async_unblocker( @@ -441,7 +441,7 @@ async def _scrape_multiple_async_unblocker( results = await asyncio.gather(*tasks, return_exceptions=True) # Process results, converting exceptions to ScrapeResult errors - processed_results: List[ScrapeResult] = [] + processed_results: list[ScrapeResult] = [] for i, result in enumerate(results): if isinstance(result, Exception): processed_results.append( @@ -449,7 +449,7 @@ async def _scrape_multiple_async_unblocker( success=False, url=urls[i], status="error", - error=f"Exception: {str(result)}", + error=f"Exception: {result!s}", method="web_unlocker", trigger_sent_at=datetime.now(timezone.utc), data_fetched_at=datetime.now(timezone.utc), diff --git a/tests/conftest.py b/tests/conftest.py index edcce14..49e2817 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ """Shared test fixtures for Bright Data SDK tests.""" -import sys import os +import sys +from typing import Any from unittest.mock import AsyncMock, MagicMock -from typing import Any, Dict, Optional import pytest @@ -24,7 +24,7 @@ def __init__( status: int = 200, json_data: Any = None, text_data: str = "", - headers: Optional[Dict[str, str]] = None, + headers: dict[str, str] | None = None, ): self.status = status self._json_data = json_data diff --git a/tests/unit/test_async_unblocker.py b/tests/unit/test_async_unblocker.py index bc75ed1..13ac4ed 100644 --- a/tests/unit/test_async_unblocker.py +++ b/tests/unit/test_async_unblocker.py @@ -1,14 +1,13 @@ """Tests for web_unlocker/async_client.py — Trigger, status, and fetch operations.""" -import pytest from unittest.mock import AsyncMock, MagicMock -from brightdata.web_unlocker.async_client import AsyncUnblockerClient -from brightdata.exceptions import APIError +import pytest +from brightdata.exceptions import APIError +from brightdata.web_unlocker.async_client import AsyncUnblockerClient from tests.conftest import MockContextManager - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- diff --git a/tests/unit/test_cli_credentials.py b/tests/unit/test_cli_credentials.py index 3c2fd5f..c956586 100644 --- a/tests/unit/test_cli_credentials.py +++ b/tests/unit/test_cli_credentials.py @@ -13,8 +13,8 @@ import pytest import brightdata.client as client_module -from brightdata.client import BrightDataClient from brightdata.cli_credentials import _cli_credentials_path, read_cli_credentials +from brightdata.client import BrightDataClient from brightdata.core.engine import AsyncEngine from brightdata.exceptions import ValidationError diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 5e12494..4bfce4b 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,17 +1,16 @@ """Tests for client.py — BrightDataClient init, services, context manager, API methods.""" -import pytest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from brightdata.client import BrightDataClient -from brightdata.exceptions import ValidationError, AuthenticationError, APIError -from brightdata.scrapers.service import ScrapeService -from brightdata.serp.service import SearchService from brightdata.crawler.service import CrawlerService from brightdata.datasets import DatasetsClient - -from tests.conftest import MockResponse, MockContextManager - +from brightdata.exceptions import APIError, AuthenticationError, ValidationError +from brightdata.scrapers.service import ScrapeService +from brightdata.serp.service import SearchService +from tests.conftest import MockContextManager, MockResponse # --------------------------------------------------------------------------- # Token loading diff --git a/tests/unit/test_colorless_service_verbs.py b/tests/unit/test_colorless_service_verbs.py index c40eea3..85a1f7f 100644 --- a/tests/unit/test_colorless_service_verbs.py +++ b/tests/unit/test_colorless_service_verbs.py @@ -12,16 +12,16 @@ Mocked at the api_client / _poll_once seam (not raw aiohttp), per the plan. """ -import pytest from unittest.mock import AsyncMock, MagicMock -from brightdata.scrapers.amazon import AmazonScraper -from brightdata.scrapers.job import ScrapeJob -from brightdata.discover.service import DiscoverService -from brightdata.models import ScrapeResult +import pytest + from brightdata.discover.models import DiscoverResult +from brightdata.discover.service import DiscoverService from brightdata.exceptions import APIError - +from brightdata.models import ScrapeResult +from brightdata.scrapers.amazon import AmazonScraper +from brightdata.scrapers.job import ScrapeJob # --------------------------------------------------------------------------- # Helpers diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 3d7ee11..a0700f4 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -3,15 +3,13 @@ import json from datetime import datetime, timezone - from brightdata.models import ( BaseResult, + CrawlResult, ScrapeResult, SearchResult, - CrawlResult, ) - # --------------------------------------------------------------------------- # BaseResult # --------------------------------------------------------------------------- diff --git a/tests/unit/test_payloads.py b/tests/unit/test_payloads.py index 6d764ed..f89ee98 100644 --- a/tests/unit/test_payloads.py +++ b/tests/unit/test_payloads.py @@ -1,28 +1,28 @@ """Tests for payload dataclasses — Validation, defaults, and serialization.""" import pytest + from brightdata.payloads import ( # Amazon AmazonProductPayload, AmazonReviewPayload, - LinkedInProfilePayload, - LinkedInProfileSearchPayload, - LinkedInJobSearchPayload, - LinkedInPostSearchPayload, # ChatGPT ChatGPTPromptPayload, + FacebookCommentsPayload, + FacebookPostPayload, + FacebookPostsGroupPayload, # Facebook FacebookPostsProfilePayload, - FacebookPostsGroupPayload, - FacebookPostPayload, - FacebookCommentsPayload, - InstagramProfilePayload, InstagramPostPayload, - InstagramReelPayload, InstagramPostsDiscoverPayload, + InstagramProfilePayload, + InstagramReelPayload, + LinkedInJobSearchPayload, + LinkedInPostSearchPayload, + LinkedInProfilePayload, + LinkedInProfileSearchPayload, ) - # --------------------------------------------------------------------------- # Amazon # --------------------------------------------------------------------------- diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 2ee48ae..311ec7f 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -4,9 +4,8 @@ import pytest +from brightdata.exceptions import APIError, AuthenticationError, NetworkError, ValidationError from brightdata.utils.retry import retry_with_backoff -from brightdata.exceptions import APIError, NetworkError, AuthenticationError, ValidationError - # --------------------------------------------------------------------------- # Happy path diff --git a/tests/unit/test_scraper_core.py b/tests/unit/test_scraper_core.py index c074d15..3cbe3d9 100644 --- a/tests/unit/test_scraper_core.py +++ b/tests/unit/test_scraper_core.py @@ -7,12 +7,13 @@ tests pin the consolidated behavior. """ -import pytest from unittest.mock import MagicMock +import pytest + from brightdata.exceptions import ValidationError -from brightdata.scrapers.base import BaseWebScraper, ScraperCore from brightdata.scrapers.amazon.search import AmazonSearchScraper +from brightdata.scrapers.base import BaseWebScraper, ScraperCore from brightdata.scrapers.chatgpt.search import ChatGPTSearchService from brightdata.scrapers.instagram.search import InstagramSearchScraper from brightdata.scrapers.linkedin.search import LinkedInSearchScraper diff --git a/tests/unit/test_ssl_helpers.py b/tests/unit/test_ssl_helpers.py index 3f2fe4f..4645940 100644 --- a/tests/unit/test_ssl_helpers.py +++ b/tests/unit/test_ssl_helpers.py @@ -3,8 +3,7 @@ import ssl from unittest.mock import Mock, patch -from brightdata.utils.ssl_helpers import is_macos, is_ssl_certificate_error, get_ssl_error_message - +from brightdata.utils.ssl_helpers import get_ssl_error_message, is_macos, is_ssl_certificate_error # --------------------------------------------------------------------------- # Platform detection diff --git a/tests/unit/test_sync_client_coverage.py b/tests/unit/test_sync_client_coverage.py index f622c1b..4eede2e 100644 --- a/tests/unit/test_sync_client_coverage.py +++ b/tests/unit/test_sync_client_coverage.py @@ -11,26 +11,26 @@ """ import asyncio +from unittest.mock import AsyncMock, MagicMock import pytest -from unittest.mock import AsyncMock, MagicMock from brightdata import SyncBrightDataClient -from brightdata.models import ScrapeResult from brightdata.discover.models import DiscoverSnapshot +from brightdata.models import ScrapeResult from brightdata.scrapers.service import ScrapeService from brightdata.serp.service import SearchService from brightdata.sync_client import ( - SyncDatasetsClient, SyncDataset, + SyncDatasetsClient, + SyncDigiKeyScraper, + SyncInstagramSearchScraper, + SyncPerplexityScraper, SyncRedditScraper, SyncTikTokScraper, - SyncYouTubeScraper, - SyncPerplexityScraper, - SyncDigiKeyScraper, SyncTikTokSearchScraper, + SyncYouTubeScraper, SyncYouTubeSearchScraper, - SyncInstagramSearchScraper, ) diff --git a/tests/unit/test_x_scraper.py b/tests/unit/test_x_scraper.py index baf9ad2..eafdc13 100644 --- a/tests/unit/test_x_scraper.py +++ b/tests/unit/test_x_scraper.py @@ -7,12 +7,11 @@ """ import asyncio - from unittest.mock import AsyncMock, MagicMock -from brightdata.scrapers.x.scraper import XScraper -from brightdata.scrapers.registry import get_scraper_for from brightdata.models import ScrapeResult +from brightdata.scrapers.registry import get_scraper_for +from brightdata.scrapers.x.scraper import XScraper from brightdata.sync_client import SyncXScraper POSTS = "gd_lwxkxvnf1cynvib9co" diff --git a/tests/unit/test_zone_manager.py b/tests/unit/test_zone_manager.py index 0709c96..10aff3f 100644 --- a/tests/unit/test_zone_manager.py +++ b/tests/unit/test_zone_manager.py @@ -3,10 +3,8 @@ import pytest from brightdata.core.zone_manager import ZoneManager -from brightdata.exceptions.errors import ZoneError, AuthenticationError - -from tests.conftest import MockResponse, MockContextManager - +from brightdata.exceptions.errors import AuthenticationError, ZoneError +from tests.conftest import MockContextManager, MockResponse # --------------------------------------------------------------------------- # List Zones