diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f7d53e..cd1836d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Bright Data Python SDK Changelog +## Version 2.5.1 - Lint cleanup + +- **Chore**: pinned an explicit `ruff` lint rule scope (`select`) and sorted imports repo-wide. No behavior changes. + ## Version 2.5.0 - CLI-credentials auth + scraper-core dedup - **CLI-credentials auth fallback**: one `brightdata login` now authenticates the SDK too. With no token given, `BrightDataClient()` / `SyncBrightDataClient()` fall back to the token already saved by the Bright Data CLI. Resolution order: `token=` parameter → `BRIGHTDATA_API_TOKEN` / `BRIGHTDATA_API_KEY` env vars → CLI login → actionable error mentioning `brightdata login`. diff --git a/pyproject.toml b/pyproject.toml index 0583e5b..2cbdc1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ where = ["src"] [project] name = "brightdata-sdk" -version = "2.5.0" +version = "2.5.1" description = "Modern async-first Python SDK for Bright Data APIs" authors = [{name = "Bright Data", email = "support@brightdata.com"}] license = {text = "MIT"} @@ -60,6 +60,12 @@ exclude = [ ] [tool.ruff.lint] +# Pyflakes + pycodestyle errors + isort (import sorting). Explicit, so newer +# ruff releases that broaden the "no select" default (e.g. datetimez, simplify, +# pyupgrade, bugbear) don't silently start failing CI with unrelated findings, +# and don't get auto-"fixed" into rewrites (e.g. Optional[X] -> X | None) that +# were never asked for. +select = ["E4", "E7", "E9", "F", "I"] ignore = [ "E402", # Module level import not at top of file (common in notebooks/scripts) "F841", # Local variable assigned but never used (common in tests) diff --git a/src/brightdata/__init__.py b/src/brightdata/__init__.py index 83ae8c0..0f5f7c7 100644 --- a/src/brightdata/__init__.py +++ b/src/brightdata/__init__.py @@ -1,6 +1,6 @@ """Bright Data Python SDK - Modern async-first SDK for Bright Data APIs.""" -from importlib.metadata import version, PackageNotFoundError +from importlib.metadata import PackageNotFoundError, version try: __version__ = version("brightdata-sdk") @@ -9,79 +9,79 @@ __version__ = "0.0.0.dev" # Export main client (async) +from .browser.service import BrowserService from .client import BrightDataClient +from .core.zone_manager import ZoneManager -# Export sync client adapter -from .sync_client import SyncBrightDataClient +# Export Discover API models +from .discover.models import DiscoverJob, DiscoverResult, DiscoverSnapshot + +# Export exceptions +from .exceptions import ( + APIError, + AuthenticationError, + BrightDataError, + NetworkError, + SSLError, + ValidationError, + ZoneError, +) # Export result models from .models import ( BaseResult, - ScrapeResult, - SearchResult, CrawlResult, Result, + ScrapeResult, + SearchResult, ) -# Export job model for manual trigger/poll/fetch -from .scrapers.job import ScrapeJob - -# Export Discover API models -from .discover.models import DiscoverResult, DiscoverJob, DiscoverSnapshot - # Export payload models (dataclasses) from .payloads import ( - # Base - BasePayload, - URLPayload, # Amazon AmazonProductPayload, AmazonReviewPayload, AmazonSellerPayload, - # LinkedIn - LinkedInProfilePayload, - LinkedInJobPayload, - LinkedInCompanyPayload, - LinkedInPostPayload, - LinkedInProfileSearchPayload, - LinkedInJobSearchPayload, - LinkedInPostSearchPayload, + # Base + BasePayload, # ChatGPT ChatGPTPromptPayload, + FacebookCommentsPayload, + FacebookPostPayload, + FacebookPostsGroupPayload, # Facebook FacebookPostsProfilePayload, - FacebookPostsGroupPayload, - FacebookPostPayload, - FacebookCommentsPayload, FacebookReelsPayload, + InstagramCommentPayload, + InstagramPostPayload, + InstagramPostsDiscoverPayload, # Instagram InstagramProfilePayload, - InstagramPostPayload, - InstagramCommentPayload, InstagramReelPayload, - InstagramPostsDiscoverPayload, InstagramReelsDiscoverPayload, -) - -# Export exceptions -from .exceptions import ( - BrightDataError, - ValidationError, - AuthenticationError, - APIError, - ZoneError, - NetworkError, - SSLError, + LinkedInCompanyPayload, + LinkedInJobPayload, + LinkedInJobSearchPayload, + LinkedInPostPayload, + LinkedInPostSearchPayload, + # LinkedIn + LinkedInProfilePayload, + LinkedInProfileSearchPayload, + URLPayload, ) # Export Scraper Studio models -from .scraper_studio.models import ScraperStudioJob, JobStatus +from .scraper_studio.models import JobStatus, ScraperStudioJob +from .scraper_studio.service import ScraperStudioService + +# Export job model for manual trigger/poll/fetch +from .scrapers.job import ScrapeJob + +# Export sync client adapter +from .sync_client import SyncBrightDataClient # Export services for advanced usage from .web_unlocker.service import WebUnlockerService -from .scraper_studio.service import ScraperStudioService -from .browser.service import BrowserService -from .core.zone_manager import ZoneManager __all__ = [ "__version__", diff --git a/src/brightdata/client.py b/src/brightdata/client.py index 3f3c1a8..b559ba7 100644 --- a/src/brightdata/client.py +++ b/src/brightdata/client.py @@ -8,11 +8,11 @@ - Follow principle of least surprise - common patterns from other SDKs """ -import os import asyncio +import os import warnings -from typing import Optional, Dict, Any, Union, List from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Union try: from dotenv import load_dotenv @@ -21,22 +21,23 @@ except ImportError: pass +from http import HTTPStatus + +from .browser.service import BrowserService +from .cli_credentials import read_cli_credentials from .core.engine import AsyncEngine from .core.zone_manager import ZoneManager -from .web_unlocker.service import WebUnlockerService -from .scrapers.service import ScrapeService -from .serp.service import SearchService from .crawler.service import CrawlerService -from .scraper_studio.service import ScraperStudioService -from .browser.service import BrowserService -from .discover.service import DiscoverService -from .discover.models import DiscoverResult, DiscoverJob from .datasets import DatasetsClient +from .discover.models import DiscoverJob, DiscoverResult +from .discover.service import DiscoverService +from .exceptions import APIError, AuthenticationError, ValidationError from .models import ScrapeResult +from .scraper_studio.service import ScraperStudioService +from .scrapers.service import ScrapeService +from .serp.service import SearchService from .types import AccountInfo -from .cli_credentials import read_cli_credentials -from http import HTTPStatus -from .exceptions import ValidationError, AuthenticationError, APIError +from .web_unlocker.service import WebUnlockerService class BrightDataClient: diff --git a/src/brightdata/core/engine.py b/src/brightdata/core/engine.py index 32490fa..473423c 100644 --- a/src/brightdata/core/engine.py +++ b/src/brightdata/core/engine.py @@ -1,14 +1,16 @@ """Async HTTP engine for Bright Data API operations.""" import asyncio -import aiohttp import ssl import warnings -from typing import Optional, Dict, Any +from http import HTTPStatus +from typing import Any, Dict, Optional + +import aiohttp + from .. import __version__ from ..exceptions import AuthenticationError, NetworkError, SSLError -from http import HTTPStatus -from ..utils.ssl_helpers import is_ssl_certificate_error, get_ssl_error_message +from ..utils.ssl_helpers import get_ssl_error_message, is_ssl_certificate_error # Rate limiting support try: diff --git a/src/brightdata/core/zone_manager.py b/src/brightdata/core/zone_manager.py index b0a7b6f..7218fee 100644 --- a/src/brightdata/core/zone_manager.py +++ b/src/brightdata/core/zone_manager.py @@ -5,10 +5,12 @@ import asyncio import logging -import aiohttp -from typing import List, Dict, Any, Optional, Tuple from http import HTTPStatus -from ..exceptions.errors import ZoneError, APIError, AuthenticationError +from typing import Any, Dict, List, Optional, Tuple + +import aiohttp + +from ..exceptions.errors import APIError, AuthenticationError, ZoneError logger = logging.getLogger(__name__) diff --git a/src/brightdata/crawler/service.py b/src/brightdata/crawler/service.py index 50d1f44..2d3821b 100644 --- a/src/brightdata/crawler/service.py +++ b/src/brightdata/crawler/service.py @@ -19,12 +19,12 @@ import json from datetime import datetime, timezone from http import HTTPStatus -from typing import Any, Dict, List, Optional, TYPE_CHECKING, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union -from .models import CrawlJob, CrawlResult from ..exceptions import APIError, ValidationError from ..utils.function_detection import get_caller_function_name from ..utils.validation import validate_url, validate_url_list +from .models import CrawlJob, CrawlResult if TYPE_CHECKING: from ..client import BrightDataClient diff --git a/src/brightdata/datasets/__init__.py b/src/brightdata/datasets/__init__.py index af5e6c2..1c66abf 100644 --- a/src/brightdata/datasets/__init__.py +++ b/src/brightdata/datasets/__init__.py @@ -4,163 +4,163 @@ Access pre-collected datasets and filter records. """ -from .client import DatasetsClient -from .base import BaseDataset, DatasetError -from .models import DatasetInfo, DatasetField, DatasetMetadata, SnapshotStatus -from .utils import export, export_json, export_jsonl, export_csv - -# Platform-specific datasets -from .linkedin import ( - LinkedInPeopleProfiles, - LinkedInCompanyProfiles, - LinkedInJobListings, - LinkedInPosts, - LinkedInProfilesJobListings, -) +from .agoda import AgodaProperties +from .airbnb import AirbnbProperties from .amazon import ( + AmazonBestSellers, AmazonProducts, + AmazonProductsGlobal, + AmazonProductsSearch, AmazonReviews, AmazonSellersInfo, - AmazonBestSellers, - AmazonProductsSearch, - AmazonProductsGlobal, AmazonWalmart, ) -from .crunchbase import CrunchbaseCompanies -from .imdb import IMDBMovies -from .nba import NBAPlayersStats -from .goodreads import GoodreadsBooks -from .world_population import WorldPopulation -from .companies_enriched import CompaniesEnriched -from .employees_enriched import EmployeesEnriched -from .glassdoor import GlassdoorCompanies, GlassdoorReviews, GlassdoorJobs -from .google_maps import GoogleMapsReviews, GoogleMapsFullInfo -from .yelp import YelpBusinesses, YelpReviews -from .zoominfo import ZoomInfoCompanies -from .pitchbook import PitchBookCompanies -from .g2 import G2Products, G2Reviews -from .trustpilot import TrustpilotReviews -from .indeed import IndeedCompanies, IndeedJobs -from .xing import XingProfiles -from .slintel import SlintelCompanies -from .owler import OwlerCompanies -from .lawyers import USLawyers -from .manta import MantaBusinesses -from .ventureradar import VentureRadarCompanies -from .trustradius import TrustRadiusReviews -from .instagram import InstagramProfiles, InstagramPosts, InstagramComments, InstagramReels -from .tiktok import TikTokProfiles, TikTokComments, TikTokPosts, TikTokShop -from .real_estate import AustraliaRealEstate -from .walmart import WalmartProducts, WalmartSellersInfo -from .mediamarkt import MediamarktProducts -from .fendi import FendiProducts -from .zalando import ZalandoProducts -from .sephora import SephoraProducts -from .zara import ZaraProducts, ZaraHomeProducts -from .mango import MangoProducts -from .massimo_dutti import MassimoDuttiProducts -from .otodom import OtodomPoland -from .webmotors import WebmotorsBrasil -from .airbnb import AirbnbProperties -from .asos import AsosProducts -from .chanel import ChanelProducts -from .ashley_furniture import AshleyFurnitureProducts -from .fanatics import FanaticsProducts -from .carters import CartersProducts from .american_eagle import AmericanEagleProducts -from .ikea import IkeaProducts -from .hm import HMProducts -from .lego import LegoProducts -from .mattressfirm import MattressfirmProducts -from .crateandbarrel import CrateAndBarrelProducts -from .llbean import LLBeanProducts -from .shein import SheinProducts -from .toysrus import ToysRUsProducts -from .mybobs import MybobsProducts -from .sleepnumber import SleepNumberProducts -from .raymourflanigan import RaymourFlaniganProducts -from .inmuebles24 import Inmuebles24Mexico -from .mouser import MouserProducts -from .zillow import ZillowProperties, ZillowPriceHistory -from .zonaprop import ZonapropArgentina -from .metrocuadrado import MetrocuadradoProperties -from .chileautos import ChileautosChile -from .infocasas import InfocasasUruguay -from .lazboy import LaZBoyProducts -from .properati import ProperatiProperties -from .yapo import YapoChile -from .toctoc import ToctocProperties -from .dior import DiorProducts +from .apple_appstore import AppleAppStore, AppleAppStoreReviews +from .ashley_furniture import AshleyFurnitureProducts +from .asos import AsosProducts +from .autozone import AutozoneProducts from .balenciaga import BalenciagaProducts +from .base import BaseDataset, DatasetError +from .bbc import BBCNews +from .berluti import BerlutiProducts +from .bestbuy import BestBuyProducts +from .bh import BHProducts +from .bluesky import BlueskyPosts, BlueskyTopProfiles +from .booking import BookingHotelListings, BookingListingsSearch from .bottegaveneta import BottegaVenetaProducts -from .olx import OLXBrazil +from .carsales import CarsalesListings +from .carters import CartersProducts from .celine import CelineProducts -from .loewe import LoeweProducts -from .berluti import BerlutiProducts -from .moynat import MoynatProducts -from .hermes import HermesProducts +from .chanel import ChanelProducts +from .chileautos import ChileautosChile +from .client import DatasetsClient +from .cnn import CNNNews +from .companies_enriched import CompaniesEnriched +from .costco import CostcoProducts +from .crateandbarrel import CrateAndBarrelProducts +from .creative_commons import CreativeCommons3DModels, CreativeCommonsImages +from .crunchbase import CrunchbaseCompanies from .delvaux import DelvauxProducts -from .prada import PradaProducts -from .montblanc import MontblancProducts -from .ysl import YSLProducts -from .world_zipcodes import WorldZipcodes -from .pinterest import PinterestPosts, PinterestProfiles -from .shopee import ShopeeProducts -from .lazada import LazadaProducts, LazadaReviews, LazadaProductsSearch -from .youtube import YouTubeProfiles, YouTubeVideos, YouTubeComments from .digikey import DigikeyProducts +from .dior import DiorProducts +from .ebay import EbayProducts +from .employees_enriched import EmployeesEnriched +from .etsy import EtsyProducts from .facebook import ( - FacebookPagesPosts, FacebookComments, - FacebookPostsByUrl, - FacebookReels, - FacebookMarketplace, FacebookCompanyReviews, FacebookEvents, - FacebookProfiles, - FacebookPagesProfiles, FacebookGroupPosts, + FacebookMarketplace, + FacebookPagesPosts, + FacebookPagesProfiles, + FacebookPostsByUrl, + FacebookProfiles, + FacebookReels, ) -from .x_twitter import XTwitterPosts, XTwitterProfiles -from .reddit import RedditPosts, RedditComments -from .bluesky import BlueskyPosts, BlueskyTopProfiles -from .snapchat import SnapchatPosts -from .quora import QuoraPosts -from .vimeo import VimeoVideos -from .google_news import GoogleNews -from .wikipedia import WikipediaArticles -from .bbc import BBCNews -from .cnn import CNNNews +from .fanatics import FanaticsProducts +from .fendi import FendiProducts +from .g2 import G2Products, G2Reviews from .github import GithubRepositories -from .creative_commons import CreativeCommonsImages, CreativeCommons3DModels -from .google_play import GooglePlayStore, GooglePlayReviews -from .apple_appstore import AppleAppStore, AppleAppStoreReviews -from .ebay import EbayProducts -from .etsy import EtsyProducts -from .wayfair import WayfairProducts -from .bestbuy import BestBuyProducts -from .myntra import MyntraProducts -from .ozon import OzonProducts -from .wildberries import WildberriesProducts -from .tokopedia import TokopediaProducts +from .glassdoor import GlassdoorCompanies, GlassdoorJobs, GlassdoorReviews +from .goodreads import GoodreadsBooks +from .google_maps import GoogleMapsFullInfo, GoogleMapsReviews +from .google_news import GoogleNews +from .google_play import GooglePlayReviews, GooglePlayStore from .google_shopping import GoogleShoppingProducts, GoogleShoppingSearchUS -from .mercadolivre import MercadolivreProducts -from .naver import NaverProducts -from .homedepot import HomeDepotUSProducts, HomeDepotCAProducts -from .lowes import LowesProducts -from .rona import RonaProducts +from .hermes import HermesProducts +from .hm import HMProducts +from .homedepot import HomeDepotCAProducts, HomeDepotUSProducts +from .ikea import IkeaProducts +from .imdb import IMDBMovies +from .indeed import IndeedCompanies, IndeedJobs +from .infocasas import InfocasasUruguay +from .inmuebles24 import Inmuebles24Mexico +from .instagram import InstagramComments, InstagramPosts, InstagramProfiles, InstagramReels from .kroger import KrogerProducts +from .lawyers import USLawyers +from .lazada import LazadaProducts, LazadaProductsSearch, LazadaReviews +from .lazboy import LaZBoyProducts +from .lego import LegoProducts + +# Platform-specific datasets +from .linkedin import ( + LinkedInCompanyProfiles, + LinkedInJobListings, + LinkedInPeopleProfiles, + LinkedInPosts, + LinkedInProfilesJobListings, +) +from .llbean import LLBeanProducts +from .loewe import LoeweProducts +from .lowes import LowesProducts from .macys import MacysProducts -from .costco import CostcoProducts -from .bh import BHProducts +from .mango import MangoProducts +from .manta import MantaBusinesses +from .massimo_dutti import MassimoDuttiProducts +from .mattressfirm import MattressfirmProducts +from .mediamarkt import MediamarktProducts +from .mercadolivre import MercadolivreProducts +from .metrocuadrado import MetrocuadradoProperties from .microcenter import MicroCenterProducts -from .autozone import AutozoneProducts -from .zoopla import ZooplaProperties -from .booking import BookingListingsSearch, BookingHotelListings +from .models import DatasetField, DatasetInfo, DatasetMetadata, SnapshotStatus +from .montblanc import MontblancProducts +from .mouser import MouserProducts +from .moynat import MoynatProducts +from .mybobs import MybobsProducts +from .myntra import MyntraProducts +from .naver import NaverProducts +from .nba import NBAPlayersStats +from .olx import OLXBrazil +from .otodom import OtodomPoland +from .owler import OwlerCompanies +from .ozon import OzonProducts +from .pinterest import PinterestPosts, PinterestProfiles +from .pitchbook import PitchBookCompanies +from .prada import PradaProducts +from .properati import ProperatiProperties +from .quora import QuoraPosts +from .raymourflanigan import RaymourFlaniganProducts +from .real_estate import AustraliaRealEstate from .realtor import RealtorInternationalProperties -from .agoda import AgodaProperties -from .carsales import CarsalesListings +from .reddit import RedditComments, RedditPosts +from .rona import RonaProducts +from .sephora import SephoraProducts +from .shein import SheinProducts +from .shopee import ShopeeProducts +from .sleepnumber import SleepNumberProducts +from .slintel import SlintelCompanies +from .snapchat import SnapchatPosts +from .tiktok import TikTokComments, TikTokPosts, TikTokProfiles, TikTokShop +from .toctoc import ToctocProperties +from .tokopedia import TokopediaProducts +from .toysrus import ToysRUsProducts +from .trustpilot import TrustpilotReviews +from .trustradius import TrustRadiusReviews +from .utils import export, export_csv, export_json, export_jsonl +from .ventureradar import VentureRadarCompanies +from .vimeo import VimeoVideos +from .walmart import WalmartProducts, WalmartSellersInfo +from .wayfair import WayfairProducts +from .webmotors import WebmotorsBrasil +from .wikipedia import WikipediaArticles +from .wildberries import WildberriesProducts +from .world_population import WorldPopulation +from .world_zipcodes import WorldZipcodes +from .x_twitter import XTwitterPosts, XTwitterProfiles +from .xing import XingProfiles from .yahoo_finance import YahooFinanceBusinesses +from .yapo import YapoChile +from .yelp import YelpBusinesses, YelpReviews +from .youtube import YouTubeComments, YouTubeProfiles, YouTubeVideos +from .ysl import YSLProducts +from .zalando import ZalandoProducts +from .zara import ZaraHomeProducts, ZaraProducts +from .zillow import ZillowPriceHistory, ZillowProperties +from .zonaprop import ZonapropArgentina +from .zoominfo import ZoomInfoCompanies +from .zoopla import ZooplaProperties __all__ = [ # Client diff --git a/src/brightdata/datasets/amazon/__init__.py b/src/brightdata/datasets/amazon/__init__.py index 54efa19..7c773e7 100644 --- a/src/brightdata/datasets/amazon/__init__.py +++ b/src/brightdata/datasets/amazon/__init__.py @@ -1,11 +1,11 @@ """Amazon datasets.""" +from .best_sellers import AmazonBestSellers from .products import AmazonProducts +from .products_global import AmazonProductsGlobal +from .products_search import AmazonProductsSearch from .reviews import AmazonReviews from .sellers import AmazonSellersInfo -from .best_sellers import AmazonBestSellers -from .products_search import AmazonProductsSearch -from .products_global import AmazonProductsGlobal from .walmart import AmazonWalmart __all__ = [ diff --git a/src/brightdata/datasets/amazon/products.py b/src/brightdata/datasets/amazon/products.py index 058bddc..9c3b5e9 100644 --- a/src/brightdata/datasets/amazon/products.py +++ b/src/brightdata/datasets/amazon/products.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/apple_appstore/__init__.py b/src/brightdata/datasets/apple_appstore/__init__.py index 804a553..dce6b93 100644 --- a/src/brightdata/datasets/apple_appstore/__init__.py +++ b/src/brightdata/datasets/apple_appstore/__init__.py @@ -1,6 +1,6 @@ """Apple App Store datasets.""" -from .store import AppleAppStore from .reviews import AppleAppStoreReviews +from .store import AppleAppStore __all__ = ["AppleAppStore", "AppleAppStoreReviews"] diff --git a/src/brightdata/datasets/base.py b/src/brightdata/datasets/base.py index 1a913f2..2937102 100644 --- a/src/brightdata/datasets/base.py +++ b/src/brightdata/datasets/base.py @@ -4,7 +4,7 @@ import asyncio import time -from typing import Dict, List, Any, Optional, Literal, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional from .models import DatasetMetadata, SnapshotStatus diff --git a/src/brightdata/datasets/booking/__init__.py b/src/brightdata/datasets/booking/__init__.py index e003247..dd04ac1 100644 --- a/src/brightdata/datasets/booking/__init__.py +++ b/src/brightdata/datasets/booking/__init__.py @@ -1,6 +1,6 @@ """Booking.com datasets.""" -from .listings_search import BookingListingsSearch from .hotel_listings import BookingHotelListings +from .listings_search import BookingListingsSearch __all__ = ["BookingListingsSearch", "BookingHotelListings"] diff --git a/src/brightdata/datasets/client.py b/src/brightdata/datasets/client.py index 72df47b..15ee055 100644 --- a/src/brightdata/datasets/client.py +++ b/src/brightdata/datasets/client.py @@ -6,7 +6,7 @@ """ import importlib -from typing import List, TYPE_CHECKING +from typing import TYPE_CHECKING, List from .models import DatasetInfo diff --git a/src/brightdata/datasets/client.pyi b/src/brightdata/datasets/client.pyi index 4ff743d..c52a08b 100644 --- a/src/brightdata/datasets/client.pyi +++ b/src/brightdata/datasets/client.pyi @@ -1,9 +1,8 @@ """Type stub for DatasetsClient — provides IDE autocomplete.""" from typing import List -from .models import DatasetInfo -from ..core.engine import AsyncEngine +from ..core.engine import AsyncEngine from .agoda import AgodaProperties from .airbnb import AirbnbProperties from .amazon import ( @@ -143,6 +142,7 @@ from .mediamarkt import MediamarktProducts from .mercadolivre import MercadolivreProducts from .metrocuadrado import MetrocuadradoProperties from .microcenter import MicroCenterProducts +from .models import DatasetInfo from .montblanc import MontblancProducts from .mouser import MouserProducts from .moynat import MoynatProducts diff --git a/src/brightdata/datasets/crunchbase/companies.py b/src/brightdata/datasets/crunchbase/companies.py index 616d158..715c3d0 100644 --- a/src/brightdata/datasets/crunchbase/companies.py +++ b/src/brightdata/datasets/crunchbase/companies.py @@ -7,7 +7,7 @@ See FIELDS dict for all filterable fields with descriptions and fill rates. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/facebook/__init__.py b/src/brightdata/datasets/facebook/__init__.py index 2a07c8a..fa0bb08 100644 --- a/src/brightdata/datasets/facebook/__init__.py +++ b/src/brightdata/datasets/facebook/__init__.py @@ -1,15 +1,15 @@ """Facebook datasets.""" -from .pages_posts import FacebookPagesPosts from .comments import FacebookComments -from .posts_by_url import FacebookPostsByUrl -from .reels import FacebookReels -from .marketplace import FacebookMarketplace from .company_reviews import FacebookCompanyReviews from .events import FacebookEvents -from .profiles import FacebookProfiles -from .pages_profiles import FacebookPagesProfiles from .group_posts import FacebookGroupPosts +from .marketplace import FacebookMarketplace +from .pages_posts import FacebookPagesPosts +from .pages_profiles import FacebookPagesProfiles +from .posts_by_url import FacebookPostsByUrl +from .profiles import FacebookProfiles +from .reels import FacebookReels __all__ = [ "FacebookPagesPosts", diff --git a/src/brightdata/datasets/glassdoor/__init__.py b/src/brightdata/datasets/glassdoor/__init__.py index aadf858..b6cd001 100644 --- a/src/brightdata/datasets/glassdoor/__init__.py +++ b/src/brightdata/datasets/glassdoor/__init__.py @@ -1,7 +1,7 @@ """Glassdoor datasets.""" from .companies import GlassdoorCompanies -from .reviews import GlassdoorReviews from .jobs import GlassdoorJobs +from .reviews import GlassdoorReviews __all__ = ["GlassdoorCompanies", "GlassdoorReviews", "GlassdoorJobs"] diff --git a/src/brightdata/datasets/goodreads/books.py b/src/brightdata/datasets/goodreads/books.py index abcc4e1..55dd7ed 100644 --- a/src/brightdata/datasets/goodreads/books.py +++ b/src/brightdata/datasets/goodreads/books.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/google_maps/__init__.py b/src/brightdata/datasets/google_maps/__init__.py index 1453afb..3d0adec 100644 --- a/src/brightdata/datasets/google_maps/__init__.py +++ b/src/brightdata/datasets/google_maps/__init__.py @@ -1,7 +1,7 @@ """Google Maps datasets.""" -from .reviews import GoogleMapsReviews from .full_info import GoogleMapsFullInfo +from .reviews import GoogleMapsReviews __all__ = [ "GoogleMapsReviews", diff --git a/src/brightdata/datasets/google_play/__init__.py b/src/brightdata/datasets/google_play/__init__.py index ee96916..a0f07eb 100644 --- a/src/brightdata/datasets/google_play/__init__.py +++ b/src/brightdata/datasets/google_play/__init__.py @@ -1,6 +1,6 @@ """Google Play datasets.""" -from .store import GooglePlayStore from .reviews import GooglePlayReviews +from .store import GooglePlayStore __all__ = ["GooglePlayStore", "GooglePlayReviews"] diff --git a/src/brightdata/datasets/imdb/movies.py b/src/brightdata/datasets/imdb/movies.py index 21aee7a..189ee06 100644 --- a/src/brightdata/datasets/imdb/movies.py +++ b/src/brightdata/datasets/imdb/movies.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/instagram/__init__.py b/src/brightdata/datasets/instagram/__init__.py index b8504e9..a02caea 100644 --- a/src/brightdata/datasets/instagram/__init__.py +++ b/src/brightdata/datasets/instagram/__init__.py @@ -1,8 +1,8 @@ """Instagram datasets.""" -from .profiles import InstagramProfiles -from .posts import InstagramPosts from .comments import InstagramComments +from .posts import InstagramPosts +from .profiles import InstagramProfiles from .reels import InstagramReels __all__ = [ diff --git a/src/brightdata/datasets/lazada/__init__.py b/src/brightdata/datasets/lazada/__init__.py index 897bb2d..413e3c3 100644 --- a/src/brightdata/datasets/lazada/__init__.py +++ b/src/brightdata/datasets/lazada/__init__.py @@ -1,8 +1,8 @@ """Lazada datasets.""" from .products import LazadaProducts -from .reviews import LazadaReviews from .products_search import LazadaProductsSearch +from .reviews import LazadaReviews __all__ = [ "LazadaProducts", diff --git a/src/brightdata/datasets/linkedin/__init__.py b/src/brightdata/datasets/linkedin/__init__.py index b123902..e965c45 100644 --- a/src/brightdata/datasets/linkedin/__init__.py +++ b/src/brightdata/datasets/linkedin/__init__.py @@ -1,8 +1,8 @@ """LinkedIn datasets.""" -from .people_profiles import LinkedInPeopleProfiles from .company_profiles import LinkedInCompanyProfiles from .job_listings import LinkedInJobListings +from .people_profiles import LinkedInPeopleProfiles from .posts import LinkedInPosts from .profiles_job_listings import LinkedInProfilesJobListings diff --git a/src/brightdata/datasets/linkedin/company_profiles.py b/src/brightdata/datasets/linkedin/company_profiles.py index daefbd7..3a894ee 100644 --- a/src/brightdata/datasets/linkedin/company_profiles.py +++ b/src/brightdata/datasets/linkedin/company_profiles.py @@ -7,7 +7,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/linkedin/people_profiles.py b/src/brightdata/datasets/linkedin/people_profiles.py index 34bc07f..ead0d18 100644 --- a/src/brightdata/datasets/linkedin/people_profiles.py +++ b/src/brightdata/datasets/linkedin/people_profiles.py @@ -7,7 +7,7 @@ See FIELDS dict for all filterable fields with descriptions and fill rates. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/models.py b/src/brightdata/datasets/models.py index fdc56d0..f012793 100644 --- a/src/brightdata/datasets/models.py +++ b/src/brightdata/datasets/models.py @@ -3,7 +3,7 @@ """ from dataclasses import dataclass, field -from typing import Dict, Optional, Any, Literal +from typing import Any, Dict, Literal, Optional @dataclass diff --git a/src/brightdata/datasets/nba/players_stats.py b/src/brightdata/datasets/nba/players_stats.py index 19b6cf7..8e2a770 100644 --- a/src/brightdata/datasets/nba/players_stats.py +++ b/src/brightdata/datasets/nba/players_stats.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/reddit/__init__.py b/src/brightdata/datasets/reddit/__init__.py index 2c5cab3..ea94ef6 100644 --- a/src/brightdata/datasets/reddit/__init__.py +++ b/src/brightdata/datasets/reddit/__init__.py @@ -1,6 +1,6 @@ """Reddit datasets.""" -from .posts import RedditPosts from .comments import RedditComments +from .posts import RedditPosts __all__ = ["RedditPosts", "RedditComments"] diff --git a/src/brightdata/datasets/tiktok/__init__.py b/src/brightdata/datasets/tiktok/__init__.py index b7b2190..9f17544 100644 --- a/src/brightdata/datasets/tiktok/__init__.py +++ b/src/brightdata/datasets/tiktok/__init__.py @@ -1,8 +1,8 @@ """TikTok datasets.""" -from .profiles import TikTokProfiles from .comments import TikTokComments from .posts import TikTokPosts +from .profiles import TikTokProfiles from .shop import TikTokShop __all__ = [ diff --git a/src/brightdata/datasets/utils.py b/src/brightdata/datasets/utils.py index c5b70a5..38f3506 100644 --- a/src/brightdata/datasets/utils.py +++ b/src/brightdata/datasets/utils.py @@ -2,10 +2,10 @@ Dataset utilities - helpers for exporting and processing dataset results. """ -import json import csv +import json from pathlib import Path -from typing import List, Dict, Any, Union, Optional +from typing import Any, Dict, List, Optional, Union def export_json( diff --git a/src/brightdata/datasets/world_population/countries.py b/src/brightdata/datasets/world_population/countries.py index a406b23..bbc9dcd 100644 --- a/src/brightdata/datasets/world_population/countries.py +++ b/src/brightdata/datasets/world_population/countries.py @@ -6,7 +6,7 @@ See FIELDS dict for all filterable fields with descriptions. """ -from typing import TYPE_CHECKING, Dict, Any +from typing import TYPE_CHECKING, Any, Dict from ..base import BaseDataset diff --git a/src/brightdata/datasets/youtube/__init__.py b/src/brightdata/datasets/youtube/__init__.py index 29d8144..505dde0 100644 --- a/src/brightdata/datasets/youtube/__init__.py +++ b/src/brightdata/datasets/youtube/__init__.py @@ -1,7 +1,7 @@ """YouTube datasets.""" +from .comments import YouTubeComments from .profiles import YouTubeProfiles from .videos import YouTubeVideos -from .comments import YouTubeComments __all__ = ["YouTubeProfiles", "YouTubeVideos", "YouTubeComments"] diff --git a/src/brightdata/datasets/zara/__init__.py b/src/brightdata/datasets/zara/__init__.py index 504dd15..ac0d1f3 100644 --- a/src/brightdata/datasets/zara/__init__.py +++ b/src/brightdata/datasets/zara/__init__.py @@ -1,6 +1,6 @@ """Zara datasets.""" -from .products import ZaraProducts from .home_products import ZaraHomeProducts +from .products import ZaraProducts __all__ = ["ZaraProducts", "ZaraHomeProducts"] diff --git a/src/brightdata/datasets/zillow/__init__.py b/src/brightdata/datasets/zillow/__init__.py index 23b0254..bf5d4c1 100644 --- a/src/brightdata/datasets/zillow/__init__.py +++ b/src/brightdata/datasets/zillow/__init__.py @@ -1,7 +1,7 @@ """Zillow datasets.""" -from .properties import ZillowProperties from .price_history import ZillowPriceHistory +from .properties import ZillowProperties __all__ = [ "ZillowProperties", diff --git a/src/brightdata/discover/__init__.py b/src/brightdata/discover/__init__.py index afd4abf..045df99 100644 --- a/src/brightdata/discover/__init__.py +++ b/src/brightdata/discover/__init__.py @@ -1,6 +1,6 @@ """Discover API — AI-powered web search with relevance ranking.""" +from .models import DiscoverJob, DiscoverResult from .service import DiscoverService -from .models import DiscoverResult, DiscoverJob __all__ = ["DiscoverService", "DiscoverResult", "DiscoverJob"] diff --git a/src/brightdata/discover/models.py b/src/brightdata/discover/models.py index 70a2731..f9c324f 100644 --- a/src/brightdata/discover/models.py +++ b/src/brightdata/discover/models.py @@ -5,7 +5,7 @@ import asyncio import time from dataclasses import dataclass, field -from typing import Any, Optional, List, Dict, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional from ..models import BaseResult diff --git a/src/brightdata/discover/service.py b/src/brightdata/discover/service.py index 714409d..b84cd66 100644 --- a/src/brightdata/discover/service.py +++ b/src/brightdata/discover/service.py @@ -7,12 +7,12 @@ import asyncio import time -from typing import Optional, List, Dict, Any from datetime import datetime, timezone +from typing import Any, Dict, List, Optional from ..core.engine import AsyncEngine from ..exceptions import APIError -from .models import DiscoverResult, DiscoverJob +from .models import DiscoverJob, DiscoverResult class DiscoverService: diff --git a/src/brightdata/exceptions/__init__.py b/src/brightdata/exceptions/__init__.py index 9974381..a0960c0 100644 --- a/src/brightdata/exceptions/__init__.py +++ b/src/brightdata/exceptions/__init__.py @@ -1,14 +1,14 @@ """Exception classes for Bright Data SDK.""" from .errors import ( - BrightDataError, - ValidationError, - AuthenticationError, APIError, + AuthenticationError, + BrightDataError, DataNotReadyError, - ZoneError, NetworkError, SSLError, + ValidationError, + ZoneError, ) __all__ = [ diff --git a/src/brightdata/models.py b/src/brightdata/models.py index d2ebee1..86773aa 100644 --- a/src/brightdata/models.py +++ b/src/brightdata/models.py @@ -2,11 +2,11 @@ from __future__ import annotations -from dataclasses import dataclass, field, asdict -from datetime import datetime -from typing import Any, Optional, List, Dict, Union, Literal import json +from dataclasses import asdict, dataclass, field +from datetime import datetime from pathlib import Path +from typing import Any, Dict, List, Literal, Optional, Union StatusType = Literal["ready", "error", "timeout", "in_progress"] PlatformType = Optional[Literal["linkedin", "amazon", "chatgpt", "instagram", "facebook"]] diff --git a/src/brightdata/payloads.py b/src/brightdata/payloads.py index e16932f..12266b0 100644 --- a/src/brightdata/payloads.py +++ b/src/brightdata/payloads.py @@ -14,9 +14,9 @@ from __future__ import annotations -from dataclasses import dataclass, field, asdict -from typing import Optional, List, Dict, Any import re +from dataclasses import asdict, dataclass, field +from typing import Any, Dict, List, Optional from urllib.parse import urlparse # ============================================================================ diff --git a/src/brightdata/scraper_studio/__init__.py b/src/brightdata/scraper_studio/__init__.py index bedfc5a..dbc3924 100644 --- a/src/brightdata/scraper_studio/__init__.py +++ b/src/brightdata/scraper_studio/__init__.py @@ -1,6 +1,6 @@ """Scraper Studio - trigger and fetch results from user-created custom scrapers.""" -from .models import ScraperStudioJob, JobStatus +from .models import JobStatus, ScraperStudioJob from .service import ScraperStudioService __all__ = ["ScraperStudioJob", "JobStatus", "ScraperStudioService"] diff --git a/src/brightdata/scraper_studio/client.py b/src/brightdata/scraper_studio/client.py index 34d27a4..fd4276e 100644 --- a/src/brightdata/scraper_studio/client.py +++ b/src/brightdata/scraper_studio/client.py @@ -9,10 +9,10 @@ Follows the same pattern as DatasetAPIClient and AsyncUnblockerClient. """ -from typing import Dict, List, Any +from http import HTTPStatus +from typing import Any, Dict, List from ..core.engine import AsyncEngine -from http import HTTPStatus from ..exceptions import APIError, DataNotReadyError BASE_URL = "https://api.brightdata.com" diff --git a/src/brightdata/scraper_studio/models.py b/src/brightdata/scraper_studio/models.py index 2f4ff33..6364408 100644 --- a/src/brightdata/scraper_studio/models.py +++ b/src/brightdata/scraper_studio/models.py @@ -5,7 +5,7 @@ import asyncio import time from dataclasses import dataclass -from typing import Dict, List, Any, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional from ..exceptions import DataNotReadyError diff --git a/src/brightdata/scraper_studio/service.py b/src/brightdata/scraper_studio/service.py index 27b0e3c..ab8faa5 100644 --- a/src/brightdata/scraper_studio/service.py +++ b/src/brightdata/scraper_studio/service.py @@ -7,11 +7,11 @@ All methods are async-only. For sync usage, use SyncBrightDataClient. """ -from typing import Dict, List, Any, Union, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Union -from .client import ScraperStudioAPIClient -from .models import ScraperStudioJob, JobStatus from ..constants import SCRAPER_STUDIO_DEFAULT_TIMEOUT, SCRAPER_STUDIO_POLL_INTERVAL +from .client import ScraperStudioAPIClient +from .models import JobStatus, ScraperStudioJob if TYPE_CHECKING: from ..client import BrightDataClient diff --git a/src/brightdata/scrapers/__init__.py b/src/brightdata/scrapers/__init__.py index b624471..060f0a1 100644 --- a/src/brightdata/scrapers/__init__.py +++ b/src/brightdata/scrapers/__init__.py @@ -1,8 +1,8 @@ """Specialized platform scrapers.""" from .base import BaseWebScraper -from .registry import register, get_scraper_for, get_registered_platforms, is_platform_supported from .job import ScrapeJob +from .registry import get_registered_platforms, get_scraper_for, is_platform_supported, register # Import scrapers to trigger registration try: diff --git a/src/brightdata/scrapers/amazon/schemas.py b/src/brightdata/scrapers/amazon/schemas.py index 2b7c821..17c3ce7 100644 --- a/src/brightdata/scrapers/amazon/schemas.py +++ b/src/brightdata/scrapers/amazon/schemas.py @@ -6,7 +6,7 @@ """ from dataclasses import dataclass -from typing import Optional, List, Dict, Any +from typing import Any, Dict, List, Optional @dataclass diff --git a/src/brightdata/scrapers/amazon/scraper.py b/src/brightdata/scrapers/amazon/scraper.py index 91d7d49..2010607 100644 --- a/src/brightdata/scrapers/amazon/scraper.py +++ b/src/brightdata/scrapers/amazon/scraper.py @@ -14,15 +14,15 @@ """ import asyncio -from typing import Union, List, Optional, Any +from typing import Any, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("amazon") diff --git a/src/brightdata/scrapers/amazon/search.py b/src/brightdata/scrapers/amazon/search.py index 74b2a13..cfc4121 100644 --- a/src/brightdata/scrapers/amazon/search.py +++ b/src/brightdata/scrapers/amazon/search.py @@ -9,12 +9,12 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any, Dict, List, Optional, Union -from ...models import ScrapeResult +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...exceptions import ValidationError +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD from ..base import ScraperCore diff --git a/src/brightdata/scrapers/api_client.py b/src/brightdata/scrapers/api_client.py index 2443517..2b2bc02 100644 --- a/src/brightdata/scrapers/api_client.py +++ b/src/brightdata/scrapers/api_client.py @@ -7,10 +7,10 @@ - Fetching snapshot results """ -from typing import List, Dict, Any, Optional +from http import HTTPStatus +from typing import Any, Dict, List, Optional from ..core.engine import AsyncEngine -from http import HTTPStatus from ..exceptions import APIError, DataNotReadyError diff --git a/src/brightdata/scrapers/base.py b/src/brightdata/scrapers/base.py index c76b86f..9a8bf35 100644 --- a/src/brightdata/scrapers/base.py +++ b/src/brightdata/scrapers/base.py @@ -10,26 +10,26 @@ """ import asyncio +import concurrent.futures import os import time -import concurrent.futures from abc import ABC from datetime import datetime, timezone -from typing import List, Dict, Any, Optional, Union +from typing import Any, Dict, List, Optional, Union -from ..core.engine import AsyncEngine -from ..models import ScrapeResult -from ..exceptions import ValidationError, APIError -from ..utils.validation import validate_url, validate_url_list -from ..utils.function_detection import get_caller_function_name from ..constants import ( - DEFAULT_POLL_INTERVAL, - DEFAULT_MIN_POLL_TIMEOUT, DEFAULT_COST_PER_RECORD, + DEFAULT_MIN_POLL_TIMEOUT, + DEFAULT_POLL_INTERVAL, ) +from ..core.engine import AsyncEngine +from ..exceptions import APIError, ValidationError +from ..models import ScrapeResult +from ..utils.function_detection import get_caller_function_name +from ..utils.validation import validate_url, validate_url_list from .api_client import DatasetAPIClient -from .workflow import WorkflowExecutor from .job import ScrapeJob +from .workflow import WorkflowExecutor class ScraperCore: diff --git a/src/brightdata/scrapers/chatgpt/scraper.py b/src/brightdata/scrapers/chatgpt/scraper.py index d98cce4..a250667 100644 --- a/src/brightdata/scrapers/chatgpt/scraper.py +++ b/src/brightdata/scrapers/chatgpt/scraper.py @@ -8,15 +8,15 @@ """ import asyncio -from typing import List, Any, Optional, Union +from typing import Any, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_CHATGPT, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_LONG +from ...exceptions import ValidationError from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_LONG, COST_PER_RECORD_CHATGPT -from ...exceptions import ValidationError +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("chatgpt") diff --git a/src/brightdata/scrapers/chatgpt/search.py b/src/brightdata/scrapers/chatgpt/search.py index a214085..51bca89 100644 --- a/src/brightdata/scrapers/chatgpt/search.py +++ b/src/brightdata/scrapers/chatgpt/search.py @@ -10,12 +10,12 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any, Dict, List, Optional, Union -from ...models import ScrapeResult +from ...constants import COST_PER_RECORD_CHATGPT, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT from ...exceptions import ValidationError +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_CHATGPT from ..base import ScraperCore diff --git a/src/brightdata/scrapers/digikey/scraper.py b/src/brightdata/scrapers/digikey/scraper.py index 3ffeb98..2cea57e 100644 --- a/src/brightdata/scrapers/digikey/scraper.py +++ b/src/brightdata/scrapers/digikey/scraper.py @@ -13,15 +13,15 @@ """ import asyncio -from typing import List, Any, Union +from typing import Any, List, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("digikey") diff --git a/src/brightdata/scrapers/facebook/scraper.py b/src/brightdata/scrapers/facebook/scraper.py index 6473547..ca5c584 100644 --- a/src/brightdata/scrapers/facebook/scraper.py +++ b/src/brightdata/scrapers/facebook/scraper.py @@ -19,15 +19,15 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any, Dict, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_FACEBOOK, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, COST_PER_RECORD_FACEBOOK +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("facebook") diff --git a/src/brightdata/scrapers/instagram/scraper.py b/src/brightdata/scrapers/instagram/scraper.py index b7c45df..6608f37 100644 --- a/src/brightdata/scrapers/instagram/scraper.py +++ b/src/brightdata/scrapers/instagram/scraper.py @@ -9,19 +9,19 @@ """ import asyncio -from typing import List, Any, Union +from typing import Any, List, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_INSTAGRAM, - DEFAULT_TIMEOUT_SHORT, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_SHORT, ) -from ...utils.validation import validate_url, validate_url_list +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("instagram") diff --git a/src/brightdata/scrapers/instagram/search.py b/src/brightdata/scrapers/instagram/search.py index 8f5a882..daaec68 100644 --- a/src/brightdata/scrapers/instagram/search.py +++ b/src/brightdata/scrapers/instagram/search.py @@ -8,17 +8,17 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any, Dict, List, Optional, Union -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_INSTAGRAM, - DEFAULT_TIMEOUT_SHORT, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_SHORT, ) -from ...utils.validation import validate_url_list, validate_instagram_date +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ...utils.validation import validate_instagram_date, validate_url_list +from ..base import ScraperCore class InstagramSearchScraper(ScraperCore): diff --git a/src/brightdata/scrapers/job.py b/src/brightdata/scrapers/job.py index a4d079f..e9e6687 100644 --- a/src/brightdata/scrapers/job.py +++ b/src/brightdata/scrapers/job.py @@ -9,12 +9,12 @@ import asyncio import time -from typing import Optional, Any from datetime import datetime, timezone +from typing import Any, Optional -from ..models import ScrapeResult -from ..exceptions import APIError from ..constants import DEFAULT_POLL_INTERVAL +from ..exceptions import APIError +from ..models import ScrapeResult from .api_client import DatasetAPIClient diff --git a/src/brightdata/scrapers/linkedin/scraper.py b/src/brightdata/scrapers/linkedin/scraper.py index 3327e67..e3a8f1b 100644 --- a/src/brightdata/scrapers/linkedin/scraper.py +++ b/src/brightdata/scrapers/linkedin/scraper.py @@ -23,15 +23,15 @@ """ import asyncio -from typing import Union, List, Any +from typing import Any, List, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_LINKEDIN, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_LINKEDIN +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("linkedin") diff --git a/src/brightdata/scrapers/linkedin/search.py b/src/brightdata/scrapers/linkedin/search.py index f72261e..d82549b 100644 --- a/src/brightdata/scrapers/linkedin/search.py +++ b/src/brightdata/scrapers/linkedin/search.py @@ -11,12 +11,12 @@ """ import asyncio -from typing import Union, List, Optional, Dict, Any +from typing import Any, Dict, List, Optional, Union -from ...models import ScrapeResult +from ...constants import COST_PER_RECORD_LINKEDIN, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT from ...exceptions import ValidationError +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_LINKEDIN from ..base import ScraperCore diff --git a/src/brightdata/scrapers/perplexity/scraper.py b/src/brightdata/scrapers/perplexity/scraper.py index 58fc9fa..d7deeeb 100644 --- a/src/brightdata/scrapers/perplexity/scraper.py +++ b/src/brightdata/scrapers/perplexity/scraper.py @@ -13,15 +13,15 @@ """ import asyncio -from typing import List, Any, Optional, Union +from typing import Any, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_PERPLEXITY, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT +from ...exceptions import ValidationError from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_SHORT, COST_PER_RECORD_PERPLEXITY -from ...exceptions import ValidationError +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("perplexity") diff --git a/src/brightdata/scrapers/pinterest/scraper.py b/src/brightdata/scrapers/pinterest/scraper.py index 388efab..c21ddef 100644 --- a/src/brightdata/scrapers/pinterest/scraper.py +++ b/src/brightdata/scrapers/pinterest/scraper.py @@ -15,15 +15,15 @@ """ import asyncio -from typing import List, Any, Union +from typing import Any, List, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("pinterest") diff --git a/src/brightdata/scrapers/pinterest/search.py b/src/brightdata/scrapers/pinterest/search.py index c116833..1f00df8 100644 --- a/src/brightdata/scrapers/pinterest/search.py +++ b/src/brightdata/scrapers/pinterest/search.py @@ -16,16 +16,16 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any, Dict, List, Optional, Union -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( DEFAULT_COST_PER_RECORD, - DEFAULT_TIMEOUT_MEDIUM, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_MEDIUM, ) +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ..base import ScraperCore class PinterestSearchScraper(ScraperCore): diff --git a/src/brightdata/scrapers/reddit/scraper.py b/src/brightdata/scrapers/reddit/scraper.py index 3351cf6..705f7e1 100644 --- a/src/brightdata/scrapers/reddit/scraper.py +++ b/src/brightdata/scrapers/reddit/scraper.py @@ -19,15 +19,15 @@ """ import asyncio -from typing import List, Any, Optional, Union, Dict +from typing import Any, Dict, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("reddit") diff --git a/src/brightdata/scrapers/registry.py b/src/brightdata/scrapers/registry.py index 9ba05bc..b154cae 100644 --- a/src/brightdata/scrapers/registry.py +++ b/src/brightdata/scrapers/registry.py @@ -12,7 +12,8 @@ import logging import pkgutil from functools import lru_cache -from typing import Dict, Type, Optional, List +from typing import Dict, List, Optional, Type + import tldextract # Configure logger for registry operations diff --git a/src/brightdata/scrapers/tiktok/scraper.py b/src/brightdata/scrapers/tiktok/scraper.py index f250bf4..95397f0 100644 --- a/src/brightdata/scrapers/tiktok/scraper.py +++ b/src/brightdata/scrapers/tiktok/scraper.py @@ -22,15 +22,15 @@ """ import asyncio -from typing import List, Any, Optional, Union, Dict +from typing import Any, Dict, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_TIKTOK, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, COST_PER_RECORD_TIKTOK +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("tiktok") diff --git a/src/brightdata/scrapers/tiktok/search.py b/src/brightdata/scrapers/tiktok/search.py index b7eede2..6e834d1 100644 --- a/src/brightdata/scrapers/tiktok/search.py +++ b/src/brightdata/scrapers/tiktok/search.py @@ -16,16 +16,16 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any, Dict, List, Optional, Union -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_TIKTOK, - DEFAULT_TIMEOUT_MEDIUM, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_MEDIUM, ) +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ..base import ScraperCore class TikTokSearchScraper(ScraperCore): diff --git a/src/brightdata/scrapers/workflow.py b/src/brightdata/scrapers/workflow.py index 02d0952..e228e5b 100644 --- a/src/brightdata/scrapers/workflow.py +++ b/src/brightdata/scrapers/workflow.py @@ -7,12 +7,12 @@ 3. Fetch results when ready """ -from typing import List, Dict, Any, Optional, Callable from datetime import datetime, timezone +from typing import Any, Callable, Dict, List, Optional -from ..models import ScrapeResult +from ..constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT from ..exceptions import APIError -from ..constants import DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT, DEFAULT_COST_PER_RECORD +from ..models import ScrapeResult from ..utils.polling import poll_until_ready from .api_client import DatasetAPIClient diff --git a/src/brightdata/scrapers/x/scraper.py b/src/brightdata/scrapers/x/scraper.py index c25d82a..cb8c01d 100644 --- a/src/brightdata/scrapers/x/scraper.py +++ b/src/brightdata/scrapers/x/scraper.py @@ -19,15 +19,15 @@ """ import asyncio -from typing import List, Any, Union, Optional +from typing import Any, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import DEFAULT_COST_PER_RECORD, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, DEFAULT_COST_PER_RECORD +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("x") diff --git a/src/brightdata/scrapers/youtube/scraper.py b/src/brightdata/scrapers/youtube/scraper.py index fc13a03..9adcc1a 100644 --- a/src/brightdata/scrapers/youtube/scraper.py +++ b/src/brightdata/scrapers/youtube/scraper.py @@ -18,15 +18,15 @@ """ import asyncio -from typing import List, Any, Optional, Union +from typing import Any, List, Optional, Union -from ..base import BaseWebScraper -from ..registry import register -from ..job import ScrapeJob +from ...constants import COST_PER_RECORD_YOUTUBE, DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM from ...models import ScrapeResult -from ...utils.validation import validate_url, validate_url_list from ...utils.function_detection import get_caller_function_name -from ...constants import DEFAULT_POLL_INTERVAL, DEFAULT_TIMEOUT_MEDIUM, COST_PER_RECORD_YOUTUBE +from ...utils.validation import validate_url, validate_url_list +from ..base import BaseWebScraper +from ..job import ScrapeJob +from ..registry import register @register("youtube") diff --git a/src/brightdata/scrapers/youtube/search.py b/src/brightdata/scrapers/youtube/search.py index 4869770..201fa21 100644 --- a/src/brightdata/scrapers/youtube/search.py +++ b/src/brightdata/scrapers/youtube/search.py @@ -19,16 +19,16 @@ """ import asyncio -from typing import List, Dict, Any, Optional, Union +from typing import Any, Dict, List, Optional, Union -from ..base import ScraperCore -from ...models import ScrapeResult from ...constants import ( COST_PER_RECORD_YOUTUBE, - DEFAULT_TIMEOUT_MEDIUM, DEFAULT_POLL_INTERVAL, + DEFAULT_TIMEOUT_MEDIUM, ) +from ...models import ScrapeResult from ...utils.function_detection import get_caller_function_name +from ..base import ScraperCore class YouTubeSearchScraper(ScraperCore): diff --git a/src/brightdata/serp/__init__.py b/src/brightdata/serp/__init__.py index 1ee43bc..05e91da 100644 --- a/src/brightdata/serp/__init__.py +++ b/src/brightdata/serp/__init__.py @@ -1,10 +1,10 @@ """SERP API services.""" from .base import BaseSERPService -from .google import GoogleSERPService from .bing import BingSERPService -from .yandex import YandexSERPService +from .google import GoogleSERPService from .service import SearchService +from .yandex import YandexSERPService __all__ = [ "BaseSERPService", diff --git a/src/brightdata/serp/base.py b/src/brightdata/serp/base.py index 3102d2a..b47d5e4 100644 --- a/src/brightdata/serp/base.py +++ b/src/brightdata/serp/base.py @@ -1,24 +1,25 @@ """Base SERP service with separated responsibilities.""" import asyncio -import aiohttp import json import re import time import warnings -from typing import Union, List, Optional, Dict, Any, Tuple from datetime import datetime, timezone +from http import HTTPStatus +from typing import Any, Dict, List, Optional, Tuple, Union + +import aiohttp -from .url_builder import BaseURLBuilder -from .data_normalizer import BaseDataNormalizer from ..core.engine import AsyncEngine -from ..models import SearchResult -from http import HTTPStatus from ..exceptions import ValidationError -from ..utils.validation import validate_zone_name -from ..utils.retry import retry_with_backoff +from ..models import SearchResult from ..utils.function_detection import get_caller_function_name +from ..utils.retry import retry_with_backoff +from ..utils.validation import validate_zone_name from ..web_unlocker.async_client import AsyncUnblockerClient +from .data_normalizer import BaseDataNormalizer +from .url_builder import BaseURLBuilder class BaseSERPService: diff --git a/src/brightdata/serp/bing.py b/src/brightdata/serp/bing.py index 64862e9..a1d080d 100644 --- a/src/brightdata/serp/bing.py +++ b/src/brightdata/serp/bing.py @@ -1,10 +1,11 @@ """Bing SERP service.""" from typing import Optional + +from ..core.engine import AsyncEngine from .base import BaseSERPService -from .url_builder import BingURLBuilder from .data_normalizer import BingDataNormalizer -from ..core.engine import AsyncEngine +from .url_builder import BingURLBuilder class BingSERPService(BaseSERPService): diff --git a/src/brightdata/serp/data_normalizer.py b/src/brightdata/serp/data_normalizer.py index 3d3b31b..ad775bf 100644 --- a/src/brightdata/serp/data_normalizer.py +++ b/src/brightdata/serp/data_normalizer.py @@ -3,6 +3,7 @@ import warnings from abc import ABC, abstractmethod from typing import Any + from ..types import NormalizedSERPData diff --git a/src/brightdata/serp/google.py b/src/brightdata/serp/google.py index 3bf662a..29f91e5 100644 --- a/src/brightdata/serp/google.py +++ b/src/brightdata/serp/google.py @@ -1,10 +1,11 @@ """Google SERP service.""" from typing import Optional + +from ..core.engine import AsyncEngine from .base import BaseSERPService -from .url_builder import GoogleURLBuilder from .data_normalizer import GoogleDataNormalizer -from ..core.engine import AsyncEngine +from .url_builder import GoogleURLBuilder class GoogleSERPService(BaseSERPService): diff --git a/src/brightdata/serp/service.py b/src/brightdata/serp/service.py index 8aff7d7..c1ea7c3 100644 --- a/src/brightdata/serp/service.py +++ b/src/brightdata/serp/service.py @@ -6,21 +6,21 @@ All methods are async-only. For sync usage, use SyncBrightDataClient. """ -from typing import Optional, Union, List, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional, Union from ..models import SearchResult if TYPE_CHECKING: from ..client import BrightDataClient - from .google import GoogleSERPService - from .bing import BingSERPService - from .yandex import YandexSERPService from ..scrapers.amazon.search import AmazonSearchScraper - from ..scrapers.linkedin.search import LinkedInSearchScraper from ..scrapers.chatgpt.search import ChatGPTSearchService from ..scrapers.instagram.search import InstagramSearchScraper + from ..scrapers.linkedin.search import LinkedInSearchScraper from ..scrapers.tiktok.search import TikTokSearchScraper from ..scrapers.youtube.search import YouTubeSearchScraper + from .bing import BingSERPService + from .google import GoogleSERPService + from .yandex import YandexSERPService class SearchService: diff --git a/src/brightdata/serp/url_builder.py b/src/brightdata/serp/url_builder.py index c66af13..d09bc55 100644 --- a/src/brightdata/serp/url_builder.py +++ b/src/brightdata/serp/url_builder.py @@ -3,7 +3,8 @@ from abc import ABC, abstractmethod from typing import Optional from urllib.parse import quote_plus -from ..utils.location import LocationService, LocationFormat + +from ..utils.location import LocationFormat, LocationService class BaseURLBuilder(ABC): diff --git a/src/brightdata/serp/yandex.py b/src/brightdata/serp/yandex.py index 6173876..08fd932 100644 --- a/src/brightdata/serp/yandex.py +++ b/src/brightdata/serp/yandex.py @@ -1,10 +1,11 @@ """Yandex SERP service.""" from typing import Optional + +from ..core.engine import AsyncEngine from .base import BaseSERPService -from .url_builder import YandexURLBuilder from .data_normalizer import YandexDataNormalizer -from ..core.engine import AsyncEngine +from .url_builder import YandexURLBuilder class YandexSERPService(BaseSERPService): diff --git a/src/brightdata/sync_client.py b/src/brightdata/sync_client.py index 8a6995a..447ee1b 100644 --- a/src/brightdata/sync_client.py +++ b/src/brightdata/sync_client.py @@ -6,14 +6,14 @@ import asyncio import logging -from typing import Optional, List, Dict, Any +from typing import Any, Dict, List, Optional logger = logging.getLogger(__name__) -from .client import BrightDataClient from .browser.service import BrowserService -from .models import ScrapeResult, SearchResult +from .client import BrightDataClient from .discover.models import DiscoverResult, DiscoverSnapshot +from .models import ScrapeResult, SearchResult from .types import AccountInfo diff --git a/src/brightdata/types.py b/src/brightdata/types.py index 43a7807..0af877e 100644 --- a/src/brightdata/types.py +++ b/src/brightdata/types.py @@ -4,7 +4,8 @@ This module provides type definitions for API responses used internally. """ -from typing import TypedDict, Optional, List, Dict, Any +from typing import Any, Dict, List, Optional, TypedDict + from typing_extensions import NotRequired diff --git a/src/brightdata/utils/location.py b/src/brightdata/utils/location.py index 97b03d6..83ff8a3 100644 --- a/src/brightdata/utils/location.py +++ b/src/brightdata/utils/location.py @@ -1,7 +1,7 @@ """Location parsing utilities for SERP services.""" -from typing import Dict from enum import Enum +from typing import Dict class LocationFormat(Enum): diff --git a/src/brightdata/utils/polling.py b/src/brightdata/utils/polling.py index 94c4678..e916ea6 100644 --- a/src/brightdata/utils/polling.py +++ b/src/brightdata/utils/polling.py @@ -11,12 +11,12 @@ from __future__ import annotations import asyncio -from typing import Any, List, Callable, Awaitable from datetime import datetime, timezone +from typing import Any, Awaitable, Callable, List -from ..models import ScrapeResult from ..constants import DEFAULT_POLL_INTERVAL, DEFAULT_POLL_TIMEOUT from ..exceptions import DataNotReadyError +from ..models import ScrapeResult async def poll_until_ready( diff --git a/src/brightdata/utils/retry.py b/src/brightdata/utils/retry.py index 6cb91d6..47a69c5 100644 --- a/src/brightdata/utils/retry.py +++ b/src/brightdata/utils/retry.py @@ -1,7 +1,8 @@ """Retry logic with exponential backoff.""" import asyncio -from typing import Callable, Awaitable, TypeVar, Optional, List, Type +from typing import Awaitable, Callable, List, Optional, Type, TypeVar + from ..exceptions import APIError, NetworkError T = TypeVar("T") diff --git a/src/brightdata/utils/ssl_helpers.py b/src/brightdata/utils/ssl_helpers.py index a709651..6cf6a36 100644 --- a/src/brightdata/utils/ssl_helpers.py +++ b/src/brightdata/utils/ssl_helpers.py @@ -5,8 +5,8 @@ particularly common on macOS systems. """ -import sys import ssl +import sys try: import aiohttp diff --git a/src/brightdata/utils/url.py b/src/brightdata/utils/url.py index 5127b32..185c131 100644 --- a/src/brightdata/utils/url.py +++ b/src/brightdata/utils/url.py @@ -1,7 +1,7 @@ """URL utilities.""" -from urllib.parse import urlparse from typing import Optional +from urllib.parse import urlparse def extract_root_domain(url: str) -> Optional[str]: diff --git a/src/brightdata/utils/validation.py b/src/brightdata/utils/validation.py index c38a52c..818cdcf 100644 --- a/src/brightdata/utils/validation.py +++ b/src/brightdata/utils/validation.py @@ -1,8 +1,9 @@ """Input validation utilities.""" import re -from urllib.parse import urlparse from typing import List +from urllib.parse import urlparse + from ..exceptions import ValidationError diff --git a/src/brightdata/web_unlocker/async_client.py b/src/brightdata/web_unlocker/async_client.py index 3b6d295..8ba837e 100644 --- a/src/brightdata/web_unlocker/async_client.py +++ b/src/brightdata/web_unlocker/async_client.py @@ -19,7 +19,8 @@ - See devdocs/web_unlocker_async_inspection.md for details """ -from typing import Optional, Any +from typing import Any, Optional + from ..core.engine import AsyncEngine from ..exceptions import APIError diff --git a/src/brightdata/web_unlocker/base.py b/src/brightdata/web_unlocker/base.py index 31cc7be..c7283f0 100644 --- a/src/brightdata/web_unlocker/base.py +++ b/src/brightdata/web_unlocker/base.py @@ -2,6 +2,7 @@ from abc import ABC, abstractmethod from typing import Any + from ..core.engine import AsyncEngine diff --git a/src/brightdata/web_unlocker/service.py b/src/brightdata/web_unlocker/service.py index db2feca..26b821f 100644 --- a/src/brightdata/web_unlocker/service.py +++ b/src/brightdata/web_unlocker/service.py @@ -3,26 +3,26 @@ All methods are async-only. For sync usage, use SyncBrightDataClient. """ -from typing import Union, List, Optional, Dict, Any -from datetime import datetime, timezone import asyncio +from datetime import datetime, timezone +from http import HTTPStatus +from typing import Any, Dict, List, Optional, Union -from .base import BaseAPI -from .async_client import AsyncUnblockerClient +from ..exceptions import APIError, ValidationError from ..models import ScrapeResult +from ..utils.function_detection import get_caller_function_name +from ..utils.url import extract_root_domain from ..utils.validation import ( + validate_country_code, + validate_http_method, + validate_response_format, + validate_timeout, validate_url, validate_url_list, validate_zone_name, - validate_country_code, - validate_timeout, - validate_response_format, - validate_http_method, ) -from ..utils.url import extract_root_domain -from ..utils.function_detection import get_caller_function_name -from http import HTTPStatus -from ..exceptions import ValidationError, APIError +from .async_client import AsyncUnblockerClient +from .base import BaseAPI class WebUnlockerService(BaseAPI): diff --git a/tests/conftest.py b/tests/conftest.py index edcce14..278c49e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ """Shared test fixtures for Bright Data SDK tests.""" -import sys import os -from unittest.mock import AsyncMock, MagicMock +import sys from typing import Any, Dict, Optional +from unittest.mock import AsyncMock, MagicMock import pytest diff --git a/tests/unit/test_async_unblocker.py b/tests/unit/test_async_unblocker.py index bc75ed1..13ac4ed 100644 --- a/tests/unit/test_async_unblocker.py +++ b/tests/unit/test_async_unblocker.py @@ -1,14 +1,13 @@ """Tests for web_unlocker/async_client.py — Trigger, status, and fetch operations.""" -import pytest from unittest.mock import AsyncMock, MagicMock -from brightdata.web_unlocker.async_client import AsyncUnblockerClient -from brightdata.exceptions import APIError +import pytest +from brightdata.exceptions import APIError +from brightdata.web_unlocker.async_client import AsyncUnblockerClient from tests.conftest import MockContextManager - # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- diff --git a/tests/unit/test_cli_credentials.py b/tests/unit/test_cli_credentials.py index 3c2fd5f..c956586 100644 --- a/tests/unit/test_cli_credentials.py +++ b/tests/unit/test_cli_credentials.py @@ -13,8 +13,8 @@ import pytest import brightdata.client as client_module -from brightdata.client import BrightDataClient from brightdata.cli_credentials import _cli_credentials_path, read_cli_credentials +from brightdata.client import BrightDataClient from brightdata.core.engine import AsyncEngine from brightdata.exceptions import ValidationError diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 5e12494..4bfce4b 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -1,17 +1,16 @@ """Tests for client.py — BrightDataClient init, services, context manager, API methods.""" -import pytest from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from brightdata.client import BrightDataClient -from brightdata.exceptions import ValidationError, AuthenticationError, APIError -from brightdata.scrapers.service import ScrapeService -from brightdata.serp.service import SearchService from brightdata.crawler.service import CrawlerService from brightdata.datasets import DatasetsClient - -from tests.conftest import MockResponse, MockContextManager - +from brightdata.exceptions import APIError, AuthenticationError, ValidationError +from brightdata.scrapers.service import ScrapeService +from brightdata.serp.service import SearchService +from tests.conftest import MockContextManager, MockResponse # --------------------------------------------------------------------------- # Token loading diff --git a/tests/unit/test_colorless_service_verbs.py b/tests/unit/test_colorless_service_verbs.py index c40eea3..85a1f7f 100644 --- a/tests/unit/test_colorless_service_verbs.py +++ b/tests/unit/test_colorless_service_verbs.py @@ -12,16 +12,16 @@ Mocked at the api_client / _poll_once seam (not raw aiohttp), per the plan. """ -import pytest from unittest.mock import AsyncMock, MagicMock -from brightdata.scrapers.amazon import AmazonScraper -from brightdata.scrapers.job import ScrapeJob -from brightdata.discover.service import DiscoverService -from brightdata.models import ScrapeResult +import pytest + from brightdata.discover.models import DiscoverResult +from brightdata.discover.service import DiscoverService from brightdata.exceptions import APIError - +from brightdata.models import ScrapeResult +from brightdata.scrapers.amazon import AmazonScraper +from brightdata.scrapers.job import ScrapeJob # --------------------------------------------------------------------------- # Helpers diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 3d7ee11..a0700f4 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -3,15 +3,13 @@ import json from datetime import datetime, timezone - from brightdata.models import ( BaseResult, + CrawlResult, ScrapeResult, SearchResult, - CrawlResult, ) - # --------------------------------------------------------------------------- # BaseResult # --------------------------------------------------------------------------- diff --git a/tests/unit/test_payloads.py b/tests/unit/test_payloads.py index 6d764ed..f89ee98 100644 --- a/tests/unit/test_payloads.py +++ b/tests/unit/test_payloads.py @@ -1,28 +1,28 @@ """Tests for payload dataclasses — Validation, defaults, and serialization.""" import pytest + from brightdata.payloads import ( # Amazon AmazonProductPayload, AmazonReviewPayload, - LinkedInProfilePayload, - LinkedInProfileSearchPayload, - LinkedInJobSearchPayload, - LinkedInPostSearchPayload, # ChatGPT ChatGPTPromptPayload, + FacebookCommentsPayload, + FacebookPostPayload, + FacebookPostsGroupPayload, # Facebook FacebookPostsProfilePayload, - FacebookPostsGroupPayload, - FacebookPostPayload, - FacebookCommentsPayload, - InstagramProfilePayload, InstagramPostPayload, - InstagramReelPayload, InstagramPostsDiscoverPayload, + InstagramProfilePayload, + InstagramReelPayload, + LinkedInJobSearchPayload, + LinkedInPostSearchPayload, + LinkedInProfilePayload, + LinkedInProfileSearchPayload, ) - # --------------------------------------------------------------------------- # Amazon # --------------------------------------------------------------------------- diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 2ee48ae..311ec7f 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -4,9 +4,8 @@ import pytest +from brightdata.exceptions import APIError, AuthenticationError, NetworkError, ValidationError from brightdata.utils.retry import retry_with_backoff -from brightdata.exceptions import APIError, NetworkError, AuthenticationError, ValidationError - # --------------------------------------------------------------------------- # Happy path diff --git a/tests/unit/test_scraper_core.py b/tests/unit/test_scraper_core.py index c074d15..3cbe3d9 100644 --- a/tests/unit/test_scraper_core.py +++ b/tests/unit/test_scraper_core.py @@ -7,12 +7,13 @@ tests pin the consolidated behavior. """ -import pytest from unittest.mock import MagicMock +import pytest + from brightdata.exceptions import ValidationError -from brightdata.scrapers.base import BaseWebScraper, ScraperCore from brightdata.scrapers.amazon.search import AmazonSearchScraper +from brightdata.scrapers.base import BaseWebScraper, ScraperCore from brightdata.scrapers.chatgpt.search import ChatGPTSearchService from brightdata.scrapers.instagram.search import InstagramSearchScraper from brightdata.scrapers.linkedin.search import LinkedInSearchScraper diff --git a/tests/unit/test_ssl_helpers.py b/tests/unit/test_ssl_helpers.py index 3f2fe4f..4645940 100644 --- a/tests/unit/test_ssl_helpers.py +++ b/tests/unit/test_ssl_helpers.py @@ -3,8 +3,7 @@ import ssl from unittest.mock import Mock, patch -from brightdata.utils.ssl_helpers import is_macos, is_ssl_certificate_error, get_ssl_error_message - +from brightdata.utils.ssl_helpers import get_ssl_error_message, is_macos, is_ssl_certificate_error # --------------------------------------------------------------------------- # Platform detection diff --git a/tests/unit/test_sync_client_coverage.py b/tests/unit/test_sync_client_coverage.py index f622c1b..4eede2e 100644 --- a/tests/unit/test_sync_client_coverage.py +++ b/tests/unit/test_sync_client_coverage.py @@ -11,26 +11,26 @@ """ import asyncio +from unittest.mock import AsyncMock, MagicMock import pytest -from unittest.mock import AsyncMock, MagicMock from brightdata import SyncBrightDataClient -from brightdata.models import ScrapeResult from brightdata.discover.models import DiscoverSnapshot +from brightdata.models import ScrapeResult from brightdata.scrapers.service import ScrapeService from brightdata.serp.service import SearchService from brightdata.sync_client import ( - SyncDatasetsClient, SyncDataset, + SyncDatasetsClient, + SyncDigiKeyScraper, + SyncInstagramSearchScraper, + SyncPerplexityScraper, SyncRedditScraper, SyncTikTokScraper, - SyncYouTubeScraper, - SyncPerplexityScraper, - SyncDigiKeyScraper, SyncTikTokSearchScraper, + SyncYouTubeScraper, SyncYouTubeSearchScraper, - SyncInstagramSearchScraper, ) diff --git a/tests/unit/test_x_scraper.py b/tests/unit/test_x_scraper.py index baf9ad2..eafdc13 100644 --- a/tests/unit/test_x_scraper.py +++ b/tests/unit/test_x_scraper.py @@ -7,12 +7,11 @@ """ import asyncio - from unittest.mock import AsyncMock, MagicMock -from brightdata.scrapers.x.scraper import XScraper -from brightdata.scrapers.registry import get_scraper_for from brightdata.models import ScrapeResult +from brightdata.scrapers.registry import get_scraper_for +from brightdata.scrapers.x.scraper import XScraper from brightdata.sync_client import SyncXScraper POSTS = "gd_lwxkxvnf1cynvib9co" diff --git a/tests/unit/test_zone_manager.py b/tests/unit/test_zone_manager.py index 0709c96..10aff3f 100644 --- a/tests/unit/test_zone_manager.py +++ b/tests/unit/test_zone_manager.py @@ -3,10 +3,8 @@ import pytest from brightdata.core.zone_manager import ZoneManager -from brightdata.exceptions.errors import ZoneError, AuthenticationError - -from tests.conftest import MockResponse, MockContextManager - +from brightdata.exceptions.errors import AuthenticationError, ZoneError +from tests.conftest import MockContextManager, MockResponse # --------------------------------------------------------------------------- # List Zones