Skills之Scrapling:Python网络爬虫框架,支持反机器人绕过、JS渲染和Cloudflare保护网站抓取 GitHub Stars 13.9万+

pip install "scrapling[all]"scrapling install

pip install scrapling

pip install "scrapling[fetchers]"scrapling install

scrapling extract get 'https://example.com' output.md

scrapling extract get 'https://example.com' output.md \  --css-selector '.content' \  --impersonate 'chrome'

scrapling extract fetch 'https://example.com' output.md \  --css-selector '.dynamic-content' \  --disable-resources \  --network-idle

scrapling extract stealthy-fetch 'https://protected-site.com' output.html \  --solve-cloudflare \  --block-webrtc \  --hide-canvas

scrapling extract post 'https://example.com/api' output.json \  --json '{"query": "search term"}'

from scrapling.fetchers import Fetcherpage = Fetcher.get('https://quotes.toscrape.com/')quotes = page.css('.quote .text::text').getall()for q in quotes:    print(q)

from scrapling.fetchers import FetcherSessionwith FetcherSession(impersonate='chrome') as session:    page = session.get('https://example.com/', stealthy_headers=True)    links = page.css('a::attr(href)').getall()    for link in links[:5]:        sub = session.get(link)        print(sub.css('h1::text').get())

page = Fetcher.post('https://api.example.com/data', json={"key": "value"})page = Fetcher.put('https://api.example.com/item/1', data={"name": "updated"})page = Fetcher.delete('https://api.example.com/item/1')

page = Fetcher.get('https://example.com', proxy='http://user:pass@proxy:8080')

from scrapling.fetchers import DynamicFetcherpage = DynamicFetcher.fetch('https://example.com', headless=True)data = page.css('.js-loaded-content::text').getall()

page = DynamicFetcher.fetch(    'https://example.com',    wait_selector=('.results', 'visible'),    network_idle=True,)

from scrapling.fetchers import DynamicSessionwith DynamicSession(headless=True, disable_resources=True, network_idle=True) as session:    page = session.fetch('https://example.com')    items = page.css('.item::text').getall()

from playwright.sync_api import Pagefrom scrapling.fetchers import DynamicFetcherdef scroll_and_click(page: Page):    page.mouse.wheel(0, 3000)    page.wait_for_timeout(1000)    page.click('button.load-more')    page.wait_for_selector('.extra-results')page = DynamicFetcher.fetch('https://example.com', page_action=scroll_and_click)results = page.css('.extra-results .item::text').getall()

from scrapling.fetchers import StealthyFetcherpage = StealthyFetcher.fetch(    'https://protected-site.com',    headless=True,    solve_cloudflare=True,    block_webrtc=True,    hide_canvas=True,)content = page.css('.protected-content::text').getall()

from scrapling.fetchers import StealthySessionwith StealthySession(headless=True, solve_cloudflare=True) as session:    page1 = session.fetch('https://protected-site.com/page1')    page2 = session.fetch('https://protected-site.com/page2')

page.css('h1::text').get()              # 第一个 h1 文本page.css('a::attr(href)').getall()      # 所有链接的 hrefpage.css('.quote .text::text').getall() # 嵌套选择

page.xpath('//div[@class="content"]/text()').getall()page.xpath('//a/@href').getall()

page.find_all('div', class_='quote')       # 通过标签 + 属性page.find_by_text('Read more', tag='a')    # 通过文本内容page.find_by_regex(r'\$\d+\.\d{2}')       # 通过正则表达式模式

first_product = page.css('.product')[0]all_similar = first_product.find_similar()

el = page.css('.target')[0]el.parent                # 父元素el.children              # 子元素el.next_sibling          # 下一个兄弟元素el.prev_sibling          # 上一个兄弟元素

from scrapling.spiders import Spider, Request, Responseclass QuotesSpider(Spider):    name = "quotes"    start_urls = ["https://quotes.toscrape.com/"]    concurrent_requests = 10    download_delay = 1    async def parse(self, response: Response):        for quote in response.css('.quote'):            yield {                "text": quote.css('.text::text').get(),                "author": quote.css('.author::text').get(),                "tags": quote.css('.tag::text').getall(),            }        next_page = response.css('.next a::attr(href)').get()        if next_page:            yield response.follow(next_page)result = QuotesSpider().start()print(f"Scraped {len(result.items)} quotes")result.items.to_json("quotes.json")

from scrapling.fetchers import FetcherSession, AsyncStealthySessionclass SmartSpider(Spider):    name = "smart"    start_urls = ["https://example.com/"]    def configure_sessions(self, manager):        manager.add("fast", FetcherSession(impersonate="chrome"))        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)    async def parse(self, response: Response):        for link in response.css('a::attr(href)').getall():            if "protected" in link:                yield Request(link, sid="stealth")            else:                yield Request(link, sid="fast", callback=self.parse)

spider = QuotesSpider(crawldir="./crawl_checkpoint")spider.start()  # 按 Ctrl+C 暂停，重新运行以从检查点恢复

npx skills add https://github.com/nousresearch/hermes-agent --skill scrapling