Playwright 爬虫完全指南 2026 — 从安装到反爬虫机制绕过

# Playwright 설치
pip install playwright

# 브라우저 바이너리 설치 (Chromium, Firefox, WebKit 세 가지 모두)
playwright install

# Chromium만 설치하려면
playwright install chromium

# 새 프로젝트 초기화 (대화형 설정)
npm init playwright@latest

# 또는 직접 설치
npm install playwright
npx playwright install

FROM mcr.microsoft.com/playwright/python:v1.58.0-noble

WORKDIR /app
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY . .
CMD ["python", "crawler.py"]

from playwright.sync_api import sync_playwright

def scrape_headlines():
    with sync_playwright() as p:
        # headless=True 로 설정하면 브라우저 창 없이 실행
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()

        page.goto("https://news.ycombinator.com")

        # 모든 뉴스 제목 선택
        items = page.locator(".titleline > a").all()
        headlines = [item.inner_text() for item in items]

        for i, title in enumerate(headlines[:10], 1):
            print(f"{i}. {title}")

        browser.close()
        return headlines

if __name__ == "__main__":
    scrape_headlines()

const { chromium } = require('playwright');

async function scrapeHeadlines() {
  const browser = await chromium.launch({ headless: true });
  const page = await browser.newPage();

  await page.goto('https://news.ycombinator.com');

  const headlines = await page.locator('.titleline > a').allInnerTexts();

  headlines.slice(0, 10).forEach((title, i) => {
    console.log(`${i + 1}. ${title}`);
  });

  await browser.close();
  return headlines;
}

scrapeHeadlines();

# 스크린샷 저장
page.screenshot(path="screenshot.png", full_page=True)

# PDF 저장 (Chromium만 지원)
page.pdf(path="page.pdf")

from playwright.sync_api import sync_playwright

def scrape_paginated(base_url, max_pages=5):
    results = []
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()

        for page_num in range(1, max_pages + 1):
            page.goto(f"{base_url}?page={page_num}")
            # 요소가 로드될 때까지 대기
            page.wait_for_selector(".product-item")

            items = page.locator(".product-item").all()
            for item in items:
                results.append(item.inner_text())

            # 다음 페이지 버튼이 없으면 종료
            if not page.locator(".next-page").count():
                break

        browser.close()
    return results

def scrape_infinite_scroll(url, scroll_count=10):
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)

        prev_count = 0
        for _ in range(scroll_count):
            # 페이지 맨 아래까지 스크롤
            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            # 새 콘텐츠 로딩 대기
            page.wait_for_timeout(1500)

            current_count = page.locator(".item").count()
            if current_count == prev_count:
                break  # 더 이상 로딩되지 않으면 종료
            prev_count = current_count

        items = page.locator(".item").all_inner_texts()
        browser.close()
        return items

import json

def intercept_api(url):
    captured_data = []

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()

        # API 응답을 가로채는 핸들러 등록
        def handle_response(response):
            if "api/products" in response.url and response.status == 200:
                try:
                    data = response.json()
                    captured_data.extend(data.get("items", []))
                except Exception:
                    pass

        page.on("response", handle_response)
        page.goto(url)
        page.wait_for_load_state("networkidle")

        browser.close()
    return captured_data

from playwright.sync_api import sync_playwright

def scrape_parallel(urls):
    results = {}
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        # BrowserContext로 독립 세션 생성
        context = browser.new_context()

        pages = [context.new_page() for _ in urls]
        for page, url in zip(pages, urls):
            page.goto(url, wait_until="domcontentloaded")

        for page, url in zip(pages, urls):
            results[url] = page.title()

        browser.close()
    return results

pip install playwright-stealth

from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()
    stealth_sync(page)  # 스텔스 적용
    page.goto("https://target-site.com")

import random

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
]

context = browser.new_context(
    user_agent=random.choice(USER_AGENTS)
)

context = browser.new_context(
    proxy={
        "server": "http://proxy-server:8080",
        "username": "user",
        "password": "password"
    }
)

import random

# 마우스 움직임 추가
page.mouse.move(
    random.randint(100, 800),
    random.randint(100, 600)
)

# 자연스러운 타이핑 딜레이
page.locator("#search").type("검색어", delay=random.randint(80, 150))

# 스크롤 후 짧은 대기
page.evaluate("window.scrollBy(0, 300)")
page.wait_for_timeout(random.randint(800, 2000))

# Claude Desktop에 Playwright MCP 연동
npx @playwright/mcp@latest