디시인사이드 크롤링 자동화 심화 ガイド 2026 — IP 차단 회避から世論分析まで

https://search.dcinside.com/post/p/1/q/{키워드}/gallery/{갤러리ID}

import requests
from bs4 import BeautifulSoup
from urllib.parse import quote
import random
import time

def crawl_dcinside_search(keyword, gallery_id, max_pages=5):
    results = []
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    }

    for page in range(1, max_pages + 1):
        url = f"https://search.dcinside.com/post/p/{page}/q/{quote(keyword)}/gallery/{gallery_id}"
        resp = requests.get(url, headers=headers)
        if resp.status_code != 200:
            break

        soup = BeautifulSoup(resp.text, "html.parser")
        posts = soup.select(".sch_result_list li")
        for post in posts:
            title = post.select_one(".tit_txt")
            date = post.select_one(".date")
            link = post.select_one("a")
            if title and date and link:
                post_url = link["href"]
                post_id = post_url.split("no=")[-1].split("&")[0] if "no=" in post_url else post_url
                results.append({
                    "id": post_id,
                    "title": title.text.strip(),
                    "date": date.text.strip(),
                    "url": post_url
                })

        time.sleep(random.uniform(1.0, 2.5))  # 랜덤 간격으로 탐지 회피

    return results

# crontab -e 로 편집
*/30 * * * * /usr/bin/python3 /home/user/dcinside_monitor.py >> /var/log/dcinside.log 2>&1

import json
import os

SEEN_FILE = "seen_posts.json"

def load_seen():
    if os.path.exists(SEEN_FILE):
        with open(SEEN_FILE) as f:
            return set(json.load(f))
    return set()

def save_seen(seen_ids):
    with open(SEEN_FILE, "w") as f:
        json.dump(list(seen_ids), f)

def filter_new_posts(posts, seen_ids):
    new_posts = [p for p in posts if p["id"] not in seen_ids]
    seen_ids.update(p["id"] for p in new_posts)
    return new_posts, seen_ids

import requests as req

def send_slack_alert(post, webhook_url):
    message = f"새 게시글 감지\n제목: {post['title']}\nURL: {post['url']}"
    req.post(webhook_url, json={"text": message})

from transformers import pipeline

classifier = pipeline(
    "text-classification",
    model="beomi/KcELECTRA-base-v2022"  # 한국어 범용 감정분석 모델 (용도에 맞게 교체 가능)
)

def analyze_sentiment(texts):
    results = []
    for text in texts:
        result = classifier(text[:512])[0]
        results.append({
            "text": text[:50],
            "label": result["label"],
            "score": round(result["score"], 3)
        })
    return results

from collections import Counter
import re

def extract_top_keywords(texts, top_n=20):
    all_words = []
    for text in texts:
        words = re.findall(r"[가-힣]{2,}", text)  # 2글자 이상 한글만
        all_words.extend(words)
    return Counter(all_words).most_common(top_n)

import random
import time

def random_delay(min_sec=1.0, max_sec=3.0):
    time.sleep(random.uniform(min_sec, max_sec))

import random

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/119",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/118",
]

def get_random_ua():
    return random.choice(USER_AGENTS)

import itertools

proxy_pool = [
    "http://proxy1:port",
    "http://proxy2:port",
    "http://proxy3:port",
]
proxy_cycle = itertools.cycle(proxy_pool)

def get_proxy():
    p = next(proxy_cycle)
    return {"http": p, "https": p}

from curl_cffi import requests as cffi_requests

resp = cffi_requests.get(
    "https://gall.dcinside.com/board/lists/",
    params={"id": "programming"},
    impersonate="chrome120"
)

https://www.dcinside.com/robots.txt