SNS 데이터, AI로 긁는 법 — 완전 재현 가이드

# Apify: apify.com 가입 → Settings → API tokens
export APIFY_API_TOKEN="apify_api_xxx"        # 본인 토큰
# BrightData: brightdata.com → Account settings → API tokens
export BRIGHTDATA_API_TOKEN="xxxx-xxxx"        # 본인 토큰
# jq 설치 필요 (brew install jq / apt install jq)

# 실행 전 가격 모델 확인. PAY_PER_EVENT 아니면 쓰지 말 것(월구독형 과금 폭탄 방지)
curl -sS "https://api.apify.com/v2/acts/futurizerush~meta-threads-scraper?token=$APIFY_API_TOKEN" \
  | jq '.data.currentPricingInfo.pricingModel'   # => "PAY_PER_EVENT" 여야 안전

cat > input.json <<'JSON'
{ "mode": "search", "search_keywords": ["1인기업","AI 자동화","n8n","솔로프리너"],
  "search_filter": "top", "results_limit": 40 }
JSON

# 동기 실행 → 결과 dataset 다운로드 (waitForFinish=300초)
RESP=$(curl -sS -X POST \
  "https://api.apify.com/v2/acts/futurizerush~meta-threads-scraper/runs?token=$APIFY_API_TOKEN&waitForFinish=300" \
  -H "Content-Type: application/json" --data-binary @input.json)
DATASET=$(echo "$RESP" | jq -r '.data.defaultDatasetId')
echo "cost: $(echo "$RESP" | jq -r '.data.usageTotalUsd')"   # 실비용 확인
curl -sS "https://api.apify.com/v2/datasets/$DATASET/items?token=$APIFY_API_TOKEN&clean=true&format=json" -o threads_posts.json
echo "saved $(jq length threads_posts.json) posts"

cat > replies.json <<'JSON'
{ "post_urls": [{"url":"https://www.threads.com/@HANDLE/post/CODE"}],
  "max_replies": 50, "include_nested_replies": true }
JSON
RESP=$(curl -sS -X POST \
  "https://api.apify.com/v2/acts/futurizerush~threads-replies-scraper/runs?token=$APIFY_API_TOKEN&waitForFinish=300" \
  -H "Content-Type: application/json" --data-binary @replies.json)
DS=$(echo "$RESP" | jq -r '.data.defaultDatasetId')
curl -sS "https://api.apify.com/v2/datasets/$DS/items?token=$APIFY_API_TOKEN&clean=true" -o replies_out.json
# 작성자 본인 댓글만 (체인 = 글의 가치 페이로드)
jq -r '.[] | select((.author_username//"")=="HANDLE") | .text_content' replies_out.json

GET https://api.brightdata.com/datasets/list   # 전체 목록(본인 계정)

DATASET_ID="gd_l1vikfch901nx3by4"   # IG 프로필
echo '[{"url":"https://www.instagram.com/TARGET_HANDLE"}]' > bd_input.json

# 1) trigger → snapshot_id
SNAP=$(curl -sS -X POST \
  "https://api.brightdata.com/datasets/v3/trigger?dataset_id=$DATASET_ID&include_errors=true" \
  -H "Authorization: Bearer $BRIGHTDATA_API_TOKEN" -H "Content-Type: application/json" \
  --data-binary @bd_input.json | jq -r '.snapshot_id')

# 2) 완료까지 폴링
while :; do
  ST=$(curl -sS -H "Authorization: Bearer $BRIGHTDATA_API_TOKEN" \
        "https://api.brightdata.com/datasets/v3/progress/$SNAP" | jq -r '.status')
  echo "status=$ST"; [ "$ST" = "ready" ] && break; [ "$ST" = "failed" ] && exit 1; sleep 15
done

# 3) 다운로드
curl -sS -H "Authorization: Bearer $BRIGHTDATA_API_TOKEN" \
  "https://api.brightdata.com/datasets/v3/snapshot/$SNAP?format=json" -o ig_out.json

import json, re
data = json.load(open("threads_posts.json", encoding="utf-8"))
def kr(s): return bool(re.search(r"[가-힣]", s or ""))
TOPIC = ("ai","에이전트","자동화","1인","사업","마케팅","퍼널","클로드","gpt","부업","프롬프트","창업")
rows, seen = [], set()
for p in data:
    t = p.get("text_content") or ""
    if not kr(t) or len(t) < 30: continue
    if not any(k in t.lower() for k in TOPIC): continue
    u = p.get("username"); f = p.get("followers_count") or 0
    L, R, RP = p.get("like_count") or 0, p.get("reply_count") or 0, p.get("repost_count") or 0
    if f < 300 or L < 20 or u in seen: continue          # 노이즈 컷
    er = (L + 2*R + 3*RP) / f                              # 반응률 = 좋아요+2*댓글+3*리포스트 / 팔로워
    rows.append((er, u, f, L, R, RP, t)); seen.add(u)
rows.sort(reverse=True)
for er,u,f,L,R,RP,t in rows[:10]:
    print(f"ER {er*100:.0f}%  @{u} (팔{f} ♥{L} 💬{R} 🔁{RP})\n{t[:200]}\n---")

3-2 키워드 검색 → 5 ER 랭킹 → 상위 글 선별
  → (체인 있으면) 3-3 댓글 수집
  → 좋은 글의 hook·구조 역설계 → 내 콘텐츠로 재작성