#!/usr/bin/env python3
"""
大V动态监测 — 抓取微博/雪球大V最新动态并进行情绪分析
监测对象: 微博52198 / 唐史主任司马迁 / 狂龙十八段 / 雪球·质变之路
数据输出: data/weibo_dav_data.json

配置方式：
  方式1：将微博Cookie保存到 data/weibo_cookie.txt（从浏览器开发者工具复制）
        支持 weibo.com(PC版) 和 m.weibo.cn(移动版) 两种Cookie
        雪球无需Cookie，自动获取匿名token（若被WAF拦截可保存 data/xueqiu_cookie.txt）
  方式2：设置环境变量 WEIBO_COOKIE
  方式3：直接在下方 DAV_LIST 中填入已知的 uid（跳过搜索步骤）
  无Cookie时模块显示配置指引，配置后即可自动抓取。
"""
import json, os, time, re, datetime, sys
try:
    import requests
    HAS_REQUESTS = True
except ImportError:
    HAS_REQUESTS = False

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
WORK_DIR = os.path.join(BASE_DIR, "data")
OUTPUT_FILE = os.path.join(WORK_DIR, "weibo_dav_data.json")
COOKIE_FILE = os.path.join(WORK_DIR, "weibo_cookie.txt")
XQ_COOKIE_FILE = os.path.join(WORK_DIR, "xueqiu_cookie.txt")
WEREAD_COOKIE_FILE = os.path.join(WORK_DIR, "weread_cookie.txt")

# 监测的大V列表 (name=显示名, search_key=搜索关键词, uid=用户ID, source=数据源)
# 已知UID可直接填入 uid 字段，跳过搜索
# 微信公众号: uid 字段填微信读书 bookId (格式 MP_WXS_xxx), 留空则用搜狗搜索兜底
DAV_LIST = [
    {"name": "微博52198", "search_key": "wu2198", "uid": "1216826604", "source": "weibo"},
    {"name": "唐史主任司马迁", "search_key": "唐史主任司马迁", "uid": "2014433131", "source": "weibo"},
    {"name": "狂龙十八段", "search_key": "狂龙十八段", "uid": "5828706619", "source": "weibo"},
    {"name": "质变之路", "search_key": "质变之路", "uid": "7625353561", "source": "xueqiu"},
    {"name": "爱在冰川", "search_key": "爱在冰川", "uid": "MP_WXS_MzU2NzEwMDc1MA==", "source": "wechat"},
    {"name": "景峰看盘", "search_key": "景峰看盘", "uid": "MP_WXS_MzIxNjA4NjAyMA==", "source": "wechat"},
]

# 情绪关键词词典
BULLISH_KW = [
    '看涨', '牛市', '加仓', '抄底', '满仓', '暴涨', '大涨', '机会来了', '利好',
    '突破', '反转', '底部', '主升', '龙头', '打板', '连板', '涨停', '买入',
    '多头', '强势', '企稳', '反弹', '回升', '拉升', '增量资金', '入场', '建仓',
    '低吸', '埋伏', '看好', '向上', '做多', '风口', '启动', '新高', '放量',
]
BEARISH_KW = [
    '看跌', '熊市', '减仓', '清仓', '空仓', '暴跌', '大跌', '风险', '利空',
    '破位', '跳水', '割肉', '闪崩', '跌停', '卖出', '空头', '弱势', '回落',
    '缩量', '离场', '止损', '规避', '看空', '向下', '压力位', '套牢', '恐慌',
    '做空', '破发', '新低', '杀跌', '踩踏', '逼空',
]
# 否定前缀 (遇到这些前缀时翻转关键词情绪)
NEGATE_PREFIX = ['不', '别', '勿', '莫', '没', '未', '非']

# 市场相关关键词（用于过滤与市场/股市无关的微博，如生活帖、社会新闻帖）
MARKET_KW = [
    # 指数与大盘
    '大盘', '指数', '沪指', '深成', '创业板', '科创板', '北证', 'A股', 'a市',
    '港股', '恒生', '美股', '纳指', '道指', '标普', '日经', '富时', '美债',
    # 个股与板块
    '股票', '个股', '板块', '题材', '概念', '龙头', '涨停', '跌停', '连板', '打板',
    '炸板', '封板', '龙虎榜', '新股', '次新', '妖股', '牛股', '翻倍', '持仓',
    # 交易操作
    '仓位', '买入', '卖出', '加仓', '减仓', '清仓', '满仓', '空仓', '半仓',
    '止损', '止盈', '割肉', '抄底', '低吸', '高抛', '建仓', '补仓',
    '短线', '中线', '长线', '波段', '网格', '定投', '交易', '胜率', '赔率',
    # 市场状态
    '行情', '牛市', '熊市', '反弹', '回调', '调整', '震荡', '单边', '放量', '缩量',
    '成交量', '成交额', '万亿', '两市', '资金', '主力', '游资', '北向资金', '外资',
    '融资', '杠杆', '流动性',
    # 机构产品
    '基金', 'ETF', '机构', '私募', '公募', '券商', '银行', '保险', '黄金', '原油',
    # 宏观
    '通胀', '通缩', '利率', '汇率', '降息', '加息', '央行', '美联储', '加息',
    'GDP', 'CPI', 'PMI', '证监会', '财政', '货币',
    # 炒股相关
    '炒股', '股市', '股民', '散户', '肉签', '中签', '打新', '复盘', '盘面',
    '开盘', '收盘', '午盘', '早盘', '尾盘', '竞价', '基本面', '技术面',
    'K线', '均线', '支撑', '压力', '趋势', '主升', '阴线', '阳线', '补跌', '补涨',
]

def market_relevance(text):
    """统计微博文本命中的市场关键词，返回命中词列表（无命中=与市场无关）"""
    if not text:
        return []
    return [kw for kw in MARKET_KW if kw in text]

# 移动版API headers (m.weibo.cn)
MOBILE_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) '
                  'AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1',
    'Accept': 'application/json, text/plain, */*',
    'Referer': 'https://m.weibo.cn/',
    'X-Requested-With': 'XMLHttpRequest',
}

# PC版API headers (weibo.com)
PC_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
    'Accept': 'application/json, text/plain, */*',
    'Referer': 'https://weibo.com/',
    'X-Requested-With': 'XMLHttpRequest',
}

def clean_html(text):
    """清理HTML标签和特殊字符"""
    text = re.sub(r'<[^>]+>', '', text)
    text = text.replace('&nbsp;', ' ').replace('&amp;', '&')
    text = text.replace('&lt;', '<').replace('&gt;', '>')
    text = text.replace('&#39;', "'").replace('&quot;', '"')
    return text.strip()

def analyze_sentiment(text):
    """基于关键词 + 否定前缀的情绪分析"""
    if not text:
        return {'score': 0, 'label': 'neutral', 'bull_kw': [], 'bear_kw': []}

    bull_hits, bear_hits = [], []
    for kw in BULLISH_KW:
        idx = text.find(kw)
        while idx >= 0:
            prefix = text[max(0, idx - 1)]
            if prefix in NEGATE_PREFIX:
                bear_hits.append(prefix + kw)
            else:
                bull_hits.append(kw)
            idx = text.find(kw, idx + 1)
    for kw in BEARISH_KW:
        idx = text.find(kw)
        while idx >= 0:
            prefix = text[max(0, idx - 1)]
            if prefix in NEGATE_PREFIX:
                bull_hits.append(prefix + kw)
            else:
                bear_hits.append(kw)
            idx = text.find(kw, idx + 1)

    bull_n = len(bull_hits)
    bear_n = len(bear_hits)
    total = bull_n + bear_n

    if total == 0:
        score, label = 0, 'neutral'
    elif bull_n > bear_n:
        score = min(1.0, bull_n / (total + 1))
        label = 'bullish'
    elif bear_n > bull_n:
        score = max(-1.0, -bear_n / (total + 1))
        label = 'bearish'
    else:
        score, label = 0, 'neutral'

    return {
        'score': round(score, 2),
        'label': label,
        'bull_kw': list(set(bull_hits))[:5],
        'bear_kw': list(set(bear_hits))[:5],
    }

# ==================== 微信公众号 (微信读书API方案) ====================

WEREAD_HEADERS = {
    'User-Agent': 'WeRead/5.3.4 (iPhone; iOS 14.1; Scale/2.00)',
    'Accept': 'application/json, text/plain, */*',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}

_weread_session = None
_weread_available = None

def load_weread_credentials():
    """从 weread_cookie.txt 加载微信读书 skey 和 vid
    文件格式: 每行一个 key=value, 或直接粘贴完整 Cookie (自动解析 skey 和 vid)
    """
    skey, vid = None, None
    if os.path.exists(WEREAD_COOKIE_FILE):
        try:
            content = open(WEREAD_COOKIE_FILE, encoding='utf-8').read().strip()
            for line in content.split('\n'):
                line = line.strip()
                if '=' in line:
                    k, v = line.split('=', 1)
                    k, v = k.strip(), v.strip().strip(';')
                    if k.lower() in ('skey', 'wr_skey'):
                        skey = v
                    elif k.lower() in ('vid', 'wr_vid'):
                        vid = v
            # 从 Cookie 字符串中解析
            if not skey:
                m = re.search(r'(?:^|;\s*)wr_skey=([^;]+)', content)
                if m:
                    skey = m.group(1)
            if not vid:
                m = re.search(r'(?:^|;\s*)wr_vid=([^;]+)', content)
                if m:
                    vid = m.group(1)
        except Exception as e:
            print('  微信读书凭证解析失败: {}'.format(e))
    return skey, vid

def weread_session():
    """获取微信读书会话（带认证头）"""
    global _weread_session, _weread_available
    if _weread_session is not None:
        return _weread_session if _weread_available else None
    skey, vid = load_weread_credentials()
    if not skey or not vid:
        _weread_available = False
        return None
    s = requests.Session()
    s.headers.update(WEREAD_HEADERS)
    s.headers.update({'skey': skey, 'vid': vid, 'Cookies': 'wr_logined=1'})
    _weread_session = s
    _weread_available = True
    return s

def weread_get_articles(book_id, count=5):
    """通过微信读书API获取公众号文章列表
    返回列表: [{title, url, ts, abstract}]
    """
    s = weread_session()
    if not s:
        return []
    try:
        synckey = str(int(time.time()))
        url = 'https://i.weread.qq.com/book/articles?bookId={}&count={}&offset=0&synckey={}'.format(
            book_id, count * 2, synckey)
        r = s.get(url, timeout=15)
        data = r.json()
        if 'reviews' not in data:
            print('  微信读书API返回异常: {}'.format(str(data)[:200]))
            return []
        articles = []
        for item in data.get('reviews', []):
            review = item.get('review', {})
            title = review.get('title', '')
            # 微信读书文章URL: 用 reviewId 构造或用 mp 链接
            article_url = review.get('mpUrl', '') or review.get('url', '')
            if not article_url and review.get('reviewId'):
                # 尝试用微信读书的阅读页
                article_url = 'https://weread.qq.com/web/reviewer/{}'.format(review.get('reviewId'))
            ts = review.get('createTime', 0) or review.get('time', 0)
            if ts and ts > 1e12:  # 毫秒转秒
                ts = ts / 1000
            abstract = review.get('abstract', '') or review.get('content', '')[:200]
            if title:
                articles.append({
                    'title': title,
                    'url': article_url,
                    'ts': int(ts) if ts else 0,
                    'abstract': abstract,
                })
            if len(articles) >= count:
                break
        return articles
    except Exception as e:
        print('  微信读书获取文章列表失败: {}'.format(e))
        return []

def fetch_wechat_article_direct(url):
    """直接抓取微信公众号文章正文（不经过搜狗）
    返回 {title, author, text, ts}
    """
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9',
        }
        r = requests.get(url, headers=headers, timeout=25)
        html = r.text
        m_title = re.search(r"var msg_title = '([^']*)'", html) or re.search(r'<h1[^>]*>([^<]+)</h1>', html)
        m_author = re.search(r'var nickname = htmlDecode\("([^"]+)"\)', html) or \
                   re.search(r'id="js_name"[^>]*>\s*(?:<[^>]*>)*\s*([^<\s][^<]*?)\s*</', html)
        m_time = re.search(r"var ct = '(\d+)'", html) or \
                 re.search(r"var createTime = '([^']+)'", html) or \
                 re.search(r'publish_time["\']?\s*[:=]\s*["\']?(\d{10})', html)
        content = re.search(r'<div class="rich_media_content[^"]*"[^>]*>(.*?)</div>\s*(?:<script|<div id="js_tags)', html, re.S)
        text = ''
        if content:
            text = re.sub(r'<[^>]+>', ' ', content.group(1))
            text = text.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&ldquo;', '"').replace('&rdquo;', '"')
            text = re.sub(r'\s+', ' ', text).strip()
        title = (m_title.group(1).strip() if m_title else '')
        author = (m_author.group(1).strip() if m_author else '')
        ts = None
        if m_time:
            v = m_time.group(1)
            if v.isdigit() and int(v) > 1e9:
                ts = int(v)
            elif not v.isdigit():
                for fmt in ('%Y-%m-%d %H:%M', '%Y-%m-%d'):
                    try:
                        ts = int(datetime.datetime.strptime(v[:16], fmt).timestamp())
                        break
                    except Exception:
                        pass
        return {'title': title, 'author': author, 'text': text, 'ts': ts}
    except Exception as e:
        print('  微信文章直接抓取失败: {}'.format(e))
        return None

def wechat_fetch_posts_via_weread(book_id, name, count=4):
    """微信公众号主流程（微信读书版）：获取文章列表 → 抓正文 → 过滤分析"""
    posts, filtered = [], 0
    if not book_id or not book_id.startswith('MP_WXS_'):
        print('  微信读书 bookId 无效，跳过')
        return posts, filtered
    articles = weread_get_articles(book_id, count)
    if not articles:
        print('  微信读书未获取到文章')
        return posts, filtered
    print('  微信读书获取到 {} 篇文章'.format(len(articles)))
    for art in articles:
        if len(posts) >= count:
            break
        # 先尝试直接抓 mp 正文
        detail = None
        if art['url'] and 'mp.weixin.qq.com' in art['url']:
            time.sleep(2)
            detail = fetch_wechat_article_direct(art['url'])
        # 如果正文抓不到，用摘要 + 标题
        if not detail or not detail.get('text'):
            detail = {
                'title': art['title'],
                'author': name,
                'text': art.get('abstract', ''),
                'ts': art.get('ts', 0),
            }
        full_text = detail['title'] + '：' + detail['text'][:1200]
        if len(full_text) < 30:
            continue
        mk_hits = market_relevance(full_text)
        if not mk_hits:
            filtered += 1
            continue
        sent = analyze_sentiment(full_text)
        ts = detail['ts'] or art.get('ts', 0)
        posts.append({
            'text': full_text[:1500],
            'time': datetime.datetime.fromtimestamp(ts).strftime('%m-%d %H:%M') if ts else '',
            'ts': ts,
            'reposts': 0,
            'comments': 0,
            'attitudes': 0,
            'sentiment': sent['label'],
            'sentiment_score': sent['score'],
            'bull_kw': sent['bull_kw'],
            'bear_kw': sent['bear_kw'],
            'market_kw': mk_hits[:4],
            'author': detail.get('author', name),
        })
        print('  抓到文章: {} [{}]'.format(detail['title'][:35], detail.get('author', name)))
    posts.sort(key=lambda p: p.get('ts', 0), reverse=True)
    return posts[:count], filtered

# ==================== 微信公众号 (经搜狗搜索中转 - 备用方案) ====================

SOGOU_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}

_sogou_session = None

def sogou_get_session():
    """获取搜狗会话（带Cookie，降低反爬拦截概率）"""
    global _sogou_session
    if _sogou_session is not None:
        return _sogou_session
    s = requests.Session()
    s.headers.update(SOGOU_HEADERS)
    try:
        s.get('https://weixin.sogou.com/weixin?type=2&query=stock', timeout=12)
    except Exception as e:
        print('  搜狗初始化会话失败: {}'.format(e))
    _sogou_session = s
    return s

def sogou_search_articles(query, need=4):
    """搜狗微信文章搜索：返回文章列表（含标题/链接/账号/时间戳）"""
    try:
        s = sogou_get_session()
        r = s.get('https://weixin.sogou.com/weixin',
                  params={'type': 2, 'query': query},
                  headers={'Referer': 'https://weixin.sogou.com/'},
                  timeout=15)
        html = r.text
        if 'seccode' in html.lower() or 'antispider' in html:
            print('  搜狗触发验证码，跳过本轮')
            return []
        # 按结果块拆分（每块含标题链接/账号/时间戳）
        blocks = re.split(r'<li id="sogou_vr', html)[1:]
        results = []
        for block in blocks:
            m_url = re.search(r'href="(/link\?url=[^"]+)"', block)
            m_title = re.search(r'<h3>\s*<a[^>]*>(.*?)</a>', block, re.S)
            m_account = re.search(r'account_name_\d+"[^>]*>([^<]+)<', block)
            m_time = re.search(r'timeConvert\("?(20\d{2}-\d{2}-\d{2}|\d{9,10})"?', block)
            if not (m_url and m_title):
                continue
            title = re.sub(r'<[^>]+>', '', m_title.group(1)).strip()
            account = m_account.group(1).strip() if m_account else ''
            ts = None
            if m_time:
                v = m_time.group(1)
                if v.isdigit():
                    ts = int(v)
                else:
                    try:
                        ts = datetime.datetime.strptime(v, '%Y-%m-%d').timestamp()
                    except Exception:
                        pass
            results.append({'title': title, 'link': m_url.group(1), 'account': account, 'ts': ts})
            if len(results) >= need * 3:  # 多取一些，后续按账号过滤
                break
        return results
    except Exception as e:
        print('  搜狗搜索[{}]失败: {}'.format(query, e))
        return []

def sogou_follow_link(link_path, referer_query):
    """跟随搜狗/link跳转获取微信文章真实URL（JS拼接重定向）"""
    try:
        s = sogou_get_session()
        url = 'https://weixin.sogou.com' + link_path.replace('&amp;', '&')
        r = s.get(url, headers={'Referer': 'https://weixin.sogou.com/weixin?type=2&query={}'.format(referer_query)},
                  timeout=15)
        parts = re.findall(r"url \+= '([^']*)';", r.text)
        if not parts:
            return ''
        return ''.join(parts).replace('@', '')
    except Exception:
        return ''

def fetch_wechat_article(url):
    """抓取微信文章页正文，返回 {title, author, text, ts}"""
    try:
        s = sogou_get_session()
        r = s.get(url, headers={'Referer': 'https://mp.weixin.qq.com/'}, timeout=25)
        html = r.text
        m_title = re.search(r"var msg_title = '([^']*)'", html) or re.search(r'<h1[^>]*>([^<]+)</h1>', html)
        m_author = re.search(r'var nickname = htmlDecode\("([^"]+)"\)', html) or \
                   re.search(r'id="js_name"[^>]*>\s*(?:<[^>]*>)*\s*([^<\s][^<]*?)\s*</', html)
        m_time = re.search(r"var createTime = '([^']+)'", html) or \
                 re.search(r'publish_time["\']?\s*[:=]\s*["\']?(\d{10})', html)
        # 正文（微信文章正文div）
        content = re.search(r'<div class="rich_media_content[^"]*"[^>]*>(.*?)</div>\s*(?:<script|<div id="js_tags)', html, re.S)
        text = ''
        if content:
            text = re.sub(r'<[^>]+>', ' ', content.group(1))
            text = text.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&ldquo;', '"').replace('&rdquo;', '"')
            text = re.sub(r'\s+', ' ', text).strip()
        title = (m_title.group(1).strip() if m_title else '')
        author = (m_author.group(1).strip() if m_author else '')
        ts = None
        if m_time:
            v = m_time.group(1)
            if v.isdigit() and int(v) > 1e9:
                ts = int(v)
            elif not v.isdigit():
                for fmt in ('%Y-%m-%d %H:%M', '%Y-%m-%d'):
                    try:
                        ts = datetime.datetime.strptime(v[:16], fmt).timestamp()
                        break
                    except Exception:
                        pass
        return {'title': title, 'author': author, 'text': text, 'ts': ts}
    except Exception as e:
        print('  微信文章抓取失败: {}'.format(e))
        return None

def wechat_fetch_posts(query, count=4):
    """微信公众号主流程：搜狗搜索 → 跳转 → 抓正文 → 过滤分析。返回 (posts, filtered)"""
    posts, filtered = [], 0
    articles = sogou_search_articles(query, need=count)
    if not articles:
        print('  搜狗无索引结果')
        return posts, filtered
    print('  搜狗索引到 {} 篇文章'.format(len(articles)))
    for art in articles:
        if len(posts) >= count:
            break
        # 账号匹配：搜狗返回的账号名或标题需含关键词（公众号常有关联号/迁移号，宽松匹配）
        combo = art['account'] + '|' + art['title']
        if query not in combo and query.replace('看盘', '') not in combo:
            continue
        time.sleep(3)  # 限速防搜狗封禁
        article_url = sogou_follow_link(art['link'], query)
        if not article_url:
            continue
        time.sleep(1.5)
        detail = fetch_wechat_article(article_url)
        if not detail or not detail['text']:
            continue
        # 标题+正文合并分析（正文取前1200字，微信文章较长）
        full_text = detail['title'] + '：' + detail['text'][:1200]
        mk_hits = market_relevance(full_text)
        if not mk_hits:
            filtered += 1
            continue
        sent = analyze_sentiment(full_text)
        ts = detail['ts'] or art['ts'] or 0
        posts.append({
            'text': full_text[:1500],
            'time': datetime.datetime.fromtimestamp(ts).strftime('%m-%d %H:%M') if ts else '',
            'ts': ts,
            'reposts': 0,
            'comments': 0,
            'attitudes': 0,
            'sentiment': sent['label'],
            'sentiment_score': sent['score'],
            'bull_kw': sent['bull_kw'],
            'bear_kw': sent['bear_kw'],
            'market_kw': mk_hits[:4],
            'author': detail['author'] or art['account'],
        })
        print('  抓到文章: {} [{}]'.format(detail['title'][:30], art['account']))
    posts.sort(key=lambda p: p.get('ts', 0), reverse=True)
    return posts[:count], filtered

# ==================== 雪球API (xueqiu.com) ====================

XQ_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
                  'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9',
}

_xq_session = None

def xq_get_session():
    """获取雪球会话（先访问行情页拿匿名token，Referer用/hq可绕过WAF拦截API）"""
    global _xq_session
    if _xq_session is not None:
        return _xq_session
    s = requests.Session()
    s.headers.update(XQ_HEADERS)
    # 用户提供的Cookie优先（登录态可看更多内容）
    if os.path.exists(XQ_COOKIE_FILE):
        with open(XQ_COOKIE_FILE, 'r') as f:
            cookie = f.read().strip()
        if cookie:
            s.headers['Cookie'] = cookie
    try:
        s.get('https://xueqiu.com/hq', timeout=12)  # 获取 xq_a_token 等匿名Cookie
    except Exception as e:
        print('  雪球初始化会话失败: {}'.format(e))
    _xq_session = s
    return s

def xq_fetch_posts(uid, count=10):
    """获取雪球用户时间线，过滤与市场无关的内容。返回 (posts, filtered)"""
    if not uid:
        return [], 0
    posts, filtered = [], 0
    try:
        s = xq_get_session()
        for page in range(1, 4):  # 最多翻3页凑够相关帖子
            if len(posts) >= count:
                break
            r = s.get('https://xueqiu.com/statuses/user_timeline.json',
                      params={'user_id': uid, 'page': page, 'count': 20, 'type': 0},
                      headers={'Referer': 'https://xueqiu.com/hq',   # 关键：绕过WAF
                               'Accept': 'application/json, text/plain, */*'},
                      timeout=12)
            if r.status_code != 200:
                print('  雪球API状态码: {}'.format(r.status_code))
                break
            data = r.json()
            statuses = data.get('statuses', [])
            if not statuses:
                break
            for item in statuses:
                title = clean_html(item.get('title') or '')
                body = clean_html(item.get('description') or item.get('text') or '')
                text = (title + '：' + body) if title else body
                if not text or len(text) < 4:
                    continue
                mk_hits = market_relevance(text)
                if not mk_hits:
                    filtered += 1
                    continue
                # 雪球时间戳为毫秒
                ts_ms = item.get('created_at', 0)
                ts = ts_ms / 1000 if ts_ms > 1e12 else ts_ms
                sent = analyze_sentiment(text)
                posts.append({
                    'text': text[:1500],
                    'time': datetime.datetime.fromtimestamp(ts).strftime('%m-%d %H:%M') if ts else '',
                    'ts': ts,
                    'reposts': item.get('retweet_count', 0),
                    'comments': item.get('reply_count', 0) or 0,
                    'attitudes': item.get('like_count', 0),
                    'sentiment': sent['label'],
                    'sentiment_score': sent['score'],
                    'bull_kw': sent['bull_kw'],
                    'bear_kw': sent['bear_kw'],
                    'market_kw': mk_hits[:4],
                })
            time.sleep(1.5)
        posts.sort(key=lambda p: p.get('ts', 0), reverse=True)  # 置顶帖沉底
        return posts[:count], filtered
    except Exception as e:
        print('  雪球获取用户[{}]帖子失败: {}'.format(uid, e))
    return posts, filtered

# ==================== PC版API (weibo.com) ====================

def pc_get_user_info(uid, headers):
    """通过PC版API获取用户信息"""
    try:
        url = 'https://weibo.com/ajax/profile/info?uid={}'.format(uid)
        r = requests.get(url, headers=headers, timeout=12)
        if r.status_code == 200:
            data = r.json()
            user = data.get('data', {}).get('user', {})
            if user:
                return str(user.get('id', uid)), user.get('screen_name', '')
    except Exception as e:
        print('  PC版获取用户信息失败: {}'.format(e))
    return str(uid), ''

def pc_search_user(name, headers):
    """通过PC版API搜索用户"""
    try:
        url = 'https://weibo.com/ajax/profile/searchProfile?q={}&page=1'.format(name)
        r = requests.get(url, headers=headers, timeout=12)
        if r.status_code == 200:
            data = r.json()
            users = data.get('data', {}).get('users', [])
            if users:
                user = users[0]
                return str(user.get('id', '')), user.get('screen_name', '')
    except Exception as e:
        print('  PC版搜索用户失败: {}'.format(e))
    return '', ''

def parse_weibo_time(t):
    """解析微博时间字符串为时间戳"""
    if not t:
        return 0
    try:
        dt = datetime.datetime.strptime(t, '%a %b %d %H:%M:%S %z %Y')
        return dt.timestamp()
    except Exception:
        try:
            dt = datetime.datetime.strptime(t, '%Y-%m-%d %H:%M')
            return dt.timestamp()
        except Exception:
            return 0

def pc_fetch_posts(uid, headers, count=10):
    """通过PC版API获取用户微博帖子 (mymblog接口)，过滤与市场无关的内容。
    返回 (posts, filtered_count)：相关微博列表 + 被过滤的无关微博数"""
    if not uid:
        return [], 0
    posts, filtered = [], 0
    try:
        for page in range(1, 4):  # 最多翻3页凑够相关微博
            if len(posts) >= count:
                break
            url = 'https://weibo.com/ajax/statuses/mymblog'
            params = {'uid': uid, 'page': page, 'feature': 0}
            r = requests.get(url, params=params, headers=headers, timeout=12)
            if r.status_code != 200:
                break
            data = r.json()
            statuses = data.get('data', {}).get('list', [])
            if not statuses:
                break
            for item in statuses:
                if not isinstance(item, dict):
                    continue
                raw_text = item.get('text_raw', '') or item.get('text', '')
                text = clean_html(raw_text)
                if not text or len(text) < 4:
                    continue
                repost = item.get('retweeted_status')
                if repost:
                    repost_text = clean_html(repost.get('text_raw', '') or repost.get('text', ''))
                    if repost_text:
                        text += ' [转发] ' + repost_text[:100]
                # 市场相关性过滤：无任何市场关键词的微博跳过（生活帖/社会新闻帖）
                mk_hits = market_relevance(text)
                if not mk_hits:
                    filtered += 1
                    continue
                created_at = item.get('created_at', '')
                reposts = item.get('reposts_count', 0)
                comments = item.get('comments_count', 0)
                attitudes = item.get('attitudes_count', 0)
                sent = analyze_sentiment(text)
                ts = parse_weibo_time(created_at)
                posts.append({
                    'text': text[:1500],
                    'time': datetime.datetime.fromtimestamp(ts).strftime('%m-%d %H:%M') if ts else created_at,
                    'ts': ts,
                    'reposts': reposts,
                    'comments': comments,
                    'attitudes': attitudes,
                    'sentiment': sent['label'],
                    'sentiment_score': sent['score'],
                    'bull_kw': sent['bull_kw'],
                    'bear_kw': sent['bear_kw'],
                    'market_kw': mk_hits[:4],
                })
            time.sleep(1.5)
        posts.sort(key=lambda p: p.get('ts', 0), reverse=True)
        return posts[:count], filtered
    except Exception as e:
        print('  PC版获取微博失败: {}'.format(e))
    return posts, filtered

# ==================== 移动版API (m.weibo.cn) ====================

def mobile_search_user(name, headers):
    """通过 m.weibo.cn 搜索用户，返回 uid"""
    try:
        url = 'https://m.weibo.cn/api/container/getIndex'
        params = {
            'containerid': '100103type=3&q={}'.format(name),
            'page_type': 'searchall',
        }
        r = requests.get(url, params=params, headers=headers, timeout=12)
        if r.status_code == 200:
            data = r.json()
            cards = data.get('data', {}).get('cards', [])
            for card in cards:
                if card.get('card_type') == 11:
                    for item in card.get('card_group', []):
                        user = item.get('user')
                        if user and user.get('id'):
                            uname = clean_html(user.get('screen_name', ''))
                            return str(user['id']), uname
                if card.get('card_type') == 10:
                    user = card.get('user')
                    if user and user.get('id'):
                        return str(user['id']), clean_html(user.get('screen_name', ''))
    except Exception as e:
        print('  移动版搜索用户[{}]失败: {}'.format(name, e))
    return '', ''

def mobile_fetch_posts(uid, headers, count=10):
    """获取用户最新微博帖子 (移动版)，过滤与市场无关的内容。返回 (posts, filtered)"""
    if not uid:
        return [], 0
    posts, filtered = [], 0
    try:
        url = 'https://m.weibo.cn/api/container/getIndex'
        params = {'containerid': '107603{}'.format(uid)}
        r = requests.get(url, params=params, headers=headers, timeout=12)
        if r.status_code == 200:
            data = r.json()
            cards = data.get('data', {}).get('cards', [])
            for card in cards:
                if card.get('card_type') != 9:
                    continue
                mblog = card.get('mblog', {})
                raw_text = mblog.get('text', '')
                text = clean_html(raw_text)
                if not text or len(text) < 4:
                    continue
                repost = mblog.get('retweeted_status')
                if repost:
                    repost_text = clean_html(repost.get('text', ''))
                    if repost_text:
                        text += ' [转发] ' + repost_text[:100]
                # 市场相关性过滤
                mk_hits = market_relevance(text)
                if not mk_hits:
                    filtered += 1
                    continue
                created_at = mblog.get('created_at', '')
                reposts = mblog.get('reposts_count', 0)
                comments = mblog.get('comments_count', 0)
                attitudes = mblog.get('attitudes_count', 0)
                sent = analyze_sentiment(text)
                posts.append({
                    'text': text[:1500],
                    'time': created_at,
                    'reposts': reposts,
                    'comments': comments,
                    'attitudes': attitudes,
                    'sentiment': sent['label'],
                    'sentiment_score': sent['score'],
                    'bull_kw': sent['bull_kw'],
                    'bear_kw': sent['bear_kw'],
                    'market_kw': mk_hits[:4],
                })
                if len(posts) >= count:
                    break
            return posts, filtered
    except Exception as e:
        print('  移动版获取用户[{}]微博失败: {}'.format(uid, e))
    return posts, filtered

# ==================== 通用接口 ====================

def load_cookie():
    """从文件或环境变量加载微博Cookie"""
    cookie = os.environ.get('WEIBO_COOKIE', '')
    if not cookie and os.path.exists(COOKIE_FILE):
        with open(COOKIE_FILE, 'r') as f:
            cookie = f.read().strip()
    return cookie

def is_pc_cookie(cookie):
    """判断Cookie是PC版还是移动版 (PC版含 SUBP 或 SSOLoginState 等)"""
    if not cookie:
        return False
    pc_indicators = ['SUBP', 'SCF', 'SSOLoginState', '_s_tentry', 'UOR']
    return any(k in cookie for k in pc_indicators)

def search_user(name, cookie):
    """搜索用户，自动选择PC/移动版API"""
    if cookie:
        if is_pc_cookie(cookie):
            headers = dict(PC_HEADERS)
            headers['Cookie'] = cookie
            uid, screen_name = pc_search_user(name, headers)
            if uid:
                return uid, screen_name
        # 回退到移动版
        headers = dict(MOBILE_HEADERS)
        headers['Cookie'] = cookie
        return mobile_search_user(name, headers)
    else:
        return mobile_search_user(name, MOBILE_HEADERS)

def fetch_posts(uid, cookie, count=10):
    """获取用户微博帖子（已过滤与市场无关内容），自动选择PC/移动版API。
    返回 (posts, filtered_count)"""
    if cookie:
        # 先尝试PC版
        if is_pc_cookie(cookie):
            headers = dict(PC_HEADERS)
            headers['Cookie'] = cookie
            posts, filtered = pc_fetch_posts(uid, headers, count)
            if posts:
                return posts, filtered
            print('  PC版无结果，尝试移动版...')
        # 回退到移动版
        headers = dict(MOBILE_HEADERS)
        headers['Cookie'] = cookie
        return mobile_fetch_posts(uid, headers, count)
    else:
        return mobile_fetch_posts(uid, MOBILE_HEADERS, count)

def get_user_info(uid, cookie):
    """获取用户信息（用于补充screen_name）"""
    if cookie and is_pc_cookie(cookie):
        headers = dict(PC_HEADERS)
        headers['Cookie'] = cookie
        return pc_get_user_info(uid, headers)
    return str(uid), ''

def main():
    if not HAS_REQUESTS:
        print('requests 库未安装，跳过大V动态抓取')
        _save_empty('requests库未安装')
        return

    os.makedirs(WORK_DIR, exist_ok=True)
    cookie = load_cookie()
    use_pc = is_pc_cookie(cookie) if cookie else False
    if cookie:
        print('=== 大V动态情绪监测 (已配置Cookie, {}版) ==='.format('PC' if use_pc else '移动'))
    else:
        print('=== 大V动态情绪监测 (无Cookie - 微博API可能受限) ===')
        print('提示: 将微博Cookie保存到 data/weibo_cookie.txt 可启用完整抓取')
    print('监测对象: {}'.format(', '.join(d['name'] for d in DAV_LIST)))

    results = []
    for dav in DAV_LIST:
        source = dav.get('source', 'weibo')
        print('\n处理: {} [{}] ...'.format(dav['name'], {'xueqiu': '雪球', 'wechat': '微信'}.get(source, '微博')))
        uid = dav.get('uid', '')
        screen_name = dav['name']

        if source == 'xueqiu':
            # 雪球：直接用UID抓时间线
            posts, filtered_n = xq_fetch_posts(uid, 10)
            print('  保留 {} 条市场相关帖子 (过滤 {} 条无关内容)'.format(len(posts), filtered_n))
            time.sleep(2)
        elif source == 'wechat':
            # 微信公众号：优先微信读书API，失败回退搜狗搜索
            book_id = uid if uid and uid.startswith('MP_WXS_') else ''
            posts, filtered_n = [], 0
            if book_id and weread_session():
                posts, filtered_n = wechat_fetch_posts_via_weread(book_id, dav['name'], 5)
            if not posts:
                # 回退到搜狗方案
                posts, filtered_n = wechat_fetch_posts(dav['search_key'], 3)
            print('  保留 {} 条市场相关文章 (过滤 {} 条无关内容)'.format(len(posts), filtered_n))
            time.sleep(3)
        else:
            # 微博：搜索或预设UID
            if not uid:
                uid, screen_name = search_user(dav['search_key'], cookie)
                if uid:
                    print('  找到用户: {} (uid={})'.format(screen_name, uid))
                else:
                    print('  未找到用户，跳过')
                time.sleep(3)
            else:
                print('  使用预设UID: {}'.format(uid))
                if use_pc:
                    _, sname = get_user_info(uid, cookie)
                    if sname:
                        screen_name = sname
                        print('  用户名: {}'.format(sname))

            posts, filtered_n = [], 0
            if uid:
                posts, filtered_n = fetch_posts(uid, cookie, 10)
                print('  保留 {} 条市场相关微博 (过滤 {} 条无关内容)'.format(len(posts), filtered_n))
                time.sleep(3)

        if posts:
            scores = [p['sentiment_score'] for p in posts]
            avg_score = sum(scores) / len(scores)
            bull_n = sum(1 for s in scores if s > 0.15)
            bear_n = sum(1 for s in scores if s < -0.15)
            neutral_n = len(scores) - bull_n - bear_n
            if avg_score > 0.15:
                overall = 'bullish'
            elif avg_score < -0.15:
                overall = 'bearish'
            else:
                overall = 'neutral'
        else:
            avg_score, bull_n, bear_n, neutral_n = 0, 0, 0, 0
            overall = 'neutral'

        results.append({
            'name': dav['name'],
            'screen_name': screen_name,
            'uid': uid,
            'source': source,
            'posts': posts[:10],
            'overall': overall,
            'score': round(avg_score, 2),
            'bull_count': bull_n,
            'bear_count': bear_n,
            'neutral_count': neutral_n,
            'post_count': len(posts),
            'filtered_count': filtered_n,
        })

    total_bull = sum(r['bull_count'] for r in results)
    total_bear = sum(r['bear_count'] for r in results)
    total_neutral = sum(r['neutral_count'] for r in results)
    valid = [r for r in results if r['post_count'] > 0]
    overall_score = sum(r['score'] for r in valid) / len(valid) if valid else 0

    if overall_score > 0.15:
        overall_sentiment = 'bullish'
    elif overall_score < -0.15:
        overall_sentiment = 'bearish'
    else:
        overall_sentiment = 'neutral'

    output = {
        'ok': len(valid) > 0,
        'fetch_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        'davs': results,
        'overall_sentiment': overall_sentiment,
        'overall_score': round(overall_score, 2),
        'total_bull': total_bull,
        'total_bear': total_bear,
        'total_neutral': total_neutral,
        'total_posts': sum(r['post_count'] for r in results),
        'total_filtered': sum(r.get('filtered_count', 0) for r in results),
    }

    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(output, f, ensure_ascii=False, indent=2)

    print('\n=== 大V情绪数据已保存: {} ==='.format(OUTPUT_FILE))
    print('总体情绪: {} (score={})'.format(overall_sentiment, overall_score))
    print('看多: {} | 看空: {} | 中性: {}'.format(total_bull, total_bear, total_neutral))

def _save_empty(reason):
    """保存空数据结构"""
    os.makedirs(WORK_DIR, exist_ok=True)
    output = {
        'ok': False,
        'fetch_time': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
        'error': reason,
        'davs': [{'name': d['name'], 'posts': [], 'overall': 'neutral', 'score': 0,
                   'bull_count': 0, 'bear_count': 0, 'neutral_count': 0, 'post_count': 0}
                  for d in DAV_LIST],
        'overall_sentiment': 'neutral',
        'overall_score': 0,
        'total_bull': 0, 'total_bear': 0, 'total_neutral': 0, 'total_posts': 0,
    }
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(output, f, ensure_ascii=False, indent=2)
    print('大V情绪数据(空)已保存: {}'.format(OUTPUT_FILE))

if __name__ == '__main__':
    main()