#!/usr/bin/env python3
"""
获取Top50股票的K线数据和实时行情数据
使用腾讯API获取K线，腾讯/新浪API获取实时行情
同时获取行业信息（通过多种API尝试）
"""
import json
import os
import time
import re
import requests
from datetime import datetime, timedelta
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import defaultdict
import warnings
warnings.filterwarnings('ignore')
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

WORK_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
RESULT_FILE = os.path.join(WORK_DIR, "scan_results_filtered.json")
OUTPUT_FILE = os.path.join(WORK_DIR, "kline_data.json")
INDUSTRY_OUTPUT = os.path.join(WORK_DIR, "stock_industry_concepts.json")

session = requests.Session()
session.verify = False
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
})


def get_stock_secid(code):
    """获取股票的腾讯/东财secid格式"""
    if code.startswith(('00', '30')):
        return f"sz{code}", f"0.{code}"
    elif code.startswith(('60', '68', '11', '13')):
        return f"sh{code}", f"1.{code}"
    else:
        return f"sz{code}", f"0.{code}"


def get_kline_data(code, count=60):
    """通过腾讯API获取日K线数据"""
    tx_code, _ = get_stock_secid(code)
    start_date = (datetime.now() - timedelta(days=120)).strftime('%Y-%m-%d')
    end_date = datetime.now().strftime('%Y-%m-%d')

    url = f'https://web.ifzq.gtimg.cn/appstock/app/fqkline/get?param={tx_code},day,{start_date},{end_date},{count},qfq'
    for attempt in range(3):
        try:
            resp = session.get(url, timeout=10)
            data = resp.json()
            stock_data = data.get('data', {}).get(tx_code, {})
            klines = stock_data.get('qfqday', stock_data.get('day', []))

            if not klines:
                if attempt < 2:
                    time.sleep(0.3)
                    continue
                return []

            result = []
            for k in klines:
                # 腾讯格式: [date, open, close, high, low, volume]
                if len(k) >= 6:
                    result.append({
                        'date': k[0],
                        'open': float(k[1]),
                        'close': float(k[2]),
                        'high': float(k[3]),
                        'low': float(k[4]),
                        'volume': float(k[5]),
                    })

            # 计算MA线
            closes = [r['close'] for r in result]
            for i, r in enumerate(result):
                r['ma5'] = round(sum(closes[max(0,i-4):i+1]) / min(i+1, 5), 2) if i >= 0 else None
                r['ma10'] = round(sum(closes[max(0,i-9):i+1]) / min(i+1, 10), 2) if i >= 4 else None
                r['ma20'] = round(sum(closes[max(0,i-19):i+1]) / min(i+1, 20), 2) if i >= 9 else None

            return result[-60:]
        except:
            if attempt < 2:
                time.sleep(0.3)
                continue
            return []


def get_realtime_quote(code):
    """通过腾讯API获取实时行情"""
    tx_code, _ = get_stock_secid(code)
    url = f'https://qt.gtimg.cn/q={tx_code}'
    for attempt in range(3):
        try:
            resp = session.get(url, timeout=8)
            text = resp.text
            # 解析腾讯行情格式: v_sz000657="51~中钨高新~000657~..."
            match = re.search(r'="([^"]+)"', text)
            if not match:
                if attempt < 2:
                    time.sleep(0.2)
                    continue
                return {}

            fields = match.group(1).split('~')
            if len(fields) < 50:
                return {}

            quote = {
                'name': fields[1],
                'code': fields[2],
                'price': float(fields[3]) if fields[3] else 0,
                'prev_close': float(fields[4]) if fields[4] else 0,
                'open': float(fields[5]) if fields[5] else 0,
                'volume': float(fields[6]) if fields[6] else 0,  # 手
                'change': float(fields[31]) if fields[31] else 0,
                'change_pct': float(fields[32]) if fields[32] else 0,
                'high': float(fields[33]) if fields[33] else 0,
                'low': float(fields[34]) if fields[34] else 0,
                'amount': float(fields[37]) if fields[37] else 0,  # 万元
                'turnover_rate': float(fields[38]) if fields[38] else 0,
                'pe_ratio': float(fields[39]) if fields[39] else 0,
                'amplitude': float(fields[43]) if len(fields) > 43 and fields[43] else 0,
                'total_mktcap': float(fields[45]) if len(fields) > 45 and fields[45] else 0,  # 亿
                'circulating_mktcap': float(fields[44]) if len(fields) > 44 and fields[44] else 0,  # 亿
                'pb_ratio': float(fields[46]) if len(fields) > 46 and fields[46] else 0,
            }

            # 五档买卖盘
            bid_ask = {}
            try:
                # 买1-5: fields[9-18] (price, volume交替)
                for i in range(5):
                    bp_idx = 9 + i * 2
                    bv_idx = 10 + i * 2
                    sp_idx = 19 + i * 2
                    sv_idx = 20 + i * 2
                    if bp_idx + 1 < len(fields) and sp_idx + 1 < len(fields):
                        bid_ask[f'bid{i+1}_price'] = float(fields[bp_idx]) if fields[bp_idx] else 0
                        bid_ask[f'bid{i+1}_volume'] = int(fields[bv_idx]) if fields[bv_idx] else 0
                        bid_ask[f'ask{i+1}_price'] = float(fields[sp_idx]) if fields[sp_idx] else 0
                        bid_ask[f'ask{i+1}_volume'] = int(fields[sv_idx]) if fields[sv_idx] else 0
            except:
                pass
            quote.update(bid_ask)

            return quote
        except:
            if attempt < 2:
                time.sleep(0.2)
                continue
            return {}


def get_industry_sina(code):
    """通过新浪API获取行业信息"""
    tx_code, _ = get_stock_secid(code)
    # 新浪行业分类API
    url = f'https://vip.stock.finance.sina.com.cn/corp/go.php/vCI_StockHolder/stockid/{code}.phtml'
    try:
        resp = session.get(url, timeout=8, headers={'Referer': 'https://finance.sina.com.cn/'})
        resp.encoding = 'gb2312'
        # 尝试从页面中提取行业信息
        text = resp.text
        # 新浪个股页面中的行业信息
        ind_match = re.search(r'行业分类.*?<a[^>]*>([^<]+)</a>', text, re.S)
        if ind_match:
            return ind_match.group(1).strip()
    except:
        pass

    # 备用：通过股票代码推断行业板块
    industry_map = {
        '银行': ['601398', '601939', '601288', '601988', '601328', '600036', '601166', '600000'],
    }

    # 通过新浪另一个API获取
    try:
        url2 = f'https://hq.sinajs.cn/list={tx_code}'
        resp = session.get(url2, timeout=8, headers={
            'User-Agent': 'Mozilla/5.0',
            'Referer': 'https://finance.sina.com.cn/'
        })
        # 新浪行情不直接包含行业，但可以获取基本数据
    except:
        pass

    return ''


def get_industry_batch(codes):
    """批量获取行业信息 - 通过东财概念板块API的替代方案"""
    industry_map = {}

    # 方案1: 尝试东财API（可能恢复）
    try:
        for code in codes:
            _, em_secid = get_stock_secid(code)
            url = 'https://push2.eastmoney.com/api/qt/stock/get'
            params = {'secid': em_secid, 'fields': 'f57,f58,f127,f128'}
            resp = session.get(url, params=params, timeout=5)
            data = resp.json().get('data', {})
            ind = data.get('f127', '') or ''
            if ind:
                industry_map[code] = ind
    except:
        pass

    if len(industry_map) >= len(codes) * 0.8:
        return industry_map

    # 方案2: 使用新浪行业数据
    # 新浪行业分类数据
    sina_industry_url = 'https://vip.stock.finance.sina.com.cn/q/view/newSinaHy.php'
    try:
        resp = session.get(sina_industry_url, timeout=10, headers={'Referer': 'https://finance.sina.com.cn/'})
        resp.encoding = 'gb2312'
        # 解析新浪行业分类
        text = resp.text
        # 格式: {行业代码: "行业名称,股票代码1,股票代码2,..."}
        matches = re.findall(r'"(\d+)":"([^"]+)"', text)
        for ind_code, ind_data in matches:
            parts = ind_data.split(',')
            if len(parts) > 1:
                ind_name = parts[0]
                for stock_code in parts[1:]:
                    stock_code = stock_code.strip().zfill(6)
                    if stock_code in codes and stock_code not in industry_map:
                        industry_map[stock_code] = ind_name
    except:
        pass

    return industry_map


def get_concepts_batch(codes):
    """批量获取概念板块信息"""
    stock_concepts = defaultdict(list)

    # 方案1: 尝试东财概念板块API
    try:
        all_boards = []
        for page in range(1, 10):
            url = 'https://push2.eastmoney.com/api/qt/clist/get'
            params = {
                'pn': page, 'pz': 100, 'po': 1, 'np': 1,
                'fltt': 2, 'invt': 2, 'fs': 'm:90+t:3',
                'fields': 'f12,f14',
            }
            resp = session.get(url, params=params, timeout=8)
            data = resp.json()
            boards = data.get('data', {}).get('diff', [])
            if not boards:
                break
            for b in boards:
                bc, bn = b.get('f12', ''), b.get('f14', '')
                if bc and bn:
                    all_boards.append({'code': bc, 'name': bn})

        if all_boards:
            codes_set = set(codes)
            for board in all_boards:
                url = 'https://push2.eastmoney.com/api/qt/clist/get'
                params = {
                    'pn': 1, 'pz': 500, 'po': 1, 'np': 1,
                    'fltt': 2, 'invt': 2, 'fs': f'b:{board["code"]}',
                    'fields': 'f12,f14',
                }
                resp = session.get(url, params=params, timeout=8)
                stocks = resp.json().get('data', {}).get('diff', [])
                members = set(str(s.get('f12', '')).zfill(6) for s in stocks if s.get('f12'))
                matched = codes_set & members
                for code in matched:
                    stock_concepts[code].append(board['name'])
    except:
        pass

    return stock_concepts


def fetch_stock_data(stock):
    """获取单只股票的K线和实时行情"""
    code = stock['code']
    name = stock['name']

    kline = get_kline_data(code)
    quote = get_realtime_quote(code)

    return code, {
        'code': code,
        'name': name,
        'kline': kline,
        'quote': quote,
    }


def main():
    with open(RESULT_FILE, 'r') as f:
        data = json.load(f)

    top50 = data['top50']
    top50_codes = [s['code'] for s in top50]

    # ===== 1. 获取K线和实时行情 =====
    print(f"开始获取 {len(top50)} 只股票的K线和实时行情数据...")
    result = {}
    completed = 0

    with ThreadPoolExecutor(max_workers=8) as executor:
        futures = {executor.submit(fetch_stock_data, stock): stock['code'] for stock in top50}

        for future in as_completed(futures):
            code = futures[future]
            completed += 1
            try:
                ret_code, ret_data = future.result()
                result[ret_code] = ret_data
                kline_count = len(ret_data.get('kline', []))
                has_quote = bool(ret_data.get('quote', {}).get('price', 0) > 0)
                print(f"  [{completed}/{len(top50)}] {ret_data['name']}({ret_code}): K线{kline_count}条, 实时{'✓' if has_quote else '✗'}")
            except Exception as e:
                print(f"  [{completed}/{len(top50)}] {code}: 获取失败 - {e}")

    # 保存K线数据
    output = {
        'fetch_date': datetime.now().strftime('%Y-%m-%d %H:%M'),
        'stock_data': result,
    }
    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(output, f, ensure_ascii=False, indent=2)

    has_kline = sum(1 for v in result.values() if v.get('kline'))
    has_quote = sum(1 for v in result.values() if v.get('quote', {}).get('price', 0) > 0)
    print(f"\nK线数据: {has_kline}/{len(top50)} | 实时行情: {has_quote}/{len(top50)}")

    # ===== 2. 获取行业和概念信息 =====
    print(f"\n开始获取行业和概念信息...")

    # 行业
    industry_map = get_industry_batch(top50_codes)
    found_ind = sum(1 for c in top50_codes if c in industry_map)
    print(f"行业信息: {found_ind}/{len(top50_codes)} 已获取")

    # 概念
    stock_concepts = get_concepts_batch(top50_codes)
    found_con = sum(1 for c in top50_codes if c in stock_concepts and stock_concepts[c])
    print(f"概念信息: {found_con}/{len(top50_codes)} 已获取")

    # 保存行业概念数据
    industry_result = {}
    for stock in top50:
        code = stock['code']
        industry_result[code] = {
            'industry': industry_map.get(code, '未知'),
            'concepts': stock_concepts.get(code, []),
        }

    industry_output = {
        'scan_date': data.get('scan_date', ''),
        'stock_info': industry_result,
    }
    with open(INDUSTRY_OUTPUT, 'w', encoding='utf-8') as f:
        json.dump(industry_output, f, ensure_ascii=False, indent=2)

    print(f"\n数据已保存:")
    print(f"  K线数据: {OUTPUT_FILE}")
    print(f"  行业概念: {INDUSTRY_OUTPUT}")


if __name__ == '__main__':
    main()