#!/usr/bin/env python3
"""
获取Top50股票的行业和概念信息 (修复版)
使用个股详情API获取行业，使用概念板块API获取概念
"""
import json
import os
import requests
import time
from collections import Counter, defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed

WORK_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
RESULT_FILE = os.path.join(WORK_DIR, "scan_results_filtered.json")
OUTPUT_FILE = os.path.join(WORK_DIR, "stock_industry_concepts_new.json")

session = requests.Session()
session.verify = False
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
    'Referer': 'https://quote.eastmoney.com/'
})
import warnings
warnings.filterwarnings('ignore')
import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

def get_stock_industry(code, max_retries=3):
    """通过个股详情API获取行业信息"""
    if code.startswith(('00', '30')):
        secid = f"0.{code}"
    else:
        secid = f"1.{code}"

    url = 'http://push2.eastmoney.com/api/qt/stock/get'
    params = {
        'secid': secid,
        'fields': 'f127,f128,f135,f136,f137',
    }
    for attempt in range(max_retries):
        try:
            resp = session.get(url, params=params, timeout=8)
            data = resp.json().get('data', {})
            industry = data.get('f127', '') or ''
            return industry if industry else '未知'
        except:
            if attempt < max_retries - 1:
                time.sleep(0.3)
            return '未知'


def get_all_concept_boards():
    """获取所有概念板块列表"""
    all_boards = []
    for page in range(1, 10):
        url = 'http://push2.eastmoney.com/api/qt/clist/get'
        params = {
            'pn': page,
            'pz': 100,
            'po': 1,
            'np': 1,
            'fltt': 2,
            'invt': 2,
            'fs': 'm:90+t:3',
            'fields': 'f12,f14',
        }
        try:
            resp = session.get(url, params=params, timeout=10)
            data = resp.json()
            boards = data.get('data', {}).get('diff', [])
            if not boards:
                break
            for b in boards:
                bc = b.get('f12', '')
                bn = b.get('f14', '')
                if bc and bn:
                    all_boards.append({'code': bc, 'name': bn})
        except:
            break
    print(f"获取到 {len(all_boards)} 个概念板块")
    return all_boards


def get_board_members(board_code, max_retries=2):
    """获取板块成分股"""
    url = 'http://push2.eastmoney.com/api/qt/clist/get'
    params = {
        'pn': 1,
        'pz': 500,
        'po': 1,
        'np': 1,
        'fltt': 2,
        'invt': 2,
        'fs': f'b:{board_code}',
        'fields': 'f12,f14',
    }
    for attempt in range(max_retries):
        try:
            resp = session.get(url, params=params, timeout=8)
            data = resp.json()
            stocks = data.get('data', {}).get('diff', [])
            return set(str(s.get('f12', '')).zfill(6) for s in stocks if s.get('f12'))
        except:
            if attempt < max_retries - 1:
                time.sleep(0.2)
            return set()


def main():
    with open(RESULT_FILE, 'r') as f:
        scan_data = json.load(f)

    top50 = scan_data['top50']
    top50_codes = set(s['code'] for s in top50)
    print(f"需要获取 {len(top50_codes)} 只股票的行业和概念信息")

    # 1. Get industries (parallel)
    print("正在获取行业信息...")
    industry_map = {}
    with ThreadPoolExecutor(max_workers=10) as executor:
        futures = {executor.submit(get_stock_industry, s['code']): s['code'] for s in top50}
        for future in as_completed(futures):
            code = futures[future]
            industry_map[code] = future.result()

    found = sum(1 for v in industry_map.values() if v != '未知')
    print(f"行业信息: {found}/{len(top50_codes)} 已获取")

    # 2. Get concepts
    print("正在获取概念板块信息...")
    concept_boards = get_all_concept_boards()
    stock_concepts = defaultdict(list)

    total = len(concept_boards)
    for i, board in enumerate(concept_boards):
        members = get_board_members(board['code'])
        matched = top50_codes & members
        for code in matched:
            stock_concepts[code].append(board['name'])
        if (i + 1) % 50 == 0:
            print(f"  概念进度: {i+1}/{total}")

    # 3. Build result
    result = {}
    for stock in top50:
        code = stock['code']
        result[code] = {
            'industry': industry_map.get(code, '未知'),
            'concepts': stock_concepts.get(code, []),
        }

    output = {
        'scan_date': scan_data['scan_date'],
        'stock_info': result,
    }

    with open(OUTPUT_FILE, 'w', encoding='utf-8') as f:
        json.dump(output, f, ensure_ascii=False, indent=2)

    print(f"\n数据已保存到 {OUTPUT_FILE}")

    # Summary
    industries = Counter(v['industry'] for v in result.values())
    print(f"\n行业分布 (Top50):")
    for ind, count in industries.most_common(15):
        print(f"  {ind}: {count}")

    concept_counter = Counter()
    for v in result.values():
        for c in v['concepts']:
            concept_counter[c] += 1
    print(f"\n概念分布 (Top50, 前20):")
    for con, count in concept_counter.most_common(20):
        print(f"  {con}: {count}")


if __name__ == '__main__':
    main()