"""Bounded, topic-based research on automotive sites. No paid search API."""
from concurrent.futures import ThreadPoolExecutor
from datetime import date
import json
import math
import re
from urllib.parse import urlencode, urlparse, urlunparse
from urllib.robotparser import RobotFileParser

from bs4 import BeautifulSoup
import ingestion

SITES = ('www.carwale.com', 'www.bikewale.com', 'www.rushlane.com', 'www.indiacarnews.com')
HOSTS = set(SITES) | {'rushlane.com', 'indiacarnews.com'}
STOP_WORDS = set('the a an and or for of in on to with best top how what which guide blog article write about india under'.split())


def terms(topic):
    return set(re.findall(r'\w{2,}', topic.casefold())) - STOP_WORDS


def relevant(topic, text):
    wanted = terms(topic)
    matched = wanted & set(re.findall(r'\w{2,}', text.casefold()))
    return bool(wanted) and len(matched) >= max(min(4, len(wanted)), math.ceil(len(wanted) * 0.6))


def safe_article_url(url):
    ingestion.validate_url(url, HOSTS)
    p = urlparse(url)
    return urlunparse((p.scheme, p.netloc, p.path, '', p.query, ''))


def discover(topic, host):
    # Search only known source domains. Search result snippets are never used as article evidence.
    url = 'https://www.bing.com/search?' + urlencode({'q': f'{topic} site:{host}', 'format': 'rss'})
    found = []
    searches = [(url, {'www.bing.com'})]
    if host in ('www.rushlane.com', 'www.indiacarnews.com'):
        query = ' '.join(word for word in re.findall(r'\w{2,}', topic) if word.casefold() not in STOP_WORDS)
        searches.insert(0, ('https://' + host + '/?' + urlencode({'s': query, 'feed': 'rss2'}), HOSTS))
        searches.insert(1, ('https://' + host + '/?' + urlencode({'s': query}), HOSTS))
    for search_url, allowed in searches:
        try:
            data, _ = ingestion.fetch(search_url, allowed, 2_000_000)
        except Exception:
            continue
        soup = BeautifulSoup(data, 'html.parser' if b'<html' in data[:2000].lower() else 'xml')
        candidates = [(item.find('title'), item.find('link')) for item in soup.find_all('item')[:30]]
        candidates += [(a, a) for a in soup.select('a[href]')]
        for title, link in candidates:
            if not title or not link or not relevant(topic, title.get_text(' ', strip=True)):
                continue
            try:
                candidate = safe_article_url(link.get('href') or link.get_text(strip=True))
            except ValueError:
                continue
            if urlparse(candidate).hostname.removeprefix('www.') == host.removeprefix('www.'):
                found.append(candidate)
        if len(found) >= 2:
            break
    return list(dict.fromkeys(found))[:3]


def extract(data, topic):
    soup = BeautifulSoup(data, 'html.parser')
    title_node = soup.find('h1') or soup.find('title')
    title = title_node.get_text(' ', strip=True) if title_node else ''
    date_node = soup.select_one('meta[property="article:published_time"], meta[itemprop="datePublished"], time[datetime]')
    published = (date_node.get('content') or date_node.get('datetime') or '') if date_node else ''
    body = ''
    def objects(value):
        if isinstance(value, list):
            for entry in value:
                yield from objects(entry)
        elif isinstance(value, dict):
            yield value
            yield from objects(value.get('@graph', []))
    for script in soup.select('script[type="application/ld+json"]'):
        try:
            for item in objects(json.loads(script.string or script.get_text())):
                if item.get('articleBody'):
                    body = BeautifulSoup(item['articleBody'], 'html.parser').get_text(' ', strip=True)
                    published = item.get('datePublished', '')
                    break
        except (ValueError, TypeError):
            continue
    if not body:
        for node in soup.select('script,style,nav,footer,header,aside,form'):
            node.decompose()
        article = soup.select_one('article, [itemprop="articleBody"], .td-post-content, .entry-content')
        if article:
            body = '\n'.join(p.get_text(' ', strip=True) for p in article.select('p') if len(p.get_text(strip=True)) > 50)
    if len(body.split()) < 80 or not relevant(topic, title + ' ' + body[:1500]):
        raise ValueError('Page did not contain enough relevant article text.')
    return {'title': title[:250], 'published': str(published)[:40], 'text': body[:3500]}


def research(topic):
    evidence = []
    issues = []
    def from_site(host):
        articles = []
        try:
            robots_data, _ = ingestion.fetch('https://' + host + '/robots.txt', HOSTS, 500_000)
            robots = RobotFileParser()
            robots.parse(robots_data.decode('utf-8', errors='replace').splitlines())
            for url in discover(topic, host):
                if not robots.can_fetch(ingestion.scraper.HEADERS['User-Agent'], url):
                    continue
                try:
                    data, _ = ingestion.fetch(url, HOSTS, 3_000_000)
                    articles.append({**extract(data, topic), 'url': url, 'accessed': str(date.today())})
                except Exception:
                    continue
                if len(articles) == 2:
                    break
        except Exception:
            return [], host + ': unavailable or search blocked'
        return articles, None if articles else host + ': no matching accessible articles'
    with ThreadPoolExecutor(max_workers=4) as pool:
        for articles, issue in pool.map(from_site, SITES):
            evidence.extend(articles)
            if issue:
                issues.append(issue)
    domains = {urlparse(item['url']).hostname.removeprefix('www.') for item in evidence}
    if len(domains) < 2:
        raise ValueError('Could not find matching readable articles on at least two websites. Try a more specific English topic. ' + '; '.join(issues))
    # Fit a small local context while preserving breadth before adding second articles.
    primary, secondary, seen = [], [], set()
    for item in evidence:
        domain = urlparse(item['url']).hostname.removeprefix('www.')
        (secondary if domain in seen else primary).append(item)
        seen.add(domain)
    return (primary + secondary)[:4]