"""Validated source adapters. Public pages read SQLite and never fetch source pages."""
from contextlib import closing
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from urllib.parse import urlparse, urljoin
import hashlib
import html
import json
import re
import threading
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from bs4 import BeautifulSoup
import catalog
import db
import scraper

PROVIDERS = {'carwale': ('car', 'www.carwale.com'), 'bikewale': ('bike', 'www.bikewale.com'), 'cardekho': ('car', 'www.cardekho.com')}
BLOG_HOSTS = {'www.carwale.com', 'www.bikewale.com', 'www.cardekho.com',
              'www.rushlane.com', 'rushlane.com', 'www.indiacarnews.com', 'indiacarnews.com'}
ASSET_HOSTS = {'imgd.aeplcdn.com','stimg.cardekho.com','images.unsplash.com','bd.gaadicdn.com','stimg2.cardekho.com'}
MEDIA_DIR = Path(__file__).resolve().parent / 'static' / 'media'
_local = threading.local()

def session():
    if not hasattr(_local, 'http'):
        _local.http = requests.Session()
        _local.http.headers.update(scraper.HEADERS)
        _local.http.mount('https://', HTTPAdapter(max_retries=Retry(total=2, backoff_factor=1,
                           status_forcelist=[429,500,502,503,504], allowed_methods=['GET'])))
    return _local.http

def validate_url(url, hosts):
    p = urlparse(url)
    if p.scheme != 'https' or p.hostname not in hosts or p.username or p.password or p.port not in (None,443):
        raise ValueError('Use an HTTPS URL on an approved source domain.')
    return url

def validate_source(provider, kind, url):
    if provider not in PROVIDERS or PROVIDERS[provider][0] != kind:
        raise ValueError('Provider and vehicle type do not match.')
    validate_url(url, {PROVIDERS[provider][1]})
    path = urlparse(url).path
    if urlparse(url).query or urlparse(url).fragment:
        raise ValueError('Use a brand page URL without query parameters or fragments.')
    if provider in ('carwale','bikewale') and not re.fullmatch(r'/[a-z0-9-]+-(?:cars|bikes)/?', path):
        raise ValueError('Use a brand listing page for this provider.')
    if provider == 'cardekho' and not re.fullmatch(r'/cars/[A-Za-z0-9_-]+/?',path):
        raise ValueError('Use a CarDekho brand listing page.')

def validate_blog_source(provider, url):
    if provider != 'rss':
        raise ValueError('Blog sources must use an RSS or Atom feed.')
    parsed = urlparse(url)
    validate_url(url, BLOG_HOSTS)
    if parsed.fragment:
        raise ValueError('Use a feed URL without a fragment.')

def parse_blog_feed(data, source_url, tag='Imported'):
    """Parse RSS/Atom into plain-text stories without exposing source links publicly."""
    soup = BeautifulSoup(data, 'xml')
    entries = soup.find_all(['item', 'entry'])
    if not entries:
        raise ValueError('Feed contained no stories; existing posts were preserved.')
    parsed = []
    for entry in entries[:30]:
        title = entry.find('title')
        title = title.get_text(' ', strip=True) if title else ''
        link_node = entry.find('link')
        link = (link_node.get('href') if link_node and link_node.get('href') else link_node.get_text(' ', strip=True) if link_node else '')
        link = urljoin(source_url, link)
        if not title or not link_node:
            continue
        try:
            validate_url(link, BLOG_HOSTS)
        except ValueError:
            continue
        body_node = entry.find(['content:encoded', 'encoded', 'content', 'description', 'summary'])
        body_html = body_node.decode_contents() if body_node else ''
        content = scraper.html_to_text(html.unescape(body_html))
        if not content:
            continue
        image = ''
        enclosure = entry.find('enclosure')
        media = entry.find(['media:content', 'content'])
        candidate = (enclosure.get('url') if enclosure else '') or (media.get('url') if media else '')
        if candidate:
            try:
                image = cache_image(urljoin(source_url, candidate))
            except Exception:
                image = ''
        publisher = {'www.rushlane.com': 'RushLane', 'rushlane.com': 'RushLane',
                     'www.indiacarnews.com': 'IndiaCarNews', 'indiacarnews.com': 'IndiaCarNews'}.get(urlparse(link).hostname, 'Source editorial')
        parsed.append({'source_url': link, 'title': title, 'excerpt': content[:280].rstrip(),
                       'content': content, 'cover_image': image, 'author': publisher, 'tag': tag or 'Imported'})
    if not parsed:
        raise ValueError('Feed stories were missing usable titles or content.')
    return parsed

def run_blog_sync(mode='full', run_id=None):
    report = {'models': 0, 'details': 0, 'errors': []}
    if run_id:
        with closing(db.connect()) as c, c:
            c.execute("UPDATE sync_runs SET status='running' WHERE id=?", (run_id,))
    try:
        for source in catalog.blog_sources(True):
            errors = []; imported = 0
            try:
                validate_blog_source(source['provider'], source['url'])
                data, _ = fetch(source['url'], BLOG_HOSTS, 5_000_000)
                for post in parse_blog_feed(data, source['url'], source.get('tag')):
                    db.save_scraped_post(**post)
                    imported += 1
                report['models'] += imported
            except Exception as error:
                errors.append(f'{type(error).__name__}: {error}')
            with closing(db.connect()) as c, c:
                c.execute('''UPDATE blog_sources SET last_synced=CASE WHEN ? > 0 THEN datetime('now') ELSE last_synced END,
                             last_error=?,post_count=? WHERE id=?''',
                          (imported, '\n'.join(errors) or None, imported, source['id']))
            report['errors'].extend((source['url'] + ': ' + error) for error in errors)
            catalog.progress(run_id, report)
    except Exception as error:
        report['errors'].append(f'{type(error).__name__}: {error}')
    finally:
        if run_id:
            catalog.finish_run(run_id, report)
    return report

def fetch(url, hosts, max_bytes=12_000_000):
    # Validate every redirect: source settings cannot target internal services.
    for _ in range(4):
        validate_url(url, hosts)
        with session().get(url,timeout=(10,30),allow_redirects=False,stream=True) as response:
            if response.is_redirect:
                url=urljoin(url,response.headers['Location'])
                continue
            response.raise_for_status()
            chunks=[]; size=0
            for chunk in response.iter_content(65536):
                size+=len(chunk)
                if size>max_bytes: raise ValueError('Source response exceeded size limit.')
                chunks.append(chunk)
            return b''.join(chunks),response.headers.get('Content-Type','')
    raise ValueError('Too many redirects.')

def page_state(text):
    match = re.search(r'window\.__INITIAL_STATE__\s*=\s*',text)
    if not match: raise ValueError('Source layout changed: embedded catalogue data is missing.')
    state,_=json.JSONDecoder().raw_decode(text[match.end():])
    if not isinstance(state,dict): raise ValueError('Invalid source catalogue.')
    return state

def fetch_state(url):
    data,_=fetch(url,{p[1] for p in PROVIDERS.values()})
    return page_state(data.decode('utf-8'))

def image_url(path):
    if not path: return ''
    return path if path.startswith('https://') else 'https://imgd.aeplcdn.com/664x374'+path

def cache_image(url):
    if not url or url.startswith('/static/'): return url
    with closing(db.connect()) as c:
        row=c.execute('SELECT local_path FROM media_cache WHERE source_url=?',(url,)).fetchone()
    if row and (Path(__file__).resolve().parent/row[0].lstrip('/')).is_file(): return row[0]
    data,mime=fetch(url,ASSET_HOSTS,8_000_000)
    from PIL import Image
    from io import BytesIO
    with Image.open(BytesIO(data)) as picture:
        if picture.width*picture.height>30_000_000: raise ValueError('Image too large.')
        picture.verify()
        ext={'JPEG':'.jpg','PNG':'.png','WEBP':'.webp','GIF':'.gif'}.get(picture.format)
    if not ext: raise ValueError('Unsupported image format.')
    name=hashlib.sha256(url.encode()).hexdigest()+ext
    MEDIA_DIR.mkdir(exist_ok=True)
    dest=MEDIA_DIR/name
    tmp=dest.with_suffix(ext+'.tmp')
    tmp.write_bytes(data); tmp.replace(dest)
    local='/static/media/'+name
    with closing(db.connect()) as c,c:
        c.execute('INSERT OR REPLACE INTO media_cache VALUES (?,?)',(url,local))
    return local

def price_text(low, high=None):
    if not low: return 'Price unavailable'
    high=high or low
    return f'₹ {low/100000:.2f}' + (f' – {high/100000:.2f}' if high!=low else '') + ' Lakh'

def overview_specs(items):
    return {item['title']:'/'.join(str(v['text']).strip() for v in item.get('keySpecsValue',[]) if v.get('text'))
            for item in items if item.get('title') and item.get('keySpecsValue')}

def sections(items):
    result={}
    for group in items or []:
        rows=[]
        for item in group.get('items',[]):
            values=[str(v) for v in item.get('values',[]) if v not in (None,'','NA','N/A')]
            if item.get('name') and values:
                rows.append([item['name'], '/'.join(dict.fromkeys(values)) + (' '+item['unitType'] if item.get('unitType') else '')])
        if rows: result[group.get('name','Specifications')]=rows
    return result

def parse_model(state, source, url):
    page=state.get('modelPage') or {}
    model=page.get('modelDetails') or {}
    if not model.get('modelName') or not model.get('modelId'):
        raise ValueError('Model details missing; existing data preserved.')
    if model.get('makeName','').casefold().replace(' ','') != source['brand'].casefold().replace(' ',''):
        # Maruti vs Maruti Suzuki is a known naming difference.
        if {model.get('makeName'),source['brand']} != {'Maruti Suzuki','Maruti'}:
            raise ValueError('Model brand differs from the configured source.')
    versions=[]
    for v in page.get('versions',[]):
        po=v.get('priceOverview') or {}
        price=po.get('exShowRoomPrice') or (po.get('price') if po.get('priceStatus')==3 else None)
        if price and v.get('versionName'):
            versions.append({'name':v['versionName'],'price':f'{price/100000:.2f} Lakh',
                             'price_lakh':price/100000,'on_road':None,'spec':v.get('displayName',''),'waiting':None})
    if not versions: raise ValueError('No valid ex-showroom variants found; existing data preserved.')
    low=min(v['price_lakh'] for v in versions)*100000
    high=max(v['price_lakh'] for v in versions)*100000
    facts=overview_specs(page.get('keySpecs',[]))
    specs=sections(page.get('specifications')) or {'Key specifications':list(facts.items())}
    first=page.get('versions',[{}])[0]
    version_specs=[[i['itemName'],str(i['value'])+(' '+i['unitType'] if i.get('unitType') else '')]
                   for i in first.get('specsSummary',[]) if i.get('itemName') and i.get('value')]
    if version_specs: specs['Base variant']=version_specs
    description='\n\n'.join(scraper.html_to_text(s) for s in (page.get('synopsis') or {}).get('summary',[]) if isinstance(s,str))
    result={'model_no':db.slugify(source['brand']+' '+model['modelMaskingName']).upper(),
            'brand':source['brand'],'name':model['modelName'],
            'category':page.get('bodyStyleText') or ('Electric scooter' if model.get('isElectricVehicle') and source['kind']=='bike' else 'Motorcycle'),
            'price':price_text(low,high),'min_price':low/100000,'max_price':high/100000,
            'image':image_url(model.get('imagePath','')),'source_url':url,
            'description':description,'specs':specs,'variants':versions}
    if source['kind']=='car':
        result.update(car_name=source['brand']+' '+model['modelName'],fuel_type=facts.get('Fuel Type','-'),
                      mileage=facts.get('Mileage','-'),engine=facts.get('Engine','-'),
                      transmission=facts.get('Transmission','-'),seating=facts.get('Seating Capacity','-'),
                      body_type={1:'Sedan',3:'Hatchback',6:'SUV',7:'MUV'}.get(model.get('bodyStyleId'),'-'))
        result['details']={'description':description,'expert_verdict':'','expert_rating':None,
            'user_rating':model.get('modelAggregateRating') or None,'review_count':model.get('modelReviewCount'),
            'safety_rating':facts.get('Safety','-'),'mileage_summary':'','brochure_url':None,
            'variants':versions,'specifications':specs,'features':sections(page.get('features')),
            'pros':[],'cons':[],'highlights':[],'latest_updates':[],'colours':[],
            'expert_sections':[],'mileage_table':[]}
    return result

def discover(state, source):
    models=(state.get('makePage') or {}).get('models')
    if not isinstance(models,list) or not models:
        raise ValueError('Brand listing is empty or changed; existing records preserved.')
    suffix='cars' if source['kind']=='car' else 'bikes'
    host=PROVIDERS[source['provider']][1]
    found=[]
    for model in models:
        if model.get('status')!=2: continue  # do not mix upcoming/discontinued models into current catalogue
        make=model.get('makeMaskingName'); slug=model.get('modelMaskingName')
        if make and slug: found.append(f'https://{host}/{make}-{suffix}/{slug}/')
    if not found: raise ValueError('No current models found; existing records preserved.')
    return list(dict.fromkeys(found))

def save_model(result, source):
    if source['kind']=='bike':
        catalog.save_bike(result,True)
    else:
        # Match existing names before assigning new IDs so bookmarked pages survive migration.
        with closing(db.connect()) as c:
            rows=c.execute('SELECT model_no,car_name FROM cars WHERE brand=?',(source['brand'],)).fetchall()
        normalized=lambda value: re.sub(r'[^a-z0-9]','',value.lower())
        match=next((r for r in rows if normalized(r['car_name'])==normalized(result['car_name'])),None)
        if match: result['model_no']=match['model_no']
        db.save_brand(source['brand'],source['url'],[result])
        db.save_details(result['model_no'],result['details'])

def run_sync(kind='all',mode='full',run_id=None):
    if kind == 'blog':
        return run_blog_sync(mode, run_id)
    report={'models':0,'details':0,'errors':[]}
    if run_id:
        with closing(db.connect()) as c,c: c.execute("UPDATE sync_runs SET status='running' WHERE id=?",(run_id,))
    try:
        sources=catalog.sources(True,None if kind=='all' else kind)
        if not sources: report['errors'].append('No enabled sources for the selected vehicle type.')
        for source in sources:
            errors=[]; count=0
            try:
                validate_source(source['provider'],source['kind'],source['url'])
                if source['provider']=='cardekho':
                    models=scraper.scrape_brand(source['brand'],source['url'])
                    if not models: raise ValueError('No models found; existing records preserved.')
                    db.save_brand(source['brand'],source['url'],models)
                    count=len(models); report['models']+=count
                    if mode!='basic':
                        todo=models if mode=='full' else [m for m in models if not db.get_car(m['model_no'])[0]['details_updated']]
                        done,errs=scraper.scrape_details(todo);report['details']+=done;errors.extend(errs)
                else:
                    urls=discover(fetch_state(source['url']),source)
                    if mode=='missing':
                        with closing(db.connect()) as c:
                            table='cars' if source['kind']=='car' else 'bikes'
                            ready={r[0] for r in c.execute(f'SELECT source_url FROM {table} WHERE details_updated IS NOT NULL')}
                        urls=[u for u in urls if u not in ready]
                    def import_one(url):
                        try:
                            result=parse_model(fetch_state(url),source,url)
                            save_model(result,source)
                            try: cache_image(result['image'])
                            except Exception as error: return True,f"{result['name']}: image cache failed ({type(error).__name__})"
                            return True,None
                        except Exception as error: return False,f'{urlparse(url).path}: {type(error).__name__}: {error}'
                    with ThreadPoolExecutor(max_workers=3) as pool:
                        for ok,error in pool.map(import_one,urls):
                            count+=int(ok);report['models']+=int(ok);report['details']+=int(ok)
                            if error: errors.append(error)
                            catalog.progress(run_id,report)
            except Exception as error: errors.append(f'{type(error).__name__}: {error}')
            with closing(db.connect()) as c,c:
                c.execute('UPDATE scrape_sources SET last_synced=CASE WHEN ? > 0 THEN datetime("now") ELSE last_synced END,last_error=?,model_count=? WHERE id=?',
                          (count,'\n'.join(errors) or None,count,source['id']))
            report['errors'].extend(source['brand']+': '+error for error in errors)
            catalog.progress(run_id,report)
    except Exception as error: report['errors'].append(f'{type(error).__name__}: {error}')
    finally:
        if run_id: catalog.finish_run(run_id,report)
    return report

def cache_existing_media():
    with closing(db.connect()) as c:
        urls={r[0] for table,col in [('cars','image'),('bikes','image'),('posts','cover_image')]
              for r in c.execute(f'SELECT {col} FROM {table}') if r[0]}
    errors=[]
    for url in urls:
        try: cache_image(url)
        except Exception as error: errors.append(f'{urlparse(url).hostname}: {type(error).__name__}')
    return errors