"""CarDekho scraper - data scrape karke local DB (db.py) me save karta hai.

Usage:
  python scraper.py              # saare brands + saari cars ki full details
  python scraper.py --basic      # sirf brand pages (price, fuel, mileage...)
  python scraper.py --missing    # sirf un cars ki details jinki abhi nahi hain
"""

from concurrent.futures import ThreadPoolExecutor
import html
import json
import re
import sys

from bs4 import BeautifulSoup
import requests

import db

# Brands jinka data automatic fetch hota hai - URL dalne ki zarurat nahi.
# CarDekho brand pages par har model ka structured JSON milta hai.
BRAND_SOURCES = {
    'Maruti Suzuki': 'https://www.cardekho.com/cars/Maruti',
    'Mahindra': 'https://www.cardekho.com/cars/Mahindra',
    'Tata': 'https://www.cardekho.com/cars/Tata',
    'Hyundai': 'https://www.cardekho.com/cars/Hyundai',
}
SOURCE_BASE = 'https://www.cardekho.com'
HEADERS = {
    'User-Agent': (
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
        'AppleWebKit/537.36 (KHTML, like Gecko) '
        'Chrome/128.0.0.0 Safari/537.36'
    ),
    'Accept-Language': 'en-IN,en;q=0.9',
}


def parse_price_lakh(text):
  """'5.99 Lakh' -> 5.99, '1.2 Crore' -> 120.0, fail hone par None."""
  if not text:
    return None
  match = re.search(r'(\d[\d,]*(?:\.\d+)?)\s*(lakh|lac|crore|cr)?', str(text), re.I)
  if not match:
    return None
  value = float(match.group(1).replace(',', ''))
  unit = (match.group(2) or 'lakh').lower()
  if unit.startswith('cr'): return value * 100
  if not match.group(2) and (',' in match.group(1) or value >= 1000): return value / 100000
  return value


def clean(value):
  value = (value or '').strip() if isinstance(value, str) else value
  if not value or value == 'None' or str(value).startswith('null'):
    return '-'
  return value


# CarDekho ke body type naam ko ek jaisa karo (filter ke liye)
BODY_TYPES = {
    'Sport Utilities': 'SUV',
    'Sedans': 'Sedan',
    'Minivans': 'Minivan',
    'Pickup Trucks': 'Pickup',
}


def fetch_page_state(url):
  """CarDekho page ka embedded window.__INITIAL_STATE__ JSON return karta hai."""
  from ingestion import fetch_state
  return fetch_state(url)


def scrape_brand(brand, url):
  """Brand page se saare models ka basic data nikalta hai."""
  state = fetch_page_state(url)

  cars = []
  for item in state.get('items', []):
    min_price = parse_price_lakh(item.get('minPrice'))
    if min_price is None:
      continue
    max_price = parse_price_lakh(item.get('maxPrice')) or min_price
    model_url = item.get('modelUrl') or ''
    cars.append({
        'model_no': model_url.strip('/').replace('/', '-').upper()
        or item.get('slug', '').upper(),
        'car_name': item.get('modelName') or item.get('name'),
        'brand': brand,
        'price': f"₹ {item.get('priceRange') or item.get('minPrice')}",
        'fuel_type': clean(item.get('fuelType')),
        'mileage': clean(item.get('mileage')),
        'min_price': min_price,
        'max_price': max_price,
        'engine': clean(item.get('engine')),
        'body_type': BODY_TYPES.get(item.get('vehicleType'), clean(item.get('vehicleType'))),
        'seating': clean(item.get('seatingCapacity')),
        'transmission': clean(item.get('transmissionType')),
        'image': item.get('image') or '',
        'source_url': SOURCE_BASE + model_url if model_url else url,
    })
  return cars


def html_to_text(html):
  """HTML ko plain text me badalta hai, har paragraph/list item alag line me."""
  if not html:
    return ''
  soup = BeautifulSoup(html, 'html.parser')
  for tag in soup(['img', 'script', 'style', 'table']):
    tag.decompose()
  lines = (
      line.strip()
      for line in soup.get_text('\n').replace('\xa0', ' ').splitlines()
  )
  return '\n'.join(line for line in lines if line)


def spec_sections(sections):
  """[{title, items:[{text, value}]}] -> {title: [[name, value], ...]}"""
  return {
      sec.get('title', ''): [
          [html.unescape(item.get('text', '')), html.unescape(str(item.get('value', '')))]
          for item in sec.get('items', [])
          if item.get('text')
      ]
      for sec in sections or []
  }


def scrape_model_details(model_url):
  """Model page + specs page se poora data: description, expert review,

  pros/cons, variants, colours, mileage, full specifications aur features.
  """
  page = fetch_page_state(model_url)
  if not (page.get('variantTable') or {}).get('variantList'):
    raise ValueError('No variant data found. Existing details were preserved.')
  try:
    specs_page = fetch_page_state(model_url.rstrip('/') + '/specs')
  except Exception:  # specs page na mile to model page ke key specs hi sahi
    specs_page = {}

  review_tabs = {
      tab.get('id'): tab
      for tab in (page.get('expertReviewDetailV2') or {}).get('items', [])
  }
  pros_cons = (review_tabs.get('prosandCons') or {}).get('data', {}).get('items', {})
  expert_sections = [
      {
          'title': tab.get('title'),
          'rating': tab.get('rating'),
          'text': html_to_text(tab.get('description')),
      }
      for tab_id, tab in review_tabs.items()
      if tab_id not in ('expertVerdict', 'prosandCons') and tab.get('description')
  ]

  highlights = []
  for sec in (page.get('quickOverview') or {}).get('keyAndFeatureList', []):
    for item in sec.get('list', []):
      if item.get('standOutFeatures'):
        highlights.append(html_to_text(item['standOutFeatures']))

  technical = specs_page.get('specsTechnicalJson') or page.get('specsTechnicalJson') or {}
  specifications = spec_sections(technical.get('specification'))
  features = spec_sections(technical.get('featured'))
  if not specifications:
    specifications = spec_sections(technical.get('keySpecs'))

  safety_rating = '-'
  for name, value in features.get('Safety', []):
    if 'NCAP Safety Rating' in name and 'Child' not in name:
      safety_rating = f'{value} ({name.replace(" Safety Rating", "")})'
      break

  mileage_table = page.get('mileageTable') or {}
  user_reviews = page.get('userReviews') or {}
  description = (review_tabs.get('overview') or {}).get('description')
  # Expert review na ho (jaise commercial models) to latest updates / SEO text
  description = (
      html_to_text(description)
      or html_to_text((page.get('pagetitle') or {}).get('description'))
      or ' '.join(filter(None, [
          (page.get('seo') or {}).get('metaDescription'),
          mileage_table.get('description'),
      ]))
  )

  return {
      'description': description,
      'latest_updates': html_to_text(
          (page.get('pagetitle') or {}).get('description')
      ).splitlines(),
      'expert_verdict': (review_tabs.get('expertVerdict') or {})
      .get('expertVerdict', {})
      .get('content', ''),
      'expert_rating': (page.get('expertReviewDetailV2') or {}).get('overallRating'),
      'expert_sections': expert_sections,
      'pros': [p.get('pros') for p in pros_cons.get('pros', {}).get('items', [])],
      'cons': [c.get('cons') for c in pros_cons.get('cons', {}).get('items', [])],
      'highlights': highlights,
      'user_rating': user_reviews.get('overAllRating'),
      'review_count': user_reviews.get('reviewCount'),
      'safety_rating': safety_rating,
      'variants': [
          {
              'name': v.get('name'),
              'price': v.get('price'),
              'price_lakh': (v['exShowRoomPrice'] / 100000
                             if v.get('exShowRoomPrice')
                             else parse_price_lakh(v.get('price'))),
              'on_road': v.get('price_orp'),
              'spec': v.get('subText'),
              'waiting': v.get('waitingPeriod'),
          }
          for v in (page.get('variantTable') or {}).get('variantList', [])
      ],
      'colours': [
          c.get('name')
          for c in (page.get('galleryColorSection') or {}).get('items', [])
      ],
      'mileage_summary': mileage_table.get('description', ''),
      'mileage_table': [
          [cell.get('text') for cell in row]
          for row in mileage_table.get('childs', [])
      ],
      'specifications': specifications,
      'features': features,
      'brochure_url': (page.get('brochureDto') or {}).get('brochureUrl'),
  }


def scrape_brands():
  """Saare brands parallel me scrape karke DB me basic data save karta hai."""
  errors = []

  def run(entry):
    brand, url = entry
    try:
      cars = scrape_brand(brand, url)
      if not cars: raise ValueError('Empty brand listing. Existing data preserved.')
      db.save_brand(brand, url, cars)
      return len(cars)
    except Exception as e:  # ek brand fail ho to baaki chalte rahein
      errors.append(f'{brand}: {e}')
      return 0

  with ThreadPoolExecutor(max_workers=len(BRAND_SOURCES)) as pool:
    count = sum(pool.map(run, BRAND_SOURCES.items()))
  return count, errors


def scrape_details(cars):
  """Diye gaye cars (model_no, car_name, source_url) ki full details DB me."""
  errors = []

  def run(car):
    try:
      db.save_details(car['model_no'], scrape_model_details(car['source_url']))
      return 1
    except Exception as e:
      errors.append(f"{car['car_name']}: {e}")
      return 0

  with ThreadPoolExecutor(max_workers=3) as pool:
    count = sum(pool.map(run, cars))
  return count, errors


def sync(details=True, only_missing=False):
  """Poora sync: brands ka basic data, phir cars ki full details."""
  db.init_db()
  cars, errors = scrape_brands()
  report = {'cars': cars, 'details': 0}
  if details:
    todo = db.cars_needing_details() if only_missing else db.all_cars()
    report['details'], detail_errors = scrape_details(todo)
    errors += detail_errors
  report['errors'] = errors
  return report


if __name__ == '__main__':
  args = sys.argv[1:]
  report = sync(details='--basic' not in args, only_missing='--missing' in args)
  print(f"{report['cars']} models ka basic data, "
        f"{report['details']} cars ki full details save hui -> {db.DB_FILE}")
  for e in report['errors']:
    print('ERROR:', e)