Initial commit Scrapyard
This commit is contained in:
1
scrapers/__init__.py
Normal file
1
scrapers/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# scrapers package
|
||||
661
scrapers/scraper_billigteknik.py
Normal file
661
scrapers/scraper_billigteknik.py
Normal file
@@ -0,0 +1,661 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
BilligTeknik scraper
|
||||
======================
|
||||
Crawls https://www.billigteknik.se/623-begagnad-barbar-dator and stores
|
||||
refurbished laptop listings in the shared SQLite database.
|
||||
|
||||
Usage (from main.py web trigger, or standalone):
|
||||
from scrapers.scraper_billigteknik import run
|
||||
conn = init_db(DB_PATH)
|
||||
run(conn, skip_phase1=False)
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .shared import (
|
||||
log, polite_get, _sleep,
|
||||
INSERT_SQL, _ALL_COLUMNS, _safe_row,
|
||||
USER_AGENTS, map_condition_grade,
|
||||
)
|
||||
|
||||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
BILLIG_BASE_URL = "https://www.billigteknik.se"
|
||||
|
||||
|
||||
# ── Title cleaning ────────────────────────────────────────────────────────────
|
||||
|
||||
# Applied in order; each pattern is replaced with a single space then collapsed.
|
||||
_TITLE_NOISE: list[re.Pattern] = [
|
||||
# Parenthetical notes: "(beg med...)", "(gen 11)", "(låg batterihälsa)" etc.
|
||||
re.compile(r'\s*\([^)]*\)'),
|
||||
# Screen size: 14", 13,3", 11.6", 13-tum, 15,6 tum
|
||||
re.compile(r'\b\d{2}[.,]?\d?\s*[-–]?\s*tum\b\s*', re.I),
|
||||
re.compile(r'\b\d{2}[.,]?\d?\s*["""″]\s*'),
|
||||
# Resolution labels
|
||||
re.compile(r'\b(Full[\s-]?HD|FHD|QHD|WQHD|UHD|WUXGA|WXGA|4K|2K|HD[\+]?)\b\s*', re.I),
|
||||
# All GB / TB amounts (RAM and storage) — also catches digit-concatenated: 512SSD
|
||||
re.compile(r'\b\d+(?:SSD|HDD|NVMe|eMMC|SSHD)\b\s*', re.I),
|
||||
re.compile(r'\b\d+\s*[GT]B\b\s*', re.I),
|
||||
# Storage type labels
|
||||
re.compile(r'\b(NVMe|SSD|HDD|eMMC|SSHD|Flash)\b\s*', re.I),
|
||||
# OS strings — longest/most specific first
|
||||
re.compile(r'\bWindows\s+\d+\s*(Pro|Home|S|Enterprise|Education)?\b\s*', re.I),
|
||||
re.compile(r'\bWin\s*\d+\s*(Pro|Home|S|P)?\b\s*', re.I),
|
||||
re.compile(r'\bW\d+(?:Pro|Home|P|H|E)\b\s*', re.I),
|
||||
re.compile(r'\b(macOS|ChromeOS|Chrome\s+OS|Linux|Ubuntu)\b\s*', re.I),
|
||||
# CPU — full model number first, then family + standalone
|
||||
re.compile(r'\b(Intel\s+)?(Core\s+)?(Ultra\s+)?i[3579]-\d{4,5}[A-Za-z]*\b\s*', re.I),
|
||||
re.compile(r'\b(Intel\s+)?Core\s+(Ultra\s+)?i[3579]\b\s*', re.I),
|
||||
re.compile(r'\bi[3579]\b\s*', re.I),
|
||||
# Intel Core Ultra (no "i" prefix, e.g. "Ultra 7 165H", "Core Ultra 5 125U")
|
||||
re.compile(r'\b(?:Intel\s+)?(?:Core\s+)?Ultra\s+[579]\s+\d{3,4}[A-Za-z]*\b\s*', re.I),
|
||||
re.compile(r'\b(AMD\s+)?Ryzen\s+\d+\b\s*', re.I),
|
||||
re.compile(r'\b(Intel\s+)?(Celeron|Pentium|Xeon|Athlon)\b\s*', re.I),
|
||||
re.compile(r'\b(Intel\s+)?(DualCore|QuadCore|OctaCore)\b\s*', re.I),
|
||||
# CPU generation indicators (must come BEFORE bare ordinal strip)
|
||||
re.compile(r'\b\d{1,2}(th|st|nd|rd)\s*gen(eration)?\b\s*', re.I),
|
||||
re.compile(r'\bgen\s*\d{1,2}\b\s*', re.I),
|
||||
# Bare generation ordinals NOT followed by a model-style word (e.g. 10th, 11th)
|
||||
re.compile(r'\b\d{1,2}(th|st|nd|rd)\b\s*', re.I),
|
||||
# Connectivity noise
|
||||
re.compile(r'\bmed\s+(4G|5G|LTE)\b\s*', re.I),
|
||||
re.compile(r'\b(4G|5G|LTE)\b\s*', re.I),
|
||||
# Standalone brand names left after CPU model is stripped
|
||||
re.compile(r'\bIntel\b\s*', re.I),
|
||||
re.compile(r'\bAMD\b\s*', re.I),
|
||||
# Touch/screen feature words left floating
|
||||
re.compile(r'\bTouch\b\s*', re.I),
|
||||
# Condition word "beg"
|
||||
re.compile(r'\bbeg\b\s*', re.I),
|
||||
# Dangling connectors / punctuation left after stripping
|
||||
re.compile(r'\s*&(?:\s+[-\w]+)*'), # "& tangentbord", "& -modem", standalone "&"
|
||||
re.compile(r'\bmed\b(?:\s+\w+)+'), # "med tangentbord", "med Sure View" etc. (1+ words)
|
||||
re.compile(r'(?<=[a-z\d])\s*[+]\s*', re.I), # trailing "+" from "HD+"
|
||||
# GPU model names (already tracked in gpu field)
|
||||
re.compile(r'\bRTX\s+\w*\d{3,4}\w*(?:\s+\w+)?\b\s*', re.I), # RTX 3000, RTX A1000, RTX 2000 Ada
|
||||
re.compile(r'\bGTX\s+\d{3,4}\b\s*', re.I),
|
||||
re.compile(r'\bQuadro\s+\w+\b\s*', re.I),
|
||||
re.compile(r'\bMX\d{3,4}\b\s*', re.I), # MX330, MX250 etc.
|
||||
# Warranty codes like 2YW, 3YW
|
||||
re.compile(r'\b\d+YW\b\s*', re.I),
|
||||
# HP Sure View privacy screen feature — strip unit or orphaned word
|
||||
re.compile(r'\bSure\s+View\b\s*', re.I),
|
||||
re.compile(r'\bView\b\s*', re.I),
|
||||
]
|
||||
|
||||
|
||||
def _clean_title(title: str) -> str:
|
||||
"""Strip spec noise from a BilligTeknik title, keeping only brand + model name.
|
||||
|
||||
Example:
|
||||
"HP ProBook 440 G8 14\" Full HD i5 (gen 11) 16GB 256GB SSD Win 11 Pro (beg)"
|
||||
→ "HP ProBook 440 G8"
|
||||
"""
|
||||
s = title.strip()
|
||||
for rx in _TITLE_NOISE:
|
||||
s = rx.sub(' ', s)
|
||||
# Collapse runs of spaces and strip trailing punctuation
|
||||
s = re.sub(r'\s{2,}', ' ', s).strip(' -–,/|·*&')
|
||||
# Safety: never return an empty string
|
||||
return s if s else title.strip()
|
||||
BILLIG_CATEGORY_URL = "https://www.billigteknik.se/623-begagnad-barbar-dator"
|
||||
BILLIG_MAX_PAGES = 60
|
||||
|
||||
# Compatibility aliases
|
||||
BASE_URL = BILLIG_BASE_URL
|
||||
CATEGORY_URL = BILLIG_CATEGORY_URL
|
||||
MAX_PAGES = BILLIG_MAX_PAGES
|
||||
|
||||
|
||||
# ── Price parsing ─────────────────────────────────────────────────────────────
|
||||
|
||||
def parse_price(text: str) -> Optional[int]:
|
||||
"""
|
||||
Extract an integer SEK price from Swedish-formatted strings.
|
||||
'3 899 kr' → 3899
|
||||
'Nypris: 16 000 kr' → 16000
|
||||
"""
|
||||
digits = re.sub(r"[^\d]", "", text)
|
||||
return int(digits) if digits else None
|
||||
|
||||
|
||||
# ── Listing page parser ───────────────────────────────────────────────────────
|
||||
|
||||
def _extract_condition_detail(title: str) -> str:
|
||||
"""
|
||||
Pull the freeform condition note out of a product title.
|
||||
'HP EliteBook (beg med små märken skärm)' → 'små märken skärm'
|
||||
'Dell Chromebook (beg)' → ''
|
||||
"""
|
||||
m = re.search(r"\(beg([^)]*)\)", title, re.IGNORECASE)
|
||||
if not m:
|
||||
return ""
|
||||
detail = m.group(1).strip()
|
||||
detail = re.sub(r"^\*?\s*med\s+", "", detail, flags=re.IGNORECASE).strip(" *")
|
||||
return detail
|
||||
|
||||
|
||||
def parse_listing_page(html: str) -> list[dict]:
|
||||
"""
|
||||
Parse a category listing page.
|
||||
Returns list of dicts: url, title, price_sek, orig_price_sek,
|
||||
condition_grade, condition_detail, category.
|
||||
"""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
results = []
|
||||
|
||||
cards = soup.select("article.product-miniature")
|
||||
if not cards:
|
||||
cards = soup.select(".js-product-miniature")
|
||||
|
||||
if not cards:
|
||||
log.debug("No product cards via standard selectors – using href fallback")
|
||||
seen: set = set()
|
||||
for a in soup.find_all("a", href=re.compile(r"billigteknik\.se/.+\.html")):
|
||||
href = a.get("href", "").split("#")[0]
|
||||
if href and href not in seen and "/beg" in href:
|
||||
seen.add(href)
|
||||
results.append({
|
||||
"url": href,
|
||||
"title": a.get_text(strip=True),
|
||||
"price_sek": None,
|
||||
"orig_price_sek": None,
|
||||
"condition_grade": None,
|
||||
"condition_detail": _extract_condition_detail(a.get_text(strip=True)),
|
||||
"category": _category_from_url(href),
|
||||
})
|
||||
return results
|
||||
|
||||
for card in cards:
|
||||
try:
|
||||
link = (
|
||||
card.select_one("a[itemprop='url']")
|
||||
or card.select_one("a.product-thumbnail")
|
||||
or card.select_one(".product-title a")
|
||||
or card.select_one("h3 a")
|
||||
)
|
||||
if not link:
|
||||
continue
|
||||
href = link.get("href", "").split("#")[0].strip()
|
||||
if not href or ".html" not in href:
|
||||
continue
|
||||
if not href.startswith("http"):
|
||||
href = urljoin(BASE_URL, href)
|
||||
|
||||
title_el = card.select_one("h3[itemprop='name']")
|
||||
if not title_el:
|
||||
title_el = card.select_one(".product-title h3") or card.select_one("h3")
|
||||
title = title_el.get_text(strip=True) if title_el else ""
|
||||
|
||||
price = None
|
||||
price_meta = card.select_one("meta[itemprop='price']")
|
||||
if price_meta and price_meta.get("content"):
|
||||
price = parse_price(price_meta["content"])
|
||||
if price is None:
|
||||
price_el = card.select_one(".product-price-and-shipping span.price")
|
||||
if price_el:
|
||||
price = parse_price(price_el.get_text())
|
||||
|
||||
orig = None
|
||||
orig_el = card.select_one(".ppc_normal_price")
|
||||
if orig_el:
|
||||
orig = parse_price(orig_el.get_text())
|
||||
|
||||
grade_spans = card.select(".product-variant-class span")
|
||||
grade_texts = [s.get_text(strip=True) for s in grade_spans
|
||||
if s.get_text(strip=True) in {"A", "B", "C", "D", "Premium"}]
|
||||
seen_grades: dict = {}
|
||||
for g in grade_texts:
|
||||
seen_grades[g] = None
|
||||
if seen_grades:
|
||||
# Map each raw grade individually; take the best (first unique) result
|
||||
mapped = list(dict.fromkeys(
|
||||
map_condition_grade(g, "billigteknik") for g in seen_grades
|
||||
))
|
||||
grade = mapped[0] if len(mapped) == 1 else "/".join(mapped)
|
||||
else:
|
||||
grade = None
|
||||
|
||||
condition_detail = _extract_condition_detail(title)
|
||||
|
||||
results.append({
|
||||
"url": href,
|
||||
"title": title,
|
||||
"price_sek": price,
|
||||
"orig_price_sek": orig,
|
||||
"condition_grade": grade,
|
||||
"condition_detail": condition_detail,
|
||||
"category": _category_from_url(href),
|
||||
})
|
||||
|
||||
except Exception as exc:
|
||||
log.debug(f"Card parse error: {exc}")
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _category_from_url(href: str) -> str:
|
||||
"""Extract the category slug from a product URL."""
|
||||
m = re.search(r"billigteknik\.se/([^/]+)/", href)
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
# ── Product detail parser ─────────────────────────────────────────────────────
|
||||
|
||||
def parse_product_page(html: str, url: str) -> dict:
|
||||
"""
|
||||
Parse a product detail page.
|
||||
Returns a dict with all extractable fields (key names match DB columns).
|
||||
"""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
data: dict = {"url": url}
|
||||
|
||||
h1 = soup.select_one("h1") or soup.select_one(".product-name")
|
||||
data["title"] = _clean_title(h1.get_text(strip=True)) if h1 else ""
|
||||
|
||||
for block in soup.select(".product-reference-data"):
|
||||
label_el = block.select_one(".name.label")
|
||||
value_el = block.select_one(".value")
|
||||
if not label_el or not value_el:
|
||||
continue
|
||||
ltext = label_el.get_text(strip=True).rstrip(":")
|
||||
vtext = value_el.get_text(strip=True)
|
||||
if re.search(r"Tillverkare|Manufacturer|Brand", ltext, re.I):
|
||||
# Normalise legacy full names to the common short brand name.
|
||||
brand = vtext
|
||||
if re.search(r"hewlett.?packard", brand, re.I):
|
||||
brand = "HP"
|
||||
data.setdefault("brand", brand)
|
||||
elif re.search(r"^Referens", ltext, re.I):
|
||||
data.setdefault("model_ref", vtext)
|
||||
elif re.search(r"Ean13|EAN", ltext, re.I):
|
||||
data.setdefault("ean", vtext)
|
||||
|
||||
price_span = soup.select_one(".current-price span[content]")
|
||||
if price_span and price_span.get("content"):
|
||||
p = parse_price(price_span["content"])
|
||||
if p:
|
||||
data["price_sek"] = p
|
||||
else:
|
||||
price_el = soup.select_one(".current-price span.price") or soup.select_one("span.price")
|
||||
if price_el:
|
||||
p = parse_price(price_el.get_text())
|
||||
if p:
|
||||
data["price_sek"] = p
|
||||
|
||||
orig_el = soup.select_one(".ppc_normal_price")
|
||||
if orig_el:
|
||||
op = parse_price(orig_el.get_text())
|
||||
if op:
|
||||
data["original_price_sek"] = op
|
||||
|
||||
specs: dict[str, str] = {}
|
||||
|
||||
best_table = max(soup.select("table"), key=lambda t: len(t.select("tr")), default=None)
|
||||
if best_table:
|
||||
for row in best_table.select("tr"):
|
||||
cells = row.select("td, th")
|
||||
if len(cells) >= 2:
|
||||
key = cells[0].get_text(strip=True)
|
||||
val = cells[1].get_text(strip=True)
|
||||
if key.lower() in ("funktion", "function", "") or val.lower() in ("specifikation", ""):
|
||||
continue
|
||||
if key and val:
|
||||
specs[key] = val
|
||||
|
||||
if not specs:
|
||||
for dl in soup.select("dl.data-sheet, dl"):
|
||||
for dt, dd in zip(dl.select("dt"), dl.select("dd")):
|
||||
k = dt.get_text(strip=True)
|
||||
v = dd.get_text(strip=True)
|
||||
if k and v:
|
||||
specs[k] = v
|
||||
|
||||
data["specs_raw"] = json.dumps(specs, ensure_ascii=False)
|
||||
|
||||
def s(pattern: str) -> str:
|
||||
rx = re.compile(pattern, re.IGNORECASE)
|
||||
for k, v in specs.items():
|
||||
if rx.search(k):
|
||||
return v
|
||||
return ""
|
||||
|
||||
screen_raw = s(r"skärmstor|screen.?size|display.?size")
|
||||
# Extract just the numeric size anywhere in the string:
|
||||
# "14\" Full HD LED-skärm" → "14\"", "Pekskärm 11.6\"" → "11.6\""
|
||||
_sz_m = re.search(r'[\d,.]+\s*(?:[-–]?\s*tum|["\u2033\u201d\u201c])', screen_raw.strip(), re.I)
|
||||
data["screen_size"] = _sz_m.group(0).strip() if _sz_m else screen_raw
|
||||
|
||||
res_raw = s(r"upplösning|resolution")
|
||||
data["screen_resolution"] = res_raw
|
||||
|
||||
screen_all = (screen_raw + " " + res_raw).lower()
|
||||
for stype in ("oled", "retina", "ips", "tn", "va", "lcd", "led"):
|
||||
if stype in screen_all:
|
||||
data["screen_type"] = stype.upper()
|
||||
break
|
||||
|
||||
data["touchscreen"] = 1 if re.search(r"touch|pek", screen_raw, re.I) else 0
|
||||
|
||||
cpu_raw = s(r"^processor$")
|
||||
data["cpu_full"] = cpu_raw
|
||||
|
||||
cpu_model_m = re.match(
|
||||
r"((?:Intel\s+Core\s+\w[\w-]+|Intel\s+Core\s+Ultra\s+\w[\w-]+|"
|
||||
r"AMD\s+Ryzen\s+\d+[\s\w-]+?|Apple\s+M\d[\w\s]*?))\s+\d",
|
||||
cpu_raw,
|
||||
)
|
||||
if cpu_model_m:
|
||||
data["cpu_model"] = cpu_model_m.group(1).strip()
|
||||
else:
|
||||
cpu_model_m2 = re.match(r"([\w\s\-]+?)\s+\d+[.,]\d+\s*GHz", cpu_raw)
|
||||
if cpu_model_m2:
|
||||
data["cpu_model"] = cpu_model_m2.group(1).strip()
|
||||
|
||||
base_m = re.search(r"(\d+[.,]\d+)\s*GHz", cpu_raw, re.I)
|
||||
if base_m:
|
||||
data["cpu_base_ghz"] = float(base_m.group(1).replace(",", "."))
|
||||
|
||||
turbo_m = re.search(r"\((\d+[.,]\d+)\s*GHz\s*Turbo\)", cpu_raw, re.I)
|
||||
if turbo_m:
|
||||
data["cpu_turbo_ghz"] = float(turbo_m.group(1).replace(",", "."))
|
||||
|
||||
cores_raw = s(r"processorkärnor|cpu.?cores|antal.+kärn")
|
||||
cores_m = re.search(r"(\d+)", cores_raw)
|
||||
data["cpu_cores"] = int(cores_m.group(1)) if cores_m else None
|
||||
|
||||
data["cpu_cache"] = s(r"cacheminne|l[23].?cache|cache")
|
||||
|
||||
ht_raw = s(r"flertrådsteknik|hyperthread|smt")
|
||||
if ht_raw and re.search(r"stödjer|ja|yes|enabled", ht_raw, re.I):
|
||||
cores = data.get("cpu_cores")
|
||||
if cores:
|
||||
data["cpu_threads"] = cores * 2
|
||||
|
||||
for source in (cpu_raw, data.get("title", "")):
|
||||
gen_m = re.search(r"(\d{1,2})(?:th|st|nd|rd)\s*(?:gen)?", source, re.I)
|
||||
if gen_m:
|
||||
data["cpu_generation"] = f"{gen_m.group(1)}th"
|
||||
break
|
||||
apple_m = re.search(r"Apple\s+(M\d(?:\s+(?:Pro|Max|Ultra))?)", source, re.I)
|
||||
if apple_m:
|
||||
data["cpu_generation"] = apple_m.group(1)
|
||||
break
|
||||
|
||||
ram_raw = s(r"^minne$|^ram$|ram-minne|ram\s+minne")
|
||||
if not ram_raw:
|
||||
ram_raw = s(r"minne")
|
||||
|
||||
ram_m = re.search(r"(\d+)\s*GB", ram_raw, re.I)
|
||||
data["ram_gb"] = int(ram_m.group(1)) if ram_m else None
|
||||
|
||||
ram_type_m = re.search(r"(LPDDR\d+[A-Z]*|DDR\d+[A-Z]*)", ram_raw, re.I)
|
||||
data["ram_type"] = ram_type_m.group(1).upper() if ram_type_m else None
|
||||
|
||||
max_ram_raw = s(r"maximal.+minne|max.?ram")
|
||||
max_ram_m = re.search(r"(\d+)\s*GB", max_ram_raw, re.I)
|
||||
data["max_ram_gb"] = int(max_ram_m.group(1)) if max_ram_m else None
|
||||
|
||||
storage_raw = s(r"lagringsutrymme|hårddisk|storage")
|
||||
stor_m = re.search(r"(\d+)\s*(GB|TB)", storage_raw, re.I)
|
||||
if stor_m:
|
||||
val = int(stor_m.group(1))
|
||||
unit = stor_m.group(2).upper()
|
||||
data["storage_gb"] = val * 1024 if unit == "TB" else val
|
||||
if re.search(r"nvme", storage_raw, re.I):
|
||||
data["storage_type"] = "NVMe SSD"
|
||||
elif re.search(r"\bssd\b", storage_raw, re.I):
|
||||
data["storage_type"] = "SSD"
|
||||
elif re.search(r"emmc", storage_raw, re.I):
|
||||
data["storage_type"] = "eMMC"
|
||||
elif re.search(r"\bhdd\b", storage_raw, re.I):
|
||||
data["storage_type"] = "HDD"
|
||||
|
||||
data["gpu"] = s(r"grafikkort|gpu|graphics.?card")
|
||||
|
||||
wifi_raw = s(r"trådlöst|wi-?fi|wlan|wireless")
|
||||
data["wifi"] = wifi_raw
|
||||
|
||||
bt_raw = s(r"^bluetooth")
|
||||
data["bluetooth"] = bt_raw
|
||||
|
||||
eth_raw = s(r"trådat.+nät|ethernet|nätverksport|wired")
|
||||
if re.search(r"nej|saknar|no\b", eth_raw, re.I):
|
||||
data["wired_ethernet"] = 0
|
||||
elif eth_raw:
|
||||
data["wired_ethernet"] = 1
|
||||
else:
|
||||
data["wired_ethernet"] = 0
|
||||
|
||||
lte_spec = s(r"4g|5g|lte|mobilnät|modem")
|
||||
title_text = data.get("title", "")
|
||||
data["has_4g_5g"] = 1 if lte_spec or re.search(r"\b4G\b|\b5G\b|LTE", title_text) else 0
|
||||
|
||||
data["usb_ports"] = s(r"usb")
|
||||
data["thunderbolt_ports"] = s(r"thunderbolt")
|
||||
data["hdmi"] = s(r"^hdmi")
|
||||
|
||||
weight_raw = s(r"vikt|weight")
|
||||
wm = re.search(r"(\d+[.,]\d+)\s*kg", weight_raw, re.I)
|
||||
data["weight_kg"] = float(wm.group(1).replace(",", ".")) if wm else None
|
||||
|
||||
for dim, field in (
|
||||
(r"bredd|width", "width_cm"),
|
||||
(r"höjd|height|tjocklek", "height_cm"),
|
||||
(r"^djup|depth", "depth_cm"),
|
||||
):
|
||||
raw = s(dim)
|
||||
dm = re.search(r"(\d+[.,]\d*)\s*cm", raw, re.I)
|
||||
data[field] = float(dm.group(1).replace(",", ".")) if dm else None
|
||||
|
||||
data["battery"] = s(r"^batteri$|battery")
|
||||
|
||||
data["operating_system"] = s(r"operativsystem|os\b")
|
||||
|
||||
kb_raw = s(r"tangentbord|keyboard")
|
||||
data["keyboard_backlit"] = 1 if re.search(r"bakgrundsbelyst|backlit|backlight", kb_raw, re.I) else 0
|
||||
|
||||
webcam_raw = s(r"webbkamera|webcam|kamera")
|
||||
data["webcam"] = (
|
||||
0 if re.search(r"nej|saknar|no camera", webcam_raw, re.I)
|
||||
else (1 if webcam_raw else 0)
|
||||
)
|
||||
|
||||
opt_raw = s(r"optisk|optical|cd|dvd")
|
||||
data["optical_drive"] = (
|
||||
0 if (not opt_raw or re.search(r"nej|saknar|no\b", opt_raw, re.I))
|
||||
else 1
|
||||
)
|
||||
|
||||
data["warranty"] = s(r"garanti|warranty")
|
||||
data["eco_cert"] = s(r"energy.?star|epeat|miljöcert")
|
||||
|
||||
# ── Images ────────────────────────────────────────────────────────────
|
||||
# Thumbnail strip: each img.js-thumb carries data-image-large-src with
|
||||
# the full-resolution URL. Fall back to the main cover image if none found.
|
||||
large_urls: list[str] = []
|
||||
for thumb in soup.select("img.js-thumb[data-image-large-src]"):
|
||||
src = thumb.get("data-image-large-src", "").strip()
|
||||
if src and src not in large_urls:
|
||||
large_urls.append(src)
|
||||
if not large_urls:
|
||||
cover = soup.select_one("img.js-qv-product-cover")
|
||||
if cover and cover.get("src"):
|
||||
large_urls.append(cover["src"].strip())
|
||||
data["image_urls"] = json.dumps(large_urls) if large_urls else None
|
||||
|
||||
data["source"] = "billigteknik"
|
||||
data["scraped_at"] = datetime.now().isoformat()
|
||||
return data
|
||||
|
||||
|
||||
# ── Phase 1: collect product URLs ─────────────────────────────────────────────
|
||||
|
||||
def collect_urls(conn: sqlite3.Connection) -> set[str]:
|
||||
"""Crawl all listing pages, re-queue every live product, return the live URL set.
|
||||
|
||||
Uses INSERT OR REPLACE so existing queue entries have their done flag reset
|
||||
to 0 — guaranteeing that re-runs always re-scrape and update every listing.
|
||||
Returns the complete set of URLs currently live on the site so the caller
|
||||
can remove sold/delisted rows from the database.
|
||||
"""
|
||||
log.info("═══ Phase 1: Collecting product URLs ═══")
|
||||
page = 1
|
||||
live_urls: set[str] = set()
|
||||
empty_streak = 0
|
||||
|
||||
while page <= MAX_PAGES:
|
||||
url = CATEGORY_URL if page == 1 else f"{CATEGORY_URL}?page={page}"
|
||||
log.info(f"Listing page {page}: {url}")
|
||||
|
||||
resp = polite_get(url)
|
||||
if not resp:
|
||||
empty_streak += 1
|
||||
if empty_streak >= 3:
|
||||
log.error("3 consecutive failed listing pages – aborting Phase 1.")
|
||||
break
|
||||
page += 1
|
||||
_sleep()
|
||||
continue
|
||||
|
||||
products = parse_listing_page(resp.text)
|
||||
|
||||
if not products:
|
||||
log.info(f" No products found on page {page}.")
|
||||
empty_streak += 1
|
||||
if empty_streak >= 2:
|
||||
log.info(" Two empty pages in a row – end of catalog.")
|
||||
break
|
||||
else:
|
||||
empty_streak = 0
|
||||
for p in products:
|
||||
live_urls.add(p["url"])
|
||||
try:
|
||||
conn.execute(
|
||||
"""INSERT OR REPLACE INTO scrape_queue
|
||||
(url, source, title, price_sek, orig_price_sek,
|
||||
condition_grade, condition_detail, category, done)
|
||||
VALUES (?, 'billigteknik', ?, ?, ?, ?, ?, ?, 0)""",
|
||||
(
|
||||
p["url"], p.get("title"),
|
||||
p.get("price_sek"), p.get("orig_price_sek"),
|
||||
p.get("condition_grade"), p.get("condition_detail"),
|
||||
p.get("category"),
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
log.debug(f" Queue insert error: {exc}")
|
||||
conn.commit()
|
||||
log.info(f" {len(products)} products on page (running total: {len(live_urls)})")
|
||||
|
||||
page += 1
|
||||
_sleep()
|
||||
|
||||
queued = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=0").fetchone()[0]
|
||||
log.info(f"Phase 1 done. {queued} URLs queued for scraping. {len(live_urls)} live URLs found.")
|
||||
return live_urls
|
||||
|
||||
|
||||
# ── Phase 2: scrape product detail pages ──────────────────────────────────────
|
||||
|
||||
def scrape_products(conn: sqlite3.Connection) -> None:
|
||||
"""Work through scrape_queue, fetching and storing each product page."""
|
||||
log.info("═══ Phase 2: Scraping product detail pages ═══")
|
||||
|
||||
queue = conn.execute(
|
||||
"""SELECT url, title, price_sek, orig_price_sek,
|
||||
condition_grade, condition_detail, category
|
||||
FROM scrape_queue WHERE source='billigteknik' AND done=0 ORDER BY rowid"""
|
||||
).fetchall()
|
||||
|
||||
total = len(queue)
|
||||
log.info(f"{total} products to scrape.")
|
||||
|
||||
for i, row in enumerate(queue, 1):
|
||||
url = row["url"]
|
||||
log.info(f"[{i}/{total}] {url}")
|
||||
|
||||
resp = polite_get(url)
|
||||
if not resp:
|
||||
conn.execute("UPDATE scrape_queue SET done=-1 WHERE url=?", (url,))
|
||||
conn.commit()
|
||||
_sleep()
|
||||
continue
|
||||
|
||||
try:
|
||||
detail = parse_product_page(resp.text, url)
|
||||
|
||||
if row["price_sek"]:
|
||||
detail["price_sek"] = row["price_sek"]
|
||||
if row["orig_price_sek"]:
|
||||
detail["original_price_sek"] = row["orig_price_sek"]
|
||||
if row["condition_grade"]:
|
||||
detail["condition_grade"] = row["condition_grade"]
|
||||
if row["condition_detail"] is not None and row["condition_detail"] != "":
|
||||
detail["condition_detail"] = row["condition_detail"]
|
||||
if row["category"]:
|
||||
detail["category"] = row["category"]
|
||||
|
||||
if not detail.get("title") and row["title"]:
|
||||
detail["title"] = _clean_title(row["title"])
|
||||
|
||||
conn.execute(INSERT_SQL, _safe_row(detail))
|
||||
conn.execute("UPDATE scrape_queue SET done=1 WHERE url=?", (url,))
|
||||
conn.commit()
|
||||
log.info(f" ✓ {detail.get('title', '(no title)')}")
|
||||
|
||||
except Exception as exc:
|
||||
log.error(f" Error processing {url}: {exc}", exc_info=True)
|
||||
conn.execute("UPDATE scrape_queue SET done=-1 WHERE url=?", (url,))
|
||||
conn.commit()
|
||||
|
||||
_sleep()
|
||||
|
||||
done = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=1").fetchone()[0]
|
||||
failed = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=-1").fetchone()[0]
|
||||
log.info(f"Phase 2 done. ✓ {done} scraped ✗ {failed} failed.")
|
||||
|
||||
|
||||
# ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
def run(conn: sqlite3.Connection, skip_phase1: bool = False) -> None:
|
||||
"""Run the BilligTeknik scrape (Phase 1 + Phase 2 + sold-item cleanup)."""
|
||||
live_urls: set[str] = set()
|
||||
|
||||
if skip_phase1:
|
||||
queued = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=0").fetchone()[0]
|
||||
log.info(f"--skip-phase1: {queued} pending URLs in queue.")
|
||||
else:
|
||||
live_urls = collect_urls(conn)
|
||||
|
||||
scrape_products(conn)
|
||||
|
||||
# Remove any DB rows that are no longer listed on the site (sold / removed).
|
||||
if live_urls:
|
||||
db_urls = set(
|
||||
r[0] for r in conn.execute(
|
||||
"SELECT url FROM laptops WHERE source='billigteknik'"
|
||||
).fetchall()
|
||||
)
|
||||
sold = db_urls - live_urls
|
||||
if sold:
|
||||
for sold_url in sold:
|
||||
conn.execute("DELETE FROM laptops WHERE url=?", (sold_url,))
|
||||
conn.commit()
|
||||
log.info(f" Removed {len(sold)} sold/unlisted product(s) from DB.")
|
||||
else:
|
||||
log.info(" No sold/unlisted products found.")
|
||||
|
||||
scraped = conn.execute("SELECT COUNT(*) FROM laptops WHERE source='billigteknik'").fetchone()[0]
|
||||
log.info(f"BilligTeknik done. {scraped} laptops in DB.")
|
||||
758
scrapers/scraper_cpu.py
Normal file
758
scrapers/scraper_cpu.py
Normal file
@@ -0,0 +1,758 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
CPU enricher (Playwright / Firefox)
|
||||
======================================
|
||||
Looks up detailed CPU specs for every cpu_full value found in the laptops
|
||||
table and stores results in the cpu_specs table.
|
||||
|
||||
Sources:
|
||||
Intel – intel.com product spec pages, found via Google search (Playwright)
|
||||
Apple – built-in static data (M1, M1 Pro, M2)
|
||||
AMD – currently skipped (spec pages are bot-protected without a solution)
|
||||
|
||||
Usage (from main.py web trigger, or standalone):
|
||||
from scrapers.scraper_cpu import run
|
||||
conn = init_db(DB_PATH)
|
||||
run(conn, reset=False)
|
||||
"""
|
||||
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
import random
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
from urllib.parse import quote as url_quote
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from .shared import log, SCHEMA
|
||||
|
||||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
INTEL_BASE = "https://www.intel.com"
|
||||
GOOGLE_SEARCH = "https://www.google.com/search?q={query}&hl=en"
|
||||
|
||||
CPU_DELAY_MIN = 2.0
|
||||
CPU_DELAY_MAX = 5.0
|
||||
|
||||
# Persistent Firefox profile for Playwright – cookies/cache survive between
|
||||
# runs so the browser looks like a returning user, not a fresh automation session.
|
||||
BROWSER_DATA_DIR = os.path.normpath(
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "browser_data")
|
||||
)
|
||||
|
||||
# ── Playwright (deferred import) ──────────────────────────────────────────────
|
||||
|
||||
|
||||
def _import_playwright():
|
||||
from playwright.sync_api import sync_playwright, BrowserContext
|
||||
return sync_playwright, BrowserContext
|
||||
|
||||
|
||||
# ── cpu_specs INSERT helpers ──────────────────────────────────────────────────
|
||||
|
||||
_CPU_COLS = [
|
||||
"cpu_key", "cpu_model_raw", "vendor", "full_name", "brand_name",
|
||||
"codename", "lithography", "launch_date",
|
||||
"total_cores", "performance_cores", "efficient_cores", "total_threads",
|
||||
"base_freq_ghz", "max_turbo_ghz",
|
||||
"l2_cache", "l3_cache",
|
||||
"tdp_w", "max_tdp_w",
|
||||
"pcie_version", "pcie_lanes",
|
||||
"max_ram_gb", "ram_types", "ram_speeds_mhz", "ecc_support",
|
||||
"igpu_name", "igpu_base_mhz", "igpu_boost_mhz",
|
||||
"source_url", "lookup_status", "scraped_at",
|
||||
]
|
||||
|
||||
CPU_SPECS_INSERT = """
|
||||
INSERT OR REPLACE INTO cpu_specs (
|
||||
cpu_key, cpu_model_raw, vendor, full_name, brand_name,
|
||||
codename, lithography, launch_date,
|
||||
total_cores, performance_cores, efficient_cores, total_threads,
|
||||
base_freq_ghz, max_turbo_ghz,
|
||||
l2_cache, l3_cache,
|
||||
tdp_w, max_tdp_w,
|
||||
pcie_version, pcie_lanes,
|
||||
max_ram_gb, ram_types, ram_speeds_mhz, ecc_support,
|
||||
igpu_name, igpu_base_mhz, igpu_boost_mhz,
|
||||
source_url, lookup_status, scraped_at
|
||||
) VALUES (
|
||||
:cpu_key, :cpu_model_raw, :vendor, :full_name, :brand_name,
|
||||
:codename, :lithography, :launch_date,
|
||||
:total_cores, :performance_cores, :efficient_cores, :total_threads,
|
||||
:base_freq_ghz, :max_turbo_ghz,
|
||||
:l2_cache, :l3_cache,
|
||||
:tdp_w, :max_tdp_w,
|
||||
:pcie_version, :pcie_lanes,
|
||||
:max_ram_gb, :ram_types, :ram_speeds_mhz, :ecc_support,
|
||||
:igpu_name, :igpu_base_mhz, :igpu_boost_mhz,
|
||||
:source_url, :lookup_status, :scraped_at
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def _safe_cpu_row(data: dict) -> dict:
|
||||
"""Ensure every expected cpu_specs column key exists (default None)."""
|
||||
return {col: data.get(col) for col in _CPU_COLS}
|
||||
|
||||
|
||||
# ── CPU string normalisation ──────────────────────────────────────────────────
|
||||
|
||||
def normalize_cpu_key(cpu_raw: str) -> str:
|
||||
"""Return a stable lowercase dedup key from any CPU model string."""
|
||||
s = cpu_raw.strip()
|
||||
s = re.sub(r'\s+\(?\d+(?:[.,]\d+)?\s*GHz.*', '', s, flags=re.I).strip()
|
||||
s = re.sub(r'\s+\d+-Core\b.*', '', s, flags=re.I).strip()
|
||||
s = re.sub(r'\s+\d+P\s+[A-Z]\d+\b.*', '', s, flags=re.I).strip()
|
||||
for prefix_pattern in (
|
||||
r'^Intel\s+Core\s+',
|
||||
r'^Intel\s+',
|
||||
r'^AMD\s+',
|
||||
):
|
||||
s = re.sub(prefix_pattern, '', s, flags=re.I).strip()
|
||||
s = re.sub(r'\s+', '-', s)
|
||||
return s.lower()
|
||||
|
||||
|
||||
def detect_vendor(cpu_raw: str) -> str:
|
||||
if re.search(r'\bApple\b', cpu_raw, re.I) or re.search(r'\bM[123]\b', cpu_raw):
|
||||
return "Apple"
|
||||
if re.search(r'\bAMD\b|\bRyzen\b|\bAthlon\b', cpu_raw, re.I):
|
||||
return "AMD"
|
||||
return "Intel"
|
||||
|
||||
|
||||
def _intel_search_term(cpu_raw: str) -> str:
|
||||
# Safety net: truncate any marketing copy that starts after a colon.
|
||||
s = cpu_raw.split(":")[0].strip()
|
||||
# Strip trailing "-processor" / "-chip" labels.
|
||||
s = re.sub(r'[-\s]*(processor|cpu|chip)$', '', s, flags=re.I).strip()
|
||||
# Drop speed/turbo info — only the model number is needed for the search.
|
||||
s = re.sub(r'\s+\(?\d+(?:[.,]\d+)?\s*GHz.*', '', s, flags=re.I).strip()
|
||||
s = re.sub(r'\s+\d+P\s+[A-Z]\d+\b.*', '', s, flags=re.I).strip()
|
||||
s = re.sub(r'^Intel\s+', '', s, flags=re.I).strip()
|
||||
return s
|
||||
|
||||
|
||||
# ── Playwright (Firefox) for JS-heavy / bot-protected pages ──────────────────
|
||||
|
||||
_pw_instance = None
|
||||
_pw_context = None # type: ignore[assignment]
|
||||
|
||||
|
||||
def _get_pw_context():
|
||||
"""Lazy-init a persistent Firefox context stored in browser_data/."""
|
||||
global _pw_instance, _pw_context
|
||||
if _pw_context is None:
|
||||
sync_playwright, _BrowserContext = _import_playwright()
|
||||
os.makedirs(BROWSER_DATA_DIR, exist_ok=True)
|
||||
_pw_instance = sync_playwright().__enter__()
|
||||
_pw_context = _pw_instance.firefox.launch_persistent_context(
|
||||
user_data_dir=BROWSER_DATA_DIR,
|
||||
headless=False,
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0",
|
||||
viewport={"width": 1280, "height": 900},
|
||||
locale="en-US",
|
||||
timezone_id="America/New_York",
|
||||
firefox_user_prefs={
|
||||
"dom.webdriver.enabled": False,
|
||||
"useAutomationExtension": False,
|
||||
"privacy.resistFingerprinting": False,
|
||||
"media.navigator.enabled": True,
|
||||
"geo.enabled": False,
|
||||
"network.http.referer.sendRefererHeader": 2,
|
||||
},
|
||||
)
|
||||
return _pw_context
|
||||
|
||||
|
||||
def playwright_get(url: str, retries: int = 3) -> Optional[str]:
|
||||
"""Fetch a page using Firefox (Playwright) with retry and back-off logic."""
|
||||
for attempt in range(1, retries + 1):
|
||||
page = None
|
||||
try:
|
||||
ctx = _get_pw_context()
|
||||
page = ctx.new_page()
|
||||
response = page.goto(url, wait_until="networkidle", timeout=30_000)
|
||||
status = response.status if response else 200
|
||||
|
||||
if status == 200:
|
||||
return page.content()
|
||||
|
||||
if status == 429:
|
||||
wait = 60 + random.uniform(10, 30)
|
||||
log.warning(f" Rate-limited (429), sleeping {wait:.0f}s (attempt {attempt}/{retries})")
|
||||
time.sleep(wait)
|
||||
elif status == 502:
|
||||
wait = 180 + random.uniform(0, 20) if attempt == 1 else random.uniform(2, 3)
|
||||
log.warning(f" HTTP 502 – backing off {wait:.0f}s (attempt {attempt}/{retries})")
|
||||
time.sleep(wait)
|
||||
elif status in (404, 410):
|
||||
log.info(f" Not found ({status}): {url}")
|
||||
return None
|
||||
else:
|
||||
log.warning(f" HTTP {status} (attempt {attempt}/{retries}): {url}")
|
||||
time.sleep(5 * attempt)
|
||||
|
||||
except Exception as exc:
|
||||
log.warning(f" Playwright error attempt {attempt}/{retries} for {url}: {exc}")
|
||||
time.sleep(10 * attempt)
|
||||
finally:
|
||||
if page:
|
||||
try:
|
||||
page.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
log.error(f" Gave up after {retries} attempts: {url}")
|
||||
return None
|
||||
|
||||
|
||||
def close_playwright() -> None:
|
||||
global _pw_instance, _pw_context
|
||||
if _pw_context:
|
||||
try:
|
||||
_pw_context.close()
|
||||
except Exception:
|
||||
pass
|
||||
if _pw_instance:
|
||||
try:
|
||||
_pw_instance.__exit__(None, None, None)
|
||||
except Exception:
|
||||
pass
|
||||
_pw_instance = _pw_context = None
|
||||
|
||||
|
||||
def _cpu_sleep():
|
||||
time.sleep(random.uniform(CPU_DELAY_MIN, CPU_DELAY_MAX))
|
||||
|
||||
|
||||
# ── Shared parsing utilities ──────────────────────────────────────────────────
|
||||
|
||||
def _parse_freq_ghz(s: str) -> Optional[float]:
|
||||
m = re.search(r'(\d+(?:[.,]\d+)?)\s*GHz', s, re.I)
|
||||
return float(m.group(1).replace(',', '.')) if m else None
|
||||
|
||||
|
||||
def _parse_int(s: str) -> Optional[int]:
|
||||
m = re.search(r'(\d+)', s.replace(',', ''))
|
||||
return int(m.group(1)) if m else None
|
||||
|
||||
|
||||
def _spec_lookup(specs: dict, pattern: str) -> str:
|
||||
rx = re.compile(pattern, re.I)
|
||||
for k, v in specs.items():
|
||||
if rx.search(k):
|
||||
return v
|
||||
return ""
|
||||
|
||||
|
||||
# ── Intel (intel.com product spec pages) ─────────────────────────────────────
|
||||
|
||||
def _intel_find_product_url(html: str) -> Optional[str]:
|
||||
"""Scan a rendered page for the first Intel product spec link."""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
|
||||
def _clean(href: str) -> str:
|
||||
m = re.search(r'/url\?q=(https://www\.intel\.com[^&]+)', href)
|
||||
if m:
|
||||
href = m.group(1)
|
||||
if href.startswith("http"):
|
||||
return href
|
||||
return INTEL_BASE + href
|
||||
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = a["href"]
|
||||
if "/products/sku/" in href and "specifications" in href:
|
||||
return _clean(href)
|
||||
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = a["href"]
|
||||
if "/products/sku/" in href and ".html" in href:
|
||||
cleaned = _clean(href)
|
||||
base = re.sub(r'/(?:ordering|compatible|downloads|support)\.html$', '', cleaned)
|
||||
if not base.endswith("specifications.html"):
|
||||
base = re.sub(r'\.html$', '/specifications.html', base)
|
||||
return base
|
||||
|
||||
for a in soup.find_all("a", href=True):
|
||||
href = a["href"]
|
||||
if "/products/sku/" in href:
|
||||
cleaned = _clean(href)
|
||||
if not cleaned.endswith("specifications.html"):
|
||||
cleaned = cleaned.rstrip("/") + "/specifications.html"
|
||||
return cleaned
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def _intel_parse_specs(html: str, url: str) -> dict:
|
||||
"""Parse an intel.com /products/sku/.../specifications.html page."""
|
||||
data: dict = {"source_url": url}
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
|
||||
h1 = soup.select_one("h1")
|
||||
if h1:
|
||||
data["full_name"] = h1.get_text(strip=True)
|
||||
|
||||
specs: dict[str, str] = {}
|
||||
|
||||
for row in soup.select(
|
||||
".tech-section-row, .specs-list-item, li.tech-section-item, "
|
||||
".blade-content li, [class*='spec-row']"
|
||||
):
|
||||
lbl = row.select_one(".tech-label, .label, span.label")
|
||||
val = row.select_one(".tech-data, .value, span.value")
|
||||
if lbl and val:
|
||||
k = lbl.get_text(strip=True).rstrip(":")
|
||||
v = val.get_text(" ", strip=True)
|
||||
if k and v and len(k) < 80:
|
||||
specs[k] = v
|
||||
|
||||
if len(specs) < 5:
|
||||
for dl in soup.select("dl"):
|
||||
dts = dl.select("dt")
|
||||
dds = dl.select("dd")
|
||||
for dt, dd in zip(dts, dds):
|
||||
k = dt.get_text(strip=True).rstrip(":")
|
||||
v = dd.get_text(" ", strip=True)
|
||||
if k and v and len(k) < 80:
|
||||
specs[k] = v
|
||||
|
||||
if len(specs) < 5:
|
||||
_INTEL_KNOWN_LABELS = [
|
||||
"Processor Number", "Product Collection", "Code Name", "Lithography",
|
||||
"Launch Date", "Vertical Segment", "Marketing Status",
|
||||
"Total Cores", "Total Threads", "Performance-cores", "Efficient-cores",
|
||||
"Processor Base Frequency", "Max Turbo Frequency",
|
||||
"Intel Thermal Velocity Boost Frequency",
|
||||
"Cache", "TDP", "Configurable TDP-down", "Max Turbo Power",
|
||||
"Bus Speed",
|
||||
"Max Memory Size", "Memory Types", "Max # of Memory Channels",
|
||||
"Max Memory Bandwidth", "ECC Memory Supported",
|
||||
"GPU Name", "Graphics Base Frequency", "Graphics Max Dynamic Frequency",
|
||||
"Graphics Burst Frequency",
|
||||
"PCI Express Revision", "Max # of PCI Express Lanes",
|
||||
"PCI Express Configurations",
|
||||
]
|
||||
label_rx = re.compile(
|
||||
r'(' + '|'.join(re.escape(l) for l in _INTEL_KNOWN_LABELS) + r')',
|
||||
re.I
|
||||
)
|
||||
for section in soup.select("section, div[class*='section'], div[class*='specs']"):
|
||||
text = section.get_text(" ", strip=True)
|
||||
parts = label_rx.split(text)
|
||||
for j in range(1, len(parts) - 1, 2):
|
||||
k = parts[j].strip()
|
||||
v = parts[j + 1].strip().split(" ")[0].strip()
|
||||
if k and v and len(v) < 200:
|
||||
specs.setdefault(k, v)
|
||||
|
||||
if len(specs) < 5:
|
||||
for table in soup.select("table"):
|
||||
for tr in table.select("tr"):
|
||||
cells = tr.select("td, th")
|
||||
if len(cells) >= 2:
|
||||
k = cells[0].get_text(strip=True).rstrip(":")
|
||||
v = cells[1].get_text(" ", strip=True)
|
||||
if k and v and len(k) < 80:
|
||||
specs.setdefault(k, v)
|
||||
|
||||
if specs:
|
||||
_ark_map(specs, data)
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def _ark_map(specs: dict, data: dict) -> None:
|
||||
"""Map flat Intel ARK spec key→value pairs to our cpu_specs columns."""
|
||||
s = lambda pattern: _spec_lookup(specs, pattern)
|
||||
|
||||
cn = s(r'code.?name|product.?collection')
|
||||
if cn:
|
||||
data["codename"] = cn
|
||||
lit = s(r'lithograph|process.?technolog|manufactur')
|
||||
if lit:
|
||||
data["lithography"] = lit
|
||||
ld = s(r'launch.?date|announced|release')
|
||||
if ld:
|
||||
data["launch_date"] = ld
|
||||
pn = s(r'processor.?number|brand.?name')
|
||||
if pn:
|
||||
data.setdefault("brand_name", pn)
|
||||
|
||||
tc = s(r'#\s*of\s*cores|total\s*cores')
|
||||
if tc:
|
||||
data["total_cores"] = _parse_int(tc)
|
||||
pc = s(r'performance.?core|p.?core')
|
||||
if pc:
|
||||
data["performance_cores"] = _parse_int(pc)
|
||||
ec = s(r'efficient.?core|e.?core')
|
||||
if ec:
|
||||
data["efficient_cores"] = _parse_int(ec)
|
||||
tt = s(r'#\s*of\s*threads|total\s*threads')
|
||||
if tt:
|
||||
data["total_threads"] = _parse_int(tt)
|
||||
|
||||
base = s(r'processor\s+base\s+freq|base\s+freq|base\s+clock')
|
||||
if base:
|
||||
data["base_freq_ghz"] = _parse_freq_ghz(base)
|
||||
turbo = s(r'max\s*turbo\s*freq|max\s*boost|turbo\s*freq')
|
||||
if turbo:
|
||||
data["max_turbo_ghz"] = _parse_freq_ghz(turbo)
|
||||
|
||||
l3 = s(r'l3\s*cache|total.*l3|smart\s*cache|^cache$')
|
||||
if l3:
|
||||
data["l3_cache"] = l3
|
||||
l2 = s(r'l2\s*cache|total.*l2')
|
||||
if l2:
|
||||
data["l2_cache"] = l2
|
||||
|
||||
tdp = s(r'^TDP$|^PBP$|base\s*power|thermal\s*design\s*power')
|
||||
if tdp:
|
||||
m = re.search(r'(\d+)', tdp)
|
||||
if m:
|
||||
data["tdp_w"] = int(m.group(1))
|
||||
max_tdp = s(r'max\s*turbo\s*power|^MTP$|maximum\s*turbo\s*power|configurable\s*tdp.?up')
|
||||
if max_tdp:
|
||||
m = re.search(r'(\d+)', max_tdp)
|
||||
if m:
|
||||
data["max_tdp_w"] = int(m.group(1))
|
||||
|
||||
pcie_ver = s(r'pci\s*express\s*revision|pcie\s*version|pci\s*express\s*version')
|
||||
if pcie_ver:
|
||||
m = re.search(r'(\d+(?:\.\d+)?)', pcie_ver)
|
||||
if m:
|
||||
data["pcie_version"] = m.group(1)
|
||||
pcie_l = s(r'max.+pci.+lanes|pci.+express.+lanes')
|
||||
if pcie_l:
|
||||
m = re.search(r'(\d+)', pcie_l)
|
||||
if m:
|
||||
data["pcie_lanes"] = int(m.group(1))
|
||||
|
||||
max_mem = s(r'max\s*memory\s*size|maximum\s*memory')
|
||||
if max_mem:
|
||||
m = re.search(r'(\d+)\s*GB', max_mem, re.I)
|
||||
if m:
|
||||
data["max_ram_gb"] = int(m.group(1))
|
||||
ram_t = s(r'memory\s*types?\b|supported\s*memory')
|
||||
if ram_t:
|
||||
data["ram_types"] = ram_t
|
||||
ram_spd = s(r'memory\s*speed|memory.*frequency')
|
||||
if ram_spd:
|
||||
data["ram_speeds_mhz"] = ram_spd
|
||||
ecc = s(r'ecc\s*memory|ecc\s*support')
|
||||
if ecc:
|
||||
data["ecc_support"] = 1 if re.search(r'yes|supported|✓', ecc, re.I) else 0
|
||||
|
||||
igpu = s(r'processor\s*graphics?\b|integrated\s*graphics?')
|
||||
if igpu:
|
||||
data["igpu_name"] = igpu
|
||||
igpu_base = s(r'graphics\s*base\s*freq|gpu\s*base\s*freq')
|
||||
if igpu_base:
|
||||
m = re.search(r'(\d+)\s*MHz', igpu_base, re.I)
|
||||
if m:
|
||||
data["igpu_base_mhz"] = int(m.group(1))
|
||||
igpu_boost = s(r'graphics.*dynamic.*freq|graphics.*max.*freq|gpu.*max.*freq')
|
||||
if igpu_boost:
|
||||
m = re.search(r'(\d+[.,]\d*)\s*GHz', igpu_boost, re.I)
|
||||
if m:
|
||||
data["igpu_boost_mhz"] = int(float(m.group(1).replace(',', '.')) * 1000)
|
||||
else:
|
||||
m2 = re.search(r'(\d+)\s*MHz', igpu_boost, re.I)
|
||||
if m2:
|
||||
data["igpu_boost_mhz"] = int(m2.group(1))
|
||||
|
||||
|
||||
def _is_captcha_page(page) -> bool:
|
||||
"""Detect if the current page is a Google CAPTCHA / unusual-traffic page."""
|
||||
try:
|
||||
url = page.url
|
||||
if "sorry/" in url or "sorry.google.com" in url:
|
||||
return True
|
||||
content = page.content()
|
||||
if re.search(r'unusual.{0,40}traffic|g-recaptcha|recaptcha|captcha', content, re.I):
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
|
||||
def _google_find_intel_url(term: str) -> Optional[str]:
|
||||
"""Search Google for the Intel product spec page URL via Playwright.
|
||||
|
||||
If Google returns a CAPTCHA or 429, the browser window (already non-headless)
|
||||
stays open and we wait up to 5 minutes for the user to solve it manually.
|
||||
"""
|
||||
# Strip parenthetical annotations like "(4 kärnor / 8 trådar)" that some
|
||||
# retailers append to CPU names — they break the Google site: search.
|
||||
clean_term = re.sub(r'\s*\([^)]*\)', '', term).strip()
|
||||
query = url_quote(f"Intel {clean_term} site:intel.com/content/www/us/en/products/sku")
|
||||
search_url = GOOGLE_SEARCH.format(query=query)
|
||||
log.info(f" → Google: {search_url}")
|
||||
|
||||
page = None
|
||||
html = None
|
||||
try:
|
||||
ctx = _get_pw_context()
|
||||
page = ctx.new_page()
|
||||
response = page.goto(search_url, wait_until="networkidle", timeout=30_000)
|
||||
status = response.status if response else 200
|
||||
|
||||
# Google sometimes returns 429 directly; more often it redirects to
|
||||
# /sorry/... with a 200 status that contains a CAPTCHA form.
|
||||
if status == 429 or _is_captcha_page(page):
|
||||
log.warning("=" * 60)
|
||||
log.warning(" ⚠ Google CAPTCHA / rate-limit detected!")
|
||||
log.warning(" Search term: '%s'", term)
|
||||
log.warning(" The browser window is open — please solve the CAPTCHA manually.")
|
||||
log.warning(" Waiting up to 5 minutes for you to reach the search results…")
|
||||
log.warning("=" * 60)
|
||||
try:
|
||||
page.wait_for_selector("#search a, #rso a", timeout=300_000)
|
||||
log.info(" ✓ CAPTCHA solved — resuming.")
|
||||
except Exception:
|
||||
log.error(
|
||||
" Timed out (5 min) waiting for CAPTCHA solution for '%s'. Giving up.", term
|
||||
)
|
||||
return None
|
||||
|
||||
elif status not in (200, 301, 302):
|
||||
log.warning(f" Google returned HTTP {status}")
|
||||
return None
|
||||
|
||||
else:
|
||||
# Normal page — just wait a moment for JS results to settle
|
||||
try:
|
||||
page.wait_for_selector("#search a, #rso a", timeout=10_000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
html = page.content()
|
||||
|
||||
except Exception as exc:
|
||||
log.warning(f" Google search error for '{term}': {exc}")
|
||||
return None
|
||||
finally:
|
||||
if page:
|
||||
try:
|
||||
page.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _intel_find_product_url(html)
|
||||
|
||||
|
||||
def lookup_intel(cpu_raw: str) -> dict:
|
||||
"""Fetch Intel product specs from intel.com via Google → Playwright."""
|
||||
term = _intel_search_term(cpu_raw)
|
||||
log.info(f" → Intel lookup via Google: '{term}'")
|
||||
|
||||
product_url = _google_find_intel_url(term)
|
||||
_cpu_sleep()
|
||||
|
||||
if not product_url:
|
||||
log.warning(f" No intel.com product URL found for '{term}'")
|
||||
return {"lookup_status": "failed"}
|
||||
|
||||
if "/products/sku/" in product_url and not product_url.endswith("specifications.html"):
|
||||
product_url = re.sub(
|
||||
r'/(?:ordering|compatible|downloads|support)(\.html)$', '', product_url
|
||||
)
|
||||
if not product_url.endswith(".html"):
|
||||
product_url += "/specifications.html"
|
||||
else:
|
||||
product_url = product_url.replace(".html", "/specifications.html")
|
||||
|
||||
log.info(f" → Intel product: {product_url}")
|
||||
html = playwright_get(product_url)
|
||||
_cpu_sleep()
|
||||
if not html:
|
||||
return {"lookup_status": "failed"}
|
||||
|
||||
result = _intel_parse_specs(html, product_url)
|
||||
result["lookup_status"] = (
|
||||
"ok" if result.get("total_cores") or result.get("tdp_w") or result.get("full_name")
|
||||
else "failed"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# ── Apple Silicon (static data) ───────────────────────────────────────────────
|
||||
|
||||
_APPLE_STATIC: dict[str, dict] = {
|
||||
"apple-m1": {
|
||||
"full_name": "Apple M1", "vendor": "Apple",
|
||||
"codename": "Firestorm / Icestorm (M1)", "lithography": "5nm (TSMC N5)",
|
||||
"launch_date": "2020-11", "total_cores": 8, "performance_cores": 4,
|
||||
"efficient_cores": 4, "total_threads": 8, "base_freq_ghz": 3.2,
|
||||
"max_turbo_ghz": 3.2, "l2_cache": "12 MB (P-core), 4 MB (E-core)",
|
||||
"l3_cache": "N/A (Unified Memory Architecture)", "tdp_w": 15, "max_tdp_w": 20,
|
||||
"pcie_version": "4.0", "pcie_lanes": None, "max_ram_gb": 16,
|
||||
"ram_types": "LPDDR4X (Unified Memory)", "ram_speeds_mhz": "4266",
|
||||
"ecc_support": 0, "igpu_name": "Apple M1 8-core GPU",
|
||||
"igpu_base_mhz": None, "igpu_boost_mhz": None,
|
||||
"source_url": "https://www.apple.com/newsroom/2020/11/apple-unleashes-m1/",
|
||||
"lookup_status": "ok",
|
||||
},
|
||||
"apple-m1-pro": {
|
||||
"full_name": "Apple M1 Pro", "vendor": "Apple",
|
||||
"codename": "Avalanche / Blizzard (M1 Pro)", "lithography": "5nm (TSMC N5P)",
|
||||
"launch_date": "2021-10", "total_cores": 10, "performance_cores": 8,
|
||||
"efficient_cores": 2, "total_threads": 10, "base_freq_ghz": 3.22,
|
||||
"max_turbo_ghz": 3.22, "l2_cache": "28 MB (P-core), 4 MB (E-core)",
|
||||
"l3_cache": "N/A (Unified Memory Architecture)", "tdp_w": 30, "max_tdp_w": 60,
|
||||
"pcie_version": "4.0", "pcie_lanes": None, "max_ram_gb": 32,
|
||||
"ram_types": "LPDDR5 (Unified Memory)", "ram_speeds_mhz": "6400",
|
||||
"ecc_support": 0, "igpu_name": "Apple M1 Pro 16-core GPU",
|
||||
"igpu_base_mhz": None, "igpu_boost_mhz": None,
|
||||
"source_url": "https://www.apple.com/newsroom/2021/10/apple-unveils-m1-pro-and-m1-max-supercharged-for-pros/",
|
||||
"lookup_status": "ok",
|
||||
},
|
||||
"apple-m2": {
|
||||
"full_name": "Apple M2", "vendor": "Apple",
|
||||
"codename": "Everest / Sawtooth (M2)", "lithography": "5nm 2nd gen (TSMC N5P)",
|
||||
"launch_date": "2022-06", "total_cores": 8, "performance_cores": 4,
|
||||
"efficient_cores": 4, "total_threads": 8, "base_freq_ghz": 3.49,
|
||||
"max_turbo_ghz": 3.49, "l2_cache": "16 MB (P-core), 4 MB (E-core)",
|
||||
"l3_cache": "N/A (Unified Memory Architecture)", "tdp_w": 15, "max_tdp_w": 20,
|
||||
"pcie_version": "4.0", "pcie_lanes": None, "max_ram_gb": 24,
|
||||
"ram_types": "LPDDR5 (Unified Memory)", "ram_speeds_mhz": "6400",
|
||||
"ecc_support": 0, "igpu_name": "Apple M2 10-core GPU",
|
||||
"igpu_base_mhz": None, "igpu_boost_mhz": None,
|
||||
"source_url": "https://www.apple.com/newsroom/2022/06/apple-unveils-m2-with-breakthrough-performance-and-capabilities/",
|
||||
"lookup_status": "ok",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def lookup_apple(cpu_key: str) -> dict:
|
||||
result = _APPLE_STATIC.get(cpu_key)
|
||||
if result:
|
||||
log.info(f" → Apple: using static data for '{cpu_key}'")
|
||||
return dict(result)
|
||||
log.warning(f" No static data for Apple key '{cpu_key}'")
|
||||
return {"lookup_status": "failed"}
|
||||
|
||||
|
||||
# ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
def run(conn: sqlite3.Connection, reset: bool = False) -> None:
|
||||
"""Look up detailed specs for all CPU models found in the laptops table."""
|
||||
if reset:
|
||||
conn.execute("DROP TABLE IF EXISTS cpu_specs")
|
||||
conn.executescript(SCHEMA)
|
||||
conn.commit()
|
||||
log.info("cpu_specs table reset.")
|
||||
|
||||
log.info("═══ CPU Lookup – starting ═══")
|
||||
|
||||
rows = conn.execute(
|
||||
"SELECT DISTINCT cpu_full FROM laptops "
|
||||
"WHERE cpu_full IS NOT NULL AND cpu_full != '' "
|
||||
"ORDER BY cpu_full"
|
||||
).fetchall()
|
||||
|
||||
if not rows:
|
||||
log.warning("No cpu_full values found in laptops table. "
|
||||
"Has the main scraper finished Phase 2 yet?")
|
||||
return
|
||||
|
||||
key_to_raws: dict[str, list[str]] = {}
|
||||
for row in rows:
|
||||
raw = row["cpu_full"].strip()
|
||||
key = normalize_cpu_key(raw)
|
||||
key_to_raws.setdefault(key, []).append(raw)
|
||||
|
||||
log.info(
|
||||
f"Found {len(rows)} distinct cpu_full strings "
|
||||
f"→ {len(key_to_raws)} unique CPU keys after normalisation"
|
||||
)
|
||||
|
||||
for key, raws in sorted(key_to_raws.items()):
|
||||
if len(raws) > 1:
|
||||
log.info(f" Merged {len(raws)} strings → '{key}': {raws}")
|
||||
|
||||
done_keys = {
|
||||
r["cpu_key"]
|
||||
for r in conn.execute(
|
||||
"SELECT cpu_key FROM cpu_specs WHERE lookup_status = 'ok'"
|
||||
).fetchall()
|
||||
}
|
||||
todo = {k: v for k, v in key_to_raws.items() if k not in done_keys}
|
||||
log.info(f"{len(done_keys)} already done, {len(todo)} to look up")
|
||||
|
||||
if not todo:
|
||||
log.info("Nothing to do – all CPUs already in cpu_specs.")
|
||||
close_playwright()
|
||||
return
|
||||
|
||||
total = len(todo)
|
||||
done_ok = 0
|
||||
done_fail = 0
|
||||
|
||||
for i, (cpu_key, raws) in enumerate(sorted(todo.items()), 1):
|
||||
model_raw = max(raws, key=len)
|
||||
vendor = detect_vendor(model_raw)
|
||||
|
||||
log.info(f"[{i}/{total}] {cpu_key} vendor={vendor}")
|
||||
log.info(f" raw: '{model_raw}'")
|
||||
|
||||
try:
|
||||
if vendor == "Apple":
|
||||
result = lookup_apple(cpu_key)
|
||||
elif vendor == "AMD":
|
||||
log.info(f" Skipping AMD CPU (not supported): {cpu_key}")
|
||||
result = {"lookup_status": "skipped"}
|
||||
else:
|
||||
result = lookup_intel(model_raw)
|
||||
|
||||
result["cpu_key"] = cpu_key
|
||||
result["cpu_model_raw"] = model_raw
|
||||
result.setdefault("vendor", vendor)
|
||||
result["scraped_at"] = datetime.now().isoformat()
|
||||
|
||||
conn.execute(CPU_SPECS_INSERT, _safe_cpu_row(result))
|
||||
conn.commit()
|
||||
|
||||
status = result.get("lookup_status", "?")
|
||||
if status == "ok":
|
||||
done_ok += 1
|
||||
log.info(f" ✓ {result.get('full_name', cpu_key)}")
|
||||
else:
|
||||
done_fail += 1
|
||||
log.warning(f" ✗ lookup_status={status} for {cpu_key}")
|
||||
|
||||
except Exception as exc:
|
||||
log.error(f" Exception processing {cpu_key}: {exc}", exc_info=True)
|
||||
try:
|
||||
conn.execute(
|
||||
"""INSERT OR REPLACE INTO cpu_specs
|
||||
(cpu_key, cpu_model_raw, vendor, scraped_at, lookup_status)
|
||||
VALUES (?, ?, ?, ?, 'failed')""",
|
||||
(cpu_key, model_raw, vendor, datetime.now().isoformat()),
|
||||
)
|
||||
conn.commit()
|
||||
except Exception:
|
||||
pass
|
||||
done_fail += 1
|
||||
|
||||
if vendor != "Apple":
|
||||
time.sleep(random.uniform(1.0, 2.0))
|
||||
|
||||
log.info(
|
||||
f"═══ CPU Lookup done. ✓ {done_ok} ok ✗ {done_fail} failed "
|
||||
f"(total processed: {total}) ═══"
|
||||
)
|
||||
|
||||
summary = conn.execute(
|
||||
"SELECT vendor, lookup_status, COUNT(*) as n "
|
||||
"FROM cpu_specs GROUP BY vendor, lookup_status ORDER BY vendor, lookup_status"
|
||||
).fetchall()
|
||||
log.info("Summary:")
|
||||
for row in summary:
|
||||
log.info(f" {row['vendor']:8s} {row['lookup_status']:8s} {row['n']} entries")
|
||||
|
||||
close_playwright()
|
||||
606
scrapers/scraper_nuvoo.py
Normal file
606
scrapers/scraper_nuvoo.py
Normal file
@@ -0,0 +1,606 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Nuvoo scraper
|
||||
===============
|
||||
Crawls https://nuvoo.com/sv-se/collections/laptops via the Shopify JSON API
|
||||
and stores refurbished laptop listings in the shared SQLite database.
|
||||
|
||||
Usage (from main.py web trigger, or standalone):
|
||||
from scrapers.scraper_nuvoo import run
|
||||
conn = init_db(DB_PATH)
|
||||
run(conn, skip_phase1=False, reset=False)
|
||||
"""
|
||||
|
||||
import re
|
||||
import json
|
||||
import time
|
||||
import random
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from . import shared as _shared
|
||||
from .shared import (
|
||||
log, _sleep, _make_session,
|
||||
INSERT_SQL, _safe_row,
|
||||
USER_AGENTS, map_condition_grade,
|
||||
)
|
||||
|
||||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
NUVOO_BASE = "https://nuvoo.com"
|
||||
NUVOO_LOCALE = "sv-se"
|
||||
NUVOO_COLLECTION_URL = f"{NUVOO_BASE}/{NUVOO_LOCALE}/collections/laptops"
|
||||
NUVOO_PRODUCTS_BASE = f"{NUVOO_BASE}/{NUVOO_LOCALE}/products"
|
||||
NUVOO_MAX_PAGES = 30
|
||||
|
||||
# ── Nuvoo HTTP helper ─────────────────────────────────────────────────────────
|
||||
|
||||
def _nuvoo_get(url: str, retries: int = 3) -> Optional[object]:
|
||||
"""Rate-limited GET for Nuvoo requests; resets shared session on error."""
|
||||
for attempt in range(1, retries + 1):
|
||||
_shared._session.headers["User-Agent"] = random.choice(USER_AGENTS)
|
||||
try:
|
||||
resp = _shared._session.get(url, timeout=20)
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
if resp.status_code == 429:
|
||||
wait = 60 + random.uniform(10, 30)
|
||||
log.warning(f"Rate-limited (429). Sleeping {wait:.0f}s (attempt {attempt}/{retries})")
|
||||
time.sleep(wait)
|
||||
elif resp.status_code in (404, 410):
|
||||
log.info(f"Product gone ({resp.status_code}): {url}")
|
||||
return None
|
||||
else:
|
||||
_shared._session.close()
|
||||
_shared._session = _make_session()
|
||||
log.warning(f"HTTP {resp.status_code} for {url} (attempt {attempt}/{retries})")
|
||||
time.sleep(5 * attempt)
|
||||
except Exception as exc:
|
||||
log.warning(f"Request error attempt {attempt}/{retries}: {exc}")
|
||||
_shared._session.close()
|
||||
_shared._session = _make_session()
|
||||
time.sleep(10 * attempt)
|
||||
log.error(f"Gave up after {retries} attempts: {url}")
|
||||
return None
|
||||
|
||||
|
||||
# ── Phase 1: collect product handles from listing pages ───────────────────────
|
||||
|
||||
def _table_exists(conn: sqlite3.Connection, name: str) -> bool:
|
||||
r = conn.execute(
|
||||
"SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (name,)
|
||||
).fetchone()
|
||||
return r is not None
|
||||
|
||||
|
||||
def collect_handles(conn: sqlite3.Connection) -> set[str]:
|
||||
"""Paginate the Nuvoo laptop collection, re-queue every live product, return live handle set.
|
||||
|
||||
Uses INSERT OR REPLACE so existing queue entries have their done flag reset
|
||||
to 0 — guaranteeing that re-runs always re-scrape and update every listing.
|
||||
Returns the complete set of handles currently live on the site so the caller
|
||||
can remove sold/delisted rows from the database.
|
||||
"""
|
||||
log.info("═══ Nuvoo Phase 1: Collecting product handles ═══")
|
||||
|
||||
page = 1
|
||||
live_handles: set[str] = set()
|
||||
handle_re = re.compile(r"/sv-se/products/([^?/#]+)")
|
||||
|
||||
while page <= NUVOO_MAX_PAGES:
|
||||
url = NUVOO_COLLECTION_URL if page == 1 else f"{NUVOO_COLLECTION_URL}?page={page}"
|
||||
log.info(f"Listing page {page}: {url}")
|
||||
resp = _nuvoo_get(url)
|
||||
if not resp:
|
||||
log.warning(f" Failed to load listing page {page}, stopping.")
|
||||
break
|
||||
|
||||
soup = BeautifulSoup(resp.text, "lxml")
|
||||
|
||||
handles_this_page: list[str] = []
|
||||
for a in soup.find_all("a", href=handle_re):
|
||||
m = handle_re.search(a["href"])
|
||||
if m:
|
||||
handle = m.group(1)
|
||||
if handle not in handles_this_page:
|
||||
handles_this_page.append(handle)
|
||||
|
||||
if not handles_this_page:
|
||||
log.info(f" No product links found on page {page} – end of catalog.")
|
||||
break
|
||||
|
||||
for handle in handles_this_page:
|
||||
live_handles.add(handle)
|
||||
product_url = f"{NUVOO_PRODUCTS_BASE}/{handle}.json"
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO scrape_queue (url, source, handle, done) VALUES (?, 'nuvoo', ?, 0)",
|
||||
(product_url, handle),
|
||||
)
|
||||
conn.commit()
|
||||
log.info(f" {len(handles_this_page)} products on page (running total: {len(live_handles)})")
|
||||
|
||||
if len(handles_this_page) < 15:
|
||||
log.info(f" Only {len(handles_this_page)} products on page {page} — last page.")
|
||||
break
|
||||
|
||||
page += 1
|
||||
_sleep()
|
||||
|
||||
queued = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='nuvoo' AND done=0").fetchone()[0]
|
||||
log.info(f"Phase 1 done. {queued} handles queued for scraping. {len(live_handles)} live products found.")
|
||||
return live_handles
|
||||
|
||||
|
||||
# ── Phase 2: fetch product JSON and insert rows ───────────────────────────────
|
||||
|
||||
def nuvoo_scrape_products(conn: sqlite3.Connection) -> None:
|
||||
"""Work through scrape_queue (nuvoo rows), fetching product JSON and inserting rows."""
|
||||
log.info("═══ Nuvoo Phase 2: Scraping product JSON ═══")
|
||||
|
||||
queue = conn.execute(
|
||||
"SELECT url, handle FROM scrape_queue WHERE source='nuvoo' AND done=0 ORDER BY rowid"
|
||||
).fetchall()
|
||||
total = len(queue)
|
||||
log.info(f"{total} products to scrape.")
|
||||
|
||||
for i, row in enumerate(queue, 1):
|
||||
url = row["url"]
|
||||
handle = row["handle"]
|
||||
log.info(f"[{i}/{total}] {handle}")
|
||||
|
||||
resp = _nuvoo_get(url)
|
||||
if not resp:
|
||||
conn.execute("UPDATE scrape_queue SET done=-1 WHERE url=?", (url,))
|
||||
conn.commit()
|
||||
_sleep()
|
||||
continue
|
||||
|
||||
try:
|
||||
data = resp.json()
|
||||
product = data.get("product", {})
|
||||
|
||||
# Also fetch the HTML product page to parse the Specifikation tab
|
||||
html_url = f"{NUVOO_PRODUCTS_BASE}/{handle}"
|
||||
resp_html = _nuvoo_get(html_url)
|
||||
tab_specs = _parse_nuvoo_specs_tab(resp_html.text) if resp_html else {}
|
||||
|
||||
rows = _parse_nuvoo_product(product, handle, tab_specs)
|
||||
for r in rows:
|
||||
conn.execute(INSERT_SQL, _safe_row(r))
|
||||
conn.execute("UPDATE scrape_queue SET done=1 WHERE url=?", (url,))
|
||||
conn.commit()
|
||||
log.info(f" ✓ {product.get('title', handle)} ({len(rows)} variant rows)")
|
||||
except Exception as exc:
|
||||
log.error(f" Error processing {url}: {exc}", exc_info=True)
|
||||
conn.execute("UPDATE scrape_queue SET done=-1 WHERE url=?", (url,))
|
||||
conn.commit()
|
||||
|
||||
_sleep()
|
||||
|
||||
done_n = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='nuvoo' AND done=1").fetchone()[0]
|
||||
failed_n = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='nuvoo' AND done=-1").fetchone()[0]
|
||||
log.info(f"Phase 2 done. ✓ {done_n} scraped ✗ {failed_n} failed.")
|
||||
|
||||
|
||||
# ── Product parser ────────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_nuvoo_product(product: dict, handle: str, tab_specs: dict | None = None) -> list[dict]:
|
||||
"""
|
||||
Parse a Shopify product dict into one dict row per condition grade.
|
||||
Keeps the cheapest variant per grade as a separate row.
|
||||
tab_specs, if provided, overrides body_html for screen/resolution/type.
|
||||
"""
|
||||
title = product.get("title", "")
|
||||
vendor = product.get("vendor", "")
|
||||
product_url = f"https://nuvoo.com/{NUVOO_LOCALE}/products/{handle}"
|
||||
|
||||
images = product.get("images", [])
|
||||
image_urls = json.dumps([img["src"] for img in images if img.get("src")])
|
||||
|
||||
body_html = product.get("body_html", "")
|
||||
specs = _parse_nuvoo_body_html(body_html)
|
||||
|
||||
# Tab specs take priority (body_html bullets rarely contain resolution)
|
||||
if tab_specs:
|
||||
for k, v in tab_specs.items():
|
||||
if v:
|
||||
specs[k] = v
|
||||
|
||||
condition_best: dict[str, dict] = {}
|
||||
for v in product.get("variants", []):
|
||||
option1 = v.get("option1", "") or ""
|
||||
option2 = v.get("option2", "") or ""
|
||||
option3 = v.get("option3", "") or ""
|
||||
price_str = v.get("price", "0") or "0"
|
||||
orig_str = v.get("compare_at_price", "") or ""
|
||||
|
||||
grade = map_condition_grade(option1, "nuvoo")
|
||||
|
||||
try:
|
||||
price = int(float(price_str))
|
||||
except (ValueError, TypeError):
|
||||
price = None
|
||||
|
||||
try:
|
||||
orig = int(float(orig_str)) if orig_str else None
|
||||
except (ValueError, TypeError):
|
||||
orig = None
|
||||
|
||||
if grade not in condition_best or (
|
||||
price and condition_best[grade]["price_sek"] and price < condition_best[grade]["price_sek"]
|
||||
):
|
||||
condition_best[grade] = {
|
||||
"condition_grade": grade,
|
||||
"condition_raw": option1,
|
||||
"price_sek": price,
|
||||
"original_price_sek": orig,
|
||||
"ram_source": option2,
|
||||
"storage_source": option3,
|
||||
}
|
||||
|
||||
base = _nuvoo_build_base_row(title, vendor, product_url, specs, image_urls)
|
||||
|
||||
rows = []
|
||||
for grade, vdata in condition_best.items():
|
||||
row = dict(base)
|
||||
row["condition_grade"] = vdata["condition_grade"]
|
||||
row["condition_detail"] = vdata["condition_raw"]
|
||||
row["price_sek"] = vdata["price_sek"]
|
||||
row["original_price_sek"] = vdata["original_price_sek"]
|
||||
|
||||
if not row.get("ram_gb"):
|
||||
m = re.search(r"(\d+)\s*GB", vdata["ram_source"], re.I)
|
||||
if m:
|
||||
row["ram_gb"] = int(m.group(1))
|
||||
if not row.get("storage_gb"):
|
||||
m = re.search(r"(\d+)\s*(GB|TB)", vdata["storage_source"], re.I)
|
||||
if m:
|
||||
val = int(m.group(1))
|
||||
unit = m.group(2).upper()
|
||||
row["storage_gb"] = val * 1024 if unit == "TB" else val
|
||||
|
||||
row["url"] = f"{product_url}?grade={grade.lower()}"
|
||||
rows.append(row)
|
||||
|
||||
return rows
|
||||
|
||||
|
||||
def _nuvoo_build_base_row(
|
||||
title: str, vendor: str, product_url: str, specs: dict, image_urls: str
|
||||
) -> dict:
|
||||
"""Build the shared column data for all condition variants of a product."""
|
||||
row: dict = {
|
||||
"url": product_url,
|
||||
"title": title,
|
||||
"brand": vendor,
|
||||
"image_urls": image_urls,
|
||||
"source": "nuvoo",
|
||||
"scraped_at": datetime.now().isoformat(),
|
||||
"touchscreen": 0,
|
||||
"wired_ethernet": 1,
|
||||
"has_4g_5g": 0,
|
||||
"keyboard_backlit": 0,
|
||||
"webcam": 1,
|
||||
"optical_drive": 0,
|
||||
}
|
||||
|
||||
s = specs
|
||||
|
||||
# Screen size: prefer direct tab field, else extract from combined screen bullet
|
||||
if s.get("screen_size"):
|
||||
raw_size = s["screen_size"]
|
||||
size_m = re.search(r'\d+[,.]?\d*\s*(?:"|tum|inch|\u2033)', raw_size, re.I)
|
||||
row["screen_size"] = size_m.group(0).strip() if size_m else raw_size.strip()
|
||||
else:
|
||||
screen_raw = s.get("screen", "")
|
||||
screen_raw = re.sub(r"[-–\s]*sk[aä]rm\s*:?\s*$", "", screen_raw, flags=re.I).strip().rstrip(": \t")
|
||||
size_m = re.search(r'\d+[,.]?\d*\s*(?:"|tum|inch|\u2033)', screen_raw, re.I)
|
||||
row["screen_size"] = size_m.group(0).strip() if size_m else screen_raw
|
||||
|
||||
# Resolution: prefer direct tab field, else try to extract from body_html bullet
|
||||
if s.get("screen_resolution"):
|
||||
row["screen_resolution"] = s["screen_resolution"]
|
||||
else:
|
||||
combined_res = " ".join(filter(None, [s.get("screen"), s.get("screen_size")]))
|
||||
res_m = re.search(r"(\d{3,4}\s*[×x]\s*\d{3,4}(?:\s*\([^)]+\))?)", combined_res)
|
||||
if res_m:
|
||||
row["screen_resolution"] = res_m.group(1).strip()
|
||||
|
||||
# Screen type and touchscreen: search all screen-related text
|
||||
combined_screen = " ".join(filter(None, [
|
||||
s.get("screen_type"), s.get("screen_size"), s.get("screen"), s.get("screen_resolution"),
|
||||
])).lower()
|
||||
if s.get("screen_type"):
|
||||
raw_type = s["screen_type"].upper()
|
||||
# Clean up Shopify values like "IPS-skärm" → "IPS"
|
||||
type_m = re.search(r"(RETINA|OLED|IPS|TN|VA|LCD|LED)", raw_type)
|
||||
row["screen_type"] = type_m.group(1) if type_m else raw_type
|
||||
else:
|
||||
for stype in ("retina", "oled", "ips", "tn", "va", "lcd", "led"):
|
||||
if stype in combined_screen:
|
||||
row["screen_type"] = stype.upper()
|
||||
break
|
||||
|
||||
if re.search(r"pek|touch", combined_screen, re.I):
|
||||
row["touchscreen"] = 1
|
||||
|
||||
cpu_raw = s.get("cpu", "")
|
||||
# Strip parenthetical annotations (e.g. "(4 kärnor / 8 trådar)") from the
|
||||
# stored name so filters and future lookups work against a clean model string.
|
||||
# cpu_raw is kept intact below so cores/threads can still be parsed from it.
|
||||
row["cpu_full"] = re.sub(r'\s*\([^)]*\)', '', cpu_raw).strip()
|
||||
|
||||
cpu_model_m = re.match(
|
||||
r"(Apple\s+M\d[\w\s]*?(?:Pro|Max|Ultra)?|"
|
||||
r"(?:Intel|AMD)\s+(?:Core(?:\s+Ultra)?|Ryzen|Celeron|Pentium|Xeon)\s+[\w\-]+(?:\s+PRO)?)",
|
||||
cpu_raw, re.I,
|
||||
)
|
||||
if cpu_model_m:
|
||||
model = cpu_model_m.group(1).strip()
|
||||
model = re.sub(r"[-\s]*(processor|cpu|chip)$", "", model, flags=re.I).strip()
|
||||
row["cpu_model"] = model
|
||||
else:
|
||||
fb_m = re.match(r"([\w\s\-]+?)\s+\d+[.,]\d+\s*GHz", cpu_raw)
|
||||
if fb_m:
|
||||
row["cpu_model"] = fb_m.group(1).strip()
|
||||
elif cpu_raw:
|
||||
model = cpu_raw.split("(")[0].strip()
|
||||
model = re.sub(r"[-\s]*(processor|cpu|chip)$", "", model, flags=re.I).strip()
|
||||
row["cpu_model"] = model
|
||||
|
||||
cores_m = re.search(r"(\d+)\s*kärnor", cpu_raw, re.I)
|
||||
if cores_m:
|
||||
row["cpu_cores"] = int(cores_m.group(1))
|
||||
|
||||
threads_m = re.search(r"(\d+)\s*trådar", cpu_raw, re.I)
|
||||
if threads_m:
|
||||
row["cpu_threads"] = int(threads_m.group(1))
|
||||
elif row.get("cpu_cores"):
|
||||
row["cpu_threads"] = row["cpu_cores"] * 2
|
||||
|
||||
ghz_vals = re.findall(r"(\d+[.,]\d+)\s*GHz", cpu_raw, re.I)
|
||||
if ghz_vals:
|
||||
row["cpu_base_ghz"] = float(ghz_vals[0].replace(",", "."))
|
||||
|
||||
turbo_m = re.search(r"turbo\s+(?:upp\s+till\s+)?(\d+[.,]\d+)\s*GHz", cpu_raw, re.I)
|
||||
if turbo_m:
|
||||
row["cpu_turbo_ghz"] = float(turbo_m.group(1).replace(",", "."))
|
||||
|
||||
gen_m = re.search(r"(\d{1,2})(?:th|:e|e)\s*(?:gen)?", cpu_raw, re.I)
|
||||
if gen_m:
|
||||
row["cpu_generation"] = f"{gen_m.group(1)}th"
|
||||
else:
|
||||
apple_m = re.search(r"Apple\s+(M\d(?:\s+(?:Pro|Max|Ultra))?)", title, re.I)
|
||||
if apple_m:
|
||||
row["cpu_generation"] = apple_m.group(1)
|
||||
|
||||
ram_raw = s.get("ram", "")
|
||||
ram_m = re.search(r"(\d+)\s*GB", ram_raw, re.I)
|
||||
if ram_m:
|
||||
row["ram_gb"] = int(ram_m.group(1))
|
||||
ram_type_m = re.search(r"(LPDDR\d+[A-Z]*x?|DDR\d+[A-Z]*)", ram_raw, re.I)
|
||||
if ram_type_m:
|
||||
row["ram_type"] = ram_type_m.group(1).upper()
|
||||
|
||||
stor_raw = s.get("storage", "")
|
||||
stor_m = re.search(r"(\d+)\s*(GB|TB)", stor_raw, re.I)
|
||||
if stor_m:
|
||||
val = int(stor_m.group(1))
|
||||
unit = stor_m.group(2).upper()
|
||||
row["storage_gb"] = val * 1024 if unit == "TB" else val
|
||||
if re.search(r"nvme", stor_raw, re.I):
|
||||
row["storage_type"] = "NVMe SSD"
|
||||
elif re.search(r"\bssd\b", stor_raw, re.I):
|
||||
row["storage_type"] = "SSD"
|
||||
|
||||
row["gpu"] = s.get("gpu", "")
|
||||
|
||||
os_raw = s.get("os", "")
|
||||
if os_raw:
|
||||
row["operating_system"] = os_raw
|
||||
elif "win 11 pro" in title.lower():
|
||||
row["operating_system"] = "Windows 11 Pro"
|
||||
elif "win 11 home" in title.lower():
|
||||
row["operating_system"] = "Windows 11 Home"
|
||||
elif "win 10" in title.lower():
|
||||
row["operating_system"] = "Windows 10"
|
||||
|
||||
if re.search(r"\b4G\b|\b5G\b|LTE", title):
|
||||
row["has_4g_5g"] = 1
|
||||
|
||||
weight_raw = s.get("weight", "")
|
||||
wm = re.search(r"(\d+[.,]\d+)\s*kg", weight_raw, re.I)
|
||||
if wm:
|
||||
row["weight_kg"] = float(wm.group(1).replace(",", "."))
|
||||
|
||||
row["wifi"] = s.get("wifi", "")
|
||||
row["bluetooth"] = s.get("bluetooth", "")
|
||||
|
||||
return row
|
||||
|
||||
|
||||
def _parse_nuvoo_body_html(body_html: str) -> dict:
|
||||
"""Extract structured specs from Nuvoo's product body_html bullet list."""
|
||||
if not body_html:
|
||||
return {}
|
||||
|
||||
soup = BeautifulSoup(body_html, "lxml")
|
||||
specs: dict = {}
|
||||
|
||||
bullets = []
|
||||
for li in soup.find_all("li"):
|
||||
strong = li.find("strong")
|
||||
if strong:
|
||||
bullets.append(strong.get_text(strip=True))
|
||||
else:
|
||||
bullets.append(li.get_text(strip=True))
|
||||
|
||||
for b in bullets:
|
||||
bl = b.lower()
|
||||
if re.match(r"(intel|amd|apple)\s", bl) and not specs.get("cpu"):
|
||||
# Nuvoo sometimes injects marketing copy after a colon, e.g.:
|
||||
# "Intel Core i5-1135G7-processor: smidig prestanda ..."
|
||||
# Truncate at the colon and strip trailing "-processor" / "-chip" suffixes.
|
||||
cpu_val = b.split(":")[0].strip()
|
||||
cpu_val = re.sub(r"[-\s]*(processor|cpu|chip)$", "", cpu_val, flags=re.I).strip()
|
||||
specs["cpu"] = cpu_val
|
||||
elif ('"' in b or "tum" in bl or "inch" in bl) and re.search(r'\d+[,.]?\d*\s*(?:"|tum|inch|\u2033)', b, re.I) and not specs.get("screen"):
|
||||
# Take only the spec part before any marketing description after a colon
|
||||
specs["screen"] = b.split(":")[0].strip()
|
||||
elif re.search(r'retina|liquid\s+retina', bl) and not specs.get("screen"):
|
||||
specs["screen"] = b.split(":")[0].strip()
|
||||
elif re.match(r"\d+\s*gb.*ddr", bl) and not specs.get("ram"):
|
||||
specs["ram"] = b
|
||||
elif re.match(r"\d+\s*(gb|tb)\s*(nvme|ssd|hdd|emmc|sshd)?", bl) and not specs.get("storage"):
|
||||
specs["storage"] = b
|
||||
elif re.match(r"(nvidia|rtx|gtx|amd radeon rx|quadro)", bl) and not specs.get("gpu"):
|
||||
specs["gpu"] = b
|
||||
|
||||
for b in bullets:
|
||||
bl = b.lower()
|
||||
if re.search(r"wi-fi|wifi|802\.11", bl) and not specs.get("wifi"):
|
||||
specs["wifi"] = b
|
||||
if re.search(r"bluetooth", bl) and not specs.get("bluetooth"):
|
||||
specs["bluetooth"] = b
|
||||
if re.search(r"windows\s+\d+|macos", bl, re.I) and not specs.get("os"):
|
||||
specs["os"] = b
|
||||
if re.search(r"\bkg\b", bl) and not specs.get("weight"):
|
||||
specs["weight"] = b
|
||||
|
||||
all_text = soup.get_text(" ", strip=True)
|
||||
if not specs.get("wifi"):
|
||||
m = re.search(r"(Wi-Fi\s+\d+[^\s,.,]*(?:\s+\([^)]+\))?)", all_text, re.I)
|
||||
if m:
|
||||
specs["wifi"] = m.group(1)
|
||||
if not specs.get("bluetooth"):
|
||||
m = re.search(r"(Bluetooth\s+[\d.]+([\s\S]{0,20})?)", all_text, re.I)
|
||||
if m:
|
||||
parts = m.group(1).strip().split()[:3]
|
||||
specs["bluetooth"] = " ".join(parts)
|
||||
|
||||
return specs
|
||||
|
||||
|
||||
def _parse_nuvoo_specs_tab(html: str) -> dict:
|
||||
"""Parse the 'Specifikation' tab (#tab-content-for-specs) from a Nuvoo
|
||||
product page, returning a dict keyed by our internal field names.
|
||||
|
||||
Tries multiple HTML structures: table rows, definition lists, div pairs.
|
||||
Maps Swedish and English labels to our field names.
|
||||
"""
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
|
||||
container = (
|
||||
soup.find(id="tab-content-for-specs")
|
||||
or soup.find(id=re.compile(r"specs?", re.I))
|
||||
)
|
||||
if not container:
|
||||
for el in soup.find_all(["section", "div"],
|
||||
id=re.compile(r"spec|spek", re.I)):
|
||||
container = el
|
||||
break
|
||||
if not container:
|
||||
return {}
|
||||
|
||||
raw: dict[str, str] = {}
|
||||
|
||||
# Strategy 1: HTML table
|
||||
for tr in container.find_all("tr"):
|
||||
cells = tr.find_all(["td", "th"])
|
||||
if len(cells) >= 2:
|
||||
k = cells[0].get_text(strip=True).rstrip(":")
|
||||
v = cells[1].get_text(" ", strip=True)
|
||||
if k and v and len(k) < 80:
|
||||
raw.setdefault(k, v)
|
||||
|
||||
# Strategy 2: definition list <dl><dt>…</dt><dd>…</dd></dl>
|
||||
if not raw:
|
||||
dts = container.find_all("dt")
|
||||
dds = container.find_all("dd")
|
||||
for dt, dd in zip(dts, dds):
|
||||
k = dt.get_text(strip=True).rstrip(":")
|
||||
v = dd.get_text(" ", strip=True)
|
||||
if k and v and len(k) < 80:
|
||||
raw.setdefault(k, v)
|
||||
|
||||
# Strategy 3: div pairs with label/value class names
|
||||
if not raw:
|
||||
for div in container.find_all("div"):
|
||||
lbl = div.find(class_=re.compile(r"label|key|term|name", re.I))
|
||||
val = div.find(class_=re.compile(r"value|data|desc", re.I))
|
||||
if lbl and val:
|
||||
k = lbl.get_text(strip=True).rstrip(":")
|
||||
v = val.get_text(" ", strip=True)
|
||||
if k and v and len(k) < 80:
|
||||
raw.setdefault(k, v)
|
||||
|
||||
if not raw:
|
||||
return {}
|
||||
|
||||
# Map Swedish + English labels to our field names (first match wins)
|
||||
_LABEL_MAP = [
|
||||
(r"sk[aä]rmstorlek|display\s*size|screen\s*size", "screen_size"),
|
||||
(r"uppl[oö]sning|resolution", "screen_resolution"),
|
||||
(r"sk[aä]rmtyp|panel(?:\s*typ)?|display\s*type", "screen_type"),
|
||||
(r"processor|cpu", "cpu"),
|
||||
(r"ram|arbetsminne", "ram"),
|
||||
(r"lagring|ssd|hdd|storage", "storage"),
|
||||
(r"grafik(?:kort)?|gpu", "gpu"),
|
||||
(r"operativsystem|os", "os"),
|
||||
(r"vikt", "weight"),
|
||||
(r"wi.?fi|wifi", "wifi"),
|
||||
(r"bluetooth", "bluetooth"),
|
||||
]
|
||||
|
||||
result: dict[str, str] = {}
|
||||
for label, value in raw.items():
|
||||
for pattern, key in _LABEL_MAP:
|
||||
if re.search(pattern, label, re.I):
|
||||
if key not in result:
|
||||
result[key] = value
|
||||
break
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ── Entry point ───────────────────────────────────────────────────────────────
|
||||
|
||||
def run(conn: sqlite3.Connection, skip_phase1: bool = False, reset: bool = False) -> None:
|
||||
"""Run the Nuvoo scrape (Phase 1 + Phase 2 + sold-item cleanup)."""
|
||||
if reset:
|
||||
conn.execute("DELETE FROM laptops WHERE source='nuvoo'")
|
||||
conn.execute("DELETE FROM scrape_queue WHERE source='nuvoo'")
|
||||
conn.commit()
|
||||
log.info("Nuvoo rows cleared.")
|
||||
|
||||
live_handles: set[str] = set()
|
||||
|
||||
if skip_phase1:
|
||||
pending = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='nuvoo' AND done=0").fetchone()[0]
|
||||
log.info(f"--skip-phase1: {pending} handles pending in queue.")
|
||||
else:
|
||||
live_handles = collect_handles(conn)
|
||||
|
||||
nuvoo_scrape_products(conn)
|
||||
|
||||
# Remove any DB rows that are no longer listed on the site (sold / removed).
|
||||
if live_handles:
|
||||
sold_count = 0
|
||||
for (url,) in conn.execute(
|
||||
"SELECT url FROM laptops WHERE source='nuvoo'"
|
||||
).fetchall():
|
||||
handle_m = re.search(r'/products/([^?/#]+)', url)
|
||||
if handle_m and handle_m.group(1) not in live_handles:
|
||||
conn.execute("DELETE FROM laptops WHERE url=?", (url,))
|
||||
sold_count += 1
|
||||
if sold_count:
|
||||
conn.commit()
|
||||
log.info(f" Removed {sold_count} sold/unlisted product(s) from DB.")
|
||||
else:
|
||||
log.info(" No sold/unlisted products found.")
|
||||
|
||||
total = conn.execute("SELECT COUNT(*) FROM laptops WHERE source='nuvoo'").fetchone()[0]
|
||||
log.info(f"Nuvoo done. {total} Nuvoo laptop rows in DB.")
|
||||
372
scrapers/shared.py
Normal file
372
scrapers/shared.py
Normal file
@@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Shared configuration, DB helpers, and HTTP utilities used by all scrapers.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
import sqlite3
|
||||
from typing import Optional
|
||||
|
||||
import requests
|
||||
|
||||
# ── Paths ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
DB_PATH = os.path.normpath(os.path.join(_HERE, "..", "database.db"))
|
||||
LOG_PATH = os.path.normpath(os.path.join(_HERE, "..", "scraper.log"))
|
||||
|
||||
# ── Delays ────────────────────────────────────────────────────────────────────
|
||||
|
||||
DELAY_MIN = 1.5
|
||||
DELAY_MAX = 3.5
|
||||
|
||||
# ── User-Agent rotation ───────────────────────────────────────────────────────
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
# Connection: close ensures every request uses a fresh TCP socket.
|
||||
# This prevents "stale connection" 504 errors caused by the server closing an
|
||||
# idle keep-alive connection while our session still thinks it is open.
|
||||
BASE_HEADERS = {
|
||||
"Accept-Language": "sv-SE,sv;q=0.9,en;q=0.8",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xhtml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"Connection": "close",
|
||||
}
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────────────
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||||
handlers=[
|
||||
logging.FileHandler(LOG_PATH, encoding="utf-8"),
|
||||
logging.StreamHandler(),
|
||||
],
|
||||
)
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# ── Condition grade normalisation ─────────────────────────────────────────────
|
||||
|
||||
_BT_GRADE_MAP: dict[str, str] = {
|
||||
"premium": "S",
|
||||
"a": "A",
|
||||
"a/b": "A",
|
||||
"b/a": "A",
|
||||
"b": "B",
|
||||
"b/c": "B",
|
||||
"c/b": "B",
|
||||
"c": "C",
|
||||
"c/d": "C",
|
||||
"d/c": "C",
|
||||
"d": "C",
|
||||
}
|
||||
|
||||
_NUVOO_GRADE_MAP: dict[str, str] = {
|
||||
"nyskick": "S",
|
||||
"utmärkt": "A",
|
||||
"bra": "B",
|
||||
"okej": "C",
|
||||
}
|
||||
|
||||
|
||||
def map_condition_grade(raw: str, source: str) -> str:
|
||||
"""Normalise a source-specific condition label to our unified S/A/B/C scale.
|
||||
|
||||
For BilligTeknik slash-joined grades like "A/B" we always take the better
|
||||
(leftmost) component, then look up the combined key first.
|
||||
|
||||
Unknown values are returned unchanged so they surface during review.
|
||||
"""
|
||||
key = raw.strip().lower()
|
||||
if source == "billigteknik":
|
||||
return _BT_GRADE_MAP.get(key, raw.strip())
|
||||
if source == "nuvoo":
|
||||
return _NUVOO_GRADE_MAP.get(key, raw.strip())
|
||||
return raw.strip()
|
||||
|
||||
# ── Database schema ───────────────────────────────────────────────────────────
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS laptops (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
url TEXT UNIQUE NOT NULL,
|
||||
title TEXT,
|
||||
brand TEXT,
|
||||
model_ref TEXT,
|
||||
ean TEXT,
|
||||
category TEXT,
|
||||
|
||||
price_sek INTEGER,
|
||||
original_price_sek INTEGER,
|
||||
|
||||
condition_grade TEXT,
|
||||
condition_detail TEXT,
|
||||
|
||||
screen_size TEXT,
|
||||
screen_resolution TEXT,
|
||||
screen_type TEXT,
|
||||
touchscreen INTEGER DEFAULT 0,
|
||||
|
||||
cpu_full TEXT,
|
||||
cpu_model TEXT,
|
||||
cpu_cores INTEGER,
|
||||
cpu_threads INTEGER,
|
||||
cpu_base_ghz REAL,
|
||||
cpu_turbo_ghz REAL,
|
||||
cpu_cache TEXT,
|
||||
cpu_generation TEXT,
|
||||
|
||||
ram_gb INTEGER,
|
||||
ram_type TEXT,
|
||||
max_ram_gb INTEGER,
|
||||
|
||||
storage_gb INTEGER,
|
||||
storage_type TEXT,
|
||||
|
||||
gpu TEXT,
|
||||
|
||||
wifi TEXT,
|
||||
bluetooth TEXT,
|
||||
wired_ethernet INTEGER DEFAULT 0,
|
||||
has_4g_5g INTEGER DEFAULT 0,
|
||||
|
||||
usb_ports TEXT,
|
||||
thunderbolt_ports TEXT,
|
||||
hdmi TEXT,
|
||||
|
||||
weight_kg REAL,
|
||||
width_cm REAL,
|
||||
height_cm REAL,
|
||||
depth_cm REAL,
|
||||
battery TEXT,
|
||||
|
||||
operating_system TEXT,
|
||||
keyboard_backlit INTEGER DEFAULT 0,
|
||||
webcam INTEGER DEFAULT 0,
|
||||
optical_drive INTEGER DEFAULT 0,
|
||||
warranty TEXT,
|
||||
eco_cert TEXT,
|
||||
|
||||
specs_raw TEXT,
|
||||
|
||||
source TEXT DEFAULT 'billigteknik',
|
||||
image_urls TEXT,
|
||||
|
||||
scraped_at TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scrape_queue (
|
||||
url TEXT PRIMARY KEY,
|
||||
source TEXT NOT NULL DEFAULT 'billigteknik',
|
||||
-- BilligTeknik listing-page cache (NULL for Nuvoo rows)
|
||||
title TEXT,
|
||||
price_sek INTEGER,
|
||||
orig_price_sek INTEGER,
|
||||
condition_grade TEXT,
|
||||
condition_detail TEXT,
|
||||
category TEXT,
|
||||
-- Nuvoo Shopify handle (NULL for BilligTeknik rows)
|
||||
handle TEXT,
|
||||
done INTEGER DEFAULT 0,
|
||||
added_at TEXT DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cpu_specs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
cpu_key TEXT UNIQUE NOT NULL,
|
||||
cpu_model_raw TEXT,
|
||||
vendor TEXT,
|
||||
full_name TEXT,
|
||||
brand_name TEXT,
|
||||
codename TEXT,
|
||||
lithography TEXT,
|
||||
launch_date TEXT,
|
||||
total_cores INTEGER,
|
||||
performance_cores INTEGER,
|
||||
efficient_cores INTEGER,
|
||||
total_threads INTEGER,
|
||||
base_freq_ghz REAL,
|
||||
max_turbo_ghz REAL,
|
||||
l2_cache TEXT,
|
||||
l3_cache TEXT,
|
||||
tdp_w INTEGER,
|
||||
max_tdp_w INTEGER,
|
||||
pcie_version TEXT,
|
||||
pcie_lanes INTEGER,
|
||||
max_ram_gb INTEGER,
|
||||
ram_types TEXT,
|
||||
ram_speeds_mhz TEXT,
|
||||
ecc_support INTEGER,
|
||||
igpu_name TEXT,
|
||||
igpu_base_mhz INTEGER,
|
||||
igpu_boost_mhz INTEGER,
|
||||
source_url TEXT,
|
||||
lookup_status TEXT DEFAULT 'pending',
|
||||
scraped_at TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def init_db(path: str, reset: bool = False) -> sqlite3.Connection:
|
||||
conn = sqlite3.connect(path)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=NORMAL")
|
||||
if reset:
|
||||
conn.executescript("DROP TABLE IF EXISTS laptops; DROP TABLE IF EXISTS scrape_queue;")
|
||||
log.info("Database reset.")
|
||||
conn.executescript(SCHEMA)
|
||||
conn.commit()
|
||||
return conn
|
||||
|
||||
|
||||
# ── HTTP helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
def _make_session() -> requests.Session:
|
||||
"""Create a fresh requests.Session with correct headers."""
|
||||
s = requests.Session()
|
||||
s.headers.update(BASE_HEADERS)
|
||||
return s
|
||||
|
||||
|
||||
_session = _make_session()
|
||||
|
||||
|
||||
def polite_get(url: str, retries: int = 3) -> Optional[requests.Response]:
|
||||
"""Rate-limited GET with retry logic. Returns None on permanent failure."""
|
||||
global _session
|
||||
for attempt in range(1, retries + 1):
|
||||
_session.headers["User-Agent"] = random.choice(USER_AGENTS)
|
||||
try:
|
||||
resp = _session.get(url, timeout=20)
|
||||
|
||||
if resp.status_code == 200:
|
||||
return resp
|
||||
|
||||
if resp.status_code == 429:
|
||||
wait = 60 + random.uniform(10, 30)
|
||||
log.warning(
|
||||
f"Rate-limited (429). Sleeping {wait:.0f}s "
|
||||
f"(attempt {attempt}/{retries})"
|
||||
)
|
||||
time.sleep(wait)
|
||||
|
||||
elif resp.status_code == 502:
|
||||
_session.close()
|
||||
_session = _make_session()
|
||||
if attempt == 1:
|
||||
wait = 180 + random.uniform(0, 20)
|
||||
log.warning(
|
||||
f"HTTP 502 Bad Gateway – backing off {wait:.0f}s before retrying "
|
||||
f"(attempt {attempt}/{retries}): {url}"
|
||||
)
|
||||
time.sleep(wait)
|
||||
else:
|
||||
log.warning(f"HTTP 502 again (attempt {attempt}/{retries}): {url}")
|
||||
time.sleep(random.uniform(2, 3))
|
||||
|
||||
elif resp.status_code in (404, 410):
|
||||
log.info(f"Product gone ({resp.status_code}): {url}")
|
||||
return None
|
||||
|
||||
else:
|
||||
_session.close()
|
||||
_session = _make_session()
|
||||
log.warning(
|
||||
f"HTTP {resp.status_code} for {url} (attempt {attempt}/{retries})"
|
||||
)
|
||||
time.sleep(5 * attempt)
|
||||
|
||||
except requests.RequestException as exc:
|
||||
log.warning(f"Request error attempt {attempt}/{retries}: {exc}")
|
||||
_session.close()
|
||||
_session = _make_session()
|
||||
time.sleep(10 * attempt)
|
||||
|
||||
log.error(f"Gave up after {retries} attempts: {url}")
|
||||
return None
|
||||
|
||||
|
||||
def _sleep():
|
||||
"""Polite random delay between requests."""
|
||||
time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
|
||||
|
||||
|
||||
# ── INSERT helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
INSERT_SQL = """
|
||||
INSERT OR REPLACE INTO laptops (
|
||||
url, title, brand, model_ref, ean, category,
|
||||
price_sek, original_price_sek,
|
||||
condition_grade, condition_detail,
|
||||
screen_size, screen_resolution, screen_type, touchscreen,
|
||||
cpu_full, cpu_model, cpu_cores, cpu_threads,
|
||||
cpu_base_ghz, cpu_turbo_ghz, cpu_cache, cpu_generation,
|
||||
ram_gb, ram_type, max_ram_gb,
|
||||
storage_gb, storage_type,
|
||||
gpu,
|
||||
wifi, bluetooth, wired_ethernet, has_4g_5g,
|
||||
usb_ports, thunderbolt_ports, hdmi,
|
||||
weight_kg, width_cm, height_cm, depth_cm, battery,
|
||||
operating_system,
|
||||
keyboard_backlit, webcam, optical_drive,
|
||||
warranty, eco_cert,
|
||||
specs_raw, source, image_urls, scraped_at
|
||||
) VALUES (
|
||||
:url, :title, :brand, :model_ref, :ean, :category,
|
||||
:price_sek, :original_price_sek,
|
||||
:condition_grade, :condition_detail,
|
||||
:screen_size, :screen_resolution, :screen_type, :touchscreen,
|
||||
:cpu_full, :cpu_model, :cpu_cores, :cpu_threads,
|
||||
:cpu_base_ghz, :cpu_turbo_ghz, :cpu_cache, :cpu_generation,
|
||||
:ram_gb, :ram_type, :max_ram_gb,
|
||||
:storage_gb, :storage_type,
|
||||
:gpu,
|
||||
:wifi, :bluetooth, :wired_ethernet, :has_4g_5g,
|
||||
:usb_ports, :thunderbolt_ports, :hdmi,
|
||||
:weight_kg, :width_cm, :height_cm, :depth_cm, :battery,
|
||||
:operating_system,
|
||||
:keyboard_backlit, :webcam, :optical_drive,
|
||||
:warranty, :eco_cert,
|
||||
:specs_raw, :source, :image_urls, :scraped_at
|
||||
)
|
||||
"""
|
||||
|
||||
_ALL_COLUMNS = [
|
||||
"url", "title", "brand", "model_ref", "ean", "category",
|
||||
"price_sek", "original_price_sek",
|
||||
"condition_grade", "condition_detail",
|
||||
"screen_size", "screen_resolution", "screen_type", "touchscreen",
|
||||
"cpu_full", "cpu_model", "cpu_cores", "cpu_threads",
|
||||
"cpu_base_ghz", "cpu_turbo_ghz", "cpu_cache", "cpu_generation",
|
||||
"ram_gb", "ram_type", "max_ram_gb",
|
||||
"storage_gb", "storage_type",
|
||||
"gpu",
|
||||
"wifi", "bluetooth", "wired_ethernet", "has_4g_5g",
|
||||
"usb_ports", "thunderbolt_ports", "hdmi",
|
||||
"weight_kg", "width_cm", "height_cm", "depth_cm", "battery",
|
||||
"operating_system",
|
||||
"keyboard_backlit", "webcam", "optical_drive",
|
||||
"warranty", "eco_cert",
|
||||
"specs_raw", "source", "image_urls", "scraped_at",
|
||||
]
|
||||
|
||||
|
||||
def _safe_row(data: dict) -> dict:
|
||||
"""Ensure every expected column key exists (default None)."""
|
||||
return {col: data.get(col) for col in _ALL_COLUMNS}
|
||||
Reference in New Issue
Block a user