662 lines
26 KiB
Python
662 lines
26 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
BilligTeknik scraper
|
||
======================
|
||
Crawls https://www.billigteknik.se/623-begagnad-barbar-dator and stores
|
||
refurbished laptop listings in the shared SQLite database.
|
||
|
||
Usage (from main.py web trigger, or standalone):
|
||
from scrapers.scraper_billigteknik import run
|
||
conn = init_db(DB_PATH)
|
||
run(conn, skip_phase1=False)
|
||
"""
|
||
|
||
import re
|
||
import json
|
||
import time
|
||
import random
|
||
import sqlite3
|
||
from typing import Optional
|
||
from datetime import datetime
|
||
from urllib.parse import urljoin
|
||
|
||
from bs4 import BeautifulSoup
|
||
|
||
from .shared import (
|
||
log, polite_get, _sleep,
|
||
INSERT_SQL, _ALL_COLUMNS, _safe_row,
|
||
USER_AGENTS, map_condition_grade,
|
||
)
|
||
|
||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||
|
||
BILLIG_BASE_URL = "https://www.billigteknik.se"
|
||
|
||
|
||
# ── Title cleaning ────────────────────────────────────────────────────────────
|
||
|
||
# Applied in order; each pattern is replaced with a single space then collapsed.
|
||
_TITLE_NOISE: list[re.Pattern] = [
|
||
# Parenthetical notes: "(beg med...)", "(gen 11)", "(låg batterihälsa)" etc.
|
||
re.compile(r'\s*\([^)]*\)'),
|
||
# Screen size: 14", 13,3", 11.6", 13-tum, 15,6 tum
|
||
re.compile(r'\b\d{2}[.,]?\d?\s*[-–]?\s*tum\b\s*', re.I),
|
||
re.compile(r'\b\d{2}[.,]?\d?\s*["""″]\s*'),
|
||
# Resolution labels
|
||
re.compile(r'\b(Full[\s-]?HD|FHD|QHD|WQHD|UHD|WUXGA|WXGA|4K|2K|HD[\+]?)\b\s*', re.I),
|
||
# All GB / TB amounts (RAM and storage) — also catches digit-concatenated: 512SSD
|
||
re.compile(r'\b\d+(?:SSD|HDD|NVMe|eMMC|SSHD)\b\s*', re.I),
|
||
re.compile(r'\b\d+\s*[GT]B\b\s*', re.I),
|
||
# Storage type labels
|
||
re.compile(r'\b(NVMe|SSD|HDD|eMMC|SSHD|Flash)\b\s*', re.I),
|
||
# OS strings — longest/most specific first
|
||
re.compile(r'\bWindows\s+\d+\s*(Pro|Home|S|Enterprise|Education)?\b\s*', re.I),
|
||
re.compile(r'\bWin\s*\d+\s*(Pro|Home|S|P)?\b\s*', re.I),
|
||
re.compile(r'\bW\d+(?:Pro|Home|P|H|E)\b\s*', re.I),
|
||
re.compile(r'\b(macOS|ChromeOS|Chrome\s+OS|Linux|Ubuntu)\b\s*', re.I),
|
||
# CPU — full model number first, then family + standalone
|
||
re.compile(r'\b(Intel\s+)?(Core\s+)?(Ultra\s+)?i[3579]-\d{4,5}[A-Za-z]*\b\s*', re.I),
|
||
re.compile(r'\b(Intel\s+)?Core\s+(Ultra\s+)?i[3579]\b\s*', re.I),
|
||
re.compile(r'\bi[3579]\b\s*', re.I),
|
||
# Intel Core Ultra (no "i" prefix, e.g. "Ultra 7 165H", "Core Ultra 5 125U")
|
||
re.compile(r'\b(?:Intel\s+)?(?:Core\s+)?Ultra\s+[579]\s+\d{3,4}[A-Za-z]*\b\s*', re.I),
|
||
re.compile(r'\b(AMD\s+)?Ryzen\s+\d+\b\s*', re.I),
|
||
re.compile(r'\b(Intel\s+)?(Celeron|Pentium|Xeon|Athlon)\b\s*', re.I),
|
||
re.compile(r'\b(Intel\s+)?(DualCore|QuadCore|OctaCore)\b\s*', re.I),
|
||
# CPU generation indicators (must come BEFORE bare ordinal strip)
|
||
re.compile(r'\b\d{1,2}(th|st|nd|rd)\s*gen(eration)?\b\s*', re.I),
|
||
re.compile(r'\bgen\s*\d{1,2}\b\s*', re.I),
|
||
# Bare generation ordinals NOT followed by a model-style word (e.g. 10th, 11th)
|
||
re.compile(r'\b\d{1,2}(th|st|nd|rd)\b\s*', re.I),
|
||
# Connectivity noise
|
||
re.compile(r'\bmed\s+(4G|5G|LTE)\b\s*', re.I),
|
||
re.compile(r'\b(4G|5G|LTE)\b\s*', re.I),
|
||
# Standalone brand names left after CPU model is stripped
|
||
re.compile(r'\bIntel\b\s*', re.I),
|
||
re.compile(r'\bAMD\b\s*', re.I),
|
||
# Touch/screen feature words left floating
|
||
re.compile(r'\bTouch\b\s*', re.I),
|
||
# Condition word "beg"
|
||
re.compile(r'\bbeg\b\s*', re.I),
|
||
# Dangling connectors / punctuation left after stripping
|
||
re.compile(r'\s*&(?:\s+[-\w]+)*'), # "& tangentbord", "& -modem", standalone "&"
|
||
re.compile(r'\bmed\b(?:\s+\w+)+'), # "med tangentbord", "med Sure View" etc. (1+ words)
|
||
re.compile(r'(?<=[a-z\d])\s*[+]\s*', re.I), # trailing "+" from "HD+"
|
||
# GPU model names (already tracked in gpu field)
|
||
re.compile(r'\bRTX\s+\w*\d{3,4}\w*(?:\s+\w+)?\b\s*', re.I), # RTX 3000, RTX A1000, RTX 2000 Ada
|
||
re.compile(r'\bGTX\s+\d{3,4}\b\s*', re.I),
|
||
re.compile(r'\bQuadro\s+\w+\b\s*', re.I),
|
||
re.compile(r'\bMX\d{3,4}\b\s*', re.I), # MX330, MX250 etc.
|
||
# Warranty codes like 2YW, 3YW
|
||
re.compile(r'\b\d+YW\b\s*', re.I),
|
||
# HP Sure View privacy screen feature — strip unit or orphaned word
|
||
re.compile(r'\bSure\s+View\b\s*', re.I),
|
||
re.compile(r'\bView\b\s*', re.I),
|
||
]
|
||
|
||
|
||
def _clean_title(title: str) -> str:
|
||
"""Strip spec noise from a BilligTeknik title, keeping only brand + model name.
|
||
|
||
Example:
|
||
"HP ProBook 440 G8 14\" Full HD i5 (gen 11) 16GB 256GB SSD Win 11 Pro (beg)"
|
||
→ "HP ProBook 440 G8"
|
||
"""
|
||
s = title.strip()
|
||
for rx in _TITLE_NOISE:
|
||
s = rx.sub(' ', s)
|
||
# Collapse runs of spaces and strip trailing punctuation
|
||
s = re.sub(r'\s{2,}', ' ', s).strip(' -–,/|·*&')
|
||
# Safety: never return an empty string
|
||
return s if s else title.strip()
|
||
BILLIG_CATEGORY_URL = "https://www.billigteknik.se/623-begagnad-barbar-dator"
|
||
BILLIG_MAX_PAGES = 60
|
||
|
||
# Compatibility aliases
|
||
BASE_URL = BILLIG_BASE_URL
|
||
CATEGORY_URL = BILLIG_CATEGORY_URL
|
||
MAX_PAGES = BILLIG_MAX_PAGES
|
||
|
||
|
||
# ── Price parsing ─────────────────────────────────────────────────────────────
|
||
|
||
def parse_price(text: str) -> Optional[int]:
|
||
"""
|
||
Extract an integer SEK price from Swedish-formatted strings.
|
||
'3 899 kr' → 3899
|
||
'Nypris: 16 000 kr' → 16000
|
||
"""
|
||
digits = re.sub(r"[^\d]", "", text)
|
||
return int(digits) if digits else None
|
||
|
||
|
||
# ── Listing page parser ───────────────────────────────────────────────────────
|
||
|
||
def _extract_condition_detail(title: str) -> str:
|
||
"""
|
||
Pull the freeform condition note out of a product title.
|
||
'HP EliteBook (beg med små märken skärm)' → 'små märken skärm'
|
||
'Dell Chromebook (beg)' → ''
|
||
"""
|
||
m = re.search(r"\(beg([^)]*)\)", title, re.IGNORECASE)
|
||
if not m:
|
||
return ""
|
||
detail = m.group(1).strip()
|
||
detail = re.sub(r"^\*?\s*med\s+", "", detail, flags=re.IGNORECASE).strip(" *")
|
||
return detail
|
||
|
||
|
||
def parse_listing_page(html: str) -> list[dict]:
|
||
"""
|
||
Parse a category listing page.
|
||
Returns list of dicts: url, title, price_sek, orig_price_sek,
|
||
condition_grade, condition_detail, category.
|
||
"""
|
||
soup = BeautifulSoup(html, "lxml")
|
||
results = []
|
||
|
||
cards = soup.select("article.product-miniature")
|
||
if not cards:
|
||
cards = soup.select(".js-product-miniature")
|
||
|
||
if not cards:
|
||
log.debug("No product cards via standard selectors – using href fallback")
|
||
seen: set = set()
|
||
for a in soup.find_all("a", href=re.compile(r"billigteknik\.se/.+\.html")):
|
||
href = a.get("href", "").split("#")[0]
|
||
if href and href not in seen and "/beg" in href:
|
||
seen.add(href)
|
||
results.append({
|
||
"url": href,
|
||
"title": a.get_text(strip=True),
|
||
"price_sek": None,
|
||
"orig_price_sek": None,
|
||
"condition_grade": None,
|
||
"condition_detail": _extract_condition_detail(a.get_text(strip=True)),
|
||
"category": _category_from_url(href),
|
||
})
|
||
return results
|
||
|
||
for card in cards:
|
||
try:
|
||
link = (
|
||
card.select_one("a[itemprop='url']")
|
||
or card.select_one("a.product-thumbnail")
|
||
or card.select_one(".product-title a")
|
||
or card.select_one("h3 a")
|
||
)
|
||
if not link:
|
||
continue
|
||
href = link.get("href", "").split("#")[0].strip()
|
||
if not href or ".html" not in href:
|
||
continue
|
||
if not href.startswith("http"):
|
||
href = urljoin(BASE_URL, href)
|
||
|
||
title_el = card.select_one("h3[itemprop='name']")
|
||
if not title_el:
|
||
title_el = card.select_one(".product-title h3") or card.select_one("h3")
|
||
title = title_el.get_text(strip=True) if title_el else ""
|
||
|
||
price = None
|
||
price_meta = card.select_one("meta[itemprop='price']")
|
||
if price_meta and price_meta.get("content"):
|
||
price = parse_price(price_meta["content"])
|
||
if price is None:
|
||
price_el = card.select_one(".product-price-and-shipping span.price")
|
||
if price_el:
|
||
price = parse_price(price_el.get_text())
|
||
|
||
orig = None
|
||
orig_el = card.select_one(".ppc_normal_price")
|
||
if orig_el:
|
||
orig = parse_price(orig_el.get_text())
|
||
|
||
grade_spans = card.select(".product-variant-class span")
|
||
grade_texts = [s.get_text(strip=True) for s in grade_spans
|
||
if s.get_text(strip=True) in {"A", "B", "C", "D", "Premium"}]
|
||
seen_grades: dict = {}
|
||
for g in grade_texts:
|
||
seen_grades[g] = None
|
||
if seen_grades:
|
||
# Map each raw grade individually; take the best (first unique) result
|
||
mapped = list(dict.fromkeys(
|
||
map_condition_grade(g, "billigteknik") for g in seen_grades
|
||
))
|
||
grade = mapped[0] if len(mapped) == 1 else "/".join(mapped)
|
||
else:
|
||
grade = None
|
||
|
||
condition_detail = _extract_condition_detail(title)
|
||
|
||
results.append({
|
||
"url": href,
|
||
"title": title,
|
||
"price_sek": price,
|
||
"orig_price_sek": orig,
|
||
"condition_grade": grade,
|
||
"condition_detail": condition_detail,
|
||
"category": _category_from_url(href),
|
||
})
|
||
|
||
except Exception as exc:
|
||
log.debug(f"Card parse error: {exc}")
|
||
|
||
return results
|
||
|
||
|
||
def _category_from_url(href: str) -> str:
|
||
"""Extract the category slug from a product URL."""
|
||
m = re.search(r"billigteknik\.se/([^/]+)/", href)
|
||
return m.group(1) if m else ""
|
||
|
||
|
||
# ── Product detail parser ─────────────────────────────────────────────────────
|
||
|
||
def parse_product_page(html: str, url: str) -> dict:
|
||
"""
|
||
Parse a product detail page.
|
||
Returns a dict with all extractable fields (key names match DB columns).
|
||
"""
|
||
soup = BeautifulSoup(html, "lxml")
|
||
data: dict = {"url": url}
|
||
|
||
h1 = soup.select_one("h1") or soup.select_one(".product-name")
|
||
data["title"] = _clean_title(h1.get_text(strip=True)) if h1 else ""
|
||
|
||
for block in soup.select(".product-reference-data"):
|
||
label_el = block.select_one(".name.label")
|
||
value_el = block.select_one(".value")
|
||
if not label_el or not value_el:
|
||
continue
|
||
ltext = label_el.get_text(strip=True).rstrip(":")
|
||
vtext = value_el.get_text(strip=True)
|
||
if re.search(r"Tillverkare|Manufacturer|Brand", ltext, re.I):
|
||
# Normalise legacy full names to the common short brand name.
|
||
brand = vtext
|
||
if re.search(r"hewlett.?packard", brand, re.I):
|
||
brand = "HP"
|
||
data.setdefault("brand", brand)
|
||
elif re.search(r"^Referens", ltext, re.I):
|
||
data.setdefault("model_ref", vtext)
|
||
elif re.search(r"Ean13|EAN", ltext, re.I):
|
||
data.setdefault("ean", vtext)
|
||
|
||
price_span = soup.select_one(".current-price span[content]")
|
||
if price_span and price_span.get("content"):
|
||
p = parse_price(price_span["content"])
|
||
if p:
|
||
data["price_sek"] = p
|
||
else:
|
||
price_el = soup.select_one(".current-price span.price") or soup.select_one("span.price")
|
||
if price_el:
|
||
p = parse_price(price_el.get_text())
|
||
if p:
|
||
data["price_sek"] = p
|
||
|
||
orig_el = soup.select_one(".ppc_normal_price")
|
||
if orig_el:
|
||
op = parse_price(orig_el.get_text())
|
||
if op:
|
||
data["original_price_sek"] = op
|
||
|
||
specs: dict[str, str] = {}
|
||
|
||
best_table = max(soup.select("table"), key=lambda t: len(t.select("tr")), default=None)
|
||
if best_table:
|
||
for row in best_table.select("tr"):
|
||
cells = row.select("td, th")
|
||
if len(cells) >= 2:
|
||
key = cells[0].get_text(strip=True)
|
||
val = cells[1].get_text(strip=True)
|
||
if key.lower() in ("funktion", "function", "") or val.lower() in ("specifikation", ""):
|
||
continue
|
||
if key and val:
|
||
specs[key] = val
|
||
|
||
if not specs:
|
||
for dl in soup.select("dl.data-sheet, dl"):
|
||
for dt, dd in zip(dl.select("dt"), dl.select("dd")):
|
||
k = dt.get_text(strip=True)
|
||
v = dd.get_text(strip=True)
|
||
if k and v:
|
||
specs[k] = v
|
||
|
||
data["specs_raw"] = json.dumps(specs, ensure_ascii=False)
|
||
|
||
def s(pattern: str) -> str:
|
||
rx = re.compile(pattern, re.IGNORECASE)
|
||
for k, v in specs.items():
|
||
if rx.search(k):
|
||
return v
|
||
return ""
|
||
|
||
screen_raw = s(r"skärmstor|screen.?size|display.?size")
|
||
# Extract just the numeric size anywhere in the string:
|
||
# "14\" Full HD LED-skärm" → "14\"", "Pekskärm 11.6\"" → "11.6\""
|
||
_sz_m = re.search(r'[\d,.]+\s*(?:[-–]?\s*tum|["\u2033\u201d\u201c])', screen_raw.strip(), re.I)
|
||
data["screen_size"] = _sz_m.group(0).strip() if _sz_m else screen_raw
|
||
|
||
res_raw = s(r"upplösning|resolution")
|
||
data["screen_resolution"] = res_raw
|
||
|
||
screen_all = (screen_raw + " " + res_raw).lower()
|
||
for stype in ("oled", "retina", "ips", "tn", "va", "lcd", "led"):
|
||
if stype in screen_all:
|
||
data["screen_type"] = stype.upper()
|
||
break
|
||
|
||
data["touchscreen"] = 1 if re.search(r"touch|pek", screen_raw, re.I) else 0
|
||
|
||
cpu_raw = s(r"^processor$")
|
||
data["cpu_full"] = cpu_raw
|
||
|
||
cpu_model_m = re.match(
|
||
r"((?:Intel\s+Core\s+\w[\w-]+|Intel\s+Core\s+Ultra\s+\w[\w-]+|"
|
||
r"AMD\s+Ryzen\s+\d+[\s\w-]+?|Apple\s+M\d[\w\s]*?))\s+\d",
|
||
cpu_raw,
|
||
)
|
||
if cpu_model_m:
|
||
data["cpu_model"] = cpu_model_m.group(1).strip()
|
||
else:
|
||
cpu_model_m2 = re.match(r"([\w\s\-]+?)\s+\d+[.,]\d+\s*GHz", cpu_raw)
|
||
if cpu_model_m2:
|
||
data["cpu_model"] = cpu_model_m2.group(1).strip()
|
||
|
||
base_m = re.search(r"(\d+[.,]\d+)\s*GHz", cpu_raw, re.I)
|
||
if base_m:
|
||
data["cpu_base_ghz"] = float(base_m.group(1).replace(",", "."))
|
||
|
||
turbo_m = re.search(r"\((\d+[.,]\d+)\s*GHz\s*Turbo\)", cpu_raw, re.I)
|
||
if turbo_m:
|
||
data["cpu_turbo_ghz"] = float(turbo_m.group(1).replace(",", "."))
|
||
|
||
cores_raw = s(r"processorkärnor|cpu.?cores|antal.+kärn")
|
||
cores_m = re.search(r"(\d+)", cores_raw)
|
||
data["cpu_cores"] = int(cores_m.group(1)) if cores_m else None
|
||
|
||
data["cpu_cache"] = s(r"cacheminne|l[23].?cache|cache")
|
||
|
||
ht_raw = s(r"flertrådsteknik|hyperthread|smt")
|
||
if ht_raw and re.search(r"stödjer|ja|yes|enabled", ht_raw, re.I):
|
||
cores = data.get("cpu_cores")
|
||
if cores:
|
||
data["cpu_threads"] = cores * 2
|
||
|
||
for source in (cpu_raw, data.get("title", "")):
|
||
gen_m = re.search(r"(\d{1,2})(?:th|st|nd|rd)\s*(?:gen)?", source, re.I)
|
||
if gen_m:
|
||
data["cpu_generation"] = f"{gen_m.group(1)}th"
|
||
break
|
||
apple_m = re.search(r"Apple\s+(M\d(?:\s+(?:Pro|Max|Ultra))?)", source, re.I)
|
||
if apple_m:
|
||
data["cpu_generation"] = apple_m.group(1)
|
||
break
|
||
|
||
ram_raw = s(r"^minne$|^ram$|ram-minne|ram\s+minne")
|
||
if not ram_raw:
|
||
ram_raw = s(r"minne")
|
||
|
||
ram_m = re.search(r"(\d+)\s*GB", ram_raw, re.I)
|
||
data["ram_gb"] = int(ram_m.group(1)) if ram_m else None
|
||
|
||
ram_type_m = re.search(r"(LPDDR\d+[A-Z]*|DDR\d+[A-Z]*)", ram_raw, re.I)
|
||
data["ram_type"] = ram_type_m.group(1).upper() if ram_type_m else None
|
||
|
||
max_ram_raw = s(r"maximal.+minne|max.?ram")
|
||
max_ram_m = re.search(r"(\d+)\s*GB", max_ram_raw, re.I)
|
||
data["max_ram_gb"] = int(max_ram_m.group(1)) if max_ram_m else None
|
||
|
||
storage_raw = s(r"lagringsutrymme|hårddisk|storage")
|
||
stor_m = re.search(r"(\d+)\s*(GB|TB)", storage_raw, re.I)
|
||
if stor_m:
|
||
val = int(stor_m.group(1))
|
||
unit = stor_m.group(2).upper()
|
||
data["storage_gb"] = val * 1024 if unit == "TB" else val
|
||
if re.search(r"nvme", storage_raw, re.I):
|
||
data["storage_type"] = "NVMe SSD"
|
||
elif re.search(r"\bssd\b", storage_raw, re.I):
|
||
data["storage_type"] = "SSD"
|
||
elif re.search(r"emmc", storage_raw, re.I):
|
||
data["storage_type"] = "eMMC"
|
||
elif re.search(r"\bhdd\b", storage_raw, re.I):
|
||
data["storage_type"] = "HDD"
|
||
|
||
data["gpu"] = s(r"grafikkort|gpu|graphics.?card")
|
||
|
||
wifi_raw = s(r"trådlöst|wi-?fi|wlan|wireless")
|
||
data["wifi"] = wifi_raw
|
||
|
||
bt_raw = s(r"^bluetooth")
|
||
data["bluetooth"] = bt_raw
|
||
|
||
eth_raw = s(r"trådat.+nät|ethernet|nätverksport|wired")
|
||
if re.search(r"nej|saknar|no\b", eth_raw, re.I):
|
||
data["wired_ethernet"] = 0
|
||
elif eth_raw:
|
||
data["wired_ethernet"] = 1
|
||
else:
|
||
data["wired_ethernet"] = 0
|
||
|
||
lte_spec = s(r"4g|5g|lte|mobilnät|modem")
|
||
title_text = data.get("title", "")
|
||
data["has_4g_5g"] = 1 if lte_spec or re.search(r"\b4G\b|\b5G\b|LTE", title_text) else 0
|
||
|
||
data["usb_ports"] = s(r"usb")
|
||
data["thunderbolt_ports"] = s(r"thunderbolt")
|
||
data["hdmi"] = s(r"^hdmi")
|
||
|
||
weight_raw = s(r"vikt|weight")
|
||
wm = re.search(r"(\d+[.,]\d+)\s*kg", weight_raw, re.I)
|
||
data["weight_kg"] = float(wm.group(1).replace(",", ".")) if wm else None
|
||
|
||
for dim, field in (
|
||
(r"bredd|width", "width_cm"),
|
||
(r"höjd|height|tjocklek", "height_cm"),
|
||
(r"^djup|depth", "depth_cm"),
|
||
):
|
||
raw = s(dim)
|
||
dm = re.search(r"(\d+[.,]\d*)\s*cm", raw, re.I)
|
||
data[field] = float(dm.group(1).replace(",", ".")) if dm else None
|
||
|
||
data["battery"] = s(r"^batteri$|battery")
|
||
|
||
data["operating_system"] = s(r"operativsystem|os\b")
|
||
|
||
kb_raw = s(r"tangentbord|keyboard")
|
||
data["keyboard_backlit"] = 1 if re.search(r"bakgrundsbelyst|backlit|backlight", kb_raw, re.I) else 0
|
||
|
||
webcam_raw = s(r"webbkamera|webcam|kamera")
|
||
data["webcam"] = (
|
||
0 if re.search(r"nej|saknar|no camera", webcam_raw, re.I)
|
||
else (1 if webcam_raw else 0)
|
||
)
|
||
|
||
opt_raw = s(r"optisk|optical|cd|dvd")
|
||
data["optical_drive"] = (
|
||
0 if (not opt_raw or re.search(r"nej|saknar|no\b", opt_raw, re.I))
|
||
else 1
|
||
)
|
||
|
||
data["warranty"] = s(r"garanti|warranty")
|
||
data["eco_cert"] = s(r"energy.?star|epeat|miljöcert")
|
||
|
||
# ── Images ────────────────────────────────────────────────────────────
|
||
# Thumbnail strip: each img.js-thumb carries data-image-large-src with
|
||
# the full-resolution URL. Fall back to the main cover image if none found.
|
||
large_urls: list[str] = []
|
||
for thumb in soup.select("img.js-thumb[data-image-large-src]"):
|
||
src = thumb.get("data-image-large-src", "").strip()
|
||
if src and src not in large_urls:
|
||
large_urls.append(src)
|
||
if not large_urls:
|
||
cover = soup.select_one("img.js-qv-product-cover")
|
||
if cover and cover.get("src"):
|
||
large_urls.append(cover["src"].strip())
|
||
data["image_urls"] = json.dumps(large_urls) if large_urls else None
|
||
|
||
data["source"] = "billigteknik"
|
||
data["scraped_at"] = datetime.now().isoformat()
|
||
return data
|
||
|
||
|
||
# ── Phase 1: collect product URLs ─────────────────────────────────────────────
|
||
|
||
def collect_urls(conn: sqlite3.Connection) -> set[str]:
|
||
"""Crawl all listing pages, re-queue every live product, return the live URL set.
|
||
|
||
Uses INSERT OR REPLACE so existing queue entries have their done flag reset
|
||
to 0 — guaranteeing that re-runs always re-scrape and update every listing.
|
||
Returns the complete set of URLs currently live on the site so the caller
|
||
can remove sold/delisted rows from the database.
|
||
"""
|
||
log.info("═══ Phase 1: Collecting product URLs ═══")
|
||
page = 1
|
||
live_urls: set[str] = set()
|
||
empty_streak = 0
|
||
|
||
while page <= MAX_PAGES:
|
||
url = CATEGORY_URL if page == 1 else f"{CATEGORY_URL}?page={page}"
|
||
log.info(f"Listing page {page}: {url}")
|
||
|
||
resp = polite_get(url)
|
||
if not resp:
|
||
empty_streak += 1
|
||
if empty_streak >= 3:
|
||
log.error("3 consecutive failed listing pages – aborting Phase 1.")
|
||
break
|
||
page += 1
|
||
_sleep()
|
||
continue
|
||
|
||
products = parse_listing_page(resp.text)
|
||
|
||
if not products:
|
||
log.info(f" No products found on page {page}.")
|
||
empty_streak += 1
|
||
if empty_streak >= 2:
|
||
log.info(" Two empty pages in a row – end of catalog.")
|
||
break
|
||
else:
|
||
empty_streak = 0
|
||
for p in products:
|
||
live_urls.add(p["url"])
|
||
try:
|
||
conn.execute(
|
||
"""INSERT OR REPLACE INTO scrape_queue
|
||
(url, source, title, price_sek, orig_price_sek,
|
||
condition_grade, condition_detail, category, done)
|
||
VALUES (?, 'billigteknik', ?, ?, ?, ?, ?, ?, 0)""",
|
||
(
|
||
p["url"], p.get("title"),
|
||
p.get("price_sek"), p.get("orig_price_sek"),
|
||
p.get("condition_grade"), p.get("condition_detail"),
|
||
p.get("category"),
|
||
),
|
||
)
|
||
except Exception as exc:
|
||
log.debug(f" Queue insert error: {exc}")
|
||
conn.commit()
|
||
log.info(f" {len(products)} products on page (running total: {len(live_urls)})")
|
||
|
||
page += 1
|
||
_sleep()
|
||
|
||
queued = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=0").fetchone()[0]
|
||
log.info(f"Phase 1 done. {queued} URLs queued for scraping. {len(live_urls)} live URLs found.")
|
||
return live_urls
|
||
|
||
|
||
# ── Phase 2: scrape product detail pages ──────────────────────────────────────
|
||
|
||
def scrape_products(conn: sqlite3.Connection) -> None:
|
||
"""Work through scrape_queue, fetching and storing each product page."""
|
||
log.info("═══ Phase 2: Scraping product detail pages ═══")
|
||
|
||
queue = conn.execute(
|
||
"""SELECT url, title, price_sek, orig_price_sek,
|
||
condition_grade, condition_detail, category
|
||
FROM scrape_queue WHERE source='billigteknik' AND done=0 ORDER BY rowid"""
|
||
).fetchall()
|
||
|
||
total = len(queue)
|
||
log.info(f"{total} products to scrape.")
|
||
|
||
for i, row in enumerate(queue, 1):
|
||
url = row["url"]
|
||
log.info(f"[{i}/{total}] {url}")
|
||
|
||
resp = polite_get(url)
|
||
if not resp:
|
||
conn.execute("UPDATE scrape_queue SET done=-1 WHERE url=?", (url,))
|
||
conn.commit()
|
||
_sleep()
|
||
continue
|
||
|
||
try:
|
||
detail = parse_product_page(resp.text, url)
|
||
|
||
if row["price_sek"]:
|
||
detail["price_sek"] = row["price_sek"]
|
||
if row["orig_price_sek"]:
|
||
detail["original_price_sek"] = row["orig_price_sek"]
|
||
if row["condition_grade"]:
|
||
detail["condition_grade"] = row["condition_grade"]
|
||
if row["condition_detail"] is not None and row["condition_detail"] != "":
|
||
detail["condition_detail"] = row["condition_detail"]
|
||
if row["category"]:
|
||
detail["category"] = row["category"]
|
||
|
||
if not detail.get("title") and row["title"]:
|
||
detail["title"] = _clean_title(row["title"])
|
||
|
||
conn.execute(INSERT_SQL, _safe_row(detail))
|
||
conn.execute("UPDATE scrape_queue SET done=1 WHERE url=?", (url,))
|
||
conn.commit()
|
||
log.info(f" ✓ {detail.get('title', '(no title)')}")
|
||
|
||
except Exception as exc:
|
||
log.error(f" Error processing {url}: {exc}", exc_info=True)
|
||
conn.execute("UPDATE scrape_queue SET done=-1 WHERE url=?", (url,))
|
||
conn.commit()
|
||
|
||
_sleep()
|
||
|
||
done = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=1").fetchone()[0]
|
||
failed = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=-1").fetchone()[0]
|
||
log.info(f"Phase 2 done. ✓ {done} scraped ✗ {failed} failed.")
|
||
|
||
|
||
# ── Entry point ───────────────────────────────────────────────────────────────
|
||
|
||
def run(conn: sqlite3.Connection, skip_phase1: bool = False) -> None:
|
||
"""Run the BilligTeknik scrape (Phase 1 + Phase 2 + sold-item cleanup)."""
|
||
live_urls: set[str] = set()
|
||
|
||
if skip_phase1:
|
||
queued = conn.execute("SELECT COUNT(*) FROM scrape_queue WHERE source='billigteknik' AND done=0").fetchone()[0]
|
||
log.info(f"--skip-phase1: {queued} pending URLs in queue.")
|
||
else:
|
||
live_urls = collect_urls(conn)
|
||
|
||
scrape_products(conn)
|
||
|
||
# Remove any DB rows that are no longer listed on the site (sold / removed).
|
||
if live_urls:
|
||
db_urls = set(
|
||
r[0] for r in conn.execute(
|
||
"SELECT url FROM laptops WHERE source='billigteknik'"
|
||
).fetchall()
|
||
)
|
||
sold = db_urls - live_urls
|
||
if sold:
|
||
for sold_url in sold:
|
||
conn.execute("DELETE FROM laptops WHERE url=?", (sold_url,))
|
||
conn.commit()
|
||
log.info(f" Removed {len(sold)} sold/unlisted product(s) from DB.")
|
||
else:
|
||
log.info(" No sold/unlisted products found.")
|
||
|
||
scraped = conn.execute("SELECT COUNT(*) FROM laptops WHERE source='billigteknik'").fetchone()[0]
|
||
log.info(f"BilligTeknik done. {scraped} laptops in DB.")
|