373 lines
13 KiB
Python
373 lines
13 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Shared configuration, DB helpers, and HTTP utilities used by all scrapers.
|
||
"""
|
||
|
||
import os
|
||
import time
|
||
import random
|
||
import logging
|
||
import sqlite3
|
||
from typing import Optional
|
||
|
||
import requests
|
||
|
||
# ── Paths ─────────────────────────────────────────────────────────────────────
|
||
|
||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||
DB_PATH = os.path.normpath(os.path.join(_HERE, "..", "database.db"))
|
||
LOG_PATH = os.path.normpath(os.path.join(_HERE, "..", "scraper.log"))
|
||
|
||
# ── Delays ────────────────────────────────────────────────────────────────────
|
||
|
||
DELAY_MIN = 1.5
|
||
DELAY_MAX = 3.5
|
||
|
||
# ── User-Agent rotation ───────────────────────────────────────────────────────
|
||
|
||
USER_AGENTS = [
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.0.0",
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15",
|
||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||
]
|
||
|
||
# Connection: close ensures every request uses a fresh TCP socket.
|
||
# This prevents "stale connection" 504 errors caused by the server closing an
|
||
# idle keep-alive connection while our session still thinks it is open.
|
||
BASE_HEADERS = {
|
||
"Accept-Language": "sv-SE,sv;q=0.9,en;q=0.8",
|
||
"Accept": "text/html,application/xhtml+xml,application/xhtml;q=0.9,*/*;q=0.8",
|
||
"Accept-Encoding": "gzip, deflate",
|
||
"Connection": "close",
|
||
}
|
||
|
||
# ── Logging ───────────────────────────────────────────────────────────────────
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s [%(levelname)s] %(message)s",
|
||
handlers=[
|
||
logging.FileHandler(LOG_PATH, encoding="utf-8"),
|
||
logging.StreamHandler(),
|
||
],
|
||
)
|
||
log = logging.getLogger(__name__)
|
||
|
||
# ── Condition grade normalisation ─────────────────────────────────────────────
|
||
|
||
_BT_GRADE_MAP: dict[str, str] = {
|
||
"premium": "S",
|
||
"a": "A",
|
||
"a/b": "A",
|
||
"b/a": "A",
|
||
"b": "B",
|
||
"b/c": "B",
|
||
"c/b": "B",
|
||
"c": "C",
|
||
"c/d": "C",
|
||
"d/c": "C",
|
||
"d": "C",
|
||
}
|
||
|
||
_NUVOO_GRADE_MAP: dict[str, str] = {
|
||
"nyskick": "S",
|
||
"utmärkt": "A",
|
||
"bra": "B",
|
||
"okej": "C",
|
||
}
|
||
|
||
|
||
def map_condition_grade(raw: str, source: str) -> str:
|
||
"""Normalise a source-specific condition label to our unified S/A/B/C scale.
|
||
|
||
For BilligTeknik slash-joined grades like "A/B" we always take the better
|
||
(leftmost) component, then look up the combined key first.
|
||
|
||
Unknown values are returned unchanged so they surface during review.
|
||
"""
|
||
key = raw.strip().lower()
|
||
if source == "billigteknik":
|
||
return _BT_GRADE_MAP.get(key, raw.strip())
|
||
if source == "nuvoo":
|
||
return _NUVOO_GRADE_MAP.get(key, raw.strip())
|
||
return raw.strip()
|
||
|
||
# ── Database schema ───────────────────────────────────────────────────────────
|
||
|
||
SCHEMA = """
|
||
CREATE TABLE IF NOT EXISTS laptops (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
url TEXT UNIQUE NOT NULL,
|
||
title TEXT,
|
||
brand TEXT,
|
||
model_ref TEXT,
|
||
ean TEXT,
|
||
category TEXT,
|
||
|
||
price_sek INTEGER,
|
||
original_price_sek INTEGER,
|
||
|
||
condition_grade TEXT,
|
||
condition_detail TEXT,
|
||
|
||
screen_size TEXT,
|
||
screen_resolution TEXT,
|
||
screen_type TEXT,
|
||
touchscreen INTEGER DEFAULT 0,
|
||
|
||
cpu_full TEXT,
|
||
cpu_model TEXT,
|
||
cpu_cores INTEGER,
|
||
cpu_threads INTEGER,
|
||
cpu_base_ghz REAL,
|
||
cpu_turbo_ghz REAL,
|
||
cpu_cache TEXT,
|
||
cpu_generation TEXT,
|
||
|
||
ram_gb INTEGER,
|
||
ram_type TEXT,
|
||
max_ram_gb INTEGER,
|
||
|
||
storage_gb INTEGER,
|
||
storage_type TEXT,
|
||
|
||
gpu TEXT,
|
||
|
||
wifi TEXT,
|
||
bluetooth TEXT,
|
||
wired_ethernet INTEGER DEFAULT 0,
|
||
has_4g_5g INTEGER DEFAULT 0,
|
||
|
||
usb_ports TEXT,
|
||
thunderbolt_ports TEXT,
|
||
hdmi TEXT,
|
||
|
||
weight_kg REAL,
|
||
width_cm REAL,
|
||
height_cm REAL,
|
||
depth_cm REAL,
|
||
battery TEXT,
|
||
|
||
operating_system TEXT,
|
||
keyboard_backlit INTEGER DEFAULT 0,
|
||
webcam INTEGER DEFAULT 0,
|
||
optical_drive INTEGER DEFAULT 0,
|
||
warranty TEXT,
|
||
eco_cert TEXT,
|
||
|
||
specs_raw TEXT,
|
||
|
||
source TEXT DEFAULT 'billigteknik',
|
||
image_urls TEXT,
|
||
|
||
scraped_at TEXT
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS scrape_queue (
|
||
url TEXT PRIMARY KEY,
|
||
source TEXT NOT NULL DEFAULT 'billigteknik',
|
||
-- BilligTeknik listing-page cache (NULL for Nuvoo rows)
|
||
title TEXT,
|
||
price_sek INTEGER,
|
||
orig_price_sek INTEGER,
|
||
condition_grade TEXT,
|
||
condition_detail TEXT,
|
||
category TEXT,
|
||
-- Nuvoo Shopify handle (NULL for BilligTeknik rows)
|
||
handle TEXT,
|
||
done INTEGER DEFAULT 0,
|
||
added_at TEXT DEFAULT (datetime('now'))
|
||
);
|
||
|
||
CREATE TABLE IF NOT EXISTS cpu_specs (
|
||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||
cpu_key TEXT UNIQUE NOT NULL,
|
||
cpu_model_raw TEXT,
|
||
vendor TEXT,
|
||
full_name TEXT,
|
||
brand_name TEXT,
|
||
codename TEXT,
|
||
lithography TEXT,
|
||
launch_date TEXT,
|
||
total_cores INTEGER,
|
||
performance_cores INTEGER,
|
||
efficient_cores INTEGER,
|
||
total_threads INTEGER,
|
||
base_freq_ghz REAL,
|
||
max_turbo_ghz REAL,
|
||
l2_cache TEXT,
|
||
l3_cache TEXT,
|
||
tdp_w INTEGER,
|
||
max_tdp_w INTEGER,
|
||
pcie_version TEXT,
|
||
pcie_lanes INTEGER,
|
||
max_ram_gb INTEGER,
|
||
ram_types TEXT,
|
||
ram_speeds_mhz TEXT,
|
||
ecc_support INTEGER,
|
||
igpu_name TEXT,
|
||
igpu_base_mhz INTEGER,
|
||
igpu_boost_mhz INTEGER,
|
||
source_url TEXT,
|
||
lookup_status TEXT DEFAULT 'pending',
|
||
scraped_at TEXT
|
||
);
|
||
"""
|
||
|
||
|
||
def init_db(path: str, reset: bool = False) -> sqlite3.Connection:
|
||
conn = sqlite3.connect(path)
|
||
conn.row_factory = sqlite3.Row
|
||
conn.execute("PRAGMA journal_mode=WAL")
|
||
conn.execute("PRAGMA synchronous=NORMAL")
|
||
if reset:
|
||
conn.executescript("DROP TABLE IF EXISTS laptops; DROP TABLE IF EXISTS scrape_queue;")
|
||
log.info("Database reset.")
|
||
conn.executescript(SCHEMA)
|
||
conn.commit()
|
||
return conn
|
||
|
||
|
||
# ── HTTP helpers ──────────────────────────────────────────────────────────────
|
||
|
||
def _make_session() -> requests.Session:
|
||
"""Create a fresh requests.Session with correct headers."""
|
||
s = requests.Session()
|
||
s.headers.update(BASE_HEADERS)
|
||
return s
|
||
|
||
|
||
_session = _make_session()
|
||
|
||
|
||
def polite_get(url: str, retries: int = 3) -> Optional[requests.Response]:
|
||
"""Rate-limited GET with retry logic. Returns None on permanent failure."""
|
||
global _session
|
||
for attempt in range(1, retries + 1):
|
||
_session.headers["User-Agent"] = random.choice(USER_AGENTS)
|
||
try:
|
||
resp = _session.get(url, timeout=20)
|
||
|
||
if resp.status_code == 200:
|
||
return resp
|
||
|
||
if resp.status_code == 429:
|
||
wait = 60 + random.uniform(10, 30)
|
||
log.warning(
|
||
f"Rate-limited (429). Sleeping {wait:.0f}s "
|
||
f"(attempt {attempt}/{retries})"
|
||
)
|
||
time.sleep(wait)
|
||
|
||
elif resp.status_code == 502:
|
||
_session.close()
|
||
_session = _make_session()
|
||
if attempt == 1:
|
||
wait = 180 + random.uniform(0, 20)
|
||
log.warning(
|
||
f"HTTP 502 Bad Gateway – backing off {wait:.0f}s before retrying "
|
||
f"(attempt {attempt}/{retries}): {url}"
|
||
)
|
||
time.sleep(wait)
|
||
else:
|
||
log.warning(f"HTTP 502 again (attempt {attempt}/{retries}): {url}")
|
||
time.sleep(random.uniform(2, 3))
|
||
|
||
elif resp.status_code in (404, 410):
|
||
log.info(f"Product gone ({resp.status_code}): {url}")
|
||
return None
|
||
|
||
else:
|
||
_session.close()
|
||
_session = _make_session()
|
||
log.warning(
|
||
f"HTTP {resp.status_code} for {url} (attempt {attempt}/{retries})"
|
||
)
|
||
time.sleep(5 * attempt)
|
||
|
||
except requests.RequestException as exc:
|
||
log.warning(f"Request error attempt {attempt}/{retries}: {exc}")
|
||
_session.close()
|
||
_session = _make_session()
|
||
time.sleep(10 * attempt)
|
||
|
||
log.error(f"Gave up after {retries} attempts: {url}")
|
||
return None
|
||
|
||
|
||
def _sleep():
|
||
"""Polite random delay between requests."""
|
||
time.sleep(random.uniform(DELAY_MIN, DELAY_MAX))
|
||
|
||
|
||
# ── INSERT helpers ────────────────────────────────────────────────────────────
|
||
|
||
INSERT_SQL = """
|
||
INSERT OR REPLACE INTO laptops (
|
||
url, title, brand, model_ref, ean, category,
|
||
price_sek, original_price_sek,
|
||
condition_grade, condition_detail,
|
||
screen_size, screen_resolution, screen_type, touchscreen,
|
||
cpu_full, cpu_model, cpu_cores, cpu_threads,
|
||
cpu_base_ghz, cpu_turbo_ghz, cpu_cache, cpu_generation,
|
||
ram_gb, ram_type, max_ram_gb,
|
||
storage_gb, storage_type,
|
||
gpu,
|
||
wifi, bluetooth, wired_ethernet, has_4g_5g,
|
||
usb_ports, thunderbolt_ports, hdmi,
|
||
weight_kg, width_cm, height_cm, depth_cm, battery,
|
||
operating_system,
|
||
keyboard_backlit, webcam, optical_drive,
|
||
warranty, eco_cert,
|
||
specs_raw, source, image_urls, scraped_at
|
||
) VALUES (
|
||
:url, :title, :brand, :model_ref, :ean, :category,
|
||
:price_sek, :original_price_sek,
|
||
:condition_grade, :condition_detail,
|
||
:screen_size, :screen_resolution, :screen_type, :touchscreen,
|
||
:cpu_full, :cpu_model, :cpu_cores, :cpu_threads,
|
||
:cpu_base_ghz, :cpu_turbo_ghz, :cpu_cache, :cpu_generation,
|
||
:ram_gb, :ram_type, :max_ram_gb,
|
||
:storage_gb, :storage_type,
|
||
:gpu,
|
||
:wifi, :bluetooth, :wired_ethernet, :has_4g_5g,
|
||
:usb_ports, :thunderbolt_ports, :hdmi,
|
||
:weight_kg, :width_cm, :height_cm, :depth_cm, :battery,
|
||
:operating_system,
|
||
:keyboard_backlit, :webcam, :optical_drive,
|
||
:warranty, :eco_cert,
|
||
:specs_raw, :source, :image_urls, :scraped_at
|
||
)
|
||
"""
|
||
|
||
_ALL_COLUMNS = [
|
||
"url", "title", "brand", "model_ref", "ean", "category",
|
||
"price_sek", "original_price_sek",
|
||
"condition_grade", "condition_detail",
|
||
"screen_size", "screen_resolution", "screen_type", "touchscreen",
|
||
"cpu_full", "cpu_model", "cpu_cores", "cpu_threads",
|
||
"cpu_base_ghz", "cpu_turbo_ghz", "cpu_cache", "cpu_generation",
|
||
"ram_gb", "ram_type", "max_ram_gb",
|
||
"storage_gb", "storage_type",
|
||
"gpu",
|
||
"wifi", "bluetooth", "wired_ethernet", "has_4g_5g",
|
||
"usb_ports", "thunderbolt_ports", "hdmi",
|
||
"weight_kg", "width_cm", "height_cm", "depth_cm", "battery",
|
||
"operating_system",
|
||
"keyboard_backlit", "webcam", "optical_drive",
|
||
"warranty", "eco_cert",
|
||
"specs_raw", "source", "image_urls", "scraped_at",
|
||
]
|
||
|
||
|
||
def _safe_row(data: dict) -> dict:
|
||
"""Ensure every expected column key exists (default None)."""
|
||
return {col: data.get(col) for col in _ALL_COLUMNS}
|