648 lines
27 KiB
Python
648 lines
27 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Laptop Browser & Scraping Server
|
||
===================================
|
||
Flask web server that:
|
||
1. Serves the filterable, searchable laptop browser UI at http://localhost:5000
|
||
2. Exposes a /api/scrape/* REST interface so scraping can be triggered
|
||
directly from the browser — no CLI needed.
|
||
|
||
Run:
|
||
python main.py
|
||
Then open: http://localhost:5000
|
||
"""
|
||
|
||
import os
|
||
import json
|
||
import sqlite3
|
||
import threading
|
||
from datetime import datetime
|
||
|
||
from flask import Flask, jsonify, request, send_from_directory
|
||
|
||
# ── Scrapers (import at module-level; they are lightweight until run() is called)
|
||
import scrapers.scraper_billigteknik as _scraper_billig
|
||
import scrapers.scraper_nuvoo as _scraper_nuvoo
|
||
import scrapers.scraper_cpu as _scraper_cpu
|
||
from scrapers.shared import init_db, DB_PATH as _SCRAPER_DB_PATH
|
||
|
||
# ── Flask app ─────────────────────────────────────────────────────────────────
|
||
|
||
DB_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "database.db")
|
||
SEED_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "seed.db")
|
||
|
||
# ── Seed bootstrap ────────────────────────────────────────────────────────────
|
||
# On first run, if no database exists yet but a seed.db is bundled, copy it
|
||
# into place so users start with pre-populated CPU data.
|
||
if not os.path.exists(DB_PATH) and os.path.exists(SEED_PATH):
|
||
import shutil
|
||
shutil.copy2(SEED_PATH, DB_PATH)
|
||
|
||
app = Flask(__name__, static_folder="static", static_url_path="/static")
|
||
|
||
|
||
# ── Database helpers ──────────────────────────────────────────────────────────
|
||
|
||
def get_db():
|
||
conn = sqlite3.connect(DB_PATH, check_same_thread=False)
|
||
conn.row_factory = sqlite3.Row
|
||
return conn
|
||
|
||
|
||
def row_to_dict(row):
|
||
d = dict(row)
|
||
if d.get("specs_raw"):
|
||
try:
|
||
d["specs_raw"] = json.loads(d["specs_raw"])
|
||
except (json.JSONDecodeError, TypeError):
|
||
pass
|
||
if d.get("image_urls"):
|
||
try:
|
||
d["image_urls"] = json.loads(d["image_urls"])
|
||
except (json.JSONDecodeError, TypeError):
|
||
pass
|
||
return d
|
||
|
||
|
||
# Normalises the raw gpu string into a clean family name.
|
||
_GPU_FAMILY_SQL = """
|
||
CASE
|
||
WHEN l.gpu LIKE '%NVIDIA%' OR l.gpu LIKE '%GeForce%'
|
||
OR l.gpu LIKE '%Quadro%' OR l.gpu LIKE '%RTX A%'
|
||
OR l.gpu LIKE '%RTX 1000%' OR l.gpu LIKE '%RTX 2000%'
|
||
THEN 'NVIDIA'
|
||
WHEN l.gpu LIKE '%AMD Radeon Pro%'
|
||
OR (l.gpu LIKE '%AMD Radeon%' AND (l.gpu LIKE '%GDDR%' OR l.gpu LIKE '%VRAM%'))
|
||
THEN 'AMD Radeon'
|
||
WHEN l.gpu LIKE '%AMD Radeon%'
|
||
THEN 'AMD Radeon'
|
||
WHEN l.gpu LIKE '%Iris Xe%'
|
||
THEN 'Intel Iris Xe'
|
||
WHEN l.gpu LIKE '%Iris Plus%'
|
||
THEN 'Intel Iris Plus'
|
||
WHEN l.gpu LIKE '%UHD%' OR l.gpu LIKE '%Intel HD%'
|
||
THEN 'Intel UHD / HD'
|
||
WHEN l.gpu LIKE '%Apple%'
|
||
THEN 'Apple GPU'
|
||
WHEN l.gpu IS NOT NULL AND l.gpu != ''
|
||
THEN 'Other'
|
||
ELSE NULL
|
||
END
|
||
"""
|
||
|
||
_DEDICATED_GPU_SQL = """
|
||
(l.gpu LIKE '%GDDR%' OR l.gpu LIKE '%VRAM%'
|
||
OR l.gpu LIKE '%NVIDIA%' OR l.gpu LIKE '%GeForce%'
|
||
OR l.gpu LIKE '%Quadro%'
|
||
OR l.gpu LIKE '%Radeon RX%'
|
||
OR l.gpu LIKE '% MX%'
|
||
OR l.gpu LIKE '%RTX A%')
|
||
"""
|
||
|
||
# Inline CASE expression returning 1/0 for use in SELECT lists
|
||
_DEDICATED_GPU_CASE_SQL = """
|
||
(CASE WHEN (l.gpu LIKE '%GDDR%' OR l.gpu LIKE '%VRAM%'
|
||
OR l.gpu LIKE '%NVIDIA%' OR l.gpu LIKE '%GeForce%'
|
||
OR l.gpu LIKE '%Quadro%'
|
||
OR l.gpu LIKE '%Radeon RX%'
|
||
OR l.gpu LIKE '% MX%'
|
||
OR l.gpu LIKE '%RTX A%') THEN 1 ELSE 0 END)
|
||
"""
|
||
|
||
_GPU_FAMILY_CLAUSES = {
|
||
'NVIDIA': "(l.gpu LIKE '%NVIDIA%' OR l.gpu LIKE '%GeForce%' OR l.gpu LIKE '%Quadro%')",
|
||
'AMD Radeon': "l.gpu LIKE '%AMD Radeon%'",
|
||
'Intel Iris Xe': "l.gpu LIKE '%Iris Xe%'",
|
||
'Intel Iris Plus': "l.gpu LIKE '%Iris Plus%'",
|
||
'Intel UHD / HD': "(l.gpu LIKE '%UHD%' OR l.gpu LIKE '%Intel HD%')",
|
||
'Apple GPU': "l.gpu LIKE '%Apple%'",
|
||
}
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ── API: filter options ───────────────────────────────────────────────────────
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
|
||
@app.get("/api/filters")
|
||
def api_filters():
|
||
"""Return all distinct values for every filterable field."""
|
||
conn = get_db()
|
||
|
||
def distinct(col, table="laptops"):
|
||
rows = conn.execute(
|
||
f"SELECT DISTINCT {col} FROM {table} "
|
||
f"WHERE {col} IS NOT NULL AND {col} != '' ORDER BY {col} ASC"
|
||
).fetchall()
|
||
return [r[0] for r in rows]
|
||
|
||
def distinct_int(col):
|
||
rows = conn.execute(
|
||
f"SELECT DISTINCT {col} FROM laptops "
|
||
f"WHERE {col} IS NOT NULL ORDER BY {col} ASC"
|
||
).fetchall()
|
||
return [r[0] for r in rows]
|
||
|
||
brands = distinct("brand")
|
||
conditions = distinct("condition_grade")
|
||
screen_sizes = distinct("screen_size")
|
||
cpu_models = distinct("cpu_model")
|
||
ram_types = distinct("ram_type")
|
||
storage_types = distinct("storage_type")
|
||
os_list = distinct("operating_system")
|
||
ram_values = distinct_int("ram_gb")
|
||
storage_values = distinct_int("storage_gb")
|
||
cores_values = distinct_int("cpu_cores")
|
||
|
||
gpu_rows = conn.execute(f"""
|
||
SELECT ({_GPU_FAMILY_SQL}) AS fam, COUNT(*) AS n
|
||
FROM laptops l
|
||
WHERE fam IS NOT NULL
|
||
GROUP BY fam
|
||
ORDER BY n DESC
|
||
""").fetchall()
|
||
gpu_families = [{"name": r[0], "count": r[1]} for r in gpu_rows]
|
||
|
||
source_rows = conn.execute("""
|
||
SELECT COALESCE(source, 'billigteknik') AS src, COUNT(*) AS n
|
||
FROM laptops
|
||
GROUP BY src
|
||
ORDER BY n DESC
|
||
""").fetchall()
|
||
sources = [{"name": r[0], "count": r[1]} for r in source_rows]
|
||
|
||
price = conn.execute("SELECT MIN(price_sek), MAX(price_sek) FROM laptops WHERE price_sek IS NOT NULL").fetchone()
|
||
ram_r = conn.execute("SELECT MIN(ram_gb), MAX(ram_gb) FROM laptops WHERE ram_gb IS NOT NULL").fetchone()
|
||
stor_r = conn.execute("SELECT MIN(storage_gb), MAX(storage_gb) FROM laptops WHERE storage_gb IS NOT NULL").fetchone()
|
||
|
||
conn.close()
|
||
|
||
return jsonify({
|
||
"brands": brands,
|
||
"conditions": conditions,
|
||
"screen_sizes": screen_sizes,
|
||
"cpu_models": cpu_models,
|
||
"ram_types": ram_types,
|
||
"storage_types": storage_types,
|
||
"os_list": os_list,
|
||
"ram_values": ram_values,
|
||
"storage_values": storage_values,
|
||
"cpu_cores_values": cores_values,
|
||
"gpu_families": gpu_families,
|
||
"sources": sources,
|
||
"price_min": price[0] or 0,
|
||
"price_max": price[1] or 99999,
|
||
"ram_min": ram_r[0] or 0,
|
||
"ram_max": ram_r[1] or 256,
|
||
"storage_min": stor_r[0] or 0,
|
||
"storage_max": stor_r[1] or 8000,
|
||
})
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ── API: laptops list (with filters) ─────────────────────────────────────────
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
|
||
@app.get("/api/laptops")
|
||
def api_laptops():
|
||
"""
|
||
Return filtered list of laptops.
|
||
Query params:
|
||
q – free text search (title, brand, cpu_model, gpu)
|
||
brand – exact match (multi: brand=Dell&brand=HP)
|
||
condition – condition_grade exact (multi)
|
||
ram_min / ram_max
|
||
price_min / price_max
|
||
storage_min / storage_max
|
||
cpu_cores_min
|
||
ram_type – multi
|
||
storage_type – multi
|
||
screen_size – multi
|
||
gpu_family – multi
|
||
dedicated_gpu – 1 = only laptops with discrete GPU
|
||
sort – price_asc | price_desc | ram_desc | storage_desc | brand_asc
|
||
page / page_size
|
||
"""
|
||
p = request.args
|
||
|
||
clauses = []
|
||
params = []
|
||
|
||
q = p.get("q", "").strip()
|
||
if q:
|
||
clauses.append(
|
||
"(l.title LIKE ? OR l.brand LIKE ? OR l.cpu_model LIKE ? "
|
||
"OR l.gpu LIKE ? OR l.cpu_full LIKE ?)"
|
||
)
|
||
like = f"%{q}%"
|
||
params += [like, like, like, like, like]
|
||
|
||
def multi(param, col):
|
||
vals = p.getlist(param)
|
||
if vals:
|
||
placeholders = ",".join("?" * len(vals))
|
||
clauses.append(f"l.{col} IN ({placeholders})")
|
||
params.extend(vals)
|
||
|
||
multi("brand", "brand")
|
||
multi("condition", "condition_grade")
|
||
multi("ram_type", "ram_type")
|
||
multi("storage_type", "storage_type")
|
||
multi("screen_size", "screen_size")
|
||
multi("source", "source")
|
||
|
||
def range_filter(param_min, param_max, col):
|
||
lo = p.get(param_min)
|
||
hi = p.get(param_max)
|
||
if lo:
|
||
clauses.append(f"l.{col} >= ?")
|
||
params.append(int(lo))
|
||
if hi:
|
||
clauses.append(f"l.{col} <= ?")
|
||
params.append(int(hi))
|
||
|
||
range_filter("price_min", "price_max", "price_sek")
|
||
range_filter("ram_min", "ram_max", "ram_gb")
|
||
range_filter("storage_min", "storage_max", "storage_gb")
|
||
|
||
min_cores = p.get("cpu_cores_min")
|
||
if min_cores:
|
||
clauses.append("l.cpu_cores >= ?")
|
||
params.append(int(min_cores))
|
||
|
||
gpu_families = p.getlist("gpu_family")
|
||
if gpu_families:
|
||
family_parts = []
|
||
for fam in gpu_families:
|
||
clause = _GPU_FAMILY_CLAUSES.get(fam)
|
||
if clause:
|
||
family_parts.append(clause)
|
||
if family_parts:
|
||
clauses.append("(" + " OR ".join(family_parts) + ")")
|
||
|
||
if p.get("dedicated_gpu") == "1":
|
||
clauses.append(_DEDICATED_GPU_SQL)
|
||
|
||
if p.get("touchscreen") == "1":
|
||
clauses.append("l.touchscreen = 1")
|
||
if p.get("backlit") == "1":
|
||
clauses.append("l.keyboard_backlit = 1")
|
||
if p.get("webcam") == "1":
|
||
clauses.append("l.webcam = 1")
|
||
|
||
sort_map = {
|
||
"price_asc": "l.price_sek ASC",
|
||
"price_desc": "l.price_sek DESC",
|
||
"ram_desc": "l.ram_gb DESC",
|
||
"storage_desc": "l.storage_gb DESC",
|
||
"brand_asc": "l.brand ASC, l.title ASC",
|
||
"title_asc": "l.title ASC",
|
||
}
|
||
sort_col = sort_map.get(p.get("sort", "price_asc"), "l.price_sek ASC")
|
||
|
||
where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
|
||
|
||
try:
|
||
page = max(1, int(p.get("page", 1)))
|
||
page_size = min(200, max(10, int(p.get("page_size", 50))))
|
||
except ValueError:
|
||
page, page_size = 1, 50
|
||
offset = (page - 1) * page_size
|
||
|
||
conn = get_db()
|
||
|
||
count_sql = f"SELECT COUNT(*) FROM laptops l {where}"
|
||
total = conn.execute(count_sql, params).fetchone()[0]
|
||
|
||
data_sql = f"""
|
||
SELECT
|
||
l.id, l.url, l.title, l.brand, l.model_ref,
|
||
l.price_sek, l.original_price_sek, l.condition_grade, l.condition_detail,
|
||
l.screen_size, l.screen_resolution, l.screen_type, l.touchscreen,
|
||
l.cpu_full, l.cpu_model, l.cpu_cores, l.cpu_threads,
|
||
l.cpu_base_ghz, l.cpu_turbo_ghz, l.cpu_cache, l.cpu_generation,
|
||
l.ram_gb, l.ram_type, l.storage_gb, l.storage_type,
|
||
l.gpu,
|
||
(CASE
|
||
WHEN l.gpu LIKE '%NVIDIA%' OR l.gpu LIKE '%GeForce%'
|
||
OR l.gpu LIKE '%Quadro%' OR l.gpu LIKE '%RTX A%'
|
||
THEN 'NVIDIA'
|
||
WHEN l.gpu LIKE '%AMD Radeon%' THEN 'AMD Radeon'
|
||
WHEN l.gpu LIKE '%Iris Xe%' THEN 'Intel Iris Xe'
|
||
WHEN l.gpu LIKE '%Iris Plus%' THEN 'Intel Iris Plus'
|
||
WHEN l.gpu LIKE '%UHD%' OR l.gpu LIKE '%Intel HD%' THEN 'Intel UHD / HD'
|
||
WHEN l.gpu LIKE '%Apple%' THEN 'Apple GPU'
|
||
ELSE NULL END) AS gpu_family,
|
||
{_DEDICATED_GPU_CASE_SQL} AS has_dedicated_gpu,
|
||
l.wifi, l.bluetooth, l.wired_ethernet, l.has_4g_5g,
|
||
l.usb_ports, l.thunderbolt_ports, l.hdmi,
|
||
l.weight_kg, l.operating_system,
|
||
l.keyboard_backlit, l.webcam, l.optical_drive,
|
||
l.warranty, l.eco_cert,
|
||
COALESCE(l.source, 'billigteknik') AS source,
|
||
l.image_urls,
|
||
cs.full_name AS cs_full_name,
|
||
cs.codename AS cs_codename,
|
||
cs.lithography AS cs_lithography,
|
||
cs.launch_date AS cs_launch_date,
|
||
cs.total_cores AS cs_total_cores,
|
||
cs.performance_cores AS cs_performance_cores,
|
||
cs.efficient_cores AS cs_efficient_cores,
|
||
cs.total_threads AS cs_total_threads,
|
||
cs.base_freq_ghz AS cs_base_freq_ghz,
|
||
cs.max_turbo_ghz AS cs_max_turbo_ghz,
|
||
cs.l2_cache AS cs_l2_cache,
|
||
cs.l3_cache AS cs_l3_cache,
|
||
cs.tdp_w AS cs_tdp_w,
|
||
cs.max_tdp_w AS cs_max_tdp_w,
|
||
cs.pcie_version AS cs_pcie_version,
|
||
cs.max_ram_gb AS cs_max_ram_gb,
|
||
cs.ram_types AS cs_ram_types,
|
||
cs.igpu_name AS cs_igpu_name,
|
||
cs.igpu_boost_mhz AS cs_igpu_boost_mhz,
|
||
cs.igpu_exec_units AS cs_igpu_exec_units,
|
||
cs.igpu_max_vram_gb AS cs_igpu_max_vram_gb,
|
||
cs.igpu_directx AS cs_igpu_directx,
|
||
cs.source_url AS cs_source_url,
|
||
cs.lookup_status AS cs_lookup_status
|
||
FROM laptops l
|
||
LEFT JOIN cpu_specs cs
|
||
ON LOWER(TRIM(l.cpu_model)) LIKE '%' || LOWER(TRIM(cs.brand_name)) || '%'
|
||
OR cs.cpu_model_raw = l.cpu_full
|
||
{where}
|
||
ORDER BY {sort_col}
|
||
LIMIT ? OFFSET ?
|
||
"""
|
||
rows = conn.execute(data_sql, params + [page_size, offset]).fetchall()
|
||
conn.close()
|
||
|
||
return jsonify({
|
||
"total": total,
|
||
"page": page,
|
||
"page_size": page_size,
|
||
"pages": max(1, (total + page_size - 1) // page_size),
|
||
"laptops": [row_to_dict(r) for r in rows],
|
||
})
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ── API: single laptop detail ─────────────────────────────────────────────────
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
|
||
@app.get("/api/laptops/<int:laptop_id>")
|
||
def api_laptop_detail(laptop_id):
|
||
conn = get_db()
|
||
row = conn.execute(f"""
|
||
SELECT l.*,
|
||
{_DEDICATED_GPU_CASE_SQL} AS has_dedicated_gpu,
|
||
cs.full_name AS cs_full_name,
|
||
cs.codename AS cs_codename,
|
||
cs.lithography AS cs_lithography,
|
||
cs.launch_date AS cs_launch_date,
|
||
cs.total_cores AS cs_total_cores,
|
||
cs.performance_cores AS cs_performance_cores,
|
||
cs.efficient_cores AS cs_efficient_cores,
|
||
cs.total_threads AS cs_total_threads,
|
||
cs.base_freq_ghz AS cs_base_freq_ghz,
|
||
cs.max_turbo_ghz AS cs_max_turbo_ghz,
|
||
cs.l2_cache AS cs_l2_cache,
|
||
cs.l3_cache AS cs_l3_cache,
|
||
cs.tdp_w AS cs_tdp_w,
|
||
cs.max_tdp_w AS cs_max_tdp_w,
|
||
cs.pcie_version AS cs_pcie_version,
|
||
cs.pcie_lanes AS cs_pcie_lanes,
|
||
cs.max_ram_gb AS cs_max_ram_gb,
|
||
cs.ram_types AS cs_ram_types,
|
||
cs.ram_speeds_mhz AS cs_ram_speeds_mhz,
|
||
cs.ecc_support AS cs_ecc_support,
|
||
cs.igpu_name AS cs_igpu_name,
|
||
cs.igpu_base_mhz AS cs_igpu_base_mhz,
|
||
cs.igpu_boost_mhz AS cs_igpu_boost_mhz,
|
||
cs.igpu_exec_units AS cs_igpu_exec_units,
|
||
cs.igpu_max_vram_gb AS cs_igpu_max_vram_gb,
|
||
cs.igpu_directx AS cs_igpu_directx,
|
||
cs.source_url AS cs_source_url,
|
||
cs.lookup_status AS cs_lookup_status
|
||
FROM laptops l
|
||
LEFT JOIN cpu_specs cs
|
||
ON LOWER(TRIM(l.cpu_model)) LIKE '%' || LOWER(TRIM(cs.brand_name)) || '%'
|
||
OR cs.cpu_model_raw = l.cpu_full
|
||
WHERE l.id = ?
|
||
""", (laptop_id,)).fetchone()
|
||
conn.close()
|
||
|
||
if not row:
|
||
return jsonify({"error": "Not found"}), 404
|
||
|
||
return jsonify(row_to_dict(row))
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ── API: alternatives ─────────────────────────────────────────────────────────
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
|
||
@app.get("/api/laptops/<int:laptop_id>/alternatives")
|
||
def api_laptop_alternatives(laptop_id):
|
||
"""
|
||
For a given laptop, find:
|
||
- cheaper_same_spec : same model, same-or-better RAM/storage, ≥5% cheaper
|
||
- better_spec_same_price : same model, within ±10% price, strictly better RAM or storage
|
||
"Same model" = all words of the shorter cleaned title are a subset of the longer.
|
||
Works cross-source (BilligTeknik + Nuvoo).
|
||
"""
|
||
conn = get_db()
|
||
|
||
target_row = conn.execute(f"""
|
||
SELECT l.id, l.title, l.brand, l.price_sek, l.ram_gb, l.storage_gb,
|
||
l.condition_grade, l.source,
|
||
{_DEDICATED_GPU_CASE_SQL} AS has_dedicated_gpu
|
||
FROM laptops l
|
||
WHERE l.id = ?
|
||
""", (laptop_id,)).fetchone()
|
||
|
||
if not target_row:
|
||
conn.close()
|
||
return jsonify({"error": "Not found"}), 404
|
||
|
||
t = dict(target_row)
|
||
t_price = t.get("price_sek") or 0
|
||
t_ram = t.get("ram_gb") or 0
|
||
t_storage = t.get("storage_gb") or 0
|
||
t_words = set((t.get("title") or "").upper().split())
|
||
|
||
# Fetch all other laptops from the same brand that have a price
|
||
candidates = conn.execute(f"""
|
||
SELECT l.id, l.title, l.brand, l.price_sek, l.ram_gb, l.storage_gb,
|
||
l.condition_grade, l.source, l.url, l.image_urls,
|
||
{_DEDICATED_GPU_CASE_SQL} AS has_dedicated_gpu
|
||
FROM laptops l
|
||
WHERE l.brand = ? AND l.id != ? AND l.price_sek IS NOT NULL
|
||
ORDER BY l.price_sek ASC
|
||
""", (t["brand"], laptop_id)).fetchall()
|
||
conn.close()
|
||
|
||
def same_model(c_title: str) -> bool:
|
||
"""All words of the shorter cleaned title must be in the longer one."""
|
||
if not c_title or not t_words:
|
||
return False
|
||
c_words = set(c_title.upper().split())
|
||
shorter = t_words if len(t_words) <= len(c_words) else c_words
|
||
longer = c_words if len(t_words) <= len(c_words) else t_words
|
||
# Require at least 3 words to avoid matching on brand name alone
|
||
return len(shorter) >= 3 and shorter.issubset(longer)
|
||
|
||
def prep(row, reason: str, reason_type: str) -> dict:
|
||
d = dict(row)
|
||
if d.get("image_urls"):
|
||
try:
|
||
d["image_urls"] = json.loads(d["image_urls"])
|
||
except Exception:
|
||
d["image_urls"] = []
|
||
d["_reason"] = reason
|
||
d["_reason_type"] = reason_type # "cheaper" | "better"
|
||
return d
|
||
|
||
cheaper = []
|
||
better = []
|
||
|
||
for row in candidates:
|
||
c = dict(row)
|
||
c_price = c.get("price_sek") or 0
|
||
c_ram = c.get("ram_gb") or 0
|
||
c_storage = c.get("storage_gb") or 0
|
||
|
||
if not same_model(c.get("title")) or t_price <= 0:
|
||
continue
|
||
|
||
price_diff_pct = (t_price - c_price) / t_price
|
||
|
||
# ── Cheaper with same-or-better specs (at least 5% cheaper) ──────────
|
||
if price_diff_pct >= 0.05 and c_ram >= t_ram and c_storage >= t_storage:
|
||
savings = int(price_diff_pct * 100)
|
||
cheaper.append(prep(row, f"{savings}% cheaper", "cheaper"))
|
||
|
||
# ── Same-ish price (±10%) with strictly better RAM or storage ─────────
|
||
elif abs(c_price - t_price) / t_price <= 0.10:
|
||
upgrades = []
|
||
if c_ram > t_ram:
|
||
upgrades.append(f"{c_ram} GB RAM vs {t_ram} GB")
|
||
if c_storage > t_storage:
|
||
upgrades.append(f"{c_storage} GB storage vs {t_storage} GB")
|
||
if upgrades:
|
||
better.append(prep(row, " · ".join(upgrades), "better"))
|
||
|
||
cheaper.sort(key=lambda x: x["price_sek"])
|
||
better.sort(key=lambda x: (-(x.get("ram_gb") or 0), -(x.get("storage_gb") or 0)))
|
||
|
||
return jsonify({
|
||
"cheaper_same_spec": cheaper[:5],
|
||
"better_spec_same_price": better[:5],
|
||
})
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ── API: scrape triggers ──────────────────────────────────────────────────────
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
|
||
_scrape_status: dict = {
|
||
"billigteknik": {"running": False, "last_run": None, "last_count": 0, "error": None},
|
||
"nuvoo": {"running": False, "last_run": None, "last_count": 0, "error": None},
|
||
"cpu": {"running": False, "last_run": None, "last_count": 0, "error": None},
|
||
}
|
||
|
||
_status_lock = threading.Lock()
|
||
|
||
|
||
def _do_scrape(name: str, skip_phase1: bool, reset: bool) -> None:
|
||
"""Worker function executed in a background daemon thread."""
|
||
conn = init_db(_SCRAPER_DB_PATH)
|
||
try:
|
||
if name == "billigteknik":
|
||
_scraper_billig.run(conn, skip_phase1=skip_phase1)
|
||
count = conn.execute(
|
||
"SELECT COUNT(*) FROM laptops WHERE source='billigteknik'"
|
||
).fetchone()[0]
|
||
elif name == "nuvoo":
|
||
_scraper_nuvoo.run(conn, skip_phase1=skip_phase1, reset=reset)
|
||
count = conn.execute(
|
||
"SELECT COUNT(*) FROM laptops WHERE source='nuvoo'"
|
||
).fetchone()[0]
|
||
elif name == "cpu":
|
||
_scraper_cpu.run(conn, reset=reset)
|
||
count = conn.execute(
|
||
"SELECT COUNT(*) FROM cpu_specs WHERE lookup_status='ok'"
|
||
).fetchone()[0]
|
||
else:
|
||
count = 0
|
||
|
||
with _status_lock:
|
||
_scrape_status[name].update({
|
||
"running": False,
|
||
"last_run": datetime.now().isoformat(),
|
||
"last_count": count,
|
||
"error": None,
|
||
})
|
||
|
||
except Exception as exc:
|
||
with _status_lock:
|
||
_scrape_status[name].update({
|
||
"running": False,
|
||
"last_run": datetime.now().isoformat(),
|
||
"error": str(exc),
|
||
})
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
@app.post("/api/scrape/<scraper_name>")
|
||
def api_scrape_start(scraper_name: str):
|
||
"""
|
||
Trigger a scraper in a background thread.
|
||
Body (JSON, optional):
|
||
skip_phase1: bool – skip listing-page collection (billigteknik/nuvoo)
|
||
reset: bool – clear existing data before scraping
|
||
Returns 202 if started, 409 if already running, 400 if unknown scraper.
|
||
"""
|
||
if scraper_name not in _scrape_status:
|
||
return jsonify({"error": f"Unknown scraper '{scraper_name}'. "
|
||
"Valid: billigteknik, nuvoo, cpu"}), 400
|
||
|
||
with _status_lock:
|
||
if _scrape_status[scraper_name]["running"]:
|
||
return jsonify({"error": f"'{scraper_name}' is already running"}), 409
|
||
_scrape_status[scraper_name]["running"] = True
|
||
_scrape_status[scraper_name]["error"] = None
|
||
|
||
body = request.get_json(silent=True) or {}
|
||
skip_phase1 = bool(body.get("skip_phase1", False))
|
||
reset = bool(body.get("reset", False))
|
||
|
||
t = threading.Thread(
|
||
target=_do_scrape,
|
||
args=(scraper_name, skip_phase1, reset),
|
||
daemon=True,
|
||
name=f"scraper-{scraper_name}",
|
||
)
|
||
t.start()
|
||
|
||
return jsonify({"status": "started", "scraper": scraper_name}), 202
|
||
|
||
|
||
@app.get("/api/scrape/status")
|
||
def api_scrape_status():
|
||
"""Return the current running/idle status for every scraper."""
|
||
with _status_lock:
|
||
return jsonify(dict(_scrape_status))
|
||
|
||
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
# ── Serve frontend ────────────────────────────────────────────────────────────
|
||
# ══════════════════════════════════════════════════════════════════════════════
|
||
|
||
@app.get("/")
|
||
def index():
|
||
return send_from_directory("static", "index.html")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
app.run(debug=True, port=5000, use_reloader=True)
|