majjo/Scrapyard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: expand iGPU fields (Exec Units, Video RAM, DirectX, Burst Freq fix)

Browse Source
This commit is contained in:
Majjo Duran
2026-04-13 18:04:10 +02:00
parent a23245f6e0
commit e7aa272ffc
5 changed files with 45 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
main.py
View File

@@ -359,6 +359,9 @@ def api_laptops():
cs.ram_types AS cs_ram_types,
cs.igpu_name AS cs_igpu_name,
cs.igpu_boost_mhz AS cs_igpu_boost_mhz,
cs.igpu_exec_units AS cs_igpu_exec_units,
cs.igpu_max_vram_gb AS cs_igpu_max_vram_gb,
cs.igpu_directx AS cs_igpu_directx,
cs.source_url AS cs_source_url,
cs.lookup_status AS cs_lookup_status
FROM laptops l
@@ -414,6 +417,9 @@ def api_laptop_detail(laptop_id):
cs.igpu_name AS cs_igpu_name,
cs.igpu_base_mhz AS cs_igpu_base_mhz,
cs.igpu_boost_mhz AS cs_igpu_boost_mhz,
cs.igpu_exec_units AS cs_igpu_exec_units,
cs.igpu_max_vram_gb AS cs_igpu_max_vram_gb,
cs.igpu_directx AS cs_igpu_directx,
cs.source_url AS cs_source_url,
cs.lookup_status AS cs_lookup_status
FROM laptops l

22
scrapers/scraper_cpu.py
View File

@@ -63,6 +63,7 @@ _CPU_COLS = [
"pcie_version", "pcie_lanes",
"max_ram_gb", "ram_types", "ram_speeds_mhz", "ecc_support",
"igpu_name", "igpu_base_mhz", "igpu_boost_mhz",
"igpu_exec_units", "igpu_max_vram_gb", "igpu_directx",
"source_url", "lookup_status", "scraped_at",
]
@@ -77,6 +78,7 @@ INSERT OR REPLACE INTO cpu_specs (
pcie_version, pcie_lanes,
max_ram_gb, ram_types, ram_speeds_mhz, ecc_support,
igpu_name, igpu_base_mhz, igpu_boost_mhz,
igpu_exec_units, igpu_max_vram_gb, igpu_directx,
source_url, lookup_status, scraped_at
) VALUES (
:cpu_key, :cpu_model_raw, :vendor, :full_name, :brand_name,
@@ -88,6 +90,7 @@ INSERT OR REPLACE INTO cpu_specs (
:pcie_version, :pcie_lanes,
:max_ram_gb, :ram_types, :ram_speeds_mhz, :ecc_support,
:igpu_name, :igpu_base_mhz, :igpu_boost_mhz,
:igpu_exec_units, :igpu_max_vram_gb, :igpu_directx,
:source_url, :lookup_status, :scraped_at
)
"""
@@ -326,7 +329,7 @@ def _intel_parse_specs(html: str, url: str) -> dict:
"Processor Number", "Product Collection", "Code Name", "Lithography",
"Launch Date", "Vertical Segment", "Marketing Status",
"Total Cores", "Total Threads", "Performance-cores", "Efficient-cores",
"Processor Base Frequency", "Max Turbo Frequency",
"Processor Base Frequency", "Max Turbo Frequency", "Burst Frequency",
"Intel Thermal Velocity Boost Frequency",
"Cache", "TDP", "Configurable TDP-down", "Max Turbo Power",
"Bus Speed",
@@ -399,7 +402,7 @@ def _ark_map(specs: dict, data: dict) -> None:
base = s(r'processor\s+base\s+freq|base\s+freq|base\s+clock')
if base:
data["base_freq_ghz"] = _parse_freq_ghz(base)
turbo = s(r'max\s*turbo\s*freq|max\s*boost|turbo\s*freq')
turbo = s(r'max\s*turbo\s*freq|max\s*boost|turbo\s*freq|^burst\s*freq')
if turbo:
data["max_turbo_ghz"] = _parse_freq_ghz(turbo)
@@ -467,7 +470,7 @@ def _ark_map(specs: dict, data: dict) -> None:
m = re.search(r'(\d+)\s*MHz', igpu_base, re.I)
if m:
data["igpu_base_mhz"] = int(m.group(1))
igpu_boost = s(r'graphics.*dynamic.*freq|graphics.*max.*freq|gpu.*max.*freq')
igpu_boost = s(r'graphics.*dynamic.*freq|graphics.*max.*freq|gpu.*max.*freq|graphics.*burst.*freq')
if igpu_boost:
m = re.search(r'(\d+[.,]\d*)\s*GHz', igpu_boost, re.I)
if m:
@@ -476,6 +479,19 @@ def _ark_map(specs: dict, data: dict) -> None:
m2 = re.search(r'(\d+)\s*MHz', igpu_boost, re.I)
if m2:
data["igpu_boost_mhz"] = int(m2.group(1))
exec_units = s(r'execution\s*units?')
if exec_units:
m = re.search(r'(\d+)', exec_units)
if m:
data["igpu_exec_units"] = int(m.group(1))
vram = s(r'graphics.*video.*mem|gpu.*video.*mem|graphics.*max.*mem')
if vram:
m = re.search(r'(\d+)\s*GB', vram, re.I)
if m:
data["igpu_max_vram_gb"] = int(m.group(1))
directx = s(r'directx')
if directx:
data["igpu_directx"] = directx.strip()
def _is_captcha_page(page) -> bool:

14
scrapers/shared.py
View File

@@ -215,6 +215,9 @@ CREATE TABLE IF NOT EXISTS cpu_specs (
igpu_name TEXT,
igpu_base_mhz INTEGER,
igpu_boost_mhz INTEGER,
igpu_exec_units INTEGER,
igpu_max_vram_gb INTEGER,
igpu_directx TEXT,
source_url TEXT,
lookup_status TEXT DEFAULT 'pending',
scraped_at TEXT
@@ -232,6 +235,17 @@ def init_db(path: str, reset: bool = False) -> sqlite3.Connection:
log.info("Database reset.")
conn.executescript(SCHEMA)
conn.commit()
# Migrate: add new iGPU columns if they don't exist yet
for _col, _typedef in [
("igpu_exec_units", "INTEGER"),
("igpu_max_vram_gb", "INTEGER"),
("igpu_directx", "TEXT"),
]:
try:
conn.execute(f"ALTER TABLE cpu_specs ADD COLUMN {_col} {_typedef}")
conn.commit()
except sqlite3.OperationalError:
pass # column already exists
return conn

BIN
seed.db
View File

Binary file not shown.

8
static/index.html
View File

@@ -724,7 +724,7 @@ async function loadDetail(id) {
${l.cs_l2_cache ? specItem('L2 Cache', l.cs_l2_cache) : ''}
${(l.cs_l3_cache || l.cpu_cache) ? specItem('L3 Cache', l.cs_l3_cache || l.cpu_cache) : ''}
${specItem('TDP', l.cs_tdp_w ? l.cs_tdp_w + ' W' : null)}
${specItem('Max Turbo Power', l.cs_max_tdp_w ? l.cs_max_tdp_w + ' W' : null)}
${l.cs_max_tdp_w ? specItem('Max Turbo Power', l.cs_max_tdp_w + ' W') : ''}
${specItem('PCIe', l.cs_pcie_version ? 'PCIe ' + l.cs_pcie_version : null)}
</div>
${hasCpuSpecs && l.cs_source_url ? `<a class="detail-link secondary" href="${escHtml(l.cs_source_url)}" target="_blank" rel="noopener" style="margin-top:10px;font-size:12px">Intel spec page ↗</a>` : ''}
@@ -755,7 +755,11 @@ async function loadDetail(id) {
<div class="spec-grid">
${l.has_dedicated_gpu ? specItem('GPU', l.gpu) : ''}
${specItem('iGPU', l.cs_igpu_name || (!l.has_dedicated_gpu ? l.gpu : null))}
${specItem('iGPU Boost', l.cs_igpu_boost_mhz ? (l.cs_igpu_boost_mhz / 1000).toFixed(2) + ' GHz' : null)}
${l.cs_igpu_base_mhz ? specItem('iGPU Base', l.cs_igpu_base_mhz + ' MHz') : ''}
${l.cs_igpu_boost_mhz ? specItem('iGPU Boost', (l.cs_igpu_boost_mhz / 1000).toFixed(2) + ' GHz') : ''}
${l.cs_igpu_exec_units ? specItem('Exec Units', l.cs_igpu_exec_units) : ''}
${l.cs_igpu_max_vram_gb ? specItem('Video RAM', l.cs_igpu_max_vram_gb + ' GB') : ''}
${l.cs_igpu_directx ? specItem('DirectX', l.cs_igpu_directx) : ''}
</div>
</div>