majjo/Scrapyard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: expand iGPU fields (Exec Units, Video RAM, DirectX, Burst Freq fix)

Browse Source
This commit is contained in:
Majjo Duran
2026-04-13 18:04:10 +02:00
parent a23245f6e0
commit e7aa272ffc
5 changed files with 45 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
scrapers/scraper_cpu.py
View File

@@ -63,6 +63,7 @@ _CPU_COLS = [
"pcie_version", "pcie_lanes",
"max_ram_gb", "ram_types", "ram_speeds_mhz", "ecc_support",
"igpu_name", "igpu_base_mhz", "igpu_boost_mhz",
"igpu_exec_units", "igpu_max_vram_gb", "igpu_directx",
"source_url", "lookup_status", "scraped_at",
]
@@ -77,6 +78,7 @@ INSERT OR REPLACE INTO cpu_specs (
pcie_version, pcie_lanes,
max_ram_gb, ram_types, ram_speeds_mhz, ecc_support,
igpu_name, igpu_base_mhz, igpu_boost_mhz,
igpu_exec_units, igpu_max_vram_gb, igpu_directx,
source_url, lookup_status, scraped_at
) VALUES (
:cpu_key, :cpu_model_raw, :vendor, :full_name, :brand_name,
@@ -88,6 +90,7 @@ INSERT OR REPLACE INTO cpu_specs (
:pcie_version, :pcie_lanes,
:max_ram_gb, :ram_types, :ram_speeds_mhz, :ecc_support,
:igpu_name, :igpu_base_mhz, :igpu_boost_mhz,
:igpu_exec_units, :igpu_max_vram_gb, :igpu_directx,
:source_url, :lookup_status, :scraped_at
)
"""
@@ -326,7 +329,7 @@ def _intel_parse_specs(html: str, url: str) -> dict:
"Processor Number", "Product Collection", "Code Name", "Lithography",
"Launch Date", "Vertical Segment", "Marketing Status",
"Total Cores", "Total Threads", "Performance-cores", "Efficient-cores",
"Processor Base Frequency", "Max Turbo Frequency",
"Processor Base Frequency", "Max Turbo Frequency", "Burst Frequency",
"Intel Thermal Velocity Boost Frequency",
"Cache", "TDP", "Configurable TDP-down", "Max Turbo Power",
"Bus Speed",
@@ -399,7 +402,7 @@ def _ark_map(specs: dict, data: dict) -> None:
base = s(r'processor\s+base\s+freq|base\s+freq|base\s+clock')
if base:
data["base_freq_ghz"] = _parse_freq_ghz(base)
turbo = s(r'max\s*turbo\s*freq|max\s*boost|turbo\s*freq')
turbo = s(r'max\s*turbo\s*freq|max\s*boost|turbo\s*freq|^burst\s*freq')
if turbo:
data["max_turbo_ghz"] = _parse_freq_ghz(turbo)
@@ -467,7 +470,7 @@ def _ark_map(specs: dict, data: dict) -> None:
m = re.search(r'(\d+)\s*MHz', igpu_base, re.I)
if m:
data["igpu_base_mhz"] = int(m.group(1))
igpu_boost = s(r'graphics.*dynamic.*freq|graphics.*max.*freq|gpu.*max.*freq')
igpu_boost = s(r'graphics.*dynamic.*freq|graphics.*max.*freq|gpu.*max.*freq|graphics.*burst.*freq')
if igpu_boost:
m = re.search(r'(\d+[.,]\d*)\s*GHz', igpu_boost, re.I)
if m:
@@ -476,6 +479,19 @@ def _ark_map(specs: dict, data: dict) -> None:
m2 = re.search(r'(\d+)\s*MHz', igpu_boost, re.I)
if m2:
data["igpu_boost_mhz"] = int(m2.group(1))
exec_units = s(r'execution\s*units?')
if exec_units:
m = re.search(r'(\d+)', exec_units)
if m:
data["igpu_exec_units"] = int(m.group(1))
vram = s(r'graphics.*video.*mem|gpu.*video.*mem|graphics.*max.*mem')
if vram:
m = re.search(r'(\d+)\s*GB', vram, re.I)
if m:
data["igpu_max_vram_gb"] = int(m.group(1))
directx = s(r'directx')
if directx:
data["igpu_directx"] = directx.strip()
def _is_captcha_page(page) -> bool:

14
scrapers/shared.py
View File

@@ -215,6 +215,9 @@ CREATE TABLE IF NOT EXISTS cpu_specs (
igpu_name TEXT,
igpu_base_mhz INTEGER,
igpu_boost_mhz INTEGER,
igpu_exec_units INTEGER,
igpu_max_vram_gb INTEGER,
igpu_directx TEXT,
source_url TEXT,
lookup_status TEXT DEFAULT 'pending',
scraped_at TEXT
@@ -232,6 +235,17 @@ def init_db(path: str, reset: bool = False) -> sqlite3.Connection:
log.info("Database reset.")
conn.executescript(SCHEMA)
conn.commit()
# Migrate: add new iGPU columns if they don't exist yet
for _col, _typedef in [
("igpu_exec_units", "INTEGER"),
("igpu_max_vram_gb", "INTEGER"),
("igpu_directx", "TEXT"),
]:
try:
conn.execute(f"ALTER TABLE cpu_specs ADD COLUMN {_col} {_typedef}")
conn.commit()
except sqlite3.OperationalError:
pass # column already exists
return conn