From eedb96c53a92beb6d8de665320a734819a9bdaf3 Mon Sep 17 00:00:00 2001 From: Majjo Duran Date: Mon, 13 Apr 2026 11:19:31 +0200 Subject: [PATCH] Add AMD CPU support (CSV import + TechPowerUp scraper), fix alternatives accordion, untrack database.db --- .gitignore | 2 + database.db | Bin 77824 -> 0 bytes scrapers/scraper_cpu.py | 369 +++++++++++++++++++++++++++++++++++++++- static/index.html | 50 ++++-- 4 files changed, 408 insertions(+), 13 deletions(-) delete mode 100644 database.db diff --git a/.gitignore b/.gitignore index 872a2b9..cfe5a23 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ ENV/ # The live database is never committed – only cpu_specs is preserved via the # bundled seed dump (if one is created). See purge_db.py before committing. database.db +database.db-shm +database.db-wal # ── Logs ────────────────────────────────────────────────────────────────────── *.log diff --git a/database.db b/database.db deleted file mode 100644 index 393efaa765e8279eb83f598365016df7d696b544..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 77824 zcmeHwdvF}ddEa0c?kTT;-JhNG^_@#HxyA zS6p&QCFl2b&wF>VizUy}9cPPN>~!}`cmMjE?ytY^>+iez^rb@07H^a*Te+I335SG4 zLYNapK@h%%|3~oO{XC2xz3v76_5^;n`uQ~>^~s}sRCwaMRAzAEyL~%-JFSYx=`jK^ z0x<$_djyoe{>gKX_6UX2ntiMKY7ukrO0Hfjv+I@oc73I~ZRe{iiubdBG1)))_~Sj- zYq`~;?c7qeU#;7veDt#S%rocam(Gbx^QSMJ6C-7(O1Uk226s;X;s<&LCXXHK`RXQn zW-+&2D{ohwKZ(#YoJ4VISj4BWM&FUo#f7DF=g&POE|CFMi6rZdYi{gc|;-zzE z#fytqCH~}7toP2NDt5J8tP>`LY12oYR5Ge8-VznrlEo?KnbRkOPpRGB-lsvR8pvIt zR?Lb^3;24LV134}VaV{quH{qgX|-}azrhGW{B|3ZV$@%3=OQJmE(bAd4Wj>Oa z9BxowgP&+$SOJYq3iR$|Z^V9ga(cc9dM%1)$`xBI7?L8ZioDE0pb06D^lI=S z9`R8km8fl0?A%(QOhbv)T-9D#-?-yF22+;v&a(&9E34&TemFay%Wnh*!5K`im+N-P zuH@W-Mio(fPAYP38#@MU`oGXT4)qS8L@;Zrx^=RsK8NIPVT>r&}V_ zxF{wI&tA60t5=C``L})fGR*`R&Yty%@VrBW>)UnS|KK-!pjQ<0Psz^L3O5Tij^9q< zMj=?7{pR=3&(&hxu9eHRjbIr+d#6yb*H-M>hFvMyHK&ZzNip!cP|ilKx}vYI80*1D zH7Vm9qW zzENA*47JlI^qs<5ZDS?B6>N*>XK(Pk!})&pn!UZ>-K)7;&5o>X{5A1L@M`%+Z6}8* z;O~i;zAHHnQF)uaTBY@s>g{UH=B@KF*tBo2mUET0mDOB+6MPM8d=u;`eaBwS=fd6d zmb&PembYt!+say{aMOG9TRk^d0bfvSvT!rl?RGBnkb8U2E;1hoIWh|Af8qEE7HBM>7HBM>7HBM>7HBM>7HBM>9-KqD|ry^r_* z4>S$LosSWS5r`3p5r`3p5r`3p5r`3p5r`3p5qSF}K>Gj0#2doIU*aqN#0bO)#0bO) z#0bO)#0bO)#0bO)#0bO)#0bO)yhRZhPb42rc*jVE&yV&bk0qRA0K( z&+yyuYq3(sLEkElVlIeR?MlI}DzaGGdRjT9D!si=eEakC`G@_3hm^E<{D*sqtw}Yb=EG;Zdb}{^?a>5TivYBs*0+q znORm|a(jakyal{plD;Leb9hpHTdI{MRkCCWC;DdT5Dd_YZ>V zPYeI9E80{trY7%0o7A{QDAOI%CZJTUl&R^8sS%Vz2Y^y;gc3vjP-My(}Ce<7+O5U-KqZi4q;nn=VzF_Y+Fh%2?KZ zP&RwPJElrAYhcDyOvRJ|<@tgAp?o;71ZkQPeZeUyNR@iS3+~<25mjoZl=OXsGGiE- zeUPNKNosWM(K~&=&8%jnJtfgi_rHO0=yJO5N%hN?A?e^;U@^`be_(gmC=v$>YL%Gsk-H z85lT!rKjhJ(DUAh2Ms~kePRA^ZhO0EOIzwZ{S_~(q9u{L26pMDox5qnQA3Rm^yhZ? z+4`x=>a>cOftLhvYUwHt`7aoY(_qn5@MnX)y@PWjfBvx_`K_nF<^BandHJ+B zwJ4F}hv^wnXP4*r<-+W|IJH~?!w2Ue++Bh=U&(L4p-H}0NBN1PdP;sLb97)tOHQYC zS$3NXryJ~%BD@R)sXa$sWL#^>-l=ECwv=b`D3W;TpxR(sKfQsh-a2MoLYm4OPJ?3?@$qj~SDX z3FoI9M&bP!h4bDhC=z&MryB}!>Y|oXX;Nir!Hjtv7SEN)c?Ddp6bpCm;HbPfb(!77 z1|~Eh3S4gt47g3-MmO5Ko*655@Z*9@H=tJ^oO5 zX!94M=}0^YXAdeMK6a!J?M~wR@Gg1mJ9Hag6D53&=kYbRjIYrde2t8e{r@Wi z|3;YjtBF6I_%9Q`HSvjw&536w9vT1J@xL7ZX#5l7JLC5F)$#f9+3^$O zLt}q2_Qu%nkNx`Cm&QIh_Mx%uvHaNb*zDNk==VqeV)TvCKOFtHqrW-&h0%|Xj*fhP z)bNSnqr(G3-x>PfL*E?wk3+vR^y@=k8v3Q7j}P4%dS&Rvp@pIJ z(A3bep~1mF8~nY&e>wQY!Ji+zIk-IdSeBWpM`vL`oUy9zRNACN@=zXk!@acz=r<`^gl95)8y6_9(BpJQ=2f-u~ zo7RO-`4wusp-2yQ@b5|(Nz;oiEKQ{_J$(AOWw>4Gx*|FsDL?I(V#3OGA zpPWo8iGnF*fFm{-9*&=2rJQu(XPhewgRmj|w0lMO2p@N^2pQpH?v;^7pFcVdg$XoAOid#yimiwh%bxX1IZLd^#Ov+vg-Q4mr(~4X# zdRI`<)HhjQ4atJEss4(4p_-X`!MgxH8{UOs*4OA5?E|uz1wT40!TiVPOH*XhLi@7zaj=NF(?(6ilnSl0t1_BeZ* zn=#=PQ^Li5j5V2xg)^qT+bMnAy8=uS$B(ta-=DX4>~9qTJ;&Mw2TID38$FT*(-sHLpOf#c=8GViL^Qg z+j}*Y%(cvwrce{edxomOQa_!&^zP*QTNZ!uQ1Y=*W-`!cCBZIqDT9&8zc4oOb)l#4 zdqUs$CcfVHYq4+sw;`JtcPU07M&O4Ff!P1Q@A4ME$m?zF|3CKsFNs%U|Nj(3!}a+b z`~P?63E#0f*|Ns||6~9Epo|v&-(&y(z$W(p|3LZw2Y+iG%>Vx&vBYBopA-7~-Xl!> z!Ni9L{`0^W``$CK^25a{$E}GGh!Kbp_<|x|+_P2BBg^&N5F?Xgvz|DlXJ-mdj$Lyi?`-&W1>PJZufMBN%6nwaekV{BU&5 zL(x%LKSrDrc+P*UpXG#)3uHw@@A+BiByalFjq-Nk#%)AJ#=0N}CQ;)TZ$`d4# z6(x8@W_fIV?m|%Xsp;N<$wwdU`KKosy?RIgydQlbO7)Uk26uG92#{(tvZEU}X#qBs zIglMpfs5txW__ClNvhK|(HzjLZx@}DVd7M-xRbkGMado5gCZW>G>?pj5#~|o+|YA% zip|FD8Vu5mNz7h=5*NRGq&}vDxOy8k*fn&0V2ro^DwlzstSComXz)7+xV)#2d^G_lZ=R z-OB=JhHj45iq{JvvOMDku~gq$wJUBAM|Wu4sDKEx8EQ#GQ!W?s7`R-(Fq}2UecqY! z7NWXc%5B;-tk1(TUFG8mL+y<+rjG3mcTA%VUF7x`2&=hfyN^@k(iT0)#2?n*Vy+IW z>lJ!K9ekpXIr%h>v1CpH;ggQxFZ~V$10)0cDx#q+@#u!^!L@P?QOU?i*Nqt$nj8bx z=T0Fvn>E85U2a=n6{i#x5%6G{|M;{s;;V&qOrm%^tIlR43L^5VZ2mlMoF95#pt-^D zjPCtj*tavqa(2mHCKlR9~!o7KVM19+XYVpiP-mUF?!Ct{>i>eb_v@zl&pV44= zPh@G}4{HRw4+jD~;_+JU)!`hefYCe!CiI#f4iV`#on^aWC|!bV)fkFADQT)K&p2Ty zZ%ME{H+jqGA)E$Qws@4z=nKPp#P9d-v-9~Cr>)M5=CsXQCId1}n%VGf*D#a1ktGXI zafjw5K9jJHu{=hHPLq7Js4kvehE0mwWDgd0I|$JazQ+RWQT#lom4RybLL3at7obtL zs;$5brdH?pQl8hhQ`?x$!E&9#Q&F9B1H8nl`G9c% zQ&Mc5LJ`!yg#@+kt=8|4QGZ$v;ed zB=-N`Y*=Fd|CQMPe|G^DV*mdXq2hixt&jcx!yoPafim|0e}ABi{r|fmXzuj|kNyAO zSEu!{|9|%dzvoa2vH$vxpdPrvHCL#3V}P&7)pyE z7n-3XRv7L7M}=#`_?O2njC^DG)uD$6exv_d?{|`4Ogt%k3mM&gb|0D%g)4}w)9N$- zh?cGcCkIphO6HnSeDBu_QAqcK?pT9W27nQA|9sC0A&HPJgKe1ks!-lf-xgqD&IaiZ;+N&c^{CIlsu)(vkn z;E-*kf6-unos+GW$r?H$SY>L=2^liN_W|4I4mZJ8@F!juTR2FMVynr7EeotB0^5dIiws*7IyklnyGYpf zaK>wH#K9J`?x?7t!qeT)x|*3W%@EycmV(ah2W_XpEDa^ArBi9s(9;z6_x7Qa!tveX zpxgChlN2{lV2c`srsfRpG-q(h-+UNbazRhif`hG|G8o-*uLOK{m_Z~pjiqwm0--qr zD;_;T(GEb>yagt4y;^@FF*x@S-z-d+M_;H_2)tpk=(g7<+&Q`Z-XelMJpP~8LD;f8X_!qW4PxEzjR$6d z!hh&@7jA%X13{QGnvx+N`1*Kf*b)yM#g;f+5H^Q&3LHvPY#$6h*u_?tQyE#+HG}W} zCxo9C#=kUncI1Bye{|@vfv@-9=o?6WHJM6$y626ag76(=zuBMivM8A5e&PZ*WJ4%w z0`rA~;HQm=0a<{mWmP?^m?;ETl`V_Q0?JDrRv3!mkaa3-lGwaL*+^@=!kw$4@Wc}x z^pJh+Fl4%FH%&V}2kV1qUH2!tw`{m;EV(g9JviU#&Es5p4HWqfiOH)hS$6IlHu-F z`RX!&g`1Xx}iaEIfM^e`&G(xnbKelBLwvN~8N?B6p?9H6`V)q?;W8CnLT z&tcW;n}G`Mh;#c`!8wn*MDC`hA8?2jprI+UEEAKY zo)Au~Ptt!+xZNW_TjdC}9s%!u;y&RUmT?ol zp}263@OG>{P_tSlC8yJRhW4#5${oWS;_o@+jKB8;E{%I9w^SE&>%i+6wzztk`vq@U z7R%ucD3@xqL8qnYfTo3r~F4 zq29w>`Vy9C^}g2C{GiH;m1(43^sx2xpi3RdUEJwG1}KQKsWIju<#K0%>7fb0VQFQQ zJ0oFc182yl70T!WUWa=B#t~}0C8V(&UL~tz3QI$>!gL9e%bOhp-sn6WM%QB;80rrg zxT=xTNnVGgjr9NH!mkSBcgL=c{NC^@Lz9DF8B7iQ_P}EQK=1D-ul4*1GJo)&y=)#* zW39Dh(^-KjqH1K|k+kfMHeIw8QU>=q{9#;3Ih;t0&OPMlH11hQsVZt%ENrlNuz~4G zQKLf2j@DPmJ0KS}ARpJPv`O0e?bSANVHZ8bgpH!7F}W}o{DS$0T=v2pF)A14t3ngG zFct^I^1N5bIoEjaa$!|5vDISB}` zbwN%NPQq!p&Z{IQ%9&zM}~&-34B{5jMW9`Q{#$~5#-$)pt67Z{4;nR##VdCqgT(xf^o zX83iCa`ZHn_%Ew3Zhb(e7}dx^i^;dWg60tj6X;0#`)I#aRA)t;zt~wtJ-I8 z2-G;D7iYkYa6c`&oQP9Kg?IA`f_iM_N_pEWcTY=i;ilb44tD|Ig_8*UCT*UR@aF4J=FDw_s=kM+@iXB$;E-4MT=PG!;= z#USxv_SuudZhdcLzjcXb7N}>|gX%A-x>Z;!AUqO+Yt#@#C@0;x`+7yPrEP=|n32k* zVxeSj-F>}gm!xujL%RFrQmHOgYbXS3>djnn^LBBym|K(TxEsaqzD`fOiDET`0a{B% zUNlr6(!PnjdW2YUsu!tx(WzbxRxhG@kyS4`)r)@hB2|y#ta>q0eR!pV9^8PDk3m5j z_|i)I|B;@4I{$xZJEFowH|ANxD@Sr6nHW z*1)7ZF`+y$4S6FttWMems7{>Ml^ae4UsktFTc+4-u%{#eY5%DEj`m3qiX9SuN>QPm zIo9g9qL)&C57M4fypZ4t^lwGcE`{S4)A7Y07Hcn+8h&fNb3M#D%AzRj)ecs0GLh-=wT~L08BmH zkAe>Xu!0pllU8&R-8}b76Pw_nuuZTbS(*{XmnQ}EQ<^gi2d7`BP&=F8pg2r%B@!ld ztFCrx)Diq*4)$z30M(( zENWzMLTI4cjwb|cX(2_k6ji6~({t-hplw36hS@;1hDp#mRND$#6zU3EGPd|@hg}Fl z>rrjD)+|`N!Yr9#spwp5s_p3f1z*gxtZbwVHLaO;h788PXm&t9mjCCnj&zOd0gjX@vw*b-VVL{gcRxvfJ zS}7UZSCuR*uC29176-QzdmLFbCyFd31^6$u34?1$tdP+SvPPo!377jC;ASinILI-2 zNWoju{h=Wos*W7TGFaaN$ZBC^)lQCM@##R#Dhd>pvaZ9pg6;o>|0;}sZS360pAPR0 zogDas{$gJ;`Gv$JlHbCgz3gq!OdFE{aOhA_hV+3v$v4#fZa7NqE*&dCsU10AhH(H6!EtDi@E>3t;I=|Hb;~5# zW^Tt7Of^4N_BbB~#fr$00T|LCNA^S~jTXIbnYL2oAayIv4F8jJNBHFDI9BL;vO5Zg zrYIeyaMrbbq2)%s2T-0ECbYJxblS?WEz*sf?)E+;tB(r*(2xeL4{E%y@if{P?q3rncE?LxI;+7hAfsi{WZs*rG@X8% z;@b>WOuwG^I>1mkXM<>FcrY=BJjH;ekWM?(7hiR0IG`eEqTdS7{1LQeTI`z4yez#7 z>9wQX>T~SVk8%>ee@d=Qc-s&0b+~&CBn%yKr%JALR>oqVF=R6O+IxB1CE@n)HUkwT z?3}PLnjBCE2|tJs!@>my6{c!w%nw#MLc#lJXd^Vre6NYF{Fwr%*c~Y-m&GD!fQ_;L5Wc1~_bj+pSuL91MkRnN8L^ z|5a_Y3+jLYww7cWo}04pm_C? zHQmqgQtc$Mjrfuz7Kh|rR?RsK%ho$bHS|;B63ccW@Tgg%^(V z;7`J$14Da@yNL+xIy13*(3oVgreqrFux^GY`H~@Qmbt^5pt9B})E%~va*y)YzL#t` zd@XHvM+hO_)*~d#Iu*dQVPo8ZkqpNa5(u!s7Fb|f++Nu2>4aqA5wd|~DUFe=!$36C z9hed?m2M;AZX{V~6Ah58%c42TgD}Z^6mxTjR-S4(jK_*>M&~2;V6*U zvuAe&mp^On7t7hgX|yK~&j|3rfhob4@EA?p4}d>orP>}W3*yVG;Rnmc`}>d19p&}~ zQuW>3`&HjJyOZc>s?s1sV;u_s8aFA9x!MJ_|=7=h-bF#}n=e^1l0lb;Quv z1WC5Ry5re7oz-zfHe+CaSKi!x*N0%jxO-w>^I{O9O1sNjk?DgHFFc3&!*NjHN{FLy zw>m-T+3)hV58jfnLsT_Wb&kgDz4ntQdE-t5ef}jJ(^WXNKISNDXl`q49VRzwo#T4$ zpsk}l6}Az0Mpv2{2msVWgM;sM1=^JD&U`{Rf$78b64bW}6k>dVjl=p87?a_uo@hF>Gi>ir#%Ew*w$;(RoGkxMqNSY5#vj_yuA7SH_+k`9H%y zJ0wH@zt?}f?;rOa>-}Qyd*BC9Onf2nNYCeb9zwA<|MS}Jq$r$uYQMvNmD_i4B7>de zkaS3H0t+mi|9ghAMK(|(WiR`0eemcXn@|-T-ZWE?Fd-U(bM$XFvHK2Dm|Z_m3qtii zbA{InzQ$Q(!`t+xt$^;`Qgt=swBohhhdYD8px#Lk6f}o{nYD_P-3|sLl~!?(TJ~W$ z+8GRi7Wgm(>ctf2DWQYDyPCoPgFRga9@&2n?(R zAsC1nsNNtC5Zw4Sg#p@W6EL_md1v>qD4bepPm-`@f^kAU*Z`Ig90NWWeAnHqa`Mci zYL-WecXkhTUb!qsDwl(XZDHqaO)H0J5>{W9>EM5BcLJ5a&`HmABT_jf1gz>fi?vU` z(m9E0_e*KJkwoqyhT=4WR5c z#E&x%=)XS+t?rHj{Mq*4Ym%Xdz}L`UfZue4p_Ve!N=9==c<-s*krTo(9KJf1ceE$8 zl?hm+9-cqEg~%kgH8sFoQ_6L?=JtG6V{;UB?R%M*X(LVAYI5v~OeUXj+Vie&WhAXQ7>8MuJU7 zSk}RlNYvrY)&VZb(WW|uGHnD&nP!k?V8P-VS2&zfRM*#?mk}I+hCzNv{F!0h3!tk+ z#Gf7UOD>B5dr3pf-~|F_dTn>;grLw#>xgvOJu1jI-Wczoc}IY7!D>=w6Eze|ZU=3U z0p$k1beO_FB43v&j=na+W~Q0LoO7BfVH^jagy@(a6z z?J1X|7m?C2MPq zhHZaG*wU;J!ItI%VCzW%8jo2)p+|+f#@3B$;rP@EW839fAPow=$ni^R@$S zf?X9AF3A)1r*%Y(Jk;O+4%Zb&-@B`LxMMdr_`R+K^eWwC1QO6JQX-jya|1a@INB+>bOdFBqhj4va{YgQ C5DM@B diff --git a/scrapers/scraper_cpu.py b/scrapers/scraper_cpu.py index da8a635..d1a2d34 100644 --- a/scrapers/scraper_cpu.py +++ b/scrapers/scraper_cpu.py @@ -637,6 +637,359 @@ def lookup_apple(cpu_key: str) -> dict: return {"lookup_status": "failed"} +# ── AMD (TechPowerUp) ───────────────────────────────────────────────────────── + +TPU_BASE = "https://www.techpowerup.com" + + +def _amd_search_term(cpu_raw: str) -> str: + """Strip AMD prefix and parenthetical junk for TechPowerUp Google search.""" + s = cpu_raw.strip() + s = re.sub(r'\s*\([^)]*\)', '', s).strip() + s = re.sub(r'[-\s]*(processor|cpu|chip)$', '', s, flags=re.I).strip() + return s + + +def _tpu_find_url(html: str) -> Optional[str]: + """Extract the first techpowerup.com/cpu-specs/ URL from a search results page.""" + soup = BeautifulSoup(html, "lxml") + + def _clean(href: str) -> str: + m = re.search(r'/url\?q=(https://www\.techpowerup\.com[^&]+)', href) + if m: + href = m.group(1) + return href + + for a in soup.find_all("a", href=True): + href = _clean(a["href"]) + if re.search(r'techpowerup\.com/cpu-specs/', href, re.I): + # Strip any trailing Google redirect junk + href = href.split("&")[0] + return href + return None + + +def _google_find_tpu_url(term: str) -> Optional[str]: + """Search Google for the TechPowerUp CPU spec URL via Playwright.""" + clean_term = re.sub(r'\s*\([^)]*\)', '', term).strip() + query = url_quote(f"{clean_term} site:techpowerup.com/cpu-specs") + search_url = GOOGLE_SEARCH.format(query=query) + log.info(f" → Google (TPU): {search_url}") + + page = None + html = None + try: + ctx = _get_pw_context() + page = ctx.new_page() + response = page.goto(search_url, wait_until="networkidle", timeout=30_000) + status = response.status if response else 200 + + if status == 429 or _is_captcha_page(page): + log.warning("=" * 60) + log.warning(" ⚠ Google CAPTCHA / rate-limit detected!") + log.warning(" The browser window is open — please solve the CAPTCHA manually.") + log.warning(" Waiting up to 5 minutes for you to reach the search results…") + log.warning("=" * 60) + try: + page.wait_for_selector("#search a, #rso a", timeout=300_000) + log.info(" ✓ CAPTCHA solved — resuming.") + except Exception: + log.error(" Timed out waiting for CAPTCHA. Giving up.") + return None + elif status not in (200, 301, 302): + log.warning(f" Google returned HTTP {status}") + return None + else: + try: + page.wait_for_selector("#search a, #rso a", timeout=10_000) + except Exception: + pass + + html = page.content() + + except Exception as exc: + log.warning(f" Google (TPU) search error for '{term}': {exc}") + return None + finally: + if page: + try: + page.close() + except Exception: + pass + + return _tpu_find_url(html) + + +def _tpu_parse_specs(html: str, url: str) -> dict: + """Parse a TechPowerUp CPU spec page into our column dict.""" + data: dict = {"source_url": url} + soup = BeautifulSoup(html, "lxml") + + # CPU name — try the dedicated heading first + for sel in ("h1.cpuname", "h1.cpu-name", "h1", ".cpu-name"): + el = soup.select_one(sel) + if el: + data["full_name"] = el.get_text(strip=True) + break + + specs: dict[str, str] = {} + + # Strategy 1:
definition lists + for dl in soup.select("dl"): + for dt, dd in zip(dl.select("dt"), dl.select("dd")): + k = dt.get_text(strip=True).rstrip(":") + v = dd.get_text(" ", strip=True) + if k and v and len(k) < 80: + specs[k] = v + + # Strategy 2: two-column rows + if len(specs) < 5: + for table in soup.select("table"): + for tr in table.select("tr"): + cells = tr.select("td, th") + if len(cells) >= 2: + k = cells[0].get_text(strip=True).rstrip(":") + v = cells[1].get_text(" ", strip=True) + if k and v and len(k) < 80: + specs.setdefault(k, v) + + # Strategy 3: labeled divs/spans (TPU uses a section-based layout) + if len(specs) < 5: + for section in soup.select(".details, .specs, [class*='specification'], .cpu-details"): + for row in section.select("li, div[class*='row']"): + parts = row.find_all(["span", "div"], limit=2) + if len(parts) >= 2: + k = parts[0].get_text(strip=True).rstrip(":") + v = parts[1].get_text(" ", strip=True) + if k and v and len(k) < 80: + specs.setdefault(k, v) + + if specs: + _tpu_map(specs, data) + + return data + + +def _tpu_map(specs: dict, data: dict) -> None: + """Map TechPowerUp spec key→value pairs to our cpu_specs columns.""" + s = lambda pattern: _spec_lookup(specs, pattern) + + cn = s(r'codename|code.?name|die.?name') + if cn: + data["codename"] = cn + lit = s(r'process\s*node|manufactur|process\s*tech|lithograph') + if lit: + data["lithography"] = lit + ld = s(r'release\s*date|launch|announced') + if ld: + data["launch_date"] = ld + + tc = s(r'^(?:#\s*)?cores?$|total\s*cores?|cpu\s*cores?') + if tc: + data["total_cores"] = _parse_int(tc) + tt = s(r'^(?:#\s*)?threads?$|total\s*threads?') + if tt: + data["total_threads"] = _parse_int(tt) + + base = s(r'base\s*(?:clock|freq(?:uency)?)|core\s*speed') + if base: + data["base_freq_ghz"] = _parse_freq_ghz(base) + boost = s(r'boost\s*(?:clock|freq)|turbo\s*(?:clock|freq)|max\.?\s*boost') + if boost: + data["max_turbo_ghz"] = _parse_freq_ghz(boost) + + l3 = s(r'l3\s*cache') + if l3: + data["l3_cache"] = l3 + l2 = s(r'l2\s*cache') + if l2: + data["l2_cache"] = l2 + + tdp = s(r'^tdp$|thermal\s*design\s*power|default\s*tdp') + if tdp: + m = re.search(r'(\d+)', tdp) + if m: + data["tdp_w"] = int(m.group(1)) + max_tdp = s(r'configurable\s*tdp|ctdp|max\.?\s*tdp|tdp.?up') + if max_tdp: + nums = re.findall(r'(\d+)', max_tdp) + if nums: + data["max_tdp_w"] = max(int(n) for n in nums) + + pcie_ver = s(r'pci\s*-?e(?:xpress)?\s*(?:version|rev(?:ision)?|gen)?$|pcie\s*(?:ver|rev)') + if pcie_ver: + m = re.search(r'(\d+(?:\.\d+)?)', pcie_ver) + if m: + data["pcie_version"] = m.group(1) + pcie_l = s(r'pci\s*-?e\s*lanes?|pcie\s*lanes?') + if pcie_l: + m = re.search(r'(\d+)', pcie_l) + if m: + data["pcie_lanes"] = int(m.group(1)) + + max_mem = s(r'max\.?\s*(?:memory|ram)|memory\s*size') + if max_mem: + m = re.search(r'(\d+)\s*GB', max_mem, re.I) + if m: + data["max_ram_gb"] = int(m.group(1)) + ram_t = s(r'memory\s*type|supported\s*memory|ram\s*type') + if ram_t: + data["ram_types"] = ram_t + ram_spd = s(r'memory\s*(?:speed|freq)|ram\s*speed') + if ram_spd: + data["ram_speeds_mhz"] = ram_spd + + igpu = s(r'gpu\s*(?:model|name)|integrated\s*graphics?|graphics?\s*model') + if igpu: + data["igpu_name"] = igpu + igpu_base = s(r'gpu\s*base|graphics?\s*base\s*(?:clock|freq)') + if igpu_base: + m = re.search(r'(\d+)\s*MHz', igpu_base, re.I) + if m: + data["igpu_base_mhz"] = int(m.group(1)) + else: + gv = re.search(r'(\d+[.,]\d*)\s*GHz', igpu_base, re.I) + if gv: + data["igpu_base_mhz"] = int(float(gv.group(1).replace(',', '.')) * 1000) + igpu_boost = s(r'gpu\s*boost|graphics?\s*(?:boost|max|dynamic)\s*(?:clock|freq)') + if igpu_boost: + m = re.search(r'(\d+)\s*MHz', igpu_boost, re.I) + if m: + data["igpu_boost_mhz"] = int(m.group(1)) + else: + gv = re.search(r'(\d+[.,]\d*)\s*GHz', igpu_boost, re.I) + if gv: + data["igpu_boost_mhz"] = int(float(gv.group(1).replace(',', '.')) * 1000) + + +def lookup_amd(cpu_raw: str) -> dict: + """Fetch AMD CPU specs from TechPowerUp via Google → Playwright.""" + term = _amd_search_term(cpu_raw) + log.info(f" → AMD lookup via TechPowerUp: '{term}'") + + tpu_url = _google_find_tpu_url(term) + _cpu_sleep() + + if not tpu_url: + log.warning(f" No TechPowerUp URL found for '{term}'") + return {"lookup_status": "failed"} + + log.info(f" → TechPowerUp: {tpu_url}") + html = playwright_get(tpu_url) + _cpu_sleep() + + if not html: + return {"lookup_status": "failed"} + + result = _tpu_parse_specs(html, tpu_url) + result["lookup_status"] = ( + "ok" if result.get("total_cores") or result.get("tdp_w") or result.get("full_name") + else "failed" + ) + return result + + +# ── AMD CSV import ──────────────────────────────────────────────────────────── + +def import_amd_csv(conn: sqlite3.Connection, csv_path: str) -> None: + """Seed cpu_specs from the official AMD Processor Specifications CSV. + + Sets lookup_status='csv' so the online TechPowerUp scraper can later + upgrade entries to 'ok' with richer data (PCIe lanes, codename, etc.). + Rows already marked 'ok' are never overwritten. + """ + import csv as _csv + + log.info(f"═══ AMD CSV import from '{csv_path}' ═══") + + def _parse_ghz(s: str) -> Optional[float]: + m = re.search(r'(\d+(?:[.,]\d+)?)\s*GHz', s or '', re.I) + return float(m.group(1).replace(',', '.')) if m else None + + def _parse_mhz_or_ghz(s: str) -> Optional[int]: + m = re.search(r'(\d+)\s*MHz', s or '', re.I) + if m: + return int(m.group(1)) + g = re.search(r'(\d+[.,]\d*)\s*GHz', s or '', re.I) + return int(float(g.group(1).replace(',', '.')) * 1000) if g else None + + def _parse_tdp(s: str) -> Optional[int]: + m = re.search(r'(\d+)\s*W', s or '', re.I) + return int(m.group(1)) if m else None + + def _parse_ctdp_max(s: str) -> Optional[int]: + # "15-54W" → 54 + nums = re.findall(r'(\d+)', s or '') + return max(int(n) for n in nums) if nums else None + + def _parse_pcie_ver(s: str) -> Optional[str]: + m = re.search(r'(\d+(?:\.\d+)?)', s or '') + return m.group(1) if m else None + + def _clean_ram_types(s: str) -> str: + # "DDR5 (FP8) , LPDDR5X (FP8)" → "DDR5, LPDDR5X" + s = re.sub(r'\s*\([^)]*\)', '', s or '').strip() + return re.sub(r'\s*,\s*', ', ', s) + + rows_ok = 0 + rows_skip = 0 + + with open(csv_path, newline='', encoding='utf-8-sig') as f: + reader = _csv.DictReader(f) + for row in reader: + name = (row.get("Name") or "").strip() + if not name: + continue + + cpu_key = normalize_cpu_key(name) + + existing = conn.execute( + "SELECT lookup_status FROM cpu_specs WHERE cpu_key = ?", (cpu_key,) + ).fetchone() + if existing and existing["lookup_status"] == "ok": + rows_skip += 1 + continue + + data = { + "cpu_key": cpu_key, + "cpu_model_raw": name, + "vendor": "AMD", + "full_name": name, + "brand_name": (row.get("Series") or "").strip() or None, + "codename": None, + "lithography": (row.get("Processor Technology for CPU Cores") or "").strip() or None, + "launch_date": (row.get("Launch Date") or "").strip() or None, + "total_cores": _parse_int(row.get("# of CPU Cores") or ""), + "performance_cores": None, + "efficient_cores": None, + "total_threads": _parse_int(row.get("# of Threads") or ""), + "base_freq_ghz": _parse_ghz(row.get("Base Clock") or ""), + "max_turbo_ghz": _parse_ghz(row.get("Max. Boost Clock") or ""), + "l2_cache": (row.get("L2 Cache") or "").strip() or None, + "l3_cache": (row.get("L3 Cache") or "").strip() or None, + "tdp_w": _parse_tdp(row.get("Default TDP") or ""), + "max_tdp_w": _parse_ctdp_max(row.get("AMD Configurable TDP (cTDP)") or ""), + "pcie_version": _parse_pcie_ver(row.get("PCI Express® Version") or ""), + "pcie_lanes": None, + "max_ram_gb": None, + "ram_types": _clean_ram_types(row.get("System Memory Type") or "") or None, + "ram_speeds_mhz": (row.get("System Memory Specification") or "").strip() or None, + "ecc_support": None, + "igpu_name": (row.get("Graphics Model") or "").strip() or None, + "igpu_base_mhz": _parse_mhz_or_ghz(row.get("Graphics Frequency") or ""), + "igpu_boost_mhz": _parse_mhz_or_ghz(row.get("Graphics Boost Frequency") or ""), + "source_url": "https://www.amd.com/en/products/specifications/processors", + "lookup_status": "csv", + "scraped_at": datetime.now().isoformat(), + } + + conn.execute(CPU_SPECS_INSERT, _safe_cpu_row(data)) + rows_ok += 1 + + conn.commit() + log.info(f" Done: {rows_ok} inserted/updated, {rows_skip} skipped (already 'ok')") + + # ── Entry point ─────────────────────────────────────────────────────────────── def run(conn: sqlite3.Connection, reset: bool = False) -> None: @@ -700,12 +1053,24 @@ def run(conn: sqlite3.Connection, reset: bool = False) -> None: log.info(f"[{i}/{total}] {cpu_key} vendor={vendor}") log.info(f" raw: '{model_raw}'") + # Check for existing CSV-seeded data so we don't overwrite it on failure + existing_status = None + existing_row = conn.execute( + "SELECT lookup_status FROM cpu_specs WHERE cpu_key = ?", (cpu_key,) + ).fetchone() + if existing_row: + existing_status = existing_row["lookup_status"] + try: if vendor == "Apple": result = lookup_apple(cpu_key) elif vendor == "AMD": - log.info(f" Skipping AMD CPU (not supported): {cpu_key}") - result = {"lookup_status": "skipped"} + result = lookup_amd(model_raw) + # If TechPowerUp failed but we already have CSV data, keep it + if result.get("lookup_status") != "ok" and existing_status == "csv": + log.info(f" TechPowerUp failed — keeping existing CSV data for {cpu_key}") + done_fail += 1 + continue else: result = lookup_intel(model_raw) diff --git a/static/index.html b/static/index.html index ae837e6..a15725f 100644 --- a/static/index.html +++ b/static/index.html @@ -162,6 +162,18 @@ ::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } /* ── Alternatives ───────────────────────────────────────────── */ + .alt-accordion { border: 1px solid var(--border); border-radius: var(--radius); margin-bottom: 16px; overflow: hidden; transition: border-color .2s, box-shadow .2s; } + .alt-accordion.has-deal { border-color: #f59e0b; box-shadow: 0 0 0 2px rgba(245,158,11,.18); } + .alt-accordion-hdr { display: flex; align-items: center; gap: 8px; padding: 11px 14px; cursor: pointer; user-select: none; font-weight: 600; font-size: 13px; background: var(--surface); } + .alt-accordion-hdr:hover { background: var(--card); } + .alt-accordion.has-deal .alt-accordion-hdr { background: rgba(245,158,11,.08); } + .alt-accordion-hdr .alt-hdr-icon { font-size: 14px; } + .alt-accordion-hdr .alt-count { background: var(--accent); color: #fff; font-size: 10px; font-weight: 700; padding: 2px 8px; border-radius: 10px; } + .alt-accordion.has-deal .alt-accordion-hdr .alt-count { background: #f59e0b; } + .alt-accordion-hdr .alt-chevron { margin-left: auto; font-size: 10px; color: var(--muted); transition: transform .2s; } + .alt-accordion.open .alt-chevron { transform: rotate(180deg); } + .alt-accordion-body { display: none; padding: 0 14px 12px; } + .alt-accordion.open .alt-accordion-body { display: block; } .alt-group-label { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .8px; color: var(--muted); margin: 14px 0 6px; } .alt-card { display: flex; gap: 10px; align-items: flex-start; padding: 10px 12px; background: var(--card); border: 1px solid var(--border); border-radius: var(--radius); cursor: pointer; transition: border-color .15s; margin-bottom: 6px; } .alt-card:hover { border-color: var(--accent); } @@ -680,6 +692,8 @@ async function loadDetail(id) { +
+

Display

@@ -781,16 +795,15 @@ async function loadDetail(id) { ${specItem('Eco Cert', l.eco_cert)}
-
`; $('detail-overlay').classList.add('open'); $('detail-panel').classList.add('open'); - loadAlternatives(id); + loadAlternatives(id, l.price_sek); } // ── Alternatives panel ─────────────────────────────────────────────────── -async function loadAlternatives(id) { +async function loadAlternatives(id, currentPrice) { const section = $('alt-section'); if (!section) return; try { @@ -800,7 +813,13 @@ async function loadAlternatives(id) { const better = data.better_spec_same_price || []; if (!cheaper.length && !better.length) return; - const renderCard = (a, type) => { + // Hot deal: any alternative is ≥15% cheaper, or any better-spec alternative exists + const hasDeal = cheaper.some(a => currentPrice && a.price_sek <= currentPrice * 0.85) + || better.length > 0; + const totalCount = cheaper.length + better.length; + const icon = hasDeal ? '🔥 ' : '✦ '; + + const renderAltCard = (a, type) => { const imgs = Array.isArray(a.image_urls) ? a.image_urls : []; const thumb = imgs[0] ? `` @@ -829,17 +848,26 @@ async function loadAlternatives(id) { `; }; - let html = '

Alternatives

'; + let cardsHtml = ''; if (cheaper.length) { - html += `
Same specs \u2014 lower price
`; - html += cheaper.map(a => renderCard(a, 'cheaper')).join(''); + cardsHtml += `
Same specs \u2014 lower price
`; + cardsHtml += cheaper.map(a => renderAltCard(a, 'cheaper')).join(''); } if (better.length) { - html += `
Same price \u2014 better specs
`; - html += better.map(a => renderCard(a, 'better')).join(''); + cardsHtml += `
Same price \u2014 better specs
`; + cardsHtml += better.map(a => renderAltCard(a, 'better')).join(''); } - html += '
'; - section.innerHTML = html; + + section.className = 'alt-accordion' + (hasDeal ? ' has-deal open' : ''); + section.innerHTML = ` +
+ ${icon} + Alternatives + ${totalCount} + ▼ +
+
${cardsHtml}
+ `; } catch (e) { console.error('Failed to load alternatives', e); }