From 2d87f6732245a1db596bab83ecdcd76e01c7ab0b Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 9 May 2019 16:58:25 +0200 Subject: [PATCH] Better database cache --- database.py | 19 +++++++++++++++---- scrobblers/maloja-scrobbler.zip | Bin 26991 -> 27068 bytes settings/default.ini | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/database.py b/database.py index c50df47..4a3146f 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,11 @@ from urihandler import uri_to_internal # doreah toolkit from doreah.logging import log from doreah import tsv -from doreah.caching import Cache +from doreah.caching import Cache, DeepCache +try: + from doreah.persistence import DiskDict +except: pass +import doreah # technical import os import datetime @@ -888,14 +892,18 @@ def sync(): ### ## Caches in front of DB -## these are intended mainly for excessive site navigation during one session (e.g. constantly going back to the main page to click the next link) +## the volatile caches are intended mainly for excessive site navigation during one session +## the permanent caches are there to save data that is hard to calculate and never changes (old charts) ### import copy cache_query = {} -cache_query_permanent = Cache(maxmemory=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +if doreah.version >= (0,7,1): + cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*10,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +else: + cache_query_permanent = Cache(maxmemory=1024*1024*10) cacheday = (0,0,0) def db_query(**kwargs): check_cache_age() @@ -918,7 +926,10 @@ def db_query(**kwargs): return result cache_aggregate = {} -cache_aggregate_permanent = Cache(maxmemory=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +if doreah.version >= (0,7,1): + cache_aggregate_permanent = DiskDict(name="dbaggregate",folder="cache",maxmemory=1024*1024*10,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +else: + cache_aggregate_permanent = Cache(maxmemory=1024*1024*10) def db_aggregate(**kwargs): check_cache_age() global cache_aggregate, cache_aggregate_permanent diff --git a/scrobblers/maloja-scrobbler.zip b/scrobblers/maloja-scrobbler.zip index b23ab38f62d4e999950f25115fccc47c246f06e4..bd80f15f4a11c74784024a2550ac2326d68c7ec8 100644 GIT binary patch delta 2416 zcmZXWXFMAS1I800RK-Or_K0<%QKMBkRLeD1tEs(JBGewCb~V&!NY{xv+N8!+DW&#S zjZ!nVpn9Qp4ktuYyYKbG`}@83d!G-_^X++_kN?e4VCxu=V1133PmC7;5Cml6UZcIQ z9&Eb^0suqO000;O0C-_>fxh9Pks;oi0TJacGmpl!AqmVoslB`brj|+O+5C#eiH#W~ zK>7aM&mn;p8yG`3q4oCj=j)%>hD0|hV$xT<4hC{@aBr6;?KpLz?&A94zY^^5@X~uu z6A}JY>kfi&ZCrOTP5UDAM2vFwevl_XN)2<7DbnO=5L-{k7C2e4eB@$PMoGzYZG$kj zo2JrZODWnJlQf|use$c#h#2i`YNf)83*$ZSk)dIq=j6Q10qoUYaEw8p%-`Bc^w7Lp z=Iqw6imHZg98=enw<6WB-(2RJ*XsRYzgc||8li{peuSI!bmL(ZdLi$u^C0aBVRWM< z>dX|0(YKwMdUmAPd5vL$viWO?`{1=cHDRL%p#kfki`EN-+M6O6T3kT>9oL7>mJYF{ zx8gc6em!@MJ4a#$-jE~d8lFOFlCvLZwurJooAJ0yJqb$7x*w}39=NPicbA>Eu~90% zgrMOvr>Z&b#ML&vX3>x3M~7D^n|??R9IO686fIo0;iF$vX_+J*MzpwhTJ($9_ZO5X z!5gFNZ*^E1ZuKuI(B~CW#b0sSnwOo0%dxM(G8FuW#omXwSokf?fP&=^yvAD(PCW78 ztgD>_1HZ5?z!FcE3_AnIHuPpNPmQ}0BK^j=z4#%f8_(T2BS%|@I0xr1OcO025+s3o z5W;N4<6^9m%o-VLhbytm-(RZvKV|4?gGzAL{4bxK_i&2e;Bl~S~Y*>o=DCmY=-SLO2Zn_usDl9k^)l0?lXzdw5C?S2N_5~LXz zY?{g(-%F@|lrviPsQ-vDOD23a__MWQFRrg&EBT_r(%zGY-Ny7xH*2Vv_>%Q$ptKQ{ z@&16nRu_c{_l2#GW+%=je9lY`!*otMIwPxpb?5-dt4v)B^*LXnQPZKHGtyBKaAHc* zp)r!ThgvBsPdE4$7VhKJ0mv^VFQ~=6D75`LDJ3Xx{mCt{B$ZaI=0?D!gn!+yGp7|3 z?ZmGKN+gfXS#`;5pqNOsYlGoFy4lq#&BdEqQr@Ur6R>9Rk#AzA=>hDO=!$|qCe;X& z5}b(U`3qbd{wvL94X&?X2A#QgIYws|CcuV+)4ltqhw?P=tg*B98OETwgsqda6)bfL zZsKioC-b#J&=UtZ!C0_s6X|iTwSTBcMBkkk~Q5LP^ZTY#=9gas6N=kY= zMfYaMWqj!68|rfE0-F7~^|uYqC=8y4Bs++U>g~tMnkQ$>Jzmfd;hqpTYG`US-2a>h z#b-^Lu`_lx^&(Ug0Gz>?;1+d^=isKS6*8)(4Sdfw5XJs3k}Z{E~Nd5 zbsVp`n?x9n_{Ly6x6l`xBdvyV^oF;U9WC9$?SS`J%^*Wum|sqO@$S)djZu+hwDj6A zw4kL@II|keQB#pXNVdd1<@t$v!h6j;Xd6!e1&!iDMIQTio9q*=cpcQB`j^InUX(Jz zF8;RAdQ7?WX;AFkt_1?H(ibNWYmxs=SwDJ>1C-YP+?mS z?`W4Z1z3~U4>E5TCr^G`%;($MzhFU=R+W(|vh0v6_=6O-iL8!w9;W*9Qyq`uW}E43 z0{FcYpIu?6S&S)kdZvv1v<-T^H|8Ws_g5uiep#@R4$BUB1^%i{q~%sZXmaUJi#eX) z^qf(|+x@8+Fyn@d-emJtgL8_z!#9M|p#$pF#kxby)$1zN@Gp~~j5{&FX8*6D`Kn7? z%b?7J6;Z9}+1tFr$i^ z$MW>}SL*vN)<$;xx?%&NE%~1&>3Ypo1+VO~u4D8Uf&%-RAZEir5jQI0%}{`%>;v1N zH+Ipx$kw{VU!LTT_k%p`ji02HVuFM%VfiSg{LiNLEoxCCq=b-i63eqKolY2;_RoYx zOR%8t%9T~`J~w2+vZ_CwHK2z4CTcVA#=Zy>LrFVHQzK%Z>3|u&9qeYa7sLMeD%azO z&c_1fhYGf_FFd09=^b3pwmZkHu%+)FAl4G6L6OgFwy@;#M@1?itkJq<5a|VJuN3l) zA1DmetJiv++&=13pSU7DMGz$tZJ*0UQi{8}(_mi`EIwc%P&R)w$;&Y?804T+yqPkm z{pv=Nj^p?voRvWS8H3?*W=DnC0gcEH^JxkRx^}9mZPNgW5&2ZwR{_a)7n2Bk%vk@O#2=R zao*bM;=tKEA3c2pj%Bp<#hOKh-hF!)cc7_Yf=1nTzupajB61$HHP*U@oia`S7QL9h zlo5tYD@&2eRcakM1${dLH}q*0${Eh-CVBHOceWGV+W|F5!Co6dJ-^rJRgMFQEa>XT z8`{@h6XA+HDfj=FqVf-@{17bhCAr&2L7c?%)dn;HO#F9}4}B286tdbacVGoM^&fX3 z&)+hF{dWWb1OP7c{GguysY|-zO1|tXD{jhp>iDDa-$};%B7nJMs_#u;GWpQg9T-M- d@$&#aBRBYY^Hkx;i8x8}1%F-8n$M4S{{c+_iGTnA delta 2372 zcmZXWdpr{g8^&(ljhoGZcDi)b&xZYCJZAETSgOVti`fMF|w3xat@;;w~$Mb zDfbn5&8-tPVQz=0DI!U|#~<(K^Pcy8{&>Eh&-2Igd!GNkPlth$As`)t21#px0023F zPI{W>-Qtx9ML7UKRtEr30ssJEp?DJULQH%#!Tc<_%3l<%j#Rnq+0wBDa(5X8WO0&J z;EX33$pGKWv2|=hDBhx>QUB;q-tV=%cU`DnC2q-L)94ZM+~}JmI^QxuzKN}#NBYuY zUYufZ+x1h?BV`SGE(0r>G15+Ta{$JKmiMpA)G$ELr$5|G^Y$x{6wVI*QgP`bba=1d zR4LY?9cP!u%O$>Kc-1Lj6_=(>E$(;7;sv!|&kssQzP8cV6W`=1w50E_?D;t5ZSmm# zJ!$La#Eh(GQR1L;k{d7Ts#kPQlP?yqtaG{sl*EJ8WZe$>Eh`=2(kss6-jCsjf4nK0 zl~Q`M=iO712|U=kAwRgb#&~0f^v*W3yELK65r5W`7~-(@8FG}mTS zY1qsz&V$b$eXGAt4q(2wF~_Nmt(PhH~kWhbkhvP2LIjRujS_27x!mKK6v_d(hvQ<5mDcFc~j#2V3)thGsj!mp|>YlSrru* zW{^K5sUL=l>kn?i-1*s9Q#+!*2}%FbmWD^~4rnJSH_{ON*AVmb*VkT`EW(8d47t&E zf6Ul&B6G73<~{owHNBKJWClh~J~|9(K7ozvz|cv57O_Oc7mf}w+G~tPSS-uy>z>zd zf1Yft@WtK1P%2yRV$J^yny(mzB!{>IHEl<|<->M*22(R5;WLM2DYd8PvFUjuh8Tk<+`GQRn zf_}qp{uWj^Um>ObhlW+GM_|miMzd$jN+qlmb*R*MXh3EXp}tOvBNJ%9tP}boXa?F- zwu`9c{GL{Hs&`Li^nQJ>AD!=Wk5$4=^#aS{<@_Qcg2MR@?pA9rrE#y>#s%`l^22KG z4{;?In-fo-U5wi*$zDv)PufX%v$r=j>@fZKl;?d{G5iDO24M$dn2nO@FgyB6`&bXJ zy#@gT)A?m4#CL6NBj8F{07gdJCFRhig{rXBcr9pt=T5U56{(>tucp8erl_`id9c!I zM~M$b1}r}8=%B!dzW1*9Xb;)~xhIOVE@;s{N8%Ot`c+L(HE0>OXM$}gNLwW#J%~T( zIpaL-zDRMK;8rKshdNzQ+Yj*z$DZ&Nbjw|{p9g*$a*P}T~gam$ROk<7?+*Y7Ju_@s`YPgpI zJAJ<7cl%xOx8~%T;{7XfFuI;#@Q(iK`Bzyfr;}a#&zXO`(XzY;rOV|411u*(t^9^$ zZ6zMq2Aj5qEsyRZi+0Bixn;EPp7N_W^Gi0NugbBII<74a; z^)81#xD1ICMc|mo%f_I!VkI^@@&3kMBaEyo(a$7fnsE!1)HYL8zrj3%OOzAMu9TGfVM3gSZ4Ek5>Ux|!Kk&}? z+g@yQo<~x_)j~8((mhg#XlnDBFNoBy^=WB}{B7J6nazRKKqJUP4*NMnAe@G$_70ms zBaH^|BJ%~VCdO_$iX?0TfM~cW>|Yu0>v2~)z34Ys@n=uI#aTmD-PAn3)Rl;w6x7P3 zUOdm029HVyD^&#JWt#d7;HR01j zEPeLpX>cNErcvMJ_=LF3>A5}+aQ=x;v$=Jc%7jO9{jTs&yqveU|7>s7uMK1LYt|X6 z+$wIEAlda_&r)jd*vw7qgzuX~FF-dchu$h?(3GG2Ft*Q(nc&AG_wfhskn;pmNeCO4=L0!!J3tm=kKDi|# zW?_aONzHcD4REbO=5`AC)R9I{m|f|alV@C+$_D~&hy2E4P0h)Iqr9());ewa3!FBt zmTyf)t{!m~i8)Zk3O9TOC?HWX(Go$XjP)0}B;8y8z|hN(8t z!Pg%8hZH{TJ)X-}*`>us9v3Kt8`}5skx(CG_u=FhwLz-UFJpN~y!TZb+|;3O)Y8lm zRdu};YHj6Y{_6FfFOY4GU}M6y>+i@_dC8F@(>%w{EKu0KxCoF_PF{T*&IWSo{{_S7 zZ^hUy_?1VDZT>?y6aet;82|{-23$#*7kYnq1^gefQQ-)n4Ld$O2pGy9`nxN#Er^cr z|HLDJ0DzO!w#oc=&{_aHlc)<#=&m@hz44!9w-6CP5B7UvFwmNfiU diff --git a/settings/default.ini b/settings/default.ini index 13f2f7f..ecf76f7 100644 --- a/settings/default.ini +++ b/settings/default.ini @@ -12,7 +12,7 @@ CACHE_EXPIRE_POSITIVE = 300 # after how many days positive results should be ref [Database] -DB_CACHE_SIZE = 1024 # how many MB the database cache should have available. +DB_CACHE_SIZE = 8192 # how many MB on disk each database cache should have available. [Local Images]