From 2d87f6732245a1db596bab83ecdcd76e01c7ab0b Mon Sep 17 00:00:00 2001 From: Krateng Date: Thu, 9 May 2019 16:58:25 +0200 Subject: [PATCH] Better database cache --- database.py | 19 +++++++++++++++---- scrobblers/maloja-scrobbler.zip | Bin 26991 -> 27068 bytes settings/default.ini | 2 +- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/database.py b/database.py index c50df47..4a3146f 100644 --- a/database.py +++ b/database.py @@ -9,7 +9,11 @@ from urihandler import uri_to_internal # doreah toolkit from doreah.logging import log from doreah import tsv -from doreah.caching import Cache +from doreah.caching import Cache, DeepCache +try: + from doreah.persistence import DiskDict +except: pass +import doreah # technical import os import datetime @@ -888,14 +892,18 @@ def sync(): ### ## Caches in front of DB -## these are intended mainly for excessive site navigation during one session (e.g. constantly going back to the main page to click the next link) +## the volatile caches are intended mainly for excessive site navigation during one session +## the permanent caches are there to save data that is hard to calculate and never changes (old charts) ### import copy cache_query = {} -cache_query_permanent = Cache(maxmemory=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +if doreah.version >= (0,7,1): + cache_query_permanent = DiskDict(name="dbquery",folder="cache",maxmemory=1024*1024*10,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +else: + cache_query_permanent = Cache(maxmemory=1024*1024*10) cacheday = (0,0,0) def db_query(**kwargs): check_cache_age() @@ -918,7 +926,10 @@ def db_query(**kwargs): return result cache_aggregate = {} -cache_aggregate_permanent = Cache(maxmemory=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +if doreah.version >= (0,7,1): + cache_aggregate_permanent = DiskDict(name="dbaggregate",folder="cache",maxmemory=1024*1024*10,maxstorage=1024*1024*settings.get_settings("DB_CACHE_SIZE")) +else: + cache_aggregate_permanent = Cache(maxmemory=1024*1024*10) def db_aggregate(**kwargs): check_cache_age() global cache_aggregate, cache_aggregate_permanent diff --git a/scrobblers/maloja-scrobbler.zip b/scrobblers/maloja-scrobbler.zip index b23ab38f62d4e999950f25115fccc47c246f06e4..bd80f15f4a11c74784024a2550ac2326d68c7ec8 100644 GIT binary patch delta 2416 zcmaEViE+Ny#hyQoMeDisWEmI4{M(z{|kEkd&C5onDk*nwO%N zRlGLxcHd-jY|o zQ!%0QdD8DQGn3U*B6k{8X>uK3zU{)R+TcRJ%~dvUbX%^iQr{2tHOjW~C3Lj9$;yUVn0bYDC5X;Q21 z#pMyNuQ|E|zIyttXQk!b+P5dPbreoNC^yR%4RO`JrvHs$K}}R|glA~wsXu- znzM=bwyfSQPQHWxBZaGNp14x0^HP@CTYq@RFUsVAslDM637I)O_IYzLXd(-rYHR*}1N*_pbb`-PAYZ((1m`f3Mx2Tz|*v?w)l&t4^KPZ?=(t z`g20>afh>$q60)l#GVESGfLUcy>{R?>lPE2granXH_wqxLhN;KDnjn>#wcu2bXsQi}_}YH(!3>b5#1R%RO_ysO{Fj{B}k8 zv_+=OU9onD$>yxrR>xSb-rm`xuuSl|yk$bGZ9+?KgCEl?-YrE=2LoTLTgtnM-fpt5 zFubq8{z09$Gv(CH^Ye8IUtZP?>2hFuQ2UYl{<@c%;@=ud+{4{pFJhYzs$TEFarC`; ztkKEcty_HZ{4Qy~Su5b({bfGaoF@U*L9E|y9t!(=RY}S0%hIO%SEN(Vu6?U1tI4i+ zdgitgD=qnR!UD~qVuEJBDrMZ8yB_pE*5UicDQ3HU=ML*%cjk*0PPy*(q3eg9S+Qy@ z!{>7qxw|zJ;?KR8@i8ykyqC8rsI>56y>h_&m(E?Umv`$d_?XeLz41iM6pbXd5IHV& zkn4B83o2~msee9k?aa)gNum57@vv97&>uW62%C7N- z>RW7GbbqP(^|XEeeqK#HDEfG5cBQPtjcHf;7?`@HmCv)3+-H0wWV(HpylLCr#IP%y z>l*7X6o0?=A!7I0C%a00&d)Hr@JTt$GrB04vHgXc!1-?q88fPv|M+`zv+YvPa;euB zL>KPfz%ywR?`I7aaczm+Ra2Sx=W?dJcF*}#SkIPoDLJ37Kl8ZbuX^XC-zs(u4-ajI=clUV6n4wqssG(^cTfG zT$r25IydZZ)%{&(Kh*OckYov7bi}Q~RrKcV)gPwr75(?~a=66VOx2DAi^o1v1=F&Y z@qRVvI5=~Iz(Lv0@Fz3kc{^uZ(q8}TMg{LRe`&MpyS%K76n|Xs=js$aqdE7<)<1eL zd{s87KfKP?Ra3#ZEAwmq0@bJAJaZ=1J~xT@TXb=bnBbd% zv~O;Ysx`$sxc(-asXRMnZq9e+wyMjcM5Ys&8!XeGY$-kD^gK}D;L}XuZj+=38*`uk zc3CdX$T>is+&(;Qu2@9^AW{VGTn9#B+A&19P zVS&p%IW^ai-5O;VyKD2-cE60@+j+MB;?2xS3gzN2Mfa^$R;`C(4}I9>Ht#}a;l`+cy%GP|*Zz?Ys+<>FcKYn0Z}EF; z{=F=C`ahQK< zm%33e*wGO*Tefu0@}tMw6du-k98MGv{lHq@ofMXx%NA<5{C&#73tPt952u7l1oU6K!oEpvZ~f;zr*9U|cj^4o_CH@B;^W7o zpVdEnIToMF{!iM#Qe>AP*S_!js($OqJNmhl#P}W;5Y?X1|3T;V(F@^|9N!7rbUy7W zNN!u*A~{oO&qX28^%vExQ}=MqxG>{*cM9vXBl|mI_A_iY=QtV2VV1Gd@5Sd(w#gh( z8O-^oj&ClGGUt}XY^JC%K!8HSlF7$Y<;A+0z6P>gWo$sDC;v&+W^9?Pkru7m;~Jf#CfldWh`D~A8U`{Em7ZLfuFW`e^4#<|#^%X? k(qkA4CP!w(GR~U3JtKu_WAfyNWQob9nI>$nQ$g+n08IgjfB*mh delta 2372 zcmdmUneqK4#`*wnW)=}<1_llWsm^x4g-bqX@Nh6NuuCy8@G>wkBqb(irx)dy=B4Om z6|arBohNK4Q0uqv&>Lnimm3WIQsn5@5TyQ0JYxidG;y?ey- zH1E`9uXo&Hb~_(bYwG={x@q9W1eZq1mV zxbpmW)(wr!#~#0(&>kE<@BRFz7oApBmWy512!FIZO#fh%W&7M2>8Bb4H*tmWym_pn zzw{7$^3jby3NOCD{^LOX_v)Lox%PGP8=QRiC|G~>(#0%hyQ{nU_vSo|$$j533;1|86dN%EWt9{qCM@HHA2{ZFAx_ zZczRCB>S#K&xz$#J8hG*{L$qlEo}gDb>$lJT;nYcgaq&pRZNF*3eAcH=Ht`91mLw_7x%7VX zWwD2wO@2#z?V1#(W0|g~maSO%SH$Nuzu4jI8JWs#&o6a+`f)9=Qa|mSc0loV3$6S+ zAF4ZkpOp@Ndd>6koAyh(Y=#fE*l_Rm4J$bmP@jEuQSa^a%af@RiRM}0?O9xUl^u$HRi>t8{AFtUW}p^H-b5nB!iO~FY7ZIa{NA8(4M^QKGP4@ zYl}km1@y2uyY+KgUNCK#I!z+iTJ+yrrN4nw%Qn=oY6r?6DRvb-V&)#veX7{&VAq_C z+nn=$XKUB@S-zMUcJ9S0CgE8k#sxmn`M1UG^Q)RgL8sN%BH5%O9ewXA%1wSU&@lE2F)z3!Y-Cz*0n<8)h+O}*PAzoo7Z72XE)rSLai zp6tnbNY~+tgyYG%2RCR-v(@ifp_YF4z=7**YosFsSS4JVO)FonO=>F@7n^ffQnzna zTVikE&gknMw>O^naB+<>|FV!WRoWKIQ`a?ZPP8u)(&7$J3-b*^Mw zCXfCoJFi*c%yUEPBuhlT%2ejo${FD_1+$! z$tKzO{pk5xHE_^=Ps6Ho8R^nn6jw1 zTduL{yj*aMwTwRV#DyxZXBgHPh;dA*{~&j?qs5z1B*Zd@rL5-)Z=}GflwH#11rvYg z*IhsI`dCG;?%sPv8QQyc88-H=Ep0@~1zy&Rk#JwKc*+^~n{}O&ol1Mv3KO0Ez)k1d#7ag<8K@?DiXY#Z_J&3x@kwBkojPYM_Q zURS}rEix$is1~QK@Ycr9LFU_{J{uIx*<6w0oOCPn){_?P-;*o_**3-P=J}>;^d?$r zqrzG5q~p7*Ufi}@S9R@zxZc`lDRH_JekM)m>fNCqR3F>Lp7-cn)eOFScP2LrAF)2G zz?wZJCuNEnCwsoPc)@#fsnQ75!0V0u4S9Rt-w`c&S=M$y$Md|>o&c9`Wm+}4@vSdU zJms2fxt2w^CvDDDwz=M>m)ojuIF<5=DKAN0+9C6Kb>h*0s;8>cq}DI{m8twJO5D?3 zjYa;Kr{Ts!mfrQ@r&j&ptNGdg#`DM4MkVREro5=X(J7ugYh`Aq+<({}wm-&bxvN|8 ziwk!{e`$p6p6ydNuVucs^!pRnH)-!Y5b|PPrsBqseLFJ~?&%m#nj*bHETj0`l*z{$ zkDhz1(02N=npmdFh2-0MFQMeo1+>I2iEuXmS*|iiZT-u zbQkjZx@p;MdoH0BOvjJSU}0W%^{P70_m&#|eZ767Wt6qQ(35vYdl%`S zaOu4N%+IQ0sZLqGc(E^@@+>*qM@_ctc@4-XAXS5$y(0t(iQtaoNOSgEs zYWQ~iQ?=^JzrU+gdzWZ{rRSPXmCX0wtMmUXSzgy@H>GUq9-fX0xjjO;^SK{-GY7TR zOTXE`B>yq_#N?8>hwldN|5@?6=efd)m3j~U-cIiixV(kw>7h**rW(gTlRUSp;`(+! zY0KrQ{;94Vd`6Me;}hogK6=K^mJ{51``cmj^YiR~wyga7AoGiz^X-RI#CTS@C$D0T ztbTugUq*4+o%4%a>K1>0+bAj@zptXpKmTuN*@1BH8*`7{R<#1#=CPgImm_9-02CSl)A8P+0DB#l|%c=6vZs z`IQ^gjF=pdCb4+|$J#)S;>>oxJE3d14D%Y0u&myG=f+t z%?&OF1&|m6BZCA(L-UJs!EdiHqS2E*)3h1QCzqzhFeXmElorFtGg&{~Rsm+cHX{Rr zJrk%UgKW{a$jLqFGGbLHRvCdzM5QP1OV?)fnfxj}j?rwgXGRR8^5kV1v5dZxSu#_Y UQj#a1O_rG4lxe~?FBRlA0OY@6ng9R* diff --git a/settings/default.ini b/settings/default.ini index 13f2f7f..ecf76f7 100644 --- a/settings/default.ini +++ b/settings/default.ini @@ -12,7 +12,7 @@ CACHE_EXPIRE_POSITIVE = 300 # after how many days positive results should be ref [Database] -DB_CACHE_SIZE = 1024 # how many MB the database cache should have available. +DB_CACHE_SIZE = 8192 # how many MB on disk each database cache should have available. [Local Images]