From 2ea63c0e99a5358eaf38785ea83b9c5923fcc9cd Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Wed, 12 Mar 2014 21:57:34 -0700 Subject: [PATCH 1/4] Better VOC handling and resizing --- Makefile | 4 +- dog.jpg | Bin 22323 -> 0 bytes src/connected_layer.c | 19 +++-- src/connected_layer.h | 3 +- src/convolutional_layer.c | 79 +++++++++++------ src/convolutional_layer.h | 4 +- src/data.c | 24 ++++++ src/data.h | 1 + src/image.c | 2 +- src/maxpool_layer.c | 13 ++- src/maxpool_layer.h | 4 +- src/network.c | 67 ++++++++++++--- src/network.h | 5 +- src/parser.c | 18 ++-- src/softmax_layer.c | 41 ++++----- src/softmax_layer.h | 3 +- src/tests.c | 174 ++++++++++++++++++-------------------- test.jpg | Bin 9756 -> 0 bytes test_color.png | Bin 1097 -> 0 bytes test_dog.jpg | Bin 30762 -> 0 bytes test_hinton.jpg | Bin 16456 -> 0 bytes 21 files changed, 288 insertions(+), 173 deletions(-) delete mode 100644 dog.jpg delete mode 100644 test.jpg delete mode 100644 test_color.png delete mode 100644 test_dog.jpg delete mode 100644 test_hinton.jpg diff --git a/Makefile b/Makefile index 4c1bb148..a02d7ef7 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,9 @@ UNAME = $(shell uname) ifeq ($(UNAME), Darwin) COMMON += -isystem /usr/local/Cellar/opencv/2.4.6.1/include/opencv -isystem /usr/local/Cellar/opencv/2.4.6.1/include else -COMMON += -march=native +COMMON += -march=native -flto endif -CFLAGS= $(COMMON) -Ofast -flto +CFLAGS= $(COMMON) -Ofast #CFLAGS= $(COMMON) -O0 -g LDFLAGS=`pkg-config --libs opencv` -lm VPATH=./src/ diff --git a/dog.jpg b/dog.jpg deleted file mode 100644 index 3b9f7abdd8b2315e3fbb6f99e7db08004726428d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22323 zcmbTcWmFtp6eZfYy9BpJf?IG45i~$>cc*c84elO1xVyW%1sZoJIE^+wd^59V)_T9* zd#7&IkE&a>&c0Rm)VXKxx23l&0Je;Tv;+VK1^|G0-vDo`fUf{JSlIvCyTQLV1Y`sR zcz6Uo%Fq?} z41|n6l)eE_mx7ypl2_R(asw`=xF5~0E2=}Ot#56t#&{9=x0~S>9dj|>0I{%sB!_<% zYs&$t?PKk!uT*0TiAN@kLl3GW%Y~$vp#=Y-RrO5 z!uUO{ru@P*)y@1JpB)=;5gkdfEIcnq-{+NOE3_BsEJY-*pdkLZQvj*J74`4qr8iB$ zb&bjCP?_Ft=TRjMcK1k^ggSm_ebqha9Yu@3S?W-`^B=shtHpyLuJ<@om`DWCDkD4l!JxpG8hY%}NFdS6sBNBpfCB2k=71&SGgX zw9#Y0dSXn^LiNkGCoo>X>x1onh5@8uhN4}YLHN<$%U;e^k1TW3qPdx?+@o$yIH5(% z<99t_&ppF+74OR88K0tdD#KF9)@O2qikMjY#WUEyThE4szWwQ zanXU2Ql*j{HAzOCe|&x(lFO^8UZ(sna_$@9K6qb&4xOz;M^OPU7+$*J>%tDOzeZ_F zrB!MKOyOelVW8?*kmCN6Z81_!jbU8LrXg;iM1V?4ihPgf$LDeqzB{K|)0SoM4Q)ea z<6wF+s`tP{H%Z}>q@|nI8=xQL#mql3+msKt<*H9F?{`r|nMIeG(THZoa6U>X_Ti^r zmS;@Xb}UpD*ix_Iy}eh?Y-_l`Riqs>O-%N2y#{Mv^{~Ft}^6HE$ts<{KZag3? zfKQ^E#K#PjY7X4N9UsTXVWu+h!tLKYtkhsC+KLlyoc%7W?G4cLrHb*6nuq<28Qvuu?VkzVGKUTm6Z##J>*wJ@ zpGiK@W-H{(@u@RXjf-ueDbPH7We46(JRhax_ZDk@od)w*d`XTv#-KHylSXG#comi0 z*1pfC>Y9Pr5Q+{v1Ag;SH({&15*U$cRC+Vy$)cMg3@{(?AF{NoWcw!nb^q{10b57D z?w3Wb#@xt&sT<$KXJKc89m8#F<8I@w>Px%j=*GCVl?j!yeGP+cDaFiJ00py?+@(!Z zIPoHlS2#oa1fN3*B0sHilzBA}1G<0@Z7^?^+@LlmEzn8$Y5Hp%)sp%lo_NrhF@#g1ExLys8C-Eca<&&NqOMx)@^I zD?|LJ-?kW~-SI}alpXjZXvRUyAO~lm@D89;*~J?`Qn`KnR8wQ`4Zt*XX>#VUlkrhW z86_(N6a+?|jvclYBT>zJ=_J(UvyeZe^!X#bEoXQmyIRzi0yZLMSkVaN2U@@tG1tvD z9I-Z_RcdMEUztc5hxgk|(ZY0fmb33PIBI!|XN&RwP(!Bla%ydY49g=TZR;)tt|$a@ z$)P9o({)s!+LB7~p^A3kVr?sr&6P{KF79b3&yoCd(vpv~+094-+ajgZd;B0px(JPm z!pXMhjBY79PFHvat^_ZnFZF!rEwTj?Q->O{**< z&5O@hV+_@~&e{(%cHycNb~Q^ju~0}SeIebz8pvG$ zX8h%_R7;afHZfa}@Hj9*ODd{m50WkD=UV;@tntw}=DqynXT?l79}kDXa$0Z=j`DK& z0S)lHxEz;I;Q0YO?QTO==MPSz=(KlG6~)@jv9+&$%tPPpD6=;eMXhE9yuX zx|FTbjuW1|ceopJq6uW4^<@Zco@aqu4db+nE1i{e0z~qp;fYjV%sZGcERFx2j!|&C9B=cHqf;xR}zPu5P>x3Yt z+R;t0BCm>N1(Qd*due0~1;tW=NqfE!=RI0{?ef%Klt-V61pIMnf1{>w>(qmr907RF*#cayhl zNf3y&HTGv)YpVo&8NUc;cCW@OU?I9F(aqecfZ5Ivvs9#lkqEVbxJWTYV0t2aQ5Y_! zm`_;lPWHu3von~%xoyrYot6ve9c}j!Aq?PiRvzU?2x43ON!Z0)-<-gcaeB{ee>bt& z&UCPCGXUPw?ki;?cdcB7bRUdrhU(r%|#90u1;{)ZJ$Jafki4eN^ZP1<%v;-r-2*H zeL&8|qB3X~ZJoYHL$Y)fWCbInX49&M5MnRBC*rs>n?!T!wzaC{aBlAz<>gBX)%uJL zly)LEWnc|=6#EQv+ftN0nY1iu)*SdLY2O~h1@~)6Y|G{Am%lDzAH2!I1>jRQw*8v< z{a+2B7K2>lo_m_M>Htct#!2=8FmJBi9Wu$%aM%6SF=HY#ri6T`DRAg5J|nIL|t8Jv7A_-%IAB zl3S~dNfHkB61>RF`h6mJCYxI`FloylrqH--1Wqvd)OZ(7CZ4@?uY&Us!MOT z3X5zjV6SeDJCs^6qj8uvHY01C2|rKp@2SQTfwBo86jeO@&bo#s94ywUA2`{8rG0@u zB4t^m+Yf@d*HtbwXpDy$+b*X@KY=lfOA_kE-g{n)D3`DQilaVtNXS;4Gz1Fn#e|BK zw>LqHLHYmi{@uOefe)MFH8szR(^fHhr`IVfoi3p>m2A{G@-sKuD7ywbQZ{-L$|ouG zWknR+S&V*6z!gntH_k4uL1yS_O8AyvU4MTSmjdl!vD3q%`F1t>u~u=Pb~V$ObezJ0 zhhqK*su7rfD0@v-BFa0lVJ@nghaIX0kysfRNmBORZb-IXFwlkXhf&_9C1dg5%XPov z7=US;6A~ou8k!3i*5OOu?Rb}=ub~x5=qvqKG+C+1}22l^0z7XvR(XajmRcR6wzY8_jYU6 zrvdZKZ-5U_b;->BQc#oY)#pDA)R$jwbxPq6feTr}B23Qt3N?pXP7A{}RThGx<=a^w zzWU)=D4yU9+FpGp?~M-`pq|2}xHa$5;*H$W&M2*F9BTL2(HPUVyyER6b2euZBpHBU z<`%ju_uOr;wQFWbrhSoggAHYE;q1{lWI)PJ9@X@AzI=JjDA)hOwiEj!z?N2fy{6%* zAVwtqv5V4G1Si&N`3bpUEqNwKL{<}63KoXjZWsE!ZtF7MEQuS~bU9RT!Gh?hVESC1 z5Am(sw)s}DC@8lE{z?@6ZE@RCq|e_g`*x@44bUL*)EEfv56Js~N^!8WWkAhT5QXW< zcc|*^1JT>KR}TdD+>cit4Lb1b@7Dua3d8%VK2tI3%|{*ojTVr$ZvJ$Cc;O=1Xz?lt zzD<(|Hs1z)52VL9_(7j0J>`@Dol+>YvwWp)2Qy@xl9=Tf<|KDyNCot~0X$GHvKy~O zrUzZ$*A-ge?@b|IV#M`n+Nu#egq0KreZc~Uq#na|o^tHFlHuM+=A~9{i~V}$vDRw$ zBd|bwl+XT7i;2e;(Y7iDU0;(-z_O(`*_G8+eM_gUIKy#pJ@6iMbsU0;EjJX6*OY@8 zo}lgzN0;Mt1F>YbnnkpP)2Ts0)5d3mP7+UQK3~SQF(KV$2;Ot!+27##%kH9FTv(OklcAmL@v~=a zn1+pG3=wHU@C~4KSS(ZE9LSvNM(ZgN3!z`ypt9jpY;N56x*`901##yO`rdx+;zB{r z4qL8QiF}hQM>Gxbrp;P~_d>X1yZ=!_dHi{gdXCkZLp9V<>g>~Mb)(KF*+%33Kj^Lh zoFXRX898D955+|plV>3Kv~m2rw9X0wknr9U%nj8p+XYx{_7 zM7t05o>N9K>i~WKmNB5LRT(qTR^*~0Q+@CPhjzna)XR|f0jH6`PY?r~aLKi_QZL01* z=%1UAkAmg1=P$nK%n{sMJT$YJJ2eM-6m>;x8+>c+z#a#D{XM@FPm^N+`2+9SEAl8;m=C@Qt{sEQcM0(*kpV(|c$!1*(|A)dtORiwkR`E0sob)#fv* z@79n;h0cdFB&IK44hJ{*NaqR8F)eZsXq#B?P*sJEC&)YPbNj8z3!+9e0!j2WMqdI0jqeT2=P~z`ojk6;4FC8mc;Y5i+4Vpn}CaK#Vf~;-XjA;Zb#gp`JXO`t!;qdB0 zU*m|88PJiZh@$RJ7CL?#3E(O(-7}DOPUk&q?uuab;Y}KrDu9Zukb~1#j+#b@lg_9) zv*odPCXan_A$}3wRImKAjdnHeqI3ycqllZlJN)bM{$HH3Ms}lId3F^+_oAY$8M=!X zc#8ApKJ_|NswM^iv?b@hn>PSSsN%fsr*)ZLxzpO^VWx%>6FU}+o^IkHOeAu_?1M&^ zb#=3bD>`La_1fcCT*z%mZH1=g?3z_2{x!0;VrzgZJ{cBA3?_F|^|K~K$(>TCcJ{_A zL;pj^U5KT5Q;Vq-E$tsAWZ^@^7DW@AohKDVlnJ0?TdvXP+bu|@Z}K2pC0$i{X-AW~ zQ~nxkw$`4h|1BJx@Zlz0rOC0%oh`e)47uhR1uehnR-DbSQ zBpeuxdTM~uLp*&5b>W${`0zpSa0PdtWIdmmV%0tJWp8-$pGO6OTQRqkyt83`f}mJL zHb*-Iv$~i!nR><`X4J@~COI^RiuQsw8>R0#Kh`R@@L7A|lNj3MmK)*y0!e>E{o2O4 zS8g|wh0_SJ-hi6OeQ;x%IrvW#H@rc?7$g|R&0jERWLU59yD&y6H1ua(wnf(@W!yvp zTFwSbW^gQw^uZ7IrsFi#pyYj}VPQy%r`IU9u4V1N%SkWTPpPg0pByQ4DAI`UXnlDq zK`uyPg(QVw$-}m=%?V1e@t)}eknwk@Uj=oHq=gL)a@3VI+-fe@4s zDM7n@QpUw6YKYm(C5m3VgkNAUc(@6~ZX+Rvf9@p9+drr)wd*3Hx~Q-1ac(YXI~o5F zGl;)SWw7Urn5sD=)^KA-GnXoxP?gZOYuQWgbfLTB%{fzTYpZ)png(=Sb#mGF=WbN7 zA7jwn(Y~bZmTb1g#`|IE=($R~AIKnqD9=O8 z?ah~!jguGWhnB0%M6t)_ZB*JCPdQWC=p-Z0Bp0Z(ag9`AuiOi z<+dyE2C$g-viOE8Mc5@eh6vK3;b=hK2-hb)A(-mNB2<0MLao7lV$P>MC)9laa?;b)CJsuog7+vFi(sS zv94pET(&ZOrvp%qCAGTiCt6et=E9T^+`T1OS9e|WH-$6>J*K*6g0E4lfO1UnZFPe% z+LHhjPo`EfVTR{G_@(SZNF(QtJLWez^Ne!OD`^OAgb0?L6!^SMO9F6ocFH_on5vL6 z&%|qe=^ioTQ2O(^V(wEo?yeeVb$VIt+Sm{GJ%$N~*K1a*bqi3bo`tfcH9|{!f`$qt zfayI$yV=-y#%8|J9?%Co<5iCf&sxOyI&mdi+DJ#m4;y^BBX3tvhBn`Qsh?jJG;6q< zVeMP{jy@C0^etilw$NehmAw4;Ul~VHUPhZpI;RyX>#27zJmRKYTYOZakQPtUNlRcA zrLAkW{kK1mkrkLqFY^r97+;!fmA2$>@l`Na`|%DZ&qULWx{&IS3t`e7l!kI>#Bq7) zcTP8AQrRykPXn3B@P}0b6(<7bX?eUGy8RQ^C2Pg8X%db;Xm}3`9?>6!nQNyb79#7` z#_OT_f5gQSJqhjjX#>pTgK&!(NTr4xVuxDaL-#Y?Y&NODjjqJzk5(1GhF;ezX|goC zcp`OmGk6Ip#ZKTtHj)LrvLdF>k=tcl+69dTF>2;6>z_M zfi}sz&|b&ZxCh=C4z^SMGlVyJ$3}qtCh{MTkR^)7VYAOJF){q(BDj~o(#VZ*hlzR4 z^+r?&;&)1IPMBryEl}QZC79LZ8D;e~GiV^vA8QuqwuSIYtmt-`|oVu`NzSI zQWoJKEvicRK3l}VJx{O9*9>UTprB-MrE$mn1x;`|2#x*vX=+3?1JyKRkn^R&ps}SpnBSf_gW7@Yb$Y`X15w87O_DeX zLWc5!WI?(tu4*;?EPY?b%DihZH#z0`pT&_NJgE&J-3f5Y?A)GtiihIaK6V@RZej}h zvfA+9ka7-5%q&{Ge4w0FKZsx~1*4so0(qKWSnc%g1mT$Ts+s9`a~!6_kc4RRlmc0Z zY5<>KM_+$Zg58Tu2Fz+VcA4Sqt^Crn|G^HFaL;BNQ;!JfH1>WQup-j{$UWP3~Okb8{ic0;*yZo`;)Bv7?d+q zZ$k|_fw+&NTj1L{1D(hcGcl32)!zV(es%Pks#*PUqcvMWaoi6M5epnaZl17Uqu0a& zD;%AvaZ)A;Qm3f;dV?FvaIs*vLNCFoNrnq_fkekZOjU8JyV2L=EXruRDqA&K>ArKg zKxEovz?ImBSk{+jt&l>pZcBiWwKeQ#T0Y|EvV&6Vc(=dJb|#hN6>d%*T{>k6CEhmu z=3?L&233 zk#?M+2^;1cYcOQS&G4^03HMr3(#y-Ip2Pib02qDudKGl!|>F-u#0FZ zqOp<{tZG^cALa}wc!&M$_a95m=!y@n_sVQiEATTZKya{#p!)n4kcj7_?R5KC$(Lw$ z5P|D-RReqME7E0Z)Th{xqbffOP#=QiQa76llYyfE3re+hk~F%R+*u>$4v|5QU9n$Z zm*-{o>E_lO0Q*@na)^&au3%QXI|t-MX6#(>2EZB4eRa7B$CjlI^1SL8&K|(gd(T+i znCWsm3dv^u~)USMnvQw~%U9lL2oj-;;CfX}LE#i%(X0Po2^rs5*F5Tul3EUz3sFOx~f%su-m z=#HyEzO)R@@4x%2W5rwc4Z;=6rNP-^9DmLDb9a$w&q1GR*w880C)gG}Y(_`dE-H3| zE9B=G$I`~`d8``IpwHmtomXV1(mYL*bi-ZvPRHgo@`f;t&9qT&SL;EW1e9(|J#+L8 zfV>`%TyXt6Nx1cut*fu0I4mlkNJU+OC_*uE@1L(deWYVFe$7NF#Z|9E)>`6_Z*sd| z(J=|uFCN+7B(luGVRh@yMj!qV-G0RQ`e%J*a~h#@@RX4782RbPu||I&hrQtbZ`5K) z{s5mKUugABy<@5oPl(jPJ4Mlo6M?x4*v$rQUE&{IGll1HCm81I<4m*0pX4Apx4!aw z!*pJr*VxJ>)pdalIV02^_tPRV|JV+Rl>gfd*hKq;wS3ZtlDA^Tr*D?@aN2y0$h7_d z2is-#1_<dxi9(n_yfYM{~ zt(q83xrWhY2KEdR{A7aeDk0B7QVYgm4S13W$28?Q{x$`=OOG|rC#a>fvA`vVblbm6 zdkqRW$FBlVW1jm+)`bHaNMq*?_N?}Pld+o;8cZ+kJ6A8WWz(p!;~XN^5!M6!mHcfIGD#*P1lJmuBe^5Alg&1g(vZ24hgudm6 zjnqPl^>V9^k~SkR!&s^CLAW5FU!G|&BL|=0T6~2-#RnYZzss<;)qJS&^TctwX^Z;@ z946UgR1_AJz!{@Zdv!y&Xq854Uj^6s>6!BBFk-w_R?04~1&6@p;Y;4WJRVWcosN1m ziE~M{$J&F$zH8X5z;?5`ku2^Hhf)y&slD61_y^Umrrd)ZafFuejsXYGR97ON@8=4B zx1_8G7<~w4umc2oklk4O5>&!zENL@IaFgDGUSR9z+0%Q}^{XLCu^J1tO#{RaZFNMs4%;R(i`I}-S*UE=c zJ>YI%m?T=F&lfzX(Q_lH;N#>`Fm86rj@2$F3R{ACD-3AN_lSF5Jm(-p`*GR>BtErq zg;J#MZvYX{%)WB;W84M3c3tfzqp}nHcN6U4=45o&GqF&Gp~|H%;9j`SBTKShO!;W> zQI{Sk>7_5fQg|UA-*=PylI_J;**DbQ0A2BZsh-9w(@b`fIPYH?7omXhbv3c@6}M&? z$?>>!SKqb~B6mAyhf$IKqedbI841oa{-Ra#P*Y1G%s9~IiWvA^pS2KZ z74x5_p#-XJ=mJ4xRroSjrJ9#LWg1*WyOozhE+W5jTjOG*__g(>)y&{uqnz#Zx4AAh z_x!%@T5f3m_sp+5a9qQrQv53k@QFv6iBQN&hyI<;OU*0QvlMiBJDNcydv;Me{d>%n znfe-U)o!5lh%jpV&dD19-^vdKyuBWhil6#(Lr2bq_E*w3n+*GIMLmYCSOBzr+qPUZ z-<&tY#XGu-UsIQuXv>(q4fTmniDL9?>6z?$hWWAN8T0N*Hvo#KG-qRsUQW{Wl7+IZ z;@2U;z3SqQ6x1M#q28ZO(gfNTfwDXA=g>|}X^uXN-&Ko3+jlpKp2aWSLYZ~Ij%vY7q%8w~Mx)$imYlGTfqhD;GM9NI2I zAJ?sX__l;BI{1p-gLBsx+EE60oY*7X#I*+tI&w@zP(uB0u5tL4d>LHooNTVLduCF(ql zGs?b);u!|IIBdG)p^&cj_rW?ro`8-mbZE17A-D>vJ+x9B&dR3AvSJjtxQt&SR;=%- zzY^K64T=?Zv-Ia=|QV( zEUqMZgY+khnY;$4RGpewdx68XT}XA6i^?vEUG7ctk^Y9ReDj?sM6SC}4o?xh9q3gl zVa(Rb&S0*Qd(lbf$k!Db zeU|Oz(|M$fwcfPj8<|JpJg7vEhBLbM0q7hw*2KB3Pd^FAQ$eT1}%(J}{6rQEvB@c|qCtX{nCM zRMB=^gMQ25j@26~UD$1JPAVIk!iTr{!=cpQ?F;wn+m|cbz{l_NB%8uThLZ^p&BEZ_ zl&v4r{BC%&X?R`!eTT`?)jg!etR7sNout!tI;E6$KZ^RR|p-Tv4!i?e0W*1n>38>XRg}EEHj8n z@Y%S<>6q+NtWXQDC!(ZAIPsUkrs#Z+DpFA9qgs5?y*2B^4Y^(Z7a)os*Z4v8L`IC% z+J~T{jZWa!kU!fyS?fBuWm$O;FGIfSR8KkScaeTwYTNbqfSpG8{cqwCBhSHsMBe7h z^G^(s2NeBb_zsdUQLDrKjUFil9r;B{?b%_rlZ;hfYevDi8~G*6qRHZzf;m!hzPbo! zfCr8!x@Bn(uU*U^RSc`r`1qcJGDpvA^n5^_?Xn<50weU{E7mm6%(L;OiF%^6{Rf|k zDn*l@pHVyg;;S=V^pTF+V#D;FH~xTkevd5ViEJ27Nl5jQ#cDlSdZp8~-m?jh9Tt7v ziEUof)t@${S(OLYH^$?enwScS!L?NHX3rnJ0nnVOi=h6!ccpAbIx@1}mCXQ(?CAK5 z2uL+5xxzr|LZsI|Yhy#)v@O+GxTv~f9F~5X_8VYcU;3YxsBrnXq-sm^JLAEHctA{r zElw6|JNMwsTde9V5^{d0%`9H0z~^ikx|Pr}d_$1vYQZ#AiRv4 z4DNYNmd&*f>Ttxv@&)R1zVuJu&w_^&H1{*^r3xHYos5g$09CE__?aB5w%$W3NtQB_ z<8J_h4g#m0O#+bH^+evF^$tjUB-u=XbB$7`Xu^DN(ZI5>i-I!U5{KZlFcqifW1=CZT-@#Gh#enwpYjS8zCM@t zLp{nu8+FXS6|7|UGN$bDC2E~D?V3xQuVRSjZ3sQ+%LVxGlXSmkW{QrdK+ z5Ea1HyJ{OBPo*tCYm#S-{}la)x>Qx%pRT)X%1D`1j1;gK@XDjg*fJ==pvqKsddsYx zbMu)z+)QpT^pa9$Jwjhd{!5i(P|#XVwbm&3Q-ZU9)c|*{z>vq8VplP1bcO5HnNk>> zap?$8uOtep({ze zf-|~ID8)EM`3^0sC1$GE1g$HGPO)}B7UxN-s@W20`KW72_$5%e;}jkaDvuW))uWwD zowMY=L7MjUL`|mzkV-B_4$~hx)DQ{4Np8ZYzH*X;5BcYf^Dd$y(A2zt>EiV>H_Jw? zUa}^9>XCepqhsaLBsn{%vflmV_D+M2xWjg)s2++Pl{mD*rCD8OD7NlmZe)*(E$$b0 z9$lA((L=RG3W()J5_M_0E5<2WU0h=2gfnJd%#`;`hjB1~nE$Div{o%TWoPx4j ztZe=st18l6yCeb4%NnxG*zwE|@9vu5H(KkJj55y&$ z?DMOdnOd7F2wbcNsofIOg4qYS>Ux`<%2P%RV>1A*qXdbc{;mw;xFH-k+v0@~k3K_U zhdPo`#GAOk`1!gPo zukG!w&ejjE^u3xe(ky6uq6GS;@TX=JBm(sKxL~qDx3)BH;zBk9!jz~ z4`;K9M(hqbKK?>2#h~_3F!a~aok;BK!xL(;Ly3&d9^o6H^;fw3yrb*KpEv@0hwzUO z#Z61t#|_J0#b@SRffe$kKG_iOd@;gW?m z{6B5k(nY%YAfX}EEn5UGMfwW|-DzrQ2NAm<>Z>&|3DE z=^*FIHb*Fv4e_Y+)1~>Z#}8lLR92&&EG%dZxZO3wx-`?9{LC!Hx(YbCIu(m{#O<|C zpC6NSpk0Wjju*ZJ{2aLIvBxCK7VA9IYD%Ba{E%z2eU;-SQPaXimf&AAY)y60neHf~ zm}ss-+37nxE6Uy$qkz>oFm{~KV8p}Pn2%c*YNQLi_iUNU}fDLNPp_qF-*d13JjHE|$f>PHT3EdM_g zy9|_yXL=di?d30W=gEqB^&T_{#5iO7-u_tf-MqWgM$D7e4RUJJ&Q&HZ(Xq*gWHr)vzbr9U6{e_OQr3Lu)m}FAi^jV*)VZs=5TGP8&tyAB!CG1iDBX&c-l3E{_wn zE})egiFxQPN zbvGA=8-jNZ~=J3?q7tPq0t6#4?cdx}2f(k4`uO@UlB}3rh zaVR<^l)%09yR(AE@0KSqYvlQg~Xyc3n z-Uuo19K90Eot;xRYCISOXk-t=cDXlQzVs&VMWuSk|MW6O${Bi>3nQJV3;HZ96P?SW zW)D9{IPVoRkD@vzsfErq^Tk^}ejTZvPErrrq-j|?PJ;fz zOuy^!O{EJ{mT1oCc~&HKhfAlob#zFYCbe0%rvMC3JzZEw={I%=+FkjU1pB>XLtn7P zHsBsfu9H8doeULpjkMV&`}lm+PEA+wB(cNaa`nGA`hPe^fG zUv9V6)cLbCEU@OQI0Bb|B;w=U0Y*JhY#=NSPTSs(F0>d{@s4)!tsDEhex@EiQJcep z7YZMrBacL&r-8dwJld%|9v`w6mm3O>@n zm?N9hAhKBDnXOh5>sTb0l!3RDVZJm?VCM{^R_))5(`m?8JG#AJIeK^OK0EQ1cQRtU8o;mgxX>k$wh55QE%9hR@~8gRvc<(i-OMo#Q*Ad5-gfp=cmlF z({az~xBANSjGZbJ`9V4M3c)7@i?uMMBU8EcV%u>^+!sOtXZJ*C@#vG8?G|qK^kX$E zJlM}w;b%UT@#Zs#*Kb*@hqpec)IC+LyVYoLmROO`yb5+N;t}adp}+Zve%*8a4!zbg zz_vd&BgRR`E&S6NCaK3c4TP!o=S0sMJ!eB4XRMA(X^MWZlFjG4e_J&w-~*Y%DL-5> z(@E0r4cgTxo5yAcqwq!3N*GLIVc&iQen}YnU{CbWlzixl6@IK#cpDjvd&M!(_N(L+H2X^RI&-f@*z91qjb}>B>zL>b(iRo!1~8=NY1 z5MBi0>PpELa0b$%`z;UgTR)k@b{TU=3);>#i0X62e8`eX`4@4#Id0j{h&|!jk-(@s zKo-392shm*ZfQ_UM6zR+*^|KBoz{$G2}jY*b%+#Ud>l9IWOczNI4El!@qDHCd(|_@ZCdaqb}={0%>$3M zeE(dB(+`vBbjuUfR@&G>zd=<{J$?w#Fplkv6O4N0neb+ZbcZ|(*ERNcB&-qON_@x* z=;__MAt?4LvGk>#m+MLth75AW0mT?8;B!`788tXyPu0-(zhq9rz1~K{B(S;vz@5v<_zJKSqNqwoDWr}+SRjpAWP}~OGkpV+iyo-rzD}&6Cgz=seU$M+&ScGBUTwx?YIvne1Z18% zcdf>>2_orO_>mx4bIW*LJk%hHymk?$4YZS%Dw-6ID|Sv3M?aE>a`}07)(h)O)1R+y z>#WlRHw-~H4koNsbOsvFTYU1rDNB(qtId+tP80s_mLg8efNwl18(3w}@V=rwuow^9 zeO4%RNWYK81#cVoGe7Xz(G1~7QKkwEH#unLeCTjZX|vK=)_GuOsU1O2LsgU@aya&Q zX5P=ifBrdl%iJDjrWP_WGD!F}8%2rtesx;kN~5$-=zKi*BKPJ)kf~Su@1hp!&-U@9 zVqQpM<2*N5A%|%!6f0C^^l{0kg0Qk!i#%63y_@)2pd$VsUFOTHSb{ao2}0_vgAl@% zJ}(Oz1T98kvwBT{@pGS8RNE5%vb4tURrxdRH%v+~U=~FWBUON$r4P z&aF%rvTTR=Y^uR-V0lv7QXW6{wR0W zkkcZMMb1Y#RG+m6KHt+0hSh*jXR^cHS11#`i&RR6J3m>?daychXri-RG%(Vo(gqDH zq`JLaUuZk2CHXvXBdHbK9N=w&z1nawx)S90$2 ziBWd0Q)+SJ(ITQ?(uCu`HH23wE0dYs@}X56k~+#Zz|PLF<<-@?QN$&*8{ChM-0i|w z(&U$F87_{0qso)hxr_|R_HcXspTkNTUevTtu<5_r-FI2kb^x-CustEQlCyR4jx#`w z<6EAocma)V%Uz{EA6;N{GhB&kjsXOAJu5#R{lag`8|QvpNjugM4QU&)bhBSSm!n>2 zX?wXP-25?%-cR^$G@?e2uw3^-UmQbe*ihA&6Flx?R@ zL`nQXFJWtC5Z-d$={#-vwXdkfXYNaqb-bQx?Q9{eP}*350~<}n)(^FRSB>sAt`{c! zv;X)D7Yi+CPF-Pp8gs=$%PJ`VVeaH`jYIg4nNc}dgqc!v@?|81B`j&Lxkh;L$ocYh z(n*?939Nh7LkC{Mb8@5k7N9^?`v$mOoZeu71P_O}RLcHnan+|TkSnH@ZUA64 zIi=4)#~~k~%7b0xDU1{ve^#A7fnt^LeFfkklOD~{&(egQ)p{EPd0WFa_$;bAxTCPq zFMeT$wHGme7SkdJ1Q_r}2Ijik@s2K5s)t<{d2K*u$JD>V)DRx*g8*K})rb15n(i?e zG(@+Eru~8s0UCHYznv+Ar6KJ@AJXr-aQnV>{%Wxdq`;BxA$wGxzH_}_6l|RPB5iYK z8n8cV9n8;98DRJZ@Lpu2uNHFr)p+F8R_H0i8zj00m4j$L%T@TKfCeno8OfLz;o!t# zK&=oJ2i_t=0r+5`CA@hx__jKvgLoJ81_+&%xnoLLtACIN?$wY>y-4os)n6tK>(M~e zBlBhrrCnIfm6E=qP$hqn;unqJY{~h~bN07V0SCFiS~)$2Mt$y);hog>CslzE98`M# zs9A1Kh}zvZ+e$!^-|{6m(W6?x6R(M}RD@Too!a$>nWk^^62sJXr|e&GjfQ;~wdBn_ z2@=D4m8H<{_hEXw|IVV}z78Q^sX`+`qiDm?sa2Z-T5WRK99!KXahGnJV_?fx06!|g zKt&xNb;{<8`L#h6DpycSL%gtZ4#|OWwk@DHoh5C zaf(WbgT9OMR~B@Ktw+@2x6xWsXUwD zt>(=r`ou{K(XDIZ?fzZZ(oYsBa(xt_r?f)8zkj$w)2N8==ec4g*3NwJ;k)%-$xn5L zS5Na2qJEA+dHcNryQpCqy?tIS!OqH#x`K&vhSXZkMInC??!PayVkFk73l>*Cd5qM6 z*p`mCKQ%*wu=-@G{LYdI#JSn@adM5xIp<{^8IN~Q$9|l%75TrZap}=202ju|I)pIt zzG?4M67>PK!PLf0)MsPQGbPvoU!A5;HLmH%CK+Y}3KMg2pRAyF&V)8D%`#-0X9>)r zbkq(C>nD`^^9ujC713;1hQk7;N)ZtK9|89j298GyFA=)%~438T> zmdk<$MtZK&4nNuVIk|gp6n|+kGe8I?6_X@jDF!w@c~!ta)|NYx>*gnh@D1BUJGZr!>jY$06i*rr;W74 z^BLm{v?%8>pSo8)4??*=-Oq7avgpZkGHMaWBf)5V%?k^RER7PVljaD6aU-DGGx&Se zE16Pw5oBeE%E=Fx3*naujiZeH(7fllt2TOlm6fU`sFkhpy(N)G9ZAROgWIC}oXG?h z_l@>e4Yk>#3y{8B0@!eUeq|qpa?c8-C1+u!XJaM}eJyo+YmX^h+Q?k1s<3jdq%R<< zl1DiONX|z}`iu5!@t=ymChK$fa`xs#@ZF?PT*{WAV}=%vLLyI)5A}`^H%tU10bU#6 z?}dI4bksF}5%`j8D?2%&Ni?fi%g-b%@rj%;Bb8Szo}*=bk>QVyp9wx1_*Cis9JG%} zTc#83LLON|lpbZlIVHfCI3(bf00CcJmC*XU{i2?StA}ulyOKvm;;7@&Vw&SK7%`m5 z$qRyiHNpG}`qz(m=f)P^8_@L^xw%U%LNsEk-6TV~IQI{a!A)|%6TfLYzZnawO#(5e z+a&lGmpqjWuebnsBO`7|Jq3LEe%h<+(od>h8{;r>wVcZu{p_$kIXO7T@mH^o&T!PC z;ZCL5=v2nRN8NWa^_gybP2xAdxH33)7S9-5g-!}>&mYF8pYEL0dK_04?=*AVNZwMF z@`97K_uvNgB}|7t{{TwWjy==d#vF)3o!eM~!+1_J-*57#X;I7exLnR!NPl>%pY@wn z!3Q1x0Cr9~eBVm=&#p?Wz2m`koad>3Qd?;~c+NV4KM#6}(?yF> zzhq~O0~4Y;IO;)Qa0mCe{{Vq~E6^wSZ>3%%-dox$`D7%p?%k8PZfqW$X0)w*3vZ+= zNe2*H$b`g5AftT8C$9j3^%d1ZoM&dGriwleUtwH1pW9^c9z=bjxMd#K<;dG{o^FMljkt{5EDMl>TATl3j7k(J}CT0vhcOF zr1o)78c%ttuphaD%K}V-3hyDpC?k?aYxL7hxA4EiF9~Vh9n#*`#?H#$%D1;N<=KXC zx$0e<{KS*VYWj@29?J7*_b_Vfo?9MQ@l(aH=sHcr4?0CZmdsve3KwW6uW~@gzaE0V zQ23AWhg$JAhyMTw?X>oHKWVpsxwbK>-bIxhatH3l(1daUj#2oJTI9!O6(=VMm5IzbqwL^$ zfw5~_b-B7j9jBTiNsC~kC>dtQVx@WOpQ)~oPtjq6Sy=T5kZ)-t`%jsVEt||EBzG#u zAD>VN+0^vAYs9@vf|n3XyF>wYmItmo1DtYwK9!SkY$elyhSbd)$~?4`{HcGu-LZs#{>F z4HS$SGE{H}3gV)vC2X35n;)`8oJ1cU5 zoW|?qxR8=Z7(e*xG0%Fla@O{jcGIlNqBT#O1cMgtPJIS?8q2?tu2wnT6oKL=%||<2 zIAsPbbUDJ{^~0Wcu31IN-&ua451i<{>tt5+s<=Z#$M5EL)_=4@Z|RwDzV6|6|fQ&b~}kY#80SZ7(SWpRBW{C z-w$e~-xEaA80L_YQcbeq)9aNWdsk6!G?!L$n{ui4e=;_bLGv>3%15Xed}r6`P~YiN zTi#BQOsl7vgpAU4R^TxJ`{dJJ;3Z(i> zS2k+a*C>xE`B@W&!NayX0z*3S=~??$^hMm}t}K!CXv}j3nkHlB{GcpqBm^)V@~5}g ztwpIkwiZV2NYY5|GU+!Y9H~7ALJJNJX|IQ^FKiY+v>0W*jUr8{!WLbs#PuK(pZ>LJ zP2st8`QVgnG2Akj7;S}^1n5o%7%4sQ54|dKQ_@Anw(QI}r<(MJl{Q3Kw>HpKx8wMJ zhw`Mkx4gbv#aNzahBjX+6$;X_h2#!|oxYy9r)tpM-^(VOIQu>1cZ(dDjX+bgXvxL^ z!ze!LY4#dr{+q5FkFnfB@l2j`M!S#@aydNXEHX!?JJ8`b{5Ic^-JimSOQc^#6I)%z zJirWbv@!-N`P?7!;P%O?G27jF@GSm&C>spYwo0!98SBp#)WKspuuW3&qbl~1g-9{O z?gIxL&uyp?I6_f5JynH=YUqbH8|uQshW7iSgfK3->J zVo9e@dh=YzzB3L=Zpn)bL0pl}cCkNN)P*I}G<$ijZDffOHX>Yn?99CG0A+EO$I`n? zkB5F9&^$A(3y&JT*1K(d@k42E0T$A#mrz4>>Q8>aRweI?3|g6McwOG%W!#EBdAzoE z<#W{V2eCB$ot@JvP*j%Vv^^}?SV=2NN<1lxqy>bTR#wlUIsGd}ElkC}(ZZR8Wuo~v z2>}~+=cZ2X{+`u(Yvj`{<$}^T@+Dy)N5=t=-N5a^KaEuvW?57$IEF_n8%6;hXF#A1 zgz_?Y9c$C4T~u~jByvVc9;M*Fk5^v~beS~mBGfI#%#Y^W9jhxL*jEjVj9`#+!9K$r zF2C^uS=ZJZtxs5*-Zlq?8_1Ii`Oi>yDLaR$^{!ys$u6lAFec1JEKMY+G95v7h>^iz zjC<#fm7l0+a9vzkJ+8}3IH|26xOiRe8qO=0U?Ytjl~4vpd~i1}{{X003x8{O z9-sD@V^!P&$f`>q1-NA$a6Vta4^u{$sA{&_dAMIDTO}_AVwtc{}sa!G=07lwy-u*h7!H-Znr0u@z$}po_%N(j53wdl} zVmkr^Y^T2@}=*s zSm&D0$ca#StOFTi!fXVVUWWjCow=r3`L}ROZ7eXyEQ&WsSTf@~cW?$*<^vs$?@H0S z)+CNYEu_kd>JaSCTLhhsIuV>7Oo8oDCF{#*>29d7waFnqvTexp7$D$r^BT_4RK2nv1hU6Gcs>E^pSW{1iiWDH_5gYvi;Bo*m`Ks`EudeX<@OQf}HDJ{IqXh|6i2gtc6>Pb=0 zUz@FO8Nv@x(3#QVSrQ9bl2%u@Cze7;#uFX##Pi15=S|+De|-dZVWG}d(6}N}a(5~E zuo(8M@2$?dg@nyKkrg9+kUrIlSx^89cNlDaKovr75(}BFW135IakLL1U^on;C70Np z&7V!awKAM%q74lTA!il9wK6PnM*eyNftg5=Sasu)2>fa^(4o0=G%`mTmSEP!-Z2A* z%Z%q3JF)G*Jq0$$OOW`G~ZHvq6?!jk~6H73t3a zp4hW}Wq+jHrogk?&g{WLhC&k|Sp9a7`|C9dlBDj$Ieol#inpOJvpjq`cGb78zM$dsdrpGNTW^ zJ@#%SbjlHqYofF8$J(vjTHN`QYzopgSXl2w<;5~xKR z)zvv-Jr6=UWEzi4@dOgt&tmX>j!n$vqGBBg;Qj<|KZZJ1exlRf>c;3t{+1z@)@a8e zmjs~3dSru<&&^J_@=_5bb0pKV=120tU`rAh`i?qu`Ms-0O+hYA&Ub#MxQ-jk86cKd zwYic+ZHXuD!>GcNJwPM=`ZU>e@3h$)DKB9~yOHNNBr@#s&}>#C_(@3z)rX+QM7nHi zE!dLb*xgFE5g4(ws)olsxn%sRYfGn&7)y=Pn7psEU&z?UGLe?RKA`9Is4o&bi})nEQzA(!ym;nV_8yy$%eZg*r-R=$ z+wCdFE0ui9i%A!n{bjbgM7ewfTZWaJF3iZ|U_D3703dJ(C#V?B5e7)2M)H#B-Z_={ zjU_p3C#tE4kE>(7HSKOR`Qb5?e=q`~IP#8KR^Cva+ntBao_Avct<7`&o#aC$u?gpf zRhJC$?v;v_;DeWS$RnZltmh`x+qmwTB5BH+queBszuPvUjDXul;kW7(D<~(R`qdpm zE06SBY(^D_+1fdV8VPr{9Zy}Pu=;U|M7EaSRE%to?N0(cNV1R--(c9spvLc}MQfC8 zXLCvz6_d=3ie;i85~%s>+ZgM%x+1L`vX!?hN2|kcdWmxBI!x^!0O1J*gYn3~WaFn8 z#{#Vj`>8yMK3B+5Kow(%n9lE%`>QXv z9@LsJcg=P6St_Wr&h{@!TIvUan5>2C2;?0RQdvCG3fQV&N zBdYR2azh2~IT^s?n#!@biNse{*D^c=G^$AX&=m(5JY#v!@?b)&5tueO8-YKddIT4Z9izxwcIef}$H>?ozwHmx+|#Cl3#dy($RI`g zG3Oz{(= zq@Fh1%HJ~brG8B<-B>z>sm^fGUfMTx}onHEDL;H1b^mAK$56<|Q)59~3mN=@0sH9E-W zwHE**Zeoz)Nt>v?*JRFuBo=Dl~F_CAhTSX1zaNSPQOwmaUa{mCU zMKOg4$EX8uUb)AuQa6`bwtJgbl|+I?wz+6q0;F~oqc?|BRY@+b%Grt8 zB#1d8Lb=_vuS{TW$sKSgo4w1oa#qytV^@16Q7egMYi*!Nfmed0^ugqjgRnQHW1kOA zs7mt4mcsfu4Demb5VQQi{P_n207y7I43K%Lbe%&}hAE=GNMU&2a+ydwP;W^vlT^fb#w%I9sI;$$k~=$=c41GHOo!&FL{^Erzy!l1nhCZ`CN=P zdS|tDT8-qI9;DWbXKQN%n4o_tj@S#3I3)ERXI`W)Jq1*cP?a>9Q*whceW_+-+)FHE zWr*VeNk4@WbeiR!%dw7;(H!NP)vK z?dG|jWwv?c0RfbUQ^^Dzj9_-h7^(FLEwveKU};v>=Xp$f(5nUcfXA=SLi6Z-1}fcY o*ZNr4kM_8tlOhyVEH;htxm;j382W-b5-MF)JsZ%S8`Yox*#Wn(rT_o{ diff --git a/src/connected_layer.c b/src/connected_layer.c index 07fad695..16a39be9 100644 --- a/src/connected_layer.c +++ b/src/connected_layer.c @@ -7,16 +7,17 @@ #include #include -connected_layer *make_connected_layer(int inputs, int outputs, ACTIVATION activation) +connected_layer *make_connected_layer(int batch, int inputs, int outputs, ACTIVATION activation) { fprintf(stderr, "Connected Layer: %d inputs, %d outputs\n", inputs, outputs); int i; connected_layer *layer = calloc(1, sizeof(connected_layer)); layer->inputs = inputs; layer->outputs = outputs; + layer->batch=batch; - layer->output = calloc(outputs, sizeof(float*)); - layer->delta = calloc(outputs, sizeof(float*)); + layer->output = calloc(batch*outputs, sizeof(float*)); + layer->delta = calloc(batch*outputs, sizeof(float*)); layer->weight_updates = calloc(inputs*outputs, sizeof(float)); layer->weight_adapt = calloc(inputs*outputs, sizeof(float)); @@ -78,14 +79,14 @@ void forward_connected_layer(connected_layer layer, float *input) { int i; memcpy(layer.output, layer.biases, layer.outputs*sizeof(float)); - int m = 1; + int m = layer.batch; int k = layer.inputs; int n = layer.outputs; float *a = input; float *b = layer.weights; float *c = layer.output; gemm(0,0,m,n,k,1,a,k,b,n,1,c,n); - for(i = 0; i < layer.outputs; ++i){ + for(i = 0; i < layer.outputs*layer.batch; ++i){ layer.output[i] = activate(layer.output[i], layer.activation); } //for(i = 0; i < layer.outputs; ++i) if(i%(layer.outputs/10+1)==0) printf("%f, ", layer.output[i]); printf("\n"); @@ -94,12 +95,12 @@ void forward_connected_layer(connected_layer layer, float *input) void learn_connected_layer(connected_layer layer, float *input) { int i; - for(i = 0; i < layer.outputs; ++i){ + for(i = 0; i < layer.outputs*layer.batch; ++i){ layer.delta[i] *= gradient(layer.output[i], layer.activation); - layer.bias_updates[i] += layer.delta[i]; + layer.bias_updates[i%layer.batch] += layer.delta[i]/layer.batch; } int m = layer.inputs; - int k = 1; + int k = layer.batch; int n = layer.outputs; float *a = input; float *b = layer.delta; @@ -113,7 +114,7 @@ void backward_connected_layer(connected_layer layer, float *input, float *delta) int m = layer.inputs; int k = layer.outputs; - int n = 1; + int n = layer.batch; float *a = layer.weights; float *b = layer.delta; diff --git a/src/connected_layer.h b/src/connected_layer.h index 4b17c59b..83ae914f 100644 --- a/src/connected_layer.h +++ b/src/connected_layer.h @@ -4,6 +4,7 @@ #include "activations.h" typedef struct{ + int batch; int inputs; int outputs; float *weights; @@ -25,7 +26,7 @@ typedef struct{ } connected_layer; -connected_layer *make_connected_layer(int inputs, int outputs, ACTIVATION activation); +connected_layer *make_connected_layer(int batch, int inputs, int outputs, ACTIVATION activation); void forward_connected_layer(connected_layer layer, float *input); void backward_connected_layer(connected_layer layer, float *input, float *delta); diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c index 8d8efc11..f7c9c102 100644 --- a/src/convolutional_layer.c +++ b/src/convolutional_layer.c @@ -31,7 +31,7 @@ image get_convolutional_delta(convolutional_layer layer) return float_to_image(h,w,c,layer.delta); } -convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride, ACTIVATION activation) +convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, int n, int size, int stride, ACTIVATION activation) { int i; size = 2*(size/2)+1; //HA! And you thought you'd use an even sized filter... @@ -40,6 +40,7 @@ convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int si layer->w = w; layer->c = c; layer->n = n; + layer->batch = batch; layer->stride = stride; layer->size = size; @@ -56,12 +57,12 @@ convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int si //layer->biases[i] = rand_normal()*scale + scale; layer->biases[i] = 0; } - int out_h = (h-size)/stride + 1; - int out_w = (w-size)/stride + 1; + int out_h = convolutional_out_height(*layer); + int out_w = convolutional_out_width(*layer); - layer->col_image = calloc(out_h*out_w*size*size*c, sizeof(float)); - layer->output = calloc(out_h * out_w * n, sizeof(float)); - layer->delta = calloc(out_h * out_w * n, sizeof(float)); + layer->col_image = calloc(layer->batch*out_h*out_w*size*size*c, sizeof(float)); + layer->output = calloc(layer->batch*out_h * out_w * n, sizeof(float)); + layer->delta = calloc(layer->batch*out_h * out_w * n, sizeof(float)); layer->activation = activation; fprintf(stderr, "Convolutional Layer: %d x %d x %d image, %d filters -> %d x %d x %d image\n", h,w,c,n, out_h, out_w, n); @@ -70,21 +71,39 @@ convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int si return layer; } +void resize_convolutional_layer(convolutional_layer *layer, int h, int w, int c) +{ + layer->h = h; + layer->w = w; + layer->c = c; + int out_h = convolutional_out_height(*layer); + int out_w = convolutional_out_width(*layer); + + layer->col_image = realloc(layer->col_image, + layer->batch*out_h*out_w*layer->size*layer->size*layer->c*sizeof(float)); + layer->output = realloc(layer->output, + layer->batch*out_h * out_w * layer->n*sizeof(float)); + layer->delta = realloc(layer->delta, + layer->batch*out_h * out_w * layer->n*sizeof(float)); +} + void forward_convolutional_layer(const convolutional_layer layer, float *in) { int i; int m = layer.n; int k = layer.size*layer.size*layer.c; - int n = ((layer.h-layer.size)/layer.stride + 1)* - ((layer.w-layer.size)/layer.stride + 1); + int n = convolutional_out_height(layer)* + convolutional_out_width(layer)* + layer.batch; memset(layer.output, 0, m*n*sizeof(float)); float *a = layer.filters; float *b = layer.col_image; float *c = layer.output; - - im2col_cpu(in, layer.c, layer.h, layer.w, layer.size, layer.stride, b); + for(i = 0; i < layer.batch; ++i){ + im2col_cpu(in+i*(n/layer.batch), layer.c, layer.h, layer.w, layer.size, layer.stride, b+i*(n/layer.batch)); + } gemm(0,0,m,n,k,1,a,k,b,n,1,c,n); for(i = 0; i < m*n; ++i){ @@ -97,9 +116,10 @@ void forward_convolutional_layer(const convolutional_layer layer, float *in) void gradient_delta_convolutional_layer(convolutional_layer layer) { int i; - int size = convolutional_out_height(layer) - *convolutional_out_width(layer) - *layer.n; + int size = convolutional_out_height(layer)* + convolutional_out_width(layer)* + layer.n* + layer.batch; for(i = 0; i < size; ++i){ layer.delta[i] *= gradient(layer.output[i], layer.activation); } @@ -107,15 +127,17 @@ void gradient_delta_convolutional_layer(convolutional_layer layer) void learn_bias_convolutional_layer(convolutional_layer layer) { - int i,j; + int i,j,b; int size = convolutional_out_height(layer) *convolutional_out_width(layer); - for(i = 0; i < layer.n; ++i){ - float sum = 0; - for(j = 0; j < size; ++j){ - sum += layer.delta[j+i*size]; + for(b = 0; b < layer.batch; ++b){ + for(i = 0; i < layer.n; ++i){ + float sum = 0; + for(j = 0; j < size; ++j){ + sum += layer.delta[j+size*(i+b*layer.n)]; + } + layer.bias_updates[i] += sum/size; } - layer.bias_updates[i] += sum/size; } } @@ -125,8 +147,9 @@ void learn_convolutional_layer(convolutional_layer layer) learn_bias_convolutional_layer(layer); int m = layer.n; int n = layer.size*layer.size*layer.c; - int k = ((layer.h-layer.size)/layer.stride + 1)* - ((layer.w-layer.size)/layer.stride + 1); + int k = convolutional_out_height(layer)* + convolutional_out_width(layer)* + layer.batch; float *a = layer.delta; float *b = layer.col_image; @@ -137,10 +160,12 @@ void learn_convolutional_layer(convolutional_layer layer) void backward_convolutional_layer(convolutional_layer layer, float *delta) { + int i; int m = layer.size*layer.size*layer.c; int k = layer.n; - int n = ((layer.h-layer.size)/layer.stride + 1)* - ((layer.w-layer.size)/layer.stride + 1); + int n = convolutional_out_height(layer)* + convolutional_out_width(layer)* + layer.batch; float *a = layer.filters; float *b = layer.delta; @@ -150,8 +175,10 @@ void backward_convolutional_layer(convolutional_layer layer, float *delta) memset(c, 0, m*n*sizeof(float)); gemm(1,0,m,n,k,1,a,m,b,n,1,c,n); - memset(delta, 0, layer.h*layer.w*layer.c*sizeof(float)); - col2im_cpu(c, layer.c, layer.h, layer.w, layer.size, layer.stride, delta); + memset(delta, 0, layer.batch*layer.h*layer.w*layer.c*sizeof(float)); + for(i = 0; i < layer.batch; ++i){ + col2im_cpu(c+i*n/layer.batch, layer.c, layer.h, layer.w, layer.size, layer.stride, delta+i*n/layer.batch); + } } void update_convolutional_layer(convolutional_layer layer, float step, float momentum, float decay) @@ -225,7 +252,7 @@ void update_convolutional_layer(convolutional_layer layer, float step, float mom void test_convolutional_layer() { - convolutional_layer l = *make_convolutional_layer(4,4,1,1,3,1,LINEAR); + convolutional_layer l = *make_convolutional_layer(1,4,4,1,1,3,1,LINEAR); float input[] = {1,2,3,4, 5,6,7,8, 9,10,11,12, diff --git a/src/convolutional_layer.h b/src/convolutional_layer.h index 8ca69b1b..4e69dcfd 100644 --- a/src/convolutional_layer.h +++ b/src/convolutional_layer.h @@ -5,6 +5,7 @@ #include "activations.h" typedef struct { + int batch; int h,w,c; int n; int size; @@ -24,7 +25,8 @@ typedef struct { ACTIVATION activation; } convolutional_layer; -convolutional_layer *make_convolutional_layer(int h, int w, int c, int n, int size, int stride, ACTIVATION activation); +convolutional_layer *make_convolutional_layer(int batch, int h, int w, int c, int n, int size, int stride, ACTIVATION activation); +void resize_convolutional_layer(convolutional_layer *layer, int h, int w, int c); void forward_convolutional_layer(const convolutional_layer layer, float *in); void learn_convolutional_layer(convolutional_layer layer); void update_convolutional_layer(convolutional_layer layer, float step, float momentum, float decay); diff --git a/src/data.c b/src/data.c index f44f5daf..39ece116 100644 --- a/src/data.c +++ b/src/data.c @@ -119,6 +119,30 @@ data load_categorical_data_csv(char *filename, int target, int k) return d; } +data load_cifar10_data(char *filename) +{ + data d; + d.shallow = 0; + unsigned long i,j; + matrix X = make_matrix(10000, 3072); + matrix y = make_matrix(10000, 10); + d.X = X; + d.y = y; + + FILE *fp = fopen(filename, "rb"); + for(i = 0; i < 10000; ++i){ + unsigned char bytes[3073]; + fread(bytes, 1, 3073, fp); + int class = bytes[0]; + y.vals[i][class] = 1; + for(j = 0; j < X.cols; ++j){ + X.vals[i][j] = (double)bytes[j+1]; + } + } + fclose(fp); + return d; +} + void randomize_data(data d) { int i; diff --git a/src/data.h b/src/data.h index 4df0c687..dfbbf72f 100644 --- a/src/data.h +++ b/src/data.h @@ -17,6 +17,7 @@ data load_data_image_pathfile_part(char *filename, int part, int total, char **labels, int k, int h, int w); data load_data_image_pathfile_random(char *filename, int n, char **labels, int k, int h, int w); +data load_cifar10_data(char *filename); list *get_paths(char *filename); data load_categorical_data_csv(char *filename, int target, int k); void normalize_data_rows(data d); diff --git a/src/image.c b/src/image.c index 16679776..24e32922 100644 --- a/src/image.c +++ b/src/image.c @@ -136,7 +136,7 @@ void show_image(image p, char *name) } } free_image(copy); - if(disp->height < 500 || disp->width < 500){ + if(disp->height < 500 || disp->width < 500 || disp->height > 1000){ int w = 1500; int h = w*p.h/p.w; if(h > 1000){ diff --git a/src/maxpool_layer.c b/src/maxpool_layer.c index 8c409b94..413816a6 100644 --- a/src/maxpool_layer.c +++ b/src/maxpool_layer.c @@ -17,10 +17,12 @@ image get_maxpool_delta(maxpool_layer layer) return float_to_image(h,w,c,layer.delta); } -maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride) +maxpool_layer *make_maxpool_layer(int batch, int h, int w, int c, int stride) { + c = c*batch; fprintf(stderr, "Maxpool Layer: %d x %d x %d image, %d stride\n", h,w,c,stride); maxpool_layer *layer = calloc(1, sizeof(maxpool_layer)); + layer->batch = batch; layer->h = h; layer->w = w; layer->c = c; @@ -30,6 +32,15 @@ maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride) return layer; } +void resize_maxpool_layer(maxpool_layer *layer, int h, int w, int c) +{ + layer->h = h; + layer->w = w; + layer->c = c; + layer->output = realloc(layer->output, ((h-1)/layer->stride+1) * ((w-1)/layer->stride+1) * c * sizeof(float)); + layer->delta = realloc(layer->delta, ((h-1)/layer->stride+1) * ((w-1)/layer->stride+1) * c * sizeof(float)); +} + void forward_maxpool_layer(const maxpool_layer layer, float *in) { image input = float_to_image(layer.h, layer.w, layer.c, in); diff --git a/src/maxpool_layer.h b/src/maxpool_layer.h index 27d6f55a..92d41e66 100644 --- a/src/maxpool_layer.h +++ b/src/maxpool_layer.h @@ -4,6 +4,7 @@ #include "image.h" typedef struct { + int batch; int h,w,c; int stride; float *delta; @@ -11,7 +12,8 @@ typedef struct { } maxpool_layer; image get_maxpool_image(maxpool_layer layer); -maxpool_layer *make_maxpool_layer(int h, int w, int c, int stride); +maxpool_layer *make_maxpool_layer(int batch, int h, int w, int c, int stride); +void resize_maxpool_layer(maxpool_layer *layer, int h, int w, int c); void forward_maxpool_layer(const maxpool_layer layer, float *in); void backward_maxpool_layer(const maxpool_layer layer, float *in, float *delta); diff --git a/src/network.c b/src/network.c index b2fc9225..e2c44b05 100644 --- a/src/network.c +++ b/src/network.c @@ -10,10 +10,11 @@ #include "maxpool_layer.h" #include "softmax_layer.h" -network make_network(int n) +network make_network(int n, int batch) { network net; net.n = n; + net.batch = batch; net.layers = calloc(net.n, sizeof(void *)); net.types = calloc(net.n, sizeof(LAYER_TYPE)); net.outputs = 0; @@ -25,10 +26,11 @@ void print_convolutional_cfg(FILE *fp, convolutional_layer *l, int first) { int i; fprintf(fp, "[convolutional]\n"); - if(first) fprintf(fp, "height=%d\n" + if(first) fprintf(fp, "batch=%d\n" + "height=%d\n" "width=%d\n" "channels=%d\n", - l->h, l->w, l->c); + l->batch,l->h, l->w, l->c); fprintf(fp, "filters=%d\n" "size=%d\n" "stride=%d\n" @@ -44,7 +46,7 @@ void print_connected_cfg(FILE *fp, connected_layer *l, int first) { int i; fprintf(fp, "[connected]\n"); - if(first) fprintf(fp, "input=%d\n", l->inputs); + if(first) fprintf(fp, "batch=%d\ninput=%d\n", l->batch, l->inputs); fprintf(fp, "output=%d\n" "activation=%s\n", l->outputs, @@ -58,17 +60,18 @@ void print_connected_cfg(FILE *fp, connected_layer *l, int first) void print_maxpool_cfg(FILE *fp, maxpool_layer *l, int first) { fprintf(fp, "[maxpool]\n"); - if(first) fprintf(fp, "height=%d\n" + if(first) fprintf(fp, "batch=%d\n" + "height=%d\n" "width=%d\n" "channels=%d\n", - l->h, l->w, l->c); + l->batch,l->h, l->w, l->c); fprintf(fp, "stride=%d\n\n", l->stride); } void print_softmax_cfg(FILE *fp, softmax_layer *l, int first) { fprintf(fp, "[softmax]\n"); - if(first) fprintf(fp, "input=%d\n", l->inputs); + if(first) fprintf(fp, "batch=%d\ninput=%d\n", l->batch, l->inputs); fprintf(fp, "\n"); } @@ -191,11 +194,11 @@ float calculate_error_network(network net, float *truth) float *out = get_network_output(net); int i, k = get_network_output_size(net); for(i = 0; i < k; ++i){ - printf("%f, ", out[i]); + //printf("%f, ", out[i]); delta[i] = truth[i] - out[i]; sum += delta[i]*delta[i]; } - printf("\n"); + //printf("\n"); return sum; } @@ -258,19 +261,26 @@ float train_network_sgd(network net, data d, int n, float step, float momentum,f int i; float error = 0; int correct = 0; + int pos = 0; for(i = 0; i < n; ++i){ int index = rand()%d.X.rows; - error += train_network_datum(net, d.X.vals[index], d.y.vals[index], step, momentum, decay); + float err = train_network_datum(net, d.X.vals[index], d.y.vals[index], step, momentum, decay); float *y = d.y.vals[index]; int class = get_predicted_class_network(net); correct += (y[class]?1:0); + if(y[1]){ + error += err; + ++pos; + } + + //printf("%d %f %f\n", i,net.output[0], d.y.vals[index][0]); //if((i+1)%10 == 0){ // printf("%d: %f\n", (i+1), (float)correct/(i+1)); //} } - printf("Accuracy: %f\n",(float) correct/n); - return error/n; + //printf("Accuracy: %f\n",(float) correct/n); + return error/pos; } float train_network_batch(network net, data d, int n, float step, float momentum,float decay) { @@ -304,7 +314,7 @@ void train_network(network net, data d, float step, float momentum, float decay) } visualize_network(net); cvWaitKey(100); - printf("Accuracy: %f\n", (float)correct/d.X.rows); + fprintf(stderr, "Accuracy: %f\n", (float)correct/d.X.rows); } int get_network_output_size_layer(network net, int i) @@ -330,7 +340,8 @@ int get_network_output_size_layer(network net, int i) return 0; } -int reset_network_size(network net, int h, int w, int c) +/* +int resize_network(network net, int h, int w, int c) { int i; for (i = 0; i < net.n; ++i){ @@ -357,6 +368,34 @@ int reset_network_size(network net, int h, int w, int c) } return 0; } +*/ + +int resize_network(network net, int h, int w, int c) +{ + int i; + for (i = 0; i < net.n; ++i){ + if(net.types[i] == CONVOLUTIONAL){ + convolutional_layer *layer = (convolutional_layer *)net.layers[i]; + resize_convolutional_layer(layer, h, w, c); + image output = get_convolutional_image(*layer); + h = output.h; + w = output.w; + c = output.c; + } + else if(net.types[i] == MAXPOOL){ + maxpool_layer *layer = (maxpool_layer *)net.layers[i]; + resize_maxpool_layer(layer, h, w, c); + image output = get_maxpool_image(*layer); + h = output.h; + w = output.w; + c = output.c; + } + else{ + error("Cannot resize this type of layer"); + } + } + return 0; +} int get_network_output_size(network net) { diff --git a/src/network.h b/src/network.h index c75804d3..5acee61b 100644 --- a/src/network.h +++ b/src/network.h @@ -14,13 +14,14 @@ typedef enum { typedef struct { int n; + int batch; void **layers; LAYER_TYPE *types; int outputs; float *output; } network; -network make_network(int n); +network make_network(int n, int batch); void forward_network(network net, float *input); float backward_network(network net, float *input, float *truth); void update_network(network net, float step, float momentum, float decay); @@ -41,7 +42,7 @@ int get_predicted_class_network(network net); void print_network(network net); void visualize_network(network net); void save_network(network net, char *filename); -int reset_network_size(network net, int h, int w, int c); +int resize_network(network net, int h, int w, int c); #endif diff --git a/src/parser.c b/src/parser.c index cf35a94a..cf64b553 100644 --- a/src/parser.c +++ b/src/parser.c @@ -52,6 +52,7 @@ convolutional_layer *parse_convolutional(list *options, network net, int count) h = option_find_int(options, "height",1); w = option_find_int(options, "width",1); c = option_find_int(options, "channels",1); + net.batch = option_find_int(options, "batch",1); }else{ image m = get_network_image_layer(net, count-1); h = m.h; @@ -59,7 +60,7 @@ convolutional_layer *parse_convolutional(list *options, network net, int count) c = m.c; if(h == 0) error("Layer before convolutional layer must output image."); } - convolutional_layer *layer = make_convolutional_layer(h,w,c,n,size,stride, activation); + convolutional_layer *layer = make_convolutional_layer(net.batch,h,w,c,n,size,stride, activation); char *data = option_find_str(options, "data", 0); if(data){ char *curr = data; @@ -90,10 +91,11 @@ connected_layer *parse_connected(list *options, network net, int count) ACTIVATION activation = get_activation(activation_s); if(count == 0){ input = option_find_int(options, "input",1); + net.batch = option_find_int(options, "batch",1); }else{ input = get_network_output_size_layer(net, count-1); } - connected_layer *layer = make_connected_layer(input, output, activation); + connected_layer *layer = make_connected_layer(net.batch, input, output, activation); char *data = option_find_str(options, "data", 0); if(data){ char *curr = data; @@ -120,10 +122,11 @@ softmax_layer *parse_softmax(list *options, network net, int count) int input; if(count == 0){ input = option_find_int(options, "input",1); + net.batch = option_find_int(options, "batch",1); }else{ input = get_network_output_size_layer(net, count-1); } - softmax_layer *layer = make_softmax_layer(input); + softmax_layer *layer = make_softmax_layer(net.batch, input); option_unused(options); return layer; } @@ -136,6 +139,7 @@ maxpool_layer *parse_maxpool(list *options, network net, int count) h = option_find_int(options, "height",1); w = option_find_int(options, "width",1); c = option_find_int(options, "channels",1); + net.batch = option_find_int(options, "batch",1); }else{ image m = get_network_image_layer(net, count-1); h = m.h; @@ -143,7 +147,7 @@ maxpool_layer *parse_maxpool(list *options, network net, int count) c = m.c; if(h == 0) error("Layer before convolutional layer must output image."); } - maxpool_layer *layer = make_maxpool_layer(h,w,c,stride); + maxpool_layer *layer = make_maxpool_layer(net.batch,h,w,c,stride); option_unused(options); return layer; } @@ -151,7 +155,7 @@ maxpool_layer *parse_maxpool(list *options, network net, int count) network parse_network_cfg(char *filename) { list *sections = read_cfg(filename); - network net = make_network(sections->size); + network net = make_network(sections->size, 0); node *n = sections->front; int count = 0; @@ -162,18 +166,22 @@ network parse_network_cfg(char *filename) convolutional_layer *layer = parse_convolutional(options, net, count); net.types[count] = CONVOLUTIONAL; net.layers[count] = layer; + net.batch = layer->batch; }else if(is_connected(s)){ connected_layer *layer = parse_connected(options, net, count); net.types[count] = CONNECTED; net.layers[count] = layer; + net.batch = layer->batch; }else if(is_softmax(s)){ softmax_layer *layer = parse_softmax(options, net, count); net.types[count] = SOFTMAX; net.layers[count] = layer; + net.batch = layer->batch; }else if(is_maxpool(s)){ maxpool_layer *layer = parse_maxpool(options, net, count); net.types[count] = MAXPOOL; net.layers[count] = layer; + net.batch = layer->batch; }else{ fprintf(stderr, "Type not recognized: %s\n", s->type); } diff --git a/src/softmax_layer.c b/src/softmax_layer.c index b6b7ff35..12684238 100644 --- a/src/softmax_layer.c +++ b/src/softmax_layer.c @@ -3,13 +3,14 @@ #include #include -softmax_layer *make_softmax_layer(int inputs) +softmax_layer *make_softmax_layer(int batch, int inputs) { fprintf(stderr, "Softmax Layer: %d inputs\n", inputs); softmax_layer *layer = calloc(1, sizeof(softmax_layer)); + layer->batch = batch; layer->inputs = inputs; - layer->output = calloc(inputs, sizeof(float)); - layer->delta = calloc(inputs, sizeof(float)); + layer->output = calloc(inputs*batch, sizeof(float)); + layer->delta = calloc(inputs*batch, sizeof(float)); return layer; } @@ -28,28 +29,30 @@ void forward_softmax_layer(const softmax_layer layer, float *input) */ void forward_softmax_layer(const softmax_layer layer, float *input) { - int i; - float sum = 0; - float largest = 0; - for(i = 0; i < layer.inputs; ++i){ - if(input[i] > largest) largest = input[i]; - } - for(i = 0; i < layer.inputs; ++i){ - sum += exp(input[i]-largest); - //printf("%f, ", input[i]); - } - //printf("\n"); - if(sum) sum = largest+log(sum); - else sum = largest-100; - for(i = 0; i < layer.inputs; ++i){ - layer.output[i] = exp(input[i]-sum); + int i,b; + for(b = 0; b < layer.batch; ++b){ + float sum = 0; + float largest = 0; + for(i = 0; i < layer.inputs; ++i){ + if(input[i+b*layer.inputs] > largest) largest = input[i+b*layer.inputs]; + } + for(i = 0; i < layer.inputs; ++i){ + sum += exp(input[i+b*layer.inputs]-largest); + //printf("%f, ", input[i]); + } + //printf("\n"); + if(sum) sum = largest+log(sum); + else sum = largest-100; + for(i = 0; i < layer.inputs; ++i){ + layer.output[i+b*layer.inputs] = exp(input[i+b*layer.inputs]-sum); + } } } void backward_softmax_layer(const softmax_layer layer, float *input, float *delta) { int i; - for(i = 0; i < layer.inputs; ++i){ + for(i = 0; i < layer.inputs*layer.batch; ++i){ delta[i] = layer.delta[i]; } } diff --git a/src/softmax_layer.h b/src/softmax_layer.h index bfcd390f..414030c6 100644 --- a/src/softmax_layer.h +++ b/src/softmax_layer.h @@ -3,11 +3,12 @@ typedef struct { int inputs; + int batch; float *delta; float *output; } softmax_layer; -softmax_layer *make_softmax_layer(int inputs); +softmax_layer *make_softmax_layer(int batch, int inputs); void forward_softmax_layer(const softmax_layer layer, float *input); void backward_softmax_layer(const softmax_layer layer, float *input, float *delta); diff --git a/src/tests.c b/src/tests.c index 557f0fbf..91217d42 100644 --- a/src/tests.c +++ b/src/tests.c @@ -77,7 +77,7 @@ void verify_convolutional_layer() int size = 3; float eps = .00000001; image test = make_random_image(5,5, 1); - convolutional_layer layer = *make_convolutional_layer(test.h,test.w,test.c, n, size, stride, RELU); + convolutional_layer layer = *make_convolutional_layer(1,test.h,test.w,test.c, n, size, stride, RELU); image out = get_convolutional_image(layer); float **jacobian = calloc(test.h*test.w*test.c, sizeof(float)); @@ -200,7 +200,7 @@ void train_full() while(1){ i += 1000; data train = load_data_image_pathfile_random("images/assira/train.list", 1000, labels, 2, 256, 256); - image im = float_to_image(256, 256, 3,train.X.vals[0]); + //image im = float_to_image(256, 256, 3,train.X.vals[0]); //visualize_network(net); //cvWaitKey(100); //show_image(im, "input"); @@ -247,30 +247,75 @@ void test_full() fclose(fp); } +void test_cifar10() +{ + data test = load_cifar10_data("images/cifar10/test_batch.bin"); + scale_data_rows(test, 1./255); + network net = parse_network_cfg("cfg/cifar10.cfg"); + int count = 0; + float lr = .000005; + float momentum = .99; + float decay = 0.001; + decay = 0; + int batch = 10000; + while(++count <= 10000){ + char buff[256]; + sprintf(buff, "images/cifar10/data_batch_%d.bin", rand()%5+1); + data train = load_cifar10_data(buff); + scale_data_rows(train, 1./255); + train_network_sgd(net, train, batch, lr, momentum, decay); + //printf("%5f %5f\n",(double)count*batch/train.X.rows, loss); + + float test_acc = network_accuracy(net, test); + printf("%5f %5f\n",(double)count*batch/train.X.rows/5, 1-test_acc); + free_data(train); + } + +} + +void test_vince() +{ + network net = parse_network_cfg("cfg/vince.cfg"); + data train = load_categorical_data_csv("images/vince.txt", 144, 2); + normalize_data_rows(train); + + int count = 0; + float lr = .00005; + float momentum = .9; + float decay = 0.0001; + decay = 0; + int batch = 10000; + while(++count <= 10000){ + float loss = train_network_sgd(net, train, batch, lr, momentum, decay); + printf("%5f %5f\n",(double)count*batch/train.X.rows, loss); + } +} + void test_nist() { srand(444444); srand(888888); - network net = parse_network_cfg("nist.cfg"); + network net = parse_network_cfg("cfg/nist_basic.cfg"); data train = load_categorical_data_csv("mnist/mnist_train.csv", 0, 10); data test = load_categorical_data_csv("mnist/mnist_test.csv",0,10); normalize_data_rows(train); normalize_data_rows(test); //randomize_data(train); int count = 0; - float lr = .0005; + float lr = .00005; float momentum = .9; - float decay = 0.001; - clock_t start = clock(), end; - while(++count <= 100){ - //visualize_network(net); - float loss = train_network_sgd(net, train, 1000, lr, momentum, decay); - printf("%5d Training Loss: %lf, Params: %f %f %f, ",count*100, loss, lr, momentum, decay); - end = clock(); - printf("Time: %lf seconds\n", (float)(end-start)/CLOCKS_PER_SEC); - start=end; - //cvWaitKey(100); - //lr /= 2; + float decay = 0.0001; + decay = 0; + //clock_t start = clock(), end; + int batch = 10000; + while(++count <= 10000){ + float loss = train_network_sgd(net, train, batch, lr, momentum, decay); + printf("%5f %5f\n",(double)count*batch/train.X.rows, loss); + //printf("%5d Training Loss: %lf, Params: %f %f %f, ",count*1000, loss, lr, momentum, decay); + //end = clock(); + //printf("Time: %lf seconds\n", (float)(end-start)/CLOCKS_PER_SEC); + //start=end; + /* if(count%5 == 0){ float train_acc = network_accuracy(net, train); fprintf(stderr, "\nTRAIN: %f\n", train_acc); @@ -279,6 +324,7 @@ void test_nist() printf("%d, %f, %f\n", count, train_acc, test_acc); //lr *= .5; } + */ } } @@ -439,91 +485,35 @@ image features_output_size(network net, IplImage *src, int outh, int outw) { int h = voc_size(outh); int w = voc_size(outw); - printf("%d %d\n", h, w); + fprintf(stderr, "%d %d\n", h, w); IplImage *sized = cvCreateImage(cvSize(w,h), src->depth, src->nChannels); cvResize(src, sized, CV_INTER_LINEAR); image im = ipl_to_image(sized); - reset_network_size(net, im.h, im.w, im.c); + resize_network(net, im.h, im.w, im.c); forward_network(net, im.data); image out = get_network_image_layer(net, 6); - //printf("%d %d\n%d %d\n", outh, out.h, outw, out.w); free_image(im); cvReleaseImage(&sized); return copy_image(out); } -void features_VOC(int part, int total) +void features_VOC_image_size(char *image_path, int h, int w) { - int i,j, count = 0; + int j; network net = parse_network_cfg("cfg/voc_imagenet.cfg"); - char *path_file = "images/VOC2012/all_paths.txt"; - char *out_dir = "voc_features/"; - list *paths = get_paths(path_file); - node *n = paths->front; - int size = paths->size; - for(count = 0; count < part*size/total; ++count) n = n->next; - while(n && count++ < (part+1)*size/total){ - char *path = (char *)n->val; - char buff[1024]; - sprintf(buff, "%s%s.txt",out_dir, path); - printf("%s\n", path); - FILE *fp = fopen(buff, "w"); - if(fp == 0) file_error(buff); + fprintf(stderr, "%s\n", image_path); - IplImage* src = 0; - if( (src = cvLoadImage(path,-1)) == 0 ) - { - printf("Cannot load file image %s\n", path); - exit(0); - } - int w = src->width; - int h = src->height; - int sbin = 8; - int interval = 10; - double scale = pow(2., 1./interval); - int m = (wnext; + IplImage* src = 0; + if( (src = cvLoadImage(image_path,-1)) == 0 ) file_error(image_path); + image out = features_output_size(net, src, h, w); + for(j = 0; j < out.c*out.h*out.w; ++j){ + if(j != 0) printf(","); + printf("%g", out.data[j]); } + printf("\n"); + free_image(out); + cvReleaseImage(&src); } void features_VOC_image(char *image_file, char *image_dir, char *out_dir) @@ -531,9 +521,9 @@ void features_VOC_image(char *image_file, char *image_dir, char *out_dir) int i,j; network net = parse_network_cfg("cfg/voc_imagenet.cfg"); char image_path[1024]; - sprintf(image_path, "%s%s",image_dir, image_file); + sprintf(image_path, "%s/%s",image_dir, image_file); char out_path[1024]; - sprintf(out_path, "%s%s.txt",out_dir, image_file); + sprintf(out_path, "%s/%s.txt",out_dir, image_file); printf("%s\n", image_file); FILE *fp = fopen(out_path, "w"); if(fp == 0) file_error(out_path); @@ -543,10 +533,11 @@ void features_VOC_image(char *image_file, char *image_dir, char *out_dir) int w = src->width; int h = src->height; int sbin = 8; - int interval = 10; + int interval = 4; double scale = pow(2., 1./interval); int m = (w= interval"); image *ims = calloc(max_scale+interval, sizeof(image)); for(i = 0; i < interval; ++i){ @@ -642,10 +633,13 @@ int main(int argc, char *argv[]) //test_split(); //test_ensemble(); //test_nist(); + //test_cifar10(); + //test_vince(); //test_full(); //train_VOC(); - features_VOC_image(argv[1], argv[2], argv[3]); - printf("Success!\n"); + //features_VOC_image(argv[1], argv[2], argv[3]); + features_VOC_image_size(argv[1], atoi(argv[2]), atoi(argv[3])); + fprintf(stderr, "Success!\n"); //test_random_preprocess(); //test_random_classify(); //test_parser(); diff --git a/test.jpg b/test.jpg deleted file mode 100644 index f7b6cb8de4d405581a2ee6802c04c091b42a4395..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9756 zcmV+%Cga)v*#F=F5K2Z#MgRc;0RTtFhMaiIXgZ+J|!+hNkcqHH$6T(|G)qX2mty3 z`T+qE0RO}Q8~^|U0s{d70000000000000970tEpE2LK5F!~jDO00IF60|EsB1qB5L z1qA>A0RjU61Q7)iAu&M)6CzPzaTGFvk+BCOLZQJGBvN9s79}%7a?vIo(4z z`hO1}olbcdanJZp1y^bsu~1;QJmU=C&`_bcbhu?6rgc&Ls1wVEY@RdQy7WX?~%nr0B3hvJ)kW3RmnAd3?Wbwr|yD46&crT8H^GST#QvitdmTG%K$gUDLe)_ z3U>yKEC$eBz|XY^+r%3i*wE#RX*+oz6a!fmEMmQii&hO{yk>-#A$8IrI&%1Z4t$V#56nPjT44=}itSl|%C(VvYSbKfn&Y6MLBB>f+c1^x%Mad(C$3ek$}KZdB+q7WQeX*9(bi*+%@7zeM-kU_Nhv~ zS-~rwnf9v7lp%?%K*5_~=XVreu`Yc>uS#BMJfp$m+|V&u&Ic79y=xWfcH2$+L_$wF zKs|uwnN2F)PpO^1l^5)G$@gL(k2T{G?H~X@3aAT*EAvbeK{yq;>il?*P#r&iA^!mP zG>rRqDPxWg=~IhmlB9c3Efq|!pKt3!)S^a182%+9wW1#0QZt5Aip2h4nY_l_-AN}k zJB7`|ejlZHs+kB*odsQZa@H9xTkRio`gk}aX2RZgQ2TlbDTgFlxe#;(L>MF)a@X~>^2q(MrB=x7&8OA@V|b^?=<-+4G*MN&$F}UZSl6gUxt=J7%KFJJlH&xjzP-NYJ8^dB-)iYVV&BY6EXeiGS8= zzUbu>BLte9KnRqPk+z3ngEQms{OVWn;nU~-^+Bmy?7ESCjVpN@Ez9t6$LUzO$hmy` z(JfPSqvEeL9p>SbDQ-#Nbf+~NMf2K0F@eu5hT!o*w6&7qqHqa~7X`h|TA~#Jat_R6 z=laxk0_B-MPZX%}xnSGpvi)*2J;1F5%yr zkU*m>agK6LL0x3$_)kOcRI-d+$@MhO>f3cRXYn?q2WyEx)tW;6Knau1Xpksa6VeVx zy$;0}&V2X(0KHEA7)=*B1pfe9Q(M!UBI!DGVO4i2o2fYO+LXHk&(1NPYtS$a$j=pL zr1`#XQLq}FytsKA$!q|5IP56)moCa6i0}?QJ%w0p+Sn&K90A^kpS(hxb|W+IKb2ByN`^Muj3a`KIuL0I zP8**5;8(7*B1TB$0<{}t$myDP&K%5gI2_aI5Qsg*rwFlit5un81ol zs90NRI*gdJfb_`Zd;6Xb8LAy!t?i+f-ZC@1ZRaNgrBcf-ew7rMx!de>in6hYv2X@( zaaL(;v3~FP-><-`op(?Ue#r^+kG+odOIe~K*O*ZI)tq-bbY+isCCdI3lO8jUD$h%t z$&kmV6br@0`}~~$04k6xj!!kYYKxx}>OFzEll@elqKo!!I^+;A5ierm5#)r^g@+aUJA=BVZjWXrbaCz=G!xsK7muUZY$fsVC# zr?7|5mmM1e>r3A20U~%U$!25fdZZ+il5$OYixOny(e`FN+h|TXp^?oR6*57y+4EDG zHbw(*eWw=2R#I4? zv5DQl^mOh|#a5Py+kDa(({4%*t&tC$opdP(B9Vr=@34#YBXwGRX`Q>NEeStZk85JYK1C!U9QcQ!* zK~#v#;EeP%&W?g@S_C9uw=~YLy4Euq90!%L?~10cRHIGpigq}?EuIMeb>{TvBaXBt z9Ow0*H{PkhaG6rVb}mEUOlU!>X{xf)Km}o@6A{0 zw-Kyj5O+vOBq{s6_oWfyh)1-tb-nWb}#ocqv-!OJ()&}6}3 z&S;a6u_^++KN=mK%h}zl-!G z`tyyWoYgd}e6tmlLP*08HfH`VYST@ywu0SQ02#(f?r2(e)xNnI<2+#e(J#|=^}8=S zPJf+d{Y;9|;ElZjpi16Wf@NR^25V9UWSkN?egQSLY1SJf3@~y#4mjv&I=QxjCWq|4 zRQcSJsK)9Ao~wS)$pXBS$0F@0eL!=Z^V9I4?DOPAOs?~vrYM?}f(uz94q5Yp2G#cF6~0=j+kde-@^TOCWq zT7t+B5HQI6X^S}Hdv(Tf+LO~_15vu~&rDWz1aIvNZbwmp zkD6lkH%lkT91c(8R9e);X;!x5_|!9VTJ*{uHOCm~{{VWu@aCZJn@O&=Km4%#hxpHG zy}h@R;sI%M7*;(8Omoy%t#m}wUK^`>*Nwlyv>g2@ZB3$zN0>USj&fZi#($-2SB|na z0VgN#H+l#!8t=y5Fa-gwr|mA#J*X1O&-b01<&9zp_sR6E+zc@kxa(PQijQiXNMw~0 zyAUg3eKPLOAvfPc)DG1jFU?KEk@{79v6Und$m>bT)~97|*H=pt^);|X;nh?7$M3~j z$72i#6b^c3yhbuJir2N}+jfjQr}d%7U|J zt`6_Ui)eqlMn*XYBiQDX(^-b2cZD3DarL8J0Qb=D9D+LiX`N6H?D={Q1!}snGCV4$ z0V4w#>r{6@n~j6{Q*c2Xg;1TH%MPpbt80B;`qTdaFoN<-J}cKgB$w?1OHDcA zPU^&;!{U^^)y?=DZDbAYgY8kbvC6cv;K&9-_5?@Y?5EkKsxFRlBlo;0gqm*i({8 z&%P;R6GliJrg*ANdOzPo@}z${kDas8pS#6wTK-3j?xP0<<|jXitm9xz5I8fM?^E3kMWzd9w}KecVR;g1!aRzS8=4w>M7b*5b087|8)9EVog@pFMR8@YW zfzCqq{HlPAOOKi?gVUM-6|#6X)6LXcQZuk|TlbKRJvj&KRQ@H08hCA_ZE?@mwDTIM zFUD%BVoQ@xczBVn+U@Sde=6VJA{kBytVma$x%yIX8mo2dRGRF0n%mfuRV_M}J$lzf z@f2Wq_CdhQx9VtmvbyQ>EFY z@F+H!mPC&nW~`!R5jt)dBLkdJE{FG5mpL1HR=PG zOCG|u3BPpwekT;Jqa2r7Y_JdxwK2EW0@qr_vl2#%bRxa)Yw_(Bi zz#b^j<<-etmqIc-ccDW9+(^Pv$Q&QkRYs`hIr#5_Gur~IZZ00>hEGl@n=jq4Zk%yV zgS88@anynBRrl9Lkj4nd2P3^!p0zjW^%TyPV)xh4#HvUqoOTt|bmaRzx<78z3W=8MxMbuSaOt{8J?$0^v z#bZT*MwM(;$Qyh8YL08yx4yk-o-j9@$o&)#Z=Z@r8v}o;TgPz|BvS&wxgn&-bL~*^ z#K9KBPDNEn!#^K93e}KeL5?ZZLW{ieDeHOHV=E}bVE#F)I3*avdXFG`Qd40k>T;(T z_NT0-+X>0WMmqe^9_?3q9PZpjYnPL~#lw(rDlo>|ho|HUutX^cE;^mxl@4kBJ3-)# znyiy0vrsiK3+))K`@&i(+S>i5U-*&_yN-gkK@QQo0R9~+3hWyV!1t**!u{dzS`Ud; zQ(K!I)k*r|wfVVv-~2733mqvXmx(9!5$rovt40fS1M#59rZK~RI$A-!^N&#WrElPp zYbjE0Bio+5Fh$XuSM+|`x(f8tvNj{^}kY92d$ z_={k{*%FM_yc^a#Rh%VWFD##$E_}2g43pZMv7Mb%lftPOs9d-+@}xF$e!kV^c8!1l zjEsIYD}W%kW<0Ymr@tq^){?Tbh^w=XgS|Hd5ugP}U^07r&~CtTdjY}E>s4|oQH=9I zVS&v+k7~Z~wv+v*dAa`p;l;`Mu8$B9AVgA&j+A`vUMNg~o1rxah)OnlY0Zmq6P?9{8p7TWGGe+el>qqshqkuA0$-fW0w+UNDwkf`=q~LELyA!j_Y1`;F48 z^*JQ_1mhpUjMlb`9>vOx;8yG6*%5qHCUdnTnVE}r8LC}ZQG1*PiNr;794b9>f0@0O zt$aVE!h$R57|kn!z`r2|dI3!7D#u*Z#xaIoGhVs$_^QZpkqg#`<0tbxQFH(RF4g{3 zQ4~!f;C(8}+AXP^QLDQZn zL{|IzKeiubl2X5Uy8Kq&E!3&t@j{ds`9eiTH+Ed^8TwbQMBinXX5HHx8280;bOhy3 z-soyDG?D617>N=ri?<&6uCW}LQ|75&dy2`HGz5oVerahZX;iA?A6iwmgU&PXX-!5w z_A_HYaMr4==^sklyh(8_--+&RL5J-gdwDUAK<626eD%dyCX*JKrd>O~opU5@=1kp5 z$7AjBMDXuAJ-ac_82mFq(eeGis)qjn8h({4lmo^#Kw#$G&o$3{T!nm1b@E5)N!iB7 z83P|RWf&X2?lJ!W&XpsFkS8RQ-l20D5|mO7PC(5bN0QQEDr81_0q6Qt4UpNyzx5fZE)gcBsrjBNPK&N5Qt= zXV7COIg^^$OKjt|DQkjRa0LleAOxIZrn5$RGMskJa;ekHxtj-%F@}E{&WRVD;wvTh zm>RY_s+XH%f?r5T#iQN zoff_!ZX z#AhIKXbT0=nOMNyzfj8c`c=N237RI3;Yjk(l>~Lb=|Q)Qtc}SD$ldXt)Z(K&Y=|+n zN7AWY02>TCccL_5FhK<4jPZj@Pj2q4S$$njIL%eM6(>JRUC7JLwLr}WB&lQf7Yx2 z0B9Cr^d*>ZW0SwNbE{p<+i_Ml@|$^)gqUSLny0C1*HX%oEy*mM2*yQh9v<<^k1Eev zw@G5i2`Eli>sw~LY|fr8Z8j?cSh*dsPuln|Me!Dm4b7gU(Mh)_V=DlDM{28!!B@67 zO&^*jjZZ8_0Vn6)wat4)yRf}xXwQ;)+@m9&^n(H|!5>aB$o`a*ZVUq+2KtOlz+KwO!fCKoG*0FDG$qYZ9 zXpbCm1~}*mIj&5n%x8QLQ?VbNJK>*CimdWKi2jDUmzC(`k)Fb#<*!rLfgl@!1pO&? zU|VX`09~bBPEBby7TCFzW22wex1SnC8fy0CM_cT%_opr){{VK}Hr(z;KS6^^cyTk~ z+doSWN#q_n()v_EyhW@jW*8}t%DsBuX7P2q%OF)tIT@M2z!|Mi#r`QxQr1f=bNhsG zgNzqG{{TwXEeXceCnwsaEK)^+NF};p)XJnNHx2IH25T1Db@rol*6|@%UT}Elu0FMY zCH&gPfi0!X%?;g%^&mfda0W$F>0Thzw0U<}+`LEFF#y$+-YoGH(;I`T$fq3R(mx?Y z(fn7aY5Ix1dxn^>A<{#YaqK~+@3nh7xf)nPL%d`l7y~^;Rr6l4>^5Il{Dxr2BJSg$LQfsOUJN$Pz^a0PMVuE5RMH*9WsxP}`Kr>%~@h zitt(Jw^HN-<&|Hl;<==1sTe?Vam^COQD@={DM=s+S0sF#e_HCw`xpG9j%a>sNuCDW z<063W454syj+GL-GQo094Uw0gwM9LPHN=gMu1{Z@**tWg-)+e}o8e4f%F)@)7(epa zf03lTGV+}U+BIRc0y+B7bot@3*EPBB zBYEoT4o~Ak@&21*r)m0hc93~wGblaC$A5F4^{86{nLUj)B#D4YIUe-1CRs0{Sg_I; z0W;G*vt1lp+jv&uz)IQj?PZcyA9_b(0sMP?D_OtOueCi+;z;2!-%0D4SA~uu1_Alx zRT|h{`OOq|%<=<+fu3o|ygRGR!90Sbp7hK3H&|5#V-2`x9@M1zE~#djp3T_@QU+;M zl31H4ODW?AwMo)h7lKgx9+aG_=-6?{scGxF{@F)UD*mJvF6%(5{&@XyH~lmR^It2mb)d=F4>?XM_3Cqq}KjQr%aPitMmA+AQKE zmEM86`+-vXJED{i$6AW2p*$S_0E)cXBZ`UI8?WI@lmwHGw0|BP-$Bm_dcV$|x`WSx zZNQKbj~|UzXxEmO9u;zf=m4aw`&;18PBZ0ygBwcDB0v=%$VNV~mEIuFLS$gE(t z@dcf{5TZnuj1^a~%}{>Sp~aq_;9xAS58^9CgmmwkoT_pMy%$DBU4G2I09ipK^Z-`f zeW|tXuMOmu=2+DkEFvTRE97#1GtF#TRh!tsC!72GiI15)xa6_^gW8+5w}J*@VcJ=; zeLZRDyMD)5V?{xUgw;8&zMkj$a3-KGm&5 zHstP20&-mQfsbl7gBP3P8wmdZ#XA1A+^`i!K*xS4EZNdWogslRFECGhktqa`i}KY^}qZb1X-Me4wwlwB!pq}F0Lk{Iof=0ZA=>^@C& zy&lugm}+;ih-BI2+Q3Ex5o3Y%J?hx0A8@V@#SsvKpn!4TrAiZt90kI$!RPd%o*1Yk z4{TJx-n<&4@r+V8gmnSfAU!@sayxPABC#i@Jq~Hl3PRI(qQof9**71cuEle1Wq8=T zua-*yr-Q|MAV@hmAd^xo!~>o?iil->r>W0MLtoMxTGF1)W%A@C72|{P@m#k50K+%7 zcZ~x1t6i8XS&;@Ts(0#xak=l#yOALv3Jt zV>qRxNZo$woKkYy#v2M9N$#f=ApytyX@5Fr_Kmg+N19BWU84c+0VLwJn_FAY8(ub{ zd6+cFvLcKhz3@M&{3`pxzF6?xxWK}f0DF3kPeqMe#TvUF6J%KYsFx~(#Z*_0PyU-gR03w>@-@t?{X`0Fml^l1lfd2|7xzznakZd@@T%Pz$9#NOMhLASVi%sc ztccTj+CUUQ%YZrKpRIZ*a-s2&)KI5aMkfS;-!&MdXZHu*k?|CbCWm_o84U|pACH0$ zxTsF#^sR^Xa){m|X%2_Kgy@~!F!A&vnYAyP270R{zC_&~@0 z&c)~_5=b+T^BT^a3wX0v0AmrtK>Q6Ja8Hcwa0Y(GC;FM1J@`rF#6S-t0CQkgWDY{d!vu<9vj$D{PjIE&PGV5qEeVf z=l~VxRghzf14j~l(lS12E96NmZcx5=M8l`Waj2IAIL^`SOWYp9w*LUxwTIdzy`7k< z0IQMQ=QYt%8DZu>QjW%sCsurNgm!8pDPfj8epIxv0)KXLd;Czm=>QwYBif}Kle>-! zcBuq0{n4D%yvA+-RnNUgYZH&p9qM8|%YrdW$rnD5c%>u-RFQx?)&|^t@kjCWQKp-9 zwNwFcjnn+Y{{WwjIwAepXrE!)a-+W$QQ+s0Xes-)+viEhUOw@oX}FWc+P+RVLjnD3 z+NI^To8sGhtG5djb`s?0solUH=Cv;v>US3wo@L=NdEm9Zv0N077~J0^){&>7e1Y4N zb562vbwSSJ1$Qeub+@#JD9@%i)qf#Sak=*>+t}dLN<}J^a7WE7np|r-IF*m9g(sz0 zh|2OCg4GE^xa1uB3Rp+%xW~VGBU^8TgXs(i z4v9kN?GDV}hXeGj!Zudtkly*Jv>V3khWgjGJY$Stb|R!B7|%@gr0yVn-i&QMIH}o! zvE}~u+Vl?+z)#sfGI9+~Wt9h{D|_Oz6sshp5r-or8Z*V7{8URS{pP~` zD~8HLPjC+-%xr#jPbmHA`L4&|IDD@RSRw1>Zv4$$>;siAlDB{_tMR zN6%b#s=wO1GI(J?ZYH@^=c&LwYm-|s0~Kj!#;0t{%L!8-T#D`l6_~zPuPcs})MF}3 zbnRInV`u{h-m9*VJ?79i0gfo?1ZYTTfX{CAmbHyZWn!oNrlJyXJu`J2cdG4XS*^5- z%a)K63`~CRI}kbitBdX;K2Il_N=$i%yN-pd;e2R2BXzfsBt=aoUv3fE-3!lbj!#!!*QkZrBx+mOK%LMpKVi>-3@_c2Wr( z`&Kpq#_m2SG9w;(;)S($ecF(Zj-r_H(=pI>P|1K3sQQXx8;D}lqq&+-E+mOriT2xC zUWsk?>v?YVDV9AN(&9-(WaG_ZbC&n*ihoSf*I(DA)$XmNj>~Ercw2>XG67-w4xEo_ z+`@??2%~5RKU(G=we3@N#g2}hV+MIg$>4s5x%3eoyN~5t4}>g#v^-B~2^k8;LNnP$ zYqiA)*v!EC{my7|UE9L@7>(Y;B-9%2E$S7s?~$6s1QX^J193Ps!pDMT=NRbJyhdqv z<}?a3ik{G*;DSeSTKA3JS%`}jDbOGB_k*Z`VuQE0YS7gu0PG1kd9CY^LKm@C~(I9M&MH!J*q}0 zW#tDwaQ^`J6ge#eC+{L`j)V-6Kv*SHo&5Z%6%%x^%jHRs*v@gqPs~?7lBeisP8es7 zzZDNHy^S;Dv;N}Mzx-$VR=eW1Z|pk=_Z^QUVV68G+BhSY!#ahB?Slqi4VC;db9ucILY?9u-K&_7DUfw1w~q2?p zUk71ECym(^Ktah8*NBqf{Irtt#G+J&^73-M%)IR4WeJci>kgcJb1w=%hbZh z=+w@z(7EA)qQeUTg%S<{D;AECDZJrFN&2^zryu`4`ug$3)q8Iw**`se)%x-Eo1||& z&gGS_58jw*`$!_XW}jj8p-&aHpTGUsw%x4y*uk2)whuC5ZpX<_KX&}4ao^g>UHJ#^bZK fg+@Z+`@`rrOC_$#`=cT-+cS8&`njxgN@xNAn@w@% diff --git a/test_dog.jpg b/test_dog.jpg deleted file mode 100644 index aa98311a8cc997d3e130afbd3bb776cff6489092..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 30762 zcmbTdXH*k!`1hF*dItf6gc=Y+lM*_?08&CH^rk?N5_)gqk0Pj%-ie5G2u*qsP(Zp; z3`M#~0!UT5SeO6v?AbkMU+wNaXXee6bAQgvHCOpw_rHaI>i|{*U6d{W2m}CtR~z8p zGC&(ZPeVgXLrqUhOUpn{&%g|3WoBYx=H=vK14H>mgoXHp1aC?x-MR^b!vuw7G-cr` zY8o0EBGPwI+UojB2o1IWc?gi6fq|Kc8N$j6Q4t=L0{E}yNd^301Mq(iAO(n$ikgO&j-KJ_f;LtF1rP+HpafA-QBq!A9dmUaK*>hM zE-0%>ea+m3MktU&E-s^l7OK_M&x!u{U0B{VD4vd<>pGbGhR97(F_<`9K~YIrMOFKb zj;@|Q%D}?X%G$=(&K~1--`&I03mY5~8WtWA8I|xTG3oJ>r?||l?3~=Z`~rMwS$Rce zRW+fexuvzOy`!`1^}xHqq2ZCyv2o(e?A-jPg~g>W8zl1P*7nz(-J|1^)3fu7AD2J> zhYJV*{co(R<(;ML0mu>o!aX05^f zqAEKk3rcQ92zgzMW5#s5+~BGiU?TTj}yx z4}z|9BaYBHm-@`IpE9h9R|}iQM1Q21!Z68lSv#q%eNRR)OC^>`r<~#sFH|lKE{3tM zbsgFJN@%L;jixq7T7e0(*CybC);h_o=|;~>`#f5PpB{hL{~B!-Q7*PzzS=PzUK`xk`JSRUY!Bt{ z!zbtHVxpp;$nB&wx_IX&bDI}sqRkKFuG)>p++|HM8QCF(g}-^0WjdQAUB{MV!csvn z!0OgTESPUj@;KWxL+M65AId{_j8r&1-C&(c6C@|o8;p)*m)NA?n+s&Xc#AFXn9|pE zCa1z|>2Ed5@zkaoGicmrwh1SpopiMseYel5i3qX}FeX?Z^EbL6R+1VaZhM{i!#x0m zk*_t4wjHf63ay!uB1uoQ2s1~X6*HJ#B(N(*p!Hc8p4%9$)SQ zYuIcxx%L4LKhbYqq;uO+?VVVZ8 z%J+iM*!~aDgS`75-D;lRyNgQgbV z*a~O9=RCp~oN4xpqxGY;Okpj#2#wjAI69~>Uwl3-kLVV6F)&kWCq{qE;sm4pkg|&*6`@ zvcfqECIEt!MP4IW#-ey?H85mGe?-%;5a;L~M_UGW-mVv;im6d=f&)$pk4(UaFwTgv zlGG)D-d1W4Gt+j< zDf=##4`q~`kQ=PDX6u=YyuDcV_yD8l*|&`55`>_B{9&az9|rgR^WOH_K1#jA&Y^!y zgBC||k|mmyq@%VgG{9xD>lAM)>q<+RsS#f(UOP0WPhRrn2pvZV(?3XQy*Bm}4YM{I zNj|Tnd>W5c@@+tFy=-eb2AEcQDaM=VG6A$>Cr+3VVxh`F0Pk^>5-(RvUNDfrcuZ}hDQ>!XBG7O)ck4ooMuWY!r{eqi=3ab`YYk|FD1P{5#W!#`c?XN&?Ux? zX>fp4wmgtJiCB@*5^E#CZ-CLQ{Wav;d0L7X($1YUrHB9b5tQNGz9tZ}2;Yw7G<@0}HmRXAUt!3VxpC64iBpZ@p!Wo~}LwkRVt z6V*TwU?XMOF`ka^RAU%%+W-T;>F#q=77cyF{AI1o(#m(0O7!^+jXCw7=+0-LqE_k1 zEy7p~#DgbE?=JSmaAUZu>=c2JP2ycNV9y<0kf!0Tmi8}hGd?k>Es8B_OH}wXQ2C&* z{6hWZK+8Ph7~UZVPx-BuW-xBG(XNm}hnUDaqDwz$eKhuWCO}*;xOzp|Bm2(VRdY+1 zpmIqJB#iSu_uTOZltMy1jHgC6^(THl*sxQg{oBK>)?wsWP1FNnfk=&`uL$uk;D(y; z&2AN}7{Qb2B)AamyHyy7*X*x18j*;aiM9S_UdT2i3WX4Xo5)H$DC}m!SuuvYiCFkn zX3r?FDEp=kz%cHsXZea}B{Ypk{|gE}&Q0c1~kCD=z7+BZzW z`(7UpZ%44wZS#6%$g8D)P&xEE>|b_eS96brkXxw6l*z#sBkHMK0>vJk)%LuJ=3V~w zA?E(+=~_9JkL&_m?@5{bjjSra<1!Zu`bHrxC?^pUBR*&%g4R@~DdBV#)#Wh&rfF*i zaw%obihhVRfjP__rS_Roa3jgbRlTx@Y!`rhapju{U8AFd(Kxdv0ar7s5fwO)vzG)( zuZwvyWc#_38sgDZt}paJW|GC=`THeVoF}berz`!@p0QmYO*r_QNrp`TQz6@|m80(S z*s$C7Ml@}slKsq@r!~F;DZ{LV;g|`NXN|H;We8@kF@I3Cf6r%oK$QL~a5@D|;GT)D zUSMbXo9u-$BIbNJ<%6V<_ z@uPkWy7Wa=$}-B- zroCKowhX>2$qL(z9Qk5vlh>?zz%BVsc-ElwX`NRW*EfZP8@W<{cmDzGr_yXQzu-oG zG}WEAXIk`<$J96Kbl8_0uIH*d+rVqXRH3@dnT8>ogI-G)1pa$`RGsX&x)_w1`eZ~k=EEe9^3c7<`t-nzuB&_eNZdUAKP{ihWN6%t z8LdZTLYLko+6n4g(9J*h07kx(5yi*6TPG>kwzECxKALaY?5TUU)#^7vyG&0daqT76 zMNK!_B95`^m8OJe!IvMpLK|KCKn^1TXMtEaP9A((O{ zB^LRYou{i-{~(>iIb6A#ryPeQ(0ZAhAD+feDw2q0*^=op?o-6lFjr+=Y*xS$hlJmVfi%UkxHjvb zFQRR?514HMgTJP6GM#9e2gG&Ij1Yk~YTVW(yZWL$g6+vAu)=+q`om2B_Oux#i>KR{ z&eL+P`}_%sLk zd*u7fdEAg-dXnRJOGuPOyNvmC>NUMHOI7;Y6@_jHu4IxV8K2Y{agQkY zUH$=L@!WXztWOYpp3tS9(dG-s@2WREmRJqrBWo_?_yr6213}81qu3k2986OA1!P9+ zHYLiO(r}?RV7Or z58EJ^`?L^$gk?K73FT5TVuydM4jPZr4*7L5D^0 zW)oJiAwf>^3D71Lu(_36enFa=C1YsarT=-ecoAT$#QSLjZ^S@W-{IWoW%-#ZG1)q4 zz@WKPyx%M^Ypq_pRf+b);(Ex?&1c1{PBP)XwSSiOB_G`O_XDPLDSV`t$~P%Z-QULG z!Y>OGd9y<}F5*VSmG~>YYcnMxY$!xPSMw;4N#4d%{R4Ra1Jpa2$Lp!(DOS?rkU!k- zPc_y)6MDx#899RXeC_$skX@g5tnR6Fd3TQUqiWs-`T~Jxv%Qo_$0V$W>!01Pf=f7R zS-WQv*Rt@@ls@dV z#T3vvQ*o6Yv@V;OY=Gw3ff)e`RX(jn0X^fJ6M4hk^W)6v#t(y5fD4vSGG$?clASw6 zRCEG~@j0R^R>ix>0h0QRKCM}P>J}w)4%~TEyU=&vSVt1;Ku{q|sZ-qFZ3Myv z9v`fjN$G!QT`&B(l@aezJ!Bgc3u=*CBCK2va@o#&H3n!HR(}kLK(rVeKKZPytdZ39 z?2p!+EKHf;6fy`VfeVR?jI!Uq8FY z`rM1VUKo>Frf#Fg+BH`x&3Yg;p&HS1?%>()jyYjlu<`nG0I^@HL4ogsR1n@uq8PtB z1^>OX*KcfN{GF+aU$Y}pSfv8D6;M?U5uCvBpH>XDbIvqoh@X~(&h8RrnCE2Tt#u!! zJ&W?1Ra&9Wr_#1EmNa{&4m%$epMsfxD12lzj4tDL>6UNvVaplP+ZN(4m<4`wRL9TF zI_tJ5(wLhw5Ds;2)W!o46{5Q0Zu= zdjfxz@cFmIi|*$-3nz9i5#{d=$95c4Me=jW@YLYcgpM}bIZ3t#^#O3~L%#s=&9p_5 zc#R$?R+*wTsiHTPWY~6GKby!ff_<}Z9yQOLN9g$3?rIrQA1J5HlkbO!Ygp)ot6vVS z?Z1-HAK{OLl#w)TD%#P8>P5Xz zC>pLUOADgWH2vIjVw-DF1&MoIp5`f7J_)N%{ZP$p)s1UYX!Qa=Xp)$|X40nxO*R6v zTmEFu2{n@eFDLAo_5ByxlMeXW2Gvg=>YB-#wi1-xTD~awlJ|y5VbF8a>U-dYvDT+V zR@F1VcWu@z!W}%r!8!v&+hD6D+otkVSsvED_NeXo0;YcX#PrjzR2tSQCu~o+cZ{?% zm^VsvX$-h-UL|~g3V=boxa*G5=~HQq=KZe zIK?i|;uFFvPjX*#$sj@h+f4v6tSBIw$Q>BBRM!KVohe6~>GPz^$Qz{yvmIjMqb-8E zhmVoGn%|Jumdl@`vIsjCv23R57TV&mhRzm!%*tdd+dGj{nrVcZaQRDtE&=+s&?X^~ zd@)dicV73rPl;4-!A}wd_Y|e4MdBI_^{PKy%qcMcx!u$d>49H!jeBEiG-vE>e_h&o ziMNPEsMDFPOo<;g`4&tpHO!yCmA#+ox(gn)+0kAtWvXUO*^Q##Dw};{-gUu$m|)RH zdL4J6xi4|ucx6d}^O=%-YeTrwI6@UYnCdDAOxzc*)1Mqq$Q-$y=+~B2QS_b0hpF92TK4aSNhXgjo2HlNPO*g^2yyNb zEB6s@&i3=6SqNm#%jZ~kJ+iWucSgkvcrE_js|p`mhgwl+^=PHzl3*8)jbNa?qV=*H z@7#YcEON#S20z-u#F60BT9}UoRK*@o3EIR=24SbIKz=YG2bf&d8htqW(dUaU`zCP@ z@!}Y)h=?TtV!`(EKj((z>IBhVmt{Nk>j}}m$cMj@A8Pl8j@JzOTeL=5E4DVQUm94x zI#SZK;XPyaX#4?D7)t@iMoby3Hf5MMURT~lZFK#|Qhc37YOo?7wrAgrr5CnLUhmME z!tN&LG<>My`RZpX8xSe}W_FZ}45JoB?6Ubwey>de>S=UU4tdD8lXJpZz6d=v)Yh=H z$B*%%PV|3Ti>$WF8kK!;$2fpHP5Ax)oLC+nIF+-jB|3%--M=2mw5k^xP0|ATO;;s&y(nJS& zJwgNbnym9b7-Ar3zY%JD)HLWj-9@n&$Fx_!F#x^|w|xGv)r^L^j>fM|=yk&AR_W8( zH?(74HCpxS(oO*KJ9Zo<9X!`J`Tt{fhEHw9U-UFlj}@zT+CSSlF2vP<-b-H?2{OI8UA@8|t>&!yL)O_9GUl zq=#(=D0}Obe*i@qtZzItz)h|2WXD*$F*Xv~3k@A3!4nFPgVA#D40{gE!ZDm8wUgU{ z>LC=h#cou?@@G}$nS@bHe3UDmV(+mthnT>TspnkNkliD_vzqa#z#FN*%VV+b_>TU#B-OJ?3 zgJqhAoj2Hj>RQyAZ!)xskdivr!tp@fKKR71)-r0F<*?M#znXPhY@_Q!WQ7!~IzL-_ z87f~dyqbZi>vjM@@Uu_fSA$*!+gTe-DU+!t{PnPwFo=5t%63vo;nYw!#EWMJF!%f^ z-}xUGy@0o&v46krS;G$qRWh0FgE!xp`M>K@LdlZq#|h%)rlofb;m$QQiIYt4yT+1&>62w0AczN5-gPqw+zTeck)6 zF~s$XyG51}b{Eby+no8wd@|A2(uxYn4Y)1VL;W)nzkO?k%Rs8;-dXjvTXLktd++es zFn5cN&25f4i~FNTNju%t+-F@G3?#nd*Tt!GEN?R*p{>c}v1!m!sg)-zATdow;1 znmICU^?reP#BkjXoHl5!w>U4yX&8DaiznO_(IG%vXWDY3Lp6t^Jlc%%Y*4TeD1KYg zKq$FM*!rMww8XC81x?AD78mNs>V?hK(*1h5i>^gvIEY7S#J z%PQ470Mw8ytIb{Y&plB(szoM4Zd_wAd(;uJ2A~PC$}hE`^ZC-UGQ)c5cf&4dBJKbgN7gdC zhLb9SW8NUh{Jqxf?m3e@*dkdZhd<{U5R$NY-XLLllg{0C1Qd7oS)?BqVkikA>*(G^ z={YnB&Wzx^=49+OzQh+DwSPGNAH+Hu&=Y-PpZC2rs9lfptZuc)Z2>|*f6JIQGh6}Q zZMI+0@~OB!@6U4e)ZJJsUvYtqbpXP&UUf6A$S%!KLPZU)W5>&v&9KnUZ0HsvQ8=gF zP~LQ?W);O!9})#*!dmjpJ$ukNR^RCgL~>;9%y(`a+zf*kntK~1a_~H*FBO=yJu5Rm zaz$1uTWT=N#YiZjXUjsk8XK20($(g#4R5)h+Zc#S@7~@Sxe}4^5B#NR-O+;{ zJ5DBwdP)|bu32f{;QcI7P5yX3I>6c>CqH|>Zo_lF81~3&pn5suw3h2aFRB`I$THu# z&1+KK%lz`d#rDA8Rs8hEBe8N>5Df@V7Vb_rczX+>r=V`p(zKrO?Z<3&%IN2=c~ZQd zhqG0ug`;)ZgS^-M?wb~y<~x6btk{#!=K9<$xPnw2vk#OOsSI?2=^a*Sc;oeGRiAD9n9$KF{gOqWcs`x*@@9e*Z-p|pW*fLCuvl*JX!;;%p9l=?drG!k@_ zMXa1(lok-A^+}a=WKuS?XqPe&z2#LsoGGU)^}3_J%^~-|JG%)(%b+bt=0Xs z;Ltrt(xq9kTn6-k8#=oSiPo;bjIj#Zffp^#6{8AnDdPIoqy5872KsliCx%i3ae$|J z2pafw`Hh0cU}KjNidaU78~W9RL77k?0SCaYXTBYT8LZD7>3cm9t2`Df_TV}XHz5U# zA$DnS`tNneHNSe7EG+-Rvp0exw@B8n3Ad$YVUw<8V}He7OLIDKc9Cn3?WYDTC)dDr`f** zn+-Cp4d73xM!*L;b=P4lNa7p)e$jgC95%4@v29Uh>;2%4N2~j4R1mq zKV;#R>i{G@z&JQ~NjO|rIWKG`er|qI`2<%zUi4;mc|y=(+YGSRw`WU$?}v1rKyMOn zuezmO3GEJ59$9?9=byhyPgLA!Dc3q_pte2sHfCIk=Dq%y$F~dd50K-$+k{fAfR4;f zg8wRXRe?MzdeZv)DF~6mge;I~k$1)$JH=3)yD+()pAZido2Rtt4?dU*&r{Y42ju`> zr`{GnmAd^tlP1mIF_avGtshSFQPTJjixQoGO)xF6D{O;$lnj;`jR~W=*pW)repddY2~{} zDzt(*3Rjr9g-&?dYB7Tg!yBR(u_01-6JJUdA%{$R@3n9XqR3Jxdfj(`xM3ar3;+s1 zz3-Wvw4o8LEgVU+b>LV((bKpRi6)4R(9BSN@~8dGHxJn$e6|&?p&QmbtxfK7dYT=X zF=qd~_C8+1j@w#ynB@pqXu0&TK=PAI#rwwF;;9LHPCD&Lse=c@GdUV_ZB+|;J|>p3 z>Ze0~TbVM3XdWl_Rd}V|6M5M&|EQSCbfEb9QQs7H)s<2Q$w7vmDP4H%Ed3k;Yw!OTWb&Hxu1D+bXY?1&O&WR<3?jvG>@KD5O3OclFr_& zb$w}ia6C(eb?eiL`>v|ig8^co`36|Ci16bwb~(LGi$hX^5v`tq)54e92%SQrr;Ai~ zbydO)e1zUav-0qwVH8?qvj3t0xra{+V=;KrF;zJ8J_m4Q&wg8lW0)fe`7SO&Q<&Kc z5F2$`(f^f6rj35PgaatRO@jZ8rB9ZffVp}CweR;5Y)KU|33yt8cu(ME>GR$gu}Y|D zFxR@pY&d*k*;qPA9Z*hVlS8qI+Ic7kD;w%h~j z_KgWu=%`$b*oOgYDFfGQq)pqD9sYj5%_-o>msBpOm1)cc?3Q_GueXAvbbqpyGDaVy zXCDyY0C9f?=u7yfT%Hk|eCb^{Th6XhzCdFD81s0s@_k{%U~*p4{0sMgdE7psKW%g` zX_PlwZcXdgXDWpb8QOCcRjK(_l?S%Sr5sGsxGY#edD9qDqHzX>`nqBcJ z_M`IsCDx3%tlzsldzJB4PUJ7h5z>UJ*Y)>Hk0e4xi3Ga$zs<>UUg_GK7 zR1y@xH)nn*V?Z-Zqg*Z>?k8G3L@kV_6KH{iG;~8gK~aoZHPS)eHBBwYE{r?r0F^oi zczf&f5B_+Q#rpivcG?Ka7;y;XnT0vr_al$!``eD@a+hv0J3Dq&ce7R30rE4PfeQNm z*WGG3yV(pEQJe=B(;bis?!XyNS6ybHXwpV=rc-9K>=cCGzJurYb5zMlXRrgtWGSXF z2nn%zk0RQ&dvur@48@Hbo+#cPXp^cRXv#2U>O<_hEg>^~=9Agi6Z$q92AL!GEQ^uF z&V_Sx&Q!snvt2%#QRM{6GtOrMQsP?7wlBv!ZvNS(d{Ek)IhTcIXFJa5(6}UIc25(T zKPvbN(Botia4oF~<9{uiUP|fGa0-O@<;;M;2Ai0WMdE|NuVyW7IK5+P#oKA0G|jzT zL|_T-W8<;jwu6(VTE^>m%_3Cg&EU}?RFN|0bn=}kifkqInWz9AD06-#=vdB}DlOOu z6K5=q@TAaMKzL~fnsYHti1z{z<^3CZ6U#g1nRKUBoTF#ObQ%5kqIfEhaw+pg!Usqe zUa{)m?{sA36M}Jp0-@EPpcxa&?NLPFyR-^=Ij^oMZNoyDtyn%)vltk#tZ-2bgb&tPfR8Xy%Wpl>AE3aQ)X>TS9cDo_}0Frmvx4>#L*=>#yLYX7@HP z%dRoZbd-(yi_~iGuF2-ephTKi0&`XC5^Tq(8_&Kzc$&BMo^PPs%1cNXG(#-G3qAi* z7@RUv8EqA#>qF4Iu*86p(JL)_VvL$Y7QN49a(yfE*MSZ*)-u)O_q^!Id=Y)4Rhkc! zS{u3~1`fX(-$%*X#ZE=Fo)e|AgCD6WK^o5*S~JGU_9;u=Ns;`*FC9Tnd@n0{a@_;; za+w)C1fpydb2FVdxd_5_y7~rhOR;EhdECP9E`as;=)}tE-!TE0mjxwra58*G0~WX8W%S`uSr`S94%w-UK~e}j!%>9=*LMh*)za?R7RD@sp} zg5NQF%k|%lSTe4oNSIW!eCVb9X{j;W&zl>bn(%>{ZNz-O{Xk?KMpGYbo%yZfp5yv_ zkBy*A?Q%y2M(jf2nW~`&^F#kH8?WMNN9)wMoz6+>$KLnp#Fe)>$F9T@W*^_N>hfeN z#od>DFwp<6h>P;V!b;S=-6vhE=x@&TBFLdd5^w)d zh?Wywc2p@T52`%Xv?~s3EhI5UB@;!yge~4=`DNK1^vAnYY&+|#3N!6SxhrZ%OMvEP zA#&2&LQb8pY_2Sj3%G&TWhk`NX)ZUT02rLaAt&|u9f|CL zxP)kGfg6CvM_c9S#XYM#DQJl<4&ePHk}>Qvyl;E{#a7rT9I)~d3tO0in^8Xu`%Wb& zT2kSsce_Hbn*(?g$x)shWhpZJNN>DCf&*n$%7uw;n6>owtI}aYv_d`&h(4-QBS2*%$93{eq)) zBjsxqRg!o15p{2%YpEgTt^e9RgQ1S2=PNGvscb#G)Ao#SN)*NC$#yHwbJKvU8$?qY z3pSrpbpB8`_L+Hj9%8(*nJN?b`S=eush-3WA#A}|lnKn zxQi7F=KpX=!%G#p|5s|Z!M0RQkNIHPrmLR8*x>LNt+1e;bEHDy$2td}y{aCRFdcYx z3j2nil~I5x)i~JfWphm}1qxAHoEqTU->3ENr~y_h-_8TZxPU zlVt8bUOn}BSGZT7dcC{1YfI*D zDPDW-2F;5b@@=ivg|lGSKHjvevmdth&u1@6P?T@;|HPUMa@OeZY&^Xhoz3F!s|m2scf#mM$_fs5jdjreT1_Ohh}oYnQJe1t!ljB1G49WZ>$6J#7Qr zAF@dwCFhCq5=32;Pp3xjZ;Judt4wk7Yh(lV{Vpn1Id@F0`?d;AY>KjDn&} z%$2!GZ8GA10wF^tX+yGOh_1@SE4z3YXwZehr%;5-)+2HAYk)hgh)p$iyxV_eS z`D!_kmf}~cc>U1IC#rDc=Wj$9%ChAYRw> zxB@x(51{5_dhQ9F=|o$ZuoO2ufA>XF>P}|ti9x`#BJYBD@cYAk6@_Upxr6}zZ?{)z zX1>>LH{UF(lLd!>GQ~*f!C|}O( zSj{rl%vIlB@eY00gXD)dyT$|~4pCZ2ZgSKOUk8`UBrQ&;OGc{9)h94Piewr+N9+{q z<3OXdQ&;Wp58VU8e%;93=V?K!>vJEy*bp6veke_{~6n z>w9e|I}W@|PvIZ&7w<4t_A@y_Ds|6}$38aMj@jKHK3LB9Q+h{P8M-RetEtaL6Y#ZC?+iGfF98OwgQ+IX zA&*?VaRZsH2vF5V;!|&2t?smaUG+h&e{@Jl=y>5WEvmYMC!Ip^B%R8kJ@11_p+zUF z{ucM&9q}0PC1qz8SD2k9FYFq47`C95sb`YC61y_(XVU`H=lV+#Iqrl?TW1?-5-<~`a8tfW3L$Ayy!0(qCiBRPl&L3T~x@vU4=nFXvfvPbgUkKyn zPp+j+=E5aWlQg|l^tj;Hrrl_N#{5QkU79y7&f3WYO)&G3YlZUnm& zidBqh=7(OeO>wUrnPzXye5vy}-Rhldhk#P2~%>nC#1$7tL+wm0F(~A$yGLXcvs%FP#9K6=OZnJq5_BQOxrYyBnMSZ zswlh+|8kbmM@#=lqLlOgHEu0A_2H#`GPTa~OM{ixw)o&J3CVlX`|fRLx2Sc$+H=Zw z5SuH$pI7-HwG4{>Q{7GRY$HsGqaAJ-`m*2Ij*9O&+m&I)CU?++gjfnX$5gg|Y&|`o*xd}1^w?6* zaw^LC67h(C_jr|o&Ea)eZo6+=Q=(#$S~!;>woRvXuoM6uZ~SGJP{|KFTnnX6Z67 z#J}_$nJ>5sA1UnwW^v3aa9`+Ba=ZSXK27~C%J1Sb>z36VCq^J&PC)}PP*UQ!G`BLL z=XjTuC68F5_A>D+0@l|#5cfIFezXi(jazkkx^z3KHl{_E^A1HJ;|WwEnF>N7YpFLr zh6Flm>DHr9l$JBK26Cdte+#A0%jf~E`I zHUyp9CjhZPjltg`N4}A`%Nb2MmO>&{CjdreZTk0Iof$nUborbA!nq`snjitC`EkLY z8luny*FG2mD`%HF9dU?VZH6|!Kj6rp%(^49TvNh_X`1|O@yGkW$op!mmVf>;*piZF zU)T^aFH&k6t5^233_2w8AClr6kCQ5oGT~?pk00g1Il|j@;Hxi{9^c9*MSo^uH>c%4 zKzQ!&Uls>#Hcw;CM;<`_0nTcRLOyFp%<6wTaD1IQ&}wd0JxcirTInsJ2%?rcYe=16 zXJrV8VRC!L%MgW^@BvVnDyYj)ZFVV8hV#6Z5qhNsbZn|G!@!&v@slb8zkB8)TU|*E zt)xnCW0`vbA%x(*dKC-VZLhDG%H(y}r6zU#%^Wc3KlRhv_k}>ysvCi>p>CsJQp^>r zhnZk^H3@)?=k}e#$%hvlH+12Tao$DZA~$m$wLvqDdv^<47>lrxn%^coCaH}BCyn+i zIP@gT5Z;PApIw%awd}H#SIgnf3{%qO<#WtGamIIhSU(wgbJgK0d-DBV#}C|OjGyib zh90%o@%D>tRK@s&JUICgXiE976;)vxB=${MN7P}J$%qtcujHdG7A4Ymf}s!YSGV!6 z7`s+1@{}*}$KTM+Pkc?qYgC`A_)FcgE$ zFN4t}goMWUIh~b(ANSwyL-#&sq(@W=Ud6*1sj|rTd|pF=$l-W-!OUDP?mcoo}tttV~euY2H-y6>l=Y!^~#i<6J=NEXLws`Pifp4Hkc z2C%r1Wka(50#A+Ai;UmXx}=|$284K;tIC{c_K2ihpk21gnx0Rd`b&Fn_9v`0=_*rE zmdV|iD#9sY?V~|_S|PKc8I={k5?!V@dPx)qdVKl3>A;dC+Xqr!89X|$Pp4hgW;A;= zI?04FzPP0XWRB)~K-c`>7_o7F~+VJ#;a8v^i{SER+;=ok_`Ajcz( zoaz;@YYy1(b|M#h2QOl;@uRxg@qt~3-&fQ(M}?&^rgyglwoCS5=Qf!-x5ph4X2LlO zcL+jXl(?errxxFk`4AxCzO=6~Q>4+h(Ox9kjf(H&u_>1H?wEdH%K6bMQ-^8QGWaC6v4@L7NK4X?VJNktXm+?=`(nA?hXu_@zjAv=!8Pv?vmCJKvpy%`)* zMO%I1F6Vo<%sdi)e7a3o*`l@vY$sovZ$S}wuTOswj${8uVIqH_i~TVpe;pr(D~bs` ztFcJ?k^i8$dP)T$5!S!TsjouH37oa5V0rZA<>38EnM5!Zsc}@{rPuAnR~&itUF&f( zzr8DG_es%VnN-IsLGKs-NTk0Q4Cgub<0e}t`JBC=_y?Hm(9C&I-OtTAGyE$XAdCB= z&Mj$C?dDosEkd{BM3m-mo=tyh)fk9(TGW^W?-@i?nrmoo!$(v!T4w{hi?aWM-ONCi zZXVz=%2<{i18N%kt3H1su#=Qv&aSypc^?p3{ap+w6HLs$)dBA*^6DeI<})qopgN<> zkr0_I5Gbv}zCA!{rHOo@0Li!`+mh+&d0vxpDf&YXJ6ckwd0RU0^IE;+M=-s3qw*gt zr*2Azy4$N>$#C<5gC}Fuv9wZ)i+6Pgw%VRJm0e1KW z`8CPpha6LRYAqw!I=iNU2d!h9{*vMky1rQoEhP$cM0w{$pYXhC`W84U1S@tqFcGGQr)0>aX zs_Aw;ca3GAW+1nFWzH&i#en?W8wF^&i|UCh)6>HOf{JKCS=MpmHL2a$V)bnWHx)PG zfCvu2??*Ri%=elv>kE?!s$WkumPX+#>m)Rp`Nu|2hfc81 zp0EcfcA??tq`%B9m5oj~TJ1(ps47DwL-km##00wCrOoE8eNQ2NC#gz!F8?swTqmR; zj)O?0TqMqwMYGiWTpQJ!G%C}D5Knw}KneT3`=J9=Wp?G%hKV-nVRcu$7P=+`xqWq} zJ{zgf0+;lcJ&&QhL;kx|FAOOpgno>0W`qL{sZ7K%)MV_YXW?val9#rw+~S0nL)j&t zB_X~{{0BM4>$P#k*lO6bUQ05pp_#!`dL6d~hY*-$*Iw6=x;~uElD7)h6gyNtr}cf3 z4X29)Hv%6s$~v_Zu+YGf4^>H^4(!mIIH%upMl_H=(eqDopY~i=8;KA?YcmHg^Ck7# zvm1GjV2@uhExl*%X2iXf4j?Vh_xiFuNE~rQ@49!e%5Ar)C7AEO`#%EmHVw&AjoAeL zloTNusTUh)12mvXNVxKj~nuOyR>zc0jxKEXN9A`8D`BpN_mUn!>??wij^c6Gxu$?)Id;i3#yKLCf=)#s`_#a)!<7Tl zlV?0qs4hVm98|kUuhM`klsmtz47ejaW~6xV4|7c`@|>P%0yjOeQ5u_WJL02Zj(bo8 zXrh5Y3b@WqS%iGbpmIp6%5ZA^vW1azfJY>n0FGt{a{JRAKv1eV!6uz2GLlXPbM>cA z2I8^*0DBYxSdZO|XLfm~DcOUb2*zqjAt?~S9G}*aqAd!z?T}~!_L$(yu$+?M5A*cR zYdJSJGMpW&@`34CRt|pG7SV%@9CxmX3Pm_>g^uCQdjA0WsW36G?baf)0yh#tT>5{X z(xZ-GmjTswscuKNrCN1ci>YGS^AtE^+mLhqeQKqxv71r1wUF%+N~+{`1mM$+-h;P8 zLjM3yNkY#60bFG<(>|1FtVPL2BsU`-m8SNRcz$yY*cTQuHr$Q~JpMSxj^{tH( z<|J)}wtz-fxF<$Ym6|)-_fJ8wZ(wimuDNKurw zMI|Ozy}49oMIhwh*E0VAYQCSzjTx7IaqN3l-xe9{6DW4EU_%4kpZ>9}e+|zPSgi6w zk8vT!z8%W3k6Z>}v&Lc1a^B za!x?Ut#3nW>gF*c4#iuLX_p#o!w#xLDabvD^sE$@LR?g9>`E1Lmb0Temcfl*9$G$>~jJBd1DI zryVJTW0Sz30Pk)|;+?==4b`Zmo2pwzB{A;N>FqE>{oIp3 z89ylW9jiR#=AVtseQA!^JXHYE>FG{I%V#`N9&me8(NCvZ0G2NDD@6m7e^P_iCXnFQNCqn z#{!iXdX{hD;15b5V5Altu>(HUHyw+YR_JZnmphaKaqGobf)gdY#4pNJbB_6~ErH=c z2rVnGALr>$iWWAr2*4jiaUFIpyHkaEC!_7pnH5+*Y0>A13vI&=DvDwVycv%8Y@w@nM- zWDU_q>||8W4;aSlQ8*~8n<=Ten6qH+;SRqu#4qyTS6t=MGLE7|%K5+ks4V zoizz{NUEP{bXB>(0IZXOh8;b=P zEXrH4AXVrN^a1`>9Ys>qnc+gqgSXeEV%uda8CTK7s0MbhyE!# z!dsayF2WTUC$X-Y$})%%zVPlVT5;D?Dw6D2wX;~}^H&O6?*-^;`?&AgnG$7ideqE{ z7pFDQj0T1qXjfcgy)=WIP$Tv0KoVpX;L{}esX~#!G%^q~)_^L!%7+YaDoEX2U>+*1 z?W4>U<93adMG5=U~WUr11sF^dxCMn7#@`H<=`ry?p}th zJ+hye1IHXxQK1UTI9$*ovfI0iNH+}RiZ>u8DB}YjwR9b)ZYrhN5hhcHW7d^`si@q2 zniec@2SR#Rmuq*eeW<*|DcUx!I%AsZyj`lv3?F1x!^knX9&ubsqdQ>q?^J-SdLL6v zc*(_W+vz`OVaLil^r}}d2xer)LY!uV(;bQdGsjO#dZ7myt3F9Y+*djEr`Xyx#QSl_ zb3q(8Y@e(ds2L$yVwaU0C?5DBC2ny~9JQ77Pj8%s~ zN2NqlC#@F&fa0t}C{c=pAV#}zZ`SvRi=x}ZZnFLdMDoN?M`P|S7|&` z5fo$rpGwiUCQuWw9X&nkH2l9PJXVE+013~`dQbw(7nY5mP5`JBl7M#xtW12xz-*6S zTCRi_vxZ!ND?$Edy3BFU$8&BiA3LL1(7Js@st) z?X)i4fkNPdK?mr6oowlZ8+9JliKyKthqcw5?iTlq7~`V1L;2uWWNOK+SVb5R2vSp(-1$A(w^!2S2O}QS2rB`H9 z$FmOAFq{+$-+6e?sHwMdu1mNOdm5#GDDp~3K40$+)K7D?u2XRy-5=`H_Y^mUi zb_>t$P~a+v_+Fz;Oy5n(|b)HW7H7w(*T9@V1~0R1p4T5U5LiJ(Owa(Jng#^0Nz zDUW_YraXI78RoV!6t;7UVlmImX%!eL=9`1wfG1W5wIC#)YDMYqQmDf6C;_<71aVAc z1Cn2c2~A;WGw53On=;@km&#Ry>M5t4J$wO5TCIb)5dwJDNAl`GV6MJ#II<&Fns zqQV`q7?vG-)eTZZ6b!(8sxoT(kfVWI*NZQWwv#lU?2s|<#ZZ~&8om6N`jnEf<8cGn zQd%f^R2<-*wM1?tlh&_TF$^-j=`o|Ih9p=?Pr3(F-mx_pOOU~M3`st@t;=9!vSk^` z6b~!r1I3Z2XjfX(zz0$BN+TeO%ypeY<*2yf*8UOPbaArbu{*I z<{<@eF~w&oSh&LG9n1v?BPZ6ORw{7S(>|2K9j=`J0C}1|xwrRXUk*}fDdYx*HC*~g>VZK(0XR2 z!s|km?9W-Ykv0IV4iC&p>P=~v549BF6a1@zw%1U>0+$16Ea&OXbedkJ=fbYd&%eJl zlZ!@)-(#RC6*p5Qi*k>8OM8`-S&0}20-lnqp*j1xskVbnm73fu1|9nQQyD&DJ*w(q zhv8MhdTU z;PG2F`fvm~W1dbs)V6jCxZI+&r1u+N~n)T;Op)4SlLGMKCI|llfGIOSUREk7>q8paywu>qR?h{`C~V=H<3W zTyDu7syOXk7zp2-WYu`x@>zFab*o5Zz~teF2BZZDBN=7RJBn|UB#*ms3)BqNn2-g) z%MRw6?y)N~DI?HO13pv)FDD%-RBb)YM2@V?`4~T?O72P>gFfP#0t=~1CRdvC&mHPb zBE@Kp^0O9TJ*&97k;0Hq09Tjzwov-hC^-s(f?noCXCklLU=Mnu0L%Sq@?ixj7LD60 zlByB81JbUe#$0t?#OJMI!77ymcC92~w{YYiL0HR1YZ6Hu% zn~2KDRdfFU)~inBv^EwB!-3CQzvU~&xFdJ9XQoxE?YrcT+;ycP#C@zxvTR|;Q=dxh zbUO&{VL%rE9E{gD1WPM#&nhupmxdEOHcl&=q18FhZqMuRYb5>Mj&#>ABATW)6k~wtj`^}(PXu^GB!bDjAE!IxmVag9`);*wty~d z_AFy^ox-_0{{Rd#?vVfmS35;-8NHc=ptnaUEGZHO-Sbl*K+^7D3G~lOwsxjWBNJF>Dw1d~)vQukgX9U)#B=(X* z*>DIQc&3FR$pG#g;=2p2EKI+BS%~NTk?1Oe=**Z%*FWBK)YkB=+~=Jh#xxPR?nncg z=&WxiwzL75?#RtYZ>E-zBBm4(xZ<=Xv%;!@^D}22g0*ssGf-&K)3v5VD94Tn_w}n* z8px3wEJonNpRHm?VB|3ekk}PHzMmnsnHPo}t!C^}xzku`OCqc8$zhJZl*^0J9ITth zK|6WuYXa8V;xz#?l_AHt?@(Nen~8U*A%Vs-){O^dNvdC%7)Yc6$s^a=vYSy1@v4)x z&N=m}y1bLfHc1<+V-25CS+)%&#-SL&^5Q$6KBKj12X|s*tc`7BP%^~|hHR>pY!7k& z0N1R?)g331H#XG4FPOR}g`pPVz3>&<1~X;%=-Bf1pbr_|^yECR49 zyVRV3M?+m@ouFh;=v0cSqzKs*v~8bpT`VDXj3z#{oMFwl z@u%KkDkr~*Mgb(&>Jd*v2{!i&srWp83#VpFm0rRLL)ah=71cJxC5LT zS5W;&KD4Z$b?Z?`?gkQ=2F!mI6&7Ul{b-;LUMYxx1x_>0YFlNebhX*T+kELec$DKweL?C0IO0{mq2r78=t8Z-` z%53|s{4MBeV(p<)SJbT~v3UI1BrI`_zLkM{X4{M8^Uh6d%GWmX7-v#IJUAQ?QCyKC zDx(rdYMy5qYEp`lx;d6ZxSn>7)gl}>0EjIlqkvOwlu4Ugn4dAAoE=HT6-N7 zlUg&nj&RDjKb>5Ml^9-Y0>TXy`fg!od7viC#lA3O4#l>&U4nO!tBmNFe9cr)#y+M0!YanxvXcY zv@|rKQsaWe4%MSw9Rx%P1$dIjNr2 zWFR8#=bDn*HDWfpcpLs93Y2<{&GakBYX!39SEqAT zu5YpOAf&vVkIER2rBVbbJ%hiYk#VKfOf&b1xpn+@$1pCY>G1BxE}R-o~K0UFuHe$>=JHM~^3ICZgbIDURXT zw=S&FTU97(_^(WS@0Iq&hz!gn0!~ihe zs?2J@=Om0!fQsfY(c?ABc#ls@n<*1%EC%KE&2^K5xUP6Tt0Mh3913D*kz--O7(MD( z*j1fw)Y@uuCejtyvi%KIjE*> zu6-)C)Xlj1X1FGtU5`c$N;fpqQS%6nM`EXfGuE?pD+Z3_$pZY2Hv&Iegueiu8@3Hy zwz>>~B*5-F)j~@4VyPEv#9cn~W2)@Q^)=PlTy9k?-#{yqcv)l&gXvnOTl zsF|p?cO^>bnL>`)BwS$8rxnUOt_Z8~xGFgn%f;pv z>4WvHD`PkJzgnnN@+NRLK?aNpiWQKk-BWF@m8pXiP-zDgGX({YK}E?Clkb{rOXkQx997V} zMmn0RH)`yP-xZL_r%QOjEB^r3teczJC32&MJ*xHO4KM^g2Tv z@zRh2#uOZLns7il2P9Jg8S9FWE;=45fX8efUutp_jAMgCrs64GiY`=V9cTdxq2~gM zkIe_tiU8oA;G=FMCm5%joLk{z+(_yvjFGNGXWJA9HjGg|UN<}Pzog6++sykL>ixl2diwZ|T!x>&~e)rY5QxV;RdwKH`%yti3X_)jO= zt4Q&(244QPt95QBg_XOC<`ze_xo9Tn4|?aV9;ZzSbSIIzl1H^7NwLRXNT`|j6SNWm zrwHVi8yA2&*Da3Anvh;DHj)csuSFnOw<*gWl}6IQFy7pA&1l0KCRZT)*6^uq8O>Ij zdZFl84Cj(DlTrm|U{r9OGt)HRERry4WEfm_uC#GB1qfb7MQ7>}l7OE4R*vn!b5*qn zneB3blDy)HE2|DJ&gK$ARrn|ASMA~~HsB8Q)lKNGJvy4dYZfuM173WUndrtzp;p~r z0i)^OtRx>dFR@2#b*l=1FF*x2x)$08K~u2arR!i3Frzu|inOL6K4LRe?Z#9fJPM(C zaUxH#3HeQH1mx^vHlnU++}y^b?E^iHQ){peb`1K}VdhEH9%@grzSElMRF_ko)qM%Y z**`GrRv^1pU>nl2Ryk9Rxy3mh)z8XT9ewEbNe|lG=`GSZ03!g^=wn1V;<>0VvzOr;XX^O89m8ha#8=s=QN!j)tS#-#sZElw+CzlW`}5 zigSQed-kXi$ZD0?IODAVGl9k^rj+NL(Lfv^NhBw4KhmW@(JKU8umi1o~1y z=X)k{UZD(@J<=bXsK;tXOiH9S4tV#WZHh1q2|a3g8Z-*Ueqqv>iKAuS6z4v*D>!k1 zRia?Pxg=+bnH5*2PvcAlBN^tZzFe`$$I_-?cN_vuGhtt2Kp9tY@7%;=)84or73oGf z{@ED$TZi^Owdu3O2sy_!iKtmeaTCVtm0^QHJx?iKIH#ej`o5Ifea_MMAMPLNS>qtk zQEn$3WL3x?4aH_FV2ZO8m^CdI7Ic;uBrrAB>C=fK3ReVot_sfM1anA|j~s=v+Fvo#1-NSRJZ#wpER69(e$QG&)Z3b#|iIIB^_ zk$|JW=}D!rR#an^=xYA}lNol%+5!CQ&yuq}D6+U$Aa`E4s|gT&rFbK)NG;s91~3-2 zw5>PKQtiVIwK0os!6_uJW;6~EJ`UdW&z$fE1$3`rEBq3Ci(SAdyQ2X6a46w%i2Bd0MWH zH!C;iTyziU91aQZQb+)ui0DRZY7gEy>#uZcMyZl0LJ99q&m4}lpang7^seKGLb=HU z6sr=j1d2%;ZA%lYt~eCHVYf%>xTs#;aD7c(euL7hO_Vu3C<2pO0NfgSTPX(zpHohE zIpd`;NK`gX0`rX-Q-V=OrM<+C0b)P{udQj?qew>X1!h^PEUHHst;==Uw>)!7o4l39QCcjptFUoO z2?SHwRE%UAYhgzMy+nDS>PXzY(7AFlDad#osz3s%&nAE&dwASokP+`rNYXXhg94-& z$21XC&_2gb~uU*alKDRpx-UO8!&|#}nc$H|>^85lNCW zxd8oZfpAs11RmhmvRv8j1nuX&aN7R>hTW%oi$FQ~ojP;vK$eFn4tc2+jWyBXa*`=l z9f+Ve%}k_dTFHkT*GXY;K2wpA-nkgxYXe(0PTPhL(xSy&^jjHL<^|i!0nJBw31js3 ztlbjiMIwf8y}_=6L{?MIbKaPlg=|Dlz#L%Kw08bs1ahaE%$nJ})(QyRcs03U43OII zen8~c4M^HYZ7C-auqITeQVug(nsdB22!#BtfNG7V6K~6s!*Q(}d`lo8VX_V?R+1al z8+T}l04LC&(zvOB-o-j(;<}x7V;-9x6z+E0`d2K<#B@J}cO!|i26xFArN&69M0jJ7 z+LHsS{{SkYNaY{9JRhK}N!mn^h8ua~-nSed;;g+zM$F_AyzyC7jnS<+a>#9@yT>j; z9R@h91bxI|zZ%1gkU?ID-mlxg&AV~nSDz!%iN9#)aL+Y%auHR?KJ}YnI3NSpp*5;i zWI_iboF&Tch@5YuBgtWqb4>)2Na{Ul(GrOiX$axs#Rgz7Ev#u%$zHmVmIm>a{nIhzRpK4%dvDwsdRc>X(EC)g= z)S#>L)84ah9h5c)7^LQ=~Kndry07k(I=Nyjo(sv9BLJn|6IT#LuG=NZe??oHG*`k0sIqp^4 zOx~a^uZEAo%qMC zF_klwAOp^50@NE(PEKi(zUfp3`F$$8+=XC~g4C#FXHY@g*BS3X77-O3u;i20r&k-j zO;_6^anVISILbES)B$5~_NFrq-My*1O9FGnMUWqMNTvZQNAh6dYrM~az6^GHLzgc zE_2w>BgujgIqRCgWjh7}y1hfg)|Vl+D2`svzu{P^rORg~$&KJ}t2zS9(2>Z^XDBIdRAl3^Ewc$O1B`9J81$`ZqePKp2LPT=rC{ul8~0h} zp7UN@Gg`3OZc_!p`L}ykN|vUss%Y5O?iS^uL|{S!NglP9k>N-+A|})d3E*R%w5odN zwQQnb*$YAdz%>l3v>Xbtzmh4Axg1ppuBusfkPTP3w?{$<0kP?tvT~&M6uVTT6V&@r zYk|o!m}F$mcD5?Q$iR%$x6pZ4xgc$mkbUTmBrS(OOm(j|tZgQGbftKgtY%>92Xn~H zbXL-VfN%wKa0ZR>v}3XFT5!oD0oid`UdD}F(MLwPVe8VQbjKp8+hwJ#0IXPp)K_*M zYEk7lYr~dTBG@s-Dkcvnii+mr?D!;7+PIZ6Iv-l;IqXsr7}3{ zNxL9WWZTw|3@TJ-G~jv{#4Jj`D7^p zf$C~TQ@Sq0Ip@-n5$5ej>C~FDG%Y9+BIBBAfz|Rm^{E0_;fYDW8KI8H;X$bOGIE*4 zScw`%4E=jzs6z$DK^VZM7vJ|^6p}>VV*oe@ijGuh*OB>91pfeORgVZu9+{^^1Exso zYOHdgJS|*|91^3769O`rY>u@emgss^iksW8skQ*P#V{+FQP-Znl?>LQRGfC=t}K1W zS{*>(jL-)>EeUc+r^#U)>#{@)wWV-_A1J278)N3_+K?1r(yilGTyDtYJ*rzPSuLgg z+4+0cf<&Nu(bE{|Km#sV5!$1bbMtk@NXMQjc5n8i_5o8n4;-3V9|{N@(~&lTgPLpf z+ebl--<+J_;(!lx!K5my-kzs&?EoCpLL=Hsa0e8-3kV&bQLqGiky|h>G20zQaq^vQi3B@9IOeUy+vZ|=(v6{dijFlR zK;xYCttm^EMm1y25)0cOw8*()syg}_vu$i4K)B#k&GWFwN|68q73i+#p2mzHIbaX?wW7@fE z1NLjCLUw|3E2_7YNe2fc3dKcpYHH;9O4e!TJ4$+%Ak??^%{8eB_mu7f)|qDO6z>V+ zCvdCPQG{9(lT4|ny1l+Y!RSZ57ppepD8MFuaC_6zF%;)--EN|!oC664stU@)%U5!k zB8Fd=ifeJ&kz1!VWQePV4BA(W=A649jEssK4Ug$av6Owlb3hV; zK%*x#+-I*^dM0QA6~8)1Ld{O-)NTcjJv&nY3hfmZ!cAByo<%WGj!!gzY9~C^r9q5} zY9hB%F;gl92X9JXAs8SNN?A_>oK$ir0F#<}aJd429lP3!k3OVPKpb?WyoIuVr9_dU zHueR%?Ni&Zc=i#~y(vZAg%u+KFx*rx%j-*zm+L|hn}Up*r*$i|n?nvgC<5V#6O}x7 zsUS>b9<%w4c^q&Q<2X~Ks2ByJtzt@pU$4D zz;ZaJ>>!c_Fe1U*$Kg@BoOP=#pq|wmAZGNS2m%&h2>fbQUcgiK1s?R^qa08KYP(yW zb5FoOoj3pvG18H+2OOFJib53dDhEKwN$b|7IcDOT$Xhw-Q*bga|K)AOk~s3Vt)vf-t-uYhxlkgdEb7!Ob~XY&q*q zW10Y=b8IEa91+s1z}khA?~U^w4)t!og=#WVYy~hzX;^n6w47bPm>8>f6~7Y}jY;c@ z=3dyN`H>Qhn!1;_YMU@K*osMVvYWo<#Ihu5y}EkUCA*R4mN#GBf(kdNs3JEm8E#ow zcV!6l_o=o3xE*S|$s9mo#YR^Kf*CqcbDncRi4nNwn80<#C}1fMKGeWP#(gS4M@o-x zPg-yTJd;2URr#?`VS~p?FbLzVAUtwt0zOMqlnnAe8aDDs`qNpC(M$<M%KWc+dBKy zkfDr_<35#TS3LHh2;grec3|VEr{QGDanq$El1~_@*~+I%U?@=RRB=Osm{sJ{5s5v& z3Rz1Q0CAH@2A9a&o@vSGdQ%w-Hb~~AL~MX~_n-%WPCY6_P+I^|8mK#XH8D^J0~qg2 z25rD5nm{6?X8Cxi_pfn4120N2MJjy8o>w#h5-DO&5Lys)R#QRSWuIm2LM@u&>fLxqn#;Bmz# zB!-pA9SV&1H4!Wc;;gpg&v8r$38o^s7!B=1BRousa7gP`j7mm3dQ@UEr}|JbqS>NE z7dgjz#MJG|s{zz=T`3%3ilKXKB#h%3sgk;|)3%}EtHqWB1Ah)HT4;(&RCLa32g4Ah zwd035IpKZlqjhDt%kjIKNzL!RQ&2PLG-DhV&I16tF^i%!Ey%SkQ=@|E2@>c9J0F6 zfEgTdO>E|#%f}Sb3gA~kn8Kj)4k@G$N><>~fssfA37#oJ;NpcietT4al6e#X3jhxS zozQ0#_Fi+=mx3q&-U;S_)fpIL#Q==rfF(I6iWm^NCXq?bDa7%LU`5UcT5vcV(&IFD z#Q-kuhLPL2qa}_9Y6_eXywU(k2t3l48@b|`xyLlDp_-k*c){p;(v-sW%|@dH0C}l* z1J;-Z+<`zqI_ET_2chXoa;v(40NmpoQ%W&&j+F4hh7Sgwzy$<)&;%oLoSufKMnE&( znYmkoLMY^c(vTK1J!qz29>$7ba}j`UI(pTacvD3c5dyFoIj3g^iYkCds`7OriYNil z4z(n`MHEs2BI)?l%|#ReUJ`uc%~_D0pk|6F7~%ISBf|QUQ1<&?h=xvrp<;2?iYsCSJdsK|(M13|zvEIGD4+(KXYmRsrU6l&9*QXeDxN9C z{IpR_2Cv-|AG$uYQAh$a)`EC6Q9ujYo!*Kl03A(9fu596Oa&CJ+KMTFGPt`tGk6Af8QelP z|L57Wd(OVv-P_gmrmOqh&*|#wd(ZvW-^IU3G!k`XHDxq(bTl;de+Sy%6*NUOj2AEd zr~Na`e+>&43kwqy3l9ee8{|2za8u8vySY&Smg_J4CNqY+@B|9g2Dq-b(z57;Xjv(6&98fBJ8 zgmNiT<|`8v6AMTlPGdH2Lsr3L*?q1TCxeELbH21vHSXP`nY3QX+2#ryF?p<^KS~}a zdlH{gJtA1@`CJ0h-pvV1VS7a_n2zaR)F5!nxqWd8uoh8sZbK$WXuBrxHCVhb23N{h z^eZt((8eFg>;l~}3V*jx0Mj`+(d_n3%&Lk+4KbEUf`#&@0b2vYbPk!zsZ9`qNBHhj zWQcd+xuBl)yFm=Y!cMqrd__lX5p|XSyT@cc&kfE%rCk(MGc_o$Cx9Frs%9vNwm~Sjsrpy>DeM+{y^jOD1A4Jc2bf!Zb!$V@3VqBp3GRTc<=JGDP?A^a3H>QC4)%3@1A77veigoMUSxIb`ug%V68uj1|V zz{L?~!8A3M^Nw3JSd?aXKh=h0-a<;{2b9RWD~}Kz@0;Ba%&o89Z~G`JgbBs!6x&JZ z_M$I;uVBG+GZ`?7;~iq!f=0D6_2U!QJKTinPbwx71s7*u&`m8}OS2T18QE`5G`g;v zIsX=4S3ddOt9N3C0*gyPi`6lfi?i3*64KLLhtRIJO5YcO%N+&ui-~P^d~fyx|4@a( zf|&Xcq9>L8p7(vLlEx6Cd``wsr07rFH36|l!B^k>ha)Xz+9lrmvW7t zBE!?CyxgUpbtFBuBON93Mwm84j*Jl{EUpS|vC3cH*H(wu}F8xJHKj$x2ty>=bl0}qwjI><83 zHE5VtG&};v9!-}5`xBSnxgxj5d&H5iE+FXLIN}s64peDBb!aGi-#cZQ1eS(Q?l@l4 zKQq*t6G!u2<-gHEQW`2rqr(*4w~4}K=ao$S3FmYSvxT2jHt$_H*3Fco%P0M+Whv*m zy=cfJz8a(7$H(&jR??2n~ER3Tj3BV}qWf{-aUb!8e7H8=5uPkSh+f>JN1Omz`JF|jfA z7Xs5JNxg>mn6_~b<3+XjDz(^q|5>3}AwG6JELKdn6pm8A(=PH%+GH30A<}#I^0Q!y z#Y5319u{R&Pn>jjf5zUHp}9|(b>rqn8&M(B`U+I5%coM{FS1`ZpB2$kY>1O_j|`k^ zL~6J^jkd^EI%&*X_h#GH9CqP-$L^_u!DjHNgsLTnnIgilOc=J(ifJ42WOaEH> zHt0U?E1S-|R5Sj_>WF0q_J@C;RTlDDoot$iObjXrL}Rs_#V*cxy|igG=D2tLc>JJu zzB1?@D{B-jv01g+pCo8H^qS>N702N)g4tuzN0bLLFwCZ`?7xsbp0UXzN{;3>k2R@Q zZGe!okC(;UtF88@UaVL3rv(XMKsXJL{R;Qh=VLcCcvUPN3bcevMneY!;^B=mS`)az zt9t5UT$0Y#CU+W2Vah+aKPRkaSmCU5Kvewqqz%t!=L~e+NcroOM`(vGB8Mf!874n! z>YCfYn*ka!58q+avZhsrN%f~3x_oDlvshIZ9H&a9wtHx7w4P&0i5ETw- z#7mjoa*!j7`5=duTXQ&qdTSA}Y9V)LSbjGVypYhON+jyDRT?5lfdcc!>V6Rx;NeuP~oG%^16xrO`V1DkP86%TYu-d#LzL-l+n|?*yF&I znsRxKobw$TH3o$Zm`i-czk;PkVl*^N$;^09tqD*IML>e80S0QRIG*0hG+Qj#r7bOt zt}&FF-r57oHz_&pbG6Cd3|t?q3-=_;`)do7XDkWtSeY_eE*Sum(%PuF_Zj;tKuN#z zq}FnpsV4u!V1UWH46KMO3*!iI&RdB>^Wha>cJ3a|PF?%OCaMCVTac~(Kr8;sk_CVTG<2AQpc3xI$ zE$q;1M8GMK3n3PX*n*~%*jo1nlGY@6n5V9>zoZ+=%8?lJH|)Bz9)C0eqMUs7<$1u* z=@()VKOs+4DD-1z4CYf}Ix#KuzTjpkM4AKL>K6S*AVC!^7`9`enLT{kf%K=>4T_y* zcSN6#hubW-j``Q|p+9R0&8@eXhxEVDqnsUHHw#Q?r$=O|Z90A21o0gY0n4;{|MQrB z6%TJfo_Np{!=0sVsUNM!e1Ac+6$&PeI1h|XqGLty>D)Y=i#>lzOo?AZG~|C}^vrHv z)y%O_*>JFM)Yt?4vrq)z5EsXd=^}?Ko4@`)5Fi^zQm+xdb~9 z&s!?XYf<>|@(Xv{vSt>>5XIJSDrt64x{ZpST{<;%ASK%*j^$u``txgt8`_2x2Qkw# zW_LlRujtSEr|Wnq^QHklys1*N_gGIDuC{x=L#28}Wi7^Y_%oReru9fZqJxYFcthgo zj3oc6ey|1XR;;%}M}}Cw4yG19NJ1Yz5olg+mdEJAyzaaFey7tSI2WP+8C4WQV5Ug7 zED;m@7$rUhh5eUfH~Ln2XrJDKqbZSkj8aEMciP}PRi!IM`xHN8u=aB2$7xwX<0EfZ zoDKtza2UOPXB#eG4=LaCTgCqDwTyAnI2%hx&5FQ&Q@{Be$rO8*P0|%ZSrIU z$n8b}3ya^cD7PkUzemOg?iE-Lb^`tsbveEDTr&X4FdIbK(^q6m1jMpPj4Uy;sS6l> zS$+7~%HC*;;aCq*41L`#rvT`E(kPK9`URgN2foS|Tp`S`<-lYrH@Rc(+sf~+@!xR? z{w=v8bsv5kLD@6mv#dGfY*}^+D`6Xl2|5fi#Yzid?Zu}V59N>z zaXShedORI1HDErEOlJV81*9;8uZOdEcsy4c>B_QM9v6mQr44WdG zIf1~Pd4zrNi$@Q)3PlcAkIcmf+)L?7jXmnU4#iCd{}Tug&zlr0Zk37YlQZK-Q^&W; zyU$%d*fC{eZ+$0vWGt&Hk1YHLK2O;ad<^U9&-| z;gI~K%n{_=BE_~6NNQP#INLPMIc|314bc4d-VACP@c-DfS+ZblZw?p48DIJX>V1-@ z*v+pTVHIOy=^9nkO*+G&Zg34Dft4%-T7v=z>y7;?0{82@Ur3(F{JAtY427*E;U5_^ z!80)}GB@Z3#*=yG1 zN&J^aL1Ucs2j=j%7-&JD_Vd!F8&B;6zW!93A?ZH65L(auB@5+r2jeA1%bGyR)X`Z9 z&uZfg^DUdkJ+33L$3DoydQ*m$x>zjqgw9F>%pf%2L*;)2th#A5#Tbt)3H^RTK)kLa zN!ypNXGvmj2=?Hq&(O^Bxcou`QNPMmsMO2UeYH6lz6r3zHd8A(tmkQ*zNEhA8~O-B zu32j~i9PbN*tI;Gv}EY!jrRRs@6wbOccNSb-e!ViYWgiK{8&=LEyaVUwt9{StXJ#)ZHUX-sbK0{6;}Hexq8TiSuu9ozxtM&7 zSO#rp`01Km#ya(n_e4g#y~M$ivDcI=OLI+J{5OVq@S*D9spa`(2g+O4G<<6u89G={ zNHOTF(nQlO^Rl)fE$O{oJJp6<5BnV1u<+ySc!2C_S!Y7s+?zSlH?c2JpGXcRH#^;3 zvY~?h3ID6Gnv$@rI%C z0oCFpZslh$!p2I6rv|~ECXON`yj`cXv>o-sXOM%tN-mnY4u)%@S_7YubJv_|_S}UW zhz@!Qedl5uS(czD@_#^xsIAk|HY?Hh%p){l&3UDEzPtTjbNy`cGmp#?Yf!R_4(p&S zE6O@2^h9NW?4`R(myj|e^C^nP4UU4NAsT4Bv=7u`gkqB-L)5P>k{A^P3M10>R?!X9 z-jxc8{fkD!bL_~GD3LHR-$d7q{7PIPvUB-72j-e>kO~r9;|-}dl*(9lN=mxCGsx^s zQ0b-h0A+Sc&YpS=QVI@coVVy=4sqdQC8II*SXOSkY5N{^J4T1s z?p|!8US0?t4ta4$v^De>jf_ccn)C^tv+X6v#khF2H`}EYNPONlXhC)ktAUh4O zw*7J7nQy*fv0uyqA*Gj;OeX0Cm1st-&f!`$!N7v!`ryx@X@g20l@dIJGTHC1gTC|& z#x*Tce01xc1o7z#j*-2e-UcHXLD`M@IxTa-3#4SL1^3o#BQ-9>A)pa{b2T`_YJ~TR z*{$2+zMJZHphuzEk^g@QE`bCr#KiWV%qmi^yHXb%9L?6X zf3808C(5!4(|c8It7C<`Hw|{UurT|*k00V8OroBWTJh%I6(=zE(n)Z^tGP7I$cV#O ztv|NgXGn3jRU$PZ{^u%W6v4>i%WC>7<=Y;eg7Q5EW6R0M$brrn?o3xi3Hh76PvksW>yzUuHa;h-4eHL2ym9_8OvDp= zey2l|cX9PyIoHp{?kQM6R1S!}`qP2PPi?ZVOmE!Rq!4)nwuG>uo$|a zNxQ(7&R0_-(->Z(uNJyraVmcnb6|PeFWgE@fRK>X z=fSy|l@9D3fD6JFz`mU1)e+rU-8)|AcqajxWo*4afRLUCY5zI3iuIQ-adsM?>HXn# zY1MRsuWkYu^^1Jngms`^bc49u_>6M23vmWbZP}bWwuF*lFTw=w4a*$-ozh1InwxrP zq!V!|_NZDdVmw@LDfiS{Q+ z<&GdipXpLM^Q=j;a+UI)8wOhM1oLo)Xx}YYCSEz(!-o3G#ibaG`){Un#+~~E&ih~j zpk9Nctr|7C3KOTpe3D8cO_|`P3eKBXWlFZO%fs;SIc-n0BEGBSV+aw2N{uGlz$(={ zQxk?*S$$_5_|L`F3G$+4)f8Jf!EY_yw>44{($zjqdHCvTY>!*BW#4rZvOd&PK0hOL z>OcV%(;3w6ArXiPu*@AxnZxDkqWtczp&;P3IOZ+X>@MBb&DQ?}$Qs(3F3oAN4VSif z{X9d0Z zdJ+zANcbYd-rs@nE#9_V8oIG=yZg+wzS_~!Y-P$yAAXd53lvVUQJCCe<62P!yy}vX zL=?!EnksqF+gbVc#LT0fMM~gq+$ednBIV+E=zD~W@#Vpr0rx{ck%WXuSl;Ym0e7*q znDot0x+zQ(-`MqoH!%Aw_jtn{+0BkLlk>Igsdk0<*J#UNoFuAHd>ZeD+h!&{qm-@BM>72x$*w4j+|q zXGxu)xiZmkF;@N1N5FW4hH*`LP2>+H!)=MTX-Ba`r0S#l@N`g{;H+7%=MPHVy8pIu zoRvJ*U&H3wKa{jimNanVa!CfjI|fRCRlmSX$>Ps)t-;wpzRPmT)%o(Dv(re&Z20ThUbsw#ymV*2r4ATOQJWBh>QwANEGVy7wZ`*vXp(=Nv z$^jqAwY6}25)xzfsVtcbAv3C>Kg#%PjwXuSZez*QgQ{G=W1xm9j6MRpq`qadrrJ)G z>&0azFa)Siu}?X?;JUd^^$-ZCB-YIJHL>x^6)MtWZ`#Q#F#n;qWs< zCqFBZem{A`_a}~N$pKbX41$x?c1?{iV?~1clRg{6_bjjI>bnDdHqKy{Y9920ue}LM zn7(gSQ0)Ij!wHnVuvFq1mNl!ss3sCJ4~|1h={G9pN{74ICdwxRLi$czN+)ZxFoSgi zqswV(ShuK`@Y6dcZ-z9Y)@NAaxlXPq$o38PxBb*(HP^2v)JiQ%hd-w@-_*PyrjaHB z#X3L6{TQ;&$ox+ktnv+a)vb*|rStq+aM>H1DAT3$+_4c=LnefAICUg{jMnjriUYXR!~_{WqKTH~mP} zr@%i+4(B0p&$KOJyHejEHh#UkI;zV-UrU?%L6pq z4&g6-toF^nBI!v$Q_>=vi|rZ(8g{)BJ5BH22l$Cg!+jzHE#@55%1$+C^oMcvJf}=X z_4Gez&z#U4HToYvM31mjD!qOwG&KuTZmo7Wjko)OPA$?e8|m(}mZ~Cp=;JqYJz<>G z=lD$98KAZo^%w1@g`1SxfO;u!kw>iL2~|IiPC`F?v``Z(i-30T$p^Z;Ps*Aq39H2v zy6$6Kh08?{6?9U4uT%2yqK{`wPZYR^%O!d-oj`Hd287+d&7XXt>XbY^G1ZMz-AiHW z`-{c^DY#sI_eNDifQ}Vw>i(*Vk+bfR@KSwLqh%Fh$Z?N|+4q)xGtDOVxjqp;XjggV z<&Rab$eKcv|fdJ&NRTJxgR?URk>#(O8Ve%(?v2?G3*B@v{RM{bPQO zwPsZiL@D7LZ@9BNb_myXo&xUcxh06_zSv1`J1A+kJ^L@3W<&O~O2SqWVO~NHRI*go z*n-26>+RrB_>(w_=Neq*v?oMiMNlog^q<+ylx3-B7hOY_=<@PmFJ?9eo?>fwk~X** zN{%?=^<23-u}ib6-WCQ$NuLXu6URLniq_j&Q>OdzE^gxQq*6x>xvjTCJd9>q5kHG& z$Yy3BE(Nyujoj+;#{w(s9W31j(}$A9>qle*Hc4dP!W&JQ6uY6=wVm^x>9L2K5|n3B z6hC|@r#?vqi!9#h4r&Gj@aB0r+Bin5D2)+oIW#TuPPGR!Iy9PC#L>+58@@~adHnR} zgQwPrpO5g2aQ~WWcPN#TrO5?TwA=oTsOX3jcMTwtyF8)hAoc;jJNsp0L|lii&sa(~ z{iPXe{;TqU5&ZWo$u0Le^sz}=sAnBM{vAB9-nE@thWqPN$nJ# z56;!wy?UAr_lM2HjMuPYd+%0AExR7iawl?cM|?^KG#v;+b^tys$Rfxs590kqj~dX= zVt2RZwpvlFhU_mFt0D=izskZqk^s}qV_DZ9u$MMxtragFhsF|{YuaQ4C6FMGcA?9b zRvw!J|Mb!b?IG!Ie#=(>&DQ&;@Ur{1HI_l-sGPcivQGUX@=0?e$5cjCm0eJ!J=Hep zt4b$eLRz!vKWQeOq6U<3xGVp#`}!ZytXG~D;PEA;F^A|V&KWsT%YmijlQM#BPFls* z1>`WijE&|kyRKV^L{4CO~==PiL62+ z4$79Nft7Cb_AeUkk#nGvtzSt+;wYzcNs?p2t!npCicu>?65=z zeK{MIoxZvLNb&qx4RX!PL#A6*G3(I4(Ko*3*EQdGWmESKv$g}ZC zXPpF~fB096VXHV|#sCO^wjCj$cDvIz*n#yXOrfDH@|-FTG1sJ)OY-{-*^Kn$7Eg$y z3Vq+iOABGp|H9Z6pYmAl80&vv&jC5gHEiJ%)P|MYPr5BWAaFB&0kx*At$i23-c z_U$!IA~t`oPJNQ6%<8;;ornCFvGNwY`j9tE_b+d$jEC;>@oyzYoMmX``fL&zFAK zM&rLVUz~DyWZ5_=T>ytAc}}p-lBvE9EQSTfJ@Pl^+uHtwd_R-?Q-EzJm#_Y?)dObN z=wKEvF|RvaiHGx8D4!uEk95j}nj)u|;~pD+H3`bCohA0hc^Va5VV*)l2E?YEiq}3) zDyT4?zr#6F1Jq;m$#VMs)D(f|m0^RgW2`ilD7s;hhl63K*7W6cO|EFN={t<9cS9Av za{VSX3)|=x#>PcFYiixt?Pt1wieL$b2HBro6B7k?z~QqU>Uy&VUu4wl_k-M+qL46< zn_pQ+kN%kCzk)rHhaQYj>_Z+E-OnXcyhM+C!^vfp{w`|YB&GVMZ}kP-m~Lq;p(o(7 z1`#PsCI{?QYW4(@FJbbGKOhwwOA{wd23{6y@CGM(D<*jZp zxhui5F>JqzY?|%#M>!{K(wUC%Iv3_e;Q}etD273@+zZ^^>TSgdQG)L1%r(=RoAeHT z`TlpWZXy~#*om*t$9$y9q|*amUx_4rV?`vS@r+h{!e>tQ$bO2OYxm=Y(khR`FQIFx zqJx@_h=q_k%K@G*1GF`rneK;a{oX=tHovBDDE8+2DxPp`B_YfCAuB5lZfK_NTg>&R z=?r=LU$=ILH}ma|Gd5Tqn=x{)%T`-nCsLh5gj;-gq`=x+#WLVZvcAVASGQ)u8olE) zUT*8Gj@GuS(>JQaj73es%m+}vau@o0ofm@shJfmiH`!K}riIOn7DHb2iIN6m>xOn8 zU`3)=G9Skb=Lyod-;Xx;Y(!s``yz(mT8kgbim6M)^)FVWL%1&Dx~R;T%;dj(c4q7N zPU#oxDuWKu?3cwya5RcdtDku6MrDh-S5ZkLeI5*QOWJCtG+I!rCOqHIg_PXvtN6p2 zyc+wAMuxc2*}*ttr~>yTDMrhxMsh|mVK zH$nNxpc&!#p2;>3ovfLRrO}_nU4>U=RBf?~-T$ke=aGd+ z-0kOg2NNE9y}=lD2qw)I6l=C;TJs`n<0$e5i72@9*jPx4 z;Vo)x&Ne22Y_7QD&?8cx%&0He75^nfS1msb4 zNVkc0Z}aJb7`s*&N;VZztm0~Jh90@1{}&B*a?QYdC~GMd=gdTr5EbXonwg7xrDxzy$a;Jn^KI@a^Xoxydr`%8!bG%IQe+W1KQ^1<3 z9cKC;^UQghpqh2kWG5bY%~@oOybdc@m6|O0Ocr*Pm|wkOf13gIEHGCg%d&n=v6fiY zx^Ix)u|=T7DWj>2`f{Q|l(2!{0nnOr>OWnIS(4bRh2{i)Mt!|&mbaf)B*)dj`funE zz_htu?^b;mpA+^%mPohM9WKb~AHnk)oAhhDr|fl}!$ru;QatjLf2rQ@6fVgqrNh+} zOReVt$-a=@2}O=^=~S9-Xtr%*GaOP7%Lf3}K{0+;LZ|i0tjD8+It^Tx*BM~W;`-fY%vfMq0#HTOBaMwppDjC195OD0mnF+P0U`Z5&W zj$j>)tKI7&A?Yg};kTmp!k9a-kl5m}0}vAkCy$=DupSfk$X=gWh|R#w1vg6Qraq?% zSqk1|JoMt<)^^mVdIv@dsDAuiF$sWBy44A#2?olzNDWZT8)lN34X@;iiart`3Rdp= zlrq>dP7ln!zb-UO|EFad43yuTpi`WIFwcq1&t8kY|LHU2UU{`eWUr+>eH=3+^G^{(CBz!OC7yY!Wj@SPagA%L}=z2`S z+@4thmF68CTNEuUc)EDR9qQ@=Z49Lxg{E2!b)C)4q6f@jD{(?!F#g5QZ+GTYk`oJE zy?DiZ9zg+~ic;rLLURslDw4Yg7+|`OTy~03<^K&&L~0022NW0p2y6LF!url7l}W zMh;B^u=y#M%He+jcZ&14?M}L08TPz=ZMtL7?^a(ZW&bi&v^0-*hUHR<$rP)kUX2}m z%8P6-@GUo}QBQ@dZNxyFg+6!byfbGjALourxf3q@LGbavF1TF7v4}>wO{zpm0)=(D zdHVSf!`F=-M`Waq!D6fbzW0ONw~!6$c^#<$AkmIEuP@p)bGzwH3aWPgC;Nq!K;-Zn zO^IuYLq-u|Kf2YG5r#f2Y-1i{3s&oSWvwQR&aLc=l{0)}U&xeJ)bRwv(7Bvea0?_> zWQTKcH&?>eoA0<~EAE43FWLUBdL)lDr;%Dy+ByUL+!`)2CLsQ_KTYBq`V0B-BEG)wV7c2t@fNVE*7?nI|= zyG(_Dg(;~mE1R^4R#9c6~}It+zt=kr5ol} zrqwj=(Nf877uN%h<*QkPr!&Ir=N!>jH~o$^9 zIo0L-o$SfT%4-qX;*BL)gmX=VW{yYY$fTqpkBUpgY3!{N^I#!z5w?QMsJJugw>}8< zbwLJQmFc66*%_ENw^ zgg~_cpzs%okMNj7IRa^{^2PPV-v={|Wso+ulm%nTcWzBCg>Rytu)CC=iOft6ywM@J zYGwc3>wV&tvW5kQJj(f+j3VAQexqo#!bUakUQ0 zEsV0{8&9VK#01I7ILB?=^v^`>Pkz-=Du4beW!F+26s+cn+;XUf;AN|R~*t>1kNbgOqhV`O|>zusVcI}M-RH2ropmS;hu|4oDQFWM~H zkOa=_q=E^r5V9vjw$22x)DmihM^>Wd#}a>Hho?U%#0kxopR}z=Q}lICjy=*UC5mf& z^+KNIs|)~=R|J`fs`q{mUyvmPS_q?f7IxD%Pc$@ttR)+POt=rI_fq!0QYGDn&>lJk zGNE2=tWx&Q4r-jRu`csAJPrRC#2}qVKT{4xp`Jbvwug{hZBH}=VAqzZf668csJnL2 zjlBwD96f}~SNaAF=sIYBh}QoRUL3x7!w7~=um1ZwzHhZ8RNOfn^&8yOA4%#A52!qR zXmd;>CMJkNF3JLVals*c4bXwGV`JMqBjHa(qQXqq!L9A%g#A&KcDm*b{4=h zB8wM+-k`k!>7O5c-bm;k656sCK!zyZIsz?xZI{t@4PJ4Mtamd({1I88Td^(&@1|rm zZl{Lg{H!D^P#xk)m#79~rKnyFjfJk3{r8cDSJf)pU$jgBRn$B6N-t^f63aazJg#bo z=Sn7)JX0573TJCoPE@l^beF)A|D)t`rwqt{qlmlB>IV3fJ(#5arv0}-=%FHM!=s^m zxzY`XYP_}e=ZLm%dvmkxi-Q-tQtZnHN_p8TgD#k+Qz4r&O#NWKrin@yl9Mo_mxNNx zg$gt`>HW1$lk_33m7BrTzG7x5%ZQUx;XD<2)0#IE!xt}`vH&=wuzp#C-ZADkI2fz~ z|MYsMoAys{QAq>qj2-nep4DXJQ$Qd^wuXpZOB?HN<1% zFIo%_%$H?Ggpz+S_CKO3{(f(fLmzijnbPMBu~6~lic($fj{=q=hqOVbNk6CT{^fNz z)^xw5zcijkbr>zgBinKVTMrv#6JQJ5GfJhydXyPMV^+nCcABt&kSLZ$)N^WMrR-yB zPPo8^I5ahw$~WWEq$yv1eo;B2f(_`=+```R7p>dm13bZxVFK188iCOuDr6DK6^jt8 zjC41fQTg^{eLh*a!AG%vUtuA$sDmwZCDP-d`(uGV8c0nzx^M#>?7~+>(e=i8lvm{` z%-g1`aA3$K1O{!I(&gY6Vm4l|E-h0;t!D2xW~$_qLq`IHUOIaI`8c1v{{GPIBHrT~ z!o2(ZJl0?PGj@sQjZv=kUo@yiTdA+1cIzC?r&{hwzh1Hs$*b^3>E9z8BWEt$OE5!y z7jDi`Uh${HQvB>egPArk9c9LHX}gQR0Hfkti-opwQzIaoiEzARCFi3G$Lxz9`AT}4%7YR#@)+<|3l zDZf{N28|^AKbExX9>0) z-Gou~YiPsFcAzsHrM`AO!9Mc_&b+5Ri}e4=)Y{dUd&F8r_9O4&O;LF z4rHQKYJVQr+@+y6k?spwst3~Ed9D+E<6B`(9fzFNOCn!gUEkJJMy3g87b#>oeCu@Q zTjK#2%fiME|Dutclw3TIxpDj8=G3*=P6lkj?7su6i&H;(EZzpV)bX@P^)06Z1VhLW z#88L0jj>6VfH=jxfq=@tXn2=#d`yIDb^+DDnYG@H`)T#9>QQCa3|zxtClP1TuA?(* zY02*5|5QC`u{9H?17HYkPHg*(&j5&UXPY)M?zOF)06pc6coijOn&|#^-g}o+6*K>Z z4m#BjsjcdL^zFoB{|z-yX@OLUzyJ4!?)$drstqD>zO>~~xcJx9@ukf-hSwQX)Hw7> z?o#>;*7{8xcr$h)ulo(d4P3k^?6WkTST*y%7o&fb@<536gV+x*kW@`;=s34MzkVxV zy%_mC9QrVOx^@EN%-+O(9?Q!3yxUzok!eq@H_VK6Jp zRUXTCrEwI*YZ{l&q19ds!aXfn?9mZ%S?iCWK&#|vBNK;2_xd-M7X>me@OVy`%Fpvc zOF(6@%U@zIM0lDxe>A?fkXTig*!Ivs@ofaH7xHHDAcY*39aXgDmM%r|+9^pRM(S-- zw(xfaPqipDTNmjrq|7--OSi8i^o>OrL}j8a?x3GH6ZD^?D(>8m`l4o+Wwc_JK{sFy zq+qU z%EhN5rc{kqtw~B;Dka2c_miVHYa*361z9;3Af0mQcT(gn;+b&HD_{%+d|v!nyVgf1 z8ABd8>_^>09CeTm<+`iiknl_*{&u>lDTqCO@rfuTf(W*Fx76-b?%QV`_OPco9VkSx zykZ10^w*VXZ>)RTU{;-Ts9i-^8;o-5j?<}+(y+J7$vcdD z!#x8X#b8t`Wcz3~vd`h4I6+^2+;ZN@M^gr?R-<&>{p<({GE7zS-}@#@a3m>kDIBoH zYMp|Qo>c0;*Gv3j*;litk<#0hRlqM`w1Q!(;>`Tm-(4thl-i%vQl46w-{MQLx-;pH z{k?9VuJ2f{ZhGsS!A_~Ov9dCt+9s)AuphBb5bI~XH*t}fq&o}^4%dh^HeO~M-R5J! zvP@D9B^O8#T=hIKCryoTj0Y&kuiX<<&9rkLNQccS#m*>1(WPM=m|3(+2bA1JSPlRg z7QT(PtGB0EYD;VU+vmprk$ASM&z62#b{ms&K-MEqnKW>eYiLA_RxYyr1CSr?$F zgar=6qDy6FF5$(SoG0wCq<3??JcSs$7xQ10Kr0;IAe0hMq2xH3ev(Q4H**_toa*FF z7@vpMPCm|i?mtg3JEAkFve0WC--`MA>usI#XpZH_Gm0-QP8LKv4RV(Ea`8y6cwS2YvZJRJS6-oxkWTjNzT=>>3-_yfMg z?zMYKQ?58UIkT+)z#XDqTV!%XsPJ(+FYS(AdaI71?4!n1MEe1i44p?jzGan8gz3lt zP$;=GKe_h}Bdm7QbY;RevZEn^@gu9kkV_}ABm8%CAQj&Jv)N*EwooSri@5VxevKam zit3ixN4uz~<;M^{gYsXrvj{E~qJ4C^WXN8xC@Jv4YnZQ7_pPzM3IXobz8v0CV{N$h z+)O%!tyxJ*N&@&go&85ZsRdugOxDOS+V#KXF_yHxeR5<(U6kFZeA)%>CpTNn!d6V< z)cF@peWrDxaQ>LhKDm*IqWF!m=4^?5|IMeOj{1T5U-w^SG@E|{#ynHDZQgB_RFvDK zfX8K2{Iek|cvKm?M77nc6I=sc_NNgSf2g;n6VojQ>E80(nzko!PVTR@FsVBRdr}E< zPcc`KjoR$7pnhs3k!Tc;&yG2k5j;UEFNymNToE3imCw)kch-FIODCUT^_JXp0ezmo zza)xL_uQx1JLcN1y_|d}vhQ+!v4_-*^2(V|uxa}>|617>!WtH!fbBE%!Zs0URLesd z9R5%6Xh;?B=^LnvOltIX|NSVRvFX)n*d9~7IIb|v`<-64C-WZ zQV{KUw97A)1#@V?Uo(mRw#tj8743H{#|q_%y*gBW3vAgNbRHU8r|)W-7X}8(1r_ci z3`kb*fAQ7^>j5i_ip`i)YrgUmsPiw4=zt)IXPnjLJ1!=wzG?^mCq!!&i_-aZ+L$Z6 z&qMgL9u!NSBQfE~_%GV46PP90)5m!)d^xNH(L=wmJjO%ZP2HWm78`Hnc+sA2aVeo5 zR`^-Mz_;`0==Qe(M0v)X2tjUWl5POMOo^M0->Y6l+A4M?Xl<{o7GodWw6tKSN%YIjTbDl&ez^1w&-_1))9sY2kE`M+Sp&`qZE zM(UZ_Co?lQG4ww5CJMMlrQAaO^)q$=9Kve&eB;QZzzgOH}DLam_K zC&{@f#Y=s^ou!s6aIP-p=QrI9O3rWP>oj5vICDvJNn^9=mB={vKe`?0K{Chhbt|{$ zx47FfA^dVRn4^>lF(qhVVg@%Y1q~YNT>3fIl&y3_XCa-zd;rs6)mH4U-*2T?!G@I- zdm#+6%!=o8J??7~3D+;RhMe$2UOoH)%-&)~mVOdNa}NR`q01NpVj+~zfOeChc}4Z< zA2naCpIa3qTn>y!omutrTXz2yD?2Zz<0}PuSz_N4B z$NE>BvNQDihZFP2M8Ficuy}(#gY#?Oi$YdgYGawp69b#`3nfxfn^asc*6`R1j&G?x z(${KwJ}H>3r?usW@+Vo}#xlsTcYM4Ux0b6xWa|#Lyte5vmR-MYYmUa;fb<`cLQXsH z4jtChKBbwALdt|VURxI)gR*ri^^NO2)8FE6IAKgsz4CaiK&$tf{tdlV>jwc!{4WhD z-lE58b1aDSDhtwu*K?iY3|+nyMag0c9Wf6EjZIaGj1o(h#f(>EX$t`EqIZHC-@~Vg73B7sM)r{D|!Jk7A6G)#!2SaPZUtc(Ag^+e?fO z^il{TpI08%(TBcxBa)!nmaPk9scUWlep+#RywGbqu05Ni1I35y|bZ@D)H-&LR7=1U7UG+ieCKI!k${{b~B BTFU?c From cc06817efa24f20811ef6b32143c6700a91c5f2a Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Fri, 11 Apr 2014 01:00:27 -0700 Subject: [PATCH 2/4] Attempt at visualizing ImageNet Features --- Makefile | 10 ++- src/convolutional_layer.c | 69 ++++++++--------- src/convolutional_layer.h | 2 +- src/cpu_gemm.c | 86 +++++++++++++++++++++ src/gemm.cl | 72 ++++++++++++++++++ src/gpu_gemm.c | 153 ++++++++++++++++++++++++++++++++++++++ src/image.c | 94 +++++++++++++++++++++++ src/image.h | 5 ++ src/mini_blas.c | 148 ++++++++++++++++++++++++++---------- src/mini_blas.h | 13 ++++ src/network.c | 4 +- src/opencl.c | 77 +++++++++++++++++++ src/opencl.h | 21 ++++++ src/tests.c | 75 ++++++++++++++++--- 14 files changed, 737 insertions(+), 92 deletions(-) create mode 100644 src/cpu_gemm.c create mode 100644 src/gemm.cl create mode 100644 src/gpu_gemm.c create mode 100644 src/opencl.c create mode 100644 src/opencl.h diff --git a/Makefile b/Makefile index a02d7ef7..07cf79fb 100644 --- a/Makefile +++ b/Makefile @@ -2,17 +2,19 @@ CC=gcc COMMON=-Wall `pkg-config --cflags opencv` UNAME = $(shell uname) ifeq ($(UNAME), Darwin) -COMMON += -isystem /usr/local/Cellar/opencv/2.4.6.1/include/opencv -isystem /usr/local/Cellar/opencv/2.4.6.1/include +COMMON+= -isystem /usr/local/Cellar/opencv/2.4.6.1/include/opencv -isystem /usr/local/Cellar/opencv/2.4.6.1/include +LDFLAGS= -framework OpenCL else -COMMON += -march=native -flto +COMMON+= -march=native -flto +LDFLAGS= -lOpenCL endif CFLAGS= $(COMMON) -Ofast #CFLAGS= $(COMMON) -O0 -g -LDFLAGS=`pkg-config --libs opencv` -lm +LDFLAGS+=`pkg-config --libs opencv` -lm VPATH=./src/ EXEC=cnn -OBJ=network.o image.o tests.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o softmax_layer.o mini_blas.o convolutional_layer.o +OBJ=network.o image.o tests.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o softmax_layer.o mini_blas.o convolutional_layer.o opencl.o gpu_gemm.o cpu_gemm.o all: $(EXEC) diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c index f7c9c102..40d58584 100644 --- a/src/convolutional_layer.c +++ b/src/convolutional_layer.c @@ -285,52 +285,47 @@ image get_convolutional_filter(convolutional_layer layer, int i) return float_to_image(h,w,c,layer.filters+i*h*w*c); } -void visualize_convolutional_layer(convolutional_layer layer, char *window) +image *weighted_sum_filters(convolutional_layer layer, image *prev_filters) { - int color = 1; - int border = 1; - int h,w,c; - int size = layer.size; - h = size; - w = (size + border) * layer.n - border; - c = layer.c; - if(c != 3 || !color){ - h = (h+border)*c - border; - c = 1; + image *filters = calloc(layer.n, sizeof(image)); + int i,j,k,c; + if(!prev_filters){ + for(i = 0; i < layer.n; ++i){ + filters[i] = copy_image(get_convolutional_filter(layer, i)); + } } - - image filters = make_image(h,w,c); - int i,j; - for(i = 0; i < layer.n; ++i){ - int w_offset = i*(size+border); - image k = get_convolutional_filter(layer, i); - //printf("%f ** ", layer.biases[i]); - //print_image(k); - image copy = copy_image(k); - normalize_image(copy); - for(j = 0; j < k.c; ++j){ - //set_pixel(copy,0,0,j,layer.biases[i]); - } - if(c == 3 && color){ - embed_image(copy, filters, 0, w_offset); - } - else{ - for(j = 0; j < k.c; ++j){ - int h_offset = j*(size+border); - image layer = get_image_layer(k, j); - embed_image(layer, filters, h_offset, w_offset); - free_image(layer); + else{ + image base = prev_filters[0]; + for(i = 0; i < layer.n; ++i){ + image filter = get_convolutional_filter(layer, i); + filters[i] = make_image(base.h, base.w, base.c); + for(j = 0; j < layer.size; ++j){ + for(k = 0; k < layer.size; ++k){ + for(c = 0; c < layer.c; ++c){ + float weight = get_pixel(filter, j, k, c); + image prev_filter = copy_image(prev_filters[c]); + scale_image(prev_filter, weight); + add_into_image(prev_filter, filters[i], 0,0); + free_image(prev_filter); + } + } } } - free_image(copy); } + return filters; +} + +image *visualize_convolutional_layer(convolutional_layer layer, char *window, image *prev_filters) +{ + image *single_filters = weighted_sum_filters(layer, 0); + show_images(single_filters, layer.n, window); + image delta = get_convolutional_delta(layer); image dc = collapse_image_layers(delta, 1); char buff[256]; sprintf(buff, "%s: Delta", window); - show_image(dc, buff); + //show_image(dc, buff); free_image(dc); - show_image(filters, window); - free_image(filters); + return single_filters; } diff --git a/src/convolutional_layer.h b/src/convolutional_layer.h index 4e69dcfd..7404defd 100644 --- a/src/convolutional_layer.h +++ b/src/convolutional_layer.h @@ -30,7 +30,7 @@ void resize_convolutional_layer(convolutional_layer *layer, int h, int w, int c) void forward_convolutional_layer(const convolutional_layer layer, float *in); void learn_convolutional_layer(convolutional_layer layer); void update_convolutional_layer(convolutional_layer layer, float step, float momentum, float decay); -void visualize_convolutional_layer(convolutional_layer layer, char *window); +image *visualize_convolutional_layer(convolutional_layer layer, char *window, image *prev_filters); void backward_convolutional_layer(convolutional_layer layer, float *delta); diff --git a/src/cpu_gemm.c b/src/cpu_gemm.c new file mode 100644 index 00000000..437b39a4 --- /dev/null +++ b/src/cpu_gemm.c @@ -0,0 +1,86 @@ +#include "mini_blas.h" + +void cpu_gemm_nn(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + int i,j,k; + for(i = 0; i < M; ++i){ + for(k = 0; k < K; ++k){ + register float A_PART = ALPHA*A[i*lda+k]; + for(j = 0; j < N; ++j){ + C[i*ldc+j] += A_PART*B[k*ldb+j]; + } + } + } +} + +void cpu_gemm_nt(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + int i,j,k; + for(i = 0; i < M; ++i){ + for(j = 0; j < N; ++j){ + register float sum = 0; + for(k = 0; k < K; ++k){ + sum += ALPHA*A[i*lda+k]*B[k+j*ldb]; + } + C[i*ldc+j] += sum; + } + } +} + +void cpu_gemm_tn(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + int i,j,k; + for(i = 0; i < M; ++i){ + for(k = 0; k < K; ++k){ + register float A_PART = ALPHA*A[k*lda+i]; + for(j = 0; j < N; ++j){ + C[i*ldc+j] += A_PART*B[k*ldb+j]; + } + } + } +} +void cpu_gemm_tt(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + int i,j,k; + for(i = 0; i < M; ++i){ + for(j = 0; j < N; ++j){ + for(k = 0; k < K; ++k){ + C[i*ldc+j] += ALPHA*A[i+k*lda]*B[k+j*ldb]; + } + } + } +} + + +void cpu_gemm(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + // Assume beta = 1 LULZ + if(!TA && !TB) + cpu_gemm_nn( TA, TB, M, N, K, ALPHA,A,lda, B, ldb,BETA,C,ldc); + else if(TA && !TB) + cpu_gemm_tn( TA, TB, M, N, K, ALPHA,A,lda, B, ldb,BETA,C,ldc); + else if(!TA && TB) + cpu_gemm_nt( TA, TB, M, N, K, ALPHA,A,lda, B, ldb,BETA,C,ldc); + else + cpu_gemm_tt( TA, TB, M, N, K, ALPHA,A,lda, B, ldb,BETA,C,ldc); +} diff --git a/src/gemm.cl b/src/gemm.cl new file mode 100644 index 00000000..7c868f41 --- /dev/null +++ b/src/gemm.cl @@ -0,0 +1,72 @@ + + +__kernel void gemm(int TA, int TB, int M, int N, int K, float ALPHA, + __global float *A, int lda, + __global float *B, int ldb, + float BETA, + __global float *C, int ldc) +{ + __local float Asub[BLOCK][BLOCK]; + __local float Bsub[BLOCK][BLOCK]; + + float val = 0; + + int row_block = get_group_id(0); + int col_block = get_group_id(1); + + int sub_row = get_local_id(0); + int sub_col = get_local_id(1); + + int row = row_block*BLOCK + sub_row; + int col = col_block*BLOCK + sub_col; + + int i,j; + for(i = 0; i < K; i += BLOCK){ + int arow = row_block*BLOCK + sub_row; + int acol = i + sub_col; + + int brow = i + sub_row; + int bcol = col_block*BLOCK + sub_col; + + Asub[sub_row][sub_col] = TA ? A[arow + acol*lda] : A[arow*lda + acol]; + Bsub[sub_row][sub_col] = TB ? B[brow + bcol*ldb] : B[brow*ldb + bcol]; + + barrier(CLK_LOCAL_MEM_FENCE); + + for(j = 0; j < BLOCK && i+j +#include +#include +#include +#include + +#include "opencl.h" +#include "mini_blas.h" + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +#define BLOCK 8 + +cl_kernel get_gemm_kernel() +{ + static int init = 0; + static cl_kernel gemm_kernel; + if(!init){ + gemm_kernel = get_kernel("src/gemm.cl", "gemm", "-D BLOCK=" STR(BLOCK) ); + init = 1; + } + return gemm_kernel; +} + +void gpu_gemm(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + cl_setup(); + cl_kernel gemm_kernel = get_gemm_kernel(); + cl_context context = cl.context; + cl_command_queue queue = cl.queue; + + size_t size = sizeof(float)*(TA ? lda*K:lda*M); + cl_mem A_gpu = clCreateBuffer(context, + CL_MEM_READ_ONLY|CL_MEM_COPY_HOST_PTR, + size, A, &cl.error); + check_error(cl); + + size = sizeof(float)*(TB ? ldb*N:ldb*K); + cl_mem B_gpu = clCreateBuffer(context, + CL_MEM_READ_ONLY|CL_MEM_COPY_HOST_PTR, + size, B, &cl.error); + check_error(cl); + + size = sizeof(float)*(ldc*M); + cl_mem C_gpu = clCreateBuffer(context, + CL_MEM_WRITE_ONLY|CL_MEM_COPY_HOST_PTR, + size, C, &cl.error); + check_error(cl); + + cl_uint i = 0; + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(TA), (void*) &TA); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(TB), (void*) &TB); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(M), (void*) &M); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(N), (void*) &N); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(K), (void*) &K); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(ALPHA), (void*) &ALPHA); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(A_gpu), (void*) &A_gpu); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(lda), (void*) &lda); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(B_gpu), (void*) &B_gpu); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(ldb), (void*) &ldb); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(BETA), (void*) &BETA); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(C_gpu), (void*) &C_gpu); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(ldc), (void*) &ldc); + check_error(cl); + + const size_t global_size[] = {ceil((float)M/BLOCK)*BLOCK, ceil((float)N/BLOCK)*BLOCK}; + const size_t local_size[] = {BLOCK, BLOCK}; + //printf("%zd %zd %zd %zd\n", global_size[0], global_size[1], local_size[0], local_size[1]); + + clEnqueueNDRangeKernel(queue, gemm_kernel, 2, 0, global_size, local_size, 0, 0, 0); + check_error(cl); + clEnqueueReadBuffer(queue, C_gpu, CL_TRUE, 0, size, C, 0, 0, 0); + check_error(cl); + + clReleaseMemObject(A_gpu); + clReleaseMemObject(B_gpu); + clReleaseMemObject(C_gpu); + +} + +/* +cl_kernel get_gemm_kernel_slow() +{ + static int init = 0; + static cl_kernel gemm_kernel; + if(!init){ + gemm_kernel = get_kernel("src/gemm.cl", "gemm_slow"); + init = 1; + } + return gemm_kernel; +} + +void gpu_gemm_slow(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) +{ + cl_setup(); + cl_kernel gemm_kernel = get_gemm_kernel_slow(); + cl_context context = cl.context; + cl_command_queue queue = cl.queue; + + size_t size = sizeof(float)*(TA ? lda*K:lda*M); + cl_mem A_gpu = clCreateBuffer(context, + CL_MEM_READ_ONLY|CL_MEM_COPY_HOST_PTR, + size, A, &cl.error); + check_error(cl); + + size = sizeof(float)*(TB ? ldb*N:ldb*K); + cl_mem B_gpu = clCreateBuffer(context, + CL_MEM_READ_ONLY|CL_MEM_COPY_HOST_PTR, + size, B, &cl.error); + check_error(cl); + + size = sizeof(float)*(ldc*M); + cl_mem C_gpu = clCreateBuffer(context, + CL_MEM_READ_ONLY|CL_MEM_COPY_HOST_PTR, + size, C, &cl.error); + check_error(cl); + + cl_uint i = 0; + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(TA), (void*) &TA); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(TB), (void*) &TB); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(M), (void*) &M); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(N), (void*) &N); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(K), (void*) &K); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(ALPHA), (void*) &ALPHA); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(A_gpu), (void*) &A_gpu); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(lda), (void*) &lda); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(B_gpu), (void*) &B_gpu); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(ldb), (void*) &ldb); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(BETA), (void*) &BETA); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(C_gpu), (void*) &C_gpu); + cl.error = clSetKernelArg(gemm_kernel, i++, sizeof(ldc), (void*) &ldc); + check_error(cl); + + const size_t global_size[] = {M, N}; + + clEnqueueNDRangeKernel(queue, gemm_kernel, 2, 0, global_size, 0, 0, 0, 0); + clEnqueueReadBuffer(queue, C_gpu, CL_TRUE, 0, size, C, 0, 0, 0); + + clReleaseMemObject(A_gpu); + clReleaseMemObject(B_gpu); + clReleaseMemObject(C_gpu); + +} +*/ diff --git a/src/image.c b/src/image.c index 24e32922..5c138d33 100644 --- a/src/image.c +++ b/src/image.c @@ -113,6 +113,7 @@ image copy_image(image p) return copy; } + void show_image(image p, char *name) { int i,j,k; @@ -152,6 +153,30 @@ void show_image(image p, char *name) cvReleaseImage(&disp); } +void save_image(image p, char *name) +{ + int i,j,k; + image copy = copy_image(p); + normalize_image(copy); + + char buff[256]; + //sprintf(buff, "%s (%d)", name, windows); + sprintf(buff, "%s.png", name); + + IplImage *disp = cvCreateImage(cvSize(p.w,p.h), IPL_DEPTH_8U, p.c); + int step = disp->widthStep; + for(i = 0; i < p.h; ++i){ + for(j = 0; j < p.w; ++j){ + for(k= 0; k < p.c; ++k){ + disp->imageData[i*step + j*p.c + k] = (unsigned char)(get_pixel(copy,i,j,k)*255); + } + } + } + free_image(copy); + cvSaveImage(buff, disp,0); + cvReleaseImage(&disp); +} + void show_image_layers(image p, char *name) { int i; @@ -227,6 +252,18 @@ image make_random_image(int h, int w, int c) return out; } +void add_into_image(image src, image dest, int h, int w) +{ + int i,j,k; + for(k = 0; k < src.c; ++k){ + for(i = 0; i < src.h; ++i){ + for(j = 0; j < src.w; ++j){ + add_pixel(dest, h+i, w+j, k, get_pixel(src, i, j, k)); + } + } + } +} + void add_scalar_image(image m, float s) { int i; @@ -404,6 +441,20 @@ image get_image_layer(image m, int l) } return out; } +image get_sub_image(image m, int h, int w, int dh, int dw) +{ + image out = make_image(dh, dw, m.c); + int i,j,k; + for(k = 0; k < out.c; ++k){ + for(i = 0; i < dh; ++i){ + for(j = 0; j < dw; ++j){ + float val = get_pixel(m, h+i, w+j, k); + set_pixel(out, i, j, k, val); + } + } + } + return out; +} float get_pixel(image m, int x, int y, int c) { @@ -595,6 +646,49 @@ void print_image(image m) printf("\n"); } +image collapse_images(image *ims, int n) +{ + int color = 1; + int border = 1; + int h,w,c; + int size = ims[0].h; + h = size; + w = (size + border) * n - border; + c = ims[0].c; + if(c != 3 || !color){ + h = (h+border)*c - border; + c = 1; + } + + image filters = make_image(h,w,c); + int i,j; + for(i = 0; i < n; ++i){ + int w_offset = i*(size+border); + image copy = copy_image(ims[i]); + normalize_image(copy); + if(c == 3 && color){ + embed_image(copy, filters, 0, w_offset); + } + else{ + for(j = 0; j < copy.c; ++j){ + int h_offset = j*(size+border); + image layer = get_image_layer(copy, j); + embed_image(layer, filters, h_offset, w_offset); + free_image(layer); + } + } + free_image(copy); + } + return filters; +} + +void show_images(image *ims, int n, char *window) +{ + image m = collapse_images(ims, n); + show_image(m, window); + free_image(m); +} + void free_image(image m) { free(m.data); diff --git a/src/image.h b/src/image.h index 9f7d74d4..9d064c36 100644 --- a/src/image.h +++ b/src/image.h @@ -21,9 +21,13 @@ void rotate_image(image m); void subtract_image(image a, image b); float avg_image_layer(image m, int l); void embed_image(image source, image dest, int h, int w); +void add_into_image(image src, image dest, int h, int w); image collapse_image_layers(image source, int border); +image get_sub_image(image m, int h, int w, int dh, int dw); void show_image(image p, char *name); +void save_image(image p, char *name); +void show_images(image *ims, int n, char *window); void show_image_layers(image p, char *name); void show_image_collapsed(image p, char *name); void print_image(image m); @@ -39,6 +43,7 @@ image ipl_to_image(IplImage* src); float get_pixel(image m, int x, int y, int c); float get_pixel_extend(image m, int x, int y, int c); +void add_pixel(image m, int x, int y, int c, float val); void set_pixel(image m, int x, int y, int c, float val); image get_image_layer(image m, int l); diff --git a/src/mini_blas.c b/src/mini_blas.c index 262798bc..bac3e226 100644 --- a/src/mini_blas.c +++ b/src/mini_blas.c @@ -3,6 +3,8 @@ #include #include #include +#include +#include "mini_blas.h" void pm(int M, int N, float *A) { @@ -17,42 +19,12 @@ void pm(int M, int N, float *A) } void gemm(int TA, int TB, int M, int N, int K, float ALPHA, - float *A, int lda, - float *B, int ldb, - float BETA, - float *C, int ldc) + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc) { - // Assume beta = 1 LULZ - int i,j,k; - if(TB && !TA){ - for(i = 0; i < M; ++i){ - for(j = 0; j < N; ++j){ - register float sum = 0; - for(k = 0; k < K; ++k){ - sum += ALPHA*A[i*lda+k]*B[k+j*ldb]; - } - C[i*ldc+j] += sum; - } - } - }else if(TA && !TB){ - for(i = 0; i < M; ++i){ - for(k = 0; k < K; ++k){ - register float A_PART = ALPHA*A[k*lda+i]; - for(j = 0; j < N; ++j){ - C[i*ldc+j] += A_PART*B[k*ldb+j]; - } - } - } - }else{ - for(i = 0; i < M; ++i){ - for(k = 0; k < K; ++k){ - register float A_PART = ALPHA*A[i*lda+k]; - for(j = 0; j < N; ++j){ - C[i*ldc+j] += A_PART*B[k*ldb+j]; - } - } - } - } + gpu_gemm( TA, TB, M, N, K, ALPHA,A,lda, B, ldb,BETA,C,ldc); } void im2row(float *image, int h, int w, int c, int size, int stride, float *matrix) @@ -150,16 +122,26 @@ float *random_matrix(int rows, int cols) void time_random_matrix(int TA, int TB, int m, int k, int n) { - float *a = random_matrix(m,k); - float *b = random_matrix(k,n); + float *a; + if(!TA) a = random_matrix(m,k); + else a = random_matrix(k,m); + int lda = (!TA)?k:m; + float *b; + if(!TB) b = random_matrix(k,n); + else b = random_matrix(n,k); + int ldb = (!TB)?n:k; + float *c = random_matrix(m,n); int i; clock_t start = clock(), end; for(i = 0; i<1000; ++i){ - gemm(TA,TB,m,n,k,1,a,k,b,n,1,c,n); + cpu_gemm(TA,TB,m,n,k,1,a,lda,b,ldb,1,c,n); } end = clock(); printf("Matrix Multiplication %dx%d * %dx%d, TA=%d, TB=%d: %lf ms\n",m,k,k,n, TA, TB, (float)(end-start)/CLOCKS_PER_SEC); + free(a); + free(b); + free(c); } void test_blas() @@ -167,9 +149,97 @@ void test_blas() time_random_matrix(0,0,100,100,100); time_random_matrix(1,0,100,100,100); time_random_matrix(0,1,100,100,100); + time_random_matrix(1,1,100,100,100); - time_random_matrix(0,1,1000,100,100); + time_random_matrix(0,0,1000,100,100); time_random_matrix(1,0,1000,100,100); + time_random_matrix(0,1,1000,100,100); + time_random_matrix(1,1,1000,100,100); + } +void time_gpu_random_matrix(int TA, int TB, int m, int k, int n) +{ + float *a; + if(!TA) a = random_matrix(m,k); + else a = random_matrix(k,m); + int lda = (!TA)?k:m; + float *b; + if(!TB) b = random_matrix(k,n); + else b = random_matrix(n,k); + int ldb = (!TB)?n:k; + + float *c = random_matrix(m,n); + int i; + clock_t start = clock(), end; + for(i = 0; i<1000; ++i){ + gpu_gemm(TA,TB,m,n,k,1,a,lda,b,ldb,1,c,n); + } + end = clock(); + printf("Matrix Multiplication %dx%d * %dx%d, TA=%d, TB=%d: %lf ms\n",m,k,k,n, TA, TB, (float)(end-start)/CLOCKS_PER_SEC); + free(a); + free(b); + free(c); +} + +void test_gpu_accuracy(int TA, int TB, int m, int k, int n) +{ + srand(0); + float *a; + if(!TA) a = random_matrix(m,k); + else a = random_matrix(k,m); + int lda = (!TA)?k:m; + float *b; + if(!TB) b = random_matrix(k,n); + else b = random_matrix(n,k); + int ldb = (!TB)?n:k; + + float *c = random_matrix(m,n); + float *c_gpu = random_matrix(m,n); + memset(c, 0, m*n*sizeof(float)); + memset(c_gpu, 0, m*n*sizeof(float)); + int i; + //pm(m,k,b); + gpu_gemm(TA,TB,m,n,k,1,a,lda,b,ldb,1,c_gpu,n); + //pm(m, n, c_gpu); + cpu_gemm(TA,TB,m,n,k,1,a,lda,b,ldb,1,c,n); + //pm(m, n, c); + double sse = 0; + for(i = 0; i < m*n; ++i) { + //printf("%f %f\n", c[i], c_gpu[i]); + sse += pow(c[i]-c_gpu[i], 2); + } + printf("Matrix Multiplication %dx%d * %dx%d, TA=%d, TB=%d: %g MSE\n",m,k,k,n, TA, TB, sse/(m*n)); + free(a); + free(b); + free(c); +} + +void test_gpu_blas() +{ + test_gpu_accuracy(0,0,17,10,10); + test_gpu_accuracy(1,0,17,10,10); + test_gpu_accuracy(0,1,17,10,10); + test_gpu_accuracy(1,1,17,10,10); + + test_gpu_accuracy(0,0,1000,10,100); + test_gpu_accuracy(1,0,1000,10,100); + test_gpu_accuracy(0,1,1000,10,100); + test_gpu_accuracy(1,1,1000,10,100); + + time_gpu_random_matrix(0,0,1000,1000,100); + time_random_matrix(0,0,1000,1000,100); + + time_gpu_random_matrix(0,1,1000,1000,100); + time_random_matrix(0,1,1000,1000,100); + + time_gpu_random_matrix(1,0,1000,1000,100); + time_random_matrix(1,0,1000,1000,100); + + time_gpu_random_matrix(1,1,1000,1000,100); + time_random_matrix(1,1,1000,1000,100); + +} + + diff --git a/src/mini_blas.h b/src/mini_blas.h index ff82a60c..56e4fa72 100644 --- a/src/mini_blas.h +++ b/src/mini_blas.h @@ -4,6 +4,7 @@ void gemm(int TA, int TB, int M, int N, int K, float ALPHA, float *B, int ldb, float BETA, float *C, int ldc); +float *random_matrix(int rows, int cols); void im2row(float *image, int h, int w, int c, int size, int stride, float *matrix); void im2col(float *image, int h, int w, int c, int size, int stride, float *matrix); void im2col_cpu(float* data_im, const int channels, @@ -13,3 +14,15 @@ void col2im_cpu(float* data_col, const int channels, const int height, const int width, const int ksize, const int stride, float* data_im); void test_blas(); + +void gpu_gemm(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc); +void cpu_gemm(int TA, int TB, int M, int N, int K, float ALPHA, + float *A, int lda, + float *B, int ldb, + float BETA, + float *C, int ldc); +void test_gpu_blas(); diff --git a/src/network.c b/src/network.c index e2c44b05..edae3c7b 100644 --- a/src/network.c +++ b/src/network.c @@ -428,13 +428,14 @@ image get_network_image(network net) void visualize_network(network net) { + image *prev = 0; int i; char buff[256]; for(i = 0; i < net.n; ++i){ sprintf(buff, "Layer %d", i); if(net.types[i] == CONVOLUTIONAL){ convolutional_layer layer = *(convolutional_layer *)net.layers[i]; - visualize_convolutional_layer(layer, buff); + prev = visualize_convolutional_layer(layer, buff, prev); } } } @@ -506,3 +507,4 @@ float network_accuracy(network net, data d) return acc; } + diff --git a/src/opencl.c b/src/opencl.c new file mode 100644 index 00000000..193fba32 --- /dev/null +++ b/src/opencl.c @@ -0,0 +1,77 @@ +#include "opencl.h" +#include +#include +#include + +cl_info cl = {0}; + +void check_error(cl_info info) +{ + if (info.error != CL_SUCCESS) { + printf("\n Error number %d", info.error); + } +} + +cl_info cl_init() +{ + cl_info info; + info.initialized = 0; + cl_uint platforms, devices; + // Fetch the Platform and Device IDs; we only want one. + info.error=clGetPlatformIDs(1, &info.platform, &platforms); + check_error(info); + info.error=clGetDeviceIDs(info.platform, CL_DEVICE_TYPE_ALL, 1, &info.device, &devices); + check_error(info); + + cl_context_properties properties[]={ + CL_CONTEXT_PLATFORM, (cl_context_properties)info.platform, + 0}; + // Note that nVidia's OpenCL requires the platform property + info.context=clCreateContext(properties, 1, &info.device, 0, 0, &info.error); + check_error(info); + info.queue = clCreateCommandQueue(info.context, info.device, 0, &info.error); + check_error(info); + info.initialized = 1; + return info; +} + +cl_program cl_fprog(char *filename, char *options, cl_info info) +{ + size_t srcsize; + char src[8192]; + memset(src, 0, 8192); + FILE *fil=fopen(filename,"r"); + srcsize=fread(src, sizeof src, 1, fil); + fclose(fil); + const char *srcptr[]={src}; + // Submit the source code of the example kernel to OpenCL + cl_program prog=clCreateProgramWithSource(info.context,1, srcptr, &srcsize, &info.error); + check_error(info); + char build_c[4096]; + // and compile it (after this we could extract the compiled version) + info.error=clBuildProgram(prog, 0, 0, options, 0, 0); + if ( info.error != CL_SUCCESS ) { + fprintf(stderr, "Error Building Program: %d\n", info.error); + clGetProgramBuildInfo( prog, info.device, CL_PROGRAM_BUILD_LOG, 4096, build_c, 0); + fprintf(stderr, "Build Log for %s program:\n%s\n", filename, build_c); + } + return prog; +} + +void cl_setup() +{ + if(!cl.initialized){ + cl = cl_init(); + } +} + +cl_kernel get_kernel(char *filename, char *kernelname, char *options) +{ + cl_setup(); + cl_program prog = cl_fprog(filename, options, cl); + cl_kernel kernel=clCreateKernel(prog, kernelname, &cl.error); + check_error(cl); + return kernel; +} + + diff --git a/src/opencl.h b/src/opencl.h new file mode 100644 index 00000000..59efbae0 --- /dev/null +++ b/src/opencl.h @@ -0,0 +1,21 @@ +#ifdef __APPLE__ +#include +#else +#include +#endif + +typedef struct { + int initialized; + cl_int error; + cl_platform_id platform; + cl_device_id device; + cl_context context; + cl_command_queue queue; +}cl_info; + +extern cl_info cl; + +void cl_setup(); +void check_error(cl_info info); +cl_kernel get_kernel(char *filename, char *kernelname, char *options); + diff --git a/src/tests.c b/src/tests.c index 91217d42..5d9136de 100644 --- a/src/tests.c +++ b/src/tests.c @@ -220,6 +220,14 @@ void train_full() //lr *= .99; } } + +void test_visualize() +{ + network net = parse_network_cfg("cfg/imagenet.cfg"); + srand(2222222); + visualize_network(net); + cvWaitKey(0); +} void test_full() { network net = parse_network_cfg("cfg/backup_1300.cfg"); @@ -265,7 +273,7 @@ void test_cifar10() scale_data_rows(train, 1./255); train_network_sgd(net, train, batch, lr, momentum, decay); //printf("%5f %5f\n",(double)count*batch/train.X.rows, loss); - + float test_acc = network_accuracy(net, test); printf("%5f %5f\n",(double)count*batch/train.X.rows/5, 1-test_acc); free_data(train); @@ -316,15 +324,15 @@ void test_nist() //printf("Time: %lf seconds\n", (float)(end-start)/CLOCKS_PER_SEC); //start=end; /* - if(count%5 == 0){ - float train_acc = network_accuracy(net, train); - fprintf(stderr, "\nTRAIN: %f\n", train_acc); - float test_acc = network_accuracy(net, test); - fprintf(stderr, "TEST: %f\n\n", test_acc); - printf("%d, %f, %f\n", count, train_acc, test_acc); - //lr *= .5; + if(count%5 == 0){ + float train_acc = network_accuracy(net, train); + fprintf(stderr, "\nTRAIN: %f\n", train_acc); + float test_acc = network_accuracy(net, test); + fprintf(stderr, "TEST: %f\n\n", test_acc); + printf("%d, %f, %f\n", count, train_acc, test_acc); + //lr *= .5; } - */ + */ } } @@ -516,6 +524,48 @@ void features_VOC_image_size(char *image_path, int h, int w) cvReleaseImage(&src); } +void visualize_imagenet_features(char *filename) +{ + int i,j,k; + network net = parse_network_cfg("cfg/voc_imagenet.cfg"); + list *plist = get_paths(filename); + node *n = plist->front; + int h = voc_size(1), w = voc_size(1); + int num = get_network_image(net).c; + image *vizs = calloc(num, sizeof(image)); + for(i = 0; i < num; ++i) vizs[i] = make_image(h, w, 3); + while(n){ + char *image_path = (char *)n->val; + image im = load_image(image_path, 0, 0); + printf("Processing %dx%d image\n", im.h, im.w); + resize_network(net, im.h, im.w, im.c); + forward_network(net, im.data); + image out = get_network_image(net); + + int dh = (im.h - h)/h; + int dw = (im.w - w)/w; + for(i = 0; i < out.h; ++i){ + for(j = 0; j < out.w; ++j){ + image sub = get_sub_image(im, dh*i, dw*j, h, w); + for(k = 0; k < out.c; ++k){ + float val = get_pixel(out, i, j, k); + //printf("%f, ", val); + image sub_c = copy_image(sub); + scale_image(sub_c, val); + add_into_image(sub_c, vizs[k], 0, 0); + free_image(sub_c); + } + free_image(sub); + } + } + //printf("\n"); + show_images(vizs, 10, "IMAGENET Visualization"); + cvWaitKey(1000); + n = n->next; + } + cvWaitKey(0); +} + void features_VOC_image(char *image_file, char *image_dir, char *out_dir) { int i,j; @@ -627,6 +677,9 @@ int main(int argc, char *argv[]) //test_distribution(); //feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW); + //test_blas(); + //test_visualize(); + //test_gpu_blas(); //test_blas(); //test_convolve_matrix(); // test_im2row(); @@ -638,7 +691,9 @@ int main(int argc, char *argv[]) //test_full(); //train_VOC(); //features_VOC_image(argv[1], argv[2], argv[3]); - features_VOC_image_size(argv[1], atoi(argv[2]), atoi(argv[3])); + //features_VOC_image_size(argv[1], atoi(argv[2]), atoi(argv[3])); + //visualize_imagenet_features("data/assira/train.list"); + visualize_imagenet_features("data/VOC2011.list"); fprintf(stderr, "Success!\n"); //test_random_preprocess(); //test_random_classify(); From 738cd4c2d7abcf62b85b759a5765b08380ee90e8 Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Wed, 16 Apr 2014 17:05:29 -0700 Subject: [PATCH 3/4] Visualizations? --- Makefile | 7 +- src/convolutional_layer.c | 7 +- src/image.c | 82 +++++++++++++++++++++-- src/image.h | 8 ++- src/network.c | 136 +++++++++++++++++++++++++++----------- src/network.h | 3 +- src/normalization_layer.c | 96 +++++++++++++++++++++++++++ src/normalization_layer.h | 26 ++++++++ src/parser.c | 36 ++++++++++ src/tests.c | 92 +++++++++++++++++++++++++- 10 files changed, 438 insertions(+), 55 deletions(-) create mode 100644 src/normalization_layer.c create mode 100644 src/normalization_layer.h diff --git a/Makefile b/Makefile index 07cf79fb..640f3082 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,21 @@ CC=gcc COMMON=-Wall `pkg-config --cflags opencv` UNAME = $(shell uname) +OPTS=-O3 ifeq ($(UNAME), Darwin) COMMON+= -isystem /usr/local/Cellar/opencv/2.4.6.1/include/opencv -isystem /usr/local/Cellar/opencv/2.4.6.1/include LDFLAGS= -framework OpenCL else -COMMON+= -march=native -flto +OPTS+= -march=native -flto LDFLAGS= -lOpenCL endif -CFLAGS= $(COMMON) -Ofast +CFLAGS= $(COMMON) $(OPTS) #CFLAGS= $(COMMON) -O0 -g LDFLAGS+=`pkg-config --libs opencv` -lm VPATH=./src/ EXEC=cnn -OBJ=network.o image.o tests.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o softmax_layer.o mini_blas.o convolutional_layer.o opencl.o gpu_gemm.o cpu_gemm.o +OBJ=network.o image.o tests.o connected_layer.o maxpool_layer.o activations.o list.o option_list.o parser.o utils.o data.o matrix.o softmax_layer.o mini_blas.o convolutional_layer.o opencl.o gpu_gemm.o cpu_gemm.o normalization_layer.o all: $(EXEC) diff --git a/src/convolutional_layer.c b/src/convolutional_layer.c index 40d58584..6916eebc 100644 --- a/src/convolutional_layer.c +++ b/src/convolutional_layer.c @@ -320,11 +320,12 @@ image *visualize_convolutional_layer(convolutional_layer layer, char *window, im image *single_filters = weighted_sum_filters(layer, 0); show_images(single_filters, layer.n, window); - image delta = get_convolutional_delta(layer); + image delta = get_convolutional_image(layer); image dc = collapse_image_layers(delta, 1); char buff[256]; - sprintf(buff, "%s: Delta", window); - //show_image(dc, buff); + sprintf(buff, "%s: Output", window); + show_image(dc, buff); + save_image(dc, buff); free_image(dc); return single_filters; } diff --git a/src/image.c b/src/image.c index 5c138d33..453919fb 100644 --- a/src/image.c +++ b/src/image.c @@ -264,7 +264,7 @@ void add_into_image(image src, image dest, int h, int w) } } -void add_scalar_image(image m, float s) +void translate_image(image m, float s) { int i; for(i = 0; i < m.h*m.w*m.c; ++i) m.data[i] += s; @@ -645,15 +645,49 @@ void print_image(image m) for(i =0 ; i < m.h*m.w*m.c; ++i) printf("%lf, ", m.data[i]); printf("\n"); } +image collapse_images_vert(image *ims, int n) +{ + int color = 1; + int border = 1; + int h,w,c; + w = ims[0].w; + h = (ims[0].h + border) * n - border; + c = ims[0].c; + if(c != 3 || !color){ + w = (w+border)*c - border; + c = 1; + } -image collapse_images(image *ims, int n) + image filters = make_image(h,w,c); + int i,j; + for(i = 0; i < n; ++i){ + int h_offset = i*(ims[0].h+border); + image copy = copy_image(ims[i]); + //normalize_image(copy); + if(c == 3 && color){ + embed_image(copy, filters, h_offset, 0); + } + else{ + for(j = 0; j < copy.c; ++j){ + int w_offset = j*(ims[0].w+border); + image layer = get_image_layer(copy, j); + embed_image(layer, filters, h_offset, w_offset); + free_image(layer); + } + } + free_image(copy); + } + return filters; +} + +image collapse_images_horz(image *ims, int n) { int color = 1; int border = 1; int h,w,c; int size = ims[0].h; h = size; - w = (size + border) * n - border; + w = (ims[0].w + border) * n - border; c = ims[0].c; if(c != 3 || !color){ h = (h+border)*c - border; @@ -665,7 +699,7 @@ image collapse_images(image *ims, int n) for(i = 0; i < n; ++i){ int w_offset = i*(size+border); image copy = copy_image(ims[i]); - normalize_image(copy); + //normalize_image(copy); if(c == 3 && color){ embed_image(copy, filters, 0, w_offset); } @@ -684,11 +718,49 @@ image collapse_images(image *ims, int n) void show_images(image *ims, int n, char *window) { - image m = collapse_images(ims, n); + image m = collapse_images_vert(ims, n); + save_image(m, window); show_image(m, window); free_image(m); } +image grid_images(image **ims, int h, int w) +{ + int i; + image *rows = calloc(h, sizeof(image)); + for(i = 0; i < h; ++i){ + rows[i] = collapse_images_horz(ims[i], w); + } + image out = collapse_images_vert(rows, h); + for(i = 0; i < h; ++i){ + free_image(rows[i]); + } + free(rows); + return out; +} + +void test_grid() +{ + int i,j; + int num = 3; + int topk = 3; + image **vizs = calloc(num, sizeof(image*)); + for(i = 0; i < num; ++i){ + vizs[i] = calloc(topk, sizeof(image)); + for(j = 0; j < topk; ++j) vizs[i][j] = make_image(3,3,3); + } + image grid = grid_images(vizs, num, topk); + save_image(grid, "Test Grid"); + free_image(grid); +} + +void show_images_grid(image **ims, int h, int w, char *window) +{ + image out = grid_images(ims, h, w); + show_image(out, window); + free_image(out); +} + void free_image(image m) { free(m.data); diff --git a/src/image.h b/src/image.h index 9d064c36..fe257425 100644 --- a/src/image.h +++ b/src/image.h @@ -1,6 +1,7 @@ #ifndef IMAGE_H #define IMAGE_H + #include "opencv2/highgui/highgui_c.h" #include "opencv2/imgproc/imgproc_c.h" typedef struct { @@ -12,7 +13,7 @@ typedef struct { image image_distance(image a, image b); void scale_image(image m, float s); -void add_scalar_image(image m, float s); +void translate_image(image m, float s); void normalize_image(image p); void z_normalize_image(image p); void threshold_image(image p, float t); @@ -23,6 +24,8 @@ float avg_image_layer(image m, int l); void embed_image(image source, image dest, int h, int w); void add_into_image(image src, image dest, int h, int w); image collapse_image_layers(image source, int border); +image collapse_images_horz(image *ims, int n); +image collapse_images_vert(image *ims, int n); image get_sub_image(image m, int h, int w, int dh, int dw); void show_image(image p, char *name); @@ -30,6 +33,9 @@ void save_image(image p, char *name); void show_images(image *ims, int n, char *window); void show_image_layers(image p, char *name); void show_image_collapsed(image p, char *name); +void show_images_grid(image **ims, int h, int w, char *window); +void test_grid(); +image grid_images(image **ims, int h, int w); void print_image(image m); image make_image(int h, int w, int c); diff --git a/src/network.c b/src/network.c index edae3c7b..7d4b1fac 100644 --- a/src/network.c +++ b/src/network.c @@ -8,6 +8,7 @@ #include "convolutional_layer.h" //#include "old_conv.h" #include "maxpool_layer.h" +#include "normalization_layer.h" #include "softmax_layer.h" network make_network(int n, int batch) @@ -40,6 +41,17 @@ void print_convolutional_cfg(FILE *fp, convolutional_layer *l, int first) fprintf(fp, "data="); for(i = 0; i < l->n; ++i) fprintf(fp, "%g,", l->biases[i]); for(i = 0; i < l->n*l->c*l->size*l->size; ++i) fprintf(fp, "%g,", l->filters[i]); + /* + int j,k; + for(i = 0; i < l->n; ++i) fprintf(fp, "%g,", l->biases[i]); + for(i = 0; i < l->n; ++i){ + for(j = l->c-1; j >= 0; --j){ + for(k = 0; k < l->size*l->size; ++k){ + fprintf(fp, "%g,", l->filters[i*(l->c*l->size*l->size)+j*l->size*l->size+k]); + } + } + } + */ fprintf(fp, "\n\n"); } void print_connected_cfg(FILE *fp, connected_layer *l, int first) @@ -48,9 +60,9 @@ void print_connected_cfg(FILE *fp, connected_layer *l, int first) fprintf(fp, "[connected]\n"); if(first) fprintf(fp, "batch=%d\ninput=%d\n", l->batch, l->inputs); fprintf(fp, "output=%d\n" - "activation=%s\n", - l->outputs, - get_activation_string(l->activation)); + "activation=%s\n", + l->outputs, + get_activation_string(l->activation)); fprintf(fp, "data="); for(i = 0; i < l->outputs; ++i) fprintf(fp, "%g,", l->biases[i]); for(i = 0; i < l->inputs*l->outputs; ++i) fprintf(fp, "%g,", l->weights[i]); @@ -61,13 +73,27 @@ void print_maxpool_cfg(FILE *fp, maxpool_layer *l, int first) { fprintf(fp, "[maxpool]\n"); if(first) fprintf(fp, "batch=%d\n" - "height=%d\n" - "width=%d\n" - "channels=%d\n", - l->batch,l->h, l->w, l->c); + "height=%d\n" + "width=%d\n" + "channels=%d\n", + l->batch,l->h, l->w, l->c); fprintf(fp, "stride=%d\n\n", l->stride); } +void print_normalization_cfg(FILE *fp, normalization_layer *l, int first) +{ + fprintf(fp, "[localresponsenormalization]\n"); + if(first) fprintf(fp, "batch=%d\n" + "height=%d\n" + "width=%d\n" + "channels=%d\n", + l->batch,l->h, l->w, l->c); + fprintf(fp, "size=%d\n" + "alpha=%g\n" + "beta=%g\n" + "kappa=%g\n\n", l->size, l->alpha, l->beta, l->kappa); +} + void print_softmax_cfg(FILE *fp, softmax_layer *l, int first) { fprintf(fp, "[softmax]\n"); @@ -88,6 +114,8 @@ void save_network(network net, char *filename) print_connected_cfg(fp, (connected_layer *)net.layers[i], i==0); else if(net.types[i] == MAXPOOL) print_maxpool_cfg(fp, (maxpool_layer *)net.layers[i], i==0); + else if(net.types[i] == NORMALIZATION) + print_normalization_cfg(fp, (normalization_layer *)net.layers[i], i==0); else if(net.types[i] == SOFTMAX) print_softmax_cfg(fp, (softmax_layer *)net.layers[i], i==0); } @@ -118,6 +146,11 @@ void forward_network(network net, float *input) forward_maxpool_layer(layer, input); input = layer.output; } + else if(net.types[i] == NORMALIZATION){ + normalization_layer layer = *(normalization_layer *)net.layers[i]; + forward_normalization_layer(layer, input); + input = layer.output; + } } } @@ -135,6 +168,9 @@ void update_network(network net, float step, float momentum, float decay) else if(net.types[i] == SOFTMAX){ //maxpool_layer layer = *(maxpool_layer *)net.layers[i]; } + else if(net.types[i] == NORMALIZATION){ + //maxpool_layer layer = *(maxpool_layer *)net.layers[i]; + } else if(net.types[i] == CONNECTED){ connected_layer layer = *(connected_layer *)net.layers[i]; update_connected_layer(layer, step, momentum, decay); @@ -156,6 +192,9 @@ float *get_network_output_layer(network net, int i) } else if(net.types[i] == CONNECTED){ connected_layer layer = *(connected_layer *)net.layers[i]; return layer.output; + } else if(net.types[i] == NORMALIZATION){ + normalization_layer layer = *(normalization_layer *)net.layers[i]; + return layer.output; } return 0; } @@ -233,6 +272,10 @@ float backward_network(network net, float *input, float *truth) maxpool_layer layer = *(maxpool_layer *)net.layers[i]; if(i != 0) backward_maxpool_layer(layer, prev_input, prev_delta); } + else if(net.types[i] == NORMALIZATION){ + normalization_layer layer = *(normalization_layer *)net.layers[i]; + if(i != 0) backward_normalization_layer(layer, prev_input, prev_delta); + } else if(net.types[i] == SOFTMAX){ softmax_layer layer = *(softmax_layer *)net.layers[i]; if(i != 0) backward_softmax_layer(layer, prev_input, prev_delta); @@ -272,7 +315,7 @@ float train_network_sgd(network net, data d, int n, float step, float momentum,f error += err; ++pos; } - + //printf("%d %f %f\n", i,net.output[0], d.y.vals[index][0]); //if((i+1)%10 == 0){ @@ -341,34 +384,34 @@ int get_network_output_size_layer(network net, int i) } /* -int resize_network(network net, int h, int w, int c) -{ - int i; - for (i = 0; i < net.n; ++i){ - if(net.types[i] == CONVOLUTIONAL){ - convolutional_layer *layer = (convolutional_layer *)net.layers[i]; - layer->h = h; - layer->w = w; - layer->c = c; - image output = get_convolutional_image(*layer); - h = output.h; - w = output.w; - c = output.c; - } - else if(net.types[i] == MAXPOOL){ - maxpool_layer *layer = (maxpool_layer *)net.layers[i]; - layer->h = h; - layer->w = w; - layer->c = c; - image output = get_maxpool_image(*layer); - h = output.h; - w = output.w; - c = output.c; - } - } - return 0; -} -*/ + int resize_network(network net, int h, int w, int c) + { + int i; + for (i = 0; i < net.n; ++i){ + if(net.types[i] == CONVOLUTIONAL){ + convolutional_layer *layer = (convolutional_layer *)net.layers[i]; + layer->h = h; + layer->w = w; + layer->c = c; + image output = get_convolutional_image(*layer); + h = output.h; + w = output.w; + c = output.c; + } + else if(net.types[i] == MAXPOOL){ + maxpool_layer *layer = (maxpool_layer *)net.layers[i]; + layer->h = h; + layer->w = w; + layer->c = c; + image output = get_maxpool_image(*layer); + h = output.h; + w = output.w; + c = output.c; + } + } + return 0; + } + */ int resize_network(network net, int h, int w, int c) { @@ -381,16 +424,21 @@ int resize_network(network net, int h, int w, int c) h = output.h; w = output.w; c = output.c; - } - else if(net.types[i] == MAXPOOL){ + }else if(net.types[i] == MAXPOOL){ maxpool_layer *layer = (maxpool_layer *)net.layers[i]; resize_maxpool_layer(layer, h, w, c); image output = get_maxpool_image(*layer); h = output.h; w = output.w; c = output.c; - } - else{ + }else if(net.types[i] == NORMALIZATION){ + normalization_layer *layer = (normalization_layer *)net.layers[i]; + resize_normalization_layer(layer, h, w, c); + image output = get_normalization_image(*layer); + h = output.h; + w = output.w; + c = output.c; + }else{ error("Cannot resize this type of layer"); } } @@ -413,6 +461,10 @@ image get_network_image_layer(network net, int i) maxpool_layer layer = *(maxpool_layer *)net.layers[i]; return get_maxpool_image(layer); } + else if(net.types[i] == NORMALIZATION){ + normalization_layer layer = *(normalization_layer *)net.layers[i]; + return get_normalization_image(layer); + } return make_empty_image(0,0,0); } @@ -437,6 +489,10 @@ void visualize_network(network net) convolutional_layer layer = *(convolutional_layer *)net.layers[i]; prev = visualize_convolutional_layer(layer, buff, prev); } + if(net.types[i] == NORMALIZATION){ + normalization_layer layer = *(normalization_layer *)net.layers[i]; + visualize_normalization_layer(layer, buff); + } } } diff --git a/src/network.h b/src/network.h index 5acee61b..f6dac7e6 100644 --- a/src/network.h +++ b/src/network.h @@ -9,7 +9,8 @@ typedef enum { CONVOLUTIONAL, CONNECTED, MAXPOOL, - SOFTMAX + SOFTMAX, + NORMALIZATION } LAYER_TYPE; typedef struct { diff --git a/src/normalization_layer.c b/src/normalization_layer.c new file mode 100644 index 00000000..2d844e0e --- /dev/null +++ b/src/normalization_layer.c @@ -0,0 +1,96 @@ +#include "normalization_layer.h" +#include + +image get_normalization_image(normalization_layer layer) +{ + int h = layer.h; + int w = layer.w; + int c = layer.c; + return float_to_image(h,w,c,layer.output); +} + +image get_normalization_delta(normalization_layer layer) +{ + int h = layer.h; + int w = layer.w; + int c = layer.c; + return float_to_image(h,w,c,layer.delta); +} + +normalization_layer *make_normalization_layer(int batch, int h, int w, int c, int size, float alpha, float beta, float kappa) +{ + fprintf(stderr, "Local Response Normalization Layer: %d x %d x %d image, %d size\n", h,w,c,size); + normalization_layer *layer = calloc(1, sizeof(normalization_layer)); + layer->batch = batch; + layer->h = h; + layer->w = w; + layer->c = c; + layer->kappa = kappa; + layer->size = size; + layer->alpha = alpha; + layer->beta = beta; + layer->output = calloc(h * w * c * batch, sizeof(float)); + layer->delta = calloc(h * w * c * batch, sizeof(float)); + layer->sums = calloc(h*w, sizeof(float)); + return layer; +} + +void resize_normalization_layer(normalization_layer *layer, int h, int w, int c) +{ + layer->h = h; + layer->w = w; + layer->c = c; + layer->output = realloc(layer->output, h * w * c * layer->batch * sizeof(float)); + layer->delta = realloc(layer->delta, h * w * c * layer->batch * sizeof(float)); + layer->sums = realloc(layer->sums, h*w * sizeof(float)); +} + +void add_square_array(float *src, float *dest, int n) +{ + int i; + for(i = 0; i < n; ++i){ + dest[i] += src[i]*src[i]; + } +} +void sub_square_array(float *src, float *dest, int n) +{ + int i; + for(i = 0; i < n; ++i){ + dest[i] -= src[i]*src[i]; + } +} + +void forward_normalization_layer(const normalization_layer layer, float *in) +{ + int i,j,k; + memset(layer.sums, 0, layer.h*layer.w*sizeof(float)); + int imsize = layer.h*layer.w; + for(j = 0; j < layer.size/2; ++j){ + if(j < layer.c) add_square_array(in+j*imsize, layer.sums, imsize); + } + for(k = 0; k < layer.c; ++k){ + int next = k+layer.size/2; + int prev = k-layer.size/2-1; + if(next < layer.c) add_square_array(in+next*imsize, layer.sums, imsize); + if(prev > 0) sub_square_array(in+prev*imsize, layer.sums, imsize); + for(i = 0; i < imsize; ++i){ + layer.output[k*imsize + i] = in[k*imsize+i] / pow(layer.kappa + layer.alpha * layer.sums[i], layer.beta); + } + } +} + +void backward_normalization_layer(const normalization_layer layer, float *in, float *delta) +{ + //TODO! +} + +void visualize_normalization_layer(normalization_layer layer, char *window) +{ + image delta = get_normalization_image(layer); + image dc = collapse_image_layers(delta, 1); + char buff[256]; + sprintf(buff, "%s: Output", window); + show_image(dc, buff); + save_image(dc, buff); + free_image(dc); +} diff --git a/src/normalization_layer.h b/src/normalization_layer.h new file mode 100644 index 00000000..fcf8af11 --- /dev/null +++ b/src/normalization_layer.h @@ -0,0 +1,26 @@ +#ifndef NORMALIZATION_LAYER_H +#define NORMALIZATION_LAYER_H + +#include "image.h" + +typedef struct { + int batch; + int h,w,c; + int size; + float alpha; + float beta; + float kappa; + float *delta; + float *output; + float *sums; +} normalization_layer; + +image get_normalization_image(normalization_layer layer); +normalization_layer *make_normalization_layer(int batch, int h, int w, int c, int size, float alpha, float beta, float kappa); +void resize_normalization_layer(normalization_layer *layer, int h, int w, int c); +void forward_normalization_layer(const normalization_layer layer, float *in); +void backward_normalization_layer(const normalization_layer layer, float *in, float *delta); +void visualize_normalization_layer(normalization_layer layer, char *window); + +#endif + diff --git a/src/parser.c b/src/parser.c index cf64b553..4aa0a79b 100644 --- a/src/parser.c +++ b/src/parser.c @@ -7,6 +7,7 @@ #include "convolutional_layer.h" #include "connected_layer.h" #include "maxpool_layer.h" +#include "normalization_layer.h" #include "softmax_layer.h" #include "list.h" #include "option_list.h" @@ -21,6 +22,7 @@ int is_convolutional(section *s); int is_connected(section *s); int is_maxpool(section *s); int is_softmax(section *s); +int is_normalization(section *s); list *read_cfg(char *filename); void free_section(section *s) @@ -152,6 +154,30 @@ maxpool_layer *parse_maxpool(list *options, network net, int count) return layer; } +normalization_layer *parse_normalization(list *options, network net, int count) +{ + int h,w,c; + int size = option_find_int(options, "size",1); + float alpha = option_find_float(options, "alpha", 0.); + float beta = option_find_float(options, "beta", 1.); + float kappa = option_find_float(options, "kappa", 1.); + if(count == 0){ + h = option_find_int(options, "height",1); + w = option_find_int(options, "width",1); + c = option_find_int(options, "channels",1); + net.batch = option_find_int(options, "batch",1); + }else{ + image m = get_network_image_layer(net, count-1); + h = m.h; + w = m.w; + c = m.c; + if(h == 0) error("Layer before convolutional layer must output image."); + } + normalization_layer *layer = make_normalization_layer(net.batch,h,w,c,size, alpha, beta, kappa); + option_unused(options); + return layer; +} + network parse_network_cfg(char *filename) { list *sections = read_cfg(filename); @@ -182,6 +208,11 @@ network parse_network_cfg(char *filename) net.types[count] = MAXPOOL; net.layers[count] = layer; net.batch = layer->batch; + }else if(is_normalization(s)){ + normalization_layer *layer = parse_normalization(options, net, count); + net.types[count] = NORMALIZATION; + net.layers[count] = layer; + net.batch = layer->batch; }else{ fprintf(stderr, "Type not recognized: %s\n", s->type); } @@ -216,6 +247,11 @@ int is_softmax(section *s) return (strcmp(s->type, "[soft]")==0 || strcmp(s->type, "[softmax]")==0); } +int is_normalization(section *s) +{ + return (strcmp(s->type, "[lrnorm]")==0 + || strcmp(s->type, "[localresponsenormalization]")==0); +} int read_option(char *s, list *options) { diff --git a/src/tests.c b/src/tests.c index 5d9136de..a6c3cd32 100644 --- a/src/tests.c +++ b/src/tests.c @@ -1,4 +1,5 @@ #include "connected_layer.h" + //#include "old_conv.h" #include "convolutional_layer.h" #include "maxpool_layer.h" @@ -223,7 +224,7 @@ void train_full() void test_visualize() { - network net = parse_network_cfg("cfg/imagenet.cfg"); + network net = parse_network_cfg("cfg/voc_imagenet.cfg"); srand(2222222); visualize_network(net); cvWaitKey(0); @@ -445,6 +446,12 @@ void test_im2row() } } +void flip_network() +{ + network net = parse_network_cfg("cfg/voc_imagenet_orig.cfg"); + save_network(net, "cfg/voc_imagenet_rev.cfg"); +} + void train_VOC() { network net = parse_network_cfg("cfg/voc_start.cfg"); @@ -498,6 +505,7 @@ image features_output_size(network net, IplImage *src, int outh, int outw) IplImage *sized = cvCreateImage(cvSize(w,h), src->depth, src->nChannels); cvResize(src, sized, CV_INTER_LINEAR); image im = ipl_to_image(sized); + normalize_array(im.data, im.h*im.w*im.c); resize_network(net, im.h, im.w, im.c); forward_network(net, im.data); image out = get_network_image_layer(net, 6); @@ -523,6 +531,69 @@ void features_VOC_image_size(char *image_path, int h, int w) free_image(out); cvReleaseImage(&src); } +void visualize_imagenet_topk(char *filename) +{ + int i,j,k,l; + int topk = 10; + network net = parse_network_cfg("cfg/voc_imagenet.cfg"); + list *plist = get_paths(filename); + node *n = plist->front; + int h = voc_size(1), w = voc_size(1); + int num = get_network_image(net).c; + image **vizs = calloc(num, sizeof(image*)); + float **score = calloc(num, sizeof(float *)); + for(i = 0; i < num; ++i){ + vizs[i] = calloc(topk, sizeof(image)); + for(j = 0; j < topk; ++j) vizs[i][j] = make_image(h,w,3); + score[i] = calloc(topk, sizeof(float)); + } + + while(n){ + char *image_path = (char *)n->val; + image im = load_image(image_path, 0, 0); + n = n->next; + if(im.h < 200 || im.w < 200) continue; + printf("Processing %dx%d image\n", im.h, im.w); + resize_network(net, im.h, im.w, im.c); + //scale_image(im, 1./255); + translate_image(im, -144); + forward_network(net, im.data); + image out = get_network_image(net); + + int dh = (im.h - h)/h; + int dw = (im.w - w)/w; + for(i = 0; i < out.h; ++i){ + for(j = 0; j < out.w; ++j){ + image sub = get_sub_image(im, dh*i, dw*j, h, w); + for(k = 0; k < out.c; ++k){ + float val = get_pixel(out, i, j, k); + //printf("%f, ", val); + image sub_c = copy_image(sub); + for(l = 0; l < topk; ++l){ + if(val > score[k][l]){ + float swap = score[k][l]; + score[k][l] = val; + val = swap; + + image swapi = vizs[k][l]; + vizs[k][l] = sub_c; + sub_c = swapi; + } + } + free_image(sub_c); + } + free_image(sub); + } + } + free_image(im); + //printf("\n"); + image grid = grid_images(vizs, num, topk); + show_image(grid, "IMAGENET Visualization"); + save_image(grid, "IMAGENET Grid"); + free_image(grid); + } + //cvWaitKey(0); +} void visualize_imagenet_features(char *filename) { @@ -566,6 +637,20 @@ void visualize_imagenet_features(char *filename) cvWaitKey(0); } +void visualize_cat() +{ + network net = parse_network_cfg("cfg/voc_imagenet.cfg"); + image im = load_image("data/cat.png", 0, 0); + printf("Processing %dx%d image\n", im.h, im.w); + resize_network(net, im.h, im.w, im.c); + forward_network(net, im.data); + + image out = get_network_image(net); + visualize_network(net); + cvWaitKey(1000); + cvWaitKey(0); +} + void features_VOC_image(char *image_file, char *image_dir, char *out_dir) { int i,j; @@ -693,7 +778,10 @@ int main(int argc, char *argv[]) //features_VOC_image(argv[1], argv[2], argv[3]); //features_VOC_image_size(argv[1], atoi(argv[2]), atoi(argv[3])); //visualize_imagenet_features("data/assira/train.list"); - visualize_imagenet_features("data/VOC2011.list"); + visualize_imagenet_topk("data/VOC2011.list"); + //visualize_cat(); + //flip_network(); + //test_visualize(); fprintf(stderr, "Success!\n"); //test_random_preprocess(); //test_random_classify(); From 5e468d1c13d6f3cd64747dfa0ef1910a1165941e Mon Sep 17 00:00:00 2001 From: Joseph Redmon Date: Thu, 17 Apr 2014 09:51:38 -0700 Subject: [PATCH 4/4] LRNorm layer, Viz, better cfg --- src/tests.c | 63 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/src/tests.c b/src/tests.c index a6c3cd32..eb85a5f6 100644 --- a/src/tests.c +++ b/src/tests.c @@ -548,7 +548,9 @@ void visualize_imagenet_topk(char *filename) score[i] = calloc(topk, sizeof(float)); } + int count = 0; while(n){ + ++count; char *image_path = (char *)n->val; image im = load_image(image_path, 0, 0); n = n->next; @@ -560,37 +562,46 @@ void visualize_imagenet_topk(char *filename) forward_network(net, im.data); image out = get_network_image(net); - int dh = (im.h - h)/h; - int dw = (im.w - w)/w; - for(i = 0; i < out.h; ++i){ - for(j = 0; j < out.w; ++j){ - image sub = get_sub_image(im, dh*i, dw*j, h, w); - for(k = 0; k < out.c; ++k){ + int dh = (im.h - h)/(out.h-1); + int dw = (im.w - w)/(out.w-1); + //printf("%d %d\n", dh, dw); + for(k = 0; k < out.c; ++k){ + float topv = 0; + int topi = -1; + int topj = -1; + for(i = 0; i < out.h; ++i){ + for(j = 0; j < out.w; ++j){ float val = get_pixel(out, i, j, k); - //printf("%f, ", val); - image sub_c = copy_image(sub); - for(l = 0; l < topk; ++l){ - if(val > score[k][l]){ - float swap = score[k][l]; - score[k][l] = val; - val = swap; - - image swapi = vizs[k][l]; - vizs[k][l] = sub_c; - sub_c = swapi; - } + if(val > topv){ + topv = val; + topi = i; + topj = j; + } + } + } + if(topv){ + image sub = get_sub_image(im, dh*topi, dw*topj, h, w); + for(l = 0; l < topk; ++l){ + if(topv > score[k][l]){ + float swap = score[k][l]; + score[k][l] = topv; + topv = swap; + + image swapi = vizs[k][l]; + vizs[k][l] = sub; + sub = swapi; } - free_image(sub_c); } free_image(sub); } } free_image(im); - //printf("\n"); - image grid = grid_images(vizs, num, topk); - show_image(grid, "IMAGENET Visualization"); - save_image(grid, "IMAGENET Grid"); - free_image(grid); + if(count%50 == 0){ + image grid = grid_images(vizs, num, topk); + //show_image(grid, "IMAGENET Visualization"); + save_image(grid, "IMAGENET Grid Single Nonorm"); + free_image(grid); + } } //cvWaitKey(0); } @@ -644,7 +655,7 @@ void visualize_cat() printf("Processing %dx%d image\n", im.h, im.w); resize_network(net, im.h, im.w, im.c); forward_network(net, im.data); - + image out = get_network_image(net); visualize_network(net); cvWaitKey(1000); @@ -778,7 +789,7 @@ int main(int argc, char *argv[]) //features_VOC_image(argv[1], argv[2], argv[3]); //features_VOC_image_size(argv[1], atoi(argv[2]), atoi(argv[3])); //visualize_imagenet_features("data/assira/train.list"); - visualize_imagenet_topk("data/VOC2011.list"); + visualize_imagenet_topk("data/VOC2012.list"); //visualize_cat(); //flip_network(); //test_visualize();