From 1112c43ffe04a6451616af1270427b145bba37b6 Mon Sep 17 00:00:00 2001 From: Phawin Khongkhasawan Date: Tue, 17 Sep 2024 19:54:00 +0700 Subject: [PATCH 01/11] Allow all autodisabled webhooks to self-heal https://gitlab.com/gitlab-org/gitlab/-/issues/396577 Changelog: changed Co-authored-by: Luke Duncalfe Co-authored-by: Ashraf Khamis Co-authored-by: Ashraf Khamis Co-authored-by: Luke Duncalfe --- .../concerns/web_hooks/auto_disabling.rb | 11 ----- app/services/web_hook_service.rb | 4 +- .../web_hooks/log_execution_service.rb | 6 +-- ...bled_webhook_into_temporarily_disabled.yml | 9 ++++ ...abled_webhook_into_temporarily_disabled.rb | 27 ++++++++++ db/schema_migrations/20240925134655 | 1 + .../integrations/img/failed_badges_v14_9.png | Bin 15999 -> 0 bytes doc/user/project/integrations/webhooks.md | 17 ++----- ...abled_webhook_into_temporarily_disabled.rb | 20 ++++++++ ..._webhook_into_temporarily_disabled_spec.rb | 46 ++++++++++++++++++ ..._webhook_into_temporarily_disabled_spec.rb | 27 ++++++++++ spec/services/web_hook_service_spec.rb | 2 +- .../web_hooks/log_execution_service_spec.rb | 22 --------- .../auto_disabling_hooks_shared_examples.rb | 30 +----------- .../web_hooks/web_hook_shared_examples.rb | 45 ----------------- 15 files changed, 140 insertions(+), 127 deletions(-) create mode 100644 db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml create mode 100644 db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb create mode 100644 db/schema_migrations/20240925134655 delete mode 100644 doc/user/project/integrations/img/failed_badges_v14_9.png create mode 100644 lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb create mode 100644 spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb create mode 100644 spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index 3499f0056fd9d2..b52e3b911ce867 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -120,17 +120,6 @@ def backoff! save(validate: false) end - def failed! - return unless auto_disabling_enabled? - return unless recent_failures < MAX_FAILURES - - attrs = { disabled_until: nil, backoff_count: 0, recent_failures: next_failure_count } - - assign_attributes(**attrs) - logger.info(hook_id: id, action: 'disable', **attrs) - save(validate: false) - end - def next_backoff # Optimization to prevent expensive exponentiation and possible overflows return MAX_BACKOFF if backoff_count >= MAX_BACKOFF_COUNT diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index 9fa870c9eac841..95f780c0577589 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -202,10 +202,8 @@ def queue_log_execution_with_retry(log_data, category) def response_category(response) if response.success? || response.redirection? :ok - elsif response.internal_server_error? - :error else - :failed + :error end end diff --git a/app/services/web_hooks/log_execution_service.rb b/app/services/web_hooks/log_execution_service.rb index db614f37db95b3..84f230285f4e6e 100644 --- a/app/services/web_hooks/log_execution_service.rb +++ b/app/services/web_hooks/log_execution_service.rb @@ -52,10 +52,10 @@ def update_hook_failure_state case response_category when :ok hook.enable! - when :error + # TODO remove handling of `:failed` in 17.8 + # https://gitlab.com/gitlab-org/gitlab/-/issues/396577 + when :error, :failed hook.backoff! - when :failed - hook.failed! end hook.parent.update_last_webhook_failure(hook) if hook.parent diff --git a/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml b/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml new file mode 100644 index 00000000000000..9b4c8432c413cb --- /dev/null +++ b/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled +description: | + This migration will fan out the migration of permanently disabled webhooks into temporarily disabled webhooks. +feature_category: integrations +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329 +milestone: '17.7' +queued_migration_version: 20240925134655 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb b/db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb new file mode 100644 index 00000000000000..5c494e61e0a973 --- /dev/null +++ b/db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled < Gitlab::Database::Migration[2.2] + milestone '17.7' + + restrict_gitlab_migration gitlab_schema: :gitlab_main + + MIGRATION = 'FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled' + DELAY_INTERVAL = 4.5.minutes + BATCH_SIZE = 500 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :web_hooks, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :web_hooks, :id, []) + end +end diff --git a/db/schema_migrations/20240925134655 b/db/schema_migrations/20240925134655 new file mode 100644 index 00000000000000..325a1df92e184f --- /dev/null +++ b/db/schema_migrations/20240925134655 @@ -0,0 +1 @@ +f589843d4a368084ce22ab0e273e566a6b2f4ddf39efc9ffb7bf8985e392b6cb \ No newline at end of file diff --git a/doc/user/project/integrations/img/failed_badges_v14_9.png b/doc/user/project/integrations/img/failed_badges_v14_9.png deleted file mode 100644 index 5a1f481e54c3f992035ef08b634d3960ab34d831..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15999 zcmeAS@N?(olHy`uVBq!ia0y~yU}|SzV07nTW?*1=a!Twc0|Ns~x}&cn1H;CC?mvmF z3=9m6#X;^)4C~IxykuZtW(e>JasB`Q|Ni~^)6&xJYV-ZQa@N4W;Oos5zkdDt@ZrPl z+qb`d{rda&@1mljH*em2`t)hmtXTyG1=p`%udS`UfB$}4TU%aUUQ0{M#*G^C>mb zeECveU;pC8iy1R!Y}vBq*|TRyj~=b4sL0RH@9OI6@9#f;{CG)8NqKqs;lqdL%$c)j z(W3S1*S~)K`bBQ!%9ShMzI|(MZa#0`JWETl5u z-S)~yKE^9nte6qP|M~OhQ?nDl?B4XIwdBK=Rsa6{KH#jsO;h+psPjDok)JP4|9krI z`MgQf0=d6`|9;@WflpT!ZB-Nae&WEdvqx`kteWD-aoEG;{q)YIX(|tFlzyK-vA)FQ z@8#3q4)1=J3K?%&$o=J93U*8ktWzN*T2e_`&LLWBK|I&aU;+FWII zeo^L&W0UXq)PLHsexIGzjq02~uV1{I)cmqI;l8=_Q9tYJ>&h+{Bz(KS_3N#be{Ws+ z`~A(|tLOIGsDHk`^ipoji~N|Mx2`TuRyZB)d9^gP-<{#|ih^TPBM(mqz1>uDJ~ia= z{>~lx5_2L&KU|vscxuNaZs9m5E=c?L^T&@T2mU>{{X8@D;qKOl z6PmZxJM1!*|FCFkzX#(%w(UC@7!(*hT^vIyZoQcs8yxH@b$oWf|5pwbootF#CZEKn zYUndM@d&wmWR*N7A)`Edrq4YeCeDPn0xawYeHIBlj!1nO5?GuVxz$>{xOsZ@*685i z-@Iktna|X9uDrGTZLw2}#s(Fw=)*SqyT6>Yyz4)IN&U;^)&Zi&IGPkN&=1~;FZ7i= z?k=|9xUwzv=^TZ!{!g3Ey1Y^6?Ti)MSTD?f`bDi`*-WEP6BPJ4n`~50I5#QqbAFnj zaKgFCMuh`IxGq{Q?)`aL)A@@O75TS+xxlL+_qyVB&WW}6l(LlPJPPF9Q>@zAR(?)) z;+j8+TJPR&`FLT?tYa-UcWxcNxkb7#_MT_3!oP2CQd_nc@E>nF{&R1Sx}>A|b=K?* zpA(1I+U7i7mK-2sII}1{`kv<^*UE}R%bztjdfrntU-$9L{M~83^h3bj9y_~-MLXK3+~EBD zFGv0NhBG@w=YtKZjOdQjV|=(4IB>CIi@2nqxP(dah!^GnaL{_A|Q=83w2 zN_B)nCh9aam};05)XbIEZh0c;@Hzg(i=)T zi!x`jwc6ZC+n>|dId!f|;k||>Ip-VREDYOt_9nwrj}5D(ueAs1hhov7~y&*A1fO$BrN8H&@tp)K0aN z?_BJr`kh8II9FJ%U&EJb)c$l=nMq;X{nW=cO=9O3$4QBNDo~dT^Ln;AY2%6~Pk;Y7 zbmVq*n#%m%?7POs9DANVtlX&f=A(u1PNPmw=Decc4WT|QY&+kj-mnZ$oMG;Du;|=j zZWE0s5B+RJ_}nfVgrAa=I$N;RcyA5EG}e}iLyZ^i{@?0vXti`(ThDgmxp$?`cii1P zL*(gu{_3ac>KOqi4$D?%tnRIFx}UOtduF%gnYPt?jwYDJn-u-aJYxR++IKU}^OqjH znjI9U9_Or8c;RV>oWF))tY_ua7hLPUbiVu$S$5rNLc8_f-mQo+vuX39(=ZyH^_?&XVi2f7DvuO z75+A*U2R6WNY2dza*tk~vtIA~_MCRU>`tkxUMoL^-DzHUiU+SEPQ{h zYI1sVb-ui$u5Qo8i9f%8*&tTh6M13t&)EwOCzfSSt5xCO-Z$s-1zDS~*55a*S{WGZ zV(gn79IB{jc5S|Vtxaq~`Q?81=l09mojl~^2 zJMp+{$@hvS5&Y|QS=(m*(N1IATrp!!+rdW`%hT3ZDL=byW%k@!f0b^?hSM*Es@7Tr z`L5yL`-ZE}@UiRs{09?eY{|O(G30n|`_ks8QK#jy*rp%Sx_hf-RT0a(;QyH7OTHXm%^uU-8?K^@!Nq=H~j851tuM` ztljVYFLHr>m&XT*miDs+t7kq^(TW%J= z_r3k8o{e9&W}xoeIJUpcdAUhTb~T~ntTUwgCilRKMa_rkr; zRd?O==17)&zx$BT%ng?gX-$d!a&TIVaE|xn!Gq9*@%AG8sh5l7@lH|f{dJOl!NR;80`meJlIcn{< z&3E2DXPU0CZAGDD{!u59!#Vs_6`60J$N3(uj+ogy;cUon-Zgq#_O{1QpH}z(kG}ny z>k>b|DOYHSMg0Dlv$#FUb6d|pMV3%~;hRw#Z@)1)^5#)++yTbQs_yPN*QZS}-g#*e z5BDbB^rE$`?XfqfSF7&6FCnzrcK@av?MBCn6A`T&_WSwF^?H)Er)z>xG*{UmloZWB#mqg^OLU z=8Lmh6L*FFXxn(RlwWp|-TA|>O!+_O*^5Vf+#CJz!o)-p|Pimk1$t6s`?VsA~iH}ydIi_t|FtK;)=ERkgT(e!?3e39sC|#;d zBrl{RMkZxc32HdTh_|Aj_HS?w+R zlV~I3^kiQ|W+Y#j@?^dRZfCsPtou0jw})Mm5}qBoF*x=6gihVZkxPy9N*1p*(vbbk zG^>Vx^V+Air)n*hPg6+_y^|xDb=qXOc$5Fj+5s*`f0Xv#wT$FLaIb@{YcJY1+Rnx?&oi7K{FBcFJ7(DL6s1XMbso2#d?jKuII3b;gE< z?^1-zZ~Dyf*(9v*^DXUB+>Em-Rl7Tli%n;C&;6hDEp792+xNHQFL7yDwA$V8ewKUx zbB*_{{_~dSUsn3W=JU4q-IrgT^lCQyvhUqX=gX}AGwI@s{JFxh$?HrjZu|Kg_Nr%G zTyJMrS@{2C;76n4mXM{l?we}7?|QmZ=F`QJ`^&yPdQzx+a#gs$QSzeX%DXnLWBMf*vqtH);=9P3xhpSVzyPx5Ly^FAHEndjH*Y4mZ+r~W$cFImna&bG$r zRO0QGO`1vK1uOE?XM8F?*3VyJbcWfpDlTuv(#g4pXIsv(sr^-=Yg{|8=2O+Fhrd1> zecs=GMx6g7*Q@>ai)HrJoHU64|AXI3fBCxmt%ncUC#IC#f1i}}s<$>Kd-3(lZgVdu znfl!3+}Nh6yZ79~t0x;98_y^CJZ{>uOlPy*8lm$={GqR}Yf7ov9A3|FeoXOXd%Ufb zj7-g!NsmAH-=A=LZ+rk>@b%kgAEtfDyC{9}?6c1Y`A@vsa>bHIy5p9_u?5!EuL_p= zDJncmil~UYC)4g;pRlj0uI+{B(+}}?o~F)oezb8L5BSo(SjcZ{CxjcW<%$;&JGvfr;d#r z-p_aZ{G0{T=l^{^y?))l+1E7t_t}2_aZ~%*@2julm#>?6Sm?=wi)Q|2ldaBtzIe#h zh;{1yxQbmLt7fiheRrGV^QXVhiu;;m-rfD3G4eJ}_UA@nCPCM~1|7_yTYZrsI z$u7H{5x1?_;?C3O36jflwIefc^Igt0xo>%U!tOo#4|;dH#{Qlrxr}F<>+OVn$wrU# z-cQ!@uT{LNbn)%Yb51gob}I#%od3Q;((lQ>fO)xDt8a2}vz|T9cToDwYUz{C%WdC3 z2s;;)a^msLteEVb#twI`)-HbeBxr4`%kt_q0`0e$cuu;k+se5tXR?f#k@>BMyw9#K zI+w#&%(lHuyvlfU-f=y_@)c%B5?%RtzR67$U2vMOZNlbNtXCGMrk|MZxO7@VYBhuS znu&@D%8x^puScx(Sar#zS+H~$*I6@FnNNOy1P;a+FTEh#!spr)=zHr1zcM;nv#i9~WM6Xmsx6RGF&H`by+pr@YdP z{tI8{c`{AA_)_QdzlRJQ-#1x2dHngBX4Hh2;>Dj6PiL2&I&neN;OvD-c83#VSUF=k zUQbDSwBe5_pRV!Owhgj-oM-)-xm9MD*2(F2lVT^y8gi|$(9_LqJN#KK+q#qSVZ=WdwSuG=W51V!>3Q^ zv1Z|G&w8x4{J7xfcUx*_-#wdXwRee`e_u${;<5=9a~tQqshk;|KQX=6@Y55&12=Cd zE?|26PIS8cj5nuuU-hwgGPgx>Pi2xyRT}{p+T$V11?HVFQ9Zr0KD`KPcnWt@7QT|{9PnYM8rQwTYwmBAZ)<4qlfEtPg;(i4N)D%X8$Cioq3G*b+1d3vGTwPNQqYqr-NDW(CX zt&F>_ChJ!I>Yt(=eNa)ZQPYv{^2>rP*V8$^JI%jtaR2(ehZ`naJ8Ts?WAfCpMrTJ{ zexJ~!L<~!@>zr|YoE1roc?!54vW$%|6s|9xLxFh?i@$~#RbzC{ACK>sQWG*e! zHeg#4sQEzf<)o(ubNjw&hwo^vmUnx#!+CiopJEnU`&H!D$Iu-7MwV$ zrh055U-K2`&xuoytFE2QdsEhF{$*3n=L#)qPx;nMovG_hS3B3W<>NlfKIeTePc7Q< zaP|bIZ9NL@Qd{i;7Awv?DLrRHO5qDmhuR~GB?^V?$;L91zOc`o#K-LLW5R)lysCv> zo1B;%3e4MWw-qPk{J;neZylUx@8{iITL=cICx9a&ZP1bi;{?&D_4fF z2kEst-pZb0u>JL1O+NX8w==$lTxMABvtp+>&p)lbD>{wTW0k^cE{m2=K3XaHS|sDq zfo$G}-jwYv9`hawl+=cWu42y2wM%(_UFZ1DX*+%FNT4b^3#Uz(mXCC&6uAKGS ze$ng0a*=1|z0yz2Janc-h;OR1*xHOv(Yu8+r_K?*U*dYLUH0#e{NO_}%Y+xHyy2Ua zVsU?diT3M-{n^HwK9+xp36{OzIOX(0omp@0F{`bwF0AkQCSUk<)7)#PZQ6ym#GPvq z;tT(B@M9pqw1ofF(_3D0?b@}}>dtY!RKAbpwjqH99nWO~Tn{SF`|Tvk3QN)&PXtte+f0e3biZr)w>tnbIaWB2#Ct-13f;oQ$- zACvzc3ck5gq4)dz&mugTqTT$@s{dZEeeb$|-~aWSokF*#e_jl!9c_uTj<$cU=P24Z?d85xIl}EncIG(LY;&uq`hOr)>b~E-Ee(Eklja|G zy7@uB{#@0k@B7~!=!pnSubTazd+nW&@Qsq)Z$4gUe;<4A$?e#E<`uQpmOXkkf9k)T zvt8Di!~5X=w7P{s^VZt1pZ0cH)!?c%?Rco8NN2hB3#Hq4s+xR4`17gVofbBi zA2QT^XFs!6f3EVcR;v{fuhQppdjDU0T=vG@k8AsicC3w>%Q)}Jp)Qq_jklRygMPBf znW(=|scs2a=%sj9D>l)1om$1*sn=T^h13p+#jrih)VlhM;e*5cTbE{qRzJBEns9BI z%+j!IOQ*@sPfuKBpX(^I{IWqYTX>{M;->LB(dn;^PhKieRZ=Z-?=~FE>?4G2H+mdcanwv^lkCiZjZ*!d zADP<)cXSnS*e^(s<~qPH`mth~?aBSU4+~bT{mjXG>dE`Nmd@4Y+kY=zxP0o<4@(bT zIF~D7^5u@NKUuBAuiXG+_C-`9BC?$#}{Q%}F~OTXjYP__R5vwizAB@Ev_+^&D- zq51rEKYQo?ohx%|?&M~fQ>g_khjj{XGAAnrt$a8=(cq83ZJzEJhG|-NKbNdzaFy8n zjOo}xR+S@7#h1h^LY6O;`p9xqOZ3#4TP=%M7rk9%s+8O(-f>Uhq(^1XH1}^`Yz@vR zJ`iA=o@5ztT73gUg1eCZda=DZ7E`((y8kH^5Bv1_l!4vx%a@DUTTaI|J`Grtt|BG0 z=kkeyq#Jkt}^UYPto{BE0lRXf}78wz*JmcPDN zd%5cL+3k12-`@Fl?cTa`V(&g|zBe&!(e=BREEAsc*UwkwGhY~Zq_6lz+LnZ#5GKJN zVm@ad)~PQ)_A7h3uI;vepCm0*elBG2k1T%Scq;jiyH%EjIG<#-=vzhBER%*Vxd!vs z_q^^hEaKlhaZZ-bA+LrQACqk1zA$P2a_;N{%$Ge2CyMvW6wT34l405*^sBb>^^$3t z?e=TeuTy?_v+b}?S;fhhGmh-ze)PE0Q9fcl&oY@!nMsx#<<2CePStH!u{9~!-LYQU zF8|4gen-9UcZ1*UWT`g%LX_UYTsX|$hA zwy^1bxZ$Jq+?Su+jBYFX`Oca-BPJq3g4ama$SEW2obKuY{#N2_(w0kH z&uQtT)h^a;zhdoIZ^3L^}6!ob;!-ds$b@HR4_mhj?^Q!T= z=SlIjU5|NmSx2UCkIH)g$St=r>n~e9{h;}_?0qit>U0SMo}QEZ>4tjwr{(tFwBuW> z8_lZp%Sg4|{iM?VkLSc66_~$1(Zp$FoyZ$p6*`40NwdwVk2}R6Z$Z3ImH3086}Jwy zD}*f2?mWtR!KTl7)m(RrqClfK1?3X;bxR5()pVa33YACn+|IFBGA)vC^^~3+tBRcX zm^Ui)_ns~}(;g;}XrMJwed6lZb!m*D6B>Kh2fMZHj-1u=Z^LnmM&q6KdoCxvk<+?QY=3%T#o3|~0psq~GYh^4b=b0*rrJLpyX?(k>Fz=T9p~II0g6g&RKRuN@ z?Qm@*@6YY8^#A`n9bZ4|w(;S_l!-A67cZT;EB7Kw`Fl zb$|V~p!XBE3-db}$+bDO9(gXom1}8|b-8l3!d@fm2^Cj2*_?QI;_Aj4f%jK5vrq26 z(dE22_0+>fO=ljN+%aTqFIcqZiuG-)c>;XGPiHb#2E|0Tv3r!xPkAfQEA#EUL|Z)H z%-2sH4<|6Mo!+k5uA6%}?CYl!*{`c^&Te9L5=o%PafX_<1>I_-;C81|Vx z5!lT4;L0NY!pKe2*`iNgkeA z&bV>ux_x@n{sc5s-)v3zWu?jx^j+ff;r_=HejN7hjf{-!Esx~$k5}|v&UR|y&c8b? zx47H+RtK$|^vdkXq$wp;GuO{AmN=f6WgsZ&@L>|uS1IqgP7Yg+7JS-~Bi|k^e{iYv zPZzlWzWj}6g_PTueG5OJn4>r0`Ly~gi@!g6`F!88q8%?eB2AhN7ICfGwB7MRyN8ZL zG>`i8f^t(&&Fj*ZJ+u1`-|iKE_@lJwiRu==$`a;`$xkobnHR|?p5k{Zg75E)B@Y~~ z`sdwRaG`fntiYj`4?eSlgO;rgF59>-L}#+j_fr=nEP1S+9iLzG&*1;Je^pE|C%-<| zubR!4G51o1?*B($**|Q^yq{WJ-FSHY1ZR#ZT1#6QR%>~GRblzqKjmN6^S@Intn!n) zPOC4ybz^>(}Z`Nnpu^KTl`e%`$BdH!L^8o9lvpBq?}vpq5DmO1zN{6VG@X`gnT zwrjt*(^%o=n?j58`^62G&i}w)bRl)enYro@5;A3+-*5Wx%#^A zF~ysw9xm}ONMtIj$^Om1ulM5Di7Aphby8Pld*@UyUo&rBV|}#k8-)g0{^bQtahAys zH%SUq2dsQ-x#j(bG|nTHfA?PG?@KCFcz?Sz&~kF2gaxNe-khnAZQG7!6xu|#i9{Dl zNLuzCc8Gg4spX~Rmzx=97Zyy4Q#@T6%#&fn_tt6;>qc+8F6Z*u-WzK+-#3!We_Rw- zaEi;(=DhvW2VGNKPPcM>QWbwMl7D^EmbW(Rx4dV)A6l`qDCRHICG#ae>ov4^Sb_;qgd!rh5g8 z0hhOjWK%BBL|KEaQ}*&LlR2ImUUb7tZ=LP$32zkJW4`bEzII8`z032q7u}sVQ!dj% zNkVeNw3`3S%~5yne=)G%_fGC>d@biy?cSIJNv~J=E@Qpt^tsUA$#e~qvw6c!o}if` z_8(Own;q(lPE2`z;k8*)_|mCaS99&8JX~K-%Fes?J4iN3#!O|Cv*d>o|Li#hPo9dp zI?uIfKefi>*3SPjJNds z^Gkj=)h#;d5F+f+c!k5nuk_CL#d6n9uF6+?0V}S7Z+%9C9}xmdHd!2 z%#RE1C4atkq;TdX1^)#xl6Lo2wp&IOrAMq5(&jSHDv09dEA`}KcMjcfeebNKR>2*a z3=GnOptS(K4wu=QEg9Cj=_PC|wwkf*aNer)0R9bApHGPRQId-d*fC(fRp?N3rKuU1m_U zKmU7It8{9=hwzGBy+;%u7lqzeJHzrlXwAd}iF4gs8bm!dGEecncl`6ehJsT}?Lm^= z`e*oeA6oU$M^U~ibl04~i<8R29t6Dl`|xp52UEp*E9C->FV8woYj?F-Rr2Y}J(%#6 zF`2FG&nqQ9>9x{UGiFcG-XtCRxIyad<4LclgihXV{BWs^S1$`+d4%1ZkU5ce6zpGkpaS!y1p8E5GWj!pir}GuO zh|1sk)y%Qt(v@9tXA-;H?mhMUzRfBv-Tc(IL}^TFVZcITi6IWUkeVYb%|Dx%isvth&6mMxs|XTBx_`HvqU<+^Yh^e2KHOFe!bnlXV0RH6_cME%-4G2E^gRcy@E6A(7q!Jiekjn z0=|T5=%!3<(HBWK=X&<1KYG{Xn-eGX_iDCp&NgdwUODlA;)Sn9PbY|OFm|te{G!S0 z@fWW5b*q^2x9bZTHHRE~wEown3Gbde|NNO)%HVS|EMaRzy0PW_@K?2+e7$^`n+~fb z?p-5z?eyWTv%*11-yM|n)Bo?BH+`Am>3r6OkB#cqec%1~! z9rgTR`QYb+4<@?4^)}s;A3Rrj-gsD9DAfI{XUe>yO$#&Y7Oec5d+zMEzt`EkBNiJd zhe@=v*9OVveRbsfUDcgeIDN<217VMzmEZLK(e+84Q{|7Z@wFH~m_ieLAKdedst}9Z32Hku9_;#B_ow_tk?dc=AMwP z2L<=rFa6BCWb-Um{V$P|`WG$A&aC6|NxSOMbFJ9y>-wg1x>m0uf;VzpdbdV6MOx(P z2Gc{)(F(~=5C39#w|boiAOEihoM@MJG~hY7EK+ti*SytP z;X&F@4)2+%`TqHY`h4$Yk6WD^oo`o(d^^OYKXv*c<2Nj)5|?|4pEo@+Ir`O=jwfO> znGRp=yV)prVTR%g#>;&Ij(J*QHzUdk7F)D(yg&UZkXRuEgxvPT%s-HL?@Y)M#VwJw3v+)i_@L0f zI>C6shaks!hj$7xBv0Sfrsli)%tJSE2OX6wr%!x)+{`edOoDoHQ+|=_?nt zE$NdLkzsQxtdDuPRI5PZ*5q$pPnU&O@|jLKHE&LUf9%^yOd_)`EecFO@@TTfn$H&k zHmYxF&pWYA)8a_s&Z6s@Tzu7LGk#~tw_l#ieogc8wIOkg9x~6)`nbNHqM2y8 zHa~%#wY|{JZFvW){!}G-KgJigdh%u`%nf>Sc)ina3mucCu5Ip;2PYlR2{4iOidR=O zIb7;>)PB2W!HxB$XDw!eOb?pabXd?{cIDw`7iU@OKQ6d5>*Va2UGr`qG28NL!TA$> zr{1%@e7f2J?1wkkcOGKg9c^y?STNT6(4PV|yV)-iFU&lWFRf{^)b-rGAIYI}bDCH2 z?%HInRxRZgB)e9sCNKEm`HiA7uMXS9>U+OiTq|RwJ^9a{4ZK_IOk-1R9jn7iD`!>F-W1`gHkg zckq##;HW1Yzs=V#NtK?mAujKexvlv1)3blb>~^x$`FCP%PyXz~=f4KzpH95ko~M3X z@m;aQ)cx;Yw^$shjbHJ&umjgf8F;)6GRfouqE-o6vWUFOY5ghC`q&6n zIPCoO^y~R``wFCf{ak*1ze4DVhqGUPZM}RY`^c*jiyXD<_cU$;j>BW@j^bl|V@P=5JSe=1r@eTVxjgUuaUa*x*8$Y5u*c`TK4=-a7hi ztI7qRe-_!i zg+D%}O3M8EvA%qcfb{9V*N%Dj9_J2RVPCkt=kndT8k3!iUaS4QAbMN%pTl>hi_+^S z^y|t8Tu@TE>5^A-#p9Z#-mlc2?wZ4ewkaP^7XAHpwEgR?sa4Y3z1gmLOxt9hR_S%( zxy{alC)cv}IXyY{{Ok68!us#nc2;&@TCBc(pF;P6tCM5BZ6kYZmT+)REZv%-`6X%c zopo#Dv+f;V7_jl)#XTVuO9I@Ug_WrEBSlaK48+jhLLSS2Ts!d7i{a!Oq2{h7w= zXY@7BySVN8nFkHOjtOkNc)Vxd%QqKh{L4+Wi23$&*6B2h`zc}Xe9}%XG!p7tAi8T$ zPG{3W>-8P+?N261*D(5=Ww`T6hwGFO}zMyob~ zdskSW)DPBD4V&AiPH8c3dnzkDZK27Bl~c63JU;oUtrkA9>*w7Ghu<0h%@0+#vuUxO6e*R|m^|^Ns*UHam+_&!Cx$S%9=Nsr(&*(d+xNL2+WX8K!8OeUD zI_}l(t@!c#P|*&l!bir{e5`X=^@2H56XK0b+pB$j&TWm`zh?Hvchf5$Ot{kSbY;=` zg{qnT7t1wQbd>*7FkhX;?y`WPp=KB3ww#L9q6{oDX~vdU4P1E{q64Rd%`Vt7>Gqwi zYZ$$!INiF_RB%N3c4tb9Gefb{ep}ADSB#z(gx~q6HfbKOg-?38z8b6R=KD<_3X&#> z%nSd?QWF%gOzDteLrj;H6tAVq(}lAnYfc+5zqjDKziK|u_x;xjemr73%u~KkDL-zD z;h6`2|Cz_~T)up9bg{cl2|7LH@Zl52 z3yx^CPm|)?In`{l1_O79+b#8=E0ex6ay-4T*Gf;H-+aF1;&&IGJbh;t8b7_wd78;H z%Wg4Q!}TFwil-gU<87~9Z&y}Uw(sTJO7nGgF+cXqnPyq|@r(t}1zpioFZOF%+IYov zo&WoqyKqM@Z%R>8+}>6|uQ!93&4gJ$DXj?Yw7rXFbJ zlnHvvbZBDayem>loOc83O!VE`!je}WV#@JwyZU@WRGMYZnaRIQ%p{&(5S%Kqp)zvz z>?n_ww-_Q8>g4?DWHo7DbK{6z>XPN%E|z_V4=OI(|8{AdO-jhEiXXrB#@Wa`F8IJ3 z?Q4`{Bl9Zuhg~XD`^HjN-X&YN+QrRI_P>3mOR`u(Fe>Y*KJ)T`gwL~%PLVlu*`4KB zw0pFFvCO~ES1txuKiJsWd9v(mMV^I^V=BAHLJ6&HF22XPC-Q3EPx*Rk)6?G*?0F3EzI<+Dm6M~a7b~^iq%`s%A8ieE;Xf9Q(&pjbdi=t9adZVOTJ(H z#W?YC!3JO34~Z_5r@M!|l8ci6wuRwU+Pxx01en3Nx#xyxsYJ zwY%6in{O3-sjC@&EY)LQx{*P5S#o7$p!4LlNm+cKySlpOBt}c#{A_gU;g?Fuf48}o zM|-kY zIXc=T{N_zhKlX5!d%?>v!-?!u4&Q!LV10aH+8^Emj%=?f(XZ`aZmj73vTyxInRRwf zae56WHVaA_Up`Q%Bg}pLKBJLl`|WR&CmHoUHb}AJdp~D4NB!QP`3-Nk_*a&2znJhe zzu+|@8@oorRUapb`=#&+?8fE;L!G~T}O8dtBGu?8fdev*qw_mDGZH{)&kIP^3JXdd9eCB?RfII(w zG=dD+e)M6E>HcT0PkV1V@=&PVd()AHe$QT9JkiE)Ht+j$-TK6G?!OGrkDh-hW2bd5 z$F8T&^qtt3#&6w!>>?UY?6h&1_&M>*>O%92AE!@t{kOY8Q@Qg-<+O>1Rs?3R{>kMN zoa-!+U+8kg^`EuozOv)DK2*48C8&41=P|a!{6DJPV$CLV?sJq_zO%KG z!-DT*!yY$*;^=>m>`ppP+ibZ?zEC*nvHF9bzC0pPivzE|`w{T9swZ5vl0D;*qT~kmQqyCf zTuyshiui6ia+KTuUEc~$B^#OaqYKllZ+tb<%k$%oQ~EdmlzL%CqQk7~QtbRallN?} zIT$X{ca)R8MtD()jMeSrH#eJ3+fFi1eEmeeWp;V=?9XRpRWv`Ie6%qy=zc~o_c@Mj zhwsb(?0eu~QPNxdWXFQz)+^a&ryZ@1oE$jkXQh*-sm-y4+uNDHnQfo;CqujY=%ZiE ztA!UG`FK_C$Ak;!j(KNfL%Y9TxV6!1^3k;mWvt$8@fFD3P5|1xjDP#VZ*?q+H;&nag>SZ@nob;hv*E^d#jNEK#~;qp(dnGdK4X2!RM&{lNri3dy~?M{ey!ai+r8wM+hwB- zcNad1eK_0jX+T1`+-nP-&ppS=lHPxfym7aDe!^POXFRVTCBCbAqi}d-MPeanF{P zrax_IUUTf|;aLhf^P67JWid3naoO#GT>jZ(&J**u?GE2L@3HDkc0SMA9Y3Y670h_8 z`F+~uJ#W73FFUtYy;rs8vE`ns4c@1wZSHVcS9JS;5@S=CfFp~(w1=?5RF9hbE{}CL zDjEh(`WAY5$^0iZ$Beh9inplQ-n;PmPy7xeYisMYdK=^OZ~lDkebycrtGRyXLU%8} zZ>gQ3OU~zS*LJ>P*n3$2y6bZD4bJm4A8!x|=jHP(KIgz#r2FAY;`d3ZhIvk|^ zOJC@8C}&To_AY5Hs;HY#B$8Idam!)L<1>sq%1{2yyYnO_{+{oFI^iiYFSaSkXeESQ z&ed#ou6kbkLdMxn#BMLs`$!YNWlN@8PdS`m^h8%|;^F1;(+@uQa7ujl83R7$-_tgK z)Su{jBCBhXP9UGNv`wGR-0)@nr`yl1)v2*Rr)SKUdr~Umkz#1o?%NX*G|yQ);hp_4 zU~0^CxqwMsi`I6n3*h5t{`=fQnx>2c4vSWcUgS^U*`N+sTo&g zw!NsZkqPMvdcE2=aEh_@(+hi#K5KtnnHhh)@TaCl%!~&CEtdU!=CdBlwQDyPi+{Q0 z=6Z0UEl=6=0WiL)jm&wxm|H-2L-YbE9UIogJ604Q`e_mP1P;$LzN@wwk z>IF}4&zrb>g5I&7GL3dlWw}+glAC-#sTy0FtiJnLVAor2j`Ql06^jsm_IrMBwPd1aZ<^pF_sG1dEuP6v z&w1A?smBJTvovMCE?jYEioK=lVKF{qDNd)iZ;O6fXzbb)*12qwv3IAlp_-0GZ;APY!?gGKvoly7ZgrnmlEviek!v(q`-_dLA#>!H@nD@z%pj!!aP)V1!#bWhVOSMPkL=OX+wz`XSaHl^N&WGRzXt3@lg^YkZ})nvuPk@!tCY-@ zsX2m&A5Jc?5P$zQMCkXQ{#t7*y?)R2+vDC>?=?UFQ07hXhi7lYx86&>pEt>Q@8^G_ z?~5yb%bYv7Z~fCQAq~54Z`CaK{1d1-`g`9$cKDbhc%XqPhY(tVTQ%<`P~ifb;4KQ8gl|5qpJ zc0~N~hr2U`wLVqzPYIYDR_gzF!rYaAR?g*;ai0`aEZIKMUBzMhgH@&7mEE(}NO`ws z2iy3(dzf6%aXTZf<8bSJi-pH*0#;>RDl^oU>oTs-Sl*OoS8?@@0iQD8_80nxvrb=X zH(8<&hw>BYdxa4Vs)D0yn@p2Ij)g^D5rL5 zw$rw%d4gZ>k&g(s>dR(VtlJurJM{#4+YrN}+Sy!b_q zXC4?l3fP&O(c>JKA1gKG|F(jRt-+Bw9-H5W#J|$!`zh^DZMw>75ycgK6A8C1}aRaa9W(WS-PkZhO&J5E#>i+q#?&`UxlALGdE^i9h zS0!S~r`A4`Ysu?XM>SK6Jk(jEzZD5A*rsKnQ?IOC_HBBBS;ybMZ-N4!C~fz+^^*PX z76*$bwoBGAN?OiYUl>`@zN0pCj&q{4RCoKHxSGr(505UHdn)VyjLhXqMX@R&g$9-( z$)7$NM$8FPYY!ERh@Qw>f9XNMy2(QEM{a1GJEAY@dhinae%<3UCbP9T-*djTuIKTm znx~p~tX1B4i_ANFKtpccR<+L}f#-$z_0`^ni1DT0UinC6&DI>f!VQ1?S>uhLZM0f? z?D4`s#UI1k-%P2ncr+nit1C-gOT$u*|FntRCb@UJ4E<*=FO@!~_??gQx6oIzy8JZKh0H z(cXEMzxG$v*X0fC4s<-9-lbG-y(>rh#J2Ys=_sL$J=M!$X9Xpm9_Bs$qx~zWV zxj|~}eS7U*`dF-w-EW^Dr#4Mr+a&(kg|!=MX2xrEwf)O_U+lK-OHa+r?*+>he-_C8 z*4=Ym*>TfP_xBbb_R1Cg=*a)p!gljTVlwM@%LQ^r9yWiEtE{bE+5CH70RIz#R%~nP zI-K|>f>!2%s4Fpr5yxuZuXrvxWx2?`n}RM^HYa85pB+{czHM>9`Q@Kv>%;>!M7Ua= h76xFU_S*gB{?n7MQCE5{hJk^B!PC{xWt~$(696(hhur`G diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 8229ef648c51fa..57efdf8cd13444 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -481,6 +481,7 @@ To optimize your webhook receivers: - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) for project webhooks in GitLab 15.7. Feature flag `web_hooks_disable_failed` removed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385902) for group webhooks in GitLab 15.10. - [Disabled on GitLab Self-Managed](https://gitlab.com/gitlab-org/gitlab/-/issues/390157) in GitLab 15.10 [with a flag](../../../administration/feature_flags.md) named `auto_disabling_web_hooks`. +- Permanently disabled webhooks [removed](https://gitlab.com/gitlab-org/gitlab/-/issues/396577) in GitLab 17.7. {{< /history >}} @@ -498,12 +499,7 @@ To view auto-disabled webhooks: 1. On the left sidebar, select **Search or go to** and find your project or group. 1. Select **Settings > Webhooks**. -In the webhook list, auto-disabled webhooks display as: - -- **Fails to connect** for [temporarily disabled](#temporarily-disabled-webhooks) webhooks -- **Failed to connect** for [permanently disabled](#permanently-disabled-webhooks) webhooks - -![Badges on failing webhooks](img/failed_badges_v14_9.png) +In the webhook list, [temporarily disabled webhooks](#temporarily-disabled-webhooks) display as **Fails to connect**. #### Temporarily disabled webhooks @@ -515,10 +511,6 @@ Webhooks are temporarily disabled if they: These webhooks are initially disabled for one minute, with the duration extending on subsequent failures up to 24 hours. -#### Permanently disabled webhooks - -Webhooks are permanently disabled if they return response codes in the `4xx` range, indicating a misconfiguration. - #### Re-enable disabled webhooks {{< history >}} @@ -528,10 +520,7 @@ Webhooks are permanently disabled if they return response codes in the `4xx` ran {{< /history >}} -To re-enable a temporarily or permanently disabled webhook: - -- [Send a test request](#test-a-webhook) to the webhook. - +To re-enable a temporarily disabled webhook, [send a test request](#test-a-webhook) to the webhook. The webhook is re-enabled if the test request returns a response code in the `2xx` range. ### Delivery headers diff --git a/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb b/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb new file mode 100644 index 00000000000000..dfa4fc6ed79585 --- /dev/null +++ b/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled < BatchedMigrationJob + operation_name :fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled + scope_to ->(relation) { relation.where('recent_failures > 3').where(disabled_until: nil) } + feature_category :integrations + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all( + disabled_until: Time.current + rand(1..30).minutes, + backoff_count: ::WebHooks::AutoDisabling::MAX_FAILURES + ) + end + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb b/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb new file mode 100644 index 00000000000000..9fd8552c493351 --- /dev/null +++ b/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled, :freeze_time, feature_category: :integrations do + let!(:web_hooks) { table(:web_hooks) } + + let!(:non_disabled_webhook) { web_hooks.create!(recent_failures: 3, backoff_count: 3) } + let!(:permanently_disabled_webhook) { web_hooks.create!(recent_failures: 4, backoff_count: 5) } + + let!(:temporarily_disabled_webhook) do + web_hooks.create!(recent_failures: 4, backoff_count: 5, disabled_until: Time.current + 1.minute) + end + + let!(:migration_attrs) do + { + start_id: web_hooks.minimum(:id), + end_id: web_hooks.maximum(:id), + batch_table: :web_hooks, + batch_column: :id, + sub_batch_size: web_hooks.count, + pause_ms: 0, + connection: ApplicationRecord.connection + } + end + + it 'migrates permanently disabled web hooks to temporarily disabled' do + described_class.new(**migration_attrs).perform + + [non_disabled_webhook, temporarily_disabled_webhook, permanently_disabled_webhook].each(&:reload) + + expect(non_disabled_webhook.recent_failures).to eq(3) + expect(non_disabled_webhook.backoff_count).to eq(3) + expect(non_disabled_webhook.disabled_until).to be_nil + + expect(temporarily_disabled_webhook.recent_failures).to eq(4) + expect(temporarily_disabled_webhook.backoff_count).to eq(5) + expect(temporarily_disabled_webhook.disabled_until).to eq(Time.current + 1.minute) + + expect(permanently_disabled_webhook.recent_failures).to eq(4) + expect(permanently_disabled_webhook.backoff_count).to eq(100) + expect(permanently_disabled_webhook.disabled_until).to be_within(30.minutes).of(Time.current) + + expect(web_hooks.where(disabled_until: nil).where('recent_failures > 3').count).to eq(0) + end +end diff --git a/spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb b/spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb new file mode 100644 index 00000000000000..3dabd422d62f67 --- /dev/null +++ b/spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled, feature_category: :integrations do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :web_hooks, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main + ) + } + end + end +end diff --git a/spec/services/web_hook_service_spec.rb b/spec/services/web_hook_service_spec.rb index e1532656a18255..2b561ba201895b 100644 --- a/spec/services/web_hook_service_spec.rb +++ b/spec/services/web_hook_service_spec.rb @@ -627,7 +627,7 @@ response_status: 400 ).deep_stringify_keys ), - 'failed', + 'error', '' ) diff --git a/spec/services/web_hooks/log_execution_service_spec.rb b/spec/services/web_hooks/log_execution_service_spec.rb index ff54508f1af7f0..c2f0b8f67e5924 100644 --- a/spec/services/web_hooks/log_execution_service_spec.rb +++ b/spec/services/web_hooks/log_execution_service_spec.rb @@ -116,28 +116,6 @@ end end - context 'when response_category is :failed' do - let(:response_category) { :failed } - - before do - data[:response_status] = '400' - end - - it 'increments the failure count' do - expect { service.execute }.to change { project_hook.recent_failures }.by(1) - end - - it 'does not change the disabled_until attribute' do - expect { service.execute }.not_to change { project_hook.disabled_until } - end - - it 'does not allow the failure count to overflow' do - project_hook.update!(recent_failures: 32767) - - expect { service.execute }.not_to change { project_hook.recent_failures } - end - end - context 'when response_category is :error' do let(:response_category) { :error } diff --git a/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb b/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb index 33b62564e5f987..ce10299f7a958a 100644 --- a/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb @@ -167,6 +167,7 @@ describe '#enable!' do it 'makes a hook executable if it was marked as failed' do hook.recent_failures = 1000 + hook.disabled_until = 1.minute.from_now expect { hook.enable! }.to change { hook.executable? }.from(false).to(true) end @@ -206,6 +207,7 @@ include_examples 'is tolerant of invalid records' do def run_expectation hook.recent_failures = 1000 + hook.disabled_until = 1.minute.from_now expect { hook.enable! }.to change { hook.executable? }.from(false).to(true) end @@ -283,34 +285,6 @@ def run_expectation expect { hook.backoff! }.to change { hook.backoff_count }.by(1) end end - - context 'when the flag is disabled' do - before do - stub_feature_flags(auto_disabling_web_hooks: false) - end - - it 'does not increment backoff count' do - expect { hook.failed! }.not_to change { hook.backoff_count } - end - end - end - end - - describe '#failed!' do - include_examples 'is tolerant of invalid records' do - def run_expectation - expect { hook.failed! }.to change { hook.recent_failures }.by(1) - end - - context 'when the flag is disabled' do - before do - stub_feature_flags(auto_disabling_web_hooks: false) - end - - it 'does not increment recent failure count' do - expect { hook.failed! }.not_to change { hook.recent_failures } - end - end end end diff --git a/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb b/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb index 7ec0564eafbfcd..f8259861b9ac7c 100644 --- a/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb @@ -659,49 +659,4 @@ end end end - - describe '#failed!', if: auto_disabling do - it 'increments the recent_failures count but does not disable the hook yet' do - expect { hook.failed! }.to change { hook.recent_failures }.to(1) - expect(hook.class.executable).to include(hook) - end - - context 'when hook is at the failure threshold' do - before do - WebHooks::AutoDisabling::FAILURE_THRESHOLD.times { hook.failed! } - end - - it 'is not yet disabled' do - expect(hook.class.executable).to include(hook) - expect(hook).to have_attributes( - recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD, - backoff_count: 0, - disabled_until: nil - ) - end - - context 'when hook is next failed' do - before do - hook.failed! - end - - it 'causes the hook to become disabled' do - expect(hook.class.executable).not_to include(hook) - expect(hook).to have_attributes( - recent_failures: (WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1), - backoff_count: 0, - disabled_until: nil - ) - end - end - end - - it 'does not do anything if recent_failures is at MAX_FAILURES' do - hook.recent_failures = WebHooks::AutoDisabling::MAX_FAILURES - - sql_count = ActiveRecord::QueryRecorder.new { hook.failed! }.count - - expect(sql_count).to eq(0) - end - end end -- GitLab From 8c306b4b090ed295c2d81efa1a2a61587009f6a2 Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Wed, 26 Feb 2025 17:47:08 +1300 Subject: [PATCH 02/11] Implement new permanently disabled logic Based on the discussion here: https://gitlab.com/gitlab-org/gitlab/-/issues/503733#note_2217234805 --- .../concerns/web_hooks/auto_disabling.rb | 50 +-- .../web_hooks/log_execution_service.rb | 4 +- ...bled_webhook_into_temporarily_disabled.yml | 4 +- ...bled_webhook_into_temporarily_disabled.rb} | 2 +- db/schema_migrations/20240925134655 | 1 - db/schema_migrations/20250311134655 | 1 + doc/user/project/integrations/webhooks.md | 2 + ...abled_webhook_into_temporarily_disabled.rb | 9 +- ..._webhook_into_temporarily_disabled_spec.rb | 22 +- .../auto_disabling_hooks_shared_examples.rb | 303 ++++++++---------- .../unstoppable_hooks_shared_examples.rb | 8 +- .../requests/api/hooks_shared_examples.rb | 2 +- 12 files changed, 202 insertions(+), 206 deletions(-) rename db/post_migrate/{20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb => 20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb} (97%) delete mode 100644 db/schema_migrations/20240925134655 create mode 100644 db/schema_migrations/20250311134655 diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index b52e3b911ce867..ab6b367b9113b8 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -6,9 +6,11 @@ module AutoDisabling include ::Gitlab::Loggable ENABLED_HOOK_TYPES = %w[ProjectHook].freeze - MAX_FAILURES = 100 - FAILURE_THRESHOLD = 3 - EXCEEDED_FAILURE_THRESHOLD = FAILURE_THRESHOLD + 1 + + FAILURE_THRESHOLD = 3 # Hooks that fail above this are temporarily disabled for a period of time. + PERMANENTLY_DISABLED_FAILURE_THRESHOLD = 39 # Hooks that fail above this are permanently disabled. + MAX_FAILURES = 100 # Stop counting failures at this level + INITIAL_BACKOFF = 1.minute.freeze MAX_BACKOFF = 1.day.freeze MAX_BACKOFF_COUNT = 11 @@ -32,40 +34,40 @@ def enabled_hook_types included do delegate :auto_disabling_enabled?, to: :class, private: true - # A hook is disabled if: + # A webhook is disabled if: # - # - we have exceeded the grace FAILURE_THRESHOLD (recent_failures > ?) - # - and either: - # - disabled_until is nil (i.e. this was set by WebHook#fail!) - # - or disabled_until is in the future (i.e. this was set by WebHook#backoff!) - # - OR silent mode is enabled. + # - it has exceeded the grace FAILURE_THRESHOLD (recent_failures > ?) + # - AND the time period it was disabled for has not yet expired (disabled_until >= ?) + # - OR it has reached the failure threshold where it is permanently disabled (recent_failures > ?) scope :disabled, -> do return all if Gitlab::SilentMode.enabled? return none unless auto_disabling_enabled? where( - 'recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)', + '(recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)) OR recent_failures > ?', FAILURE_THRESHOLD, - Time.current + Time.current, + PERMANENTLY_DISABLED_FAILURE_THRESHOLD ) end - # A hook is executable if: + # A webhook is executable if: # - # - we have not yet exceeeded the grace FAILURE_THRESHOLD (recent_failures <= ?) - # - OR we have exceeded the grace FAILURE_THRESHOLD and neither of the following is true: - # - disabled_until is nil (i.e. this was set by WebHook#fail!) - # - disabled_until is in the future (i.e. this was set by WebHook#backoff!) - # - AND silent mode is not enabled. + # - it has not exceeeded the grace FAILURE_THRESHOLD (recent_failures <= ?) + # - OR it has exceeded the grace FAILURE_THRESHOLD and: + # - it was temporarily disabled but can now be triggered again (disabled_until < ?) + # - AND has not reached the failure threshold where it is permanently disabled (recent_failures <= ?) scope :executable, -> do return none if Gitlab::SilentMode.enabled? return all unless auto_disabling_enabled? where( - 'recent_failures <= ? OR (recent_failures > ? AND (disabled_until IS NOT NULL) AND (disabled_until < ?))', + '(recent_failures <= ? OR (recent_failures > ? AND disabled_until IS NOT NULL AND disabled_until < ?)) ' \ + 'AND recent_failures <= ?', FAILURE_THRESHOLD, FAILURE_THRESHOLD, - Time.current + Time.current, + PERMANENTLY_DISABLED_FAILURE_THRESHOLD ) end end @@ -79,13 +81,19 @@ def executable? def temporarily_disabled? return false unless auto_disabling_enabled? - disabled_until.present? && disabled_until >= Time.current && recent_failures > FAILURE_THRESHOLD + disabled_until.present? && disabled_until >= Time.current && + recent_failures > FAILURE_THRESHOLD && + recent_failures <= PERMANENTLY_DISABLED_FAILURE_THRESHOLD end def permanently_disabled? return false unless auto_disabling_enabled? - recent_failures > FAILURE_THRESHOLD && disabled_until.blank? + recent_failures > PERMANENTLY_DISABLED_FAILURE_THRESHOLD || + # Keep the old definition of permanently disabled just until we have migrated all records to the new definition + # with `QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled` + # TODO Remove the next line as part of https://gitlab.com/gitlab-org/gitlab/-/issues/525446 + (recent_failures > FAILURE_THRESHOLD && disabled_until.blank?) end def enable! diff --git a/app/services/web_hooks/log_execution_service.rb b/app/services/web_hooks/log_execution_service.rb index 84f230285f4e6e..dcd6ada463392f 100644 --- a/app/services/web_hooks/log_execution_service.rb +++ b/app/services/web_hooks/log_execution_service.rb @@ -52,8 +52,8 @@ def update_hook_failure_state case response_category when :ok hook.enable! - # TODO remove handling of `:failed` in 17.8 - # https://gitlab.com/gitlab-org/gitlab/-/issues/396577 + # TODO remove handling of `:failed` as part of + # https://gitlab.com/gitlab-org/gitlab/-/issues/525446 when :error, :failed hook.backoff! end diff --git a/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml b/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml index 9b4c8432c413cb..f7699ccc4c760f 100644 --- a/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml +++ b/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml @@ -4,6 +4,6 @@ description: | This migration will fan out the migration of permanently disabled webhooks into temporarily disabled webhooks. feature_category: integrations introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329 -milestone: '17.7' -queued_migration_version: 20240925134655 +milestone: '17.11' +queued_migration_version: 20250311134655 finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb b/db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb similarity index 97% rename from db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb rename to db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb index 5c494e61e0a973..1c84209416ccc6 100644 --- a/db/post_migrate/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb +++ b/db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled < Gitlab::Database::Migration[2.2] - milestone '17.7' + milestone '17.11' restrict_gitlab_migration gitlab_schema: :gitlab_main diff --git a/db/schema_migrations/20240925134655 b/db/schema_migrations/20240925134655 deleted file mode 100644 index 325a1df92e184f..00000000000000 --- a/db/schema_migrations/20240925134655 +++ /dev/null @@ -1 +0,0 @@ -f589843d4a368084ce22ab0e273e566a6b2f4ddf39efc9ffb7bf8985e392b6cb \ No newline at end of file diff --git a/db/schema_migrations/20250311134655 b/db/schema_migrations/20250311134655 new file mode 100644 index 00000000000000..e6d774ce2a07f5 --- /dev/null +++ b/db/schema_migrations/20250311134655 @@ -0,0 +1 @@ +c37dddf687af231b44ceda39f3e43e52675fe6abd462c87383a19363a592f243 \ No newline at end of file diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 57efdf8cd13444..368f2c1698b502 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -492,6 +492,8 @@ For more information, see the history. {{< /alert >}} +TODO update documentation according to [new plan](https://gitlab.com/gitlab-org/gitlab/-/issues/503733#note_2217234805). + GitLab automatically disables project or group webhooks that fail four consecutive times. To view auto-disabled webhooks: diff --git a/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb b/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb index dfa4fc6ed79585..04c4f9753e0fa0 100644 --- a/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb +++ b/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb @@ -7,11 +7,16 @@ class FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled < BatchedMi scope_to ->(relation) { relation.where('recent_failures > 3').where(disabled_until: nil) } feature_category :integrations + # Values are based on constants from ::WebHooks::AutoDisabling + NEW_RECENT_FAILURES = 40 # PERMANENTLY_DISABLED_FAILURE_THRESHOLD + 1 + NEW_BACKOFF_COUNT = 37 # PERMANENTLY_DISABLED_FAILURE_THRESHOLD - FAILURE_THRESHOLD + def perform each_sub_batch do |sub_batch| sub_batch.update_all( - disabled_until: Time.current + rand(1..30).minutes, - backoff_count: ::WebHooks::AutoDisabling::MAX_FAILURES + disabled_until: 1.minute.ago, # disabled_until value doesn't matter + recent_failures: NEW_RECENT_FAILURES, + backoff_count: NEW_BACKOFF_COUNT ) end end diff --git a/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb b/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb index 9fd8552c493351..1d98d53ad6974f 100644 --- a/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb +++ b/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb @@ -6,10 +6,10 @@ let!(:web_hooks) { table(:web_hooks) } let!(:non_disabled_webhook) { web_hooks.create!(recent_failures: 3, backoff_count: 3) } - let!(:permanently_disabled_webhook) { web_hooks.create!(recent_failures: 4, backoff_count: 5) } + let!(:legacy_permanently_disabled_webhook) { web_hooks.create!(recent_failures: 4, backoff_count: 5) } let!(:temporarily_disabled_webhook) do - web_hooks.create!(recent_failures: 4, backoff_count: 5, disabled_until: Time.current + 1.minute) + web_hooks.create!(recent_failures: 4, backoff_count: 5, disabled_until: Time.current + 1.hour) end let!(:migration_attrs) do @@ -24,10 +24,10 @@ } end - it 'migrates permanently disabled web hooks to temporarily disabled' do + it 'migrates legacy permanently disabled web hooks to new permanently disabled state' do described_class.new(**migration_attrs).perform - [non_disabled_webhook, temporarily_disabled_webhook, permanently_disabled_webhook].each(&:reload) + [non_disabled_webhook, temporarily_disabled_webhook, legacy_permanently_disabled_webhook].each(&:reload) expect(non_disabled_webhook.recent_failures).to eq(3) expect(non_disabled_webhook.backoff_count).to eq(3) @@ -35,12 +35,18 @@ expect(temporarily_disabled_webhook.recent_failures).to eq(4) expect(temporarily_disabled_webhook.backoff_count).to eq(5) - expect(temporarily_disabled_webhook.disabled_until).to eq(Time.current + 1.minute) + expect(temporarily_disabled_webhook.disabled_until).to eq(Time.current + 1.hour) - expect(permanently_disabled_webhook.recent_failures).to eq(4) - expect(permanently_disabled_webhook.backoff_count).to eq(100) - expect(permanently_disabled_webhook.disabled_until).to be_within(30.minutes).of(Time.current) + expect(legacy_permanently_disabled_webhook.recent_failures).to eq(40) + expect(legacy_permanently_disabled_webhook.backoff_count).to eq(37) + expect(legacy_permanently_disabled_webhook.disabled_until).to eq(Time.current - 1.minute) expect(web_hooks.where(disabled_until: nil).where('recent_failures > 3').count).to eq(0) + + expect(ProjectHook.executable.pluck_primary_key).to contain_exactly(non_disabled_webhook.id) + expect(ProjectHook.disabled.pluck_primary_key).to contain_exactly( + temporarily_disabled_webhook.id, + legacy_permanently_disabled_webhook.id + ) end end diff --git a/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb b/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb index ce10299f7a958a..8eedb25a9890d6 100644 --- a/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb @@ -8,144 +8,93 @@ allow(logger).to receive(:info) end - shared_examples 'is tolerant of invalid records' do - specify do - hook.url = nil - - expect(hook).to be_invalid - run_expectation - end - end + shared_context 'with failure thresholds' do + where(:recent_failures, :disabled_until, :executable) do + past = 1.minute.ago + now = Time.current + future = 1.minute.from_now - describe '.executable/.disabled', :freeze_time do - let!(:not_executable) do [ - [4, nil], # Exceeded the grace period, set by #fail! - [4, 1.second.from_now], # Exceeded the grace period, set by #backoff! - [4, Time.current] # Exceeded the grace period, set by #backoff!, edge-case - ].map do |(recent_failures, disabled_until)| - create( - hook_factory, - **default_factory_arguments, - recent_failures: recent_failures, - disabled_until: disabled_until - ) - end + # At 3 failures the hook is always executable + [3, nil, true], + [3, past, true], + [3, now, true], + [3, future, true], + # At 4 failures the hook is executable only when disabled_until is in the past + [4, nil, false], + [4, past, true], + [4, now, true], + [4, future, false], + # At 39 failures the logic should be the same as with 4 failures (testing the boundary of 40) + [39, nil, false], + [39, past, true], + [39, now, true], + [39, future, false], + # At 40 failures the hook is always disabled + [40, nil, false], + [40, past, false], + [40, now, false], + [40, future, false] + ] end + end - let!(:executables) do - expired = 1.second.ago - borderline = Time.current - suspended = 1.second.from_now + describe '.executable and .disabled', :freeze_time do + include_context 'with failure thresholds' - [ - # Most of these are impossible states, but are included for completeness - [0, nil], - [1, nil], - [3, nil], - [4, expired], - - # Impossible cases: - [3, suspended], - [3, expired], - [3, borderline], - [1, suspended], - [1, expired], - [1, borderline], - [0, borderline], - [0, suspended], - [0, expired] - ].map do |(recent_failures, disabled_until)| - create( - hook_factory, - **default_factory_arguments, + with_them do + let(:web_hook) do + factory_arguments = default_factory_arguments.merge( recent_failures: recent_failures, disabled_until: disabled_until ) - end - end - - it 'finds the correct set of project hooks' do - expect(find_hooks.executable).to match_array executables - expect(find_hooks.executable).to all(be_executable) - - # As expected, and consistent - expect(find_hooks.disabled).to match_array not_executable - expect(find_hooks.disabled.map(&:executable?)).not_to include(true) - - # Nothing is missing - expect(find_hooks.executable.to_a + find_hooks.disabled.to_a).to match_array(find_hooks.to_a) - end - - context 'when the flag is disabled' do - before do - stub_feature_flags(auto_disabling_web_hooks: false) + create(hook_factory, **factory_arguments) end - it 'causes all hooks to be considered executable' do - expect(find_hooks.executable.count).to eq(16) + it 'scopes correctly' do + if executable + expect(find_hooks.executable).to match_array([web_hook]) + expect(find_hooks.disabled).to be_empty + else + expect(find_hooks.executable).to be_empty + expect(find_hooks.disabled).to match_array([web_hook]) + end end - it 'causes no hooks to be considered disabled' do - expect(find_hooks.disabled).to be_empty - end - end + context 'when the flag is disabled' do + before do + stub_feature_flags(auto_disabling_web_hooks: false) + end - context 'when silent mode is enabled' do - before do - stub_application_setting(silent_mode_enabled: true) + it 'causes all hooks to be scoped as executable' do + expect(find_hooks.executable).to match_array([web_hook]) + expect(find_hooks.disabled).to be_empty + end end - it 'causes no hooks to be considered executable' do - expect(find_hooks.executable).to be_empty - end + context 'when silent mode is enabled' do + before do + stub_application_setting(silent_mode_enabled: true) + end - it 'causes all hooks to be considered disabled' do - expect(find_hooks.disabled.count).to eq(16) + it 'causes all hooks to be scoped as disabled' do + expect(find_hooks.executable).to be_empty + expect(find_hooks.disabled).to match_array([web_hook]) + end end end end describe '#executable?', :freeze_time do - let(:web_hook) { create(hook_factory, **default_factory_arguments) } - - where(:recent_failures, :not_until, :executable) do - [ - [0, :not_set, true], - [0, :past, true], - [0, :future, true], - [0, :now, true], - [1, :not_set, true], - [1, :past, true], - [1, :future, true], - [3, :not_set, true], - [3, :past, true], - [3, :future, true], - [4, :not_set, false], - [4, :past, true], # expired suspension - [4, :now, false], # active suspension - [4, :future, false] # active suspension - ] - end + include_context 'with failure thresholds' with_them do - # Phasing means we cannot put these values in the where block, - # which is not subject to the frozen time context. - let(:disabled_until) do - case not_until - when :not_set - nil - when :past - 1.minute.ago - when :future - 1.minute.from_now - when :now - Time.current - end - end - - before do - web_hook.update!(recent_failures: recent_failures, disabled_until: disabled_until) + let(:web_hook) do + factory_arguments = default_factory_arguments.merge( + recent_failures: recent_failures, + disabled_until: disabled_until + ) + create(hook_factory, **factory_arguments) end it 'has the correct state' do @@ -164,25 +113,17 @@ end end - describe '#enable!' do - it 'makes a hook executable if it was marked as failed' do - hook.recent_failures = 1000 - hook.disabled_until = 1.minute.from_now - - expect { hook.enable! }.to change { hook.executable? }.from(false).to(true) + describe '#enable!', :freeze_time do + before do + hook.recent_failures = WebHooks::AutoDisabling::FAILURE_THRESHOLD + hook.backoff! end - it 'makes a hook executable if it is currently backed off' do - hook.recent_failures = 1000 - hook.disabled_until = 1.hour.from_now - + it 'makes a hook executable' do expect { hook.enable! }.to change { hook.executable? }.from(false).to(true) end it 'logs relevant information' do - hook.recent_failures = 1000 - hook.disabled_until = 1.hour.from_now - expect(logger) .to receive(:info) .with(a_hash_including( @@ -197,20 +138,20 @@ end it 'does not update hooks unless necessary' do - hook + hook.recent_failures = 0 + hook.backoff_count = 0 + hook.disabled_until = nil sql_count = ActiveRecord::QueryRecorder.new { hook.enable! }.count expect(sql_count).to eq(0) end - include_examples 'is tolerant of invalid records' do - def run_expectation - hook.recent_failures = 1000 - hook.disabled_until = 1.minute.from_now + it 'is tolerant of invalid records' do + hook.url = nil - expect { hook.enable! }.to change { hook.executable? }.from(false).to(true) - end + expect(hook).to be_invalid + expect { hook.enable! }.to change { hook.executable? }.from(false).to(true) end end @@ -235,13 +176,15 @@ def run_expectation end end - context 'when we have exhausted the grace period' do + context 'when failures exceed the threshold' do before do hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD) end - it 'disables the hook' do + it 'temporarily disables the hook' do expect { hook.backoff! }.to change { hook.executable? }.from(true).to(false) + expect(hook).to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end it 'increments backoff_count' do @@ -280,37 +223,70 @@ def run_expectation end end - include_examples 'is tolerant of invalid records' do - def run_expectation - expect { hook.backoff! }.to change { hook.backoff_count }.by(1) + context 'when the flag is disabled' do + before do + stub_feature_flags(auto_disabling_web_hooks: false) + end + + it 'does not disable the hook' do + expect { hook.backoff! }.not_to change { hook.executable? }.from(true) + expect(hook).not_to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end end + + it 'is tolerant of invalid records' do + hook.url = nil + + expect(hook).to be_invalid + expect { hook.backoff! }.to change { hook.backoff_count }.by(1) + end end end - describe '#temporarily_disabled?' do - it 'is false when not temporarily disabled' do + describe '#temporarily_disabled? and #permanently_disabled?', :freeze_time do + it 'is initially not disabled at all' do expect(hook).not_to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end - it 'allows FAILURE_THRESHOLD initial failures before we back-off' do + it 'becomes temporarily disabled after a threshold of failures has been exceeded' do WebHooks::AutoDisabling::FAILURE_THRESHOLD.times do hook.backoff! + expect(hook).not_to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end hook.backoff! + expect(hook).to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end - context 'when hook has been told to back off' do + context 'when the flag is disabled' do before do + stub_feature_flags(auto_disabling_web_hooks: false) + end + + it 'is not disabled at all' do hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD) hook.backoff! + + expect(hook).not_to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end + end - it 'is true' do - expect(hook).to be_temporarily_disabled + context 'when hook exceeds the permanently disabled threshold' do + before do + hook.update!(recent_failures: WebHooks::AutoDisabling::PERMANENTLY_DISABLED_FAILURE_THRESHOLD) + hook.backoff! + end + + it 'is permanently disabled' do + expect(hook).to be_permanently_disabled + expect(hook).not_to be_temporarily_disabled end context 'when the flag is disabled' do @@ -318,25 +294,22 @@ def run_expectation stub_feature_flags(auto_disabling_web_hooks: false) end - it 'is false' do + it 'is not disabled at all' do expect(hook).not_to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled end end end - end - describe '#permanently_disabled?' do - it 'is false when not disabled' do - expect(hook).not_to be_permanently_disabled - end - - context 'when hook has been disabled' do + # TODO Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/525446 + context 'when hook has no disabled_until set and exceeds FAILURE_THRESHOLD (legacy state)' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::EXCEEDED_FAILURE_THRESHOLD) + hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1) end - it 'is true' do + it 'is permanently disabled' do expect(hook).to be_permanently_disabled + expect(hook).not_to be_temporarily_disabled end context 'when the flag is disabled' do @@ -344,8 +317,9 @@ def run_expectation stub_feature_flags(auto_disabling_web_hooks: false) end - it 'is false' do + it 'is not disabled at all' do expect(hook).not_to be_permanently_disabled + expect(hook).not_to be_temporarily_disabled end end end @@ -354,14 +328,14 @@ def run_expectation describe '#alert_status' do subject(:status) { hook.alert_status } - it { is_expected.to eq :executable } + it { is_expected.to eq(:executable) } - context 'when hook has been disabled' do + context 'when hook has been permanently disabled' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::EXCEEDED_FAILURE_THRESHOLD) + allow(hook).to receive(:permanently_disabled?).and_return(true) end - it { is_expected.to eq :disabled } + it { is_expected.to eq(:disabled) } context 'when the flag is disabled' do before do @@ -372,13 +346,12 @@ def run_expectation end end - context 'when hook has been backed off' do + context 'when hook has been temporarily disabled' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::EXCEEDED_FAILURE_THRESHOLD) - hook.disabled_until = 1.hour.from_now + allow(hook).to receive(:temporarily_disabled?).and_return(true) end - it { is_expected.to eq :temporarily_disabled } + it { is_expected.to eq(:temporarily_disabled) } context 'when the flag is disabled' do before do diff --git a/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb b/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb index cce52fd5fbd4ff..48ad472044de8f 100644 --- a/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true RSpec.shared_examples 'a hook that does not get automatically disabled on failure' do + let(:exeeded_failure_threshold) { WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1 } + describe '.executable/.disabled', :freeze_time do let(:attributes_for_webhooks) do attributes_list = attributes_for_list(hook_factory, 13) @@ -159,7 +161,7 @@ # Initially expect(hook).not_to be_permanently_disabled - hook.update!(recent_failures: WebHooks::AutoDisabling::EXCEEDED_FAILURE_THRESHOLD) + hook.update!(recent_failures: exeeded_failure_threshold) expect(hook).not_to be_permanently_disabled end @@ -172,7 +174,7 @@ context 'when hook has been disabled' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::EXCEEDED_FAILURE_THRESHOLD) + hook.update!(recent_failures: exeeded_failure_threshold) end it { is_expected.to eq :executable } @@ -180,7 +182,7 @@ context 'when hook has been backed off' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::EXCEEDED_FAILURE_THRESHOLD) + hook.update!(recent_failures: exeeded_failure_threshold) hook.disabled_until = 1.hour.from_now end diff --git a/spec/support/shared_examples/requests/api/hooks_shared_examples.rb b/spec/support/shared_examples/requests/api/hooks_shared_examples.rb index 45637423c9c8c9..1c6b957136726a 100644 --- a/spec/support/shared_examples/requests/api/hooks_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/hooks_shared_examples.rb @@ -223,7 +223,7 @@ def hook_param_overrides context 'the hook is disabled' do before do - hook.update!(recent_failures: hook.class::EXCEEDED_FAILURE_THRESHOLD) + hook.update!(recent_failures: hook.class::FAILURE_THRESHOLD + 1) end it "has the correct alert status", :aggregate_failures do -- GitLab From 0f8b6a52080e8cb9839bf139fc1e74ac71071002 Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Mon, 10 Mar 2025 17:01:10 +1300 Subject: [PATCH 03/11] Refactor more specs --- .../concerns/web_hooks/auto_disabling.rb | 29 ++-- spec/factories/project_hooks.rb | 2 +- spec/models/project_spec.rb | 2 +- .../webhook_autodisabling_shared_context.rb | 32 ++++ .../auto_disabling_hooks_shared_examples.rb | 141 ++++++++++++------ .../unstoppable_hooks_shared_examples.rb | 112 +++++--------- .../has_web_hooks_shared_examples.rb | 2 +- .../web_hooks/web_hook_shared_examples.rb | 101 +------------ .../requests/api/hooks_shared_examples.rb | 4 +- .../merge_request_cleanup_refs_worker_spec.rb | 4 +- 10 files changed, 185 insertions(+), 244 deletions(-) create mode 100644 spec/support/shared_contexts/models/concerns/webhooks/webhook_autodisabling_shared_context.rb diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index ab6b367b9113b8..f5feff593df8a9 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -7,9 +7,8 @@ module AutoDisabling ENABLED_HOOK_TYPES = %w[ProjectHook].freeze - FAILURE_THRESHOLD = 3 # Hooks that fail above this are temporarily disabled for a period of time. - PERMANENTLY_DISABLED_FAILURE_THRESHOLD = 39 # Hooks that fail above this are permanently disabled. - MAX_FAILURES = 100 # Stop counting failures at this level + TEMPORARILY_DISABLED_FAILURE_THRESHOLD = 3 + PERMANENTLY_DISABLED_FAILURE_THRESHOLD = 39 INITIAL_BACKOFF = 1.minute.freeze MAX_BACKOFF = 1.day.freeze @@ -36,7 +35,7 @@ def enabled_hook_types # A webhook is disabled if: # - # - it has exceeded the grace FAILURE_THRESHOLD (recent_failures > ?) + # - it has exceeded the grace TEMPORARILY_DISABLED_FAILURE_THRESHOLD (recent_failures > ?) # - AND the time period it was disabled for has not yet expired (disabled_until >= ?) # - OR it has reached the failure threshold where it is permanently disabled (recent_failures > ?) scope :disabled, -> do @@ -45,7 +44,7 @@ def enabled_hook_types where( '(recent_failures > ? AND (disabled_until IS NULL OR disabled_until >= ?)) OR recent_failures > ?', - FAILURE_THRESHOLD, + TEMPORARILY_DISABLED_FAILURE_THRESHOLD, Time.current, PERMANENTLY_DISABLED_FAILURE_THRESHOLD ) @@ -53,8 +52,8 @@ def enabled_hook_types # A webhook is executable if: # - # - it has not exceeeded the grace FAILURE_THRESHOLD (recent_failures <= ?) - # - OR it has exceeded the grace FAILURE_THRESHOLD and: + # - it has not exceeeded the grace TEMPORARILY_DISABLED_FAILURE_THRESHOLD (recent_failures <= ?) + # - OR it has exceeded the grace TEMPORARILY_DISABLED_FAILURE_THRESHOLD and: # - it was temporarily disabled but can now be triggered again (disabled_until < ?) # - AND has not reached the failure threshold where it is permanently disabled (recent_failures <= ?) scope :executable, -> do @@ -64,8 +63,8 @@ def enabled_hook_types where( '(recent_failures <= ? OR (recent_failures > ? AND disabled_until IS NOT NULL AND disabled_until < ?)) ' \ 'AND recent_failures <= ?', - FAILURE_THRESHOLD, - FAILURE_THRESHOLD, + TEMPORARILY_DISABLED_FAILURE_THRESHOLD, + TEMPORARILY_DISABLED_FAILURE_THRESHOLD, Time.current, PERMANENTLY_DISABLED_FAILURE_THRESHOLD ) @@ -82,7 +81,7 @@ def temporarily_disabled? return false unless auto_disabling_enabled? disabled_until.present? && disabled_until >= Time.current && - recent_failures > FAILURE_THRESHOLD && + recent_failures > TEMPORARILY_DISABLED_FAILURE_THRESHOLD && recent_failures <= PERMANENTLY_DISABLED_FAILURE_THRESHOLD end @@ -93,7 +92,7 @@ def permanently_disabled? # Keep the old definition of permanently disabled just until we have migrated all records to the new definition # with `QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled` # TODO Remove the next line as part of https://gitlab.com/gitlab-org/gitlab/-/issues/525446 - (recent_failures > FAILURE_THRESHOLD && disabled_until.blank?) + (recent_failures > TEMPORARILY_DISABLED_FAILURE_THRESHOLD && disabled_until.blank?) end def enable! @@ -107,7 +106,7 @@ def enable! save(validate: false) end - # Don't actually back-off until a grace level of FAILURE_THRESHOLD failures have been seen + # Don't actually back-off until a grace level of TEMPORARILY_DISABLED_FAILURE_THRESHOLD failures have been seen # tracked in the recent_failures counter def backoff! return unless auto_disabling_enabled? @@ -115,7 +114,7 @@ def backoff! attrs = { recent_failures: next_failure_count } - if recent_failures >= FAILURE_THRESHOLD + if recent_failures >= TEMPORARILY_DISABLED_FAILURE_THRESHOLD attrs[:backoff_count] = next_backoff_count attrs[:disabled_until] = next_backoff.from_now end @@ -156,11 +155,11 @@ def logger end def next_failure_count - recent_failures.succ.clamp(1, MAX_FAILURES) + recent_failures.succ.clamp(1, PERMANENTLY_DISABLED_FAILURE_THRESHOLD + 1) end def next_backoff_count - backoff_count.succ.clamp(1, MAX_FAILURES) + backoff_count.succ.clamp(1, PERMANENTLY_DISABLED_FAILURE_THRESHOLD + 1) end end end diff --git a/spec/factories/project_hooks.rb b/spec/factories/project_hooks.rb index 8691a79a158704..482cec1195d5ce 100644 --- a/spec/factories/project_hooks.rb +++ b/spec/factories/project_hooks.rb @@ -39,7 +39,7 @@ end trait :permanently_disabled do - recent_failures { WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1 } + recent_failures { WebHooks::AutoDisabling::PERMANENTLY_DISABLED_FAILURE_THRESHOLD + 1 } end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index d015e99d88c74a..923c0a4eccd27f 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -6409,7 +6409,7 @@ def has_external_wiki it 'executes hooks which were backed off and are no longer backed off' do project = create(:project) hook = create(:project_hook, project: project, push_events: true) - WebHooks::AutoDisabling::FAILURE_THRESHOLD.succ.times { hook.backoff! } + WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD.succ.times { hook.backoff! } expect_any_instance_of(ProjectHook).to receive(:async_execute).once diff --git a/spec/support/shared_contexts/models/concerns/webhooks/webhook_autodisabling_shared_context.rb b/spec/support/shared_contexts/models/concerns/webhooks/webhook_autodisabling_shared_context.rb new file mode 100644 index 00000000000000..b559374ced9d95 --- /dev/null +++ b/spec/support/shared_contexts/models/concerns/webhooks/webhook_autodisabling_shared_context.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +RSpec.shared_context 'with webhook auto-disabling failure thresholds' do + where(:recent_failures, :disabled_until, :executable) do + past = 1.minute.ago + now = Time.current + future = 1.minute.from_now + + [ + # At 3 failures the hook is always executable + [3, nil, true], + [3, past, true], + [3, now, true], + [3, future, true], + # At 4 failures the hook is executable only when disabled_until is in the past + [4, nil, false], + [4, past, true], + [4, now, true], + [4, future, false], + # At 39 failures the logic should be the same as with 4 failures (testing the boundary of 40) + [39, nil, false], + [39, past, true], + [39, now, true], + [39, future, false], + # At 40 failures the hook is always disabled + [40, nil, false], + [40, past, false], + [40, now, false], + [40, future, false] + ] + end +end diff --git a/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb b/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb index 8eedb25a9890d6..465209541a0d60 100644 --- a/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/auto_disabling_hooks_shared_examples.rb @@ -8,39 +8,8 @@ allow(logger).to receive(:info) end - shared_context 'with failure thresholds' do - where(:recent_failures, :disabled_until, :executable) do - past = 1.minute.ago - now = Time.current - future = 1.minute.from_now - - [ - # At 3 failures the hook is always executable - [3, nil, true], - [3, past, true], - [3, now, true], - [3, future, true], - # At 4 failures the hook is executable only when disabled_until is in the past - [4, nil, false], - [4, past, true], - [4, now, true], - [4, future, false], - # At 39 failures the logic should be the same as with 4 failures (testing the boundary of 40) - [39, nil, false], - [39, past, true], - [39, now, true], - [39, future, false], - # At 40 failures the hook is always disabled - [40, nil, false], - [40, past, false], - [40, now, false], - [40, future, false] - ] - end - end - describe '.executable and .disabled', :freeze_time do - include_context 'with failure thresholds' + include_context 'with webhook auto-disabling failure thresholds' with_them do let(:web_hook) do @@ -48,6 +17,7 @@ recent_failures: recent_failures, disabled_until: disabled_until ) + create(hook_factory, **factory_arguments) end @@ -86,7 +56,7 @@ end describe '#executable?', :freeze_time do - include_context 'with failure thresholds' + include_context 'with webhook auto-disabling failure thresholds' with_them do let(:web_hook) do @@ -94,6 +64,7 @@ recent_failures: recent_failures, disabled_until: disabled_until ) + create(hook_factory, **factory_arguments) end @@ -115,7 +86,7 @@ describe '#enable!', :freeze_time do before do - hook.recent_failures = WebHooks::AutoDisabling::FAILURE_THRESHOLD + hook.recent_failures = WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD hook.backoff! end @@ -155,16 +126,33 @@ end end - describe '#backoff!', :freeze_time do + describe '#backoff!' do + around do |example| + if example.metadata[:skip_freeze_time] + example.run + else + freeze_time { example.run } + end + end + context 'when we have not backed off before' do it 'does not disable the hook' do expect { hook.backoff! }.not_to change { hook.executable? }.from(true) + expect(hook.class.executable).to include(hook) end it 'increments recent_failures' do expect { hook.backoff! }.to change { hook.recent_failures }.from(0).to(1) end + it 'does not increment backoff_count' do + expect { hook.backoff! }.not_to change { hook.backoff_count }.from(0) + end + + it 'does not set disabled_until' do + expect { hook.backoff! }.not_to change { hook.disabled_until }.from(nil) + end + it 'logs relevant information' do expect(logger) .to receive(:info) @@ -178,19 +166,27 @@ context 'when failures exceed the threshold' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD) + hook.update!(recent_failures: WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD) end it 'temporarily disables the hook' do expect { hook.backoff! }.to change { hook.executable? }.from(true).to(false) expect(hook).to be_temporarily_disabled expect(hook).not_to be_permanently_disabled + expect(hook.class.executable).not_to include(hook) end it 'increments backoff_count' do expect { hook.backoff! }.to change { hook.backoff_count }.from(0).to(1) end + it 'increments recent_failures' do + expect { hook.backoff! }.to change { + hook.recent_failures + }.from(WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD) + .to(WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1) + end + it 'sets disabled_until' do expect { hook.backoff! }.to change { hook.disabled_until }.from(nil).to(1.minute.from_now) end @@ -201,7 +197,7 @@ .with(a_hash_including( hook_id: hook.id, action: 'backoff', - recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1, + recent_failures: WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1, disabled_until: 1.minute.from_now, backoff_count: 1 )) @@ -209,17 +205,71 @@ hook.backoff! end + it 'is no longer disabled after the backoff time has elapsed', :skip_freeze_time do + hook.backoff! + + expect(hook).to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled + expect(hook.class.executable).not_to include(hook) + + travel_to(hook.disabled_until + 1.second) do + expect(hook).not_to be_temporarily_disabled + expect(hook).not_to be_permanently_disabled + expect(hook.class.executable).to include(hook) + end + end + + it 'increases the backoff time exponentially', :skip_freeze_time do + hook.backoff! + + expect(hook).to have_attributes( + recent_failures: (WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1), + backoff_count: 1, + disabled_until: be_like_time(Time.zone.now + 1.minute) + ) + + travel_to(hook.disabled_until + 1.second) do + hook.backoff! + + expect(hook).to have_attributes( + recent_failures: (WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 2), + backoff_count: 2, + disabled_until: be_like_time(Time.zone.now + 2.minutes) + ) + end + + travel_to(hook.disabled_until + 1.second) do + hook.backoff! + + expect(hook).to have_attributes( + recent_failures: (WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 3), + backoff_count: 3, + disabled_until: be_like_time(Time.zone.now + 4.minutes) + ) + end + end + context 'when the hook is permanently disabled' do before do allow(hook).to receive(:permanently_disabled?).and_return(true) end - it 'does not set disabled_until' do - expect { hook.backoff! }.not_to change { hook.disabled_until } + it 'does not do anything' do + sql_count = ActiveRecord::QueryRecorder.new { hook.backoff! }.count + + expect(sql_count).to eq(0) + end + end + + context 'when the hook is temporarily disabled' do + before do + allow(hook).to receive(:temporarily_disabled?).and_return(true) end - it 'does not increment the backoff count' do - expect { hook.backoff! }.not_to change { hook.backoff_count } + it 'does not do anything' do + sql_count = ActiveRecord::QueryRecorder.new { hook.backoff! }.count + + expect(sql_count).to eq(0) end end @@ -232,6 +282,7 @@ expect { hook.backoff! }.not_to change { hook.executable? }.from(true) expect(hook).not_to be_temporarily_disabled expect(hook).not_to be_permanently_disabled + expect(hook.class.executable).to include(hook) end end @@ -251,7 +302,7 @@ end it 'becomes temporarily disabled after a threshold of failures has been exceeded' do - WebHooks::AutoDisabling::FAILURE_THRESHOLD.times do + WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD.times do hook.backoff! expect(hook).not_to be_temporarily_disabled @@ -270,7 +321,7 @@ end it 'is not disabled at all' do - hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD) + hook.update!(recent_failures: WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD) hook.backoff! expect(hook).not_to be_temporarily_disabled @@ -302,9 +353,9 @@ end # TODO Remove as part of https://gitlab.com/gitlab-org/gitlab/-/issues/525446 - context 'when hook has no disabled_until set and exceeds FAILURE_THRESHOLD (legacy state)' do + context 'when hook has no disabled_until set and exceeds TEMPORARILY_DISABLED_FAILURE_THRESHOLD (legacy state)' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1) + hook.update!(recent_failures: WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1) end it 'is permanently disabled' do diff --git a/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb b/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb index 48ad472044de8f..05943e563b64d9 100644 --- a/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/unstoppable_hooks_shared_examples.rb @@ -1,99 +1,57 @@ # frozen_string_literal: true RSpec.shared_examples 'a hook that does not get automatically disabled on failure' do - let(:exeeded_failure_threshold) { WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1 } + let(:exeeded_failure_threshold) { WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1 } describe '.executable/.disabled', :freeze_time do - let(:attributes_for_webhooks) do - attributes_list = attributes_for_list(hook_factory, 13) - - merged_attributes = attributes_list.zip([ - [0, Time.current], - [0, 1.minute.from_now], - [1, 1.minute.from_now], - [3, 1.minute.from_now], - [4, nil], - [4, 1.day.ago], - [4, 1.minute.from_now], - [0, nil], - [0, 1.day.ago], - [1, nil], - [1, 1.day.ago], - [3, nil], - [3, 1.day.ago] - ]) - - merged_attributes.map do |attributes, (recent_failures, disabled_until)| - attributes.merge(**default_factory_arguments, recent_failures: recent_failures, disabled_until: disabled_until) - end - end + include_context 'with webhook auto-disabling failure thresholds' - let(:webhooks) { described_class.create!(attributes_for_webhooks) } - - it 'finds the correct set of project hooks' do - expect(find_hooks).to all(be_executable) - expect(find_hooks.executable).to match_array(webhooks) - expect(find_hooks.disabled).to be_empty - end + with_them do + let(:web_hook) do + factory_arguments = default_factory_arguments.merge( + recent_failures: recent_failures, + disabled_until: disabled_until + ) - context 'when silent mode is enabled' do - before do - stub_application_setting(silent_mode_enabled: true) + create(hook_factory, **factory_arguments) end - it 'causes no hooks to be considered executable' do - expect(find_hooks.executable).to be_empty + it 'is always enabled' do + expect(find_hooks).to all(be_executable) + expect(find_hooks.executable).to match_array(find_hooks) + expect(find_hooks.disabled).to be_empty end - it 'causes all hooks to be considered disabled' do - expect(find_hooks.disabled).to match_array(webhooks) + context 'when silent mode is enabled' do + before do + stub_application_setting(silent_mode_enabled: true) + end + + it 'causes no hooks to be considered executable' do + expect(find_hooks.executable).to be_empty + end + + it 'causes all hooks to be considered disabled' do + expect(find_hooks.disabled).to match_array(find_hooks) + end end end end describe '#executable?', :freeze_time do - let(:web_hook) { build(hook_factory, **default_factory_arguments) } - - where(:recent_failures, :not_until) do - [ - [0, :not_set], - [0, :past], - [0, :future], - [0, :now], - [1, :not_set], - [1, :past], - [1, :future], - [3, :not_set], - [3, :past], - [3, :future], - [4, :not_set], - [4, :past], # expired suspension - [4, :now], # active suspension - [4, :future] # active suspension - ] - end + include_context 'with webhook auto-disabling failure thresholds' with_them do - # Phasing means we cannot put these values in the where block, - # which is not subject to the frozen time context. - let(:disabled_until) do - case not_until - when :not_set - nil - when :past - 1.minute.ago - when :future - 1.minute.from_now - when :now - Time.current - end - end + let(:web_hook) do + factory_arguments = default_factory_arguments.merge( + recent_failures: recent_failures, + disabled_until: disabled_until + ) - before do - web_hook.update!(recent_failures: recent_failures, disabled_until: disabled_until) + build(hook_factory, **factory_arguments) end - it 'has the correct state' do + it 'is always executable' do expect(web_hook).to be_executable end end @@ -131,7 +89,7 @@ context 'when we have exhausted the grace period' do before do - hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD) + hook.update!(recent_failures: WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD) end it 'does not disable the hook' do @@ -146,7 +104,7 @@ expect(hook).not_to be_temporarily_disabled # Backing off - WebHooks::AutoDisabling::FAILURE_THRESHOLD.times do + WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD.times do hook.backoff! expect(hook).not_to be_temporarily_disabled end diff --git a/spec/support/shared_examples/models/concerns/web_hooks/has_web_hooks_shared_examples.rb b/spec/support/shared_examples/models/concerns/web_hooks/has_web_hooks_shared_examples.rb index 113dcc266fc80b..841fe04a035c7c 100644 --- a/spec/support/shared_examples/models/concerns/web_hooks/has_web_hooks_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/web_hooks/has_web_hooks_shared_examples.rb @@ -19,7 +19,7 @@ context 'when there is a failed hook' do before do hook = create_hook - hook.update!(recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1) + hook.update!(recent_failures: WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1) end it { is_expected.to eq(true) } diff --git a/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb b/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb index f8259861b9ac7c..78698946b88e46 100644 --- a/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb +++ b/spec/support/shared_examples/models/concerns/web_hooks/web_hook_shared_examples.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.shared_examples 'a webhook' do |factory:, auto_disabling: true| +RSpec.shared_examples 'a webhook' do |factory:| include AfterNextHelpers let(:hook) { build(factory) } @@ -560,103 +560,4 @@ it { expect(hook.masked_token).to eq described_class::SECRET_MASK } end end - - describe '#backoff!', if: auto_disabling do - context 'when we have not backed off before' do - it 'increments the recent_failures count but does not disable the hook yet' do - expect { hook.backoff! }.to change { hook.recent_failures }.to(1) - expect(hook.class.executable).to include(hook) - end - end - - context 'when hook is at the failure threshold' do - before do - WebHooks::AutoDisabling::FAILURE_THRESHOLD.times { hook.backoff! } - end - - it 'is not yet disabled' do - expect(hook.class.executable).to include(hook) - expect(hook).to have_attributes( - recent_failures: WebHooks::AutoDisabling::FAILURE_THRESHOLD, - backoff_count: 0, - disabled_until: nil - ) - end - - context 'when hook is next told to backoff' do - before do - hook.backoff! - end - - it 'causes the hook to become disabled for initial backoff period' do - expect(hook.class.executable).not_to include(hook) - expect(hook).to have_attributes( - recent_failures: (WebHooks::AutoDisabling::FAILURE_THRESHOLD + 1), - backoff_count: 1, - disabled_until: 1.minute.from_now - ) - end - - context 'when the backoff time has elapsed', :skip_freeze_time do - it 'is no longer disabled' do - travel_to(hook.disabled_until + 1.minute) do - expect(hook.class.executable).to include(hook) - end - end - - context 'when the hook is next told to backoff' do - it 'disables the hook again, increasing the backoff time exponentially' do - travel_to(hook.disabled_until + 1.minute) do - hook.backoff! - - expect(hook.class.executable).not_to include(hook) - expect(hook).to have_attributes( - recent_failures: (WebHooks::AutoDisabling::FAILURE_THRESHOLD + 2), - backoff_count: 2, - disabled_until: 2.minutes.from_now - ) - end - end - end - end - end - end - - it 'does not do anything if the hook is currently temporarily disabled' do - allow(hook).to receive(:temporarily_disabled?).and_return(true) - - sql_count = ActiveRecord::QueryRecorder.new { hook.backoff! }.count - - expect(sql_count).to eq(0) - end - - it 'does not do anything if the hook is currently permanently disabled' do - allow(hook).to receive(:permanently_disabled?).and_return(true) - - sql_count = ActiveRecord::QueryRecorder.new { hook.backoff! }.count - - expect(sql_count).to eq(0) - end - - context 'when the counter are above MAX_FAILURES' do - let(:max_failures) { WebHooks::AutoDisabling::MAX_FAILURES } - - before do - hook.update!( - recent_failures: (max_failures + 1), - backoff_count: (max_failures + 1), - disabled_until: 1.hour.ago - ) - end - - it 'reduces the counter to MAX_FAILURES' do - hook.backoff! - - expect(hook).to have_attributes( - recent_failures: max_failures, - backoff_count: max_failures - ) - end - end - end end diff --git a/spec/support/shared_examples/requests/api/hooks_shared_examples.rb b/spec/support/shared_examples/requests/api/hooks_shared_examples.rb index 1c6b957136726a..af119d0cc4de56 100644 --- a/spec/support/shared_examples/requests/api/hooks_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/hooks_shared_examples.rb @@ -223,7 +223,7 @@ def hook_param_overrides context 'the hook is disabled' do before do - hook.update!(recent_failures: hook.class::FAILURE_THRESHOLD + 1) + hook.update!(recent_failures: hook.class::TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1) end it "has the correct alert status", :aggregate_failures do @@ -237,7 +237,7 @@ def hook_param_overrides context 'the hook is backed-off' do before do - WebHooks::AutoDisabling::FAILURE_THRESHOLD.times { hook.backoff! } + WebHooks::AutoDisabling::TEMPORARILY_DISABLED_FAILURE_THRESHOLD.times { hook.backoff! } hook.backoff! end diff --git a/spec/workers/merge_request_cleanup_refs_worker_spec.rb b/spec/workers/merge_request_cleanup_refs_worker_spec.rb index 6c87b6827a80b0..0b96fe66ac3d67 100644 --- a/spec/workers/merge_request_cleanup_refs_worker_spec.rb +++ b/spec/workers/merge_request_cleanup_refs_worker_spec.rb @@ -30,8 +30,8 @@ expect(cleanup_schedule.completed_at).to be_nil end - context "and cleanup schedule has already failed #{WebHooks::AutoDisabling::FAILURE_THRESHOLD} times" do - let(:failed_count) { WebHooks::AutoDisabling::FAILURE_THRESHOLD } + context "and cleanup schedule has already failed #{described_class::FAILURE_THRESHOLD} times" do + let(:failed_count) { described_class::FAILURE_THRESHOLD } it 'marks the cleanup schedule as failed and track the failure' do expect(cleanup_schedule.reload).to be_failed -- GitLab From 75ef406d86dae8cccd5276f2caa25650cbfc815c Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Wed, 12 Mar 2025 17:42:56 +1300 Subject: [PATCH 04/11] Update frontend strings --- app/views/shared/web_hooks/_hook.html.haml | 6 ++--- .../shared/web_hooks/_hook_errors.html.haml | 12 +++++---- locale/gitlab.pot | 26 +++++++++---------- .../projects/hooks/edit.html.haml_spec.rb | 4 +-- .../projects/hooks/index.html.haml_spec.rb | 18 ++++++------- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/app/views/shared/web_hooks/_hook.html.haml b/app/views/shared/web_hooks/_hook.html.haml index 384e312dce3b00..8350eb03f18b7d 100644 --- a/app/views/shared/web_hooks/_hook.html.haml +++ b/app/views/shared/web_hooks/_hook.html.haml @@ -11,11 +11,11 @@ = hook.url - if hook.rate_limited? - = gl_badge_tag(_('Disabled'), variant: :danger) + = gl_badge_tag(_('Webhooks|Rate limited'), variant: :danger) - elsif hook.permanently_disabled? - = gl_badge_tag(s_('Webhooks|Failed to connect'), variant: :danger) + = gl_badge_tag(s_('Webhooks|Disabled'), variant: :danger) - elsif hook.temporarily_disabled? - = gl_badge_tag(s_('Webhooks|Fails to connect'), variant: :warning) + = gl_badge_tag(s_('Webhooks|Temporarily disabled'), variant: :warning) %div - hook.class.triggers.each_value do |trigger| diff --git a/app/views/shared/web_hooks/_hook_errors.html.haml b/app/views/shared/web_hooks/_hook_errors.html.haml index b610f56ef1780e..52402d5a337807 100644 --- a/app/views/shared/web_hooks/_hook_errors.html.haml +++ b/app/views/shared/web_hooks/_hook_errors.html.haml @@ -1,5 +1,6 @@ - strong = { strong_start: ''.html_safe, strong_end: ''.html_safe } +- help_link = link_to('', help_page_path('user/project/integrations/webhooks.md', anchor: 'auto-disabled-webhooks'), target: '_blank', rel: 'noopener noreferrer') - if hook.rate_limited? - placeholders = { limit: number_with_delimiter(hook.rate_limit), root_namespace: hook.parent.root_namespace.path } @@ -8,14 +9,15 @@ - c.with_body do = s_("Webhooks|Webhooks for %{root_namespace} are now disabled because they've been triggered more than %{limit} times per minute. These webhooks are re-enabled automatically in the next minute.").html_safe % placeholders - elsif hook.permanently_disabled? - = render Pajamas::AlertComponent.new(title: s_('Webhooks|Webhook failed to connect'), + - failure_count = { failure_count: hook.recent_failures } + = render Pajamas::AlertComponent.new(title: s_('Webhooks|Webhook disabled'), variant: :danger) do |c| - c.with_body do - = safe_format(s_('Webhooks|The webhook failed to connect and is now disabled. To re-enable the webhook, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings.'), strong) + = safe_format(s_('Webhooks|The webhook has %{help_link_start}failed%{help_link_end} %{failure_count} times consecutively and has been disabled. To re-enable the webhook, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings.'), strong, failure_count, tag_pair(help_link, :help_link_start, :help_link_end)) - elsif hook.temporarily_disabled? - - help_link = link_to('', help_page_path('user/project/integrations/webhooks.md', anchor: 'auto-disabled-webhooks'), target: '_blank', rel: 'noopener noreferrer') - retry_time = { retry_time: time_interval_in_words(hook.disabled_until - Time.now) } - = render Pajamas::AlertComponent.new(title: s_('Webhooks|Webhook fails to connect'), + - failure_count = { failure_count: hook.recent_failures } + = render Pajamas::AlertComponent.new(title: s_('Webhooks|Webhook temporarily disabled'), variant: :warning) do |c| - c.with_body do - = safe_format(s_('Webhooks|The webhook %{help_link_start}failed to connect%{help_link_end} and is scheduled to retry in %{retry_time}. To re-enable the webhook, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings.'), retry_time, strong, tag_pair(help_link, :help_link_start, :help_link_end)) + = safe_format(s_('Webhooks|The webhook has %{help_link_start}failed%{help_link_end} %{failure_count} times consecutively and is disabled for %{retry_time}. To re-enable the webhook earlier, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings.'), retry_time, strong, failure_count, tag_pair(help_link, :help_link_start, :help_link_end)) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 32f29a8d696eae..e08715995eb3f8 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -65913,6 +65913,9 @@ msgstr "" msgid "Webhooks|Description (optional)" msgstr "" +msgid "Webhooks|Disabled" +msgstr "" + msgid "Webhooks|Do not show sensitive data such as tokens in the UI." msgstr "" @@ -65925,12 +65928,6 @@ msgstr "" msgid "Webhooks|Enable SSL verification" msgstr "" -msgid "Webhooks|Failed to connect" -msgstr "" - -msgid "Webhooks|Fails to connect" -msgstr "" - msgid "Webhooks|Feature flag events" msgstr "" @@ -65991,6 +65988,9 @@ msgstr "" msgid "Webhooks|Project or group access token events" msgstr "" +msgid "Webhooks|Rate limited" +msgstr "" + msgid "Webhooks|Regular expression" msgstr "" @@ -66030,6 +66030,9 @@ msgstr "" msgid "Webhooks|Tag push events" msgstr "" +msgid "Webhooks|Temporarily disabled" +msgstr "" + msgid "Webhooks|The URL must be percent-encoded if it contains one or more special characters." msgstr "" @@ -66039,10 +66042,10 @@ msgstr "" msgid "Webhooks|The secret token is cleared on save unless it is updated." msgstr "" -msgid "Webhooks|The webhook %{help_link_start}failed to connect%{help_link_end} and is scheduled to retry in %{retry_time}. To re-enable the webhook, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings." +msgid "Webhooks|The webhook has %{help_link_start}failed%{help_link_end} %{failure_count} times consecutively and has been disabled. To re-enable the webhook, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings." msgstr "" -msgid "Webhooks|The webhook failed to connect and is now disabled. To re-enable the webhook, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings." +msgid "Webhooks|The webhook has %{help_link_start}failed%{help_link_end} %{failure_count} times consecutively and is disabled for %{retry_time}. To re-enable the webhook earlier, see %{strong_start}Recent events%{strong_end} for more information about the error, then test your settings." msgstr "" msgid "Webhooks|Trigger" @@ -66060,13 +66063,10 @@ msgstr "" msgid "Webhooks|Webhook disabled" msgstr "" -msgid "Webhooks|Webhook failed to connect" -msgstr "" - -msgid "Webhooks|Webhook fails to connect" +msgid "Webhooks|Webhook rate limit has been reached" msgstr "" -msgid "Webhooks|Webhook rate limit has been reached" +msgid "Webhooks|Webhook temporarily disabled" msgstr "" msgid "Webhooks|Webhooks for %{root_namespace} are now disabled because they've been triggered more than %{limit} times per minute. These webhooks are re-enabled automatically in the next minute." diff --git a/spec/views/projects/hooks/edit.html.haml_spec.rb b/spec/views/projects/hooks/edit.html.haml_spec.rb index 68dd6faeb76b49..06c4598a867a79 100644 --- a/spec/views/projects/hooks/edit.html.haml_spec.rb +++ b/spec/views/projects/hooks/edit.html.haml_spec.rb @@ -39,7 +39,7 @@ it 'renders alert' do render - expect(rendered).to have_text(s_('Webhooks|Webhook failed to connect')) + expect(rendered).to have_text(s_('Webhooks|Webhook disabled')) end end @@ -52,7 +52,7 @@ it 'renders alert' do render - expect(rendered).to have_text(s_('Webhooks|Webhook fails to connect')) + expect(rendered).to have_text(s_('Webhooks|Webhook temporarily disabled')) end end end diff --git a/spec/views/projects/hooks/index.html.haml_spec.rb b/spec/views/projects/hooks/index.html.haml_spec.rb index e876ace3fcb554..fab2af6a7ca395 100644 --- a/spec/views/projects/hooks/index.html.haml_spec.rb +++ b/spec/views/projects/hooks/index.html.haml_spec.rb @@ -19,9 +19,9 @@ expect(rendered).to have_css('.gl-heading-2', text: _('Webhooks')) expect(rendered).to have_text('Webhooks') - expect(rendered).not_to have_css('.gl-badge', text: _('Disabled')) - expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Failed to connect')) - expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Fails to connect')) + expect(rendered).not_to have_css('.gl-badge', text: _('Webhooks|Rate limited')) + expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Disabled')) + expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Temporarily disabled')) end context 'webhook is rate limited' do @@ -29,10 +29,10 @@ allow(existing_hook).to receive(:rate_limited?).and_return(true) end - it 'renders "Disabled" badge' do + it 'renders "Rate limited" badge' do render - expect(rendered).to have_css('.gl-badge', text: _('Disabled')) + expect(rendered).to have_css('.gl-badge', text: _('Webhooks|Rate limited')) end end @@ -41,10 +41,10 @@ allow(existing_hook).to receive(:permanently_disabled?).and_return(true) end - it 'renders "Failed to connect" badge' do + it 'renders "Disabled" badge' do render - expect(rendered).to have_css('.gl-badge', text: s_('Webhooks|Failed to connect')) + expect(rendered).to have_css('.gl-badge', text: s_('Webhooks|Disabled')) end end @@ -53,10 +53,10 @@ allow(existing_hook).to receive(:temporarily_disabled?).and_return(true) end - it 'renders "Fails to connect" badge' do + it 'renders "Temporarily disabled" badge' do render - expect(rendered).to have_css('.gl-badge', text: s_('Webhooks|Fails to connect')) + expect(rendered).to have_css('.gl-badge', text: s_('Webhooks|Temporarily disabled')) end end end -- GitLab From 4026f833f7b4e671015569ae9ba8e4afb01d053e Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Wed, 12 Mar 2025 17:43:18 +1300 Subject: [PATCH 05/11] Update docs --- .../integrations/img/failed_badges_v17_1.png | Bin 0 -> 40445 bytes doc/user/project/integrations/webhooks.md | 49 ++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 doc/user/project/integrations/img/failed_badges_v17_1.png diff --git a/doc/user/project/integrations/img/failed_badges_v17_1.png b/doc/user/project/integrations/img/failed_badges_v17_1.png new file mode 100644 index 0000000000000000000000000000000000000000..865f8cb10d9dc031081b46bcd417b626de3a244b GIT binary patch literal 40445 zcmeAS@N?(olHy`uVBq!ia0y~yV5wnXV5;X}W?*2j6*B2$U|?WLcl32+VA$Bt{U?!? zfq{XsILO_JVcj{ImkbO{H!?#aN+NuHtdjF{^%7I^lT!66atjzhz^1~gBDWwnwIorY zA~z?m*s8)-39P~@uh`R!J3~WrBO950ao&l8= z^*65H?(FEfefw@xW7~!Em%o4exp?vN>(_3kr{!+hy#3DYd%;2Be!d~Ue*HOp>g>&% zw|lz!-@W~~XyMXdKmM#(zIxZLy$SItmo8o{E-d^1@88=u?~fim(bn2IdD66JPhX@Y zXZQ9@m^){FXJ=nu-=r&-uKoP+`^1UUmo8o2v19kl8FSB_y>Rc|gHIp7{P_O!^{cmg zcJF_1|MB+iyQWT^wQSkS*yzOavg)kNyroN4%$vLL#q-y>xy2hdY_6|s`tbf^byfY@ zGv|*SIsWzQ*TaX7Ub%dA|Gq6`wzS%g-yBKY#JpFW(hBwJ{MgC6cOI->y)Gp=qqL+lBR%KO%_~U> zX&*m)esKR$OLNECH5*>Ne7$e)f!dm;Qzy?nee(S3)$8lmZwe2KUbtY%@87@w+`nB} zS+{QO#O<#6x zJkS&N`Sa(u=jQymesPK)&xcD3KTdA@b>`@oEo%;WTKs+U`rr3&*RI}p7VW;d+U0F! z=CP^qzb~JCU7GrJ?cxV}I=o}p}%=~rvB%J(^Ql-Zf&Z2kreR!$fO^~_MHg# z*rq4_cHNRYtrgGaPCV>weYqg%XrSZImlr+UeOBk1{az?~f1t`tacArxU$%y6Kgo7(j0u& zJXE!Ly3(ja{nrYW$y(*lc1~`Q*5C1Z`hzC_2h$&o^z| z#PLjhvRSb6$#+qq4rWG%hm$nkoOV}|o>$YRwDUG=j3|SHt;U^mnQsn%@^n?&F8Gy! z;g#s$JAE!eTJvjE`ggE11nj@jzI5@Yldj($W-&0lI=;`t@2PO7!Usl%E5=_sUDnhD zd}L&}GO<`!1aX(>Zc*6XXf#Iiv-Txf3>ctl?2LAcZK7s#9aJ<`Mt~bmI@+Y20?@*Vz zdnBHrq2}MuPcvuEo*k_*ckukv>DsZqI4wQ;fp1zwoCRRcl*ant5jNuKC{{pDEtH=<#!(v=67-DyE*aSDzR@oA@vi?Zo2f5Ew{U))f5weEMOzd-1z<#S)Wn$BzJP_jwylcmbB%8Fc58QxLv_Z$EG zks1EB>vQ9Z^PxVn0@G@4b%b@MDbC;TX!o|R$yIB{$&42oPd=NUSRQGkJbQ1@_ajnYnTnIcSiCicB{HmeS6MVEC0)`zy9i=^mOmijPF+_KYZh4BsAgtSM{D6@3jVHo3yJ7*Ixd6 zWMy5+%Dc9Lz3-+UTbuEvrF$pyp^5#sJ$hRO7j05{w{`36*Y3woJ=t7nwE16wmFR@? zMMg8rt*47kTjni)UE5Y!ygGgBl|tQ_CYN<4`NUmtvzoc*Nq(hXrh0(0k>#@+vUPlX zNl$|3yo;^)SFu86{)0sG^I4-84~{SY z^P^q&*PoO9-k%;#G7QT3vF>^AY2{_l%dby9e_&nNn;ZGN%WlOc`M+f>y1h+#vc~lD zp)V)3ZtD&GUG*(E^k?ANJz4vw?JzS~u&?@5T~m#N$ETO-R*$v_*?zX#X0+tXwyIB= z=CZ}wQ$Ia?*J%0qQiJHt;?E{^>I+}q+4XJTlX+9lsmNWqUp?)`QnPEFYCe)pD~v5V zPN{Fts#>=9@VOAVZ5NW}1+~js+h(>)te9}#ZQ6YY#gLz8Zr>`JKI5oJEnnr)`I`PJ z8|Uu%+Lk`)hN!V8KI*-6F)AqWXJYjp;U$~h=D$w) z-tqj@lgZD0qqr6>i#@lc;;@UuZg!uG&wFMxByWhU{8Rk4?M33)aos@+uhIrY=dy60PTD%U2w zYAASexY+*PD-rvg<^1Pwx@W52edYV#vF^?8*Gs0Qnz27|?%w~AZM9k6qXf0;W2J2t zS2~OPraTd7t?obRAoN7!(KVyC(9K88l=byzKaN===W?z`?fWjj&!X>6)IZqpakFds z;^ue@<>L=8X9dyd9?U=^D#iAItoISLA!~xupFXt8+W{D3!}* zP1%(5wD`+}z1AyEPv&>G$ox2Q#?J*Sdgcb**_yie`nBM7ddqktfA*a2yM6rWk4Gyt z>vFo^uK9LF=v@2LVAVy!LBhZE_xW-knsnZBGS^+js84efxK#Sr*4$9cH;t1wB~oNn&O(dB!&LeAys;-}pr?#^$+ z><%72SE1}T%VkoD+AbN!wUwVHhU6UK-TB~+MPY2%{HvGxMXKwrJT0$NUTgNCaL(U5 zRhfG>E-ts|2r7$Fmfw4z-}lcFbFF*Vi)L)OCF6d|qxRR#jk)hMCOP`?o&LvOl5_Fu zAH~jzLZ<%fW6Y<1lI*=U#owfoCz!oOc|PZ4p_4q5db4_g>fF=cKF{y4KN&pdv}c;b%}C!BdN=o+{CN6+_pbcQ9af9GU*vsNC~DgF zc$Vjtrgqz-jJ>*+A0>fS56X0sl_pCws^URE*Bcb$D$vs^UUYF+rAdEbl7&j)T>w)d`) zZNzO`zn%GR%>GTc=k5HzrS)xbu}m?0i{kt};cqH4ymgFK4{pf_zol^`Ers!C$uBnP z>zBN%XCHhsdEyM!iyppz=4|fV?(%cf-Mbb?xs`VNm0g~&`RKLcES^Kjw>zJ1QV=%f ze{HF|^0;JJcV%Ks+|$e3JxnGR9h!b#!v9~n(83L-Y&-T&+njjTWz8JhW75YKt?SLc z_cQ*S!KarG?oUp;F@MW3m`U4?=hkwSAm+rQNh)RpE{7vgYMgn^X3bqzk-W?|VKvqx_YE z{NA~i=k4bt?#gL;X#H*3^POBj|0GVob=J?!g6q`tP4)Y?6uWYt&{i_Kta$jI_T=-T z)qf6F|I51Gee2W5UgM~XYbN+BPW8%sV{-0q@C<7Oo}WI`{7X-+v)Ut?JNMM{Et@{= zHs`e0oBxw5|H;-XHcDpRn-%VPXfFE{cSln5lg;xf-Nw7)o^Oe?TrZV!XiXz0_pQLk z8}4nFmw%z?wQyF8Q+4gn3hmDePaU5*Maba)7O96h>Tl=TPs)*<^KD1op2SZ_lb3nL zs>(-Nbx(K8HGer{r&O?;&%Twh=f+NOtn`0sILTL1Dn7e9q9tvU8%W!zfcyf2CG z8U&tPjxoIYeKymU?Ovavo)*u^xFP-U@u$p3JCZNBXI*(M)10DmZ|C7OYyCz)OV_@Y z7HU@OQx($#!|$$QpQ*uWR=78Q#j(V_tbd|i7DsV?`nU4Wb}hAu=XKO4_Wraw@qEjg zLeDSB-ly-Ct5t202$B1d*|bb-TlYuDpEBG2_7!JDUq5&!s5JfVot-N_S*n%3$bEmd zvnWh#$^UokPlS0-sY^}qpE&>CgUxo|SpMIW;PA8yc3J(MH+b2zq8q!P`lsuiPVfGs z^X%jCr^TB}rq6#W)UB-d^^uKwSoaj(8j;U=xx1{+33?yV+aAhm^l*BK#e}Zw7H?9M zeIM?;)~*-!V%3UIktN$KGNbwHw|WOFzyB?AwyrOg%i-C%ga58BefjvB^ryJ(M`R zy!w;xoPYfN9XLJ}O7AVr_4{m@5Mj9P=9k4vD;?v%>?^hU_w4bfoo-ta(zf3=T+r*d zNck9x`^wobN*{da|B@QseQU2vRb6QFhMH$S$Bh?FlWLDu{3T!g{M+SYQ;!#}3Op{S zQq63*-R^-Ro0EXcktKYJO=_w!K}j0ZMVwBseqvD6++M(Ga>8U`mZ8fl=fvR5#tH2! zo}@mV{w8bt>Ws}ni`IS9?Kb|XJ$5j z{`h{5rSWC)3%goP7RK#gQs2p+^0Csc`-scakJJ0-^Z)$R$?ebk*f2zQe}&w+FH-9! zEnik=c-?vZiRJT5C-Rs3dETz7T(v=b|`;o|C!5m3>0gi*x?#qq-s= z9$xlo;|yMl=tEB~Pg-`jqAHiim;H#N-Q=1ip7h4s4^287UW6QIzw2bzd&_0Ekm|0Q zKPv7g89(y!&wsHc`P%HL3(JJ0BA0gBalfD6xi82?ac)xAqN58x{0se&$6r#cxv|qB zb5Vp@UTNe*@nc7Lp7&4vV4eC{QT{aZVF4e(Xng^9X_@?G7ri}CFqBN4S(9eRn|JXn zPmS4(18JgMKYz?Tuxv}T#g6}7K3dPsY|O9zkU0GHgS(X6iyeR3IpTJ^r5BRvXl z@9AY7__E`4`hgD;8~-nnQkro3*W8%>KAibey0stsFV{)?wDHUz|I5hCk zZ9$FB>MaZAN9!~?#rnB27XRGQypMC%i8o>0%&X1+WSA+~eaZ}Mn!Yjey2rxusoT3B zrbqX3h`7B`O0DAl8F4B1){U(@+F~@5_^npD&*sWl;24v%c=G(MosAD$OSHDteEhlD z@bg`(tvvH9zrGWDzU{t-y#2`w&wIDm+_-G=%OxkhVusd5YyOsb%M4vFz3i}5{-W8w za_*n?ANV4D`6u=-UD_l0`u*ant)GA9zUTju`+kdn*`rdY>Y58C|NqVPUN$wFpHE@; z%9=a7s_(gSGjdMx&wp`dHlyLQ+us_t-99wYGuQ0?4GrH7+wGp*l{#FoX@*yk@9s?P z)zh~ARD7G}@=C_ecK(-VT1u;?EG_-bb9u#KZq1;aX)@0i7rjs0JvZs$#Gpmufd`fe z+z_jnztDH`!tLi>8ar#Mj~@PVa_0v1liRoWy2@UhD>-wHjhK+}x^uo(i&JNM&X>7q zlCtnrX3X|A((-EC-W@a5o$w;>dRmWVU(3I8_Dx#JGUZ>aii6AkFwE$1`lEUA_Uxq< zTIP3m9BgbZE><>P|6LH<(@DfIF~Fh;HKl$Si^ARr~lr|%=cM7S|oF^D9ST96fwC$`^cbfFsP3V z>N9OE)Ypgip>gN|d&yxENOvdeCq@R9V7o8|1`h=iO~BQ2 zhuXSc^6+ap%Y_x!+JytAYgAY6+qOkfwX1jO*5?!W8Pt}X_EfM{EWMece}0L}XP!@t z3QdPM%@Oi=^ZFx`=B65pccnK=^!=Cgx;Jy$+&SnjeM$HPdqYRWVPTIyw=J3FHm`UR zdi`jP{`ngC_t^ieL7Pv!C51B{V!ef6NAshig^VI@&Qvd{j~Yu+Xtnow3yL=IKc$Gg=PIPMM}4 z&(L#Xld9*5BBfVCJE!w%YPM_VI@mGnx%+A2Bo%qiPlg6&7ou9PvHx_i+wEsJ|F?wi zyEe<~*&p*me6xyemg(NLi(h^9R%-Z^#nJgT4?f@J@yz)4(=R`E^6A^_>QiJ_DBr<7&^C#Ey zMnc8Di>7ufjS{JN?AM)O^MUzk?+BlXGSS7B46POyKv`Sg=y=C}R5 zpER@b^Y`1`a(_E{JfQ;u<|IUSK?N0kXnwk1ec5Pi`@;1WG z-t$uOphj6#kNr-VukH$R5{sXxpJuk+HRqG1_#W0TC)dxu@_nAlT~qJ>U)MLEoZMeJQKrr= zdbL#f{GTiSY?rn9yJA({O+!1YEeTt{t;lNJdMP$yyTd-|bvy6c8V2Z|dbuOym*?H0 zzcc2kpT0Hc+r;-Z)AlVt>MVRyINf#jt@W#TukESbx9d`w{?8vvqukr`LcVUws92XD zQW}+#6B=rIKHzEJ&8Kg&XD&DSsk7phivRoE53vF3vy+dkySCW(=Dz5cY(HZZtz~pq za7}M1PuTMA*4nb#1AjxB^{aPo)DE%ZU1fD3Oe^H8^WHdF!Jm;kk1{X5Dg1md+s~G{ zJ%^WWz3D4&we}h3u9{V+L$|Ks&a347`Q(_$h06?UG%Zs_jpnHTFF^8$BtgOmb9Yg*U`3{i(a1p)F1xb>B(jvtuJ=5 zFV#-w6s+k8IV3u9{n{=M*{mdt&uNfch$gp0cTlh00UN2)? zV)(48X2+*%P81t05y)P=&oNOne^J?Pj~%xXmF6>T%1E{BpSX4NL9X`~&6eKe{Upg5 zwy<5wMB;@X$IlyI=NxtWStEaDheXijeF8r(aMk)BR{9mn8u?P`^rjPkw%qyo;Nb_Z zTTXs$+3V*$mR=p%WWgRQUNXDcJl5`F_D0>G0`L9@Px@C{|NVq|zj@I&`TEsHtLN1u zpMRX|9%#36<%@LP89A40CU+jccVkb&bKP|_^w{0}wobOG4b%0{%+~Lnu_=@3ZDah| zotNGTpY1dYOwBOc-lnI%_ORcrZOiAL<6HZ%bi$$^H8Y)69=65Bi}SLzoijH#P?UPI zxxZ%3{z>M3Qg!idS6#oy>|1^-P-D)4RdF-oE2niOUQw7Yt+Ib_nt!q8&ziPVtzC!0 zFUFsX5tzbnYx~v7$%`{`rON4lC;q%S@y5M4J=48t-|06gF)5*iDP`Tt9Uc#!FX(D> z7FV1Wu0H?3n~d1EPV4^t^!Rtr>igHwP18cJxzu}~G7?swf7RKyTum>O^QVlC_Urzj z?zg9o7k!WR{Iy=*hzm89qkE|*Af0SEDYj4HJN9QL>3MtNC z#aVP`XV`1iO(zy|PUf1w*Zuan%Gw7mnhR{@Z_T;4IN8)D@#CWjj}}PxbnA7-=A4jm zP?8nrZG92UxXQ}2X}bBvoLL6TS(v9!V0>=0X^FDdI?;P8w)%;`eO21|w&#=L-ZiOW zCvL32duDUl-8WC(R`uk3ZCvr|)CRTbj~3Lt5cgXf|Kg#L_hl#9#~a?W#_?Zkd?g(8 zrsK@{C(i8U_s)wQoLy&R_4DL{b}N7Wv)-z4xswi@W8`kX6eDn2dvd=s%Yv<|-#qTo zT6C7FY-8u~kDQaw+X;U2FZ?MJIK??IYh&)uh<($R?Ef==Z}IO=C2!^(8=b;~cqnY`t92xhOQ0mu;@7U1rY7XB+POe4B9j{DT*} z1npdoew_MZy$<8G+=ku>e75mh=bu~ofzz;+P4l}bsC9U??kdN8-X~jRb}37&nyzwa zTGH%aovWtaI2xkzImKl4*d%9d9T zFMHp-%&h&@UU#zp@AKMd)dk<@8z4H(IkZG~w$QI^TwW0TT zrp?QKeI@T(WIzOC*whf$y1B>00ylaciMCP7>&;#9S4AgprSoT-V8vw}{@pFFA24s< zxMY*l-L0z2-^Td&&Ocj~*YtNwZ0mWetdv#C%HJd^V_UDDmwFh?U?=mb>WlR&%XNpj zH(pu(_E>VS(jJq~DKbI&imY38ycy>*TYXr}DWe_O?9UnbEM2ZcxBcXM>nD3A{HZ?P znbfdC;p*jz=#)rizR7&>MxNw3hTDN zZF9T*W#h6X9v>FITacpZFzeK{-tAGQhnU=cm>AU;J(n#Jh`qCB#}UyVBAJ1EqhE38 zNw|gaUf&_i|?3$f> z=V)DQK3{fpv-f*(=S3BL8ml(knq+lqS>(66Ub{Nmj%PgcA9r0?{x{|1CuNE7-N_$b zHfQ{nKT^YV+<58PY+k$l+=r^awaA%l6ZsLLWx+GwOKI!T8JmJLTKo&jKHOZ~?Qrv; z`t1uDm)1ORUOjK>rxULm^7go9EZw;9we#D9>!*IR-SuO3c=LHXUZwJ@f-;6ZYl2TH zte^g|__famOXt}N4ktC2&pK7+(7V)b%W08mI#)8zOb%SK`p<&%c*Xi*(r?^5n**bUDwq-bEF&dAMc$ABio0V64e8xnV(`O}!4s zdXpaln|hLd<~JFvYwlm#WWBI#LX461>{%h4XSnR-td?)={3vOybfCS;bc@^TbXNIW zCk->ZGjG3OjI`@?Z*@D{E)`;Joh4}`eQ;yM)skMf;}SJ{&WN`#=9)1TRIFA$-S|3v z)-_S#nv4qVCzJOwzMP>WE>ODk<}$l2)q%3v6VIjy6q|*f&fAd{%-)9k=OwbzFKkRd|K?$_^F*N+XO7>qd@WLA$8kvbl*q!}Y)?2} zS#iv-Tl7AmkaywPVA&cS(U_M0kd?n|Ote$geh3^rc6;fB!+drnuGd~m&*iP@dyr)Q z&NrepFJ5V>*pG&-p*;I$D6ew9z5Hj*1$mM7uQxb$IqWxz(+i&XZT17_+UuPQmc&eZ z+I;3??#lTqmYF@R3hm#UXq2~dSD@^%OS-JP4}86>kS!%Rz4g%Ydl^3`3;9g{x^CA6 zBj@)L+6s9e6_aL zb940P$kzC2Z@cha@PY7@sVgEhw!NxUV5<2f`XeDitFh;8-zlEEx7hT9-`O$OOxj*5 zb0Q?k=d6g#iWfBti!BaM2=2J0^uK|Z^#X@=ulD`>F;YzI7D?PmpW0-@jyzGSx{@c6 zthF-G5NWGm)$8)dS`K>W*B2+!NkiyK&5}uw=EyUL(2w5k$Zj0 zZfBkko^4xls$4G5R9xt}>5Bi6uB}<;ubuqCw9Q&>!Fsz5VP{#pURH*k?0K>&;m3ss zD`L;(aqs1hot;_TYA;m%jD3E50l)RswDt$ajn}S-BzB(t))mM!|Ms@q+B%{!t^7xJ z1g=VZ`B~Hcn3ha0i~P!u`z~eVUJ$d}G552`EV&(rr?xew%zw#vma{#=I=YMJ!_B36 zD|Xo{Wd4q*opt!l}Z*?oz|Xq%z4V(ll?C_-(X2zjwQ z@ma>P=YMBtTX4wFKl^Q=%bvILdskE}m|8QLP4)LqiAPU6*mq4~p8e6yzdWUO@%qah z)n*+rhhyJOt!8#>FBU8Quq@{L!f3TC4?~rrmWu>QJSvoa!M^u*aL`8KrRxRy78;3t zp0nNY-ub2LW>++(yx@Im(Bd~u|8D3RpXEjNHCLohUn-uYXt-dLj^tcttD6f{1lmIE zqF$+AdNNNdAoZKNgnGd2o^Ok1JpUh)c{t+2(l0!D^Lyv$UTioo`boT#`MQzjZLO}Y zIe+;l?mxeMLC=p%v9}|<9*eSmm|~bb&woka89}?lF7FpMmTg%-?@-&??Gl#t-JPfV zSsxymzv1^I*|uBD1;Vaz@f|uI*Oa+bwD|NJ|C?98L}vWzZ_#_Ue5%j)#zQ-N?wkra z?$a9_bt6i@@z*z|+qV9#A3yG5|8Fww)%k#2sgC!TvX-fA7G&4@^!RK41m3mT^Dhgh zp8e{1nf0gm@>)LsV+U?sJ$YHIpi{5rh?&#;C3BZbuh17xR7sb&^e+zVj+DF5zuVl3 zeZGq8w&3Lc71!^ziy7Z(iM4l`TF(m|JEnwAMtLvkzMp7Y141bi#dK@lI6}{ zY3~={vA+2I$#woE=c1?GH#W7bJRfzi^hb@9P{8&Hg%39+{8&=IQewOLi_1*;`9b?F z8Rg_U&g~L=K0k#+((dai=D2mz4}}v&it8pl>{R!97`*j`>FETv!&-m$N?l^=eXcCi zAFZw9{-fm<_oRqzM&f#(Tm6@wJ9}r7+}p=yXG;9`^Lczo{oAyh{blBy&mUH{OghH) zVo#4m)W#=YHt4*cpC(e=bbj|n@&3~DZz^rrmg?Uqo~zzD*VM6m-o7gK7Rkh!=kD5T zFPpXUkWmZ2UD*U}yBIMR`QVJz+V$BB|8XWoZ=d$U&v=)YjM45lDH)}8i(_}*)e!sk zR`lV=`0CP)YcJMqIg$6htp7FR<@?fp+t+S*^MKjB?TX&Mbvs=6?f5wU-&?23G5_oD ztnkp*ADbQ>dU*KZ;a2W;eh?Jbld+Sri`h}|0}LO2h?M&v8Kd%pr|3t)!$s3=&R=-6 zA^pVU=Z7M4s$L3ai9IaJZjs}Y*Nge`>B8ou!UvlVeZJ7T@y3@+?6a@kohG=j|8P6E z_}(j(V(wohKUKV%Bi|jXpWov6J#_vo$A`S_{Nj3Sou8lRGS#+yvMD^*e{|pU=}!H( zzpQ*I80RHp=k5OM@LJQu+H)s7oOP|>|E8J8W%|=wPydjNdie2)h26A{y*?bx%6X5L zr=aIv$s3;X5*Dt0J?XpYf-2{xZghF^bm@a_d!~S-mwZ`Q;#@WTx2fcR9V2ntWbdAruO&-(ZJKRs^Huoft16vu3*V)D`23HQ;zDnWntx0}aWCh! z{46Pc_~GH9CFM(JUplQm=X6l}`R6z1NB8|J5}H0&cKenahhH9kD%^5@TJ*&f7Y#MF z+ci&GLhkG;z313;>`2Mk<=n4hrsdozek1??j`aQ&X5!oT?5LUbp|kVn?1whr+vjju zoH%n*bAIvh?Bt|%Gy3ODm@`9y{p+enGcJFAb;xFV(80$4o6GI&b5$Z`rt5@WX`f~z zaK>TDU56YIO~IER4hu@Q7?-sCP}SyOWb>{#*}^QjFi=IPIlv@J>6*g9gKHL@sdVo; z((BCEBCGd@`Np&M^~da*Qa-7_SzWdM*Yw;Ov8iDjZkKLYy=wRB)f4Zvf8Y0Y)!7X` z7U%vff4)CH=ITw|a2dPr5{KtA*POYrw*L3A+Mc~-*Vf6_NJ(Da&s8(8@ZA@cR}VgK zeB1DUQR%w3KA-v{KY!ZY_Hf$-DIK2Ex|wfVe2vU5DM-9Cxr)qld@TQ}JKoysy_ zf7PtBX~D;z&%WWe?CN4}#ZRi$d6}xgH+`NKbKi6h|5kYVKWj~#@wMk`V(ovt=rOJ1 z{qe?VTjRZBQpwZi%l^HbA7&@Ba+Aopit6~#;%f1O>F&1M4!MW&YtNq@#;>#Xe$45D z)1l4he;qkn@z&;l?*06)sz3DCzMj0*s$Zf+_J{G(@0t(arR9ezqzGg!(Vw@&IONgs z{fRmEBC9XlepM^+LgZ^ytovKjUCUl`FS_*cUGw)u^QQ%N*IrNk_Uz(3F@t{}o?JR` z>5*+ue97Of^*dDOWvtX|Jb!#^%l66*U$)yRe|`U0{vI!TZf9m?F5Q1Fo7>#U zYJKb2vR{iTN*<*!STg_dyZgcLn7XOkhnY_wJXmhO<>2#;3L^hAHrZa|uFI9( zzcDO#+t1@UEe|#yeD$OCY{U9S3FFk6c`k|8PPtcPsyz<$)W7h$$=A}a&uF|Wz4*`r zPWLm<*3T>Y@GZ^YMX6=7DtW_%r#I4-WtK`KewUXLG0g z!12$kv!(KOCVg1huNCY6@ONnB+4Nbn%YCN@-R0VLg7=GTgk8jk?}j{U88}N{l$98n zt-O)7`%ckeuP1CZ|7W&qCj7gv<+ve3Zh{F@qFhtA0BiAH=gAlIwEv~P`&x0hD}UYu zfd`xaJQQ`Y{P6b#ll+ItX47iDUhu_8Y_bXEcy{euZ+zqoH>aT6-+2r++;Zh-Js0#` zD_yAJy_rvE++^;4N!1@2rjAPm)y%bLTK{z5yx$PXeyWV)M~SZFJJE&9We-?h2vvM# z^ziXb|8A`+vy=Kx>{fynH@!ODteihs3Y<;Wu9?JovAo5@>h{Clm}89c=I@G5i#DHM zutMd7d|l-aj{PcDeQs>q)DkvNil||Tx6hEOl-+NzEqnb}kM)UNH4+>j)BXz7`hFHv z^6gYL3{oz+kdn1zzlhrO^o|FY6^~`Q%@C+Lcc^Shn&gXQy*OR-o+`TlGbrDqQ-8O$_%kBSLJ5BwtpbsnfyTfOr}qD z!t56Lstlu~HR7LkNv60QkWgO~IepPRiQww9H`q^CBz)B0xa+rJM)u+4ZI7H6_QkkA z3YpR5At%HqRHHHL&+G*kny##NIsVy5R>tn1*^8}utGid|ge8YAsawS&UyxzOX?JqY zi*p=R4ZM?;mYB&%Er|s%PHuMvo>>m)iRK!Nohrn4akyEq8W$_I87d>h?6T$`8$x zUkcXpKe^W<)#7vDOZKBCDVF)B1xGV#I_8}*l+M+S-YqX*5ykb$&`M}Z&F_g8SM$~c zu(IT&d*n40Id|Bo%F5W)H>yW&xL38-Yeo9K4A#9jx#kClo)xIETb&i>VXoA3s$-YB z_K8i(3m$*Wd3Hsgt;X+&V&h!nb;gTJYTq>UueR9zH0$8;>>Z|}50#tS0w0#O6ewnu zU3#~#AgJVa_W7AxZH!bhlx3f#&cD-eV9jL12b*&axW77iVD@jenyB2zvn}2%F<2K9 zExdi!v4B%8{?qrA&k@lnHvR9;IRAOuf~9wobOM((ANkFg&eoLg=Dc@q*K(m6Bej}y z3Y)Batc>PQ)D$Q?sqxn1&~u*{&GsA9mA<62*VwGf-0^hRLHBicD+2%Xe!s|7xi@!3 zR8iiUTXT5%l-X*gz2M?p{^RKfNuzwDqzMm~<<=PA`gyIy%j=fJ|In8Hjf&Se=UXzb zy;I#a{iVVW2@j*w--8bClCXQeYL}|1{5tdJK~ui%DX*%T@=VC+kb3HzeeNF^pFO#> z=gE%aC0fTG?kRfsc-m{e`J$Jz9VWm1WueYff72?2|8~KZc@6F#E@=oFzxm~NN_$Ou zf$Ed+H2v28Eib|p1%xFP7-US2o7?uWPTGEY%VM7s-l7kab2>^7J@1tKb(Q1piY3j5 zmzR2J-PmT7`!~3DmqUS_@u!MU>PbIeot}4m^ZsV-_)DJ;a-5zVezmsAKhE^DW&YfU znt3;#PJ7KKKPkg0V@Jrf$J-K`z9rwvx^Ea`xoHYZa>Jxv153BFPOaar$8ya#yIlCv zVlKDn23 z_i?;uf#19p*4I=Ht*vp~aAb1O)hiFfE*1NHxOwogt6aV0yz>vmrW$kIFIe5CZL#~Q z)xmVnL(8KMEoZ;JSdVaG<&nc_3W3~%On*~mV6LZJUPj1=FK~m zl{Z}3Z2LW3-_DTO!X%!+I0Y@gHoY(KquxU6GcyRW>Pmd!lB!$)$DvaH;^=f&*_htou!3R>tfsdV4Ce=|V!Q{p{N z(_3+uC8c>UZ;f&}V*h5F5Nq;N{a3$QCtR?owyc>keYf8C16|Ptn|SZ6IR4PZ@HWFL zyU)@@#YEShG#FhA&t>{lsipKe3JdwZU0-SrbnLnQUNwH^Jh6$?_fNukX90bT{3- zbTZG~qS^3r`KkiF~rx=d?o$)Q| z%IO;ij+|L6b|vxeg&J>-SMKfUNtJd6AFRW6OGv+$z1V0QrL46lr|J1EhxYPMHx2Ip zIC^*Ea--ary4T}p$$h*lzSV7aa+m$-4MmJKZap%~gnoQTUwPTR>Sx?!aj%_x!Dd21 zH7(D7==ffH{pa?lG}GtbxJ)OXl#wgwR8M|$muZv8Hk;Y(O>&W6pKMmIoMrZ4@x1(b zPcP3Xo)N}9`}Utdi+zkLXNWPJK0I^zNuPr=pFhr9UvPSAn}2)6#*YhM79{y_Ph|_A z_(wwI;Xhwm+`l>c3XTon!l) zB&FcxQIr4febSs@X6FCx@FlS=+u5gfKS<8&)otv*WTAI_LG8P9e3pM1zM8B%^>~(V z^S_XPQ>E0Sr}Zx0^gAVf3E!ub)KIQ-npV>mo{@VeAvx*u$~P^WiXY|al%6(iIIqVV z{;!JHTx?D8b+yK_TZ%a`%M-F5GOPVs;A(1U)_OrHa`9oUf))V-Y2PR77kGVo75`@L zd4Fk7vE#309W0+1SRBw>C~nNg6J_9?a4cRrnP_{_Tx_G{dJKc00lCS2<6{0*V&ciq1GHg9$;YexG0?+w4* z1HZ01^Lxj=)mfj}cYfN}`ZaI8o!Q~%ZPQMzyIOAH-Y3{L{mOfZtUpn6%l|%Fxy5zY z{^!$g&8qUW53>!poMkzEdHm|?mHIp@xQo|Xto62+(oR_OZeHBQYwPpgoa$xV8ueU9 z+~o7>d8-n`u3k&N`0DO~O_^unQw}}9+i$<}eD5djA34|0SNv43*%abarMK_*?frsw z_4|K(=1-dSW$~TUpMS}}FZk)buJ7;X-hZO0;`;x-#@8=fe??)ZxLw(+SyQE}?f(9$ z?6mS+v8(34^5J>UCOy5t{5SBV;T&y~>|)Nt6Zia8_<2^nFSj;s<9AQL=2aK}O5d)_ zu1NU&>Ufs2e7(rdHQUR-UT2e@eAIkZkD2=9^Tjn&GnW7ASTT8;&BXj&o8mUCdbQI$ zS^vj(_22g{U$@_5CbR$Nshan_|9X?-Ygfnq){A|1`~4U7M=wtw@%YIszqu|yt+P3* zK6pdEZB_oNZMUt9rM&E=s#jUP{y*C}ezDiD2@i`S?WSj0owENS^0NQfyj832&10^9 z!TrQjUZZB!v}x<2`akj8&DZOl>uLREeX$42{O@e@gRjm!`238^f0GF=A5O0QxBT|n z=Xd5WT<$MllaMd?cXx`=!S*lz9psbvJ0Cb(^`GcY(i1w^KIK~cxvaCVb)U}iWRaf~ z7@96z^>^a68z7GB{rwhiZ=Sy`zGeMPmk)}mFT%z3{=7UYK7TG>F&_o1*f~3xAw@|-JTP`CATi) z`NL;3CavOb`EW$sdegNKN$C$+$LCpZWNT;ZUmTzicUz6&K|?P?$h;F5gvI}?|EeLj z+H_9e@jT1vvHf2ydF48vX#JQXF;`Ndz^;LFyN~64osHY4h5m4Swb8|^y-MmRr-yOF z)`mQ)1{^@lkL6tAshB%sg{W{JO*XKm0BWhooP3!nvJq-kxa^>ihg*>aP;T1ReQG{4s6(M{do4D)WA9H45zxiqQxxgxZXYt=DDz1_C50&OH+%u z?W)fjx$b=I2-JKfYc=ufPVRZkHDMP zS2L|K+PmrbXV%oRl%nmaMoB+k^V?>=iu3P?s@Ydn`EuJI*7GIhOa1-$Y?kih=+jx)ib~#gP47@!)b2%RK6^3zdirzOqwTg%rf-?szAi*!gqW&n3E= zbKF?c%O(d;4Zl_LLuDEB?#Xe+HD?~nORcXF+jgk^qVYSG>4%n|=CfNm!z!@mk%i>p z$q^?$xZATdWKJz*eb2h*m{;7Rtwnv!iMFRAS#BifvIE z&Zh0|I(bXNrI&X*bBy7=ZO#1sORngAQFBN@(^p?t=pNRo8dS;0%Q`)#}@uiwM z*1Vq1i{G-n)@A*^QPJMS{z$OTQ+f0LK8ZJlGJ@w8HJGdoH(e{+?Z(~v!La|os7>UX z4PRz|`!ykkYrdQG4~}m#Uq8yH-eEUmcvkhZzt!hQ&j%%2)<5o>4dV};7e8Wj(f3#t z!%&xv0tR(!c~!uZy&oBdzzhPR+o^tVYq!-*y6QYT^V@WUtYFt2(e(V zaap!pqI%VV6VDt+W%*CjckMNkjF&iQ9Xxeq4QKgI z$-uP?lct|%wz_b2LFeYTljaDmh!hUa&OCc7YW9=7X<1cE+B1CjoU*>&9QiPKv0s11 zpRK8H8$aB8Eyvm8IP>#CjbNjtGnzX$zby#xX^UV!(8qq+TQ>8xFYDyRn?B31Z%p-fdiLVU!s_UWUj?wy7 zO<%Tcll%Bm_ebNrJ58%i1n&eL&Yt9WNuizUxrr}#NRpM|53JQk(PnhrFREr6`1ZcGwpg3bocs|sJtH>?eY7Ueu#JyVZHvn zSm+vyL+#ODCT*;1FxdP}VeheE^S`BEk#x^ z%&?R_aMhvZvB^~q)$(Vq9e-MKwXF!`8qVDjH=AOLg-)J5Vy3rJW2xi^&Gs!@q~1xd zlTAGERD84IL*Kn^(TTOGCwSz8vn>zn<=Xqn9@z7+##Jz|hq>lPgii^TFfre{mIOj!Z#NmEOyJ*oY*k!ee=z#y*Kl?c64^sH=im8woVlV>b8Y6WBci-^X|KPgv1>l`ImQ3O#Avzfxx1OHm!RaQhGmgFpFgCu!dhM#titYI(vR`wW zn_j=0IPaj~&W&nv(-{xS9Z=ItF+aX6=RtF})3R$tIYwWm96qw}o5Mk_s_i`2veY-V z#@gKI%wM%Ks+fP%;dI#)E!!0zt|yl!u8BMSbHUeVO@HEI)oQx9eXS?UxG(XvyRBvB z?Da!KaHpPHl$O()nQP~Ve7<_sVK$HFPe%FK$F(Oh2q)$j?pK=Dwf*PPCFXa!CC?`{ zyhzD#(=jakA*09eX3a*9{oj4oFVf|@_)|gp<7#jB#EBCn0(Y|XbK7<@SS@9>vvIk3 zlY3V5>8};8%m?m-sRSN=%(HR&L}~Wh-}`Q_3cmE)vL#ge4*tIRDA%?zY=F-l8_GOf7qh-py+v)0>1OpExouUT-W zDA*xJRzmhM!z>A7i&;GiHaTs}d-dNPn<#OU-MF*9ac1SE${Y0woAoEZ*%?|IT3&it zBc%NB_sKUxuf{&V@?Pe5jm7gTOEcNue=Im-vv(YDcj&Cf!TJt7jihuf=cbi%g zOr7!rX7@|_UQ(Nr{QLIv%%ePox3^W-ERt6~ZapJ#)-GMK?cohCCiLIzU3xT0tXTTz zt?9YK(qB9OxR|Q;$3Kkm(ob17KmX$0X9jE3G{VmB-CW17V<)Z?DyU_x6*Cq5EBV=Swe#T`rH_ov#bE6E@rxU>uQC|M2-?{kiGtPbd9g=yQL+`LShe+^QO{ZO7!U zgjYO}dYF~_GDW<7@`Yo4o2+_y1nx$=R<3Pt7jzS&1zXz#qp)4rOgy4uQ0 zc6D`BYbGa?ZOYu zS+GJZ^S^_MtF-AGn)Bl75@Gk2m-urmk?3T(ZhCerwHy^dx8hz@r#O&PjyN(Ne z&^UeR(PlgD?%UsU0;Ae$7(Tqto)WP}POVL7bKJ?O<4JW>u3z$*fAr|v*_;dCGyiPx zHM^g@x%Hs!5pJ{F2Yww-Dx0$XQqugVN8d(Ix#wiZ@M7lnqc;va_2k}bity<(Qn%%H zE}ZXKJb|B~%InbOj|=Bc65>uWv^>>wMZE3NH&0`bM2=bYj14!}bR7M!(H#`grLF5^ z$B>iv(<5uTTbs3b^5wy6kEE>^Dy~z2TqGKj&toy0e`^Ky$f? zgeTwXnzENlnnpE6YZgA#irgW(^x@*CpLeV5RXMZw|MkxemVZBedK%*5YwM-5;6B=sF*4Fg>{_=8lSVm~Dy5F5h`=2k|CCYi{pV2y{x9LY;O@VEJ$ikD)d%lZZ zcxumdl^J#BP0Ki?O#Ccys1 z1pSoW>wG&;U7hlD?zuf%*jFy{p7grJ?{sJ6*1aaH-yEyu&3tSwv^D1LP062om#6GJ z^>WUITd!7JeyQ_d_mVeizh5-3`JH6*_Ta=Y#rZy)o{N6_m+sD$E!}0c`DotRw?BQi z^7F-iJGK1I-rd{M;-|+mn@^ei;ZayzRN~@5yR%OWY;$sNz4S@fZl81R^1qjn`+mJj zU_GUh|75;RTAY1LW9LHs*ZV#5zjdDV?aOKowDYo)EadsL|3J9kx(loNG*6u^o$5cw zt9NFNP1hwcm97a-i`x!-{e1b=lg}Pj!gW8FJmGX)zN~}oq0O0XtD}Y0Zl1cy_0jQ? z!TsVOlLNavCw^IV>(4IRlF1V4+rw*2);-(x+T?FshHl?!^_hF*p3S_tKSp+!gnGrV zd-*$SuGsm0Go4syJ3TCP`;Bu!PcL6Sbbm)$&I7xrQ-aGrp7rm0@pQ86dJWSzdn7Br z8{Y0;qcZba@Vtdq8I3|eO{Mq+`k{E>pN=yNtS*(>33;Y`)=1}3+-*oua?EW{oE*YKFTX?>$H-H z+`5l3cHw*Oo)ernBlVf~{G3ZR#T5$tC#28b6X;*@a(C~i!YoPuOLFeZx0hQhK3VK` z_U&S`g8N0A9%r(DkemKQ-racJ68Vek#J(mk&9pr#y~Tji&5TTS*vV*mh>B4{4?dpd+T*+Y4aE5nk-bZJU*e~*4%f>@|II| z^e$NZbFo(3WhkDz{PDYach{_~dV1MQEYp^m^JmJ>W}Wv7j_i4i*8Sa>OV|9f8tc`N4CEYJ~q5+m91GcP4%v_mHU>T zb5rXNMz7zhv9jc;(R;HEN6dr2B=aiHx7wr;)GoK{^S{Vd-7#)Izf3q<_v@dBUt+(` z{mj?TSH9p_;dkP>P;Ih&CY>C6|X-(&9Eu?(^Ab2r&r8ZCWBJjr5DmA zb7pAGzQXT&XxYSV-PzM3a`az(X<)XBWdEtsJumI7&h0(xzRik#lOp{7O0?R1or{`s zk3VUu-v6x-xF*W(TlJ(ZN5bZv|6sYmeSOcTo$AlproF%Y_WRY!SFHb8uAOWT@r>y* zOWZYmyMS4<)k$f|-6k3HGptfKy<)zko}oDN!^*3f?=)vQFE*QcM6Km?X{T|j@Xr&K zDSuq1`zC3=P@l_e$02QaqP@(APx92IQ*ri7C)?JnuPS>oW%C|=6~5=u=X`E$e6-Dw zZ}*e+GjevF&|a-_OW9g{!91R;4|QwTFETmt@2m6HYe(95Wj;}!=YH?sjtw!nTP2?q zw}-vGf5Y4I?!T^Y7B^cGHtkhuFLN#~+V}bN0uN2D4IcwPEtJi8YWRQoN%aDe)zi;A zFXCS$`ulwTiBtI{d>+4|^yka3tWLXe+WGhArK+1YZP~Kr!TbYOKQH9nFLV==SDeW9 zap^6Np9URU9+^d^t6nSkZm8M$n=|f197}H9u5^XeXJKhqtA5M>-L$eMWBOCp`S)$t zbO&5C$~Ql;yrH*DPj$*`|L#hwJfBa?>K(pVar~^|^OQNaeU8bKGF^qeHFXIuPh@A= z%I>(h?i%CgDx)xVT>*Q2iz8d#M%(kfSIoS=yy2OZ3BUbT^hy4Pbjr5YvZH(7eU!JYol*1ca%G2J^-qnhy=wEf&ip+xGRE zp8Nm3FaCY~^rSCor?ls@UfNq(cHi!6m(Kmy&ujh&MX1j=IlNWScGqTSwNF3otDj2= zU!BTcF!}V{o#vAnUEPi@Vd~TO2y~ZI5}CSs@nV*RArdYHp|c`-Pdis0y{6R?bTRY# zstvn)r*R$i*xi|WHfZy{rAJJ+)~TA_(vQ8J^J@Cv4QbiQo1T~6FHdVYw|7m!Klvwj z4$rBU`|-5L;^_7`=)xoO!^H*6tSlj5oQ~2&b4YS!>j`45& zFT$73zjJlDcHh?XuP5!NwZp{wZszl!)Arn-vA&Jpgyp$od5O%AO{d!T&FfQm zy(Eh7=o?|dnv&OxUkHiaSFcl6SRXiJ2E+W&Md{)}^#}6qv+lb$VVml{a2cn!wnnu! zUIM+6cBzXFEdR^!_#cy0`0UINFWoMFo8S1eMdthjGyaqxeL{04g`ycz0M^zX6%)jyWfX%-LA2b%l3Kj9(H11P5 zT($eJg@jm{&CT!`tn(GyB%L^{I@jwPow1d)YtRWV{L9Z%9L*>n%l!HQv+n~n&mgD6 z$35>@o`|32xZ}{n;&At=ufqGksr{Zc>#k|y!^`YkFE7jr(kpKh5p*xRnH2i&)7Db= zIVs+$&NHn(7HY0P`9V=?Kkx5G&+=Q9hx8Y$y_B@#!DHr#o7|ggWBlx-?rqnF`{Bq* zy^T6=RGL4$?3mW>quTU=G3%^)!pwu8cfa|@Q1WxlYHvR2EcFY9Y3E-&{Jc$rb;ah4 zPqXKy9egf%IHt)b(JxGSXUlgh2|L?_{mrXaGv=<-c(6D-LAHi3gFmXJiT`8fk)-2X z8&;nxn60!XY}2pYsb6%gbZQ>_t^GO4KHMzT%tPnG0d}eE-4###*@O!hiR|BMRC6gM ze_He_KkM^WhNq4dTAO@ml$%w>WYTQ=QD-Z5OK0mXaul?j_#~Yl7=6(EGQ&h1pgy;Kg*7=T8(ssoDytVeV zP|dy2y?xmMkU+nz@Y7mY=<~H*hYzra?(p7t{87)1D_c2!m!*FF+wK4p zt1d*M9sPEN28vlZmt+I+A>aBZfV)tvjS8v;C*v&=ue>X*F8hl{2g`0V~=Y+8JS z#pK9KerEZ9{d(v4&zsLFe*B`Wgz32G5<%N3SNYkFJS=9v_>(F5N6N(h2M*$QlvcZO z#BREH{ebMLE0uP}o!8BlGR>{?VErvWBW}*x0*lY44bsKuC(0a@tccoG>6Csk?Zd&A z4BeUnp;Fe5wv*qaEG`YwZ|Aey_0^K2PLJ_R<~--3o)bZ5L_Qx_=^|h^|3Hs{XV~NW z>+&l2*_`xL+D{gIV@W;#
Tb5A$UWSPI2=d*;*>v*2xpRF3{sy_}Kkz4ue=zUR; ze+&*wF8vdJ=7y4-BNwmyJxg~Uoy58ivMZ9DcXHWnJt6vdVvI4{2S?8y+3N|t;)2e` zM}$f~HOd4`c(_?yHEZeH2OHNmOH94BD^l`p(`U!_Rar(BbDZ9;IdS!@F{k75ri%5U zv5TLiTcq-p#!KhSy>fbCbAMi~0>{A>59~U`MZ6!rS7>)$;UZwSbI(V|?!zC>Uy^xO z(W)*{eT;qn`Hzxi8t-SY%(uQVYZre+Qo)`_S`Ft54?h;&#TF)~lRJ6aGGpUI2bSlw zXE;7y_WH=Q*MUZJm2N#1ZVqsiJ-5?i<9E-6hHF3N3O)9odTe1?LA?0!snfd_>Py;9 zeKb=yrH-}YQSikxlTWKLaz3xotx1?5bNo!gtdhtJrCA}m?Q(XvzCJA4n{e}VQ+VVho>d(&u%8_ALVJ0~VoCe=?za!*{8*xOcX?q% z<8|@QeIj$(xJ6VaD#!m{>k@qGVtM&-r;V95YDbFx*l?%L^B21n@rGaK;FC%3P8F`7 zw4myMwpDwqpXL9^X(}aavi%E%%2qmbd|CML+5)@h3GHQhTK8gq9_&~8vvO6=mbsaV zna-`IPdeJG<{dCPY`tzy(%t5Y1TCGS`tZpBcG!LO^UKdw zr?meT_&2NUt005m-Z&ki=#Vq>aL0$hM$@9jlwtK&AUpslVzMcS=)UL0ES5wo&p9WP#FvwSHFYtdf z>k#XM{nv$flhl+E7A!YBBG>Tf?83?62hOEc^PhXX^Zy%GuD0Cw`@UQ?zVJ-&R<@yU2`vYR7@} z{zGSLD(o&*q-&om@w02`z4vtf6(p9lh5cWOU{@Zn;(*I(slpax31vr`>j@Y>UuLN+BfYckE(Ix?>`WcG>~jPk-Q`?hB@JqUjJ^_Nopb*p^enHy$k35#;c zbIEIkXw-nFZD_IWO1XrX|vM|>)4jv%z3%2Uo}I&e8xXV?T^8>H_qRAs8OTVzUPv$ z(aW8S6)t%6pRAtGIU&OPn9!2xyL+w{O<(+Brp%<@CRtr4gM;VCgjCyIj{1}`Z~fK; z9mT538+UK!OIG}{-Ys7wVmH}&9>2Sqt#bnuUWnGuy% zrZh$&QUi!xH>__LXlMk_ha$%S|Jg$=Ed1P>wCU2 zr+HgS+mg*|I)7{F?@o@1&A2}4bn?b2>%y+v_s_cIwD+8dT^rZrLz{lwZCD*q_HKst zw~aY_w|%*EP`xv?wd25Zg$L(0eGB^VKJBV?eUZA(xvyWhC;$9XzwhU2&FJ}wzt$;m z%|CH?d5iAlsc}|6L^kFGsyx^%7NCBQ_uGWjS<#v@cFU_pi*Dt7yHjLg*VFZD(Ra5U z{B<{e^M80>-e=#kr2mp z`Sn68uZPW|Y8!IX9t)OtTiy3J~vZJ@&1+_ENr?ZiL|Fx<^vtOPpHz@2hhpyx-E4 zdfP=^Xm-k*+<*VP75<{?*|AyuK3EMW=_T?9~ahE^LcDLTvece8gEp0<)w(hx(vaE^Qy0%QYX3tn#_UaIy z9^0D;%~+dmf9~L-i8g5pk-W=N1eFz4b}L1uC@8Fx7b;kp$IWYW=GotWbAPMVEEx{VVt5 z%bZ8=0@6h}@3;Oq^d&RDdIqNzcW}}9+l{XKHmUjVd%Cf(%W>Z(h48eLji+tDTC>mvv zhHdj7$$je9@;2eembHhv z_O0ql@V8!TpUpG*fwKEQjmef>mosgqt~$bfduHsLUbp_wmu8-De=u)Tol0;aZ>qO{jJfg^AF75H}Tg8#UFEcrT=b!`1|-S!}*aW z@2}N8HrXMlXma6t!q0?*_kA~KvwCEt-@Lf|%9Mwe)&0|~vwG^;?$^7Uv!40Sy_j9+ zmyVRGz%wq77K0hDRU>^~>=BG{yn5LvYs+2%m68Wq`~M5>k@>OjVH5w0C10nSB|i|d zdEuANf8f@x|6(T7(q>M$l((CILG@*Qeb%Iv?|F?l50uMIQ?Z}SyB_ez1IyURXS#XJx`YIcw%?Q_eLhZZ^lH7+-oRICsw z%$R*Rn6;5#-(Y6>VfDk>R!N!P8{e;Q>VIGU^Mhiw{KCnh|Jvi_?Ur~S63$6U$YzsZ4~$oRQSz^3zOz1Ims!;3EnK=gZ+Yxz{}Z%bRcy_>$0w(+IeB0r_sa9G zF)Kfd9L!9YI=H*Wxm-br{Z*+#O_g55naa4iw%O&CvGpd6{(_Q|TlzH?A2Bj-Flje5 z4%=j*z&rQCX2!KLnSxv1s%%-aqwdhH9=1)EJ97EiHkEa96*%V@ziaD1JEv#agbUnx zM*?MjC{%l?PQK0|u-R+JH051=!phIQMA_{s>Vr8g| zerRBx&+D}#{pG%71;rI}D$d;z-f*DgCo`MmVfMIXEZdjp8nef{+mv2hb?WI)M|pX> z3`pWfff6tbe?r%4*cf2^~IrEwMTFcor&fI=|%4_cI z>dtfg`e_};DOLGy+Rn=kUo`D@;cYyBUg2m;O@R131@G$63$EMWf8;*@s^Q0$n3or8 zemh#Q2A6zPmSwqZ$!UJ@qaYU zX1Nrd%JpDM}qO$TuIxV6M0#{F8}GX z<0{r{7J{bsPp9g8bVc9WtbVHY&IacTWtL|)?>_u%Rkfj6O<2y47ZpYyo@GnhUAK8` zZSh9fd|jNiOYqrE+D{KGk3YXxLr_P3_J8Js&pTx=7@gDH8nxWH>HFb%dv4qQNM2i| zRmI%?k}ZW>Zj+ayUJ5-i3lF8%aC)WmNENXMR#@2M~Pv?h=Vzs?YN1WIS7~JQ4ls&pp<>6baw;L9H2y*^$ zLpo{o!{W<@f1@<>LM}Ic*lD&{`No^zBw43W#uVZ0+RwGh^LXSt`JX$4EIIN0(dALwzZ;Vy|mYdBT*v zlK1>KJy79LBUPi~Vtk-Y`}l2Fy?ceMVQj0jQ!@Ax15h%yI|GB&5v5PoEIv~vf`P=+K|zH`S!gl zOYY7-v^?e5rJ^@xZT>B_S|Xy+X4WS@<^E{69dgV6x#L}vHh(2HtMv21>N^e^U$o;X z7fh_FzwIUcwv$w1HFI4#Ja`aG?{HMiIht($->K@k> z%0Im-`SM$}KjtvZFN`&+@Z^?T z^LeMrhQQ*sLS~&VhfkA!|JIwcsA|UbZ)|GMa|HJE1y-7w`2Fj*%5OOs@v(hZov2F8 z9D7N-Rjf--hbv56_bEw#!J3&3(#?}59$Inw=<06%(u9|Dcp9=d-rh0wNv7}*o)14t zA2gfHQ#-$r+pp?X=?4LC#kCv{-wXU&_e}rZk`o^y7M+~hyLa)i4-5YUs%kpMom0!5 z{Oy8&#sc*gHix`vHo_l`^`eeu}e`?Rst>xt>7_>J% z-yXYU&YiwB=k<-gJS=9a+0gae>A8Q(ve&`e=NZ*pdF1n6^QM2JnEjRwM$CLue)V2* zj$yRP&A(fA?vU<-a-VZ|cbqD`pb>6J-=#|C^7uI$Z|^n!p~v5=_MtN6y1~^0?Dv<}ez+)m zC;QWt71tlO^lFIQKUiJ-v*3|~@3!zMjT)^L3UwJ??^fy6oJm;{^XI@xtplN-cFbYp zd7~PWcb#pT&Ge0bcJS30ER?$*{mT1Y>l4GxqUV?Ch}R_Cx)G+j+yCV%94)&Z;3?ct2ulx|5-0`+#x01{rAUlDv5zKt#4{WxmOlJA`B$U8Ju?5W&3MyqM<=6`yZ+SN!MdcM~p{NVph z<#vX-F_kj)_A6Z`e%o-Aaa&8IkE}sPr)DE7kH!L_e>n}u1XTo-1eWN8F-5dFI5n){ zxT500=^DCNrb(vLOSN5VMb|9pc3YN}L060UCRV+9w{ZGYv-WM%Yu|rQdo%N6&&O?^ zHhy*Yna@_uudJe72upE_O#Sc=qf|{(oOQ*3B{~eLL^Y3uXJW-kUbL zsccDMaxYhTB*8i3yw{bVU1uiw)T)Kw%TL&QdUJU12g{YLdP_ITo@VqwMb=hkgHtwpfa*N(? zwNuNSZA#W$9JjVau?9Oe~ z>u(Bu{|o9ZuKm}{HNSm!lF*iqmiH$7|MU8{#D%vHUfrlIo%vcj@&4cC_Lr1RIqX)h z$PCGQV_E;!Z=IE=MbVE(yLZ{1-FcaJdXn|?Uxn)emY!8DasTvb>51v`H@B6p{cz3w zw^T^ayZFH0;df76{P_K<+r4`80*h<+KJI^!)AMAG&7$p=LOY99DxG^@=`JdpRm8e4 z$Lp69*Ds})#aeRqcW?T9>v25T|J=X)(d~b6HTyq*@k}~+_PyNR4QrpCEH*Wr^5esi z_vLb`Hn)?D)#I)w2WPi^R)2N(tF^!U>uoA(WgT*AZXY~7>&-VEaWj=qSv#{oZdTXV z>h!p!kG3D%=k<5q`?Ei<%uw?CWO8WAyj8zC_Sv>i`S`(7=e@V7OsZVXreugJ_7lr_ z=KtODQyS%U2UCENu z&QyKwGbiX*k^g#6n=(Q6Mt->Xc1B^l(^0-Vo05`slcTp! zXPdg_wDGOV;!uyyX+Nbt91*@{c0EMoRY~^ob(XePZ&qFvma%7F^Kdfrg=Mnq8PD7_ zwFoG4KEo_(@mgr{q`TK9Px|^>+=sa)?WClKU+eiTpC20M=IYMRh&i6HQFuY$e>vuS zmpSnaC$_cq&p7>Nm0;Qg_K=gI>=V8Vt?@UmS$0y=gRj-!esh!m-4h>P@qWF3_iJI? zMTgDYlcnTl#_fFi@yf|JCoW`t4LI=S{{4czn^F_wt~~lU<@9vF)Aj|uTSO|KJUKA+ zIp>B02eS5QM7+yn{ZR4e*tJKmwr?pD61#d!G<8lm(*?!z(?!JF!@>fiak0 z%BFJNu18Okj)~cvJ@Y>Ep|E!U@_-py&po1@_j&o}>hK!}Jeghidab#LOaQlD#kmbX zKc4#7p{n@?wD-jMuzPEt-DkG}3xj_r&Rhh{a~ zf;OL|<(y60-F5Jm0Lxe2-;8}P^`j5Ay9Z^7RdFu3a)tTbL-Cf%p2~&MGkT!h;oVoYUrWyTQRfXTj#;qc@^rM5kR#W0twwHUAz{ zhuM2(`H3%29-OCjA^jcW_YVph_mnv2PdjHI6}rgqDZ_S~TPJr4J$g3XjZy6MwMuop zV{5OA^JE`>^z?7bgz!GK&kQZ4k=nNf6LVbGY?Gi3&uR=#P4PMTFyT?o?N?%x70W~tj<&vB`fJD6wx)@8 zH!J&^FX*YQ*p)g@OJuv+$qti(byt!=+fRxrr|dLqMy4MGF5Gw@_=#W`Unm;OE4;6m;dJ45(=E$xYo%xEzRo=A@kF>Zdww)qO{DhcZyyqqi=JIP z!j;8-cN@c_w{%!&ZEHWT<)C}4v36#k74s%ZokmIl?eM;-F|i%%XLr zx*jI~D=!I7Tx>DD_hHoZS#s%HJ0yPENvw@;cC5UXnb)zWaf;=KnFl#tLO1_eAb#_) zr+>aujoJ@Zk^Z_8(<5;)hx|W^_AKHu)Vj3pz#P#2ldJ8Z{U;Z0{re!ux4*#a;l>l| z%Hr-Rt(vj1m4EvSy<}O2vuEx!|ESs~TBEk$%z?e@-szmtRzIK~R=~0#=E~R3X?$k4 zCzZHFJp2&pvXV=F|FM$!-|QsT@}GXWiuLIUX^)x0WgGu+?tG>)Y4+#SJC-bZqIzIC z!=p=CJ62TZN;B)NpTWzSpLIDlt8XojqMfj9_!#2YwrBeDf;FhXakCnou~VgZ83#8S3w(40>K+lF8+$xtSeGB z(a8Lv&Lu(FyIp0Q8*abnUR?d59<%|4>(FvpHviRKPYv%dWZ&v$oH<>?Ubpy^U4z`) zNw@jzeqH=DIyKK&>G?y)U~6ZxR87ydo#EZm@4i+|%)3oWRN`CXCe`5OyM9aNJQU8)oxk+k<>X}Mjd!i@9K3#P&$}7H z<~I(tHy<}|>69<#pLBS6)a}rc1$@ylLBi)aKXi)fG%-yBxgaIN)G)re-#>b-_QeOu zTlRTnoB4RZU2>+O|ESvGe)XqyGj}IVKbN(`XYR$6WU<9@XU`OfM+9tM@b25biYuFW zxYq4vkJ^3yM6cJdGKS@24G%dYQ0k zLpQtiLsiqz>^F`@KXe*=mR{HV^L^begRpf^Yc5nJJJzk$5wx>p=dqrgGVG?(JH%MxnOaOh?}!{0`ua8 z1KbL?6u2~Ad+a{p8pw8#XYr=)13gANUN@{uYwPYbOfr#l^e8j3$*n0g?>lnBZieTX ze~f9*JZo>f`JEQ8cJsdF@>RQ6@6O0Lp2zk3x91YQ*wE+UuJik!|M3e=v<`jhWxb=5 zUo?Jc==IpptxAt)rfn|W9$xZEQEk$BpZ>FDxdtcQ=lA~R54@H8+AeY35GPjdMlS!;VT<@aw6a{SZB~n&~2YzQvU|<==g7 zKTB>l<#~6Mxc*?yGWKj<`LAX3+e0rU>k`?Pd)3@}gYwxEVek__$dgAz- zgO}b$i4_aizSTC%&*6&k$Vc6dNi|_W=jpY~l9{P&f9=tVI zRBgVIT~Ua5&@CbUX?2%#jQ2PvyWEf5my}$sAe${=Q~c3ot(jTAMe~VI6K~&oGPj{- zhtC`J_FG$DIGf9Oa!o#8H~XK%Euruup~~JjR-9h$koVJldX3e5{>Rem>od3ipWe*w zeQ?zT{}0|;F;V$RdwjN9@x2gUa*1+6{4`^edcBDXH6KRT&f`9JiWRN?!DAAf~E?VNDyqZjW6uP2l9 zW=ZUPm-$8d=+zgV+)_8pOq4&(K2@7&9bU2O)7kj%2A3oDUfp_PwZf7mh<-YnRsL0x*5uYA0Cg}bYSUdIhxiuC2Y3FC=PF=fNEo$}GZPTj3MqpmE2-|`3 zi4n4)1ZndK=$1y%<`K{>ji7xdXi6tYd}3s1_`&p(fgvo&E`x=MVZvu3ub;r}43K@0 zjW81qd^tPi)?=@iKJhY1odKs-zdUKF8h6ImZ+}w_gMs4FeN0P^s7`bBs$6(vw)g~o zhQv2pZcmo8d~|BD%I@0qGs%%#rm&{IE6kVi-~RpMoJ|Zr83e9f)9Bq?)gs|_HcIuq zwboh08Kskwg2&U-uN=e?iviR%-i z!VJmkNy)Aro6a9m6q_8hZ0d3Cf?fSnGf%KLBzT1{dw5t{b!lP8(IrznmzJ~oZ4aN> zQp3P;EU$cxKv}g-5~qHuU*yEgq0bcL8HA$kPH~-d-#DkHh@HQtRf<7D{zUrCPm(fn za`zUT73bqO-*WQi`i#=bC4aRtn`<3UzwO!hMxtix(h|zy0?=*%=JoLqp;=y??u_>`8-+bPnds3 z_}JY`IcCRuB{1yK`p;{Oq<&UN6mN5~WB62gJ>pkwa_6+qnG0s!d?n6zdP4Txo~J7> zo^XGg-_u&HApc?Ar$SQ+&*;FPB5bZfNk5)=I3G@(HO2VahuKd*hXu|Pyqy2&ddH{ue7X5EyXTa(1{g!x{eo%_s*uT-xQm*Vo>Y&i^^*;$*w$8Exw>MB3Fy znNQk&I(9-~dF|e53Prn*U!5{%*SyY#8m;mjcaP@&S!`X@?VMG1t{^t>vf$p9t**L~ zM}1#eZm)J5*86f^GX+zO@CU{?LYd_?#W^G z@B4ngyq_|Ee{`JP(Wpzy8#gXu?uoq+WT$I#a^1)6BKohk-pHKbe|x#f*}&$fk4v(4 zf7$ZO*mF)W+s}%O&fSaW)ra^Q=bGqV+toP9X8oO5d%5Dgi%%AtCr%yl>6Qh7| z>v>P(^={9J+Wxm&bJp#EGwpF*-Pe!%+Xbugn(NQ~mi6goSWJzXlgPzW#|z)@Tl6RV zblP8ozaJiU@9%9rmHushe!Siu!MBTVUSGd*<(FSyU&|jZ?^sx~=_$i+wplsZlfO=0 zUSl_DGUG(;hkG^Fip^ieuQI22r_{BlZ70;5?f;xNJ$709_|~@GsFW%FUcSrHX2m>R zZNGe`&_#yFk?*DU?VtX>!sXO+#`wVVv#p~|e$IJZsD3)8Bx1_4#M93L)U`65POJN7 z8(;V9JgS|yw(*T%v#(LKfceiS8L};kn)8iTwFxfjh_dU8?lRI(o9A;%=EC3SZa?o_ zDstxDF57pzYF?FDO?!l0+(gAmc2~kAL$9X2sy?~TZ=FDU@Uq_HOM8u&bJ^cDY_a~b zIB~j^lKf6r{#uo@*A%8*cJKXkQ|GeRm#YfAU%NH3LQQ+GotnPQ_e5Rns%upfbC>u< zZpg@9d40wGRI#O*H`YpSGG!NvuoJ#xTQ6k4r{*XBAA7rh`(?a~r#$=ERup${dm_6= zqH4F!AGM!z9z3<1@c+-JNSm$~PmBA%I7Suz`7tT2?xCf^)7J?{=E4?Hc3}r_vzUY z#=`Rg_YH&dHlEX%Z@Xx_yZ!8!+Y2JzPL`gx-1B;1W6Z=?_qC(>{=fgcyv%6nzC(*l zcz0VCFsph5{FGUkb zV-IDYZ2Nfn`QN8C9G6b&wm*KH-u3D8f4j1py$dw-k28Imx%AoD$G7`7J~ycqp7WXW zw&Ju;4{eultS*K_NH3!`qvpJ)1!K}{cqpp zuk`Uu?&tLlHTN`5MZe{>-L!e>Y4x1;h*RowpVwVKCE~3YyV$KiXJ1a@9#5qcQ{S=) z|9oLuCHV70&bO$YNh&UOvU|3dPp^v1z2f`m+RKGgt$98ba{s)-np0{XW0x2>t6lfl ztM7~Y)HcuEnO?0D_<-sywq^xMN?eLd|-&Qhq-w)h>c+=Klw!77j z8C*BG<{N*o_YjU%x)3niUND%~`e5*r`5BKKBYA2rWgPF@RIm8sN%u|b2d#6u8u?={ z%{tiS{FnET^P;EzuXDTC6_kGZxH;m-jI~=#tc`zEUQu>h>;7YoS=*NhoBPj;-M&&< zH?h)=$GY&w<~?@vi(E4=w*7wFbY5q6T1CYF&C&i!AAX+tP#m)v>XIV6@&Xs>6rtdf zC$qaEav%O;p5GzCJYTlxLzh{v1z$}{)TFm|yYUZ;96d@GK+fG#D$k#SnT+Df9`+rAXzKny$Y7Go3vTeKm5*`>g8vJ>~h@X z=dwuf+s&(byZV`zPR*tpIT^E(gau}9R$^bVebT1={dR(ik`pR$Tm&?Zc`h_CLSFZDA8|wy7_#64X^c8%h?Y;&v6f4-=HdXT3e2}W?hut zdIQl7;&wStm(6H&zh19(^O4I|w?2FJlj;EPafPqdg#)nh#wa`&+S>fQu|TP z$)f+VYtIFj{`s`<+lv#sMQf7Q_GFuJ3++EvwkytkcbV2H2LIlWix#hc?|uGbro2dM zi|KD}Zj)VyY_+Wp{}lf4A@b2>>y00FZV>u#vX--CVcltO!(R3|bANu{DPvbM<8`mf zUw5C@x)SFSlh1$alDKbh&yTIym~lnE^GwIdA9G9}HPue zC&BSC8JgKiYYYyx`xW`6UvD_Vs#`umzxn-|H2Jm{tJn8FjX7x;c4orOw8Ye!&6)GJ z8ujs(&OO8~HzP&;byVA)yPlrQU4GAJH2)p^tY^s+wVXuFxBVLFyL%*`|2(oesq;f6 zM;*Ux#`BL?mA56l=9RasKdi2IfBKTc|5I1%-}|)iTewWvGp28c(itm1xl3(dcS5f9 zitNXl3+m14-`{XZ6vYI;d^~Sy#Ll9$#sBgjOm-1}l5_j^4Bv3pA_>iX&Bs4C*qSEE zU3j`m=J$imzQK~b-yZxn+_~Q5xAakt`4wR`1~qEx>h*usqoqGKIloEwu$#O88;8y8 z!{2|{%nN^iZ}Y(ohbQybRk5eu>-cHA{@L^W#^MpCKmMfUzR`Yak&)VcJYYp=v;RB! ztrdT~cSS$AI(OoSIk|Et)ECd&Ua#(KcKG?0ZJ}o-2-+2{iTLE#)-InhvtKFFzFYE) z&W2NeO^ucuwVC%_=kJ-AgtLd9bMHEIkoDpmeXH;MtzI%UR;72(Jm0=qY<|te^CjnG zlRP{$e@v7&yWqV1?}wKumVay4d}0>JSbFebdRNfy22t;(^S9Og-@pF$X6OHwr*@*h zt4I+V0l= zYTfbnp6mtLcT)T}M9LyF(@xK;TV12AU3YtZ_??Pxmvislxc0c(Mrn7|?_0*U3GeRi zo0uJ$(AyRpJ6Gy``MX1EO;M>oZYcb@yY9Yj&69-E@3O9i;;9uIS{De|am;5~&2+HY z>4!p1fJO`dA@+yOPKq@Gb{Zmw(s>W2S519j-21_C;TNsKGi&&&(_~aRHym9wjdNwX zXx?`BSq?uI1XSHE&54lxkyCKTw?S8ItIsCa*PYkam~?`4y((SP_uBYz!IxY4-J!bc zm%co*+s^j#rR)}duG9&a12=JfnWNq zPETt-9GJ6Pw=la>&-c8VkiG5Z%a@8JA2vJ9{M#Ek!B}zIk(Ezdr({4hx$ZxIchZqb z%b(=sGv!1sn09@#@MTWs^IwiU$p&dzShwEk#{!Tqxmulm+*p5h-`QnXW=+2Qy!erG zi_=Y>`HLP-O>vnZ5TA33Z?1%#U7oTdr)0dJ=(GK)3hucd%R-hvR_=8AF~h}U)x#V1 zQ8i7fpSrtmPmB5J-ZH1x_vXSRr<^aVvpjxpTGjD4`qa~}Ew2UaCTp~uPvT7zSA28) z;!`81ZF)N&MLGRS4$kJYi)>k;%{1Tb)ZAGcq~lJ1Sm>fylhf*NoN2oBnoP|sj?5^| zA3JVw?u_B=3|hedZeG9Bj~krx>zuNZgR}YVv=k$?8RuI%t^3TIa-8Gj!R11&9}LZB zd~}}eG@Cu9V2i74TYhG!?WV}3g3YU#>ibr0En1+ZZ>qc7$Z19M)`cIwsa?KyCY@8V z!}sZXZ7#{c4mj>|1Y`5C2)Z z@7jfZpFB>MU0NHu(V4gPypQX-E2-j3+m1hbVj(G2)3kLJ*S^br0UFKof6Ng0bJ0lT z&~cvmoA(5l>~Na)y^HVCv)4+DxhtA0j=LVuoSN|~ukJ(=SLNfUe7SdT`#Lolt~8FC zu5D|>wvXYo9n00AlBXTo6H5|(b~MUf`~7?e>)JJ4IZkfvKiNNP@qGBW{9{hO9}nm9 zJ%3zwU#^Hb&v}b6^wXS=rv&VcwY>U1XJ6I|se9IAQ5+&+{bu_)fe)FxT|K%R+a$mJ zYSg$Q)ym(@o7-U*DVMR(BqQ^tSe~NJ#B;YYjkn~Lx7aNkDpoY zy24mi`m1Pin)7Cv=I^_=Ts*zVeVgUfLyPA~GoM(w+%)u{H@E33*8|IEH#OJoIlwUA z$W7x{Qct$ug-v-;8?Q^+F4Mp3$Tz zYMP{s-8zGalphl}DcpJJSfdrlA#Yq`bn8i1R>k|RZ$7_VaMpUSQ<<1!^b+1LN?U3& z4%TNqUnr_@@a)~bBB#C+ZC7M+cQRVOX!zcrF@x!BefOJoF<<6MIto6FJ2-ML#BAqI z-ZOJkdP|k4v8Hxig{dfmVP=t#gk#5kAY{-WbW-Z)gC;TcOhW$&mrJDl@2<9yzgVpOrivNvlB zV{z9Juhl$U^8d{W+Daz#IO){z^geb5<^6|;om=*{@EmPEZ+9{O5a%Vimp;#mcDg9} z?5T6svbXxj{7=xDz3!l-{>Nw1Js%wU4BBE}w}h?hHdeAOn!6y}UZm74Dau}Pd&%`? zvHD5s&zJpaOWtu-sV;br{7qe}`o~vpMBAme#Q*+VPv1PBKflmvUsOXzvm`wtltA+_UUw zTgIN;Rq0(Cc=FmhMX5gp$^VYu+i7aLF{iNRSEA0nJDmsad7N@^YPfYOR$yU6@)M>p`v4pO372<-|E}cwt9jC#2a|8Svi$#_dFH$? zUrGwDo!=U^o{#xc%;I~q1HC4!-1v7^&yu#L^VaXWR(`DflF}dfHf!gt!(STZU$^{V z{_3~;+3iJ-uB2wSu5vq67PoZvj&mne&QBC(vg7}$ezV}-uettd=g+Bq3~_4c7q?5- z-L9rzQ*yyT?QUOOR7cgUqTTvUBG>P@UtSj6Vjvh4`P6{h{0ke)WlKqJyGU+3tNCXt zi!V0$ugb1LEH>Yv?0c9+APRzKO3`R+|miXg?8MxA4@|;$-=?J048f@ZfV)?81jrGvF(fJ-p^hB#AEiO{d+!aG&@rDaC6ea&t1_DIbwrJUVCh;)!rn&J2khZ^(vzT>~1ZJPHl>q z*vQ^s{`O;Tf3ow^?ZsOrFZ0oV+S*6M#ptzH>rK0UE8_VN>rpB2%TZoEd^vi}w<)ZEHBzVx<| z^Uhd?7eeYiD}(MQ`d2rKo%?*~dE2)Idp?}JrJ1`*(xzyU%e_|Sz@?YZ=j6Ur{oWRQ z_c6cYf9(&IYac$o{y}oV*EOD!dsSWsP5=7;`?PYWK!hIp&v{BIABUNGI+6 z;A!^Ksp;d4FUfs#7Q9q{Uudqq(qi==Q^xR`<@_u^WcWdAOeHSy^{@EMsd;0@XH`|% zJB1S!ZZ7(KH`FeYf3x!Di|-b%$n3itcHOM^^pE2_(^PhD?9zYrYrB)v*UN`K7@p_4 z{?tG3sIJLPt;8$Bdlt{hX$!wXr0QSK(RG)}vvQW+p$l?EUz8L;r%0&4o{o z*<_qAZECx5;brQ--Uz)U&o2jO+yVd1V2k|a+9@{Kvo@am*(Y}Da9ic)rA`+OENv#={5tCfXV`9^ z;v@~hr*ATaVx+~wyP~A)9_47+DVf!AmcQAh7hR?KNq-{W#@~zXWR?4Mo;M0A;;9h| zvJr zw9E|A$_VM{QP(+z*Umb0X=UKCT@Qa4b=^+h^*%1>7n88(Q|Y&@OOsu8+pBXOS{~D{ zEWEubX;aYg`%BLAYs$B*+UUAbe&yP`clSjW$<(}AyV3nYaFFf7;{m_#>{b8rCHu1VyP*FDr=>&KZ~wI>!SEwgmJ5R}>L zm)^8^I>TqfMSnz3P131dGWC0|<-6Bcd!+3STTP8k54AYXugY_NCFnHkHGkIKFVECV z`S;KM%dD!K6J)H!GQV$G`7`or(DWl$4#n&)dHrVXMy6w{s;XSq$Mf!Ac&NeZ??W5) zFJBJt);hzLpME4(U257#AE$=%o>#1$TWwGFeosB+$uWQa|FeOYW47JdnvgqHX8Jy- z)!$TQ+`oKt`sV#lX?ibrK6m(mD;}To-|TpEW5dnhwRu5*qyOwF-9OJ>w)o66zV^Ar z=hE(r-*|Rm?Y53*mnYf(Tl)Qma9rzs_URuSoEpy8FFl?&?MYzuHNx9^3D7# zj`o5zZDsNQUj(G2l$SI9UnTJ6Kbt&9lfnUZ2R?Kk zYWBH4-rH+@m+eW&cb{baeNm!Ke!_OCt4}-l%IU73_Hbd-g3FEmzYPS6*R*{M*Hw67 z@?djI#_EJWt^5D7J05NK*S}H3^!3rfrUgoepXWC`b@?UjFUBG&YL~$Og4xlY$*yCk zK&9h_{)Y2H6F%`eU6X9;Vx9l3K54GDMQtme+}0mukB+6eevn@OyYkwe4UhO7U7PxM z?_0j!RQAUD{1-uIo6Fa~V?Xm;d(%XZ(rV#N6y8N8owz8`t)+=uDLH5#8I4bDY8sJOm-%Ua#nJJ$WMN_n$uapmNV>-N;@ z8(n*uu?XaNrum`EXBHk`aw+`Yx0Z=#`_CDgZ8ZMbwcq9}ul(Zqp})3@ABm{6`#pDE z|EiC|k4iVy{jM-8@LFrR?)SHbVEU7SaJ}+r4?j`xjYURXft6qq;W_4beCO9XL`|NW!*XyUBy||WkZr;I_tJhxa zV3n_bl2yFo3a|YCkS$s%OE>Zb&tJ8T-}ZyyhU2nLT@3Q!z5eRSy64JfowW=$-uWQ; zNXf6Z$xoMg1p1#)nW!0Ty7kQQzlzrqf81E(F#Aev;lF8Tp4?l$Pj*q^Q~j#y;}>kT zc`N5GaI>8|!Eko}OlQW6cP4+I+p~1e=btXN;>BNLW=?(owOp`9EpypZzaMLYr-!#J zIWgyr{nJMNtXhQ?93L`EqOL|C`}SKVS3Y?V%+)$F+?-gC%}b$(!7YU-@> z2Sw)HW-UCth#9 z`80dM!}W=4;tsXfo)9{gruqDG{si|DZ>umJktur~%sXk;f2(4H$o=%>zB&A{-`l>g zcq}OReHNcekk-!iC58(3g|*YmR=t#3emh!j3G0o2%eJT6bBA2uIq&gfgZiEQo-7dx z*$2}DoU~6g?^|ud^mtCx7wOrDl#36hvz-@bskN`UBiiQ36?H~7^_=qGqPu_Bd({1q zkUZx0)09R8pjLY4pzwu>%jvT?CI!!G~s%@W?nOVMH<)S^UTit6w{~j7B9@n4w#nU<4=*2! z2y<~MocjK1msrF3hSzcj8MW5lv*()c8JX&`)aCq|jYS93?-l-De{bGgA)7T}ho85t z(h@kG&oZC!_|w{Nf369XIR<=uXgqm$#Il>4%2)axoA6QOLsAS zn7FZ5Rqpnk{K(bDr~Y5Q_jAvdiANd#>}~d4C;MihNl8NHq6bmernrR%6mEUAP@r&g zY*dS>X4}!c7t4;tZedrJUa_I#b~ab>$Mq-Vx4%!H(|I*y#*x;)HF?QtM#%|wd%o|p zzZd#*&6jlF*H$ZEu>EWan)mbeboO&RQ>)M2S{hI?x8~jNi&wPu=SOO8|NC^I-KO)~ zk2M)P{QBhCa_jszuEhQ4k^*zyzTI1(CsH% ztG#Qq-k5o$%N59;+umG#tQc*FnGI{$c1sGmA6r0Ab|PDN?j_4qKhb2TrH z`QNYjc_LYS+HKQMKd(NqoIB&7-^P#K0+WNxz65HzPf`T=a7y@!>%Q}@RV`JI{oK@* zzw}P_^!`u!OAEUvoxdOIz3BYW{;J+1NBj7e%vs^~ur9W&+WE=i=0NqI3;rBh6KX&C zlc&HP>+Rg)*;i%^4~4j?B#ZbLB3fxsw(A zz5ezmE4q8{o_>0x`N)Om=Fh4)fO0U*0+I5bxOYO};lh&mS$`kyx zCG~b0JoRA>eC z)swfs4ilUrrgiap*|rsH&wY7sw!i8|cdg0g&f^=?#I)w``y<=DmQVb%_Vp4~p6#pq zg^#y$Pd`2}c1zc`WPcU4&r`Ky&u6HyP3WrGlry=;I%6@{gh@5?HU}Ex+Btu!?%2rO<6?L7O>>y=(HQ5=u5rc(PCQxsqw?p==x6cFo7ZmlpT07g zU1DPm1H%M<2GEM4C(N)@Xg@JBJYi-4tuXq@0FrWmoNo&{zxESj!~Y#MvmWv8XJzQ= zH(!+ba1l*h}} @@ -492,8 +492,6 @@ For more information, see the history. {{< /alert >}} -TODO update documentation according to [new plan](https://gitlab.com/gitlab-org/gitlab/-/issues/503733#note_2217234805). - GitLab automatically disables project or group webhooks that fail four consecutive times. To view auto-disabled webhooks: @@ -501,17 +499,48 @@ To view auto-disabled webhooks: 1. On the left sidebar, select **Search or go to** and find your project or group. 1. Select **Settings > Webhooks**. -In the webhook list, [temporarily disabled webhooks](#temporarily-disabled-webhooks) display as **Fails to connect**. +In the webhook list, auto-disabled webhooks display as: + +- **Disabled** for [permanently disabled](#permanently-disabled-webhooks) webhooks +- **Temporarily disabled** for [temporarily disabled](#temporarily-disabled-webhooks) webhooks + +![Badges on failing webhooks](img/failed_badges_v17_1.png) #### Temporarily disabled webhooks -Webhooks are temporarily disabled if they: +{{< history >}} + +- [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to become permanently disabled after 40 consecutive failures in GitLab 17.11. + +{{< /history >}} + +Webhooks are temporarily disabled if they fail four consecutive times. +If they fail 40 consecutive times they become [permanently disabled](#permanently-disabled-webhooks) instead. + +A failure is: + +- The [webhook receiver](#webhook-receiver-requirements) returns a response code in the `4xx` or `5xx` range. +- The webhook experiences a [timeout](../../gitlab_com/_index.md#webhooks) when attempting to connect to the webhook receiver +- The webhook encountered other HTTP errors. + +Temporarily disabled webhooks are initially disabled for one minute, with the duration extending on subsequent failures up to 24 hours. +After this period has elapsed they are automatically re-enabled and can trigger again. + +#### Permanently disabled webhooks + +{{< history >}} + +- [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to become permanently disabled after 40 consecutive failures in GitLab 17.11. + Previously, webhooks were permanently disabled after four consecutive `4xx` failures. + In 17.11 webhooks that were permanently disabled underwent a data migration to update them to the new permanently disabled feature logic. + These webhooks will only list four failures in their [recent events](#view-webhook-request-history) even though the UI may say the webhook has 40 failures. + +{{< /history >}} -- Return response codes in the `5xx` range. -- Experience a [timeout](../../gitlab_com/_index.md#webhooks). -- Encounter other HTTP errors. +A webhook is permanently disabled if they failed 40 consecutive times. +Unlike [temporarily disabled](#temporarily-disabled-webhooks) webhooks, these webhooks do not automatically re-enable. -These webhooks are initially disabled for one minute, with the duration extending on subsequent failures up to 24 hours. +Webhooks created before 17.11 and with four consecutive `4xx` failures were considered permanently disabled. #### Re-enable disabled webhooks @@ -522,7 +551,7 @@ These webhooks are initially disabled for one minute, with the duration extendin {{< /history >}} -To re-enable a temporarily disabled webhook, [send a test request](#test-a-webhook) to the webhook. +To re-enable a disabled webhook, [send a test request](#test-a-webhook). The webhook is re-enabled if the test request returns a response code in the `2xx` range. ### Delivery headers -- GitLab From d36c070e5c78495cf7e554b52f8cc3fe3d6b3ed7 Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Mon, 17 Mar 2025 17:11:28 +1300 Subject: [PATCH 06/11] Update migration to regular post migration --- .../concerns/web_hooks/auto_disabling.rb | 2 +- ...bled_webhook_into_temporarily_disabled.yml | 9 --- ...abled_webhook_into_temporarily_disabled.rb | 27 --------- ...rate_old_disabled_web_hook_to_new_state.rb | 32 +++++++++++ db/schema_migrations/20250311134655 | 1 - db/schema_migrations/20250317021451 | 1 + ...abled_webhook_into_temporarily_disabled.rb | 25 -------- ..._webhook_into_temporarily_disabled_spec.rb | 52 ----------------- ..._webhook_into_temporarily_disabled_spec.rb | 27 --------- ...old_disabled_web_hook_to_new_state_spec.rb | 57 +++++++++++++++++++ 10 files changed, 91 insertions(+), 142 deletions(-) delete mode 100644 db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml delete mode 100644 db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb create mode 100644 db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb delete mode 100644 db/schema_migrations/20250311134655 create mode 100644 db/schema_migrations/20250317021451 delete mode 100644 lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb delete mode 100644 spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb delete mode 100644 spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb create mode 100644 spec/migrations/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state_spec.rb diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index f5feff593df8a9..ee4ab13dc5230f 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -90,7 +90,7 @@ def permanently_disabled? recent_failures > PERMANENTLY_DISABLED_FAILURE_THRESHOLD || # Keep the old definition of permanently disabled just until we have migrated all records to the new definition - # with `QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled` + # with `MigrateOldDisabledWebHookToNewState` # TODO Remove the next line as part of https://gitlab.com/gitlab-org/gitlab/-/issues/525446 (recent_failures > TEMPORARILY_DISABLED_FAILURE_THRESHOLD && disabled_until.blank?) end diff --git a/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml b/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml deleted file mode 100644 index f7699ccc4c760f..00000000000000 --- a/db/docs/batched_background_migrations/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -migration_job_name: FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled -description: | - This migration will fan out the migration of permanently disabled webhooks into temporarily disabled webhooks. -feature_category: integrations -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329 -milestone: '17.11' -queued_migration_version: 20250311134655 -finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb b/db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb deleted file mode 100644 index 1c84209416ccc6..00000000000000 --- a/db/post_migrate/20250311134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -class QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled < Gitlab::Database::Migration[2.2] - milestone '17.11' - - restrict_gitlab_migration gitlab_schema: :gitlab_main - - MIGRATION = 'FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled' - DELAY_INTERVAL = 4.5.minutes - BATCH_SIZE = 500 - SUB_BATCH_SIZE = 100 - - def up - queue_batched_background_migration( - MIGRATION, - :web_hooks, - :id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) - end - - def down - delete_batched_background_migration(MIGRATION, :web_hooks, :id, []) - end -end diff --git a/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb b/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb new file mode 100644 index 00000000000000..0e5e4932ce5619 --- /dev/null +++ b/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class MigrateOldDisabledWebHookToNewState < Gitlab::Database::Migration[2.2] + BATCH_SIZE = 10_000 + TABLE = 'web_hooks' + SCOPE = ->(table) { + table.where('recent_failures > 3').where(disabled_until: nil) + }.freeze + + NEW_RECENT_FAILURES = 40 # WebHooks::AutoDisabling::PERMANENTLY_DISABLED_FAILURE_THRESHOLD + 1 + NEW_BACKOFF_COUNT = 37 # NEW_RECENT_FAILURES - WebHooks::AutoDisabling::FAILURE_THRESHOLD + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + milestone '17.11' + + def up + disabled_until = Time.zone.now.to_fs(:db) # Specific time does not matter, just needs to be present + + each_batch(TABLE, connection: connection, scope: SCOPE, of: BATCH_SIZE) do |batch, _batchable_model| + batch.update_all( + recent_failures: NEW_RECENT_FAILURES, + backoff_count: NEW_BACKOFF_COUNT, + disabled_until: disabled_until + ) + end + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20250311134655 b/db/schema_migrations/20250311134655 deleted file mode 100644 index e6d774ce2a07f5..00000000000000 --- a/db/schema_migrations/20250311134655 +++ /dev/null @@ -1 +0,0 @@ -c37dddf687af231b44ceda39f3e43e52675fe6abd462c87383a19363a592f243 \ No newline at end of file diff --git a/db/schema_migrations/20250317021451 b/db/schema_migrations/20250317021451 new file mode 100644 index 00000000000000..ff5142ed7d6aae --- /dev/null +++ b/db/schema_migrations/20250317021451 @@ -0,0 +1 @@ +b90e017fcfdb70ab0d478f0d5fa2803fbcd6ee444c157902b34cab495496684b \ No newline at end of file diff --git a/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb b/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb deleted file mode 100644 index 04c4f9753e0fa0..00000000000000 --- a/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - class FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled < BatchedMigrationJob - operation_name :fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled - scope_to ->(relation) { relation.where('recent_failures > 3').where(disabled_until: nil) } - feature_category :integrations - - # Values are based on constants from ::WebHooks::AutoDisabling - NEW_RECENT_FAILURES = 40 # PERMANENTLY_DISABLED_FAILURE_THRESHOLD + 1 - NEW_BACKOFF_COUNT = 37 # PERMANENTLY_DISABLED_FAILURE_THRESHOLD - FAILURE_THRESHOLD - - def perform - each_sub_batch do |sub_batch| - sub_batch.update_all( - disabled_until: 1.minute.ago, # disabled_until value doesn't matter - recent_failures: NEW_RECENT_FAILURES, - backoff_count: NEW_BACKOFF_COUNT - ) - end - end - end - end -end diff --git a/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb b/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb deleted file mode 100644 index 1d98d53ad6974f..00000000000000 --- a/spec/lib/gitlab/background_migration/fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::FanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled, :freeze_time, feature_category: :integrations do - let!(:web_hooks) { table(:web_hooks) } - - let!(:non_disabled_webhook) { web_hooks.create!(recent_failures: 3, backoff_count: 3) } - let!(:legacy_permanently_disabled_webhook) { web_hooks.create!(recent_failures: 4, backoff_count: 5) } - - let!(:temporarily_disabled_webhook) do - web_hooks.create!(recent_failures: 4, backoff_count: 5, disabled_until: Time.current + 1.hour) - end - - let!(:migration_attrs) do - { - start_id: web_hooks.minimum(:id), - end_id: web_hooks.maximum(:id), - batch_table: :web_hooks, - batch_column: :id, - sub_batch_size: web_hooks.count, - pause_ms: 0, - connection: ApplicationRecord.connection - } - end - - it 'migrates legacy permanently disabled web hooks to new permanently disabled state' do - described_class.new(**migration_attrs).perform - - [non_disabled_webhook, temporarily_disabled_webhook, legacy_permanently_disabled_webhook].each(&:reload) - - expect(non_disabled_webhook.recent_failures).to eq(3) - expect(non_disabled_webhook.backoff_count).to eq(3) - expect(non_disabled_webhook.disabled_until).to be_nil - - expect(temporarily_disabled_webhook.recent_failures).to eq(4) - expect(temporarily_disabled_webhook.backoff_count).to eq(5) - expect(temporarily_disabled_webhook.disabled_until).to eq(Time.current + 1.hour) - - expect(legacy_permanently_disabled_webhook.recent_failures).to eq(40) - expect(legacy_permanently_disabled_webhook.backoff_count).to eq(37) - expect(legacy_permanently_disabled_webhook.disabled_until).to eq(Time.current - 1.minute) - - expect(web_hooks.where(disabled_until: nil).where('recent_failures > 3').count).to eq(0) - - expect(ProjectHook.executable.pluck_primary_key).to contain_exactly(non_disabled_webhook.id) - expect(ProjectHook.disabled.pluck_primary_key).to contain_exactly( - temporarily_disabled_webhook.id, - legacy_permanently_disabled_webhook.id - ) - end -end diff --git a/spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb b/spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb deleted file mode 100644 index 3dabd422d62f67..00000000000000 --- a/spec/migrations/20240925134655_queue_fan_out_migrate_permanently_disabled_webhook_into_temporarily_disabled_spec.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_migration! - -RSpec.describe QueueFanOutMigratePermanentlyDisabledWebhookIntoTemporarilyDisabled, feature_category: :integrations do - let!(:batched_migration) { described_class::MIGRATION } - - it 'schedules a new batched migration' do - reversible_migration do |migration| - migration.before -> { - expect(batched_migration).not_to have_scheduled_batched_migration - } - - migration.after -> { - expect(batched_migration).to have_scheduled_batched_migration( - table_name: :web_hooks, - column_name: :id, - interval: described_class::DELAY_INTERVAL, - batch_size: described_class::BATCH_SIZE, - sub_batch_size: described_class::SUB_BATCH_SIZE, - gitlab_schema: :gitlab_main - ) - } - end - end -end diff --git a/spec/migrations/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state_spec.rb b/spec/migrations/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state_spec.rb new file mode 100644 index 00000000000000..ad26f6e30769c3 --- /dev/null +++ b/spec/migrations/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state_spec.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe MigrateOldDisabledWebHookToNewState, :freeze_time, feature_category: :webhooks, migration_version: 20250206114301 do + let!(:web_hooks) { table(:web_hooks) } + + let!(:non_disabled_webhook) { web_hooks.create!(recent_failures: 3, backoff_count: 1) } + let!(:legacy_permanently_disabled_webhook) { web_hooks.create!(recent_failures: 4, backoff_count: 1) } + let!(:temporarily_disabled_webhook) do + web_hooks.create!(recent_failures: 4, backoff_count: 1, disabled_until: Time.current + 1.hour) + end + + describe '#up' do + it 'migrates legacy permanently disabled web hooks to new permanently disabled state' do + migrate! + + [non_disabled_webhook, temporarily_disabled_webhook, legacy_permanently_disabled_webhook].each(&:reload) + + expect(non_disabled_webhook.recent_failures).to eq(3) + expect(non_disabled_webhook.backoff_count).to eq(1) + expect(non_disabled_webhook.disabled_until).to be_nil + + expect(temporarily_disabled_webhook.recent_failures).to eq(4) + expect(temporarily_disabled_webhook.backoff_count).to eq(1) + expect(temporarily_disabled_webhook.disabled_until).to eq(Time.current + 1.hour) + + expect(legacy_permanently_disabled_webhook.recent_failures).to eq(40) + expect(legacy_permanently_disabled_webhook.backoff_count).to eq(37) + expect(legacy_permanently_disabled_webhook.disabled_until).to eq(Time.current) + + expect(web_hooks.where(disabled_until: nil).where('recent_failures > 3').count).to eq(0) + + expect(ProjectHook.executable.pluck_primary_key).to contain_exactly(non_disabled_webhook.id) + expect(ProjectHook.disabled.pluck_primary_key).to contain_exactly( + temporarily_disabled_webhook.id, + legacy_permanently_disabled_webhook.id + ) + end + + it 'migrates in batches' do + web_hooks.create!(recent_failures: 4, backoff_count: 1) + web_hooks.create!(recent_failures: 4, backoff_count: 1) + + stub_const("#{described_class}::BATCH_SIZE", 2) + disabled_until = Time.zone.now.to_fs(:db) + + expect do + migrate! + end.to make_queries_matching( + /UPDATE "web_hooks" SET "recent_failures" = 40, "backoff_count" = 37, "disabled_until" = '#{disabled_until}'/, + 2 + ) + end + end +end -- GitLab From eacf05d9baf9937a2abbb706bb9a770ef96083d6 Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Mon, 24 Mar 2025 16:09:56 +1300 Subject: [PATCH 07/11] Add reviewer feedback Co-authored-by: Ashraf Khamis --- .../concerns/web_hooks/auto_disabling.rb | 8 +++- ...ges_v17_1.png => failed_badges_v17_11.png} | Bin doc/user/project/integrations/webhooks.md | 43 ++++++------------ .../projects/hooks/index.html.haml_spec.rb | 2 +- 4 files changed, 22 insertions(+), 31 deletions(-) rename doc/user/project/integrations/img/{failed_badges_v17_1.png => failed_badges_v17_11.png} (100%) diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index ee4ab13dc5230f..a9a4b3d58e12ea 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -8,6 +8,10 @@ module AutoDisabling ENABLED_HOOK_TYPES = %w[ProjectHook].freeze TEMPORARILY_DISABLED_FAILURE_THRESHOLD = 3 + # A webhook will be failing and being temporarily disabled for the max backoff of 1 day (`MAX_BACKOFF`) + # for at least 1 month before it becomes permanently disabled on its 40th failure. + # Exactly how quickly this happens depends on how frequently it triggers. + # https://gitlab.com/gitlab-org/gitlab/-/issues/503733#note_2217234805 PERMANENTLY_DISABLED_FAILURE_THRESHOLD = 39 INITIAL_BACKOFF = 1.minute.freeze @@ -37,7 +41,7 @@ def enabled_hook_types # # - it has exceeded the grace TEMPORARILY_DISABLED_FAILURE_THRESHOLD (recent_failures > ?) # - AND the time period it was disabled for has not yet expired (disabled_until >= ?) - # - OR it has reached the failure threshold where it is permanently disabled (recent_failures > ?) + # - OR it has reached the PERMANENTLY_DISABLED_FAILURE_THRESHOLD (recent_failures > ?) scope :disabled, -> do return all if Gitlab::SilentMode.enabled? return none unless auto_disabling_enabled? @@ -55,7 +59,7 @@ def enabled_hook_types # - it has not exceeeded the grace TEMPORARILY_DISABLED_FAILURE_THRESHOLD (recent_failures <= ?) # - OR it has exceeded the grace TEMPORARILY_DISABLED_FAILURE_THRESHOLD and: # - it was temporarily disabled but can now be triggered again (disabled_until < ?) - # - AND has not reached the failure threshold where it is permanently disabled (recent_failures <= ?) + # - AND has not reached the PERMANENTLY_DISABLED_FAILURE_THRESHOLD (recent_failures <= ?) scope :executable, -> do return none if Gitlab::SilentMode.enabled? return all unless auto_disabling_enabled? diff --git a/doc/user/project/integrations/img/failed_badges_v17_1.png b/doc/user/project/integrations/img/failed_badges_v17_11.png similarity index 100% rename from doc/user/project/integrations/img/failed_badges_v17_1.png rename to doc/user/project/integrations/img/failed_badges_v17_11.png diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 020033ff3332d2..db6ecad8de6501 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -481,7 +481,10 @@ To optimize your webhook receivers: - [Generally available](https://gitlab.com/gitlab-org/gitlab/-/issues/329849) for project webhooks in GitLab 15.7. Feature flag `web_hooks_disable_failed` removed. - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/385902) for group webhooks in GitLab 15.10. - [Disabled on GitLab Self-Managed](https://gitlab.com/gitlab-org/gitlab/-/issues/390157) in GitLab 15.10 [with a flag](../../../administration/feature_flags.md) named `auto_disabling_web_hooks`. -- Badges [changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) from **Fails to connect** to **Disabled**, and **Failing to connect** to **Temporarily disabled** in GitLab 17.11. +- **Fails to connect** and **Failing to connect** [renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to **Disabled** and **Temporarily disabled** in GitLab 17.11. +- [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to become permanently disabled after 40 consecutive failures in GitLab 17.11. + Webhooks that were permanently disabled underwent a data migration. + Migrated webhooks may list four failures in their [recent events](#view-webhook-request-history) even though the UI may say the webhook has 40 failures. {{< /history >}} @@ -501,46 +504,30 @@ To view auto-disabled webhooks: In the webhook list, auto-disabled webhooks display as: -- **Disabled** for [permanently disabled](#permanently-disabled-webhooks) webhooks - **Temporarily disabled** for [temporarily disabled](#temporarily-disabled-webhooks) webhooks +- **Disabled** for [permanently disabled](#permanently-disabled-webhooks) webhooks -![Badges on failing webhooks](img/failed_badges_v17_1.png) +![Badges on failing webhooks](img/failed_badges_v17_11.png) #### Temporarily disabled webhooks -{{< history >}} - -- [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to become permanently disabled after 40 consecutive failures in GitLab 17.11. - -{{< /history >}} - Webhooks are temporarily disabled if they fail four consecutive times. -If they fail 40 consecutive times they become [permanently disabled](#permanently-disabled-webhooks) instead. +If webhooks fail 40 consecutive times, they become [permanently disabled](#permanently-disabled-webhooks). -A failure is: +Failure occurs when: - The [webhook receiver](#webhook-receiver-requirements) returns a response code in the `4xx` or `5xx` range. -- The webhook experiences a [timeout](../../gitlab_com/_index.md#webhooks) when attempting to connect to the webhook receiver -- The webhook encountered other HTTP errors. +- The webhook experiences a [timeout](../../gitlab_com/_index.md#webhooks) when attempting to connect to the webhook receiver. +- The webhook encounters other HTTP errors. -Temporarily disabled webhooks are initially disabled for one minute, with the duration extending on subsequent failures up to 24 hours. -After this period has elapsed they are automatically re-enabled and can trigger again. +Temporarily disabled webhooks are initially disabled for one minute, +with the duration extending on subsequent failures up to 24 hours. +After this period has elapsed, these webhooks are automatically re-enabled. #### Permanently disabled webhooks -{{< history >}} - -- [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to become permanently disabled after 40 consecutive failures in GitLab 17.11. - Previously, webhooks were permanently disabled after four consecutive `4xx` failures. - In 17.11 webhooks that were permanently disabled underwent a data migration to update them to the new permanently disabled feature logic. - These webhooks will only list four failures in their [recent events](#view-webhook-request-history) even though the UI may say the webhook has 40 failures. - -{{< /history >}} - -A webhook is permanently disabled if they failed 40 consecutive times. -Unlike [temporarily disabled](#temporarily-disabled-webhooks) webhooks, these webhooks do not automatically re-enable. - -Webhooks created before 17.11 and with four consecutive `4xx` failures were considered permanently disabled. +Webhooks are permanently disabled if they fail 40 consecutive times. +Unlike [temporarily disabled webhooks](#temporarily-disabled-webhooks), these webhooks are not automatically re-enabled. #### Re-enable disabled webhooks diff --git a/spec/views/projects/hooks/index.html.haml_spec.rb b/spec/views/projects/hooks/index.html.haml_spec.rb index fab2af6a7ca395..a6551ad86a0176 100644 --- a/spec/views/projects/hooks/index.html.haml_spec.rb +++ b/spec/views/projects/hooks/index.html.haml_spec.rb @@ -19,7 +19,7 @@ expect(rendered).to have_css('.gl-heading-2', text: _('Webhooks')) expect(rendered).to have_text('Webhooks') - expect(rendered).not_to have_css('.gl-badge', text: _('Webhooks|Rate limited')) + expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Rate limited')) expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Disabled')) expect(rendered).not_to have_css('.gl-badge', text: s_('Webhooks|Temporarily disabled')) end -- GitLab From a7c25710e89bc4f3e0458bd34e831fbe8402ed8c Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Mon, 24 Mar 2025 17:16:51 +1300 Subject: [PATCH 08/11] Add temporary index https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329#note_2405419505 --- ...rate_old_disabled_web_hook_to_new_state.rb | 21 +++++++++++++++++++ ...web_hooks_for_migrate_disabled_web_hook.rb | 21 +++++++++++++++++++ db/schema_migrations/20250317021351 | 1 + db/schema_migrations/20250317021551 | 1 + 4 files changed, 44 insertions(+) create mode 100644 db/post_migrate/20250317021351_add_temporary_index_to_web_hooks_for_migrate_old_disabled_web_hook_to_new_state.rb create mode 100644 db/post_migrate/20250317021551_remove_temporary_index_from_web_hooks_for_migrate_disabled_web_hook.rb create mode 100644 db/schema_migrations/20250317021351 create mode 100644 db/schema_migrations/20250317021551 diff --git a/db/post_migrate/20250317021351_add_temporary_index_to_web_hooks_for_migrate_old_disabled_web_hook_to_new_state.rb b/db/post_migrate/20250317021351_add_temporary_index_to_web_hooks_for_migrate_old_disabled_web_hook_to_new_state.rb new file mode 100644 index 00000000000000..07609ee57b6378 --- /dev/null +++ b/db/post_migrate/20250317021351_add_temporary_index_to_web_hooks_for_migrate_old_disabled_web_hook_to_new_state.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class AddTemporaryIndexToWebHooksForMigrateOldDisabledWebHookToNewState < Gitlab::Database::Migration[2.2] + milestone '17.11' + + INDEX_NAME = 'tmp_index_web_hooks_on_disabled_until_recent_failures' + TABLE = :web_hooks + COLUMNS = [:id, :recent_failures, :disabled_until] + + disable_ddl_transaction! + + def up + add_concurrent_index TABLE, COLUMNS, where: 'disabled_until is NULL', name: INDEX_NAME + + connection.execute("ANALYZE #{TABLE}") + end + + def down + remove_concurrent_index_by_name TABLE, INDEX_NAME + end +end diff --git a/db/post_migrate/20250317021551_remove_temporary_index_from_web_hooks_for_migrate_disabled_web_hook.rb b/db/post_migrate/20250317021551_remove_temporary_index_from_web_hooks_for_migrate_disabled_web_hook.rb new file mode 100644 index 00000000000000..b69145b318e37a --- /dev/null +++ b/db/post_migrate/20250317021551_remove_temporary_index_from_web_hooks_for_migrate_disabled_web_hook.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveTemporaryIndexFromWebHooksForMigrateDisabledWebHook < Gitlab::Database::Migration[2.2] + milestone '17.11' + + INDEX_NAME = 'tmp_index_web_hooks_on_disabled_until_recent_failures' + TABLE = :web_hooks + COLUMNS = [:id, :recent_failures, :disabled_until] + + disable_ddl_transaction! + + def up + remove_concurrent_index_by_name TABLE, INDEX_NAME + end + + def down + add_concurrent_index TABLE, COLUMNS, where: 'disabled_until is NULL', name: INDEX_NAME + + connection.execute("ANALYZE #{TABLE}") + end +end diff --git a/db/schema_migrations/20250317021351 b/db/schema_migrations/20250317021351 new file mode 100644 index 00000000000000..44ac91170f81af --- /dev/null +++ b/db/schema_migrations/20250317021351 @@ -0,0 +1 @@ +43a806f0236fcf8d57242c339a90e1afbb7c1ca5950d29423da5648eb4b855ff \ No newline at end of file diff --git a/db/schema_migrations/20250317021551 b/db/schema_migrations/20250317021551 new file mode 100644 index 00000000000000..ffaffeff340f03 --- /dev/null +++ b/db/schema_migrations/20250317021551 @@ -0,0 +1 @@ +93b32fdd10b4eddaad779c8aa8ae8f0a9f0806549ee9539659fefa09e79a87ad \ No newline at end of file -- GitLab From 6c5316d5db0ae668b4c0572adcc8fceb633fcd2d Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Tue, 25 Mar 2025 14:34:04 +1300 Subject: [PATCH 09/11] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Ashraf Khamis --- doc/user/project/integrations/webhooks.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index db6ecad8de6501..9b34fd3f005010 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -483,8 +483,6 @@ To optimize your webhook receivers: - [Disabled on GitLab Self-Managed](https://gitlab.com/gitlab-org/gitlab/-/issues/390157) in GitLab 15.10 [with a flag](../../../administration/feature_flags.md) named `auto_disabling_web_hooks`. - **Fails to connect** and **Failing to connect** [renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to **Disabled** and **Temporarily disabled** in GitLab 17.11. - [Changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166329) to become permanently disabled after 40 consecutive failures in GitLab 17.11. - Webhooks that were permanently disabled underwent a data migration. - Migrated webhooks may list four failures in their [recent events](#view-webhook-request-history) even though the UI may say the webhook has 40 failures. {{< /history >}} -- GitLab From 011d11da861036f9c6d7c72c5def107bb5ecd1ac Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Wed, 26 Mar 2025 12:38:49 +1300 Subject: [PATCH 10/11] Apply 2 suggestion(s) to 2 file(s) Co-authored-by: Ashraf Khamis Co-authored-by: Andy Schoenen --- ...250317021451_migrate_old_disabled_web_hook_to_new_state.rb | 2 +- doc/user/project/integrations/webhooks.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb b/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb index 0e5e4932ce5619..6679f25a868ba2 100644 --- a/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb +++ b/db/post_migrate/20250317021451_migrate_old_disabled_web_hook_to_new_state.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class MigrateOldDisabledWebHookToNewState < Gitlab::Database::Migration[2.2] - BATCH_SIZE = 10_000 + BATCH_SIZE = 1000 TABLE = 'web_hooks' SCOPE = ->(table) { table.where('recent_failures > 3').where(disabled_until: nil) diff --git a/doc/user/project/integrations/webhooks.md b/doc/user/project/integrations/webhooks.md index 9b34fd3f005010..f1afbfdc9aeac7 100644 --- a/doc/user/project/integrations/webhooks.md +++ b/doc/user/project/integrations/webhooks.md @@ -527,6 +527,10 @@ After this period has elapsed, these webhooks are automatically re-enabled. Webhooks are permanently disabled if they fail 40 consecutive times. Unlike [temporarily disabled webhooks](#temporarily-disabled-webhooks), these webhooks are not automatically re-enabled. +Webhooks that were permanently disabled in GitLab 17.10 and earlier underwent a data migration. +These webhooks might display four failures in [**Recent events**](#view-webhook-request-history) +even though the UI might state they have 40 failures. + #### Re-enable disabled webhooks {{< history >}} -- GitLab From c24b033a8056c50252a95f3be970642202410606 Mon Sep 17 00:00:00 2001 From: Luke Duncalfe Date: Thu, 27 Mar 2025 13:58:29 +1300 Subject: [PATCH 11/11] Add reviewer feedback --- app/models/concerns/web_hooks/auto_disabling.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/models/concerns/web_hooks/auto_disabling.rb b/app/models/concerns/web_hooks/auto_disabling.rb index a9a4b3d58e12ea..9ff6b40c266c42 100644 --- a/app/models/concerns/web_hooks/auto_disabling.rb +++ b/app/models/concerns/web_hooks/auto_disabling.rb @@ -85,8 +85,7 @@ def temporarily_disabled? return false unless auto_disabling_enabled? disabled_until.present? && disabled_until >= Time.current && - recent_failures > TEMPORARILY_DISABLED_FAILURE_THRESHOLD && - recent_failures <= PERMANENTLY_DISABLED_FAILURE_THRESHOLD + recent_failures.between?(TEMPORARILY_DISABLED_FAILURE_THRESHOLD + 1, PERMANENTLY_DISABLED_FAILURE_THRESHOLD) end def permanently_disabled? -- GitLab