
    YhR              ,          d dl mZmZmZ d dlZd dlmZ ddlmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddgZ G d de          Zd	d
e de de de de
 de dz   e_        dee         dee         dee         dee         dee         dee         dee         dee         dededee ef         dee ef         dee ef         de de dededed ef&d!Z!dee         dee         dee         dee         dee         dee         dee         dee         dededee ef         dee ef         dee ef         de de dededed ef&d"Z"dee         dee         dee         dee         dee         dee         dee         dee         dedede de dee ef         de de dededed ed#df(d$Z# ee!%          	 	 	 	 	 	 	 	 d*dee         dee         dee         dee         dee         dee         d'ee         deded(ee         dee         dee         ded edede de dee ef         de de def*d)            Z$dS )+    )castOptionalUnionN)Tensor   )_capturable_doc_default_to_fused_or_foreach_device_dtype_check_for_fused_differentiable_doc_disable_dynamo_if_unsupported_foreach_doc
_fused_doc!_get_capturable_supported_devices_get_scalar_dtype
_get_value_maximize_doc_params_doc_stack_if_compiling
_to_scalar_use_grad_for_differentiable_view_as_real
DeviceDictDeviceDtypeDict	OptimizerParamsTAdamadamc                        e Zd Z	 	 	 	 	 ddddddddded	eeef         d
eeeef         eeef         f         dededede	e         dededede	e         def fdZ
 fdZd Zedd            Z xZS )r   MbP?g?g+?:0yE>r   FN)foreachmaximize
capturabledifferentiablefuseddecoupled_weight_decayparamslrbetasepsweight_decayamsgradr"   r#   r$   r%   r&   r'   c                   t          |t                    r:|r|	st          d          |                                dk    rt          d          d|k    st          d|           d|k    st          d|           d|d         cxk    rdk     sn t          d	|d                    d|d         cxk    rdk     sn t          d
|d                    d|k    st          d|           t          |d         t                    rt          |d         t                    sEt          |d         t                    rt          |d         t                    st          d          t          |d         t                    r@|	s|rt          d          |d                                         dk    rt          d          t          |d         t                    r@|	s|rt          d          |d                                         dk    rt          d          t          ||||||||	|
||          }t                                          ||           |r)|
rt          d          d| _	        |rt          d          d S d S )NElr as a Tensor is not supported for capturable=False and foreach=Truer   Tensor lr must be 1-element        zInvalid learning rate: zInvalid epsilon value: r         ?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-element)r)   r*   r+   r,   r-   r#   r"   r$   r%   r&   r'   z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.)

isinstancer   
ValueErrornumelfloatdictsuper__init__RuntimeError_step_supports_amp_scaling)selfr(   r)   r*   r+   r,   r-   r"   r#   r$   r%   r&   r'   defaults	__class__s                 b/var/www/tools.fuzzalab.pt/emblema-extractor/venv/lib/python3.11/site-packages/torch/optim/adam.pyr9   zAdam.__init__#   s     b&!! 	@ z  [   xxzzQ !>???byy;r;;<<<czz<s<<===eAh$$$$$$$$M58MMNNNeAh$$$$$$$$M58MMNNNl""JLJJKKKa%((	Q-7a%-H-H	Q58V,,	Q1;E!Hf1M1M	Q OPPPeAh'' 	F '  a   Qx~~1$$ !DEEEeAh'' 	F '  a   Qx~~1$$ !DEEE%!)#9
 
 
 	*** 		W P"#NOOO.2D+
  W"#UVVV		W 		WW W    c                 $   t                                          |           | j        D ]d}|                    dd           |                    dd           |                    dd            |                    dd           |                    dd           |                    dd           |                    dd           }|d	         D ]}| j                            |g           }t          |          d
k    rt          j        |d                   stt          |d                   }|d         s|d         r*t          j
        |t          |          |j                  n!t          j
        |t                                |d<   fd S )Nr-   Fr#   r"   r$   r%   r'   r&   r(   r   stepis_fuseddtypedevicerF   )r8   __setstate__param_groups
setdefaultstategetlentorch	is_tensorr6   tensorr   rG   )r<   rL   groupr&   pp_statestep_valr>   s          r?   rI   zAdam.__setstate__r   s   U###& 	 	EY...Z///Y---\5111-u5555u===$$Wd33E8_  *..B//w<<1$$U_WV_-M-M$$WV_55H !.O
 38.O$"3U"C"C"C#$8    #\(:K:M:MNNN FO		 	r@   c                    d}|d         D ]J}	|	j         ?|t          j        |	          z  }|                    |	           |	j         j        rt          d          |                    |	j                    | j        |	         }
t          |
          dk    r|d         rt          |	           |d         s|d         r0t          j	        dt          |d                   |	j        	          n!t          j        d
t                                |
d<   t          j        |	t          j                  |
d<   t          j        |	t          j                  |
d<   |d         r#t          j        |	t          j                  |
d<   |                    |
d                    |                    |
d                    |d         r|                    |
d                    |d         r|
d         j        rt          d          |d         r1t          j        |d                   r|d         st          d          |                    |
d                    L|S )NFr(   zJAdam does not support sparse gradients, please consider SparseAdam insteadr   r&   r$    rC   rE   r1   rH   rB   )memory_formatexp_avg
exp_avg_sqr-   max_exp_avg_sqr%   zB`requires_grad` is not supported for `step` in differentiable moder"   r)   r/   )gradrO   
is_complexappend	is_sparser:   rL   rN   r
   zerosr   rG   rQ   
zeros_likepreserve_formatrequires_gradrP   )r<   rR   params_with_gradgradsexp_avgsexp_avg_sqsmax_exp_avg_sqsstate_stepshas_complexrS   rL   s              r?   _init_groupzAdam._init_group   s    x =	2 =	2Av!u/222 ''***6# &d   QV$$$
1u::??W~ 95a888 !.J
 38.J"3U7^"L"L"L#$8    #\#5F5H5HIII &M (-'7)>( ( (E)$ +0*:)>+ + +E,' Y' 272BU-B3 3 3./ i 0111""5#6777# D#**51A+BCCC)* uV}/J &\   )$d44 ",/
 '_   ""5=111r@   c                    |                                   d}|5t          j                    5   |            }ddd           n# 1 swxY w Y   | j        D ]}g }g }g }g }g }g }	|d         \  }
}|                     |||||||	          }t          ||||||	f|d         ||
||d         |d         |d         |d         |d         |d	         |d
         |d         t          | dd          t          | dd          |d         d |S )zPerform a single optimization step.

        Args:
            closure (Callable, optional): A closure that reevaluates the model
                and returns the loss.
        Nr*   r-   r)   r,   r+   r#   r"   r$   r%   r&   
grad_scale	found_infr'   )r-   rj   beta1beta2r)   r,   r+   r#   r"   r$   r%   r&   rm   rn   r'   ) _cuda_graph_capture_health_checkrO   enable_gradrJ   rk   r   getattr)r<   closurelossrR   rd   re   rf   rg   rh   ri   ro   rp   rj   s                r?   rB   z	Adam.step   s    	--///"$$ ! !wyy! ! ! ! ! ! ! ! ! ! ! ! ! ! ! & )	 )	E-/"$E%'H(*K,.O(*K >LE5**  K   i(';">2%Lz*i( .$%56Gn"4t<<!$T::',-E'F+    0 s   AA
A)r   r    r!   r   FN)__name__
__module____qualname__r   r   r6   r   tupleboolr   r9   rI   rk   r   rB   __classcell__)r>   s   @r?   r   r   "   s        $(COMW #' $ $',MW MW MWMW %- MW U5&=)5+??@	MW
 MW MW MW $MW MW MW MW ~MW !%MW MW MW MW MW MW^    0I I IV "9 9 9 "!9 9 9 9 9r@   af  Implements Adam algorithm.

    .. math::
       \begin{aligned}
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{input}      : \gamma \text{ (lr)}, \beta_1, \beta_2
                \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}          \\
            &\hspace{13mm}      \lambda \text{ (weight decay)},  \: \textit{amsgrad},
                \:\textit{maximize},  \: \epsilon \text{ (epsilon)}                              \\
            &\textbf{initialize} :  m_0 \leftarrow 0 \text{ ( first moment)},
                v_0\leftarrow 0 \text{ (second moment)},\: v_0^{max}\leftarrow 0          \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                                 \\
            &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do}                         \\

            &\hspace{5mm}\textbf{if} \: \textit{maximize}:                                       \\
            &\hspace{10mm}g_t           \leftarrow   -\nabla_{\theta} f_t (\theta_{t-1})         \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}g_t           \leftarrow   \nabla_{\theta} f_t (\theta_{t-1})          \\
            &\hspace{5mm}\textbf{if} \: \lambda \neq 0                                           \\
            &\hspace{10mm} g_t \leftarrow g_t + \lambda  \theta_{t-1}                            \\
            &\hspace{5mm}m_t           \leftarrow   \beta_1 m_{t-1} + (1 - \beta_1) g_t          \\
            &\hspace{5mm}v_t           \leftarrow   \beta_2 v_{t-1} + (1-\beta_2) g^2_t          \\
            &\hspace{5mm}\widehat{m_t} \leftarrow   m_t/\big(1-\beta_1^t \big)                   \\
            &\hspace{5mm}\textbf{if} \: amsgrad                                                  \\
            &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t)                  \\
            &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big)              \\
            &\hspace{5mm}\textbf{else}                                                           \\
            &\hspace{10mm}\widehat{v_t} \leftarrow   v_t/\big(1-\beta_2^t \big)                  \\
            &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/
                \big(\sqrt{\widehat{v_t}} + \epsilon \big)                                       \\
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
            &\bf{return} \:  \theta_t                                                     \\[-1.ex]
            &\rule{110mm}{0.4pt}                                                          \\[-1.ex]
       \end{aligned}

    For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_.
    z
    Args:
        a  
        lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR
            is not yet supported for all our implementations. Please use a float
            LR if you are not also specifying fused=True or capturable=True.
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        decoupled_weight_decay (bool, optional): if True, this optimizer is
            equivalent to AdamW and the algorithm will not accumulate weight
            decay in the momentum nor variance. (default: False)
        amsgrad (bool, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
        z	
        a=  
    .. Note::
        A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`.
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ

    r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   c                X   ||J t           j                                        rFt          |t                    sJ t          |
t                    sJ t          |t                    sJ nt          |          }t          |
t                    r|
j        |
j        f|
i}nd }t          |           D ]w\  }}|s||         n||          }||         }||         }||         }t           j
                                        sF|rDt                      }|j        j        |j        j        k    r|j        j        |v sJ d| d            |dz  }|dk    r|r|                    d||z  z
             nv|r]t          |t                    rH|j        r)|                    |                                |          }n/|                    ||          }n|                    ||          }t          j        |          rot          j        |          }t          j        |          }t          j        |          }|rt          j        ||                   ||<   t          j        |          }|j        }|3|j        }||f}||vr|
                    ||d          ||<   ||         }n|
}|                    |d|z
             |rxt          |t                    rc|j        r-|                    t          j        |          d|z
             n]|                    |                              ||d|z
  	           n.|                    |                              ||d|z
  	           |s|r|}|r@t          |
t                    r+|
j        rd|
|                                z  z
  } nd|
|z  z
  } nd|
|z  z
  } |r@t          |t                    r+|j        rd||                                z  z
  }!nd||z  z
  }!nd||z  z
  }!|| z  }"|"                                }#|!                                }$|r|r||                                         }%n||         }%||                             t          j        |%|                     ||                                         |$|#z  z                      ||#z            }&n0|                                |$|#z  z                      ||#z            }&|r)|                    |                                |&           n|                    ||&           nt=          |          }d|
|z  z
  } d||z  z
  }!|| z  }"|!d
z  }$|rTt          j        ||         |||                    ||                                         |$z                      |          }&n*|                                |$z                      |          }&|                    ||&|" 	           |r7t          j        | |                   rt          j        ||                   ||<   yd S )NIIf capturable=True, params and state_steps must be on supported devices: .r   r   alphaT)rG   rF   non_blocking)weight)value      ?)out) rO   jitis_scriptingr3   r6   r   r   rG   rF   	enumeratecompileris_compilingr   typemul_rc   addcmul_cloneaddr]   view_as_realtolerp_squarenegsqrtcopy_maximumadd_addcdiv_r   view_as_complex)'r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   
beta1_dictiparamr\   rY   rZ   step_tcapturable_supported_devicesrG   rF   keydevice_beta1rB   bias_correction1bias_correction2	step_sizestep_size_negbias_correction2_sqrtr[   denoms'                                          r?   _single_tensor_adamr   Y  s   , )"3"3"3y  "e$$$$$%'''''%''''''^^ %   27,1Le0T


f%% VK VK5'6uQxxeAhY1+ ^
Q ~**,, 	 	+L+N+N(!V]%777L%)EEEE{\x{{{ FEE 	!1% ?

1rL001111 " ?jv&F&F ?#1 C#}}U[[]]LII#xx\xBB88E8>>DE"" 	.%d++D(11G+J77J L%*%78J%K%K"&u--E!KE 5/C*$$"'((!T #+ # #
3 2<CLL L 	dA,---  	Ij77 	I" M   d!3!3AI FFFF&&//d!e)/LLLLOOE""++D$a%i+HHH E	= E	=D  3*UF";"; 3& 7'(5DJJLL+@'@$$'(5$;$$#$ud{?   3*UF";"; 3& 7'(5DJJLL+@'@$$'(5$;$$#$ud{? --I%MMOOM$4$9$9$;$;! ,! 8%4Q%7%=%=%?%?NN%4Q%7N"((~z)R)RSSS $A&++--1F1VW$s]*++ 
 OO%%)>)NO$s]*++   /w}}6666w....f%%D 5$; 5$;--I$4c$9! Noa0*/RSBTUUUU )+00225JJPPQTUU#**-BBHHMMNN7E)N<<<  	Ku'q	22 	K!&!6q7I!J!JOAmVK VKr@   c                  
+ t          |           dk    rd S t          t                    r8|st          d                                          dk    rt          d          t          
t                    r8|st          d          
                                dk    rt          d          t          t                    r8|st          d                                          dk    rt          d          t          j                                        sI|rGt          d	
          +t          +fdt          | |          D                       sJ d+ d            ||J |r
J d            t                    t          j        | |||||g          }t          
t                    r!t          
j                  dk    r	
j        
ind }|                                D ]\  \  }}}}}}}t%          t&          t                   |          }t%          t&          t                   |          }t%          t&          t                   |          }t%          t&          t                   |          }t%          t&          t                   |          } |d         j        }!||!|vr
                    |!d          ||!<   |r||!         n
}"|	rH|r4t%          t&          t                   |          }#t+          |||||#           nt+          ||||           |rt          j        |          }t          j                                        s9| d         j        r,t          j        | t          j        dd          d           nt          j        | d           |dk    rO|rt          j        |d|z  z
             n1|rt          j        |||           nt          j        |||          }t          j        ||d|"z
             t          j        |           t          t          j                  rt          j        |dz
            }$d}%n|}$dz
  }%t          j        ||$||%           ~~$|r`t          j        
|           }&t          j        |           }'t          j         |&d           t          j         |'d           t          j!        |'           t          j"        |&           t          j#        |&           t          j$        |'           |&}(|'})|rJt%          t&          t                   |          }#t          j%        |#|           t          j&        |#          }*nt          j&        |          }*t          j"        |*|)           t          j        |*|           t          j"        |*|(           t          j'        |||*           
fd| D             }&fd| D             }'tQ          fd|&D                       }(d |'D             })|rJt%          t&          t                   |          }#t          j%        |#|           t          j&        |#          }*nt          j&        |          }*t          j"        |*|)           t          j        |*|           t          j'        |||*|(           d S )Nr   r/   r   r0   zHbeta1 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF)supports_xlac              3   n   K   | ]/\  }}|j         j        |j         j        k    o|j         j        v V  0d S rv   )rG   r   ).0rS   rB   r   s      r?   	<genexpr>z%_multi_tensor_adam.<locals>.<genexpr>T  s]       
 
 4 HMT[-- >!==
 
 
 
 
 
r@   r~   r   z#_foreach ops don't support autogradcpuTrG   r   r2   )rG   r   c                 :    g | ]}d t          |          z  z
  S r   r   )r   rB   ro   s     r?   
<listcomp>z&_multi_tensor_adam.<locals>.<listcomp>  8          26EZ----     r@   c                 :    g | ]}d t          |          z  z
  S r   r   )r   rB   rp   s     r?   r   z&_multi_tensor_adam.<locals>.<listcomp>  r   r@   c                      g | ]
}|z  d z  S )rW   )r   bcr)   s     r?   r   z&_multi_tensor_adam.<locals>.<listcomp>  s!    ,W,W,Wb2g^,W,W,Wr@   c                     g | ]}|d z  S )r   rW   )r   r   s     r?   r   z&_multi_tensor_adam.<locals>.<listcomp>  s    $H$H$HRW$H$H$Hr@   ))rN   r3   r   r:   r5   r4   rO   r   r   r   allzipr   r   "_group_tensors_by_device_and_dtypestrrG   valuesr   listr   r   _foreach_negis_cpu_foreach_add_rQ   _foreach_mul__foreach_add_foreach_lerp__foreach_mul_foreach_addcmul__foreach_pow_foreach_sub__foreach_neg__foreach_div__foreach_reciprocal__foreach_sqrt__foreach_maximum__foreach_sqrt_foreach_addcdiv_r   ),r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   grouped_tensorsr   device_params_device_grads_device_exp_avgs_device_exp_avg_sqs_device_max_exp_avg_sqs_device_state_steps__device_paramsdevice_gradsdevice_exp_avgsdevice_exp_avg_sqsdevice_state_stepsrG   r   device_max_exp_avg_sqsscaled_device_gradsr   r   r   r   r   exp_avg_sq_sqrtr   s,             ```                              @r?   _multi_tensor_adamr     s   , 6{{a"f < 	W   88::??:;;;%   ? 	Z   ;;==A=>>>%   ? 	Z   ;;==A=>>> >&&(( 

Z 

'H(
 (
 (
$  
 
 
 
 v{33
 
 
 
 
 	
 	

 xXtwww	
 	
 	
 )"3"3"3DDDDDD	BB  B	+L O eV$$	),U\):):e)C)C 
u  ""$$_ _ 		 	T&\>::DL-88tF|-=>>!$v,0CDD!$v,0CDDq!(!fJ&>&>!&d!K!KJv-7Bz&))U  	 )-d6l<S)T)T&! #&*    !<BT    	< -l;;L ~**,, 	71CA1F1M 	7"ELU$C$C$C3      2A6661% 
#M1rL7H3HIIII  'm<XXXXX#(#5$m<$ $ $L 	_lA<LMMM.666 eU\** 	"'"4\1u9"M"MEE".IE 3\5	
 	
 	

   B	$1%9KLL$1%9KLL 0!444 0!444 0111  0"555&'7888 !1222
 )I$4! J)-d6l<S)T)T&'(>@RSSS #("56L"M"M"'"56H"I"I1FGGG555;;; #M?OTTTT       :L            :L      ,,W,W,W,WFV,W,W,WXXI$H$H7G$H$H$H! J)-d6l<S)T)T&'(>@RSSS #("56L"M"M"'"56H"I"I1FGGG555#	   u_ _r@   returnc                   | sd S |rt          d          |	|j        |ini }|	|j        |ini }t          |t                    r!t	          |j                  dk    r	|j        |ind }t          j        | |||||g          }|                                D ]\  \  }}\  \  }}}}}}}t          t          t                   |          }t          t          t                   |          } t          t          t                   |          }!t          t          t                   |          }"t          t          t                   |          }#d\  }$}%|+|
                    ||                    |d                    }$|+|
                    ||                    |d                    }%|&||vr"|                    |d          ||<   ||         }t          j        |#d           |st          j        nt          j        }& |&|| |!|"||#|||
|||||$|%           |%&t          j        |#|%gt#          |#          z             d S )	Nz9Adam with fused=True does not support differentiable=Truer   )NNT)r   r   r   )	r-   r)   ro   rp   r,   r+   r#   rm   rn   )r:   rG   r3   r   r   r   r   itemsr   r   rK   r   rO   r   _fused_adam__fused_adamw_r   rN   )'r(   re   rf   rg   rh   ri   rm   rn   r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   r'   grad_scale_dictfound_inf_dictlr_dictr   rG   r   r   r   r   r   r   r   r   r   r   r   r   device_grad_scaledevice_found_inffuncs'                                          r?   _fused_adamr     s   ,   XVWWW ,6+A	J''r  *3)>	9%%B  &b&11Wc")nn6M6MBSW   B	+L O 
			 	 3 3 
	 
	
"	T&\>::DL-88tF|-=>>!$v,0CDD!$v,0CDD.8++! / : :
f4@@! !  -88	V$??    6#8#8 ee6eEEGFOB.222)?Xu!!UEX"%(&	
 	
 	
 	
" '"%5$6=O9P9P$P  c3 3r@   )single_tensor_fnFr"   r&   c                   |	2|0t          | |d          \  }}|rt          |t                    r|sd}|	d}	|d}t          j                                        s(t          d |D                       st          d          |r-t          j        	                                rt          d          |	r-t          j        	                                rt          d          |	r&t          j        	                                st          }n/|r&t          j        	                                st          }nt          } || |||||f|||||||||||
||d dS )	znFunctional API that performs Adam algorithm computation.

    See :class:`~torch.optim.Adam` for details.
    NF)	use_fusedc              3   J   K   | ]}t          |t          j                  V  d S rv   )r3   rO   r   )r   ts     r?   r   zadam.<locals>.<genexpr>  s?       5 5()
1el##5 5 5 5 5 5r@   zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers)r-   rj   ro   rp   r)   r,   r+   r#   r$   r%   rm   rn   r'   )r	   r3   r   rO   r   r   r   r:   r   r   r   r   r   )r(   re   rf   rg   rh   ri   r"   r$   r%   r&   rm   rn   rj   r'   r-   ro   rp   r)   r,   r+   r#   r   r   s                          r?   r   r   q  s   F }1Ne
 
 

7  	z"f-- 	j 	G} >&&(( 
 5 5-85 5 5 2 2 
 ^
 
 	
  U59))++ USTTT S'')) SQRRR #UY++-- #	 #//11 #!"D !%5'     r@   )NFFNNNFF)%typingr   r   r   rO   r   	optimizerr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   __all__r   __doc__r   r{   r6   r   r   r   r   rW   r@   r?   <module>r      s   ( ( ( ( ( ( ( ( ( (                                                  0 6
m m m m m9 m m mb$J	 " 
# $ 
% & 
' ( 
) * 
+  KB NBKLBK<BK 6lBK f	BK
 &\BK fBK  BK BK BK BK BK BK 	eVmBK BK  
!BK" #BK$ %BK& 'BK( !)BK BK BK BKJpLp<p 6lp f	p
 &\p fp  p p p p p p 	eVmp p  
!p" #p$ %p& 'p( !)p p p pf]L]<] 6l] f	]
 &\] f]  ] ] ] ] ] ] 	eVm] ]  
!]" #]$ %]& ']( !)]* 
+] ] ] ]@  1DEEE #  #'"&#(!W WLW<W 6lW f	W
 &\W fW d^W W W D>W  W W W  !!W$ %W& 'W( )W* 	eVm+W, -W. 
/W0 1W W W FEW W Wr@   