o
    eizE                     @   sR  d Z ddlZddlZddlmZ ddlmZmZmZ ddl	Z	ddl
mZ ddlmZ ddlmZ ddlmZ d	d
lmZmZ d	dlmZmZmZmZmZmZ d	dlmZ ddlmZ ddl m!Z! ddl"m#Z# ddl$m%Z% ddl&m'Z' ddl(m)Z) erddl*m+Z+ G dd de,Z-G dd de,Z.ee/dZ0de	j1de2fddZ3G dd de)Z4dS ) a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)Iterable)AnyOptionalTYPE_CHECKING)TensorVariable)Source)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                   @      e Zd ZdS )ArgMappingExceptionN__name__
__module____qualname__ r"   r"   k/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/_dynamo/variables/optimizer.pyr   8       r   c                   @   r   )GuardInstallExceptionNr   r"   r"   r"   r#   r%   <   r$   r%   
perf_hintsxreturnc                 C   s\   ddl m} | jr,|| jjd}tjj| d u}|r*|j	d us"J |p)|j	
| S |S dS )Nr   )get_managerFT)torch._inductor.cudagraph_treesr)   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)r'   r)   manageris_static_addressr"   r"   r#   _is_static_for_cudagraphsC   s   
r6   c                       sZ  e Zd ZdddhejZ			d&dejjdee	e
ef  deee  dee	ejef  de
ddf fdd	Zd
ddedee de	eef ddf
 fddZd
ddedef fddZd'ddZd'ddZde
de
deee
 e	ee
f f fddZd(ddZd'ddZd
ddejdefddZd
ddee de
d ee
 d!e
ddfd"d#Zd'd$d%Z  Z S ))OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesNvaluekwargsr(   c                    s>   t  j|fi | || _|pi | _|pi | _|pt | _d S N)super__init__r;   r8   r9   setr:   )selfr;   r8   r:   r9   r<   	__class__r"   r#   r?   ^   s
   

zOptimizerVariable.__init__txr   nameargsr   c           
   
      s   |dkrlt | jdst ||||S zD| | |   | j|i |\}}| jj|i |}| | | 	||||| dt
| j }||| j | | t|W S  ttfyk }	 zW Y d}	~	nd}	~	ww t ||||S )zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)hasattrr;   r>   call_methodgraph_break_if_pending_mutationmove_step_if_cpuget_python_argsrG   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r%   )
rA   rD   rE   rF   r<   py_args	py_kwargsret_valmangled_name_rB   r"   r#   rJ   m   s&   


zOptimizerVariable.call_methodc                    s|   |dv r| j s	J t| |t| j |dS |dkr7ddlm} | jjD ]}|d D ]}||dd q(q"| | t 	||S )	NrG   )sourceparam_groupsr
   mark_static_addressparamsTguard)
rY   r   r   
decoratorsr\   r;   rZ   _set_capturabler>   var_getattr)rA   rD   rE   r\   groupprB   r"   r#   rb      s   

zOptimizerVariable.var_getattrc                 C   sp   | j jD ]1}|d D ]*}|jj}|jt|d }|r4||r4ddlm	} |dd| d| dg d q
qd S )	Nr]   r
   )unimplementedz(optimizer: pending mutation on parameterz
variable: z, parameter: zSPending mutations on a parameter (e.g. due to using closure) require a graph break.)gb_typecontextexplanationhints)
r;   rZ   outputside_effectsid_to_variablegetrP   has_pending_mutationexcre   )rA   rD   grd   rk   variablere   r"   r"   r#   rK      s   z1OptimizerVariable.graph_break_if_pending_mutationc           	         s   ddl m} dtttf dtf fdd} jjD ]
}||r#d|d< q jo,t	 jd	}|
t| jj|}|jD ]}ttd}td|j|< q<d S )
Nr   LazyVariableTrackerrc   r(   c                    sJ   d}d}|  dg D ]}||jp|jM }|| jjvM }q
d| v o$|o$|S )NTr]   
capturable)rm   r+   is_xpur;   state)rc   all_uninitializedall_gpurd   rA   r"   r#   safe_to_set_capturable   s   zAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturableTrt   rZ   ) rs   dictstrr   boolr;   rZ   rY   r   realize_allr   builditemsr   _HashableTrackerr   rS   )	rA   rD   rs   rz   rc   rY   param_groups_vtparam_group_vtkeyr"   ry   r#   ra      s    
z!OptimizerVariable._set_capturablec                    sF   dt dt ffdd  fdd|D } fdd| D }||fS )	z9Get python values equivalent to the variable tracker argsargr(   c                    st   t | tr|  r|  S t | tr| jsg S t | tr8t | jtr8t | jj	t
r8| jj	jdkr8 jj| jj S t)NrZ   )
isinstancer   is_python_constantas_python_constantr   r   r   rY   r   baser   memberr;   rZ   r-   r   )r   ry   r"   r#   map_arg   s   
z2OptimizerVariable.get_python_args.<locals>.map_argc                    s   g | ]} |qS r"   r"   ).0r   r   r"   r#   
<listcomp>   s    z5OptimizerVariable.get_python_args.<locals>.<listcomp>c                    s   i | ]	\}}| |qS r"   r"   )r   kvr   r"   r#   
<dictcomp>   s    z5OptimizerVariable.get_python_args.<locals>.<dictcomp>)r   r   )rA   rF   r<   new_args
new_kwargsr"   )r   rA   r#   rM      s   z!OptimizerVariable.get_python_argsc                 C   s@   | j j D ]\}}d|v r|d jr|d |j|d< qd S )Nstep)r;   rv   r   is_cputor,   )rA   rd   rv   r"   r"   r#   rL      s
   z"OptimizerVariable.move_step_if_cpuc                    s  ddl m  ddlm} i | _i | _dtdd f fdd}ttj	|| j
j | jo.t| jd	}|t|| j
j|}| joCt| jd
}t|| j
j|}|  |d usWJ |jj| t| j
j|jD ]\}}	t|d dkr|d D ]6}
|
jd urd }t| j
j D ]\}}||
u r|} nq|r|t|| j
j|
 t|t||  nqv|	|t d}d}g }t|d |!|D ]1\}}|j}|| j|< t"|d}|jd ur|| j|j< t#|jsd}|$| qt%|&t'j( q|st)*t+j,rdd |D }t)-d| qft| j
j. D ]>\}}t|t||}|jj| t|. D ]#\}}t/|tj	rQ|| jvrQ|| jvrQt|t||| j|< q/qd S )Nr
   r[   r   rr   r'   r(   c                    s    | dd d S )NTr^   r"   )r'   r[   r"   r#   mark_static  s   zEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_staticrZ   rv   r]   r   TgradFc                 S   s   g | ]}|j qS r"   rE   )r   srcr"   r"   r#   r   G  s    zDOptimizerVariable.map_sources_and_install_guards.<locals>.<listcomp>)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)0r`   r\   lazyrs   r8   r9   r   r	   r.   Tensorr;   rv   rY   r   r   r   r   rZ   realizerj   guard_on_key_orderaddzipr   lenr   	enumeratekeysr   r   getitem_constr   rS   unpack_var_sequencer   r6   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGwarningvaluesr   )rA   rD   rs   r   params_groups_sourcer   state_sourcestate_vtrc   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsrd   p_vtparam_sourcegrad_sourcenon_static_grad_namesidxr;   p_state_source	inner_idxr   r"   r[   r#   rN      s   










z0OptimizerVariable.map_sources_and_install_guardstensor_valuec                 C   s   ddl m} || jv r"||dd | j| }| j|j|j n&|| jv r-| j| }n||dd |	t
|}t|}| j|j|j t|||S )z%Wrap state tensor in a TensorVariabler
   r[   Tr^   )r`   r\   r9   r:   r   rj   module_key_namerE   r8   rQ   r   r   r   r   )rA   rD   r   r\   rY   global_namer"   r"   r#   wrap_tensorb  s   


zOptimizerVariable.wrap_tensorrT   rU   c              	   C   s   t ||D ]G\}}t|trLt|tsJ dt|D ]0\}}	|jj| t|	tj	r7|j
| ||	 q|jo?t|j|}
|j
t||	|
 qqdS )z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr   rj   rk   mutationr.   r   r   r   r   rY   r   r   r   )rA   rD   rF   r<   rT   rU   r   py_argr   valrY   r"   r"   r#   rO   ~  s   	
z"OptimizerVariable.update_list_argsc                    sB   | j  | j|jjdtjjdd f fdd}|j| d S )Ngmr(   c                    s"   d fdd}t | d S )Nr(   c                     sJ   D ] }  j | d   j| d  jrj  jr"j  qd S r=   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclassesr   )r   names_to_deletetcr"   r#   clear_static_tensor_refs  s   

z\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refsr(   N)weakreffinalize)r   r   r   r   r;   )r   r#   init_finalizer  s   	z:OptimizerVariable.create_finalizer.<locals>.init_finalizer)r:   r;   rj   tracing_contextr.   fxGraphModuleadd_graph_finalizer)rA   rD   r   r"   r   r#   rR     s
   z"OptimizerVariable.create_finalizer)NNN)rD   r   r(   Nr   )!r   r    r!   r   _nonvar_fieldsr.   optim	Optimizerr   r|   r   r   r@   r}   r   r   r?   r   r   rJ   rb   rK   ra   tuplerM   rL   rN   r   r   r   rO   rR   __classcell__r"   r"   rB   r#   r7   V   s    


$

 


g

r7   )5__doc__r   r   collections.abcr   typingr   r   r   r.   torch._dynamo.variables.tensorr   torch._guardsr   torch._loggingr   torch.utils._pytreer	   guardsr   r   rY   r   r   r   r   r   r   r0   r   r   r   constantr   dictsr   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r%   r   r   r   r~   r6   r7   r"   r"   r"   r#   <module>   s4     
