o ei@sTddlZddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z ddl mZmZddlmZmZmZddlZddlmmZddlmZddlmZddlmZdd lm Z dd l!m"Z"d d l#m$Z$d d l%m&Z&m'Z'm(Z(d dl)m*Z*m+Z+d dl,m-Z-m.Z.m/Z/m0Z0d dl1m2Z2gdZ3e4e5Z6ee5dZ7de8de9fddZ:de;e8dfdee$ddGdddZ?e$ddGd d!d!Z@gd"ZAe$d#dGd$d%d%ZBdFd'e=fd(d)ZCe$d#dGd*d+d+eBZDe$ddd,d-ZEe$d#dGd.d/d/eFZGe$d#dGd0d1d1ZHe$ddGd2d3d3eHZIe$d#dGd4d5d5eHZJe$ddGd6d7d7eHZKe'D] ZLd8d9ZMeMeLqKd:d;ZNe(D]ZOeNeOq]dd?ZQd@dAge.eRdeRdejSjTejSjUDZVdBdCeVeH<dDdCeVe;<dEdCeVeW<eQeVe<<eVeWeVe+<eVe<eVe*<dS)GN) OrderedDict)CallableIterator)fields is_dataclass)AnyOptionalTypeVar) _fx_map_arg)is_opaque_value_type)getArtifactLogger) tree_map_)CapturedTraceback) compatibility)Graph magic_methodsreflectable_magic_methods)immutable_dictimmutable_list)Argument base_typesNodeTarget)check_for_mutable_operation) TracerBaseGraphAppendingTracer TraceErrorProxy MetaProxy AttributeParameterProxyScopeScopeContextManager annotationobjreturncCs.ddl}ddl}t||j|j|j|j|jfS)z Returns True if obj is an arbitrary callable (function, lambda, method, etc.) that requires special tracing to handle. These cannot be symbolically traced using the standard Proxy mechanism. rN) functoolstypes isinstance FunctionType MethodTypeBuiltinFunctionTypeBuiltinMethodTypepartial)r%r'r(r/X/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/fx/proxy.py_is_arbitrary_callable1sr1args.kwargscs:dtd}d|d|ffdd }t||t||S)z Recursively searches args and kwargs for any arbitrary callable. Returns the first arbitrary callable found, or None if none exist. N_Tr%r&csdur|St|r ||SN)r1)r%foundr/r0checkQs z'_find_arbitrary_callable..check)r r )r2r3r4r8r/r6r0_find_arbitrary_callableFs   r9Fis_backward_compatiblecs*eZdZdZdedeffdd ZZS)r"a7Scope object that records the module path and the module type of a module. Scope is used to track the information of the module that contains a Node in a Graph of GraphModule. For example:: class Sub(torch.nn.Module): def forward(self, x): # This will be a call_method Node in GraphModule, # scope for this would be (module_path="sub", module_type=Sub) return x.transpose(1, 2) class M(torch.nn.Module): def __init__(self) -> None: self.sub = Sub() def forward(self, x): # This will be a call_method Node as well, # scope for this would be (module_path="", None) x = x.transpose(1, 2) x = self.sub(x) return x module_path module_typecst||_||_dSr5)super__init__r<r=)selfr<r= __class__r/r0r?xs  zScope.__init__)__name__ __module__ __qualname____doc__strrr? __classcell__r/r/rAr0r"^sr"cs:eZdZdZdedeffdd ZddZdd ZZS) r#zA context manager to track the Scope of Node during symbolic tracing. When entering a forward function of a Module, we'll update the scope information of the current module, and when we exit, we'll restore the previous scope information. scope current_scopecs0tt||_|j|_|j|_||_dSr5)r>r?copy _prev_scoper<r=_scope)r@rIrJrAr/r0r?s   zScopeContextManager.__init__cC|jSr5)rMr@r/r/r0 __enter__zScopeContextManager.__enter__cGs|jj|j_|jj|j_dSr5)rLr<rMr=)r@r2r/r/r0__exit__s  zScopeContextManager.__exit__) rCrDrErFr"r?rPrRrHr/r/rAr0r#~sr#) nn_module_stacktorch_fnsource_fn_stack original_aten recompute ac_graph_idhas_backward_hook from_nodequantization_tag_numeric_debug_handlecustompartitioner_tagTc@seZdZUeed<dZeed<dZeed<dZeed<dZ eed<dZ eed<dZ e ed <e ed <ee ee effed <ee ee effed <ed d  d0de dedeedfdee efdee deedefddZdejdejfddZed ddeddfdd Zed d   d1de dedeedfdee efdee deed!eegdffd"d#Zd$d%Zed dd&edefd'd(Z ed dd)ddefd*d+Z!ed dd)dde"fd,d-Z#ed dd)ddefd.d/Z$dS)2rgraphFrecord_stack_traces!_record_forward_stack_traces_onlycheck_mutable_operations trace_assertsproxy_buffer_attributesforwardtraced_func_namerI module_stacknode_name_to_scopeTr:Nkindtargetr2.r3name type_exprr&cCs|dkr |jr t||||j||||||}|jj|jjf|j|j<t rt }| d} | r@| |_ t j| vr@d|jd<tD]} | |vrRt|| |j| <qBt|j} | duritd| |j| |jd<t } | durd|jd<d | jvr| j d |jd <d| jvr| j d|_ n |jrt|j|jd <|jr|j st} | r|| } | rd | |_ td ||S) a' Inserts a graph node given target, args, kwargs, and name. This method can be overridden to do extra checking, validation, or modification of values used in node creation. For example, one might want to disallow in-place operations from being recorded. call_function stack_traceTis_gradient_accNzAssigning new_seq_nr %s to %sseq_nris_functional_regeneratedr]rSzcreate_node %s) rbrr_ create_noderIr<r=rhrk fx_tracebackhas_preserved_node_metaget_current_metagetrnGRADIENT_ACC_SPECIAL_STACKmeta_COPY_META_FIELDSrK _get_seq_nrannotation_logdebugget_current_replay_nodergr`rextractsummary_filter_traceback_framesjoinformatstriplog)r@rirjr2r3rkrlnode current_metarnfield new_seq_nr replay_nodeuser_stack_summaryr/r/r0rssR              zTracerBase.create_nodercsg}|jr dd|D}n d}t|D]\}}|jdkr&||d}|}nq|dkr-g}ddlmfdd|D}tj|S)NcSs&g|]}|jdks|jdr|qS)reztorch/__init__.py)rkfilenameendswith.0framer/r/r0 s  z7TracerBase._filter_traceback_frames..reruninteresting_filescsg|] }|jvr|qSr/)rrrr/r0r4s  )ra enumeraterk%torch.fx.experimental.symbolic_shapesr traceback StackSummary from_list)r@r user_frames first_forwardirr/rr0rs&      z#TracerBase._filter_traceback_framesrrcCs t||Sr5)r)r@rr/r/r0proxy< zTracerBase.proxyproxy_factory_fnc Cs~||}||} t|tstdt|t| ts&tdt| ||||| ||} |s9|| } | S|| } | S)ao Create a Node from the given arguments, then return the Node wrapped in a Proxy object. If kind = 'placeholder', then we're creating a Node that represents the parameter of a function. If we need to encode a default parameter, we use the ``args`` tuple. ``args`` is otherwise empty for ``placeholder`` Nodes. z Expected args_ to be tuple, got z!Expected kwargs_ to be dict, got ) create_argr)tupleAssertionErrortypedictrsr) r@rirjr2r3rkrlrargs_kwargs_rrr/r/r0 create_proxy@s     zTracerBase.create_proxycsFtgd}rjrtfdd|Drns s!dSS)zf Find the Python stack frame executing the user code during symbolic tracing. ) ztorch/fx/proxy.pyztorch/fx/_symbolic_trace.pyz%torch/fx/experimental/proxy_tensor.pyz torch/_ops.pyztorch/_tensor.pyztorch/utils/_python_dispatch.pyztorch/_prims_common/wrappers.pyztorch/_refs/__init__.pyz%torch/_refs/nn/functional/__init__.pyztorch/utils/_stats.pyc3s |] }jj| VqdSr5)f_code co_filenamer)rfilerr/r0 ~s z.TracerBase._find_user_frame..N)inspect currentframef_backall)r@pt_filesr/rr0_find_user_framefs zTracerBase._find_user_frameacstt}|dur|SttrjStdr"SttrHtdr;fddD}t|StfddDStt rVfddDStt r`t Stt rwt  j j jSttrt j j jSttjjtjjfrSttrStrfdd tD}d jd |StgttjRsψdusψd urшStd t)z A method that lowers the objects seen as arguments during symbolic evaluation into Argument types that can be stored in IR. Can be override to support more trace-specific types. N__fx_create_arg___fieldscg|]}|qSr/rrelemrOr/r0rz)TracerBase.create_arg..crr/rrrOr/r0rrcrr/rrrOr/r0rrcs"i|] }|jt|jqSr/)rkrgetattr)rrrr@r/r0 sz)TracerBase.create_arg..rmr/.zargument of type: )_create_arg_bypassrwrr)rrhasattrrrlistr_create_arg_dictslicerstartstopsteprangetorch_ops OpOverloadHigherOrderOperatorr rrrsrBrenumEnumNotImplementedError)r@rhandlerr2r3r/rr0rsP                    &zTracerBase.create_argr%cCtd)aFCalled when a proxy object is being converted to a boolean, such as when used in control flow. Normally we don't know what to do because we don't know the value of the proxy, but a custom tracer can attach more information to the graph node using create_node and can choose to return a value. zFsymbolically traced variables cannot be used as inputs to control flowrr@r%r/r/r0to_boolzTracerBase.to_boolcCr)aACalled when a proxy object is being iterated over, such as when used in control flow. Normally we don't know what to do because we don't know the value of the proxy, but a custom tracer can attach more information to the graph node using create_node and can choose to return an iterator. aHProxy object cannot be iterated. This can be attempted when the Proxy is used in a loop or as a *args or **kwargs function argument. See the torch.fx docs on pytorch.org for a more detailed explanation of what types of control flow can be traced, and check out the Proxy docstring for help troubleshooting Proxy iteration errorsrrr/r/r0iterrzTracerBase.itercCs t|dS)zCalled when a proxy object is has the keys() method called. This is what happens when ** is called on a proxy. This should return an iterator it ** is suppose to work in your custom tracer. keysr rr/r/r0rs zTracerBase.keysNN)NNN)%rCrDrEr__annotations__r`boolrarbrcrdrfrGr"rrrrrrrrrrrsrrrrrrrrrrrrr/r/r/r0rs          L &   %"J rrr node_namecCst}d}|dddkrtd||dd}ntjjr-td||d}ntj d }tjjsRt }|durRd|j vrRtd ||j d}|S) a: Returns the seq_nr node meta for the current proxy node that we're creating. The seq_nr number in node meta is related to but not the same as the "sequence number" in autograd. We use the seq_nr to correlate forward and backward nodes in the traced FX graphs (e.g. in copy_fwd_metadata_to_bw_nodes). The corresponding forward and backward FX graph nodes should have the same seq_nr. The ordering of the seq_nr in the graph does not indicate when the node is executed. For example, the nodes in the invoke_subgraph HOP's subgraph may be called multiple times by different HOP nodes, but these nodes only have a single seq_nr, which may be smaller, the same, or larger than the calling HOP. `node_name` is the name of the node that we're creating. It is used for logging only. N in_grad_fnrz+%s: seq_nr from current_meta grad_fn_seq_nrgrad_fn_seq_nrrz#%s: seq_nr from current_meta seq_nrrprz%s: seq_nr from replay_node) rtrvrwr|r}rfxr_is_preserving_node_seq_nrautograd_get_sequence_nrr~ry)rrrrr/r/r0r{s         r{cs"eZdZdeffdd ZZS)rr_cs0t||_tdd|_t|_i|_dS)Nrr) r>r?r_r"rI collectionsrrgrh)r@r_rAr/r0r?2s    zGraphAppendingTracer.__init__)rCrDrErr?rHr/r/rAr0r0srcCs|stddS)NzAssertion failed)rxr/r/r0 assert_fn:src@s eZdZdS)rN)rCrDrEr/r/r/r0r@src@seZdZdZeddd$deddfdd Zd efd d Zd%ddZ d e fddZ d e fddZ ddZ d&ddZd edfddZddZd efddZeddddZd d!Zed'd"d#ZdS)(ra ``Proxy`` objects are ``Node`` wrappers that flow through the program during symbolic tracing and record all the operations (``torch`` function calls, method calls, operators) that they touch into the growing FX Graph. If you're doing graph transforms, you can wrap your own ``Proxy`` method around a raw ``Node`` so that you can use the overloaded operators to add additional things to a ``Graph``. ``Proxy`` objects cannot be iterated. In other words, the symbolic tracer will throw an error if a ``Proxy`` is used in a loop or as an ``*args``/``**kwargs`` function argument. There are two main ways around this: 1. Factor out the untraceable logic into a top-level function and use ``fx.wrap`` on it. 2. If the control flow is static (i.e. the loop trip count is based on some hyperparameter), the code can be kept in its original position and refactored into something like:: for i in range(self.some_hyperparameter): indexed_item = proxied_value[i] For a more detailed description into the Proxy internals, check out the "Proxy" section in `torch/fx/README.md` Tr:NrtracerOptional[TracerBase]cCs"|dur t|j}||_||_dSr5)rr_rr)r@rrr/r/r0r?cs  zProxy.__init__r&cCd|jjdS)NzProxy()rrkrOr/r/r0__repr__kzProxy.__repr__r cCs t||Sr5r)r@kr/r/r0 __getattr__ns zProxy.__getattr__cCrNr5__dict__rOr/r/r0 __getstate__srQzProxy.__getstate__c Csddl}i}|jD](\}}z|||}Wnty.td||jj||}Ynw|||<q d|vrz Proxy.__iter__..keyUNPACK_SEQUENCEc3s|]}|VqdSr5r/)rrrOr/r0rsz!Proxy.__iter__..)rrrrrdisget_instructionsrsys version_infobisectrf_lastiopnamerargvalrr)r@r calling_frame inst_listrinst_idxinstr/rOr0__iter__s"      zProxy.__iter__cCs|jdtj|fiSNrm)rroperatorabsrOr/r/r0__abs__sz Proxy.__abs__c Cs|jjrt}|durtd|j}|durtdtt|j }t j dkr9ddl m }|||jddd}n|jd }||}|jd kr||d }|jdurVtd ||jd d }|jd kri|jdkpm|jdk} | r|jdkr|jdt|fidS|j|S)NrrrrrcSrNr5rrr/r/r0r r z Proxy.__bool__..r r POP_JUMP_IF_TRUErz%inst.arg is None for POP_JUMP_IF_TRUE LOAD_GLOBALrLOAD_ASSERTION_ERROR RAISE_VARARGSrmT)rrcrrrrrrrrrrrrrrargrrrr) r@rrinstsrcurrfirstlaststarts_with_assertr/r/r0__bool__s6        zProxy.__bool__cCs |j|Sr5)rrrOr/r/r0rs z Proxy.keyscCr)Nz'len' is not supported in symbolic tracing by default. If you want this call to be recorded, please call torch.fx.wrap('len') at module scope RuntimeErrorrOr/r/r0__len__sz Proxy.__len__cs|r|nd}|r |ni}ifdd}t||t||tdkr3tdtd|tt}t|tj j rQ|j f|}| d|j ||Stj|r`| d|j||St|tjjr|t||}|dur|td|jd |d |j d ||||j|jd S) Nr/cst|r d|j<dSdSr5)r)r)rclstracersr/r0 find_tracers z-Proxy.__torch_function__..find_tracerrz!Found multiple different tracers z" while trying to trace operations rz5Unable to symbolically trace the HigherOrderOperator z4 because it received an arbitrary callable argument z(. Use make_fx or dynamo tracing instead.rmrk)r lenr,rrnextrr)r_C ScriptMethodownerrrk overridesis_tensor_method_or_propertyrCrrr9_namer__target_to_str)r/ orig_methodr(r2r3r1r bad_callabler/r.r0__torch_function__sJ          zProxy.__torch_function__r5)r&r )r&rr)rCrDrErFrrr?rGrrrrrrrrrrrr*rr- classmethodr>r/r/r/r0rEs$  # rcsNeZdZdZ d deddffdd Zdefd d Zed fd d Z Z S)rzW A Proxy subclass that propagates metadata (meta['val']) during graph tracing. Nrrrcst||||_dSr5)r>r? fake_mode)r@rrr@rAr/r0r?s zMetaProxy.__init__r&cCr)Nz MetaProxy(rrrOr/r/r0rrzMetaProxy.__repr__cs|r|nd}|r |ni}d}|D] }t|tr|}nq|dur$tdt||||}|j|dd|Di||jjd<Wdn1sKwYt|j|j|jS)Nr/z5No MetaProxy found in arguments, but one is expected.cSs&g|]}t|tr|jjdn|qS)val)r)rrry)rrr/r/r0r3s&z0MetaProxy.__torch_function__..rA) r)rrr>r>r@rryr)r/r<r(r2r3 meta_proxyr$rrAr/r0r> s*    zMetaProxy.__torch_function__r) rCrDrErFrr?rGrr?r>rHr/r/rAr0rsrc@s<eZdZedddedefddZeddZd d Z d S) r Tr:rootattrcCs||_||_|j|_d|_dSr5)rCrDr_node)r@rCrDr/r/r0r?;s zAttribute.__init__cCs.|jdur|jdt|j|jfij|_|jSr)rErrrrCrDrrOr/r/r0rBs zAttribute.nodecOs|jd|j|jf||S)Nr)rrrDrCrr/r/r0rLszAttribute.__call__N) rCrDrErrrGr?propertyrrr/r/r/r0r 9s   r cspeZdZdZdedeffdd ZdefddZe d d Z d d Z d dZ e ddZ ddZddZZS)r!z A special proxy which lets "shape", "size", "dim", and a few other attribute accesses pass through to the underlying module parameter object, so that conditional tests on these attributes will not throw exception during tracing rrcs>t||t|tjjstdt|||_||_ dS)NzExpected Parameter, got ) r>r?r)rnn Parameterrrparamrk)r@rrrkrIrAr/r0r?Zs  zParameterProxy.__init__r&cCsd|jdS)NzParameterProxy(rr2rOr/r/r0raszParameterProxy.__repr__cC|jjSr5)rIshaperOr/r/r0rKdzParameterProxy.shapecC |jSr5)rIsizerOr/r/r0rNh zParameterProxy.sizecCrMr5)rIdimrOr/r/r0rPkrOzParameterProxy.dimcCrJr5)rIndimrOr/r/r0rQnrLzParameterProxy.ndimcCrMr5)rInumelrOr/r/r0rRrrOzParameterProxy.numelcCrMr5)rInelementrOr/r/r0rSurOzParameterProxy.nelement)rCrDrErFrrr?rGrrFrKrNrPrQrRrSrHr/r/rAr0r!Rs  r!cs4fdd}|_ddd}tt||dS)Ncs$|dj}tt}|d|||S)Nrrm)rrrr)r2r3rrjmethodr/r0impl|s  z_scope..impl___)rCrsetattrr)rUrVas_magicr/rTr0rM{s rMcs:ddd}fdd}||_||_tt||dS)N__rrXrWcs tt}|jd|||fiSr)rrrr)r@rhsrjorig_method_namer/r0rVs z!_define_reflectable..impl)rrCrErYr)r^ method_namerVr/r]r0_define_reflectables  r`cCstd|)NzJKeys for dictionaries used as an argument cannot contain a Node. Got key: r+)r$r/r/r0_no_nodes_errors racCsFi}|D]\}}t|ts||}t|t||||<q|Sr5)rr)rGrmap_argra)r@rrrrr/r/r0rs   rcCsi|]}|ddqS)cSs|Sr5r/r@rr/r/r0r sz.r/)rtr/r/r0rsrcCs|jSr5)rrdr/r/r0r r r cstfdd|DS)Nc3s|]}|VqdSr5rrrOr/r0rsz..)rrdr/rOr0r rcsfdd|DS)Ncrr/rrrOr/r0rrz..r/rdr/rOr0r s)rr)XrrKrrrloggingrrrrcollections.abcrr dataclassesrrtypingrrr rtorch.fx.tracebackrrttorch._Cr rbtorch._library.opaque_objectr torch._loggingr torch.utils._pytreer torch.utils._tracebackr_compatibilityrr_rrrimmutable_collectionsrrrrrrroperator_schemasr__all__ getLoggerrCrr|objectrr1rrrGr9r"r#rzrr{rr ValueErrorrrrr r!rUrMr`r^rarrrrrrrr/r/r/r0s            N7 L'&