o ei@@sUddlmZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z m Z ddlmZmZddlZddlZddlmZddlmZddlmmZddlmZddlmZdd lmZd d l m!Z!m"Z"m#Z#d d l$m%Z%d d l&m'Z'm(Z(m)Z)e rddl*m+Z+m,Z,m-Z-ddl.m/Z/ddl0m1Z1edZ2edZ3e4e5Z6dVddZ7edWddZ8e#dXddZ9 dYdZd'd(Z:d[d*d+Z;e#d\d,d-Ze#d]d1d2Z?e#d^d3d4Z@d_d7d8ZAejBjCZCeCjDeCjEeCjFeCjGeCjHeCjIeCjJeCjKeCjLeCjMeCjNeCjOeCjPeCjQjReCjQjSeCjTeCjUeCjVeCjWeCjXeCjYeCjZhZ[ee[Z[e#d\d9d:Z\d`d>d?Z]dadAdBZ^da_dCe`dD<dbdGdHZadcdPdQZb RdddedTdUZcdS)f) annotationsN)contextmanager)partial)Any TYPE_CHECKING) ParamSpecTypeVar)SymInt)get_decompositions) bind_symbols) aot_function aot_modulemake_boxed_compiler)strip_overloads)default_partition draw_graph#min_cut_rematerialization_partition)Callable GeneratorSequence)Node) IntLikeType_P_Rfx_gfx.GraphModulereturncCs4|jjdtjjjdD]}tjjj|_q ||S)N call_functionoptarget) graph find_nodestorchopsaten_to_copytor! recompile)rnoder+d/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/_functorch/compilers.py _canonicalize/s   r-Generator[None, None, None]c cs6tjd}z dVWtj|dStj|w)NF)r$_C_jit_set_autocast_mode)old_jit_autocast_flagr+r+r,_disable_jit_autocast8s  r2inps Sequence[Any]torch.jit.ScriptModulecCs:tt||jjdtjjjdD]}t|j dkr.t|j dkr.d|j vr.tjjj |_ q|jj D]}i}|j D]\}}t|tjrI|j}|||<q<||_ q3|j|tj|}tj|jtj|}tj|}tdd|Ds||Wd|SWd|S1swY|S)a Compiles the :attr:`fx_g` with Torchscript compiler. .. warning:: This API is experimental and likely to change. Args: fx_g(fx.GraphModule): The input Fx graph module to be compiled. Returns: Torch scripted model. rrr dtypecss|] }t|tjjVqdSN) isinstancer$ _subclasses FakeTensor).0tr+r+r, nszts_compile..N)r2rr"r#r$r%r&r'lenargskwargsr(r!nodesitemsr8devicetypelintr)jitscriptr/_jit_pass_remove_mutationfreezeevaloptimize_for_inferenceany)rr3r* new_kwargskvfr+r+r, ts_compileCs<  &          rQT_rnamestr clear_metaboolcCst|jt|||d|S)N)rU)printcoder)rrRrSrUr+r+r,_draw_graph_compiless rY5Callable[[fx.GraphModule, list[Any]], fx.GraphModule]cCsttt|dS)NrS)rrrYr[r+r+r,draw_graph_compile{sr\cCs|S)z Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler and can be used to check accuracy. .. warning:: This API is experimental and likely to change. r+rrRr+r+r,nops r^cs4eZdZddddfd d Zdfdd ZZS)DebugInterpreterNT initial_envenable_io_processingr?rradict[Node, Any] | NonerbrVrcs(t|jg|R|_tj|||dS)Nr`)r modulesymbol_mappingsuperrun)selfrarbr? __class__r+r,rgs zDebugInterpreter.runnrc sdfdd dfd d dfdd d fdd }t|}d|jvrpt|jd\}}t|\}}t|t|krNtt|dt|ttt|||D]\}} t | t j sdqX||| fddqX|S)!Nnirrintcs@t|ts|St|jjj}|jst d|t |S)Nzexpected r to be a number, got ) r8r sympyexpandr*exprxreplacere is_numberAssertionErrorrm)rlr)rhr+r, subst_symints z/DebugInterpreter.run_node..subst_symintnistuple[IntLikeType, ...]tuple[int, ...]cstfdd|DS)Nc3s|]}|VqdSr7r+)r;rlrur+r,r=szHDebugInterpreter.run_node..subst_symint_tuple..)tuple)rvryr+r,subst_symint_tuplesz5DebugInterpreter.run_node..subst_symint_tuplea torch.TensorbrVcsT|dkr(t|jD]}||||kr'||dkr'dSq dS)Nrr FT)numelrangendimstridesize)r|r~idxryr+r,check_significant_stridessz.check_significant_stridesnvrvdescCallable[[], str]Nonec st|s tdt||j|jkr"t|d|jd|j||krDt|d|d|d|||}|sct|d|d|d|dS)Nz"expected desc to be callable, got z:  != z aka )callablersrDr6rr)rrr same_strides)rr{r+r,checks * *z(DebugInterpreter.run_node..checkvalrcsddjS)Nzoutput z where )rer+)irhr+r,sz+DebugInterpreter.run_node..)rlrrrm)rvrwrrx)r|r}r~r}rrV)rr}rr}rrrr) rfrun_nodemetapytree tree_flattenr>rsziprr8r$Tensor) rhrkrrtn_vals_n_specr_vals_r_specrrri)rrrhrur{r,rs   zDebugInterpreter.run_node)r?rrarcrbrVrr)rkrrr)__name__ __module__ __qualname__rgr __classcell__r+r+rir,r_s r_DCallable[[DebugInterpreter, Any, dict[Node, Any] | None, bool], Any]cCs t|jS)z Returns a (slow) interpreter over the FX graph module that also checks various debugging properties (e.g., that tracing strides matched real strides.) )r_rgr]r+r+r, debug_nops rcCs(t|tj|}tj|}|Sr7)rr$rFrGrIrJ)rrRrPr+r+r,simple_ts_compiles rrPCallable[..., Any]cCs t|tSr7)r r)rPr+r+r,nnc_jits rcCst|j|Sr7)rWrXr]r+r+r, print_compile s rfnCallable[_P, _R] | nn.Moduler@cKsFttttd}||t|tjjrt|fi|St |fi|S)a~ Wrapper function over :func:`aot_function` and :func:`aot_module` to perform memory efficient fusion. It uses the :func:`min_cut_rematerialization_partition` partitioner to perform efficient recomputation. It uses NVFuser to compile the generated forward and backward graphs. .. warning:: This API is experimental and likely to change. Args: fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` that takes one or more arguments. Must return one or more Tensors. **kwargs: Any other overrides you want to make to the settings Returns: Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior of the original :attr:`fn`, but whose forward and backward graphs have gone through recomputation optimizations, and the graphs have been compiled with nvfuser.  fw_compiler bw_compiler partition_fndecompositions) rQrdefault_decompositionsupdater8r$nnModulerr )rr@configr+r+r,memory_efficient_fusions rSequence[torch.Tensor]cCsH|dtddd|Ddddlm}||t||S)NfooaQ ############################################################## # To minimize FX graph, copy and paste the below and run it # ############################################################## import torch import torch.fx as fx from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess inps = cSsg|]}|j|jfqSr+)shaper6)r;rr+r+r, Gsz!debug_compile..a? inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] from foo import FxModule mod = FxModule().cuda() with torch.jit.fuser("fuser2"): # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) r)FxModule) to_folderrWrrcudarQ)rr3rr+r+r, debug_compile9s    rrm graph_indexinput_data_pathlist[torch.Tensor]c Csg}t|dU}t|}g}|D]B}t|dkr"|}|t}n,|\}}}} } | tjtjtjtj tjtj tt hvrFtj dd|| | d}ntj || | d}||qWd|S1s_wY|S)zZ Return a random input for the given inputs meta generated from _save_fx_default. rbr r)r6rCN)openpickleloadr>randomr$rmint32int64rVuint8floatrandintrandappend) rinputsrP inputs_metarrDinput_r_strider6rCr+r+r, get_inputs[s6      r current_name folder_namedump_example_inputgmtorch.fx.GraphModuleexample_inputs nn.Modulec slddlm}dfdd dfdd dfdd }dfdd }d fdd }||||||tdS)!aO The forward, backward, and joint computation graph will be stored in {folder_name}/{current_name}/{current_name}_forward_{graph_index}, {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. The input shape of the graphs will be stored in the .input files. These files can be loaded with pickle, and is a list of format (type, shape, stride, dtype, device). In the case of type = int or float, it is just (type,). For joint graph input, it is a nested list [[],[]] where the two inner lists have the same format. If dump_example_input is True, example_inputs will be stored in .pt file. Since each function might produce multiple graphs, the graph_index is used to distinguish difference graphs r)aot_module_simplifiedr?rr list[Any]csg}t|dkr!t|dtr!||d7}||d7}|S|D](}t|tus1t|tur:|t|fq#|t||j||j |j fq#|S)Nrr ) r>r8rzrDrmrrrrr6rC)r? input_metaarg)get_input_metar+r,rsz(_save_fx_default..get_input_meta gm_to_saver type_namerTrcsBt|jjdkrttjd|tdSt|}|j t j j | |}tjddd|ddd|dt tddd|dtdd|dtdd}t||Wdn1sxwYrt |ddd|dtdd|dtd dSdS) Nrz!No nodes in graph {%s}_{%s}_{%s}./T)exist_okrRz.inputwbz.pt)r>r"rAlogloggingWARNINGrcopydeepcopy set_codegenr$fxCodeGenr)osmakedirsrrrdumpsave)rr?rrrrP)rrrrr+r,graph_saver_helpers< 22z,_save_fx_default..graph_saver_helperrrrcs||d|S)Nforwardr+rrrr+r,graph_saver_forwards z-_save_fx_default..graph_saver_forwardcs||dtd7a|S)Nbackwardr )rrrr+r,graph_saver_backwards z._save_fx_default..graph_saver_backward joint_args%tuple[fx.GraphModule, fx.GraphModule]cs||dt||S)Njoint)r)rrrr+r,graph_saver_joints  z+_save_fx_default..graph_saver_jointrN)r?rrr)rrr?rrrTrr)rrrrrr)rrrrrr)functorch.compilerr) rrrrrrrrrr+)rrrrrr,_save_fx_defaultzs #rF Callable[[bool, nn.Module], Any]cCsdatt|||S)as Dump the forward, backward, and joint computation graph. Example Usage: save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) optimize_ctx = torchdynamo.optimize( save_fx_func ) with torch.enable_grad(): with optimize_ctx: result = forward_and_backward_pass(model, example_inputs) r)rrr)rrrr+r+r,graph_dumper_aotsr)rrrr)rr.)rrr3r4rr5)T) rrrRrrSrTrUrVrr)rSrTrrZ)rrrRrrr)rrrRrrr)rrrRrrr5)rPrrr)rrr@rrr)rrr3rrr5)rrTrr) rrTrrTrrVrrrrrr)F)rrTrrTrrVrr)d __future__rrrrrr contextlibr functoolsrtypingrrtyping_extensionsrrrnr$torch.fxrtorch.nnrtorch.utils._pytreeutils_pytreerr torch._decompr %torch.fx.experimental.symbolic_shapesr aot_autogradr rr compile_utilsr partitionersrrrcollections.abcrrr torch.fx.noder torch.typesrrr getLoggerrrr-r2rQrYr\r^ Interpreterr_rrrr%r&detach gelu_backwardleaky_relu_backwardsigmoid_backwardthreshold_backwardhardtanh_backwardhardsigmoid_backwardhardswish_backward tanh_backward silu_backward elu_backwardcudnn_batch_normcudnn_batch_norm_backward masked_fillScalarrelu leaky_reluhardtanh hardswish hardsigmoid conj_physical is_same_sizerrrrr__annotations__rrrr+r+r+r,s              0   H      '   k