o eib3@shUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZmZddlmZmZddlZddlZddlZddlmZddlmm Z!ddl"m#Z#m$Z$ddl%m&Z&dd l'm(Z(dd l)m*Z+dd l,m-Z-m.Z.dd l/m0Z0dd l1m2Z2ddl3m4Z4m5Z5ddl6m7Z7ddl8m9Z9ddl:m;Z;ddlZ>ddl?m@Z@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHddlImJZJddlKmLZLddlMmNZNdd lOm*Z*ddlPmQZQddlRmSZSmTZTmUZUmVZVddlWmXZXddlYmZZZm[Z[m\Z\ddl]m^Z^ddl_m`Z`ddlambZbmcZcmdZdmeZemfZfdd lgmhZhmiZimjZjerAddlkZlddlmZme*jnZod!epd"<eqerZsd#epd$<ejtjuZuejtjvZveGd%d&d&ZweGd'd(d(ZxeGd)d*d*Zyd!d.d/Zzd"d2d3Z{d"d4d5Z|d#d7d8Z}Gd9d:d:Z~d!d;d<Ze~Zd$d?d@Zd%dAdBZd&dEdFZ  Gd'd(dRdSZd!dTdUZd!dVdWZd!dXdYZd!dZd[Zd!d\d]Zd!d^d_Zd!d`daZd!dbdcZd)dgdhZd*dmdnZd+dqdrZ s t d,d-d|d}Zd.ddZd/ddZd0ddZd1ddZd2ddZd3ddZd4ddZd4ddZd5ddZ d6d7ddZddd8ddZdddd9ddZedZd:ddZd#ddZd;ddZejddd„Zd?ddτZd@dd҄ZdAddԄZdAddքZdAdd؄Zd!ddڄZdBdd݄Z d6dCddZdDddZdEddZdFddZdGddZdHddZdIddZddlmZdJddZdKddZ dLdMddZdNddZdOd d ZdPddZ dQdddRddZ    G dSdTdd ZdS(U) annotationsN) defaultdictdeque)Callable) dataclassreplace)Any TYPE_CHECKING)countersis_node_meta_valid)(create_structured_trace_for_min_cut_info)is_with_effects)config)CustomKnapsackSolverCustomRuntimeEstimator)FakeScriptObject) is_builtin) LazyStringtrace_structured) trace_log)extract_tensor_metadata) BackwardState) is_sym_node py_sym_types) magic_methodsmethod_to_operator)find_symbol_binding_fx_nodes free_symbolsis_symbol_binding_fx_node size_hintstatically_known_falsestatically_known_true) graph_drawer) OrderedSet)CheckpointPolicy)GraphInfoProvider) dp_knapsackdp_knapsack_sliding_hirschberggreedy_knapsack ilp_knapsack)KnapsackEvaluator) AOTOutputSavedForBackwardsAOTOutput#SavedForBackwardsNoVcCheckAOTOutput)_is_functional_graph)get_aot_graph_name)_is_bwd_seed_offset_is_fwd_seed_offset _is_primal _is_tangentget_cuda_generator_meta_val) fx_graph_cseget_aten_targetraise_getitemsboolAOT_PARTITIONER_DEBUGzlogging.Loggerlogc@sleZdZUdZded<ded<ded<ded<ded<dd d ZdddZdddZdddZdddZ dS)OpTypesz8Class for keeping track of different operator categorieszOrderedSet[Callable[..., Any]] fusible_opscompute_intensive_ops random_opsview_opsrecomputable_opsnodefx.Nodereturnr9cCt||jvSN)r7r=selfrBrIg/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/_functorch/partitioners.py is_fusiblegzOpTypes.is_fusiblecCrErF)r7r>rGrIrIrJis_compute_intensivejrLzOpTypes.is_compute_intensivecCrErF)r7r?rGrIrIrJ is_randommrLzOpTypes.is_randomcCrErF)r7r@rGrIrIrJis_viewprLzOpTypes.is_viewcCrErF)r7rArGrIrIrJis_recomputablesrLzOpTypes.is_recomputableNrBrCrDr9) __name__ __module__ __qualname____doc____annotations__rKrMrNrOrPrIrIrIrJr<]s     r<c@s~eZdZUded<ded<ded<ded<ded<ded <ded <ejdd d ZdddZdddZdddZ dddZ dS)NodeInfo list[fx.Node]inputsOrderedSet[fx.Node]_required_fw_nodesrequired_bw_nodestangents_closureunclaimed_nodesdict[fx.Node, int]fw_orderstatic_lifetime_input_nodesrDcs tddjDfdddS)Ncss|]}|VqdSrFrI.0nrIrIrJ sz-NodeInfo.required_fw_nodes..cs j|SrF)r`rdrHrIrJ z,NodeInfo.required_fw_nodes..key)sortedr[rgrIrgrJrequired_fw_nodesszNodeInfo.required_fw_nodesrdrCr9cC ||jvSrF)r[rHrdrIrIrJis_required_fw zNodeInfo.is_required_fwcCrnrF)r\rorIrIrJis_required_bwrqzNodeInfo.is_required_bwcCrnrF)r^rorIrIrJ is_unclaimedrqzNodeInfo.is_unclaimedintcCs$||jvr td|d|j|S)NNode z not in fw nodes!)r[AssertionErrorr`rorIrIrJ get_fw_orders  zNodeInfo.get_fw_orderN)rDrX)rdrCrDr9)rdrCrDrt) rRrSrTrV functoolscached_propertyrmrprrrsrwrIrIrIrJrWws     rWc@s6eZdZUded<ded<ded<ded<ded<dS) MinCutOptionsr9ban_if_used_far_apartban_if_long_fusible_chainsban_if_materialized_backwardban_if_not_in_allowlistban_if_reductionN)rRrSrTrVrIrIrIrJrzs  rzrBrCrDcCs|jddtjtjfvS)N recompute)metagetr$MUST_RECOMPUTEPREFER_RECOMPUTErBrIrIrJmust_recomputes rfx_gfx.GraphModulecCs |jjD] }t|r dSqdS)NTF)graphnodesrrrBrIrIrJhas_recomputable_opss rcCs<|jjD]}t|rt|jdrtjj|jjvrdSqdS)NtagsTF) rrrhasattrtargettorchTagnondeterministic_seededrrrIrIrJhas_recomputable_rng_opss  rrtcCsJt|jdtjtjfrdSt|jdtjs#tdt|jddS)Nvalr%z.expected node.meta['val'] to be SymFloat, got ) isinstancerrSymIntSymBoolSymFloatrvtyperrIrIrJ sym_node_sizesrc@seZdZdddZdS)InvalidNodeBaserDstrcCsdS)Nz Invalid NoderIrgrIrIrJ__repr__szInvalidNodeBase.__repr__N)rDr)rRrSrTrrIrIrIrJrsrcCst|jdddkS)N namespace_c10d_functional)getattrrrrIrIrJis_not_collectiver getitem_nodefx.Node | NonecCs~|jtjkrdS|jd}|jd}t|tjr|jdkrdSd|jvr&dS|jd}||vr1dS||}t|tjr=|SdS)zGiven a getitem node, check if it extracts from a higher-order op that has kwargs mapping the key back to an original input. Returns the original input node if found, None otherwise. Nrr% call_functionkwargs) roperatorgetitemargsrfxNodeopr)r ho_resultrkroriginal_inputrIrIrJ_get_ho_op_original_inputs      rcCsD|jtjjjjtjjjjfvrdS|jd}t|t j sdSt |S)zCheck if node is a view/reshape of a higher-order op output that aliases an input. Returns the original input node from the higher-order op's kwargs if the pattern matches, None otherwise. Nr) rropsatenviewdefaultreshaperrrrr)rBsourcerIrIrJ_is_copy_node_bw_onlys   renvdict[fx.Node, Any]cCs`t|}|dur||vrt||ts||St|}|dur.||vr.t||ts.||SdS)aTry to find a valid input replacement for an invalid forward output. This handles cases where a forward output depends on backward nodes but semantically aliases an input. For example, a view of a getitem from a triton kernel that mutates a buffer in backward, or a direct getitem from such a higher-order op. The original input may be a primal or a valid intermediate node already present in the forward graph. N)rrrr)rBrrrIrIrJ_find_input_for_invalid_outputs   rF joint_graphfx.GraphrYrXoutputs outputs_descslist[AOTOutput]subgraph str | Noneignore_must_be_in_fw_bwcsjt}i|D]}||j}|j|_||<q|jD]{}|sBt|r1|dkr1||vr1t|<qt|rB|dkrB||vrBt|<q|vrGq|j dkrQt|<q|j dkrt j |j i|j } fdd| D} t| rrt|<q||fdd|<q|j d kr||fd d|<q|j d kr qg} t||D]\} } t| tjr| vrtd | d t| trd} | jtjjjjurt| rt| j dkrt| j dtjr| j dvrt| j dts| j d} | durt| } | dur | | qtd | d| | q| | q|t | }||jd<|!|"|S)a Given a graph, extracts out a subgraph that takes the specified nodes as inputs and returns the specified outputs. This includes specifying non-placeholder nodes as inputs. The general strategy is to initialize all inputs with proxies as we encounter them, and trace through the graph, only keeping values which take in valid proxies. Then, all dead code is eliminated. backwardforward placeholderrcs&g|]}t|tjrt|tqSrI)rrrrrcxrrIrJ Ps   z6_extract_graph_with_inputs_outputs..c|SrFrIrrrIrJrhYz4_extract_graph_with_inputs_outputs..get_attrcrrFrIrrrIrJrh\routputruz couldn't be found in envNr%rz was invalid, but is outputdesc)#rGraphrnamerr_must_be_in_backward InvalidNode_must_be_in_forwardrpytreearg_tree_leavesrrany node_copyziprr RuntimeErrorrrrrrcopy_rlenrappendrvrtupleeliminate_dead_codelint)rrYrrrr new_graphrBnew_nodeall_args output_valuesrx_desc replacementoutrIrrJ"_extract_graph_with_inputs_outputss              rcCs2tjot|jtjjrt|j p|jtjj j kSrF) ris_non_builtin_to_includerrr_ops OpOverloadrr higher_order triton_kernel_wrapper_functionalrrIrIrJrsrcCs|jdko t|jdtS)Nrr)rrrrrrrIrIrJ_is_backward_statesrcC|jdddkS)Npartitioner_tag is_backwardrrrrIrIrJ_has_tag_is_backwardrrcCr)Nr is_forwardrrrIrIrJ_has_tag_is_forwardrrcCr)Nrmust_be_in_forwardrrrIrIrJ_has_tag_must_be_in_forwardrrcCr)Nrmust_be_in_backwardrrrIrIrJ_has_tag_must_be_in_backwardrrcCs>t|rdSt|jtjjo|jjj}t| ot | o|SNT) rrrrrr_schema is_mutablerrrBrrIrIrJrs rcCs2t|rdSt|jtjjo|jjj}t|o|Sr) rrrrrrrrrrrIrIrJrs  r joint_modulenum_fwd_outputsEtuple[list[fx.Node], list[fx.Node], list[AOTOutput], list[AOTOutput]]cCstjdd|jjddD}ttt|jjddjddgt|}|d|}||d}|d|}||d}||||fS)Ncs|]}|jVqdSrFrrcrBrIrIrJrez+_extract_fwd_bwd_outputs..rrr) rrr find_nodesnextiterrrr)rrrr fwd_outputs bwd_outputsfwd_outputs_descsbwd_outputs_descsrIrIrJ_extract_fwd_bwd_outputss     r  saved_valuesrrNonecCs(|D]}|j|kr||dSqdSrF)rremove)rr saved_valuerIrIrJ_remove_by_names   rfwd_module_outputs#list[fx.Node] | tuple[fx.Node, ...]cCs@t|}tt|dddD]}t||s|d}|Sq|S)Nr%)rranger)ridxirIrIrJfind_first_sym_nodes r@-q=rtorch.fx.Graph torch.fx.Nodemaxfloatminpositionc Cs^||,|jtjjjj|fd}tjjj|jd|jd<t|jd|jd<Wdn1s4wY||2|jtjjj j|dgdfd}tjjj |jddgd|jd<t|jd|jd<Wdn1sswY||0|jtjj j j|tj fd}tjj j |jdtj |jd<t|jd|jd<Wdn1swY||.|jtjjj j||fd}tjjj |jd||jd<t|jd|jd<Wdn1swY||,|jtjjjj|fd} tjjj|jd| jd<t| jd| jd<Wdn 1s%wY|| .|jtjjjj| |fd} tjjj| jd|| jd<t| jd| jd<Wdn 1sawY|| 9|jtjj j j| tjfd|d|jd} tjj j | jdtj| jd<t| jd| jd<Wd| S1swY| S) Nrr tensor_metarTfp8_scale_pos__rr)inserting_afterrrrrabsrrramaxprimsconvert_element_typefloat64 clamp_min reciprocalmulTensorfloat32r) rrBrr r!abs_node amax_node amax_64_nodeclamp_min_nodereciprocal_nodemul_node scale_noderIrIrJcalculate_quantization_scalings                                     r8r7 quant_type torch.dtyper, clamp_maxc Csp||0|jtjjjj|tjfd}tjjj|jdtj|jd<t |jd|jd<Wdn1s8wY||1|jtjj j j ||fd}tjj j |jd|jd|jd<t |jd|jd<Wdn1svwY||.|jtjj j j||fd} tjj j |jd|| jd<t | jd| jd<Wdn1swY|| .|jtjj jj| |fd} tjj j| jd|| jd<t | jd| jd<Wdn1swY|| 7|jtjjjj| |fd|d|jd} tjjj| jd|| jd<t | jd| jd<Wd| S1s1wY| S)Nrrr"fp8_quant_pos_r$r%)r&rrrr)r*rr0rrrr.r/r,r;r) rrBr7r9r,r;r!target_node_32scaled_target_nodeclamp_min_scaled_nodeclamp_max_scaled_nodequant_activation_noderIrIrJperform_quantization-s                            rBtensor torch.TensorcCs|}|}||dS)z Calculate the size of a PyTorch tensor in megabytes (MB). Args: tensor (torch.Tensor): Input tensor Returns: float: Memory size in MB i)numel element_size)rC num_elementsrFrIrIrJcalculate_tensor_sizess  rHlist[torch.dtype]cCs.tjjjddd}dd|dD}|S)N!activation_quantization_aten_passallowed_dtypesztorch.bfloat16cSs g|] }tt|ddqS).r)rrsplit)rcdtyperIrIrJrsz&get_allowed_dtypes..;)r _inductorrpost_grad_fusion_optionsrrM)rKrIrIrJget_allowed_dtypessrRcCst}t|r|jdj|vrdStjjjddd}t |jd}tjjjddds2||kStjjjdddrJt ||kpIt ||k St ||kS)NrFrJ size_in_mbdskip_dynamo_guardsquantize_dynamic_shape) rRr rrNrrPrrQrrHr!r )rBrKsize_thresholdrSrIrIrJshould_quantizes4  rXcCs*tjjjddd}tt|ddS)NrJr9ztorch.float8_e5m2rLr)rrPrrQrrrM)r9rIrIrJget_quant_types rYrNtuple[float, float]cCst|}|j|jfS)z Calculate the range of values for a given torch.dtype. Args: dtype (torch.dtype): The input dtype. Returns: tuple: A tuple containing the minimum and maximum values. )rfinfor r)rNinforIrIrJcalculate_ranges  r]c s|jddd}|jd}t}t|\}}tg}g}t|D]\}} | jddrtj j j dddrYt || |d |} t || | ||||} t| sS|| nI|| nC|| 6|jtjjjj| |fd |d | jd } tjjj| jd || jd <t| jd | jd<Wdn1swY| |<q!fddt|D} t| } ||}|r| d| || | d} |dt| tddd7<dS)Nrrrsaved_for_quantizationFrJ use_scalingTrr<r$r%rr"csg|] \}}||qSrI)r)rcrrBposition_to_quantrIrJrsz*quantize_activation_fw..inductor%activation_quantization_fwd_aten_passr%)rrrYr]dict enumeraterrrrPrrQr8rBrrr&rrr)r*rrrr update_argrr )rrr r9r,r;tensor_scale_nodessym_scale_nodesr!rBr7 quant_nodeoutput_updated_argsr scale_nodesrIr`rJquantize_activation_fwsb           rlc sdd|jD}d}|D]^}|jddrk|jd|jd}tjjjdddr||d|j d d t fd d |D}Wdn1sSwY||.|j tj jjj||fd }tj jj|jd||jd<t|jd|jd<Wdn1swY||1|j tj jjj||fd }tj jj|jd|jd|jd<t|jd|jd<Wdn1swY||.|j tj jjj||fd }tj jj|jd||jd<t|jd|jd<Wdn 1swYnB||4|j tj jjj||fdt|j d}tj jj|jd||jd<t|jd|jd<Wdn 1sKwYt|jD]}||kri||kri|||qWq tddd7<dS)NcSsg|] }|jdkr|qS)rrrrIrIrJrz*quantize_activation_bw..r^F dequant_typerJr_ fp8_scale_ fp8_quant_c3s|] }|jkr|VqdSrFr)rc bwd_input scale_namerIrJre s z)quantize_activation_bw..rrr"dequant_r%rb%activation_quantization_bwd_aten_passr%)rrrpoprrPrrQr&rrrrrr)r*rrrdivr/rlistuserskeysreplace_input_withr ) r bw_inputsactivation_noderBrnr7divided_target_node_32 dequant_nodeuserrIrtrJquantize_activation_bws                            r fwd_module bwd_modulebwd_module_inputsdict[str, fx.Node]c stdddfdddtjtdddfdddtdddfd ddjjd d d jd }|D]P}d |jvr|tdd|j}j|jj |jd}Wdn1sdwY|j d}|j |j d|j d<||j d<| |j |q:tjjjdddrtjjdd }|d} t|D] } t| s| } nqjjd d d jd } | D].}d|jvrj| jj |jd} Wdn1swY| j |j | } qtjtdddfddddS)NartifactcS dddS)N,before_activation_quantization_fwd_aten_passstringrencodingrIrIrIrIrJrhTz5perform_fp8_activation_quantization..cjddddSNFT print_outputinclude_strideinclude_deviceprint_readablerIrrIrJrhX metadata_fn payload_fncSr)N+after_activation_quantization_fwd_aten_passrrrIrIrIrIrJrharcrrrrIrrIrJrhercSr)N,before_activation_quantization_bwd_aten_passrrrIrIrIrIrJrhlrcrrrrIrrIrJrhprrrrrpz^fp8_quant_pos_\d+_rqrrrnTr^rJr_rrrocSr)N+after_activation_quantization_bwd_aten_passrrrIrIrIrIrJrhrcrrrrIrrIrJrhr)rrlrrrrresubr&rrupdatereplace_all_uses_with erase_noderrPrrQrrzreversedr4r) rrrquant_fwd_module_outputsfwd_nodersquant_bwd_inputrnquant_bwd_module_inputs bwd_input_locbw_inputscaled_fwd_module_outputsscale_bwd_inputrI)rrrJ#perform_fp8_activation_quantizationMsv                 rraOrderedSet[fx.Node] | Nonec Cs(tjdddur dS|rdd|Dng}dd|D}tjjjdddr/dd|D}|jjd d d jd }d d|jjd d D}d}|D]<} | j |vrt | r| j |vrct d| j qKd| j d<| j dj| j d<d|| j j d<| j dj|| j j d<d}qK|rt|||dSdS)NrJcSg|]}|jqSrIrrrrIrIrJrz2enable_activation_quantization..cSi|]}|j|qSrIrrrrIrIrJ z2enable_activation_quantization..exclude_primalsFcSsi|] }d|jvr|j|qS)primalsrrrrIrIrJrsrrrcSrrIrrrrIrIrJrrz*Skipping quantization of static input %s: Tr^rrn)inductor_configrQrrrPrrrrrrXr;debugrrNr) rrrrastatic_input_namessaved_values_namesrrshould_perform_fp8_quantrBrIrIrJenable_activation_quantizationsL   r)rasaved_sym_nodes%tuple[fx.GraphModule, fx.GraphModule]c sLt||d\}}}}|jjdd} gtt| } gtt| } gtt| } gtt| } gtt| }t |j|| | ||d}t j }|jddD];}|j s`t|jt||jqN|rytdd|j Dryt|jt||jqNt|rt|j|stdqNt}g}g}|D]}t|}|r||||q||qt|j}t|| D],}d|jvrqt|jd|}t|d d d D]}||vrq|||q||O}q||||g}g}g}D]%}t|jdt r ||q|jd d r||q||q||t!|t"D]!\}}|krQ|jd d sQtd|ddt!q1t |j| | ||||fddt#t!t!|t!|Dd}t |j||| | |||d}t$j%&||}t$j%&||}t'|||||fS)Nrrrrcss0|]}|jtjjjjuot|jdkVqdSrN)rrrr wait_tensorrrr{rbrIrIrJres   z+_extract_fwd_bwd_modules..z'backward_state_inputs must not be emptyrcSs|jSrFrr)srIrIrJrh"sz*_extract_fwd_bwd_modules..rjsaved_tensor_with_no_vc_checkFzi=z, no_vc_check_start_idx=z, len(saved_values)=cs0g|]}|kr|tkrt|nt|qSrI)rr.r-rcrno_vc_check_start_idxrrIrJrZs z,_extract_fwd_bwd_modules..r)(r rrfilterr3r4r2r1rrr distributed is_availabler{rrallrvr#raddrr itertoolschainrrrlclearextendrrrrrerr_lazy_graph_module_make_graph_moduler) rrrrrar r r r  placeholders primal_inputstangent_inputsfwd_seed_offset_inputsbwd_seed_offset_inputsbackward_state_inputs bwd_graphdistributed_enabledrB saved_symbolssaved_sym_nodes_bindingsaved_sym_nodes_derivedsymbolsymbol_bindings new_symbolsrsaved_values_with_vc_checksaved_values_no_vc_checksaved_opaque_objectsr fwd_graphrrrIrrJ_extract_fwd_bwd_moduless                    r)static_lifetime_input_indicesra _joint_inputsrrlist[int] | Nonecsg}d}|jjD]}t|st|st|r|}q|dur!td|jjD]}t|s0||||ur6nq%tdd|Dt |}t |} |rft |jddur`t dt||||dSt|dd }tjsmt|t|t||dur{g}t|||} g} g} tjd%dd} d%dd}d%fdd }|jjD]}|jvrq|jdkr|jdd|Dvrq|jtjjjj tjj!j"j tjj!j#j tjj!j$j tjj!j$j%fvrqt&|r| |q||rq|j'(dt)j*kr| |q||r|r td|d|j| |q| |s"|jdkr"td|dfdd|j+D}t,dd|Dr<| -|qt.|sF| |qt/t01| 2} t/t01| 2} tj3rct3|j| } |durk| j4}t5|| | ||d\}}|jj6t7d |jj6t7d |r| rt8|||t9| \}}t:|}tj;rd!d"l|d#d$}t9| j?dkrt>|dd$}||fS)&a Partitions the :attr:`joint_module` in a manner that closely resembles the behavior observed in the original ``.forward()`` and ``.backward()`` of the callable, i.e., the resulting forward graph contains those operators that are executed in the original ``.forward()`` callable passed to :func:`aot_function`. The default partitioner collects the operators that are between the forward inputs and the forward outputs. This helps in finding the tensors which have to be stashed for the backward pass. These stashed tensors become the output of the generated forward graph. The remaining operators are then placed in the backward graph. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. Returns: Returns the generated forward and backward Fx graph modules. Nzlast_node must not be Nonecss |] }|jdkr|jVqdSrNrrrrIrIrJrez$default_partition..rzxTrying to unsafely apply AC to a non-functional graph with the default partitioner. Falling back to min-cut partitioner.)rrTis_default_partitionrBrCrDr9cSs d|jvpt|jdtjjS)Nr"r)rrrr _subclasses FakeTensorrrIrIrJ is_tensors z$default_partition..is_tensorcSs"tdd|jDot|jdkS)Ncss|] }|jtjkVqdSrF)rrrrcrrIrIrJrez=default_partition..is_multi_output..r)rr{rrrIrIrJis_multi_outputs z*default_partition..is_multi_outputcs.|jddo|jdvo p|jtjjjjuS)NF) impure_random)rr) is_impurerrrrrrrr)rrIrJrs z$default_partition..is_impurercss|]\}}|VqdSrFrI)rckvrIrIrJre rz.Trying to apply AC on a graph with impure op: z, rz Expected z to be a tensorcsg|] }|jvr|qSrIrrrb)forward_node_namesrIrJrrmz%default_partition..cs|]}t|VqdSrFrrbrIrIrJrerrra)is_impure_noder%enable_activation_offloadingFrrQ)@rrrr3r2rvr4rr#rrr/warningswarn#min_cut_rematerialization_partitioncleanup_recompute_tagsr(unsafe_allow_optimization_of_collectivesforce_save_collectivesforce_save_effectful_opsforce_save_bw_mutation_srcclassify_nodesrrrrr named_modulesrrr_assert_scalarrprofiler_record_function_enter_new_record_function_enter_record_function_exit_RecordFunctionrrrr$ MUST_SAVEr{rrrrzrdfromkeysr|_sync_decision_cross_ranksrarrrfunctionalize_rng_opsr#reordering_to_mimic_autograd_enginer,_activation_offloading.activation_offloadingr8thread_graphsafe_rng_from_hopsr\)rrrrra forward_nodes last_noderBgraph_has_recomputable_opsgraph_has_recomputable_rng_ops node_inforrrrrbackward_usages fw_module bw_modulerrI)rrrJdefault_partitionys                                r!g.ArEcCs ||jSrF)itemsize)rErNrIrIrJ_tensor_nbytesUrqr#csdddd|jvrP|jd}t|trdSt|ttfr(tfd d |DSt|tr:tfd d |DSt|tj rD|St d t |d ||j dks^|j tjjjjur`dSt d|d)NrobjectrDrtcSs(t|tjsdStt|dd|jS)Nrfallback)rrr/r#rrErNrrIrIrJ object_nbytesZs z_size_of..object_nbytesrr%c3|]}|VqdSrFrIrbr(rIrJregrz_size_of..c3s|] \}}|VqdSrFrI)rcr$rdr*rIrJreirzUnknown metadata type z on node rrruzO didn't have `val` metadata; we should always have `val` metadata on the nodes.)rr$rDrt)rrrrzrsumrditemsrr/rrrrrrr r)rBrrIr*rJ_size_ofYs"       r-cCsbddlm}|t}|jD]}|jdkr||jjd7<q tdt | t ddddS)Nr)rrr%%sTrkreverse) collectionsrrtrrrrRr;r\rlr,r itemgetter)rrcntrBrIrIrJ _count_opsvs   $r4!list[torch._ops.OpOverloadPacket]cCslg}ttjjD]+}ttjj|}t|tjjsq|D]}t||}tj j |j vr2| |nqq|SrF) dirrrrrrrOpOverloadPacket overloadsr pointwiserr)r attr_nameopoverloadpacketoverload op_overloadrIrIrJ pointwise_opss   r>rtuple[Any, ...] depth_mapr_list[tuple[fx.Node, int]]cs*fdd|D}t|tdddS)Ncs&i|]}t|tjjjr||qSrI)rrrrBr)rcargr@rIrJrs zsort_depths..r%Tr/)rlr,rr2)rr@ arg_depthsrIrCrJ sort_depthss rEgmc s0ti|jjddD]}|fdd|<q ddt|jjDdfd d }ttt |jj}d }t j }|D]}|j D]}||krS|}|}qEq@|d ur[|St|jjd |D]}|j dkr{|jtjjjjur{||qgt|jj|d D]}||qtj|}|S)a This pass finds the first bwd node in the graph (by looking at users of tangents) and then reorders the graph by walking from this node to all the way to the end of the graph. At each op in this traversal, we insert this op in a new graph and try to bring only the relevant subgraph from the other non-bwd edges relevant for this op. This closely mimics the behavior of autograd engine. Why is this pass required in the first place? This is an artifact of how partitioners work today. The starting point of partitioner is a joint graph, which is fwd and then bwd graph. In the case of checkpointing, we keep portions of fwd graph in their original place in the joint graph, while obtaining a bwd graph. As a result, the resulting bwd graph has copies of recomputed fwd subgraphs followed by the original bwd graph. If we run this naively, this leads to bad memory footprint, because the fwd subgraphs are live for way longer duration than necessary. This pass reorders the operations such that we prioritize the ops for the original bwd graph while only realizing those ops from the fwd graph that are necessary at any given point in the graph. rrcrrFrIrrrIrJrhrz5reordering_to_mimic_autograd_engine..cSi|]\}}||qSrIrIrcrrBrIrIrJrz7reordering_to_mimic_autograd_engine..rBrCrDrcs|g}t}t|dkr)|}||vs|vrq||||j7}t|dks t|fddd}|D]}|fdd|<q5dS)NrcrrFrIrf)orderrIrJrhrzSreordering_to_mimic_autograd_engine..insert_node_in_graph..rjcrrFrIrrrIrJrhr)r#rrxrall_input_nodesrlr)rB cur_nodesinsertable_nodesrrrJrIrJinsert_node_in_graphs     zAreordering_to_mimic_autograd_engine..insert_node_in_graphNrrBrCrDr)rrrrrrerrzrr4mathinfr{rrrrrrr GraphModule) rFrBrOrfirst_node_in_bwd minimum_ordertangentrnew_gmrIrNrJrs4   rrtorch.fx.GraphModuler fw_nodebw_nodedevice torch.device rng_countlast_fwd_inputlast_bwd_input#tuple[torch.fx.Node, torch.fx.Node]cCs|j}|dur td|j} |j} tjjj} |j||jd|} t || j d<| }Wdn1s9wY|j||jd|} t || j d<| }Wdn1sawYt |j }| |d<|j|| j d| |jg|jR|d}Wdn1swY||| |t |j }| |d<| |$| j d| |jg|jR|d}||| |Wd||fS1swY||fS) a% Note [CUDA Graph Safe RNG Functionalization] CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator (and its cuda-tensor RNG state during graph capture). For each RNG operation's forward/backward pair: - We create two generators initialized with identical values - Each forward and backward call advances its respective generator equally - This keeps generators synchronized so forward and backward operations use matching RNG values When forward is called multiple times before backward (causing desynchronization): - We save the forward RNG state - We update the backward Generator's state before executing backward Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator changes are reflected during replay. This function modifies both forward and backward computation graphs by: Creating RNG state placeholders for both passes Updating the forward node to use graph-safe RNG state Updating the backward node to use graph-safe RNG state For more details: https://github.com/pytorch/pytorch/issues/113541 Nzdevice_idx must not be Nonefwd_rng_state_rbwd_rng_state_ rng_staterrr)indexrvrr_prims rng_primsgraphsafe_run_with_rng_stater&rr5rrdr create_noderrrrinserting_before)rr rYrZr[r]r^r_ device_idxfw_graphbw_graphrh fwd_rng_state bwd_rng_state fw_kwargsfunctional_fw_node bwd_kwargs rng_outputrIrIrJ%apply_graphsafe_rng_functionalizationsV)           rt num_sym_nodesc' st}d#dd}d$d d d%dd}||}||}||} i} |jjD]5} t| rZt| jdrZtjj | jj vrZ| j |vsC| j | vrDq%|| j } || j } | | j }| |d| | <q%tj j j}tj j j}d}|jjddD] } d| j vry| }nqn|durtdg}tt|jjdd}tt|jjdd}tfdd| D}|tdt|dk}tjj}tjo| o|j p|jj}t| D]\}}|d} |d}| }|j}|j}|r|dur|j dkrt!||| |||||\}}q|"| M|j#d|| jg| j$R| j%d}|j#dt&j'|dfid}|||j(d <|j#dt&j'|dfid} t))| j(| _(| *| |+| |,|Wdn 1sRwY|"|d!t|}!|-|!}"|||"j(d <Wdn 1s{wY|"|#|j#d||"|jg|j$R|j%d} |*| |+|Wdn 1swYq|rtt.|jjd"d}#|#j$d}$t|$|}%|$d|%t/||$|%d}&|j0|&|j+|#|1|1||fS)&NgmodrrDrcSsFi}|jjD]}|jdkr t|jdr tjj|jjvr |||j <q|S)Nrr) rrrrrrrrrr)rv random_nodesrBrIrIrJ get_rng_opsbs    z*functionalize_rng_ops..get_rng_opsrBrCtorch.device | NonecSs^d|jvrdS|jd}t|ts|f}|D]}t|tjr)|jjdkr)|jSqtdS)zV Check the example value of the node outputs to find the device type. rNcudacpu)rrrrr/r[r)rB candidates candidaterIrIrJ get_devicems       z)functionalize_rng_ops..get_devicer[rDcSsddlm}|}|durtd|'|dur,|jdkr,|tjWdS|tWdS1s=wYdS)Nr)detect_fake_modezfake_mode must not be Nonerz) torch._guardsrrvr from_tensorrrz get_rng_state)r[r fake_moderIrIrJget_sample_rng_states  $z3functionalize_rng_ops..get_sample_rng_stater)fwdbwdrrrVzaCouldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see thisc3s|] }|dVqdS)rNrI)rc node_pairr~rIrJres  z(functionalize_rng_ops..r{r%rrrzrrdrrrng_state_output_r)rvrrDr)rBrCrDry)r[ryrDrD)2rcountrrrrrrrrrrrfrgrun_and_save_rng_staterun_with_rng_staterrrrr#valuesdiscardr[rrPrgraphsafe_rng_functionalizationfallback_random test_configs*graphsafe_rng_func_ignores_fallback_randomrerrtrjrirrrrrcopyrrrrrrr recompile)'rrr ruuidrxrjoint_graph_rng_opsfw_graph_rng_opsbw_graph_rng_opsrecomputable_rng_ops_maprB base_noderYrZrun_and_save_rngrbw_tangent_start_nodefw_rng_state_outputsr^r_devicesmulti_cuda_devices ind_config'use_rng_graphsafe_rng_functionalizationr]rr[rlrmrqstaters state_namebw_rng_state_nodefw_output_node fw_outputssym_node_start_idxrrIrrJrGs                      $         rcCsB|jjD]}t|jtjjr|jjdkrt|st j |j d<qdS)z By default, the partitioner is not allowed to recompute collectives unless they come from a user-annotated AC region. See Note [Recomputing collectives in the partitioner] rrN) rrrrrrrrrr$rrrrBrIrIrJr+s   rcs@dfdd |jjD]}t|rt|st|s|q dS) a\ Force save outputs from with_effects nodes wrapping effectful ops. Effectful ops (registered via _register_effectful_op) should not be recomputed because they may have arbitrary global side effects (I/O, RNG state, collectives, etc.). We mark the tensor outputs of with_effects as MUST_SAVE to prevent recomputation of the effectful op. The with_effects node returns a tuple (token, result). We recursively find all leaf outputs extracted via getitem and mark them as MUST_SAVE. Since these are saved, the with_effects op doesn't need to be recomputed in backward. rBrCrDrcsF|jD]}|jtjur |t|jdttfs t j |jd<qdS)Nrr) r{rrrrrrrrzr$r)rBrmark_getitem_outputsrIrJrHs   z6force_save_effectful_ops..mark_getitem_outputsNrP)rrr rrrrIrrJr:s rcCst}t|jjD]6}|jdkrq |jtjjj j u}|r=t |r(| |j dt|r<|j d|vr.csrrFrrrIrIrJrerr) rrrr{rr$rrrrr)rrrBrrIrIrJrxs4        rrmin_cut_optionsdont_ban)tuple[list[fx.Node], OrderedSet[fx.Node]]c< sB durtttr(tdd|jD}|tddjD}td|dpd d dpd d dpfdd zddlWntyR}zt d|d}~wwdq fdd dr fdd }dqfdd dsfdd } tdtdufd"d# } |jD]} | j d$krq| j vr| j vrj| jd%d&tjd'd(nj| jd)d&tjd*d(qt| rɈj| jd)d&tjd+d(qt| r| | d,n t| r| | d-|| } | r| r| | | d.| jvod/| jvpd.| jvot| jd.tj } t| rtt| } d}n"| r/t| jd.ttfr)d0} d}ntj} d1}n|| j \} }|rX| tjksE| t!krXj| jd)| jd%| d2|d(nj| jd)| jd%| d3| j"D]}j| jd%|jd)tjd4d(qiqdv fd9d: } j#r j$D]]} fd;d<|j"D} fd=d<|j"D}t%|dkr||t&|}t'|j"D]2} |r (||kr||r|vrӐqtd>| (||| (|| |qq j)rwt}|jD]} |sq (||fg} (|}t%|dkrut*+|\}}||vr!q |,| (||d?krKt%|dkrKtd@|| (| (|| |n*|j"D]} |rl||rl|vrlt*-| (||fqNt%|dksqz .dAd&\}}Wnj/y}zut0t1j2}d}d|j3} z>| r| j4dBdCdCdDnt5|t6dEdFdGfdHdGdIt7dJdK}t8|dL }!|!9Wdn 1swYWnt:y}z dM|dN}WYd}~nd}~wwdO;jt6dEdPdGfdQdGdIt?}"|"ri}#dAg}$dwdSdT}%|"D]_\}&}'}(|$@|'|&dAkr:|%|'})|#A|)g@|(q|'d&krN|%|&})|#A|)g@|(q|%|&|%|'krf|%|&})|#A|)g@|(q|%|&}*|%|'}+|#A|+g@dU|*qg},|#BD]\}-}.|,@dV|-dW|.D] }/|,@dX|/qqdO;|,}0dY;|$}1tC\}2rt6dEdZdGfd[dGdI|rd\|dOnd}3|2r|3d]|2dO7}3d}4|rd^}4t d_|0d`|1dO|3|4|tdatdbtDfdcdGtCd}~wt:y%tdatdbtDfdddGtCw|\}5 t}6fded|5DD]\}7|6E fdfd|7Dq6t}8|6D]/\}9}:|9ddg|:ddhkrqtFdi|9ddgdj|:ddh|9ddg}-|8,|-qNtG| dkdltH|jD tI fdmd|8D fdndGdo};|;fS)xNcss2|]}|jdkrt|jdrt|jjVqdS)r_overloadpacketN)rrrrrrrIrIrJres z solve_min_cut..csrrF)rrrIrIrJrerz&Ops banned from re-materialization: %sarCbrDr9cSsn|jtjjjkr dS|jd}tjj|\}}|D]}|j |}||ur(dSt |t r4||vr4dSqdSNFrT) rrrrauto_functionalizedr_higher_order_opsauto_functionalizeget_mutable_argsrrrz)rr mutable_opmutable_arg_namesr$rrBrIrIrJ!can_fuse_into_auto_functionalizeds$   z8solve_min_cut..can_fuse_into_auto_functionalizedcSs\|jtjjjkr dS|jd}|D]}|jd}|dur td||}||ur+dSqdS)NFtensors_to_clonerzkwargs must not be NoneT)rrrrrrrv)rrrrrrBrIrIrJ.can_fuse_into_triton_kernel_wrapper_functionals  zEsolve_min_cut..can_fuse_into_triton_kernel_wrapper_functionalcsht|tjkr dS||rdS||rdS|jtjur*|jdjtjj j ur*dS |o3 |S)NTrF) r7rcatrrrrrrrrrK)rr)rrop_typesrIrJrKs    z!solve_min_cut..is_fusiblerzANeed networkx installed to perform smart recomputation heuristicsrBcsv|rdSt|g}t|dkr9|}|jD]}|s(||s(dS|r2||qt|dksdSr)rOr#rrxr{rpr)rBrLcurr)rKrrrIrJis_materialized_backwardss       z0solve_min_cut..is_materialized_backwardsrcs|jdkrdS|jtjurdS|jddtjkrdStj r% |r%dS|jt j j t jj fvr2dSjr=|s.should_ban_recomputation..rz reduction op)rrrrrrr$rrrecompute_viewsrOrlift_fresh_copyr lift_freshr~rPrNrMrr}r;rrr{ dist_from_bwmax_dist_from_bwrr+rr-)rBinput_tensors_size output_size)rrrrIrJshould_ban_recomputationsF      z/solve_min_cut..should_ban_recomputationcs*jdkrdStfddjD S)NrTc3s|]}|VqdSrFrIr)rKrBrIrJreMsz9solve_min_cut..is_materialized..)rrr{r)rKrrJis_materializedIs z&solve_min_cut..is_materializedrarZtuple[float, str | None]cstjr ||vr dSt|}tjr|rtjdfSt|jdt r/t|jdt j s/t dfSt |dtt|jdd}|rF|dfS|d dfS) zReturns (weight, cannot_save_reason). cannot_save_reason is None for finite weights, or a string explaining why the node cannot be saved for infinite weights. rzview op (recompute_views=True)rz$SymFloat (non-SymInt symbolic value)g?rTr%N)r treat_parameters_as_free_to_saver-rrOrQrRrrrrrINT_INFrtrr r)rBramem_sz)rrrIrJget_node_weightOs    z&solve_min_cut..get_node_weightrqreasonrcs|rdS|vr t|jtjjo|jjdk}tjs|s dSt |r&dSd|j vr6t|j dtj r6dS |j d|jdtj|rKd|nddd S) NFrrr_inzcannot recompute: zcannot recomputecapacityrT)rOrrrrrrrrrrrradd_edgerrQrR)rBr is_collective) banned_nodesrnx_graphrrIrJban_recomputation_if_allowedzs(    z3solve_min_cut..ban_recomputation_if_allowedr_outsinkz;must be available for backward: input required for gradientrrz3must be computed in backward: required for gradientz+must recompute: marked by checkpoint policyz primal inputzforward RNG seedrr"znon-tensor outputz cannot save: )rzdata dependency start_nodesrX max_rangertc sg}|D]}t|||dfqt|dkrVt|\}}}|s(|S|jD]$}|rO||kr:q+||||f}||vrOt||q+t|dks|S)z Finds the first unfusible node in the chain of nodes starting from `start_nodes` and returns its position. Tr)heapqheappushrwrheappopr{rp) rr sorted_nodesrdr$rBnode_is_fusiblerr)rKrrIrJfind_first_unfusible s(      z+solve_min_cut..find_first_unfusiblecs g|] }|r|qSrI)rprwrrrIrJr s z!solve_min_cut..csg|] }|r|qSrI)rprrrIrJr$ s  z1used above/below fusible %s:(%s) -> %s -> %s:(%s)rTztoo long %s %s %s %srFTrrcSr)Nmin_cut_failed_fx_graphrrrIrIrIrIrJrh rzsolve_min_cut..cSrFrIrI) fx_graph_strrIrJrh rmin_cut_failed_graphz.txtwz(failed to write: ) cSr)Nmin_cut_failed_edge_listrrrIrIrIrIrJrh rcrrFrIrI) edge_list_strrIrJrh r node_namecSs.dD]}||r|dt| Sq|S)N)rr)endswithr)rsuffixrIrIrJ get_base_name s  z$solve_min_cut..get_base_namez depends on z :z - z -> cSr)Nmin_cut_failed_svgrrrIrIrIrIrJrh rcrrFrIrI) svg_contentrIrJrh rzFX graph dump: zMin-cut graph visualization: z[Production debugging: Use tlparse to extract debug artifacts (min_cut_failed_fx_graph, min_cut_failed_edge_list, min_cut_failed_svg)] aAOT Autograd failed to partition the joint forward-backward graph. The partitioner determines which intermediate values to save from the forward pass vs recompute in the backward pass. This error means a value is required for backward, but cannot be saved AND cannot be recomputed. This is a bug in PyTorch. Please file an issue at https://github.com/pytorch/pytorch/issues Nodes involved in the conflict: z [For PyTorch developers: one of the above constraints is wrong. Either the node should be recomputable, saveable, or not required for backward.] [Debug: min-cut path] z-Failed to compute min-cut on following graph:r.cdjjSNrjoin readwriteedgelistgenerate_edgelistrInxrrIrJrh rcrrrrIrrIrJrh rc3s|] }||fVqdSrFrIrb)rrIrJre rc3s |] }|vr|fVqdSrFrI)rcr) non_reachableurIrJre sz node_in[:-3]=z != node_out[:-4]=cSrGrIrIrHrIrIrJr rIz!solve_min_cut..c3s|]}|VqdSrFrIr name_to_noderIrJre! rcrrFrIr)node_idxrIrJrh! rrj)rrCrrCrDr9rQ)rBrCrDr)rBrCrarZrDr)rq)rBrCrrrDr9)rrXrrtrDrt)rrrDr)Jr#get_default_op_listr:rrAr;r\networkx ImportErrorrDiGraphrr\r]rrrQrRrr3r2rprrrr/rrrrrrrarr{r{rmrrrrwr|rrrr minimum_cutNetworkXUnboundedr9rhandlers owning_modulerrr_get_unique_pathopenwrite Exceptionrrrr_find_infinite_capacity_pathr setdefaultr,visualize_min_cut_graphrrrvget_name_to_nodererl)tj?tj@tjAtjBtjCtjDtjEtjFtGjHtjItjJtjKtjL}tjItjJtjMg}|tjNtjOtjPtQjRtjStjTtjUtjVtjWg 7}|}|gtQjtQjXtjYtjLtjZtQj[tQj@tj[tj\tQjRtjVtj]tjNtjStjOtj^tj_tj`tjatjbtjctjdtjetjftjgtjhtjitjTtjjtjktjltjmtjntQjotQjp7}|tjqtjrg7}||7}|ts7}|tjtg7}|ddtuD7}tv|}tvtwdtxftjytjztj{g}tj|tj}tj~tjtjtjtjtjtjtjtjg }||B}t|tv||tv||S)NcSg|]}t|qSrI)r)rcmrIrIrJr rz'get_default_op_list...)rrrryatan2r.rr pow remainderfmod__and____or____xor__ __lshift__ __rshift__eqnegegtleltr' bitwise_notceilfloorfracnegreluroundsilutruncr;log10log1plog2lgammaexpexpm1erferfccosacoscoshsinasinsinhtanatantanhatanhsqrtrsqrtr-sigmoidsoftplus thresholdthreshold_backwardclampwherelerpaddcmulgelu gelu_backwardr+mean_grad_sum_to_size sum_to_sizer(totype_asrrsqueeze unsqueezersub_to_copyaliasrslicetr)broadcast_in_dimexpand as_stridedpermuteselectrMr*clone full_likevarstd _unsafe_viewrbroadcast_tensors scalar_tensorones new_zerosrarangetriuvar_meanisinfrfullzerosempty empty_likeargmaxmaximumiota'_low_memory_max_pool_offsets_to_indicesregatherr> zeros_likerr#rrnative_dropout rand_like randn_likemm convolutionconvolution_backwardbmmaddmm#_scaled_dot_product_flash_attention'_scaled_dot_product_efficient_attention_flash_attention_forward_efficient_attention_forwardupsample_bilinear2d _scaled_mmr<)default_recomputable_opsrecomputable_view_opsr@rAr?r>r=rIrIrJr } s      !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKM       !"#& r cCsi}|jD]}|||j<q|SrF)rr)rr rBrIrIrJr% s  rmemory list[float]runtimes max_memoryall_recomputable_banned_nodes"tuple[float, list[int], list[int]]c Cstj}|dkr t|||S|dkrt|||S|dkr!t|||S|dkr+t|||S|dkrKtdtj ||||d}t||t |dj t|d St |t r_||||||\}} d || fStd |) Ngreedyilpdpr(dynamic_memory_budget_dpzdynamic_memory_budget_dp is an experimental solver. It does not guarantee performance improvements. Additionally, it is not guaranteed to be stable.)rr recorded_knapsack_input_memories recorded_knapsack_input_runtimes)graph_info_provider) knapsack_algomax_mem_budgetrz,Not aware of memory budget knapsack solver: )ractivation_memory_budget_solverr)r*r'r(r;warningr&inialize_from_graphr+get_knee_point_memory_budgetrrr) rrrrrrSOLVERrsaved_node_idxrecomp_node_idxrIrIrJ#_optimize_runtime_with_given_memory, sH       r no_dispatchrr'csNt|j}d fdd fdd|D}fd d|D}|j||d S) Ndtorch.SymInt | intrDrtcs t|dS)Nr&)r)rr&rIrJrealize_symbolc s z8_remove_symbols_without_guarding..realize_symbolcg|]}|qSrIrIrcrrrIrJrf rz4_remove_symbols_without_guarding..crrIrIrrrIrJrg r)stride)rrrDrt)rzshapernew_empty_strided)rr'rrrI)r'rrJ _remove_symbols_without_guarding` s rcs$tj}ddd}|dkrdS|dkrFt'd d lm}t|jjf\| fd d }|WdS1s?wYdS|d krd dl m }t|jjf\|dd}j iWdn1stwY| }t|dSt|tr|Std|)NrrrDcSst|tjrt|jdtjrt|jdddSt|tjr0t|jdtjr0t|jdddSt|tjrAt|jdtj rAdSt|tjrRt|jdtj rRdS|S)Nrr%r&?T) rrrrrr/rrrrrrrIrIrJmaterialize_argn sz)estimate_runtime..materialize_argtestingr%profiler) benchmarkercsjiSrF)rrIrrrBrIrJrh z"estimate_runtime..flops)FlopCounterModeF)displayz Not aware of runtime estimator: )rrrDr)r*activation_memory_budget_runtime_estimatorr$torch._inductor.runtime.benchmarkingrrtree_maprr benchmark_gputorch.utils.flop_counterrrget_total_flopsrrrr)rB RUNTIME_MODErrmsrmode counted_flopsrIrrJestimate_runtimek s,   $    r memory_budgetcsF|dks|dkrtd|ttjtjtjtjtjd}tjr)t |ddddd}|dkr0j St |\}}|dkr>|SdJd d j  | krR|SdK fdd dL fdd t |dddd}t |\}}||kr{|St |ddt \}} ||kr|Sddl m tfddj DdMfdd } | | } dd| D fdd| D} t| tdd tdkrшj Sfd!dD d"dDdd#lm dN fd*d+ tjrdOfd.d/ } | d0| d1g} | ddd| dddkrz| d| dfg}|rz|\}}|d|dd2krC| || |q#| |d|dd3}|dd|ddkrc|||f|dd|ddkrw|||f|s&| ddlm}d4d| D}d5d| D}|jd6d7|j||d8d9t|D]\}}|j|d:|||fd;dq|d?|d@| dA|!d|"}|#t$%}tj&durtj&}t$j'|ddBdC}t(j)*rt(j)+rdDt(j),}t$j-.|dE|dFt/dG}|0|t12dH||dIdS)PNr%rzJThe valid ranges for memory budget are 0 <= m <= 1. The provided value is )r{r|r}r~rF)r{r|r}r~rrXrDrcSsttt|dSNeA)r+mapr-)rrIrIrJestimate_activations_size rz:choose_saved_values_set..estimate_activations_sizeszcs|dSrrI)r) max_act_size min_act_sizerIrJget_normalized_size sz4choose_saved_values_set..get_normalized_size activationscs|SrFrI)r)rrrrIrJ get_mem_ratio s z.choose_saved_values_set..get_mem_ratio)r{r|r})r~get_node_storagec3r)rFrIrr rIrJre rz*choose_saved_values_set..rrZcsfdd|DS)Ncs2g|]}|jtdkr|vst|r|qS)r)rrtrrr!input_storagesrIrJr s zRchoose_saved_values_set..get_recomputable_banned_nodes..rI)rr"rIrJget_recomputable_banned_nodes s z>choose_saved_values_set..get_recomputable_banned_nodescSs$g|]}|jddtjkr|qS)rF)rrr$rrrIrIrJr s z+choose_saved_values_set..csg|]}|vr|qSrIrIr)must_save_nodesrIrJr sTr/csg|]}t|qSrIr-r)rrIrJr s cSrmrI)rrrIrIrJr rrrrrWrrtuple[list[fx.Node], float]c st|t|d|\}}}Wdn1swYt}|D]}z ||Wq'ty;Yq'w|sEtdt|||\}} trbt ||||ddD|d ||fS)Nrz:dont_ban must be a subset of all_recomputable_banned_nodescSrmrIr&rrIrIrJr> rzNchoose_saved_values_set..get_saved_values_knapsack..) rrsaved_node_idxsrecomputable_node_idxsexpected_runtimememories_banned_nodes normalized_memories_banned_nodesruntimes_banned_nodesmin_cut_saved_values) rrr#r BaseExceptionissubsetrvrFr:r ) rrrr*r(r)rrrr$)aggressive_optionsrr+rr-rIrJget_saved_values_knapsack s\    z:choose_saved_values_set..get_saved_values_knapsackrtuple[float, float, float]cs(|d\}}|t||fS)N)rr)r+)rrr*)rr2rrr-rIrJestimate_for_budgetI s  z4choose_saved_values_set..estimate_for_budgetrrgMbP?rcSg|]}|dqS)rrIrcitemrIrIrJrf rcSr5r%rIr6rIrIrJrg r) )figsizeo)markerz.4fz offset points)rr9center) textcoordsxytexthaz Memory Budgetz Runtime of Recomputed Componentsz:Pareto Frontier of Memory Budget vs. Recomputation Runtime)exist_okrq_rank_memory_budget_paretor$r]z%Generated Pareto frontier curve at %s)rrr)rrXrDr)rrrDr)rrXrDr)rrZrDrX)rrrrWrrrDr')rrrDr3)3rrzrban_recompute_used_far_apart!ban_recompute_long_fusible_chains#ban_recompute_materialized_backwardban_recompute_not_in_allowlistban_recompute_reductionsaggressive_recomputationrrYrFtorch._inductor.fx_utilsr!r#rlr-rtorch.utils._mode_utilsrvisualize_memory_budget_paretorxrsortmatplotlib.pyplotpyplotfigureplotreannotatexlabelylabeltitlegridgcfshowrTgetcwdmemory_budget_pareto_dirmakedirsrrris_initializedget_rankrUrr0savefigr;r)rrrrruntime_optimized_saved_valuesr$more_aggressive_optionsmore_aggressive_saved_values%aggressive_recomputation_saved_valuesrr$recomputable_banned_nodesr4optionsbisectslhsrhsmidpltx_valuesy_valuesrtxtfigfig_dir rank_suffixfig_namerI)r1rrrr!rr2r#rrr+rr%rrr-rJchoose_saved_values_set s          3 "             rrlist[torch.fx.Node]c sddlmddd}dfd d }tjrtjrtjd kr||r||rtd d |Dg}dd ttjD}tj ||dt |g}i}t |D]1\}} fdd | D} d} | D]} t | } | | 7} |tj kr| || j<qm| |d<|| q\tj|tjjd}tjj|tjjjjdtt|}d|d|tdddfdddfdd ||D}Wdn1swYWd|SWd|S1swY|S)Nrunset_fake_temporarilyrrrDr9cSs2|jD]}t|jtjjr|jjdvrdSqdS)N>c10d_functionalrTF)rrrrrrr)rrBrIrIrJhas_collectives s   z3_sync_decision_cross_ranks..has_collectivesc sddd|jD}t|d}ddttj Dt "tj |Wdn1s:wYWdn1sIwYt fddDS)N/csrrFrrrrIrIrJre rzE_sync_decision_cross_ranks..has_same_nodes..r[cSsg|]}dqSrFrIrcr$rIrIrJr rzF_sync_decision_cross_ranks..has_same_nodes..c3s|] }d|kVqdSrrIr all_inputsrIrJre r) rrhashlibsha256encode hexdigestrrrget_world_sizerall_gather_objectr)rnode_strrYrtrzrJhas_same_nodes sz2_sync_decision_cross_ranks..has_same_nodesr%cSrrIrrrrIrIrJr rz._sync_decision_cross_ranks..cSsg|]}gqSrIrIryrIrIrJr scg|]}|qSrIrI)rcop_namerrIrJr rz total size)r[rzpicked_rank_idx=z, saved_nodes of current rank=rcSr)N)aot_joint_graph_sync_decision_cross_ranksrrrIrIrIrIrJrh rz,_sync_decision_cross_ranks..crrFrIrI)sync_decision_cross_ranks_strrIrJrh rrcrrIrIrbrrIrJr r)rrrDr9)torch._subclasses.fake_tensorrurrrr]rrrrrrer-r^rrrCdistributed_c10d_get_object_coll_device all_reduceReduceOpMAXrtargminr7r)rrrwrobjectssaved_ops_names_all_ranks saved_sizessaved_ops_with_sizesrsaved_ops_names saved_nodes saved_sizerB size_of_nodesaved_sizes_tensorpicked_rank_idxrI)r rrurJr sn         (, ,,rmodulerc Csd}|rdnd}tt|jjdd}|jjdtjjjdD]}t||j dj }t |t j rg}|jjddD];}||jvrs|j|&|j|d|} |d 7}|jd | jd <| }|| Wd n1snwYq8|r|j|!|jdtjjjg|j |Ri} |j| d d Wd n1swY|jd} | r| \} } g| dd|DR}|| f| jd<|j|q|S)u Graph-safe RNG lets torch.compile use CUDA Graphs for graphs with RNG ops. For graphs without HOPs, the partitioner adds placeholder nodes fwd_rng_state_* and bw_rng_state_* to the forward and backward graphs. At runtime, the AOTDispatcher retrieves these RNG states and passes them to the compiled graphs. This works well for no-HOP graphs. With HOPs, the partitioner runs recursively: it first partitions the HOP (producing forward/backward HOP subgraphs) and then stitches them back into the outer joint graph. For HOPs that contain RNG ops, the outer joint graph now includes HOP subgraph modules with extra RNG placeholders. We must thread these placeholders through the outer module partitioned forward and backward graphs—this function does exactly that. It collects the RNG placeholder nodes from the HOPs and creates corresponding placeholders in the outer forward and backward graphs. There is a catch: for a short period, the joint graph is in a “bad” state. The HOP subgraphs expect additional inputs (because of the new placeholders), but the outer graph call sites don't yet provide them. We can't fix this in the joint graph because the joint graph's input signature is fixed (primals, tangents). As a compromise, we keep the joint graph in somewhat of a bad state for some time and, once the outer forward and backward graphs are partitioned, insert the corresponding RNG placeholders and wire up the calls. rrornrrr)rrr$r%rNT)propagate_metaeager_input_valscSsg|]}|jdqS)r)r)rcinprIrIrJr- rIz2thread_graphsafe_rng_from_hops..)rrrrrrrinvoke_subgraphrrrrrrSrr&rrrrirrr)rrr] rng_string last_inputhop_nodernew_rng_inputsplaceholder_nodercnew_hop_node_with_fixed_args eager_vals eager_args eager_kwargsnew_eager_argsrIrIrJr s^           r list[int]csbt|jt|jjD]%}|jdkrd|jvr|n t|r'||vr1|j q t t t |jj}t t t |jj}||}t||d\}}} } } dd|Dt|j||| d} tfdd| jDtfdd|jjD} tfd dt|D}d }i}|jjD]}|vr|||<|d 7}qt|| | ||S) Nrtangentsrcss&|]}|dur|jdkr|VqdS)Nrr)rcr<rIrIrJreU sz!classify_nodes..rc3s$|] }|jdkr|jVqdSrrrrrIrJre[ s c3s$|] }|vr|vr|VqdSrFrIr)r\rmrIrJre` sc3s |] \}}|vr|VqdSrFrI)rcrprrIrJree rrr%)rrr#rrrrrrr{rzrr3r2r rrrerW)rrrrBrrrYr r r r r]forward_only_graphr^rafw_cntr`rI)r r\rmrrJr 8 s^            r rbrcompilercCs|j||j}tjrt|}||_|j}t|}t|} |r)t|dd}tj s0t |t |t ||dur>g}t |||} t| jdkrUt||||| jdSt|jjD]+} | jdkrhtd| _q[| | sqd| _q[td| _| jD] } t| j| jd| _qyq[tj} |jD]} t| jd dtr| jd } nqt|| | d }tj rt ||}t!t"t#|}t!t"d d |}t$||||| jd \}}|r| rt%|||t|\}}t&|}tj'rddl(m'}||||| jt)|}t)|}t*|dd}t*|dd}t+rt,dd|D}t-dd|Dd}t./d|t./d|t0dd|jjD}t0dd|jjD}||@}t1t}|jjD]} | j2|vrdt3| j4drd|t5| j4j6d7<qIt./dt|t|t|t,|7t89ddd}t./d|||fS)ax Partitions the joint graph such that the backward recomputes the forward. Recomputing helps in trading off memory bandwidth with computation. To create the fwd and bwd graph, we copy the joint graph, manually set the outputs to just original forward or backward outputs. And then we run the resulting graphs through dead code elimination. .. warning:: This API is experimental and likely to change. Args: joint_module(fx.GraphModule): The joint forward and backward graph. This is the result of AOT Autograd tracing. _joint_inputs: The inputs to the joint graph. This is unused. compiler: This option determines the default set of recomputable ops. Currently, there are two options: ``nvfuser`` and ``inductor``. recomputable_ops: This is an optional set of recomputable ops. If this is not None, then this set of ops will be used instead of the default set of ops. num_fwd_outputs: The number of outputs from the forward graph. Returns: Returns the generated forward and backward Fx graph modules. FrNr)rrrarrr%r)rcSs t| SrFrrfrIrIrJrh riz5min_cut_rematerialization_partition..rrrTcSsg|] }t|t|fqSrI)r-rrrIrIrJrsz7min_cut_rematerialization_partition..csrrFr&rrIrIrJrerz6min_cut_rematerialization_partition..z'Theoretical Activations Stored: %.2f GBz,Theoretical Per Activation Storage Sizes: %scs |] }|jdkr|jVqdSrNrrrIrIrJre rcsrrrrrIrIrJrerrz# remat/fw/bw: %d/%d/%dr/zCount of Ops Rematerialized: %s):rrrrcser6rrrrrrr r rr\r!rarrrrtrrpr{r activation_memory_budgetrrrrrrrrzrrrrrrrr8rr:rlr+r;r\r#rrrrrrr,rr2)rrrrrr cse_graphrrrrrBrrrrrr r sorted_sizestotal_activations_size_gbfw_module_nodesbw_module_nodes remat_nodescountsrematerialized_opsrIrIrJry s "                   rfx_graphTtracedfnamefigname clear_metaprogstr | list[str] | Noneparse_stack_tracedot_graph_shapecCs|rt|j}t||}|jjD]}i|_qtj |\} } | s'dt j } t d| | tj||||d} | } t| d| d} | | }|durU| |dS| ||ddS)NrLzWriting FX graph to file: %s%s)rrwrite_)r)rdeepcopyrrrSrrrTrUsplitextrtorch_compile_graph_formatr;r\r" FxGraphDrawerget_main_dot_graphrlstrip)rrrrrrrrrBr5extgr write_methodrIrIrJ draw_graph%s*      rrQ)rrrDr9)rBrCrDrt)rrCrDr)rBrCrDr)rBrCrrrDr)NF)rrrYrXrrXrrrrrr9rDr)rrrrtrDr)rrXrrrDr)rrrDrt)rrr) rrrBrrrr rr!rtrDr)rrrBrr7rr9r:r,rr;rr!rtrDr)rCrDrDr)rDrI)rBrrDr9)rDr:)rNr:rDrZ)rrrDr)rrrrrrrDrrF) rrXrrrrrarrDr) rrrrXrrXrrtrarrDr) rrrrrrtrrrarrDr)rErtrNr:rDrt)rrrDr)rDr5)rr?r@r_rDrA)rFrrDr)rrXr rXrYrrZrr[r\r]rtr^rr_rrDr`) rrrrr rrurtrDr)rrrDr)rrrr9rDr) rrrrWrrzrrrDr)rrGrDrH)rRrrSrrDr)rrGrDrX)rDr<)rrrDr)rrrrrrrrrrWrrXrDr)rrDr'rtrDrD)rBrCrDrr8)rrrrWrrrDrX)rrrrsrDrs)rrrr9rDr)rrrrrrtrDrW)rb) rrrrrrrrtrrrDr)rTNFN)rrXrrrrrr9rrrr9rrrDr) __future__rrrxr|rrloggingrQrrTos.pathrrr1rrcollections.abcr dataclassesrrtypingrr rtorch._inductor.inductor_primstorch.distributedtorch.fxrtorch.utils._pytreeutils_pytreertorch._dynamo.utilsr r ;torch._functorch._activation_checkpointing.ac_logging_utilsr $torch._functorch._aot_autograd.utilsr torch._inductorrr!torch._inductor.custom_graph_passrr"torch._library.fake_class_registryrtorch._library.utilsrtorch._loggingrrtorch._logging._internalrrr%torch.fx.experimental._backward_stater"torch.fx.experimental.proxy_tensorrrtorch.fx.experimental.sym_noderr%torch.fx.experimental.symbolic_shapesrrrrr r!torch.fx.passesr"torch.utils._ordered_setr#torch.utils.checkpointr$rq-_activation_checkpointing.graph_info_providerr&"_activation_checkpointing.knapsackr'r(r)r*,_activation_checkpointing.knapsack_evaluatorr+_aot_autograd.descriptorsr,r-r._aot_autograd.functional_utilsr/_aot_autograd.logging_utilsr0_aot_autograd.utilsr1r2r3r4r5 compile_utilsr6r7r8r rsympydebug_partitionerr:rV getLoggerrRr;rrr)r<rWrzrrrrrrrrrrrrrrrrrrrr rrr8rBrHrRrXrYr]rlrrrrr!rtrr#r-r4cacher>rErrtrrrr rrrFrrrr rrrLrrrrrrrr rrrIrIrIrJs8                   !        $ o          N F    A P^5)Z      N ^ e    < z !  ' )  0 0~QRD0