o eiH @sddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z ddlZddlmZddlmZddlmmZddlmmZddlmZmZddlmZddlm Z m!Z!gd Z"d e#d e$e#e#ffd d Z%dee&dej'de(e#e ffddZ)dej'de(e#e fdejj*fddZ+d8dejj*d ejj*fddZ,dej*d ej*fddZ-dej*de.ej'de.ej'de.ej'fd d!Z/ej0ej1ej2ej3ej4ej5ej6ej7ej8ej9ej7ej:ej;g ZgZ?ej0ej@ej1ejAej2d"d#iZBde.ej'de(e#ej*ffd$d%ZCde.ej'de(e#ej*fd&e(ej*ej*ffd'd(ZDGd)d*d*ZEd9d-d.ZFd/eEd eGfd0d1ZHGd2d3d3ZIdejJfdejj*d4e e(e#e fd5e&ejJd ejj*fd6d7ZKdS):N) defaultdict)Iterable)Enum)AnycastOptional)ArgumentTarget) ShapeProp)fuse_conv_bn_evalfuse_linear_bn_eval) matches_module_patternreplace_node_modulefuseremove_dropoutextract_subgraphmodules_to_mkldnn reset_modules MklSubgraphgen_mkl_autotuneruse_mkl_length UnionFindoptimize_for_inferencetargetreturncCs*|dd^}}|r|d|fSd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnamer!l/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/fx/experimental/optimization.py _parent_name%sr#patternnodemodulescCst|jdkr dS|jd|f}t||D]2\}}t|tjs"dS|jdkr*dSt|jts3dS|j|vr;dSt ||j|urGdSqdS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)r$r%r&nodes expected_type current_noder!r!r"r /s     r new_modulecCsLt|jtstdt|jt|j\}}|||j<t||||dS)NExpected str target, got )r+rr/AssertionErrorr0r#setattr)r%r&r4 parent_namer r!r!r"rCs  rFmodelc Cs4tjtjftjtjftjtjftjtjfg}|st |}|r&t |t j j s,t |}n|}t|}t |j}|D]W}|jD]Q}t|||rt|jdjdkrTqA||jdj} ||j} | jseqA|dtjtjtjfvrwt| | } nt| | } t|jd|| ||jd||qAqeZdZdedeedfdeeefdeffdd Z Z S)z&remove_dropout..DropoutRemoverrr).kwargsrcsHt|j|tjrt|dkrtdt||dSt|||S)Nrz Expected 1 arg for Dropout, got r)r+ submodulesr:Dropoutr(r6superr')selfrr)rV __class__r!r"r'|s  z2remove_dropout..DropoutRemover.call_module) __name__ __module__ __qualname__r tuplerrGr/rr' __classcell__r!r!r[r"DropoutRemover{s  rb)r,rFrD Transformer transform)r9rQrbr!r!r"rus  r orig_moduler1inputsoutputscs|t}i|D] }||j}||<q|D]}||fdd}||<q|fdd|D|t||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. cs|SNr!)xenvr!r"z"extract_subgraph..csg|]}|qSr!r!).0outputrjr!r" z$extract_subgraph..)r,Graph placeholderr node_copyrolintrE)rer1rfrgrRinputnew_noder%r!rjr"rs     rcCs t|Srh) th_mkldnnMkldnnBatchNorm)a_r!r!r"rls rlcCsi}|D]H}|jdkrLt|jtstdt|j||j}t|tvrLtt||tj}t|t j s?tdt|t |||<t |||q|S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r'r5zExpected nn.Module, got )r.r+rr/r6r0 mkldnn_maprDfloatr:ModulerBrCr)r1r& old_modulesr% cur_moduler4r!r!r"rs      rrcCsZ|D](}|jdkr*t|jtstdt|j||j}||vr*t||||qdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. r'r5N)r.r+rr/r6r0r)r1r&rr%rr!r!r"rs   rc@seZdZdejfddZdS)rfx_graphcCs||_g|_g|_g|_dSrh)rr1 start_nodes end_nodes)rZrr!r!r"__init__s zMklSubgraph.__init__N)r]r^r_r,rrrr!r!r!r"rsr rcs*dddtdtffdd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) NrIrcs|j}dur|jj|jjtdd|Dtttj dd|j D}t |j ||fdd}|fdd}t jj t|fdd}||kS) NcSsg|]}t|jqSr!)rDrandnshapernr%r!r!r"rpsz@gen_mkl_autotuner..use_mkl_heuristic..cSsg|]}|jdqS)r)r)rr!r!r"rpscs<tD]}|qt}tD]}|qt|Srh)rangetime)fr{begin)iterswarmupr!r" benchmarks   z?gen_mkl_autotuner..use_mkl_heuristic..benchmarkcsddddDDS)NcSg|]}|qSr!)to_densernir!r!r"rp szRgen_mkl_autotuner..use_mkl_heuristic....cSrr!) to_mkldnnrr!r!r"rp rqr!r! sample_inputs submoduler!r"rl sz>gen_mkl_autotuner..use_mkl_heuristic..csSrhr!r!rr!r"rlrm)rr owning_modulerr propagaterlistr,r-rrr1rrIrGrH)rI input_nodes output_argsrmkl_time no_mkl_timeexample_inputsrQrrrrr"use_mkl_heuristics&  z,gen_mkl_autotuner..use_mkl_heuristic)rbool)rrrrr!rr"rs "rrIcCst|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )r(r1)rIr!r!r"rsrc@sFeZdZddZdefddZdedefddZd ed efd d Zd S)rcCsdg||_dg||_dS)Nrrsize)rZnr!r!r"r%s zUnionFind.__init__vcCs||j|<d|j|<dS)Nrr)rZrr!r!r"make_set)s zUnionFind.make_setrcCsF|j|}||kr |S|durtd|||j|<tt|j|S)NzParent is None)rr6findrint)rZrparr!r!r"r-s zUnionFind.findrzbcCsf||||}}||kr|S|j||j|kr ||}}||j|<|j||j|7<dSrh)rrr)rZrzrr!r!r"join6s  zUnionFind.joinN)r]r^r_rrrrrr!r!r!r"r$s  r pass_configtracerc sHdddtid}|duri}|||drt|}|dr#t|}|ddur+|St|dts6td d|dvr@td |dd}|}|t |t |j t| }Gd d d t}tjD]}|j} |jd kr||j} t| tvr|j} t| d} | dur| jtjkrtd| jtdkrtdn|jdkr|jtvr|j} n|jtvr|j} | |jkr"| |jkrt dd|j!Dsqk"|t #|j!fdd} Wdn1swYt$t%t j&j'| |_!(|)dd|f} |*| |f| _!Wdn 1swYqkt+tj|}|_,jD]B}|jdkrr|jdkrr|j!d}t|j-}|D]}|jdkrc|jdkrc|*|.|qKt/|j-dkrr.|q1t/j}t0|fddt1jD]{\}}|jdkr|jdkr||_23|q|jdkr|jdkrƈ|j!ddurtd|j!d|_4qfdd|j5D}t/|dkrِqt dd|Drtd t6|}|d|_7|d!dD] }8|d|qqt9fd"d}jD]9}t:|d#r#|;|j7j<|t:|d$r5|;|j2j=<|t:|d%rG|;|j4j><|q|?D](}||st|j=|j>D]}|j!d}|*|.|qZt@|j||qMd}jD]}|jdks|jdkr|d!7}q{tABtCDd&|Et |}|S)'a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fusermkldnn_layout_optimizeNrrrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configc@seZdZdZdZdZdS)z*optimize_for_inference..MklSupportrrN)r]r^r_NOYESUNKNOWNr!r!r!r" MklSupportlsrr'z)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functioncss|]}|jdkVqdS)rN)r)rnargr!r!r" sz)optimize_for_inference..csd|fS)Nr) call_methodrrr!r"rlsz(optimize_for_inference..rrrrcs0t|dr |jSt|dr|jSdS)Ncolor start_color)hasattrrrrr)ufr!r" get_colors    z)optimize_for_inference..get_colorz!Expected color for to_dense inputcs,g|]}t|tjr|dur|qSrh)r+r,r-r)rr!r"rps  z*optimize_for_inference..css|]}|duVqdSrhr!rr!r!r"rszFound None in cur_colorsrcstSrh)rr!rr!r"rlrmrr end_colorzmkldnn conversions: %s)Frupdaterrr+rG RuntimeErrortracerBrCr,rErootrHrrr1rr.rr0mkldnn_supportedrnext parametersdtyperDr}r6devicemkldnn_supported_unknownranyr)inserting_beforemap_argrr`r%rinserting_after create_noderLrrrJrMr(r enumeraterrrall_input_nodessortedrrrrrappendrrvaluesrlogging getLoggerr]inforu)r9rrdefault_pass_configr cur_tracerr&rr%supports_mkldnnrsample_parameter mkldnn_argsdense_xrprv_noderJuser num_nodescur_idx cur_colors other_color mkldnn_graphsrIprvmkldnn_conversionsresultr!)rrrr"r@s                                       r)FF)rr)LrBroperatorr collectionsrcollections.abcrenumrtypingrrrrDtorch.fxr,torch.nnr:torch.nn.functional functionalFtorch.utils.mkldnnutilsmkldnnrx torch.fx.noderr torch.fx.passes.shape_propr torch.nn.utils.fusionr r __all__r/r`r#r0r-rGr r~rrrrrr=rAr>ReLU MaxPool2d AvgPool2dAdaptiveAvgPool2drelu transposesigmoid avg_pool2dadaptive_avg_pool2draddmulr MkldnnConv2d MkldnnLinearr|rrrrrrrTracerrr!r!r!r"s            (   "   1