}YhE ddlZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z ddlZddlmZddlmZddlmcmZddlmcmZddlmZmZddlmZddlm Z m!Z!gd Z"d e#d e$e#e#ffd Z%d ee&dej'de(e#e ffdZ)dej'de(e#e fdejj*fdZ+d,dejj*d ejj*fdZ,dej*d ej*fdZ-dej*de.ej'de.ej'de.ej'fdZ/ej0ej1ej2ej3ej4ej5ej6ej7ej8ej9ej7ej:ej;g ZgZ?ej0ej@ej1ejAej2diZBde.ej'de(e#ej*ffdZCde.ej'de(e#ej*fde(ej*ej*ffdZDGd d!ZEd-d$ZFd%eEd eGfd&ZHGd'd(ZIdejJfdejj*d)e e(e#e fd*e&ejJd ejj*fd+ZKdS).N) defaultdict)Iterable)Enum)AnycastOptional)ArgumentTarget) ShapeProp)fuse_conv_bn_evalfuse_linear_bn_eval) matches_module_patternreplace_node_modulefuseremove_dropoutextract_subgraphmodules_to_mkldnn reset_modules MklSubgraphgen_mkl_autotuneruse_mkl_length UnionFindoptimize_for_inferencetargetreturncP|dd^}}|r|dnd|fS)zp Splits a qualname into parent path and last atom. For example, `foo.bar.baz` -> (`foo.bar`, `baz`) .r)rsplit)rparentnames t/var/www/tools.fuzzalab.pt/emblema-extractor/venv/lib/python3.11/site-packages/torch/fx/experimental/optimization.py _parent_namer$%s3 MM#q))MVT &6!99B ,,patternnodemodulescjt|jdkrdS|jd|f}t||D]x\}}t|tjsdS|jdkrdSt|jtsdS|j|vrdSt||j|urdSydS)NrF call_moduleT) lenargszip isinstancefxNodeoprstrtype)r&r'r(nodes expected_type current_nodes r#rr/s 49~~u"&)A,!5E'*7E':':  # |,00 55 ?m + +55,-s33 55  g - -55  +, - -] B B55 C 4r% new_modulect|jtsJt|j\}}|||j<t ||||dSN)r.rr2r$setattr)r'r(r7 parent_namer"s r#rrCsY dk3 ' ''''$T[11K%GDK GK $ 33333r%Fmodelctjtjftjtjftjtjftjtjfg}|stj |}|rt|tj j stj|}n|}t|}tj |j}|D]}|jD]}t'|||rt)|jdjdkr8||jdj} ||j} | jse|dtjtjtjfvrt3| | } nt5| | } t7|jd|| ||jd||tj ||S)z Fuses convolution/BN and linear/BN layers for inference purposes. Will deepcopy your model by default, but can modify the model inplace as well. rr)nnConv1d BatchNorm1dConv2d BatchNorm2dConv3d BatchNorm3dLinearcopydeepcopyr.torchr/ GraphModulesymbolic_tracedict named_modulesgraphr4rr+r,usersrtrack_running_statsr r rreplace_all_uses_with erase_node) r<inplaceno_tracepatternsfx_modelr( new_graphr&r' first_layerbn fused_layers r#rrLs BN# BN# BN# BN# H % e$$ :eUX-ABB$U++8))++,,G hn--I++O + +D%gtW== +ty|)**Q..%dil&9: T[)-1:")RY !BBB"3K"D"DKK"5k2"F"FK#DIaL';GGG**49Q<888$$T*** + >(I . ..r%ctj|}Gddtjj}||S)z5 Removes all dropout layers from the module. cPeZdZdedeedfdeeefdeffd Z xZ S)&remove_dropout..DropoutRemoverrr,.kwargsrct|j|tjrt |dksJ|dSt |||S)Nrr)r. submodulesr>Dropoutr+superr*)selfrr,r] __class__s r#r*z2remove_dropout..DropoutRemover.call_module{sY$/&12:>> A4yyA~~~~Awww**64@@@r%) __name__ __module__ __qualname__r tupler rKr2rr* __classcell__)rcs@r#DropoutRemoverr\zs} A  A(-hm(< AFJ3PS8n A  A A A A A A A A A Ar%ri)r/rJrH Transformer transform)r<rUris r#rrtsi ''HAAAAA-AAA >( # # - - / //r% orig_moduler4inputsoutputscPtj}i|D]!}||j}||<"|D] }||fd}||<!|fd|D|tj||S)z Given lists of nodes from an existing graph that represent a subgraph, returns a submodule that executes that subgraph. c|Sr9)xenvs r#z"extract_subgraph..s s1vr%c g|] }| Srqrq).0outputrss r# z$extract_subgraph..s888fc&k888r%)r/Graph placeholderr" node_copyrwlintrI) rlr4rmrnrVinputnew_noder'rss @r#rrs I"$C((44E &&t-=-=-=-=>>D  8888888999 NN >+y 1 11r%c*tj|Sr9) th_mkldnnMkldnnBatchNorm)a_s r#rtrts!:1!=!=r%ci}|D]}|jdkrt|jtsJ||j}t |t vrqt t ||t j}t|tj sJtj |||<t||||S)z For each node, if it's a module that can be preconverted into MKLDNN, then we do so and create a mapping to allow us to convert from the MKLDNN version of the module to the original. r*) r1r.rr2r3 mkldnn_maprHfloatr>ModulerFrGr)r4r( old_modulesr' cur_moduler7s r#rrs /1K?? 7m # #dk3// / / / -JJ:--'Z(8(89*ekRR !*bi88888*.- *C*C J'#D':>>> r%rc|D]Q}|jdkrDt|jtsJ||j}||vrt ||||RdS)za Maps each module that's been changed with `modules_to_mkldnn` back to its original. r*N)r1r.rr2r)r4r(rr'rs r#rrsuLL 7m # #dk3// / / / -J[((#D';z3JKKK LLr%c$eZdZdejfdZdS)rfx_graphc>||_g|_g|_g|_dSr9)rr4 start_nodes end_nodes)rbrs r#__init__zMklSubgraph.__init__s#  $& *,(*r%N)rdrerfr/ryrrqr%r#rrs1+++++++r%r rcHdddtdtffd }|S)aW This generates a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by running it with the example_inputs. Example usage: heuristic = gen_mkl_autotuner(example_inputs, iters=10) fast_model = optimization.optimize_for_inference(model, heuristic) NrMrc|j} :|jj |jj t d|Dt ttj d|j D}t |j || fd}|fd}tjj t |fd}||kS)Nc@g|]}tj|jSrq)rHrandnshapervr's r#rxz@gen_mkl_autotuner..use_mkl_heuristic..s$IIITTZ00IIIr%c(g|]}|jdS)r)r,rs r#rxz@gen_mkl_autotuner..use_mkl_heuristic..s*T*T*TD49Q<*T*T*Tr%ctD] }| tj}tD] }| tj|z Sr9)rangetime)frbeginiterswarmups r# benchmarkz?gen_mkl_autotuner..use_mkl_heuristic..benchmarksc6]]  IKKE5\\  9;;& &r%c6ddDDS)Nc6g|]}|Srq)to_denservis r#rxzRgen_mkl_autotuner..use_mkl_heuristic....s-!" r%c6g|]}|Srq) to_mkldnnrs r#rxzRgen_mkl_autotuner..use_mkl_heuristic....s 1W1W1WA!++--1W1W1Wr%rq sample_inputs submodulesr#rtz>gen_mkl_autotuner..use_mkl_heuristic..s7&/i1W1W1W1W1W&Xr%cSr9rqrsr#rtz>gen_mkl_autotuner..use_mkl_heuristic.. s =(Ar%)rr owning_modulerr propagaterlistr/r0rrr4rrMrKrL) rM input_nodes output_argsrmkl_time no_mkl_timerrexample_inputsrUrrrs @@r#use_mkl_heuristicz,gen_mkl_autotuner..use_mkl_heuristics8'  ~3H.4K h   ) ). 9 9 9II[III 4=*T*TEO*T*T*TUU $Xu{KUU  ' ' ' ' ' '9         O !4 (?(?(A(A#B#BK    i A A A A ABB +%%r%)rbool)rrrrrUrs``` @@r#rrs[HK&&&&&&&&&&&&> r%rMc2t|jdkS)z This is a heuristic that can be passed into `optimize_for_inference` that determines whether a subgraph should be run in MKL by checking if there are more than 2 nodes in it )r+r4)rMs r#rrs u{  a r%c@eZdZdZdefdZdedefdZdedefdZd S) rc2dg|z|_dg|z|_dS)Nrr!size)rbns r#rzUnionFind.__init__s ,06A: !sQw r%vc.||j|<d|j|<dS)Nrr)rbrs r#make_setzUnionFind.make_set s A ! r%rc|j|}||kr|S|J|||j|<tt|j|Sr9)r!findrint)rbrpars r#rzUnionFind.find$sOk!n 88H3 ACQ(((r%rbc||||}}||kr|S|j||j|kr||}}||j|<|j|xx|j|z cc<dSr9)rrr!)rbrrs r#joinzUnionFind.join,szyy||TYYq\\1 66H 9Q<$)A, & &aqA A !  ! $ r%N)rdrerfrrrrrrqr%r#rrs'''#)c)c))))%c%c%%%%%%r%r pass_configtracercdddtid}|i}|||drt|}|drt|}|ddur|St |dt st d d|dvrt d |dd}|}|tj |tj |j t | }Gd d t}tjD]}|j} |jd kr||j} t)| t*vrq|j} t/| d} | F| jt4jks Jd| jt5jdks Jdn6|jdkr+|jt*vr|j} n|jt:vr|j} | |jkr| |jkr t?d|j Ds!|5tj"|j fd} dddn #1swxYwYtGtHtj%j&| |_ '|5(dd|f} |)| |f| _ dddn #1swxYwYtUtj|}|_+jD]}|jdkr|jdkr|j d}t|j,}|D]B}|jdkr5|jdkr*|)|-|Ct]|j,dkr-|t]j}t_|fdtajD] \}}|jdkr(|jdkr||_12|9|jdkr?|jdkr4|j dJ|j d|_3fd|j4D}t]|dkrt?d|DrJtk|}|d|_6|ddD]}7|d| tqfd}jD]}ts|dr8|:|j6j;|ts|dr8|:|j1j<;|ts|dr8|:|j3j=;||>D]l}||s_|j<|j=zD]9}|j d}|)|-|:t|j||md}jD]}|jdks |jdkr|dz }tjAtCd |Dtj |}|S)!a Performs a set of optimization passes to optimize a model for the purposes of inference. Specifically, the passes that are run are: 1. Conv/BN fusion 2. Dropout removal 3. MKL layout optimizations The third optimization takes a function `use_mkl_heuristic` that's used to determine whether a subgraph should be explicitly run in MKL layout. Note: As FX does not currently handle aliasing, this pass currently assumes nothing aliases. If that isn't true, use at your own risk. T heuristic) conv_bn_fusermkldnn_layout_optimizeNrrrFz+mkldnn_layout_optimize config is not a dictz4Heuristic not found in mkldnn_layout_optimize configceZdZdZdZdZdS)*optimize_for_inference..MklSupportrrN)rdrerfNOYESUNKNOWNrqr%r# MklSupportrbs r%rr*z)this pass is only for torch.float modulescpuz!this pass is only for CPU modules call_functionc3,K|]}|jdkVdS)rN)r)rvargs r# z)optimize_for_inference..s)II3:3IIIIIIr%c2d|fS)Nr) call_method)rrs r#rtz(optimize_for_inference..s)=)=kA4)P)Pr%rrrrct|dr|jSt|dr|jSdS)Ncolor start_color)hasattrrrr)rufs r# get_colorz)optimize_for_inference..get_colorsT 1g   $7717## # 1m $ $ *771=)) )tr%cpg|]2}t|tjr|'|3Sr9)r.r/r0)rvrrs r#rxz*optimize_for_inference..sQa))9Q<<+ ! ,++r%c3K|]}|duV dSr9rqrs r#rz)optimize_for_inference..s&9919999999r%rc"tSr9)r)rsr#rtz(optimize_for_inference..s H@U@Ur%rr end_colorzmkldnn conversions: %s)Erupdaterrr.rK RuntimeErrortracerFrGr/rIrootrLrrr4rr1rr3mkldnn_supportedrnext parametersdtyperHrdevicemkldnn_supported_unknownranyr,inserting_beforemap_argrrgr'r inserting_after create_noderPrrrNrQr+r enumeraterrrall_input_nodessortedrrrrrappendrrvaluesrlogging getLoggerrdinfor|)r<rrdefault_pass_configr cur_tracerr(rr'supports_mkldnnrsample_parameter mkldnn_argsdense_xrprv_noderNuser num_nodescur_idx cur_colors other_color mkldnn_graphsrMprvmkldnn_conversionsresultrrrs @@@r#rr6sX&#."?  {+++>*U +,&u%%34== )*BCT J JJHIII-.FGGGQRRR+,DEkRJ e 4 455HN:?H---$()<)<)>)>$?$?GTX^$$"'"'$- 7m # # -JJ#333",.#' (=(=(?(?#F#F #/+1U[@@@CA@@,2el56I6IIII;JIIW ' '{...",. 888","4 jm + +*"444IItyIIIII**400   jIPPPP                 U27#34kBBDI))$// ' '"..}j4'RR**7333 $w  ' ' ' ' ' ' ' ' ' ' ' ' ' ' ' $D$8$8'BBK&H * * 7m # # z(A(Ay|H$$E . .7m++ {0J0J..x888''---4:!####D)))HN##I 9  B$#8>2244  7m # # {(B(B&D  KK W % %$+*C*C9TYq\**666&Yty|44DNN-J:!##99j99999 9 9 9 ++J#ADJ)!""~ 4 4  1 {3333 4-88U8U8U8U,V,VMJJ 4 ! ! B "''$*-- . 4 ; ;D A A A 4 ' ' N "''$"233 4 @ G G M M M 4 % % J "''$.11 2 < C CD I I I%%''==  '' =)EO; * *il**3///##D)))) %+w < < <$$ ;+ % % )B)B ! #  h$$%=?QRRR MMOOO ^E8 , ,F Ms$4JJ" %J" -6L//L3 6L3 )FF)rr)LrFroperatorr collectionsrcollections.abcrenumrtypingrrrrHtorch.fxr/torch.nnr>torch.nn.functional functionalFtorch.utils.mkldnnutilsmkldnnr torch.fx.noder r torch.fx.passes.shape_propr torch.nn.utils.fusionr r __all__r2rgr$r3r0rKrrrrrrrrArErBReLU MaxPool2d AvgPool2dAdaptiveAvgPool2drelu transposesigmoid avg_pool2dadaptive_avg_pool2draddmulr MkldnnConv2d MkldnnLinearrrrrrrrrTracerrrqr%r#r+s|  ######$$$$$$&&&&&&&&&& &&&&&&&&&********000000HHHHHHHH    --sCx---- d^#%759#s(^(4 '4 cN48=4444%/%/%/58?%/%/%/%/P0")0 0000&22 =2 M2"'] 2222.IINGLL J O MFL&%L(,7Iy%Iy%N== T"']T#ry.5I&L =L #ry. !Lbi*+LLLL"++++++++++++\ + $    %%%%%%%%:-1 ipp 8?p$sCx.)p Op X_ ppppppr%