o ei@sHUddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlZddlmZddlmZddlmZm Z m!Z!ddl"m#Z#ddlm$Z$m%Z%ddl&m'Z'dd lm(Z(dd l)m)Z)m*Z*dd l+m,Z,dd l-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5dd l6m7Z7m8Z8ddl9Z9ddl:m;mZ?ddl9m@Z@mAZAddlBmCZCddlDmEZEddlFmGZGmHZHmIZImJZJddlKmLZLmMZMmNZNddlOmPZPmQZQmRZRddlSmTZUddlVmWZWmXZXddlYmZZZddl[m\Z\m]Z]m^Z^m_Z_m`Z`maZambZbmcZcmdZdmeZemfZfmgZgddlhmiZiddljmkZkmlZlmmZmmnZnmoZoddlpmqZqmrZrddlsmtZtddlumvZvmwZwddlxmyZymzZzm{Z{m|Z|m}Z}m~Z~ddlmZddlmZdd lmZmZmZdd!lmZdd"lmLZdd#lmZmZmZdd$lmZmZdd%lmZdd&lmZmZmZdd'lmZd(d)lmZd(d*lmZd(d+lmZd(d,lmZd(d-lmZd(d.lmZe4d/Ze3r9dd0lmZmZmZmZdd1lmZd(d2lmZd(d3lmZd(d4lmZd(d5lmZd(d6lmZmZd(d7lmZmZd(d8lmZmZd(d9lmZd(d:lmZejd;kZeLjZe9jĠed<Ze9jĠed=Ze eơZdd@dAZddCdDZddGdHZGdIdJdJZGdKdLdLe΃ZGdMdNdNe΃ZddOdPZddSdTZdddYdZZ Uddd_d`ZdddcddZGdedfdfZ U V U dddjdkZאddmdnZ o odddudvZejGdwdxdxZېddzd{ZܐdddZGdddejރZߐdddZdddZed ddZdddZejGdddZGdddeZGdddZd ddZd ddZGddde0eZejGdddeZGdddeeZejd ddZezGdddZGdddZdaded<d ddZe jewdZe jedZejddd„ZdddĄZddddDŽZezGddɄdɃZddd΄ZezGddЄdeZezGdd҄deZezGddԄdeZdddلZddd܄ZezGddބdރZdddZed ddZGdddZe$dddZGdddZezGdddeZ ezGdddZ GdddZ Gddde Z Gddde Z dS() annotationsN) bisect_right)copy)c_void_pCDLLcdll) timedelta) lru_cachepartial)Path)_TemporaryFileWrapper)timetime_ns) ModuleType)AnycastGenericNoReturnOptional TYPE_CHECKINGTypeVarUnion)overrideSelf)SymIntTensor)get_interface_for_device) SkipFrame)CompileEventLoggercounters dynamo_timedget_metrics_context)configexcmetrics)custom_backend_codegen_configscustom_backend_passesinit_backend_registration) compile_utils)rocm_compile_command rocm_compiler)in_toplevel_process) _LINKER_SCRIPT_set_gpu_runtime_env _TORCH_PATHconvert_cubin_to_obj CppBuilder CppOptionsCppTorchDeviceOptionsget_compiler_version_infoget_ld_and_objcopy&get_name_and_dir_from_output_file_pathnormalize_path_separatorrun_asm_build_object) pick_vec_isa)CustomGraphModulePassCustomGraphPassCustomGraphPassTypeCustomPartitionerFnCustomPartitionerFnType)has_frozen_paramsis_frozen_param)_reload_python_module) cache_dirdefault_cache_dir) ALIGN_BYTESclear_on_fresh_cachedetermine_aoti_mmap_flagsis_linux is_windowsXPU_KERNEL_FORMAT)FakeScriptObject)trace_structured)extract_tensor_metadata FakeTensorTensorMetadata)log_cache_bypass)r") CacheArtifactCacheArtifactFactoryCacheArtifactManager)TensorPropertiesWeights)CUSTOM_OBJ_FILENAME_PREFIX)has_hintShapeEnv size_hint) OrderedSetCompiledFxGraph) create_cache)autotune_cache)AutotuneCacheBundler) TritonBundler)VT)Callable GeneratorKeysViewSequence)Future)_CompileFxKwargs)BuildOptionsBase GraphLowering) ChoiceCaller)CompiledFxGraphConstants OutputCode) JsonDataTy RemoteCache)HalideInputSpec HalideMeta)CachingAutotuner) InputTypewin32 output_code autotuningreturnstrcCstjjdurdSdS)N cubin_path hsaco_path)torchversionhipr~r~c/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/_inductor/codecache.pyget_cpp_wrapper_cubin_path_namesrdevicecCs,|dkrtjjdur dSdS|dkrtSdS)Ncudacubinhsacoxpu)r{r|r}rHrr~r~rget_kernel_bin_formats r device_typedict[str, str]cCs4tjtttjjtt | d}|S)zJ Gets all the current device information used to compile the .so. ) AOTI_PLATFORM AOTI_MACHINE AOTI_CPU_ISAAOTI_COMPUTE_CAPABILITY) sysplatformmachinerxr{ _inductor cpu_vec_isar8upperrget_compute_capability)rmetadatar~r~rget_device_informations rc@sVeZdZeejdddZeeejdddZdd d Z dd d Z dddZ dS) CacheBaserwdict[str, Any]c Csddlm}m}|r|}nd}z9ddid|id}tjtj}tjjdur9|j|dd<tjj|dd<n|j |dd<tjj |dd <Wn t t fyVi}Ynwt tj|d d d |d <|S)Nr) HAS_TRITON triton_keynametriton)rr|rr|rr}T) sort_keysutf-8hash)%torch._inductor.runtime.triton_compatrrr{rget_device_propertiescurrent_devicer|r gcnArchNamer}AssertionError RuntimeErrorhashlibsha256jsondumpsencode hexdigest)rrtriton_versionsystemdevice_propertiesr~r~r get_systems6 zCacheBase.get_systemr cCsttjtdtdS)Ncacher)r ospathjoinrArrr~r~r~rget_local_cache_pathszCacheBase.get_local_cache_pathNonecCst|_dSN)rrrselfr~r~r__init__szCacheBase.__init__cCsT|}|s iSt|}t|}Wd|dS1s!wY|dS)Nr)ris_fileopenrload)rlocal_cache_pathlocal_cache_fp local_cacher~r~rget_local_caches   zCacheBase.get_local_cachercCs0|}tt|tj|j|ddddddS)N)rrindentT make_dirs)r write_atomicrxrrr)rrrr~r~rupdate_local_caches  zCacheBase.update_local_cacheN)rwr)rwr rwr)rrrwr) __name__ __module__ __qualname__ staticmethod functoolsrrrDrrrrr~r~r~rrs$  rc@s eZdZd ddZdd d Zd S) LocalCachekeysrxrwdict[str, Any] | NonecGs0|}|}|D] }||vr||}qdS|Sr)r)rrr sub_cachekeyr~r~rlookups zLocalCache.lookupvaluerrcGsL|}|}|ddD] }||i||}q |||d<||dS)Nr)r setdefaultr)rrrrrrr~r~r set_values   zLocalCache.set_valueN)rrxrwr)rrxrrrwr)rrrrrr~r~r~rrs  rc@seZdZ ddd dZdS)PersistentCacheNchoiceslist[ChoiceCaller]oprxinputs benchmark1Callable[[Any], dict[ChoiceCaller, float]] | None hint_override int | Nonerwdict[ChoiceCaller, float]c st|dur|d|n|id fdd }tjr%|ni}||sl|durl|tfd d Ds@J|i|iiD]\}} | ||<qV| |S) a Check to see if we have benchmarked the given choice callers. For each choice caller: 1. Check local_cache[op][inputs][choice][precision], return benchmark if cached. 2. If benchmark is not None: a. `max_autotune_gemm=True`: benchmark the choice, update local_cache[op][inputs][choice], and return the benchmark. b. `max_autotune_gemm=False`: don't benchmark the choice, return nothing. N_rrrwboolcs\d}D]'}|}||iiivr'|||<qd}|S|S)z2Check if `cache` contains data for all the choicesTF)hash_keyget)rhitchoice choice_hash cache_keyrr precisiontimingsr~r check_cache#sz+PersistentCache.lookup..check_cachec3s|]}|vVqdSrr~).0r)rr~r 7z)PersistentCache.lookup..)rrrwr) r{get_float32_matmul_precisionr"autotune_local_cacherallritemsrr) rrrrrrrrrtimingr~rrr s  zPersistentCache.lookupr) rrrrxrrxrrrrrwr)rrrrr~r~r~rr srcCs.tjtd}tj|stj|dd|S)NlocksTexist_ok)rrrrAexistsmakedirs)lock_dirr~r~r get_lock_dirBs rdatabytescCs&tt|dddS)N3r)base64 b32encoderrdigestdecodelower)rr~r~r sha256_hashIs&rrcode str | bytesextracCsLt|tr|n|d}|r t|tr|n|d}|d|}dt|S)Nrs||c) isinstancerrr)rr  hashing_strextra_br~r~r code_hashNs   rbasename extension specified_dirtuple[str, str, str]cCsb|rtj|r |}ntjt|}n tjt|dd}tj||d|}|||fS)NrY.)rrisabsrrA)rrrsubdirrr~r~rget_pathVs  rcontent hash_typecCs:|dvr t||S|ddthvrtt|Std|)N>ptxspvramdgcnrrzUnknown hash type )rrHreprr)rr rr~r~rget_hashds   rc@s:eZdZdZ dddddd d ZdddZdddZdS)WritableTempFileao Avoid "Permission denied error" on Windows: with tempfile.NamedTemporaryFile("w", suffix=".gv") as temp_file: # Not writable on Windows: # https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile Example: with WritableTempFile("w", suffix=".gv") as temp_file: tree.to_dotfile(temp_file.name) wN)encodingsuffixmoderxr!rr"rwrcCs||_||_||_dSr)r#r!r")rr#r!r"r~r~rrxs zWritableTempFile.__init___TemporaryFileWrapper[Any]cCs tj|j|j|jdd|_|jS)NF)r!r"delete)tempfileNamedTemporaryFiler#r!r" temp_filerr~r~r __enter__szWritableTempFile.__enter__exc_typeexc_valexc_tbc CsP|jz t|jjWdSty'}z trn|WYd}~dSd}~wwr)r(closerunlinkrOSError _IS_WINDOWS)rr*r+r,er~r~r__exit__s zWritableTempFile.__exit__)r )r#rxr!rr"rrwr)rwr$)r*rr+rr,rrwr)rrr__doc__rr)r2r~r~r~rrls  rr str | Nonetuple[str, str]c CsL|dur t|||}t|||\}}}tj|s"t||dd||fS)NTr)rstriprrrrr) rrr rrrr_subdirrr~r~rwrites  r8textcCst|ddS)zT Write the `text` to a file and return the path computed based on the hash. txtrYr8)r9r~r~r write_textsr<Fpath_rr encode_utf_8rcCst|ttfs Jdt|}|r|jjddd|jdtdt d}t|tr0dnd}|j ||r9dndd  }| |Wdn1sMwYz |j |d WdSt yttsetj||d t|YdSw) Nz6Only strings and byte arrays can be saved in the cacheT)parentsrrz.tmpr wbrr!)target)srcdst)r rxrr parentmkdirrgetpid threading get_identrr8renameFileExistsErrorr0shutilcopy2remove)r=rrr>rtmp_path write_modefr~r~rrs&   rc@s"eZdZUdZded<ded<dS)TensorMetadataAndValueszk TensorMetadata plus the elements as a list of raw values. Used for hashing inlined constants. rMtensor_metadata list[Any]valuesNrrrr3__annotations__r~r~r~rrRs  rRxcC|Srr~rXr~r~r_identr[trrMcCs&t|}t|dstj|ddd}|S)zs Extracts the tensor metadata and removes fields of the TensorMetadata that are not needed for caching _is_inductor_staticrN)storage_offset storage_bytes)rKhasattr dataclassesreplace)r]metar~r~r%extract_tensor_metadata_for_cache_keys recseZdZdZ d/d0fd d Zd1ddZd2ddZd3ddZd4ddZd5ddZ d6d!d"Z d7d%d&Z d8d(d)Z d9d-d.Z ZS):FxGraphCachePicklera: Custom pickler to customize the pickling of some objects (Tensors), only for the purpose of computing a hash for keying into the FxGraphCache. Tensors contain objects that don't pickle and/or vary between runs, and we want to capture the data that allow us to compute a stable, but safe hash. Fgmtorch.fx.GraphModulehas_user_defined_triton_kernelsrrwrcst|_t|jtj|_|jt t |j t jt |jt jjjt |jt jt |jt jjjjt |jtt |ji|rQt |j|j|j<d|_dS)a2 Create an FX graph pickler. If include_non_inlined=True, then pickling will include the _values_ for all Tensors. (Note that any tensors are constants attached as attributes to the GraphModule). Otherwise, pickling will include only the metadata for these tensors. TN)ioBytesIO_streamsuperrcopyregdispatch_tablerupdaterLrr _reduce_fake_tensorr{r_reduce_tensornn parameter Parameterr_reduce_symintfx experimental_backward_state BackwardState_reduce_unsupportedrI_reduce_fake_script_object_reduce_graph_module __class__fast)rrgrir~r~rrs&      zFxGraphCachePickler.__init__r]r.tuple[Callable[[T], T], tuple[TensorMetadata]]cCst|}t|ffS)z7 Custom reducer to pickle FakeTensors. )rer[)rr]rr~r~rrqs z'FxGraphCachePickler._reduce_fake_tensorHtuple[Callable[[T], T], tuple[TensorMetadata | TensorMetadataAndValues]]cCsddlm}|jr tdt|}t|r||st|ffSt}| }t|}|dkr9t d|ddtt ||ffS)z Custom reducer to pickle Tensors. If we see tensors, we know they're constants stored as attributes on the GraphModule. rYrizmkldnn tensors unpickleableg?z0FX graph cache copying of a large constant took z.1zs. Please file an issue.) graphrj is_mkldnnBypassFxGraphCacherer?can_inline_constantr[r tolistwarningswarnrR)rr]rjrstartrUelapsedr~r~rrrs    z"FxGraphCachePickler._reduce_tensorsr#tuple[Callable[[T], T], tuple[str]]cCstt|ffS)z3 Custom reducer to pickle SymInts. )r[rxrrr~r~rrv=sz"FxGraphCachePickler._reduce_symintrrcCtd)z{ Custom reducer to handle any objects that we don't support and therefore raise to bypass caching. zReduce unsupported)rrr~r~rr{Fz'FxGraphCachePickler._reduce_unsupported&tuple[Any, tuple[dict[str, Any], str]]cCsH|\}\}}|d}tdd|}tdd|}||d<|||ffS)a Custom reducer for graph module to handle irrelevant data for user defined triton kernels Essentially what we are doing here is a huge hack where user defined triton kernel contain a dynamo time side table and the arguments to the call_function are indices into this side table. These arguments are not for hashing purposes since we included the source code into the cache key and the numbers are prone to give false negatives due to ordering. _codezkernel_idx = \d+rzconstant_args_idx = \d+) __reduce__resub)rrgfnrimportsrr~r~rr}Ms   z(FxGraphCachePickler._reduce_graph_modulerI*tuple[Callable[..., Any], tuple[Any, ...]]cCsR|jdurt|j}t|rt|rt|st|jffSt|j|j|jffSr) real_objtype opaque_objectis_opaque_type should_hoist has_membersr[script_class_name wrapped_obj)rr]clsr~r~rr|`s   z.FxGraphCachePickler._reduce_fake_script_objectobjrc CszZz|||jWW|jd|jdStttjt fy6}z t j dddt d|d}~wt yZ}zdt|vrUdt|vrUt j dddt d|d}~ww|jd|jdw)z< Pickle an object and return a byte string. rzFailed to pickle cache keyTexc_infoNpybind11zis not pickleable)dumprlgetvalueseektruncate TypeErrorAttributeErrorpickle PicklingError ValueErrorlogwarningrrrx)rrr1r~r~rrps$      zFxGraphCachePickler.dumpsrxcCs||}t|S)zE Serialize an object and return a hash of the bytes. )rr)rrserialized_datar~r~rrs zFxGraphCachePickler.get_hashinpFxGraphHashDetails list[str]c sd fdd }g}t|D]k\}}t|tr>tt|D]}||}|d|d|d|d |||qqt|tre|D]\}} | }|d|d|d|d || qGq|}|d|d|d ||q|S) z Get a printable string describing in more detail all the attributes comprising an object. Useful for debugging when one graph hashes to a different value than another. rrrwrxcst|tjr tt|St|tr(|jddd}t|dkr |S|dddSt|j vr.get_str[z] z]: z: Nrrrwrx) varsrr listrangerrappenddict) rrrlinesattrriihkvr~rr debug_liness   ,  ( "zFxGraphCachePickler.debug_linesF)rgrhrirrwr)r]rrwr)r]rrwr)rrrwr)rrrwr)rgrhrwr)r]rIrwr)rrrwrr)rrrwr)rrrr3rrqrrrvr{r}r|rrr __classcell__r~r~rrrfs  % "    rfrootslist[str] | Noneprefixhasher hashlib._Hashc Cstt||dddD]L}|j|jd}|dusJ|j}|dus%Jt|d}||j d|| Wdn1sEwY|j rXt |j |jd|q dS)NcSs|jSr)rrZr~r~rz!build_code_hash..rrbrr)sortedpkgutil iter_modules module_finder find_specroriginrrprreadispkgbuild_code_hashsubmodule_search_locations)rrrlibspecmodulerQr~r~rrs   rfuncCallable[[], bytes]cs@gd fdd }d fdd }dfd d }||_||_|S)z This function is a reimplementation of functools.lru_cache with a set function that allows prepopulating the cache. rwrcs tdkr dSNrrrr~_cacherr~rwrappers  z torch_key_cache..wrapperrrcstdksJ|dSrr)rrr~rset_valsz torch_key_cache..set_valcs dSr)clearr~rr~rr ztorch_key_cache..clearNrwr)rrrwrr)setr)rrrrr~rrtorch_key_cachesrcCsxtddd,tsddd }|tWd Sd d lm}|d dWd S1s5wYd S)zS Compute a key that contains relevant information about torch source files inductor_codecache_torch_keyF)log_pt2_compile_eventrootrxrwrc sd}tjtfdd|D}t}|tj dt |gd||D]$}tj |rKt |d}|| Wdn1sFwYq'|S)N)z"codegen/aoti_runtime/interface.cppz script.ldcsg|] }tj|qSr~)rrrrrX inductor_rootr~r sz4torch_key..get_code_hash..rrr)rrdirname__file__rrrpr{ __version__rrrrrr)r extra_filesrrrQr~rr get_code_hashs   z torch_key..get_code_hashNr)parutilztorch/src_hash.txtascii)rrxrwr) r r" is_fbcoder.libfb.pyrget_file_contentsrstripr)rrr~r~r torch_keys  $rcCs tjtSr)rrrrr~r~r~rget_inductor_rootrrc@seZdZUdZded<dS)OrderedSetHolderzb See FxGraphHashDetails. Holds a sorted list to support stable hashing of set kwargs. rTrNrVr~r~r~rrs  rc@seZdZdZdS)rzI Exception to indicate that the FxGraphCache should be bypassed. N)rrrr3r~r~r~rr src@s>eZdZdZdgZdd dZdddZdddZdddZdS) rzz Object to capture all the details for a compiled FX graph relevant to computing a safe and stable cache key. graph_idrgrhexample_inputsSequence[InputType] fx_kwargsrginputs_to_check Sequence[int]rwrcCs||_||_tj|_i|_t|D] \}}||jvr3t|t t fvr.t t||j|<q||j|<qddl m }m}m} ddlm} g|_|dur|D][} t| tjjsYqOt| jjd|d| jjd| dD]>} ddlm} || jd}d}t|| r|j rt!tdd |j D}|j"}| |}|#| jd }|j$|||fqkqO||_%t&d d |D }|rtj'(rtj')|_*t+t,tj-j.j/f|_0tj1j2j3j4tj1j2j3j5tj1j2j3j6f|_7t8|_9t:;|_d d |_?|@t=jA|_AtjBj=jC|_D|@t=jE|_E|@t=jF|_F|@t=jG|_G|Ht=jI|_I|Ht=jJ|_JtKtLtM|j@tNO|_NddtPD|_P|Qt=jR|_Si|_TtUV}|durf|jTrhddt|jTdddD|_TdSdSdS)Nr)kernel_side_table triton_kernel_wrapper_functionaltriton_kernel_wrapper_mutation)9user_defined_triton_kernel_transitive_closure_source_code call_function)rrB) Autotuner kernel_idxcss*|]}tdd|DVqdS)css|]}t|VqdSrrx)rkvr~r~rrQrz8FxGraphHashDetails.__init__...N)r all_kwargsr)rr r~r~rrPs  z.FxGraphHashDetails.__init__..constant_args_idxcss|] }t|tjVqdSr)r r{rrr~r~rrfsFignore_private_configscSs&i|]\}}|dur||jddqS)NFr)save_config_portable)rr custom_configr~r~r s  z/FxGraphHashDetails.__init__..cSsi|] \}}t||qSr~r)rsymrr~r~rrscSs t|dSrrrZr~r~rrs z-FxGraphHashDetails.__init__..r)Wrgrcconfig cache_key_tagrrrEXCLUDED_KWARGSrrrXr*torch._higher_order_ops.triton_kernel_wrapr r r torch._inductor.codegen.wrapperr user_defined_triton_sourcemodulesr r{rw GraphModule itertoolschainr find_nodestriton.runtime.autotunerr get_kernelkwargsconfigsrxrget_constant_argsrrany accelerator is_availablecurrent_device_indexdefault_cuda_device_index$are_deterministic_algorithms_enabled-is_deterministic_algorithms_warn_only_enabledutils deterministicfill_uninitialized_memory!deterministic_algorithms_settingsbackendsrmatmulfp32_precision&allow_fp16_reduced_precision_reduction&allow_bf16_reduced_precision_reductioncuda_matmul_settingsr torch_versionrr system_infor"rinductor_config_get_custom_pass_detailpost_grad_custom_pre_pass _functorchbundled_autograd_cacheprecompile_enabledpost_grad_custom_post_passjoint_custom_pre_passjoint_custom_post_pass_get_custom_pass_detail_unsafe_pre_fusion_custom_pass_fuse_ddp_communication_passesr'tuplemapr&rUr%!_get_custom_partitioner_fn_detailcustom_partitioner_fn_custom_partitioner_fnvar_to_hint_override FxGraphCache_get_shape_env)rrgrrrrrr r r r rnoderkernelr( kernel_source constant_argsno_tensor_inputs shape_envr~r~rrs      #       zFxGraphHashDetails.__init__ custom_passr Any | Nonecsf|sdSt|trfdd|DSt|tr|St|tr"|St|r(dStdtt|)Ncsg|]}|qSr~)rFrrr~rrzEFxGraphHashDetails._get_custom_pass_detail_unsafe..zunknown config type: )r rrxr:uuidcallablerrrrWr~rrrFs   z1FxGraphHashDetails._get_custom_pass_detail_unsafe+CustomGraphPassType | CustomGraphModulePasscCs"|sdSt|ttfs J|Sr)r r:r9rZr\r~r~rr>sz*FxGraphHashDetails._get_custom_pass_detailrLr=cCs|sdSt|ts J|Sr)r r<rZ)rrLr~r~rrKsz4FxGraphHashDetails._get_custom_partitioner_fn_detailN) rgrhrrrrgrrrwr)rWrrwrX)rWr]rwrX)rLr=rwrX) rrrr3rrrFr>rKr~r~r~rrs  $ rrgrhrrrrgrrtuple[str, list[str]]c Csft||||}t|jdk}t||}d||}||}d|} td|d| ||fS)z= Generate a unique hash of the FX graph for caching. rrQ z$FX graph cache hash details for key z: ) rrrrfrrrrdebug) rgrrrdetailsripicklerrr debug_strr~r~rcompiled_fx_graph_hashs    rd time_saved_nsintcCs|tjr tjs dSt|d}tr-tjd}t d|||t||d7}t d|t j t|d|S)z} Ephemerally increases the NCCL timeout when compiling for a distributed job Returns amount of seconds increased rgeAz>pytorch/remote_cache:ephemeral_timeout_fudge_factor_percentagezNEphemeral NCCL timeout increase fudge factor %d and original increase value %ddzIncreasing NCCL timeout by %d)seconds)r{ distributedr,is_initializedrfr"r_utils_internaljustknobs_getval_intrinfodistdistributed_c10d"_add_ephemeral_timeout_for_all_pgsr)reincreased_timeout_sec fudge_factorr~r~r.add_ephemeral_timeout_increase_for_distributeds$  rsc@sdeZdZdZed(ddZed)ddZed*ddZed+ddZed,d"d#Z ed-d%d&Z d'S). GuardedCachezJ Mixin for caches that have guards associated with their entries. rtype[GuardedCache[T]]_keyrxrwcCr)Nz.Implement _get_tmp_dir_for_key on parent classNotImplementedError)rrvr~r~r_get_tmp_dir_for_keyz!GuardedCache._get_tmp_dir_for_keyr local_hitr local_miss remote_hit remote_missrcCr)Nz(Implement _record_result on parent classrwrrr{r|r}r~r~r~r_record_results zGuardedCache._record_resultlocal remote_cacheRemoteCache[JsonDataTy] | None,Generator[tuple[T, bytes, bool], None, None]c cs@|rW||}tj|rWtt|D]A}|drqz)ttj||d}| }t ||dfVWdn1s@wYWqt yVt jdddYqw|rz1||}durt|tsjJ|d} t| ttfswJt| }t ||dfVWdSWdSt yt jd|jddYdSwdS) NrrTz,fx graph cache unable to load compiled graphrrFz %s unable to load compiled graph)ryrrrrlistdir startswithrrrrloads Exceptionrrrr rrxrr b64decoder) rrrrrrrQr cache_datarr~r~riterate_over_candidates"sD        z$GuardedCache.iterate_over_candidatesevaluate_guards5Callable[[str, list[int] | list[torch.SymInt]], bool]hints list[int]-tuple[T | None, bytes | None, dict[str, str]]cCsd}d}d}d} d} ||||D]1\} } } t| dsJ| js(| }| }d}nt|| j|} | r=| }| }d}| j} nd}| j} qd|i}| durO| |d<|duoT| }|duo[| }|dusb|oc|}|duok|du}|j|||||d |||fS) aY Find the first cache entry in iterate_over_candidates that passes `evaluate_guards`. Args: key: The cache key to look up local: Whether to check the local cache remote_cache: The remote cache to check, if any evaluate_guards: Function that evaluates whether a guard passes the check, given a list of hint values and the guard expression. hints: List of symint hints paired with evaluate_guards Returns: A tuple of (graph, pickled_content) if found, or (None, None) if not found N full_missF guards_exprr guard_misscache_status_detailedcache_status_guard_expr)r{r|r}r~)rrarrr)rrrrrrrpickled_content result_statussample_guards_exprin_local candidaterrrmr{r}r|r~r~r~rfind_guarded_entryFsL  zGuardedCache.find_guarded_entryrrlist[torch.SymInt]cCsdd|DS)z Get the backed SymInt objects from the input list. Note that we can never have guards that depend on unbacked symint. cSs$g|]}t|tjrt|r|qSr~)r r{rrUrrr~r~rrs$z7GuardedCache._filter_backed_symints..r~)rrr~r~r_filter_backed_symintssz#GuardedCache._filter_backed_symintsShapeEnv | NonecCs"tjj}|r |js dS|jjS)zG Helper to get the shape env from the tracing context. N)r{_guardsTracingContexttry_get fake_moderV)rctxr~r~rrPs  zGuardedCache._get_shape_envN)rrurvrxrwrx)rrurrxr{rr|rr}rr~rrwr) rrurrrrrrxrwr)rrurrxrrrrrrrrrwr)rrurrrwr)rrurwr) rrrr3 classmethodryrrrrrPr~r~r~rrts   # N  rtc@s,eZdZed ddZeed ddZdS) InductorCacheArtifactrwrcCst|j|jdSr)rO_write_to_local_cacherrrr~r~rpopulate_cachesz$InductorCacheArtifact.populate_cacherxcCdS)Ninductorr~r~r~r~rrzInductorCacheArtifact.typeNrrwrx)rrrrrrrr~r~r~rrs  rc@seZdZdZedEddZedFd d ZedGddZedHddZ e dIdJd$d%Z edKd(d)Z edLd,d-Z edMd0d1Z edMd2d3ZedNd:d;ZedOd// - On lookup, we compute the key from the graph details, iterate over all leaf files in the corresponding subdirectory, deserialize the entry, and evaluate its guards expression. If the evaluation succeeds, we have a cache hit. If it fails, we compile the graph and store a new entry. - Finally, on a cache hit, we need to make sure any guards that would have been created during compilation are added to the current context. rwrxcCstjtdS)zS Get the toplevel temporary directory for storing compiled graphs. fxgraph)rrrrAr~r~r~r _get_tmp_dirszFxGraphCache._get_tmp_dirrtype[FxGraphCache]rcCstjt|dd|S)zA Return the disk location for a given cache key. rYr)rrrrOrrrr~r~rrysz!FxGraphCache._get_tmp_dir_for_keyr{rr|r}r~rcCsp|r ttjd|rttjdttjd||r#ttjd|r6ttjdttjd|dSdS)zG Called by GuardedCache to record hit/miss statistics. !inductor_fx_local_cache_hit_count"inductor_fx_remote_cache_hit_count!inductor_fx_remote_cache_hit_keys"inductor_fx_local_cache_miss_count#inductor_fx_remote_cache_miss_count"inductor_fx_remote_cache_miss_keysN)rtry_increment_topleveladd_to_set_toplevelrr~r~rrs> zFxGraphCache._record_resultrr[ cache_infor constantsrl-tuple[CompiledFxGraph | None, dict[str, Any]]csj}r4t|}|}dur4t||d<tjd|jdtjd|jdt|jdkr4ttj dz |dd l m }|j durK| jWn tyYd|fYSwt}jtj|d tjjtd j7<td td tdddfdddtdfdddtdfddfdddtdddfdddtdddfdddtrԈjrt dj|fS)ah Cache specific post compile steps that need to run if we find a graph in the cache This includes putting bundled triton artifacts in the right place, reloading the PyCodeCache artifact, etc. These don't always happen (i.e. on a cache miss, so they are in a separate function from CompiledFxGraph.post_compile) Ntriton_bundler_metainductor_compile)cached_kernel_nameszAOTAutogradCache.inductor_loadrnum_triton_bundlesrYrirrzOutput code: %szOutput code written to: %sartifactcS dddS)Nfx_graph_runnablestringrr!r~r~r~r~rrHz5FxGraphCache.cache_hit_post_compile..cjSr)runnable_graph_strr~rr~rrLr) metadata_fn payload_fninductor_post_grad_graphcrr)inductor_post_grad_graph_strr~rr~rrPrrinductor_output_codecstjdS)N)filename file_path)rrabspathr~) artifact_pathr~rrTs cSrr~r~rr~rrXcSr)N*inductor_provenance_tracking_node_mappingsrrr~r~r~r~rr\rcrr)inductor_provenance_mapping_strr~rr~rr`rcSr)N0inductor_provenance_tracking_kernel_stack_tracesrrr~r~r~r~rrdrcrr)$inductor_provenance_stack_traces_strr~rr~rrhrinductor_provenance)!_triton_bundler_ read_and_emitrxrtry_add_pt2_compilerrrrafter_deserializationrrjsave_output_code source_coder/r]inductor_meta_from_configr^ begin_compiler$CachedMetricsHelper apply_deltasmetrics_deltasrcounter_deltasoutput_code_logr`rJr! in_progressr add_to_set)rrrbundlerrdrj inductor_metar~)rrrrcache_hit_post_compile s~                   z#FxGraphCache.cache_hit_post_compileNrrrrrr.cSr)NTr~)rXyr~r~rrrz,FxGraphCache._lookup_graph..Tz*fx graph cache key %s post-load guards: %s)rOrPrr"&unsafe_skip_cache_dynamic_shape_guardsevaluate_guards_expressionrrrprQrecord_artifactrrrrrr`guardsr)rrrrrrrVsymintsrrrr guard_infocheckr~r~r _lookup_graphss4      zFxGraphCache._lookup_graphrrcCsHt|}tj|stj|ddtj|t|}t||dddS)NTrr) rOryrrrrrrr)rrrrr~r~rrs  z"FxGraphCache._write_to_local_cachecompiled_graphrmc CsVddlm}t||sJdt|dt}|dusJt|}||}|j||d|_ t |} | zt | } Wnty[tjddd td d d7<YdSwz3tt|| |rnt|| |rt| jpud d } t| d| d} ||| WdSWdStytjddd td dd7<YdSw)z= Store a serialized CompiledFxGraph on disk. rYrZzserialization for z NYIN) placeholdersrz1fx graph cache unable to serialize compiled graphTrrfxgraph_cache_pickle_errorrg.Ar)r time_taken_msz!fx graph unable to write to cachefxgraph_cache_write_error) compile_fxr[r rrOrPrget_pruned_guardsproduce_guards_expressionrrprepare_for_serializationrrrrrrrQrrrrf_time_taken_nsr b64encoderput) rrrrrr[rVrrdisk_compiled_graphrrrr~r~r _save_graphsN        zFxGraphCache._save_graphrgrhcCs|D]<}t|tjjsq|jjD]-}t|jtjj r+|j s+t d|j |j dkr?tt||jtjjr?t dqqdS)Nz!Can't cache HigherOrderOperator: getattrzCan't cache torchbind objects)r r r{rwr!rnodesrB_opsHigherOrderOperator cacheablerrrr_C ScriptObject)rgrrQr~r~r_check_for_hops$   zFxGraphCache._check_for_hopcCstjtjfD]}|rt|tr|stdqtjtjfD]}|r/t|tr+|s/tdqtj dur?ttj ts?tdtj D]}t |rQt|tsQtdqBt |r`t jds`tdtjjrhtdd d lm}|jrxtd ttdurtd td t|dS)z Check some conditions that would preclude caching and raise BypassFxGraphCache to bypass in case caching is not possible. z!Unsupported post grad custom passzUnsupported joint custom passNz#Unsupported _pre_fusion_custom_passz(Unsupported _fuse_ddp_communication_passz,pytorch/inductor:allow_freezing_with_cachingz$Skipping graph with frozen constantszORuntime constant folding can introduce constants that aren't static across runsr)CompilerBisectorz$dont cache graph when bisect enabledzfx graph cache no shape envz No shape env)r"r?rCr r:rZrrDrErGrHr[r>r{rkjustknobs_check aot_inductoruse_runtime_constant_folding!torch._inductor.compiler_bisectorrbisection_enabledrr`rOrPr)rgprr~r~r_check_can_caches>       zFxGraphCache._check_can_cacherrgrrremote3tuple[tuple[str, list[str]] | None, dict[str, Any]]c Cszt|t||||\}}Wn8tyH}z,tddd7<td||r1tdt|dt|t d}d|fWYd}~Sd}~ww||fifS) a Checks that the inductor input is cacheable, then computes and returns the cache key for the input. Returns (key_info, cache_info) where: - key_info is (hash_key, debug_lines), and - cache_info will contain debug info in the event of BypassFxGraphCache. NB: It is possible to have this function return a union instead. But I personally believe it is more annoying/difficult to read in that format. rfxgraph_cache_bypassrYz%Bypassing FX Graph Cache because '%s'bypass_fx_graphbypass) cache_statecache_bypass_reasoncache_event_timeN) rOrrdrrrrmrNrxr) rgrrrrrrr1rr~r~r prepare_key:s$    zFxGraphCache.prepare_keycCsd}t|tddS)zK Attempts to load the remote cache, returns None on error. z fx-graph-v1FbRemoteFxGraphCacheRemoteFxGraphCache)r\r"r)cache_idr~r~rget_remote_cache_szFxGraphCache.get_remote_cacherr is_backwardc Cst||||||\}} i| ||td} |durTtd|tddd7<d| d<|j} durP| | d <ttj d | d t | } d krP| | d <|| fStd|tddd7<d| d<|| fS)z Lookup the graph with the given key, and return results and metadata. Doesn't do any logging on its own, because AOTAutograd handles a cache miss differently from FXGraphCache. )r componentsrNzfx graph cache hit for key %srfxgraph_cache_hitrYrrre distributed_ephemeral_timeout_usirephemeral_timeout_increasezfx graph cache miss for key %sfxgraph_cache_missmiss) rOrrrrmrrrrrrs) rrrrrr$rrrrreephemeral_increaser~r~r load_with_keyls>   zFxGraphCache.load_with_keycCs*z ttWdStyYdSw)z. Clear out the on-disk cache. N)rLrmtreerOrFileNotFoundErrorr~r~r~rrs  zFxGraphCache.clearr)rrrrxrwrx)rrrrxr{rr|rr}rr~rrwr)rr[rrrrlrwrr)rrxrrrrrrrrlrrrwr)rrxrrrwr) rrxrrmrrrrrrrwr)rgrhrwr) rgrhrrrrgrrrrrwr)rwr)rrxrrrrrrrrr$rrrlrrrwrr)rrrr3rrrryrrrrrrrrr#r,rr~r~r~rrOs<   * e:  :  2 $   1rOrcCsBddd} ||rtj|S|drtj|S|dfS)NrwrxcSs trdSdS)Nz.pyd.sor0r~r~r~rget_module_ext_typesz;split_aot_inductor_output_path..get_module_ext_typez.pt2rr)endswithrrsplit)rr1r~r~rsplit_aot_inductor_output_paths     r4c@sTeZdZUiZded<eejZe  ddddZ edddZ edddZ dS)CudaKernelParamCachezdict[str, dict[str, Any]]rNrrxparamsdict[str, str | None]rbin_typeasmr4asm_typerwrcCs8d}tjjrtjjsJd|dsJd|d}t|||ttjjd|d\}} t| \}}tjj rSddt dd d i} || vsEJd t j | \} }| | |} d } tjj s]tjjrtjjdusg|r|r|smJd |ssJd|dvry|nd} t||| ttjjd|d\}} | |t<| |d<||j|<dS)Nz:package_cpp_only requires triton kernel names to be unique mangled_namezMissing kernel namer)rrrrz.fatbinz.spvrz.hsacoz8multi_arch_kernel_binary only supported in CUDA/XPU/ROCmrzMissing kernel assembly codezMissing kernel assembly type>rrrrr9)r"rpackage_cpp_onlyrunique_kernel_namesr8r4 output_pathr5emit_multi_arch_kernelrHrrsplitextr{r|r}rr)rrr6rr8r9r:rrbin_pathbin_type_to_ext base_pathasm_path hash_kindr~r~rrsh        zCudaKernelParamCache.setrcCs|j|dSr)rrrr~r~rr zCudaKernelParamCache.get KeysView[str]cCs |jSr)rrrr~r~rget_keys zCudaKernelParamCache.get_keysNN)rrxr6r7rrxr8rxr9r4r:r4rwr)rrxrwr)rwrG) rrrrrWrr cache_clearrrrrIr~r~r~rr5s  K r5c@seZdZdZedddZdS)AotCodeCompilerz. Compile AOT Inductor generated code. rrj wrapper_coderx kernel_codeserialized_extern_kernel_nodesr4radditional_filesrrwlist[Union[str, Weights]] | strcasb|}tt}tddt|jdd} t| } to%dko%j t tj j \} } tj j r;d f dt d| | tj jd \} d dtd | | tj jd \}ddtj jsttj tjtjtd d dd7}tj j}d|}|dd|dd|dd|dtd| |d\}Wdn1swYtd} | ddf|tjj|jddWdn1swYtj jr| tj j s| tj js| t!"d t!"dt#d fd!d" fd#d"d$t#d fd%d"fd&d"d$tj jsBt!"d't#d fd(d"fd)d"d$t$ }t$}|j%| &sZj'd*d+t(t$d,}d fd2d3 }d4d5l)m*}t+}|tj || d6t,d7}|`|rt(|-d8}t|d9 }||Wdn 1swYtj jr| |tj j.}|d:<|/t0t(|1|j2d;}tj j.3D]\}}t4|t(rt4|t(sJd<qt|d9}|t56tj j.Wdn 1swYt(|1|j2d;}t78||tj jr0| |tj j s0| || r7tj j nt(|-d=} t9fd>d?j:DddDdEtj j;s[tj j|!}#t?|#\}$}%|$r|%rt@dLd}&|$r|j2dM}'t(|1|'}&j|%|$ |dN}(tdd*tj j dO|(})tddPd*i|(}*tj jArtBstCjdQ}+tD|+| fdRtj j i|(|)_EtF},rtD|,| fi|(|*_Ett(|j2 t(|j%|)dS}-|-}.|-G}/tt(|j2t(|j%|*dS}0|0}1|0G}2tHdT|.tHdU|1tj j r^t(|1|j2dV}3|)I|3| |3|-J||-K| | |n)z|-LWntMjNtOfy}4zdWt(|4vr|t@dX|4|4d}4~4ww|0L|%s|!}5d4}6|$rtPQdY|#}5|&dusJt|&dZ }7|7|!Wdn 1swY| |&ntRtStTUd4tTVtTjWjXd[Y}6tPQd\|#d]|6}5||5tZj[}8d4}9i}:t\j]3D]G\}9\};}tj |j%|=}?tf|?|>d*| |?q|:r`tj |j%d_}@t|@d9}|t56|:Wdn 1sVwY| |@tTjgjhrhtintj}A|Ajk8}B|Ajkl|Brtj jmrJd`g}Cg}DtBsQtn \}E}Ftotjpjqdai}Gtrjs3D]\}H}I|H|Gvrq|Idb}Jr|D |J|Itt}Ktj jmr@dckr@tTjgjhdurtuv}Ltuwdd|Jde|Kdf|Ldg|Ldf|Ldh|Ldi}Mz txjy|Mzd*d*d*djWnMtxj{y}4zt||Mdk|4j}dl|4j~tZj~dmd}4~4wwd4dnlm}Ntj&|Js(t@do|Jdp|N|J|Kddq}O|Os:t@dr|HdstH"dt|Ktj jrO|C t|K|H|E|Fqt| \}P}Qt|j du}R|/|2|8g|B|C}St|P|S|Q|RdS}T|T}U|TG} tHdv|Ut dw}|d|dx|.d|dy|UdWdn 1swYtdw}|d|dx|1d|dy|UdWdn 1swYtj j rpt(|1|j2dz}V|RI|V| |V| t|%r0t(|1|j2d{}Wt|WdZ}7|7|!|7tPQdY|6Wdn 1s%wY| |Wn | |8|TK||8tj jmrStTjgjhdurS|T||D||Dng|B|C}S||S|SD] }X|TK||Xq`|T|nn|TL|SD]}Y|Y|Bvrqvt|Yqv|%rtj jd|krt@d}ddd}Z|Z}[tXd|[}\t| d%}]|]}^|]d|\|^|\|]|!|]tPQdY|6Wdn 1swYtj jr| | Wdn 1swYtjjd4kr(tTjj}_tj |j%d}`t|`d9}|t5j6|_ddWdn 1swY| |`tj jr/|S| S)z Returns the .so path, or returns a list of files that were generated if config.aot_inductor.package=True. oi)vec_isaraot_modersources BuildOptioncpur_rz wrapper.cpp)r rrz.// Triton kernels are embedded as comments in z kernel.cppcsrcr aoti_runtimezmodel.hAOTInductorModelz<>zAOTInductorModel((zAOTInductorModel :z :rrrNzw+cpp)rzWrapper code written to: %szKernel code written to: %s graph_dumpc dddS)Ninductor_aot_wrapper_coderbrrrr~r~) wrapper_pathr~rrz)AotCodeCompiler.compile..crrr~r~)rNr~rrrrcrd)Ninductor_aot_kernel_coderbrfr~r~) kernel_pathr~rrrhcrrr~r~)rOr~rrrzHeader code written to: %scrd)Ninductor_aot_header_coderbrfr~r~) header_pathr~rrrhcrrr~r~) header_coder~rrrTrzCMakeLists.txtconstsrrrxrwcsPtjj}|dkr$jtj@rt|dkrtddndd}n|dkr-dd }n|d kr6d}d }nt d |d krCd }t|dk}t|dk}d)fdd }d*dd}d+fdd }|rn||t ||\} } n|rx|t |\} } n||t |\} } t | | t tjj d \} } t| } tjd!d"} tt | jt | t | j| d#}|}|d ur|rtt | |t | jn||r!|r!t|d$H}|d|d}tjd%kr|d&n|d'}|d(ksJ||d}|t|kr| ||d}||7}|t|ksWdn 1swYt| |S),Nlinuxi5wzPModels with buffer mutation included doesn't support constants greater than 2GB!z .ldata, "aw"z .lrodata, "a"rdarwinz __DATA,__datarrtFzUnsupported platform: rrrrnr align_bytesrf symbol_prefixrxis_large_constsrrwr5csdd}|d|d7}|d|d7}||d7}|s3|D] }|d|d7}q!|s2|d7}n|d 7}|d t|d d7}|d |d 7}||d7}|dfS)N .section r_ .balign  .globl _binary_constants_bin_start _binary_constants_bin_start: z .byte z .space 1 z .quad 0x1234567899abcdef z .space .globl _binary_constants_bin_end _binary_constants_bin_end: z weights.Sr)rnrqrrrs consts_asmr  section_attrr~rformat_consts_to_gnu_asms zRAotCodeCompiler.compile.._compile_consts..format_consts_to_gnu_asmcSst|}d}|}|d|d7}|d|d|d7}d}|D]}||d7}|d }|d dkr6|d 7}q|d 7}|d|d |d7}|dfS)Nz#if defined(__clang__) || defined (__GNUC__) #define ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize("address"))) #else #define ATTRIBUTE_NO_SANITIZE_ADDRESS #endif ATTRIBUTE_NO_SANITIZE_ADDRESS zalignas(z ) extern zunsigned char z_binary_constants_bin_start[z] = { r, rYz z}; z) extern unsigned char * z_binary_constants_bin_end; z weights.cppr})rnrqrr consts_size asan_attr const_cpp count_bytesr r~r~rformat_consts_to_cpps zNAotCodeCompiler.compile.._compile_consts..format_consts_to_cppcsxtr d}d}||fSdd}|d|d7}|d|d7}||d7}|d |d 7}||d 7}d }||fS) a This function handles zero-sized constants because the C++ standard prohibits zero-length arrays: https://stackoverflow.com/questions/9722632/what-happens-if-i-define-a-0-size-array-in-c-c On Windows (MSVC): The compiler reports error C2466 for zero-sized arrays: https://learn.microsoft.com/en-us/cpp/error-messages/compiler-errors-1/compiler-error-c2466 Solution: Use assembly compilation to handle this case. Why not use Win32 assembly for all paths? ml64 only supports alignment up to 16 bytes, which isn't optimal for performance. Cross-platform implementation: Linux: Added '-pedantic' to disable zero-sized arrays in C++ compiler Windows: MSVC naturally rejects zero-sized arrays by default z option casemap:none .data ?_binary_constants_bin_start@@3PAEA: align 16 ?_binary_constants_bin_end@@3PAEA: align 16 public ?_binary_constants_bin_start@@3PAEA public ?_binary_constants_bin_end@@3PAEA end r9rtr_rurvrwrxrzr{r|Sr0)rqrrasm_codeasm_extrr~rget_zero_consts_asm_codes   zRAotCodeCompiler.compile.._compile_consts..get_zero_consts_asm_coderaT)rrV compile_onlyuse_relative_pathrrX output_dirrYzr+blittlesͫxV4s4Vxr) rnrrqrfrrrxrsrrwr5)rnrrqrfrrrxrwr5)rqrfrrrxrwr5)r"ruse_consts_asm_buildmutated_buffersrXrrrrrrCr8rxmodel_name_for_generated_filesr r2rVr0stemrEget_target_file_pathr7buildrrrr byteorderfindrrN)rnr use_asm_buildrrrsis_zero_size_constsrrr consts_codecode_extrconsts_sobject_build_optionsobject_builderconsts_orQhdr start_idxposrc)rrspecified_sub_dirrrr_compile_constss    .              z0AotCodeCompiler.compile.._compile_constsrFileLock.locktimeoutz.jsonr AOTI_DEVICE_KEYz_metadata.jsonz"Metadata must only contain stringsr/c3s&|]}|jvr|jVqdSr)folded_constantsget_original_value_of_constantis_cudarrrr~rrs  z*AotCodeCompiler.compile..r] torch.Tensorall_cudarc Ssddd}ddl}|dkrdS|jr#tjj|}tjj|}n| }|}| }| || |j |}t|j}|rF|S||S) N raw_bytesrrwcSs$|t|tdttd}|S)NrY)ljustrrC)r padded_bytesr~r~r_pad_to_alignments zEAotCodeCompiler.compile.._to_bytes.._pad_to_alignmentr)rrrwr)ctypesnumelrr{opsmkldnndata_ptr_nbytesuntyped_storagerZnbytesrPOINTERc_ubytercontents) r]rrrrrt_cpu raw_arrayrr~r~r _to_bytess    z*AotCodeCompiler.compile.._to_bytes binary_blobrc3s*|]}|jvr|VqdSr)rrr)rrrr~rrs pickle_weightscs6i|]}|jvrj||tj|fqSr~)rallocated_constant_namerrRrrrr~rrs  z+AotCodeCompiler.compile..z?use_external_weights and use_mmap_weights cannot both be True.z _weights.blob)rVruse_mmap_weightsuse_mmap_weights_externalrrUr min_optimizer)rVrrz#aot wrapper compilation command: %sz"aot kernel compilation command: %sz_compile_flags.jsonz is too big to optimizezUPlease use torch._inductor.config.aot_inductor.compile_wrapper_opt_level = 'O0' flag.qr@)rYqqryzsaving script object %s as %szcustom_objs_config.jsonz. Check that ROCm toolchain is available and LLVM IR is valid.zCreated multi-arch bundle: %s)rUrrVrzaot linkage command: %saz// Compile cmd // z// Link cmd // z_linker_flags.jsonz_serialized_weights.binwindowszKwhen cross_target_platform is windows, use_mmap_weights should not be true.rfcsxtr2ddlm}m}m}ddlmmmGfddd|}|}|j |||j }|Sddl }| }|S)Nr)byref StructurewindllDWORDLPVOIDWORDc sReZdZdfdfdfdfdfdfdfdfd fd fd fg Zd S) zCAotCodeCompiler.compile..get_page_size..SYSTEM_INFOwProcessorArchitecture wReserved dwPageSizelpMinimumApplicationAddresslpMaximumApplicationAddressdwActiveProcessorMaskdwNumberOfProcessorsdwProcessorTypedwAllocationGranularitywProcessorLevelwProcessorRevisionN)rrr_fields_r~rr~r SYSTEM_INFOR sr)r0rrrrctypes.wintypesrrrkernel32 GetSystemInforresource getpagesize)rrrrsi sys_page_sizerr~rr get_page_sizeG sz.AotCodeCompiler.compile..get_page_sizei@za+b zkernel_information.jsonrr)rnrrrxrwrx)r]rrrrwrr~)rwrf)r-r8r0r2rVrget_command_liner"rr4rr>r<rr8rdynamic_linkagerrrrrrrcr writelinesflushr`r`rurpackagerrrmrJr rErrFrxtorch.utils._filelockrr LOCK_TIMEOUT with_suffixrrpr with_namerrr rrrLrrrpackage_constants_in_so package_constants_on_disk_formatrSrrErprecompile_headersr0_get_cpp_wrapper_header_precompile_headerprecompiled_header_get_cpp_prefix_headerrrsave_flags_to_jsonsave_compile_cmd_to_cmakesave_src_to_cmakerr#CppCompileErrorrstructpackrrfr{randintiinfoint64maxitemrr enumeratetorchbind_constants_libraryfake_class_registryrIrr r rT _pickle_saverr|r} ROCmCodeCache CUDACodeCache aot_kernels_orr?r4rrrNr5rrcuda_compile_utils_nvcc_arch_as_compile_option_cuda_compiler subprocessrunr3CalledProcessErrorprintstdoutstderr$torch._inductor.rocm_multiarch_utilsrembed_kernel_binaryr/r5r,save_kernel_asm_to_cmakeextendsave_link_cmd_to_cmakerNcross_target_platformtelltraceprovenance_tracking_levelrcreate_kernel_information_json)arrrNrOrPrrQgenerated_filespicked_vec_isavec_isa_cmd_gen cpp_commandspecified_output_pathspecified_artifact_name wrapper_keyrrQmodel_class_name class_namer]wrapper_path_operatorkernel_path_operator cmake_pathrrrlockextern_kernel_nodes_jsonr meta_jsonrrkernel_meta_json output_soserialized_weights weights_dictruse_external_weightsrexternal_weights_pathexternal_weights_filenamecompile_commandwrapper_build_optionskernel_build_options header_file cpp_prefixwrapper_builderwrapper_compile_cmd wrapper_okernel_builderkernel_compile_cmdkernel_o compile_flagsr1 aot_constants magic_number f_weightsrcustom_obj_idxqual_name_to_idrconstantcustom_obj_namecustom_obj_bytescustom_obj_pathconstants_config_json gpu_codecache gpu_kernels_ocubins_o asm_filesldobjcopykernels kernel_namerasm_file cubin_file current_archcmdrsuccess output_namerso_build_optionsobj_srcs so_builderlink_cmd linker_flags weight_filero_filer page_size_ page_sizef_soso_size kernel_infokernel_info_jsonr~) rrrrrmrlrOrjrrrNrgrcompiles                          @                                                                                 $     t    zAotCodeCompiler.compileN)rrjrNrxrOrxrPr4rrxrQrrwrR)rrrr3rrur~r~r~rrMsrMz CDLL | None_libgomprargsr list[c_void_p] | c_void_p | Nonec sPdfdd fdd|D}|dsJ|d d}t|d D]\}}|d kr1t|}t||}q$t|sAJ|d t}t|j j |D] \}}|j rW|||j <qK|rb|t | d=||i|} | durodSt| ttfrd d| D} | D]} t| tjsJ|dqtjj| St| tjsJ|dtjj| S)NargrrwcsJtt|dkrtjj|St|ttfr#t|fdd|DS|S)Nzc3s|]}|VqdSrr~rr convert_argr~rr rz9custom_op_wrapper..convert_arg..) rxrr{r _aoti&alloc_tensor_by_stealing_from_void_ptrr rrI)ryr{r~rr| s z&custom_op_wrapper..convert_argcsg|]}|qSr~r~rryr{r~rr rz%custom_op_wrapper..z torch.ops.z, can not be called through custom_op_wrapperrrz, can not be loaded through custom_op_wrappercSs"g|] }|dur tgn|qSr)r{tensor)rrr~r~rr s"z returns a list of non-tensorsz returns a non-tensor)ryrrwr)rrr3 importlib import_modulerr[rzip_schema arguments kwarg_onlyrrr rrIr{rr r}#unsafe_alloc_void_ptrs_from_tensors!unsafe_alloc_void_ptr_from_tensor) rrwconverted_argsrrTrr'func_argconv_argresultrr~r{rcustom_op_wrapper s8    rprecompiled_headersrheaderhashable_cmd_linerDc KstrJdt>}t|d}|d|dtt|ddt|tdi|ddid}|dd d }|| }Wdn1sKwYtdi|ddi}t d|dd||t | t d\} } t| | |d} tjtddttjt| d| f| S)Nz>CppBuilder does not currently support precompiling on Windows!z header.hppz #include preprocessingTrWrrxrwcSs$tjdd|fddd}|jdS)zReading the whole preprocessed header in for hashing is very expensive, but calling a fast hashing utility in a subprocess is cheap.opensslsha512T)rr9r)rrr"r3)r cmd_outputr~r~r_get_file_checksum s z._precompile_header.._get_file_checksum precompilingr)rrr rrrr~)rrxrwrx)r0r&TemporaryDirectoryr r<r0rxr2rrr8r3 get_compiler _HEADER_DIRrr_HEADER_LOCK_DIR_worker_compile_cpprr) rrrDpreprocessing_dirpreprocessing_header preprocessorrpreprocessor_hashheader_build_option header_hashheader_full_path cpp_builderr~r~rr sJ       rcCs|drdSdS)NrZz torch/csrc/inductor/cpp_prefix.h)rrr~r~rr s rrVcCsJ|jdddd}tjjo|dk}d|rdndd |r d d S|d S) zGiven a device type (and optionally whether we're in AOT Inductor mode), returns the path to the cpp_wrapper header file to be precompiled.:rY)maxsplitrrZztorch/csrc/inductor/ aoti_include cpp_wrapper/ array_refz.h)r3r"rallow_stack_allocation)rrV base_device is_array_refr~r~rr s rc@seZdZUdZiZded<eejZiZ ded<ed#d d Z e d#d dZ e d$ddZ e    d%d&ddZe d'd!d"ZdS)( CppCodeCachezCompiles and caches C++ libraries. Users of this class supply the source code to be compiled, while compilation flags are set by CppBuilder.*dict[str, Callable[[], CDLL | ModuleType]]rrcpp_compile_command_flagsrrxrrwCDLL | ModuleTypecCs t|Sr)r LoadLibrary)rrr~r~r_load_library_inner rJz CppCodeCache._load_library_innerc Csz |||}||_|WSttfyU}z;dt|vr8tjdr8t da |||}||_|WYd}~Sdt|vrPt|dt dt d|d}~ww)Ngompz/usr/lib64/libgomp.so.1z(failed to map segment from shared objectz3. The most common reason this may occur is if the zl folder is mounted with noexec (e.g., by default Docker mounts tmp file systems as noexec). Please remount zi with exec enabled, or set another temporary directory with TORCHINDUCTOR_CACHE_DIR environment variable.) rr ImportErrorr/rxrrrrrrvr& gettempdir)rrrrr1r~r~r _load_library$ s*    zCppCodeCache._load_libraryrr4cCr)z_ Given a device type, returns the path to a CPP header file to be precompiled. Nr~rrr~r~r_get_uncompiled_header; sz#CppCodeCache._get_uncompiled_headerrZNr~ main_coder submit_fnr extra_flags Sequence[str]optimized_codecsLij||ttd}ttdt||dud|}tdddi|}dd d } | |} | |} t|d |d | d\} |rPt|d| d\} }ntj }j vr!ddl m }tj td}ddtjrts|}rt|| fd|dui||_|rt|}rt|| fi||_t| \}}t|| ||d}|rt|\}} t||||d}t|||gtdi||d}tt||||ft|ntt||ft|dfdd }|dur||tdtj s |Wdn 1swY|j <j S)z\Compile and load a C++ library. Returns a callable that returns the loaded library.)rrrrUNrrT build_optionrhrwrxcSstdd|dS)zWriting the code to file will calculate a hash, which we need to vary if the command line flags change. This implements a mostly-generic way of validating that.rSrTrW)r0r)rr~r~rget_hashable_command_lineh s z:CppCodeCache.load_async..get_hashable_command_linezmain.cpprr z optimized.cpprrrr)rrXrYrrcsFdur!dur }|dusJdus!JSr)rr)r binary_pathrfuturerr worker_fnr~rload_fn s   z(CppCodeCache.load_async..load_fnrr~)rrhrwrxrwr)rr"rr8r-r2rr8rdevnullrrrrrrcpp_cache_precompile_headersr0rrrrr5r0rrr rr6rr)rrrrrrrDmain_build_optionoptimized_build_optionr main_cmd_lineoptimized_cmd_line main_pathroptimized_pathr lock_pathr main_namer main_builderoptimized_nameoptimized_builderlinkerrr~rr load_asyncB s              zCppCodeCache.load_asyncrwr'cO|j|i|Sr)rrrwr'r~r~rr zCppCodeCache.load)rrxrrxrwrrrxrwr4)rZNr~N) rrxrrxrrrrrr4rwrrwrr'rrwr)rrrr3rrWrrrLrrrrrrrr~r~r~rr s(       rr cpp_buildersSequence[CppBuilder]cCs`ddlm}||td|D]}tj|s|qWddS1s)wYdS)Nrrr)rrrrrrrr)rrrbuilderr~r~rr s "rc@seZdZUiZded<eejZdddZdZ dZ dZ e d Zed)ddZed*ddZe     d+d,d#d$Zed-d'd(ZdS).CppPythonBindingsCodeCacherrFTinclude_pytorchsharedrRzkernel({}); Py_RETURN_NONE;ra // Python bindings to call {entry_func}(): #define PY_SSIZE_T_CLEAN #include #include #include #include #ifndef _MSC_VER #if __cplusplus < 202002L // C++20 (earlier) code // https://en.cppreference.com/w/cpp/language/attributes/likely #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) #endif #else #define likely(x) (x) #define unlikely(x) (x) #endif // This is defined in guards.cpp so we don't need to import PyTorch headers that are slooow. // We manually link it below to workaround issues with fbcode build. static void* (*_torchinductor_pyobject_tensor_data_ptr)(PyObject* obj); template static inline T parse_arg(PyObject* args, size_t n) {{ static_assert(std::is_pointer_v, "arg type must be pointer or long"); return static_cast(_torchinductor_pyobject_tensor_data_ptr(PyTuple_GET_ITEM(args, n))); }} template <> inline int64_t parse_arg(PyObject* args, size_t n) {{ auto result = PyLong_AsSsize_t(PyTuple_GET_ITEM(args, n)); if(unlikely(result == -1 && PyErr_Occurred())) throw std::runtime_error("expected int arg"); return result; }} template <> inline uintptr_t parse_arg(PyObject* args, size_t n) {{ auto result = PyLong_AsVoidPtr(PyTuple_GET_ITEM(args, n)); if(unlikely(result == reinterpret_cast(-1) && PyErr_Occurred())) throw std::runtime_error("expected int arg"); return reinterpret_cast(result); }} template <> inline float parse_arg(PyObject* args, size_t n) {{ auto result = PyFloat_AsDouble(PyTuple_GET_ITEM(args, n)); if(unlikely(result == -1.0 && PyErr_Occurred())) throw std::runtime_error("expected float arg"); return static_cast(result); }} {extra_parse_arg} static PyObject* {entry_func}_py(PyObject* self, PyObject* args) {{ try {{ if(unlikely(!PyTuple_CheckExact(args))) throw std::runtime_error("tuple args required"); if(unlikely(PyTuple_GET_SIZE(args) != {arg_len})) throw std::runtime_error("requires {arg_len} args"); {call_entry_func} }} catch(std::exception const& e) {{ PyErr_SetString(PyExc_RuntimeError, e.what()); return nullptr; }} catch(...) {{ PyErr_SetString(PyExc_RuntimeError, "unhandled error"); return nullptr; }} }} static PyMethodDef py_methods[] = {{ {{"{entry_func}", {entry_func}_py, METH_VARARGS, ""}}, {{NULL, NULL, 0, NULL}}}}; static struct PyModuleDef py_module = {{PyModuleDef_HEAD_INIT, "{entry_func}", NULL, -1, py_methods}}; PyMODINIT_FUNC PyInit_{entry_func}(void) {{ const char* str_addr = std::getenv("_TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTR"); if(!str_addr) {{ PyErr_SetString(PyExc_RuntimeError, "_TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTR must be set"); return nullptr; }} char* endptr = nullptr; errno = 0; uintptr_t addr = std::strtoull(str_addr, &endptr, 10); if(errno != 0 || endptr == str_addr || addr == 0) {{ PyErr_SetString(PyExc_RuntimeError, "Failed to parse _TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTR"); return nullptr; }} _torchinductor_pyobject_tensor_data_ptr = reinterpret_cast(addr); PyObject* module = PyModule_Create(&py_module); if (module == NULL) {{ return NULL; }} #ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; }} rrxrrwrcCsttjjjjtjd<|d|j}zt j |WSt y"Ynwt j ||}|dus0Jt j |}|t j |<|jdusBJ|j||S)N'_TORCHINDUCTOR_PYOBJECT_TENSOR_DATA_PTRr)rxr{r _dynamor'_torchinductor_pyobject_tensor_data_ptrrenvironentry_functionrr KeyErrorrutilspec_from_file_locationmodule_from_specloader exec_module)rrr module_namerrr~r~rr_ s         z.CppPythonBindingsCodeCache._load_library_innerrr4cCt|Sr)rrr~r~rrr rzz1CppPythonBindingsCodeCache._get_uncompiled_headerrZrNr~argtypesrrr num_outputsrfrrrrOc svdddt|D}jjt|j|jjj|dd} j|| ||||ddd fd d } | S) aV Wrap a C++ function in fast Python bindings. Args: argtypes: The types of args to ENTRY_FUNCTION(), e.g. ["float*", "long"] main_code: C++ source code containing ENTRY_FUNCTION(). Will be built at -O3 if kernel_code is None (to maximize performance in any kernels that are present), or -O1 otherwise (to minimize compile time). kernel_code: If present, C++ source code that will be built at -O3 and linked to main_code. Returns: A python version of ENTRY_FUNCTION() rcss.|]\}}d|ddd|dVqdS)z parse_arg(args, )N)rc)rnargtyper~r~rr s  zBCppPythonBindingsCodeCache.load_pybinding_async..) array_len)arg_lencall_entry_func entry_funcextra_parse_arg)rrrNrwrcs(durttsJtjSr)r rrrr~r get_resultrr~rr s z?CppPythonBindingsCodeCache.load_pybinding_async..futurer) rrsuffix_templateformatrcall_entry_functionrrr) rrrrrrrrO parseargsr"rr~rrload_pybinding_asyncv s&   z/CppPythonBindingsCodeCache.load_pybinding_asyncrwr'cOrr)rrr~r~rload_pybinding rz)CppPythonBindingsCodeCache.load_pybinding)rrxrrxrwrr)rZrNr~N)rrrrxrrxrrfrrrrrOr4rwrr)rrrrrWrrrLrrrrtextwrapdedentrrrrrrr~r~r~rr s2  e  4rc@sNeZdZUiZded<eejZdddZdZ dZ e dZ edd d ZdS)CppWrapperCodeCacherrTrinductor_entry_cppzreturn inductor_entry_cpp({});a #include static inline std::vector unpack_tensor_handle_list(PyObject* pyvec) {{ std::vector result; size_t result_len = PyList_GET_SIZE(pyvec); result.reserve(result_len); for (size_t i = 0; i < result_len; i++) {{ // AtenTensorHandle is essentially a pointer void* elem = PyCapsule_GetPointer(PyList_GET_ITEM(pyvec, i), NULL); result.push_back(reinterpret_cast(elem)); }} return result; }} static inline PyObject* pack_tensor_handle_list(const std::array& arr) {{ PyObject* result = PyList_New({array_len}); for (size_t i = 0; i < {array_len}; i++) {{ PyObject *elem = arr[i] == nullptr ? Py_None // Store AtenTensorHandle as PyCapsulate : PyCapsule_New(reinterpret_cast(arr[i]), NULL, NULL); PyList_SET_ITEM(result, i, elem); }} return result; }} template <> inline std::vector parse_arg>(PyObject* args, size_t n) {{ return unpack_tensor_handle_list(PyTuple_GET_ITEM(args, n)); }} PyObject* inductor_entry_cpp(std::vector&& input_handles) {{ // For outputs, we only allocate an array to hold returned tensor handles, // not the actual output tensor storage. std::array output_handles{{}}; try {{ inductor_entry_impl(input_handles.data(), output_handles.data()); if (PyErr_Occurred()) {{ return nullptr; }} return pack_tensor_handle_list(output_handles); }} catch(std::exception const& e) {{ PyErr_SetString(PyExc_RuntimeError, e.what()); return nullptr; }} catch(...) {{ PyErr_SetString(PyExc_RuntimeError, "unhandled error"); return nullptr; }} }} rrxrwr4cCrr)rrr~r~rr rzz*CppWrapperCodeCache._get_uncompiled_headerNr)rrrrrWrrrLrrrrrrrrr~r~r~rr s  6rc@seZdZUiZded<eejZdZded<e dZ e e dZ e e dZ e d Zed3ddZed4ddZeejd5ddZed6ddZeejd7d d!Zeejd7d"d#Ze d8d9d(d)Zed:d,d-Zed5d.d/Zed;d1d2ZdS)<HalideCodeCachez*dict[str, Callable[[], ModuleType | CDLL]]rNr4_standalone_runtime_patha #include "{halideruntime_h}" #include "{headerfile}" #include #include namespace c10 {{ inline long div_floor_integer(long a, long b) {{ if ((a<0) != (b<0)) {{ const auto quot = a / b; const auto rem = a % b; return rem ? quot - 1 : quot; }} return a / b; }} }} z void kernel({argdefs}) {{ {buffers} int err = halide_kernel({buffer_names}); if(err != 0) throw std::runtime_error("halide_kernel failed"); }} a{ #include static const halide_device_interface_t* cuda_interface = halide_cuda_device_interface(); void kernel({argdefs}, uintptr_t stream) {{ {buffers} int err = halide_kernel(reinterpret_cast(stream), {buffer_names}); if(err != 0) throw std::runtime_error("halide_kernel failed"); }} a #include "{}" #include static int acquire_context(void* user_context, void** cuda_context_out, bool create) {{ return cuCtxGetCurrent(reinterpret_cast(cuda_context_out)); }} static int release_context(void* user_context) {{ return 0; }} static int get_stream(void* user_context, void* cuda_context, void** stream_out) {{ *stream_out = user_context; return 0; }} static int register_halide_hooks() {{ halide_set_cuda_acquire_context(&acquire_context); halide_set_cuda_release_context(&release_context); halide_set_cuda_get_stream(&get_stream); return 0; }} int inductor_register_halide_hooks_result = register_halide_hooks(); rrxryrprrrwrc Csn|jdusJ|jdurt|jt|jksJ|jdusJ|jp$|jd|j}|r:d|d}d}d}d}n d}d}d|d}d }g} t|j|jD]\} } | d | d | dqOd |d t| dkrwd|dd | dnd|d|d|d |d|d |d|d |d|d |d| d |dt| d |d|d|dg S)Nz + zreinterpret_cast(rcuda_interfacenullptrhalide_buffer_flag_device_dirty0zreinterpret_cast(halide_buffer_flag_host_dirtyzhalide_dimension_t(0, rzhalide_buffer_t ;rzhalide_dimension_t z _dims[] = {z};zhalide_dimension_t * z_dims = nullptr;z .device = z.device_interface = z.host = z .flags = z.type = z.dimensions = z.dim = z_dims;z.padding = nullptr;) shapestrideroffsetalias_ofrrrr halide_type) rrryrrrdevice_interfacehostflagsdimssizerr~r~r_codegen_bufferC s:"     zHalideCodeCache._codegen_bufferrdrq headerfileobjectc Cs|}|d|jvus Jd|jvsJg}g}t|jD]+\}}|r;|d|||d|||qd|jvsBJ||j qd dd|D }|rZ|j n|j }|j||red nd |d d d |jD|d |d} | S)N user_context no_runtimez&hl_buf_hl_buf_*r_cSsg|]}d|qS) r~)rliner~r~rrw rYz1HalideCodeCache._codegen_glue..HalideRuntimeCuda.hzHalideRuntime.hrcss.|]}|jdur|d|jVqdS)Nr)r bindings_typerrzr~r~rr s z0HalideCodeCache._codegen_glue..)halideruntime_hrargdefsbuffers buffer_names)rrBrr is_bufferrr'rctyperrlstripglue_template_cudaglue_template_cppr find_header) rrdrrr&r'rTry glue_template glue_coder~r~r _codegen_glueg s2   zHalideCodeCache._codegen_gluecCs:tddtd}|}td|j|j|j|gdS)NOIrWr_r) r0r1rrrr,r+standalone_runtime_cuda_initr)r command_gen command_liner~r~r config_hash s zHalideCodeCache.config_hashr"errmsgc Cstjjd}|dus|jstdzX|jd}t|D]H}|drezt dtj ||g}Wn t j y<Yqwtd|d}|retj tj |d|}tj |retj |WSqWt|ty{}zt||d}~ww) Nhalidez$halide python bindings not installedrr/lddz(/.*)/libHalide.sorrY)r machinery PathFinderrrrrrr2r check_outputrrSubprocessErrorrsearchrrgrouprr) r"r7rr>routmrr1r~r~r_search_for_file s4    z HalideCodeCache._search_for_filecCsVd|d}dtjvrtjtjd|}tj|r|Sd|d}t||S)Nlibautoschedule_r/ HALIDE_LIB Can't find z3, set env HALIDE_LIB to the directory containing it)rrrrrrrrB)rsofilerr7r~r~rfind_libautoschedule s    z$HalideCodeCache.find_libautoschedulecCsdtjvrtjtjd|}tj|r|Sdtjvr5tjtjtjdd|}tj|r5|Sd|d}td||S)NHALIDE_INCLUDErDz ../include/rEz7, set env HALIDE_INCLUDE to the directory containing it)rrrrrrrrB)rrr7r~r~rr- s     zHalideCodeCache.find_headerrrrCallable[[], Any]c sttt|t||fddd}tj|dddt|d}t|d}t|d}t|d }t|d } tj | } g} | rwt ||t j |d d d |ddddg } |j rf| d||j g| || ttj| dd|jD} |r| d|j| |||||f| r| jnd|rdndd| r| tt|tt| | }|r||jn|dfdd }|S)Nrr8Trzgenerate_kernel.pyzhalide_kernel.azhalide_kernel.hdoner:z-grR-oz-f halide_kernelz-ezstatic_library,h,schedulez-pcSsg|] }|jdur|qSr)rr#rr~r~rr sz9HalideCodeCache.generate_halide_async.. uintptr_trrZ)rrrrwrIcsrSrr~r~bindings_futurewait_for_compiler~rrsz3HalideCodeCache.generate_halide_async..load)rwrI)r rrrr6rrrxrrrr executable schedulerr'rGrwrrr r check_callrrrr0build_standalone_runtimetouch_worker_task_halider)rrdrrdirpathgenfilelibfilerdonefilelockfile need_compilejobsre binding_typestaskrr~rOrgenerate_halide_async sr             z%HalideCodeCache.generate_halide_asyncrwr'cOrr)rarr~r~rgenerate_haliderzHalideCodeCache.generate_halidec Cs|jr tj|jr |jStjrdnd}d}|dkrdnd}|jr0tj|jr,Jt}nt}t |d|d| }tj |dd t |d }t |d }t |d }t |d } t ||} tj|sddl } ddlm} | |t]tj|st|d} |dkr| |j|dWdn1swY| | | |t| \}}t||| g|t|dd}tt|t |Wdn1swYtj| sJ| |_| S)NrrZzlibStandaloneHalideRuntime.soz host-cudarzhalide-runtime--TrrKr:z hooks.cppzstandalone_halide_runtime.arrr r")rr)!rrrrr{rr,rBrAr r6rrxr8rrrrr8r3rr-compile_standalone_runtimeTargetr5r0r2rrTshlexr3rrV)rrlibnamerBbaserX done_file lock_file hook_filea_fileso_filehlrrQrrhalide_cmd_genr~r~rrUsf             z(HalideCodeCache.build_standalone_runtimercCr)z5Header precompiling is currently disabled for halide.Nr~rr~r~rrXrz&HalideCodeCache._get_uncompiled_header)rrxryrprrrwr)rdrqrrrwrxr)r"rxr7rxrwrx)rrxrwrxr)rdrqrrxrrrwrI)rwrr'rrwrIr)rrrrrWrrrLrrrrr,r+r3rrr0rr6rBrGr-rarbrUrr~r~r~rr sP     ! # !  D  :rr\r^list[partial[Any]]c Cslddlm}z"||t|D]}|qWdWdS1s!wYWdStjy}ztjddkrt|dd^}}}tj | drt | }d} || d ks\JGd d d } |d } t| tsoJ| || d <ttd dg|dd} || | }tdd} | |Wdn1swYtd||d}~ww)Nrr HALIDE_REPRO1re)rrrpythonz hl.main()rYc@eZdZdddZdS)z _worker_task_halide..OutrwrxcSr)Nr@r~rr~r~r__repr__or\z)_worker_task_halide..Out.__repr__Nr)rrrrur~r~r~rOutnrvrLz import sys, tempfile with tempfile.TemporaryDirectory() as out: sys.argv = zrepro.pyz? hl.main() r r zwrote repro.py: )rrrrr=rrrrrrrr read_textcountindexr rfrrrrcrr8r*r)r\r^rjobr1rsscriptrermainrvcireplfdr~r~rrW^sD  &   rWrcCs0t|d WddS1swYdS)Nr)r)rr~r~rrVs "rVc@seZdZUgZded<iZded<iZded<ed&d'd dZed&d(ddZ e  d)d*ddZ ed+d,dd Z ee j d-d$d%ZdS). PyCodeCachezlist[ModuleType]r zdict[str, ModuleType]modules_no_attrz dict[str, list[tuple[Any, ...]]]linemapsrrrxr rwr5cCst|d|dSNpyrr;)rrr r~r~rr8rFzPyCodeCache.writercCst|d|d\}}|||Sr)r8load_by_key_path)rrr rrr~r~rrs zPyCodeCache.loadNrrlinemaplist[tuple[int, str]] | Noneattrsrc Cs|durg}|dur||jvr|j|St}t|||d}|r)tt||j|<|dur<|D] \}}t|||q1|rM|durG||j|<|j ||S)N)set_sys_modules) rr+r@rrrrsetattrr r) rrrrr in_toplevelmodrrr~r~rrs    zPyCodeCache.load_by_key_pathFpurgerrc CsT|r|jD]}z |js Jt|jWqtyYqw|j|jdS)z Clear the in-memory module cache. If purge=True, also delete all the corresponding on-disk source files. N)r rrrNr.rr)rrrr~r~rrLs    zPyCodeCache.cache_clearlinenorflist[dict[str, Any]] | NonecCsn||jvrdSt|j|dkrdS|j|\}}t||}|dkr$dS||d}|s.dSd dd}||S) NrrY stack_tracerxrwlist[dict[str, Any]]cSs"d}t||}ddt|DS)Nz"File "(.+)", line (\d+), in (.+)\ncSs"g|] \}}}|t||dqS))rr!r)rf)rrQlrr~r~rrszPPyCodeCache.stack_frames_for_code..parse_stack_trace..)rfindallreversed)rregexmatchesr~r~rparse_stack_traces  z.parse_stack_trace)rrxrwr)rrr)rrrrrrentryrr~r~rstack_frames_for_codes     z!PyCodeCache.stack_frames_for_coder)rrxr rxrwr5)rrxr rxrwrrK) rrxrrxrrrrrwrr)rrrwr)rrxrrfrwr)rrrr rWrrrr8rrrLrrrr~r~r~rrs"   !rrarrrcCstt||Sr)rrr)rarr~r~r_load_triton_kernel_from_sourcesrc Cstr._wrapped_funcrwrrwr)rrrrr)rrrr~rr __getattr__Ds  zDLLWrapper.__getattr__rcCrYrr~rr~r~rr)Qr\zDLLWrapper.__enter__rwrcG |dSrr-)rrwr~r~rr2TrzDLLWrapper.__exit__cCrrrrr~r~r__del__WrzDLLWrapper.__del__N)rrxrwrr)rrxrwr)rwrr) rrrr3rr-rrr)r2rr~r~r~rrs   # rr>cCs|dS)z, standard format for the error path z.errorr~)r>r~r~rbinary_error_path[rrc@seZdZUdZejGdddZiZded<gZ ded<dZ d ed <dZ d ed <e d8ddZ eeddd9ddZe d:ddZe d;dd&d'Ze d;d?d*d+Ze d@d-d.Ze d;dAd6d7ZdS)BCUTLASSCodeCacheaB A cache for managing the compilation and loading source code specifically for CUTLASS. This class handles writing source code to files, compiling them into shared objects, and caching the results to avoid redundant compilations. It also manages error handling and logging for the compilation process. c@s*eZdZUded<ded<dZded<dS)zCUTLASSCodeCache.CacheEntryrx input_pathr>Nr4 error_json)rrrrWrr~r~r~r CacheEntryks rdict[str, CacheEntry]rrrrrx_SOURCE_CODE_SUFFIX_BACKENDrwrcCs|j|jdSr)rrrrHr~r~rrLw zCUTLASSCodeCache.cache_clearr)maxsizecaching_enabledrcaching_availablerXcCsN|s tddS|s dSz ddlm}|WSty&tdYdSw)ad Get or create the class instance of the CUTLASSKernelBinaryRemoteCache. Args: caching_enabled: Whether binary remote caching is enabled caching_available: Whether we're in fbcode environment Returns: CUTLASSKernelBinaryRemoteCache: The class instance of the kernel binary remote cache z6CUTLASSKernelBinaryRemoteCache not requested, skippingNr)CUTLASSKernelBinaryRemoteCachezECUTLASSKernelBinaryRemoteCache not available, remote caching disabled)rr`-torch._inductor.fb.kernel_binary_remote_cacherr)rrrr~r~rget_kernel_binary_remote_cache|s   z/CUTLASSCodeCache.get_kernel_binary_remote_cachecCtrrwrHr~r~r _use_re_buildzCUTLASSCodeCache._use_re_buildN src_filesdst_file dst_file_ext extra_argsOptional[list[str]]cCrrrwrrrrrr~r~r_compile_commandsz!CUTLASSCodeCache._compile_commandcCrrrwrHr~r~r_source_code_extrarz#CUTLASSCodeCache._source_code_extrarr5cCsFtjjrt|dgd|}|}n|}t||j|d\}}||fSz Writes source code into a file with dst_file_ext as the file extension. Returns the hash key of source code, and the path to the file. dummy_input dummy_outputr)r"rcutlass_hash_with_compile_cmdrrrr8r)rrrrDr rrr~r~rr8szCUTLASSCodeCache.writerrc Cs|dkr||d|\}}}|||\}}|gd}} n|||\}}|gd}} ||} | |jvrddlm} t} | tj| |dt d} | u|d t |j  |}t |}|j tjjohtj td }|d urx|||tj|rt|d d  }|}Wd n1swYt|\}}|d urtjjr||tjj|||||j| <t||tj|s|||||}t|d }|d|d|j d| d|dWd n1swYt!}t"#d|j | ||$d}z&|%r$ddl&m'}||tj(|tj)|n t*j+|t*j,tj-dWnNt*j.yS}z|/|j01d | ||||t||j0|d }~wt2y}}zdt3|vrw|/t3|| ||||t|t3|||d }~wwt!}|j d| d||d|}t"4|n t"#d|j | ||d urtjjr||tjj|||d |j| <Wd n 1swY|j| }|j5d urt|j5\}}t||6d |j| j7||fS)z Compiles CUDA source_code into a file with dst_file_ext extension. If dst_file_ext is "so", first compiles to ".o" and then links to ".so". Returns a tuple of dst_file_path, hash_key, source_code_path sorSLinking CompilationrrrrN)rrrrArr_z// rz cmd // z %s %s: %s)run_build_command)r#envzCOMPILE FAILED WITHz took z seconds. Command: z-%s %s skipped: %s since output already exists)8rur8rrrrrrrrrrrrr"ruse_binary_remote_cacheforce_disable_cachesrrrrrrrupload_to_binary_remote_cacherbinary_remote_cache_force_writerr#CUDACompileErrorrrr rr`r3rtriton.fb.re_build_helperrrrrr<STDOUTrr _record_compile_erroroutputrrrxrmrrr>)rrrrobj_pathrrrroperation_name key_with_extrrr:r> error_pathbinary_remote_cachefhr cmd_parts error_outputrerQ start_timererrorend_timelog_duration_msg cache_entryr~r~rrus          "           b zCUTLASSCodeCache.compiletuple[DLLWrapper, str, str]cC<|dkrtd|d||||\}}}t|||fSz Compiles source code and loads the generated .so file. Returns a tuple of DLLWrapper, hash_key, source_code_path rzCOnly support loading a .so file for now. Requested file extension: z. Source code: rrurrrr dst_file_pathrsource_code_pathr~r~rrD zCUTLASSCodeCache.load error_strrrrr>rrc Cst||g}|||||j|<t|}t|ddd } | |Wdn1s,wY|durCtjj rE| |tjj dSdSdS)Nr rrA) rrrrrrr8r"rrrr) rrrrrr>rrrrr~r~rrUs  z&CUTLASSCodeCache._record_compile_errorr)rrrrrwrXrwrr rrrrxrrxrrrwrxrrrxrrxrwr5rrxrrxrrrwrrrxrrxrwr)rrxrrxrrrrxr>rxrrrwr)rrrr3rb dataclassrrrWrrrrrLrr rrrrr8rurrr~r~r~rrcs@          rc@sBeZdZdZdZedddZe ddddZedddZdS)rcuCUDArwrcCstSr)r use_re_buildrHr~r~rrvrzzCUDACodeCache._use_re_buildNrrrrxrrrcCstj||||dS)N)r)rcuda_compile_commandrr~r~rrzszCUDACodeCache._compile_commandcCs"tttttg}|Sr)rrr_nvcc_compiler_options_nvcc_host_compiler_optionsr)rr r~r~rrs z CUDACodeCache._source_code_extrarrrr) rrrrrrrrrr~r~r~rrqs  rc@s~eZdZUejGdddZiZded<gZded<dZ dZ e dd d Z e dddZe ddddZe d ddZdS)!rc@seZdZUded<ded<dS)zROCmCodeCache.CacheEntryrxrr>N)rrrrWr~r~r~rrs  rrrrrrbFrwrcCstjtjdSr)rrrrr~r~r~rrLrzROCmCodeCache.cache_clearrrxrr5cCs.ttdgd|}t||j|d\}}||fSr)rr)r8r)rrr cuda_commandrrr~r~rr8s  zROCmCodeCache.writeNrrrc Csx|jsd|_tttt|||\}}||jvrddlm }t }|t j ||dtd}|v|dt|j |} t j | st|g| ||} t} | d} ztj| tjdt jd} td | Wntjy}zt| |j|d}~wwt}d || d | }t|ntd || t|| |j|<Wdn1swY|j|j ||fS) z Compiles source_code into a file with dst_file_ext extension, using the compile command specific for the ROCm platform. Returns a tuple of dst_file_path, hash_key, source_code_path TrrrrNr)r#r9rzCompilation output: %szCompilation took z seconds. Compile command: z+Skip compiling %s: output %s already exists)!_logged_compiler_versionrr`r3rxr*r8rrrrrrrrrrrr)r r3rr<rrr r#rrrmrrr>)rrrrrrrrr:r>rerrrrrrr~r~rrusN       zROCmCodeCache.compilercCrrrrr~r~rrrzROCmCodeCache.loadrrrrr)rrrrbrrrrWrrr rrLrr8rurr~r~r~rrs     1rc@rt)CodeCacheFuturerwCallable[..., Any]cCrrrwrr~r~rrr\zCodeCacheFuture.resultNrwr )rrrrr~r~r~rr rwr c@s$eZdZ d d dd Zdd d ZdS) LambdaFutureN result_fnr rFuture[Any] | NonerwrcCs||_||_dSr)rr)rrrr~r~rrs zLambdaFuture.__init__cCs|Sr)rrr~r~rrszLambdaFuture.resultr)rr rrrwrr)rrrrrr~r~r~rrs rc@s$eZdZdZd ddZd dd Zd S) StaticAutotunerFuturezM A statically launchable CachingAutotuner, loaded from TritonBundler static_autotunerrrrwrcCs||_d|_dSr)rreload_kernel_from_src)rrr~r~rrs zStaticAutotunerFuture.__init__cCsd|jdusJtd|jj|jd|jjd|jdd|jWdS1s+wYdS)Nz%StaticAutotunerFuture.warm_precompile)rF)warm_cache_only reload_kernelstatic_triton_bundle_key)rr rrecheck_autotune_cache precompilerr~r~rrs $zStaticAutotunerFuture.resultN)rrrrwr)rwrr)rrrr3rrr~r~r~rr s  rr)rrxrwrx)rrxrwr)rrrwrxr)rrr rrwrx)rrxrrxrrxrwr)rr)rrr rxrrxrwrx)rrrN)rrrrxr rxrrxrrxrr4rwr5)r9rxrwrx)FF) r=rxrrrrr>rrwr)rXrarwra)r]rrwrM)rrrrxrrrwr)rrrwrr) rgrhrrrrgrrrwr^)rerfrwrf)rrxrwr5)rrxrwrrwrx)rrxrrxrDrrwrxrr)rrxrVrrwrx)rrxrrrwr)r\rxr^rprwr)rrxrwr)rarxrrxrwrr)r>rxrwrx( __future__rrrnrbrrrimportlib.resourcesrjr"rloggingrrrrrrfrLr rrr&rrHrbisectrrrrrrdatetimerr r pathlibr r r rtypesrtypingrrrrrrrrtyping_extensionsrrr{torch._library.opaque_objectrrtorch.distributedrirnrrtorch._dynamo.device_interfacertorch._dynamo.excrtorch._dynamo.utilsrrr r!torch._inductorr"r#r$torch._inductor.codegen.commonr%r&r'torch._inductor.codegen.cudar(r,torch._inductor.codegen.rocm.compile_commandr)r*$torch._inductor.compile_worker.utilsr+torch._inductor.cpp_builderr,r-r.r/r0r1r2r3r4r5r6r7torch._inductor.cpu_vec_isar8!torch._inductor.custom_graph_passr9r:r;r<r=torch._inductor.freezing_utilsr>r?%torch._inductor.runtime.compile_tasksr@%torch._inductor.runtime.runtime_utilsrArBtorch._inductor.utilsrCrDrErFrGrH"torch._library.fake_class_registryrItorch._loggingrJtorch._subclasses.fake_tensorrKrLrMtorch._utils_internalrNtorch.compilerrtorch.compiler._cacherOrPrQ)torch.export.pt2_archive._package_weightsrRrS"torch.export.pt2_archive.constantsrT%torch.fx.experimental.symbolic_shapesrUrVrWtorch.utils._ordered_setrXrur[rr\runtimer]runtime.autotune_cacher^triton_bundlerr_ virtualizedr`racollections.abcrbrcrdreconcurrent.futuresrfrrgrrhrrjirrkrlrmrnro runtime.hintsrprqruntime.triton_heuristicsrrr1rsr0file_lock_timeoutr_logginggetArtifactLoggerrrautotuning_log getLoggerrrrrrrrrrrrrrr8r<rrrRr[rePicklerrfrrrrrrrrrdrsrtregisterrrOrr4r5rMrvrWrrrrrrrrrrrrrrWrVrrrrrrrrr rrr~r~r~rs|      (     8                           A 7   (     M  !  M  vY | 4 ?  J BDj*gK&c