o ki/@sUdZddlmZddlZddlmZddlmZddlm Z m Z ddl m Z m Z dd lmZe r7dd lmZddlZdd lmZd gZdad ed<eGdddZededdZ dfdgdd ZedhddZdidd Z !djdkd*d+Zdld4d5ZdmdZ dndAdBZ! dodpdIdJZ" KdqdrdNdOZ#dddddddPdsdTdUZ$ddddVdtdYdZZ% [ K Kdudd\dvd]d^Z&dd\dwdadbZ'ej(dcedddS)xzUBER PROTOTYPE!!!) annotationsN) dataclass)cache)Any TYPE_CHECKING) TypeVarTupleUnpack) _registry) ModuleType)Libraryregister_flash_attention_fa4 str | None_FA4_MODULE_PATHc@s eZdZUded<dddZdS) _FA4HandlezLibrary | NonelibraryreturnNonecCs d|_dSN)r)selfra/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/nn/attention/_fa4.pyremove"s z_FA4Handle.removeN)rr)__name__ __module__ __qualname____annotations__rrrrrrs rdevice torch.devicerintcCstj|\}}|Sr)torchcudaget_device_capability)rmajor_rrr_get_device_major&sr%flash_attn.cute.interface module_pathstrcCst|}|attS)z Register FA4 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA4 implementation. )_fa4_import_modulerr_fa4_register_kernels)r'r$rrrr ,s  r cCs2t|}t|drt|dstd|d|S)N_flash_attn_fwd_flash_attn_bwdzModule 'z' does not expose FA4 kernels) importlib import_modulehasattr RuntimeError)r'modulerrrr);s r)r cCsHtddd}|dtd|dtd|dtd|dtd|S)NatenIMPLCUDA_flash_attention_forward_flash_attention_backward#_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward)r impl!_fa4_flash_attention_forward_impl"_fa4_flash_attention_backward_impl4_fa4_scaled_dot_product_flash_attention_forward_impl5_fa4_scaled_dot_product_flash_attention_backward_impl)librrrr*Cs r*rquery torch.Tensortensorstuple[torch.Tensor, ...] cum_seq_qtorch.Tensor | None require_fp32$tuple[tuple[str, torch.Tensor], ...]cCstdd|Ds dStdd|DdkrdS|jtjtjfvr#dS|D]\}}|jtjkr6|d Sq%|durC|d krCd S|durO|d krOd Stj sVdSt |j dvr_dSdS)Ncss|]}|jVqdSr)is_cuda.0trrr Zsz,_fa4_common_support_error..zinputs must be CUDA tensorscSsh|]}|jqSr)rrHrrr \sz,_fa4_common_support_error..r inputs must share devicez'query dtype must be float16 or bfloat16z dtype must be float32zdense query must be 4Dzragged query must be 3DzCUDA not available) z+FA4 requires compute capability 9.0 or 10.0) alllendtyper float16bfloat16float32dimr! is_availabler%r)r?rArCrEnametensorrrr_fa4_common_support_errorTs&   r\keyvalue dropout_pfloatreturn_debug_maskbool alibi_slopes seqused_kc Csp|dkrdS|r dS|durdS|dur!|jtjkrdS|js!dSt||||f|}|dur6|dkr4dS|SdS) Ndropout_p must be 0zreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDArMz(query, key, value must be on same device)rTr int32rGr\) r?r]r^r_rarcrdrCerrorrrr_fa4_forward_support_errorns*  rigrad_outout logsumexpwindow_size_left int | Nonewindow_size_rightc CsR|dkrdS|dus| durdSt|||||||f|d|ffd} | dur'| SdS)Nrerfz windowed attention not supportedrl)rE)r\) rjr?r]r^rkrlr_rCrmrorhrrr_fa4_backward_support_errors rpTs Unpack[Ts]tuple[Unpack[Ts]]cGstdd|DS)Ncss|] }|ddVqdS)r N) transposerHrrrrKsz#_transpose_dense..)tuple)rArrr_transpose_densesrwcu_seq_qcu_seq_kscale float | None is_causal!tuple[torch.Tensor, torch.Tensor]c  Csttdurtdtt} ||||d||| dur| ndd} | dur'| | d<| j|||fi| \} } | | fS)NFA4 not registeredT) softmax_scalecausalrmro return_lse cu_seqlens_q cu_seqlens_krdrk)rr0r) contiguousr+)r?r]r^rxryrzr|rmrordrkr1kwargslserrr_fa4_run_forwards    rF deterministic/tuple[torch.Tensor, torch.Tensor, torch.Tensor]c  CsLtdurtdtt} | j|||||||| ||| d \} } }| | |fS)Nr~)rrrrr)rr0r)r,r)rjr?r]r^rkrlrxryrzr|rr1dqdkdvrrr_fa4_run_backwards"  r)rzrmrordrcrk cum_seq_kmax_qmax_kc  Cst||||| || |}|durtd|t|||||| || | | | \}}tjdtj|jd}tjdtj|jd}tjd|j|jd}|||||fS)Nz)FA4 flash_attention forward unsupported: )rt)rTrrr) rir0rr zerosuint64remptyrT)r?r]r^rCrrrr_r|rarzrmrordrcrkrhr rng_state philox_offset debug_maskrrrr:s:  r:)rzrmrorunusedc Csdt||||||| ||| }|durtd|t}t|||||||||| | \}}}|||fS)Nz*FA4 flash_attention backward unsupported: )rpr0r $are_deterministic_algorithms_enabledr)rjr?r]r^rkrlrCrrrr_r|rrrzrmrorhrrrrrrrr;$s:  r;rerzcCst|||||ddd}|durtd|t|||\}} } t|} | dd} |d} | d}t|| | dd| |||||| d \}}}}}|d}|d}| |dd|||||f S)NzFA4 SDPA forward unsupported: r rt)rzrk)rir0rwr empty_likerusizer:)r?r]r^r_r|rarzrhqkvout_bhsdout_bshd max_q_flash max_k_flashr$rrrrrrrrrr<WsV        r< philox_seedrcCst||||||| ddd }|durtd|t|||||\}}}}}|d}|d} t||||||dd|| | | | | |d\}}}t|||\}}}|||fS)NzFA4 SDPA backward unsupported: rtr)rpr0rwrr;)rjr?r]r^rkrlrCrrrr_r|rrrzrhrrrogorrrrrrr=sH     r=FA4) register_fn)rrrr)r&)r'r(rr)r'r(rr )rr )r) r?r@rArBrCrDrErFrr)r?r@r]r@r^r@r_r`rarbrcrDrdrDrCrDrr)rjr@r?r@r]r@r^r@rkr@rlr@r_r`rCrDrmrnrornrr)rArrrrsr)r?r@r]r@r^r@rxrDryrDrzr{r|rbrmrnrornrdrDrkrDrr})F)rjr@r?r@r]r@r^r@rkr@rlr@rxrDryrDrzr{r|rbrrbrr) r?r@r]r@r^r@rCrDrrDrrrrr_r`r|rbrarbrzr{rmrnrornrdrDrcrDrkrD)"rjr@r?r@r]r@r^r@rkr@rlr@rCrDrrDrrrrr_r`r|rbrr@rr@rzr{rmrnrorn)reFF)r?r@r]r@r^r@r_r`r|rbrarbrzr{)rjr@r?r@r]r@r^r@rkr@rlr@rCrDrrDrrrrr_r`r|rbrr@rr@rzr{))__doc__ __future__rr- dataclassesr functoolsrtypingrrtyping_extensionsrrr typesr r torch.libraryr __all__rrrr%r r)r*r\rirprqrwrrr:r;r<r=register_flash_attention_implrrrrsl             !  , ,B7M8