o kiK@sUdZddlmZddlZddlZddlmZerddlmZddl m Z ddl m Z ddl mZmZddlZdd lmZd d lmZd gZdad ed<dad ed<e GdddZe dqddZ drdsdd ZdtddZdud d!Zdvd.d/Zdwd6d7Zdxd>d?Z ed@Z!dydCdDZ"dzdFdGZ#    d{d|dPdQZ$ Rd}d~dWdXZ%   dddYdYddddZdd\d]Z&ddYdYddddZdd^d_Z'dddd`ddcddZ(    e R RdddfddgdhZ) e R RdddfddidjZ*ddfddmdnZ+ej,doedpdS)z PROTOTYPE! Flash Attention 3 implementation. For fp8: only supports forward pass right now. For fp16/bf16: supports forward and backward pass. ) annotationsN) TYPE_CHECKING)Callable) dataclass)cache) TypeVarTupleUnpack)Library) _registryregister_flash_attention_fa3zCallable | None _FA3_CUDA_FWD _FA3_CUDA_BWDc@s eZdZUded<dddZdS) _FA3HandlezLibrary | NonelibraryreturnNonecCsd|_tjddS)NF)rtorch_C_set_sdp_use_fa3)selfra/var/www/addictedbytheproject.nl/epg/venv/lib/python3.10/site-packages/torch/nn/attention/_fa3.pyremove*sz_FA3Handle.removeN)rr)__name__ __module__ __qualname____annotations__rrrrrr&s rdevice torch.devicerintcCstj|\}}|SN)rcudaget_device_capability)rmajor_rrr_get_device_major0sr&flash_attn_interface module_pathstrcCst|tjdttS)z Register FA3 flash attention kernels with the PyTorch dispatcher. Args: module_path: Python module path to the FA3 implementation. T)_fa3_import_modulerrrr_fa3_register_kernelsr(rrrr 6s  rcCszt|ttjdstd|dttjjds"td|dttjjds1td|dtjjjatjjj a dS)N flash_attn_3zModule 'z' does not expose FA3 kernelsfwdz%' does not expose FA3 forward kernelsbwdz&' does not expose FA3 backward kernels) importlib import_modulehasattrrops RuntimeErrorr-r.r r/rr,rrrr*Gs     r*r cCsdtddd}|dtd|dtd|dtd|dtd|dtd|d td|S) NatenIMPLCUDAz"_flash_attention_forward.quantizedz-_scaled_dot_product_flash_attention.quantized_flash_attention_forward#_scaled_dot_product_flash_attention_flash_attention_backward,_scaled_dot_product_flash_attention_backward)r impl!_fa3_flash_attention_forward_impl4_fa3_scaled_dot_product_flash_attention_forward_impl)_fa3_flash_attention_forward_impl_default<_fa3_scaled_dot_product_flash_attention_forward_impl_default"_fa3_flash_attention_backward_impl5_fa3_scaled_dot_product_flash_attention_backward_impl)librrrr+Xs0 r+query torch.Tensortensorstuple[torch.Tensor, ...] dropout_pfloat cum_seq_qtorch.Tensor | None q_descale k_descale v_descale str | NonecCs|dkrdStdd|DsdStdd|Ddkrd S|jtjkr6|dus0|dus0|dur6td t|durB|d krBd S|durN|d krNdStj sUdSt |j dkr^dSdS)Nzdropout_p must be 0css|]}|jVqdSr!)is_cuda.0trrr sz,_fa3_common_support_error..zinputs must be CUDA tensorscSh|]}|jqSr)rrRrrr z,_fa3_common_support_error..r inputs must share devicezWhen using SDPA with fp8, descale tensor should always be used for accurate dequantization. Please use _scaled_dot_product_attention_quantized and provide the descale tensors.zdense query must be 4Dzragged query must be 3DzCUDA not available z#FA3 requires compute capability 9.0) alllendtyper float8_e4m3fnwarningswarn UserWarningdimr" is_availabler&r)rDrFrHrJrLrMrNrrr_fa3_common_support_errorts*   rfkeyvaluereturn_debug_maskbool alibi_slopes seqused_kc s|rdS|dur dS|dur|jtjkrdS|jsdStjtjtjftfdd|||hDs6dStdd |||hDd krFd St ||||f|||| | } | dur_| d kr]d S| SdS)Nzreturn_debug_mask must be Falsezalibi_slopes not supportedzseqused_k must be int32zseqused_k must be CUDAc3|]}|jvVqdSr!r_rRsupported_dtypesrrrUz-_fa3_forward_support_error..inputs must be one of cSrVrrnrRrrrrWrXz-_fa3_forward_support_error..r #all inputs must have the same dtyperYz(query, key, value must be on same device) r_rint32rQr`float16bfloat16r]r^rf) rDrgrhrHrirkrlrJrLrMrNerrorrror_fa3_forward_support_errors8    rxgrad_outout logsumexpwindow_size_left int | Nonewindow_size_rightc s|jtjkr dS|jtjkrdStjtjftfdd|||||hDs,dStdd|||||hDdkr>d St|||||||f||ddd} | durT| SdS) NzHFA3 backward does not support fp8 - use inference only (torch.no_grad())zlogsumexp dtype must be float32c3rmr!rnrRrorrrUrqz._fa3_backward_support_error..rrcSrVrrnrRrrrrWrXz._fa3_backward_support_error..r rs) r_rr`float32rurvr]r^rf) ryrDrgrhrzr{rHrJr|r~rwrror_fa3_backward_support_errors,       rTs Unpack[Ts]tuple[Unpack[Ts]]cGstdd|DS)Ncss|] }|ddVqdS)r N) transposerRrrrrUsz#_transpose_dense..)tuple)rFrrr_transpose_densesrxcCs"|dur|ddkr|S|S)z2Ensure tensor is contiguous in the last dimension.Nr )stride contiguous)rrrr_maybe_contiguouss"rcu_seq_qcu_seq_kmax_qmax_kscale float | None is_causal!tuple[torch.Tensor, torch.Tensor]cCsDtdurtdt|}t|}|jtjkr(|ddkr(|ddkr(|nt|}t|}t|}t| } tg|||ddd| ||dd| ||dddddd| ||||| durr| nd| durz| ndddddtrdnddtj pdR\} }}}| |fS) zF Run the FA3 forward pass by calling the C++ kernel directly. NFA3 not registeredrr rrPT) r r4rr_rr`rr$are_deterministic_algorithms_enabledr_get_sm_carveout_experimental)rDrgrhrrrrrrr|r~rlrzrLrMrNqkv cu_seqlens_q cu_seqlens_k softmax_lse out_accumsoftmax_lse_accumrrr_fa3_run_forwards       ! " $rF max_seqlen_q max_seqlen_k deterministic/tuple[torch.Tensor, torch.Tensor, torch.Tensor]cCstdurtdt|}|ddkr|n|}|ddkr$|n|}|ddkr1|n|}t|}t|}t|}t|}t|}t|||||||||||dd|| | | | | d|tjpfd|||fS)Nrrr rPr) rr4rrrr empty_likerr)ryrDrgrhrzr{rrrrrrr|r~rdoutrrrolsedqdkdvrrr_fa3_run_backwardCsH     rrrr|r~rlrkrz cum_seq_kc Cst||||| |||| | | }|durtd|t|||||||| |||||| | | \}}tjdtj|jd}tjdtj|jd}tjd|j|jd}|||||fS)Nz)FA3 flash_attention forward unsupported: )r)r_rrr) rxr4rrzerosuint64remptyr_)rDrgrhrJrrrrHrrirLrMrNrr|r~rlrkrzrwr rng_state philox_offset debug_maskrrrr=~sJ r=c Cs.t|||||||||| ddd| | | | ||dS)Nr)r=)rDrgrhrJrrrrHrrirr|r~rlrkrzrrrr?s*r?)rr|r~runusedcCst||||||| ||| }|durtd|t}t|||||||||| || |dur/|nd|dur6|nd|\}}}|||fS)z0FA3 implementation of _flash_attention_backward.Nz*FA3 flash_attention backward unsupported: r)rr4rrr)ryrDrgrhrzr{rJrrrrHrrrrr|r~rwrrrrrrrrAsB  rArPrc Cst|||||ddd||| } | durtd| t|||\} } } |jtjkr+tjn|j}tj||d}|dd}| d}| d}t | | | dd|||||| ||||d\}}}}}| d}| d}||dd|||||f S)NzFA3 SDPA forward unsupported: rnr r)rrzrLrMrN) rxr4rr_rr`rvrrsizer=)rDrgrhrLrMrNrHrrirrwrrr out_dtypeout_bhsdout_bshd max_q_flash max_k_flashr%rrrrrrrrrr> sd       r>c Cst|||ddd||||d S)Nr)r>)rDrgrhrHrrirrrrr@gs r@ philox_seedrcCst||||||| ddd }|durtd|t|||||\}}}}}t||||||dd|| | | | | |d\}}}t|||\}}}|||fS)zCFA3 implementation of _scaled_dot_product_flash_attention_backward.NzFA3 SDPA backward unsupported: r)rr4rrA)ryrDrgrhrzr{rJrrrrHrrrrrw grad_out_tq_tk_tv_tout_trrrdq_outdk_outdv_outrrrrBs6   rBFA3) register_fn)rrrr )r')r(r)rr)r(r)rr)rr )rDrErFrGrHrIrJrKrLrKrMrKrNrKrrO)rDrErgrErhrErHrIrirjrkrKrlrKrJrKrLrKrMrKrNrKrrO)ryrErDrErgrErhrErzrEr{rErHrIrJrKr|r}r~r}rrO)rFrrr)rrKrrK)NNNN)"rDrErgrErhrErrKrrKrr rr rrrrjr|r}r~r}rlrKrzrKrLrKrMrKrNrKrr)F) ryrErDrErgrErhrErzrEr{rErrKrrKrr}rr}rrrrjr|r r~r rrjrr)NNN)&rDrErgrErhrErJrKrrKrr rr rHrIrrjrirjrLrKrMrKrNrKrrr|r r~r rlrKrkrKrzrK) rDrErgrErhrErJrKrrKrr rr rHrIrrjrirjrrr|r r~r rlrKrkrKrzrK)"ryrErDrErgrErhrErzrEr{rErJrKrrKrr rr rHrIrrjrrErrErrr|r}r~r})NNNrPFF)rDrErgrErhrErLrKrMrKrNrKrHrIrrjrirjrr)rPFF)rDrErgrErhrErHrIrrjrirjrr)ryrErDrErgrErhrErzrEr{rErJrKrrKrr rr rHrIrrjrrErrErr)-__doc__ __future__rr0ratypingrcollections.abcr dataclassesr functoolsrtyping_extensionsrrr torch.libraryr r __all__r rrrr&r r*r+rfrxrrrrrrr=r?rAr>r@rBregister_flash_attention_implrrrrs              % +&   \ FI:? K(5