ܙdz zdZddlZddlZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl Z ddlmZddlmZmZmZddlmZmZmZddlmZmZejeZdZdZd Z d Z!d Z"eZ#eZ$d %Z&e'e&D]\Z(Z)e*ej+ee)e(d Z,dZ-dZ.e j/Z0Gdde1Z2Gdde1Z3dZ4dddddddddddd Z5dZ6dZ7dZ8dZ9dZ:Gd d!e1Z;d"Zd%Z?d&Z@d'ZAd(ZBd)ZCd*ZDd+ZEd,ZFd-ZGd.ZHd/ZId0ZJejKd1ZLejKd2ZMejKd3ZNd4ZOejKeOPd5d6ZQejKd7ZRhd8ZSejKd9ZTejKd:ZUejKd;ZVejKd<ZWejKd=ZXejKd>ZYejKd?ZZejKd@Z[ejKdAZ\dBdCdDZ]dEZ^dFZ_dGZ`dHZadIZbdJZcdS)Kz( This is a direct translation of nvvm.h N)c_void_pc_intPOINTERc_char_pc_size_tbyrefc_char)ir) NvvmErrorNvvmSupportError NvvmWarning) get_libdeviceopen_libdevice open_cudalib)cgutilsconfiga NVVM_SUCCESS NVVM_ERROR_OUT_OF_MEMORY NVVM_ERROR_PROGRAM_CREATION_FAILURE NVVM_ERROR_IR_VERSION_MISMATCH NVVM_ERROR_INVALID_INPUT NVVM_ERROR_INVALID_PROGRAM NVVM_ERROR_INVALID_IR NVVM_ERROR_INVALID_OPTION NVVM_ERROR_NO_MODULE_IN_PROGRAM NVVM_ERROR_COMPILATION ze-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64ze-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64cF tdS#t$rYdSwxYw)z( Return if libNVVM is available TF)NVVMr 7lib/python3.11/site-packages/numba/cuda/cudadrv/nvvm.py is_availabler<s; t uus   ceZdZdZeeeeefeeefeeefeeee efeeee efeeeeefeeee feeefeeee feeefeeeeeeeeefeeeeefd Z dZ dZ dZ edZedZedZd Zd Zd d ZdS)rzProcess-wide singleton. ) nvvmVersionnvvmCreateProgramnvvmDestroyProgramnvvmAddModuleToProgramnvvmLazyAddModuleToProgramnvvmCompileProgramnvvmGetCompiledResultSizenvvmGetCompiledResultnvvmGetProgramLogSizenvvmGetProgramLog nvvmIRVersionnvvmVerifyProgramNct5|jt|x|_} t d|_n-#t $r }d|_d}t||zd}~wwxYw|j D]G\}}t|j|}|d|_ |dd|_ t|||Hdddn #1swxYwY|jS)Nnvvmz;libNVVM cannot be found. Do `conda install cudatoolkit`: %srr ) _nvvm_lock_NVVM__INSTANCEobject__new__rdriverOSErrorr _PROTOTYPESitemsgetattrrestypeargtypessetattr)clsinsteerrmsgnameprotofuncs rr/z NVVM.__new__s@  . .~%(.s(;(;;7".v"6"6DKK777%)CN2F*6A:666 7$(#3#9#9#;#;..KD%"4;55D#(8DL$)!""IDMD$----! . . . . . . . . . . . . . . .$~s5)C!AC! A1A,,A11A$C!!C%(C%c|}|d|_|d|_|d|_|d|_t |_dS)Nrr r)get_ir_version_majorIR_minorIR _majorDbg _minorDbgget_supported_ccs_supported_ccs)self ir_versionss r__init__z NVVM.__init__sS))++ #A #A $Q$Q/11rc&|j|jfdkS)N)r )rBrCrHs r is_nvvm70zNVVM.is_nvvm70s  t}-77rcB|j|jfdkrtStS)N)r )rBrC_datalayout_original_datalayout_i128rMs r data_layoutzNVVM.data_layouts" M4= )F 2 2' '# #rc|jSN)rGrMs r supported_ccszNVVM.supported_ccss ""rct}t}|t|t|}||d|j|jfS)NzFailed to get version.)rrr check_errorvalue)rHmajorminorerrs r get_versionzNVVM.get_versions[uU||U5\\:: 6777{EK''rcjt}t}t}t}|t|t|t|t|}||d|j|j|j|jfS)NzFailed to get IR version.)rr(rrXrY)rHmajorIRminorIRmajorDbgminorDbgr\s rrAzNVVM.get_ir_versions''''7777  ww!&x%//CC 9:::}gmX^X^KKrFc|rDt|t|}|r%t|tjddS|dS)Nr )r RESULT_CODE_NAMESprintsysexit)rHerrormsgrgexcs rrXzNVVM.check_errorsR  C!25!9::C c     r)F)__name__ __module__ __qualname____doc__ nvvm_resultrr nvvm_programrrr2r-r/rJpropertyrNrSrVr]rArXrrrrrKs $WWU^^WWU^^D*77<+@+@A +GGL,A,AB x8#E x8'E uggh.?.?A wwx'8'8&:#.|X!F#.|WWX=N=N!O*<B&wwu~~wwu~~!'%..''%..:*<%gh//1c33KlJ*22288X8 $$X$ ##X#(((LLLrrc8eZdZdZdZdZdZdZdZdZ dS) CompilationUnitct|_t|_|jt |j}|j|ddS)NzFailed to create CU)rr0rp_handlerrrX)rHr\s rrJzCompilationUnit.__init__sVff #~~ k++E$,,?,?@@ %:;;;;;rct}|t|j}||dddS)NzFailed to destroy CUT)rg)rr rrurX)rHr0r\s r__del__zCompilationUnit.__del__sJ''dl(;(;<<3 6TBBBBBrc|j|j|t|d}|j|ddS)z Add a module level NVVM IR to a compilation unit. - The buffer should contain an NVVM module IR either in the bitcode representation (LLVM3.0) or in the text representation. NFailed to add module)r0r!rulenrXrHbufferr\s r add_modulezCompilationUnit.add_modulesM k00v14VdDD %;<<<<z+CompilationUnit.compile..+s<*9*9*9./+3188F3C3C*D*D*9*9*9rzFailed to verify zFailed to compile z&Failed to get size of compiled result.zFailed to get compiled result.)categoryN)appendpopgetintboolreplacejoinmapreprkeysr formatrrzr0r)ru _try_errorr#rr$rr rYr%get_loglogwarningswarnr) rHoptionsopts other_optionskvoptstrc_optsr\reslenptxbufs rcompilezCompilationUnit.compilesD G   KK GKK$6$66 7 7 7 ;;v   : KK W[[%8%88 9 9 9  A AAG||W[[^^,,-- H #s(;(;Q'??@@@  EYYs48899F4;;FCCDD DSYY&*9*937*9*9*9:k++DL#d))VLL 1222k,,T\3t99fMM 2333k33DL%--PP EFFF6<'**k// fEE =>>><<>> 8 : M$([ 9 9 9 9aaayrcj|j||d|dS)N )r0rXr)rHr\ris rrzCompilationUnit._try_errorFs3 dllnnn%EFFFFFrct}|j|jt |}|j|d|jdkrkt|jz}|j|j|}|j|d|j dSdS)Nz#Failed to get compilation log size.r zFailed to get compilation log.r) rr0r&rurrXrYr r'decode)rHrr\logbufs rrzCompilationUnit.get_logIsk// eFmmLL %JKKK .usD111R2////////////rc2g|]}|tjk|Sr)rCUDA_DEFAULT_PTX_CC)rrs rrz(ccs_supported_by_ctk..zs/;;;Rv999999r) CTK_SUPPORTEDtupleCOMPUTE_CAPABILITIESKeyError) ctk_versionrrs @@rccs_supported_by_ctkrqs <&{311111#711122 2 <<<;;#7;;;<< < < <=%d.%d)r )rrVr enumerate)myccrVriirs rfind_closest_archrsFF(M $Qs###=)) , ,2 ::III $YYAvv+.2Ri9&s+++%QU++++  rc`tjr tj}nt||f}d|zS)z1Matches with the closest architecture option z compute_%d%d)r FORCE_CUDA_CCr)rZr[rs rget_arch_optionrs51# %00 D  rz~Missing libdevice file. Please ensure you have package cudatoolkit >= 11.0 Install package by: conda install cudatoolkit ceZdZdZdZdZdS) LibDeviceNc|j5tttt |_|j|_dSrU)_cache_r RuntimeErrorMISSING_LIBDEVICE_FILE_MSGrbcrMs rrJzLibDevice.__init__s= < &"#=>>>)++DL,rc|jSrU)rrMs rrz LibDevice.gets wr)rkrlrmrrJrrrrrrs7Grrz define internal {T} @___numba_atomic_{T}_cas_hack({T}* %ptr, {T} %cmp, {T} %val) alwaysinline {{ %out = cmpxchg volatile {T}* %ptr, {T} %cmp, {T} %val monotonic ret {T} %out }} z %cas_success = cmpxchg volatile {Ti}* %iptr, {Ti} %old, {Ti} %new monotonic monotonic %cas = extractvalue {{ {Ti}, i1 }} %cas_success, 0 zI %cas = cmpxchg volatile {Ti}* %iptr, {Ti} %old, {Ti} %new monotonic a define internal {T} @___numba_atomic_{T}_{FUNC}({T}* %ptr, {T} %val) alwaysinline {{ entry: %iptr = bitcast {T}* %ptr to {Ti}* %old2 = load volatile {Ti}, {Ti}* %iptr br label %attempt attempt: %old = phi {Ti} [ %old2, %entry ], [ %cas, %attempt ] %dold = bitcast {Ti} %old to {T} %dnew = {OP} {T} %dold, %val %new = bitcast {T} %dnew to {Ti} {CAS} %repeat = icmp ne {Ti} %cas, %old br i1 %repeat, label %attempt, label %done done: %result = bitcast {Ti} %old to {T} ret {T} %result }} a define internal {T} @___numba_atomic_{Tu}_inc({T}* %iptr, {T} %val) alwaysinline {{ entry: %old2 = load volatile {T}, {T}* %iptr br label %attempt attempt: %old = phi {T} [ %old2, %entry ], [ %cas, %attempt ] %bndchk = icmp ult {T} %old, %val %inc = add {T} %old, 1 %new = select i1 %bndchk, {T} %inc, {T} 0 {CAS} %repeat = icmp ne {T} %cas, %old br i1 %repeat, label %attempt, label %done done: ret {T} %old }} a define internal {T} @___numba_atomic_{Tu}_dec({T}* %iptr, {T} %val) alwaysinline {{ entry: %old2 = load volatile {T}, {T}* %iptr br label %attempt attempt: %old = phi {T} [ %old2, %entry ], [ %cas, %attempt ] %dec = add {T} %old, -1 %bndchk = icmp ult {T} %dec, %val %new = select i1 %bndchk, {T} %dec, {T} %val {CAS} %repeat = icmp ne {T} %cas, %old br i1 %repeat, label %attempt, label %done done: ret {T} %old }} a define internal {T} @___numba_atomic_{T}_{NAN}{FUNC}({T}* %ptr, {T} %val) alwaysinline {{ entry: %ptrval = load volatile {T}, {T}* %ptr ; Return early when: ; - For nanmin / nanmax when val is a NaN ; - For min / max when val or ptr is a NaN %early_return = fcmp uno {T} %val, %{PTR_OR_VAL}val br i1 %early_return, label %done, label %lt_check lt_check: %dold = phi {T} [ %ptrval, %entry ], [ %dcas, %attempt ] ; Continue attempts if dold less or greater than val (depending on whether min or max) ; or if dold is NaN (for nanmin / nanmax) %cmp = fcmp {OP} {T} %dold, %val br i1 %cmp, label %attempt, label %done attempt: ; Attempt to swap in the value %old = bitcast {T} %dold to {Ti} %iptr = bitcast {T}* %ptr to {Ti}* %new = bitcast {T} %val to {Ti} {CAS} %dcas = bitcast {Ti} %cas to {T} br label %lt_check done: ret {T} %ptrval }} ctjrt|St|S)NTi)rrN cas_nvvm70r cas_nvvm34rs rir_casrKs? vv(  B '''  B '''rc ft||||t|}tjdi|S)N)TrOPFUNCCASr)dictrir_numba_atomic_binary_templater)rrrrparamss rir_numba_atomic_binaryrRs7 A"$F2JJ ? ? ?F * 1 ; ;F ; ;;rc jt||||||t|}tjdi|S)N)rrNANr PTR_OR_VALrrr)rrir_numba_atomic_minmax_templater)rrrrrrrs rir_numba_atomic_minmaxrWsD A"#"---F + 1 ; ;F ; ;;rcVt||t|SN)rTur)ir_numba_atomic_inc_templaterrrrs rir_numba_atomic_incr^# ' . .rvayy . I IIrcVt||t|Sr)ir_numba_atomic_dec_templaterrrs rir_numba_atomic_decrbrrc|}t|D]G\}}|dr-d}|t j||<nHd|S)z@ Find the line containing the datalayout and replace it ztarget datalayoutztarget datalayout = "{0}"r) splitlinesr startswithrrrSr)llvmirlinesrlntmps r_replace_datalayoutrfs     E5!!2 ==, - - -Czz$&&"455E!H E  99U  rcdtddddfdtdd d d fd tddd d fd tddfdtddfdtdd ddddfdtddddddfdtdd ddddfdtddddddfdtdd ddddfdtddddddfd tdd dd!ddfd"tdddd!ddfd#g}t jsL|d$t d %fd&t d%fgz }t|}|D]\}}| ||}t jrt|}nt|}|S)'NzIdeclare double @"___numba_atomic_double_add"(double* %".1", double %".2")doublei64faddadd)rrrrzEdeclare float @"___numba_atomic_float_sub"(float* %".1", float %".2")floati32fsubsubzIdeclare double @"___numba_atomic_double_sub"(double* %".1", double %".2")z=declare i64 @"___numba_atomic_u64_inc"(i64* %".1", i64 %".2")u64rz=declare i64 @"___numba_atomic_u64_dec"(i64* %".1", i64 %".2")zEdeclare float @"___numba_atomic_float_max"(float* %".1", float %".2")rznnan oltptrmax)rrrrrrzIdeclare double @"___numba_atomic_double_max"(double* %".1", double %".2")zEdeclare float @"___numba_atomic_float_min"(float* %".1", float %".2")znnan ogtrzIdeclare double @"___numba_atomic_double_min"(double* %".1", double %".2")zHdeclare float @"___numba_atomic_float_nanmax"(float* %".1", float %".2")nanultzLdeclare double @"___numba_atomic_double_nanmax"(double* %".1", double %".2")zHdeclare float @"___numba_atomic_float_nanmin"(float* %".1", float %".2")ugtzLdeclare double @"___numba_atomic_double_nanmin"(double* %".1", double %".2"))immargrzMdeclare i32 @"___numba_atomic_i32_cas_hack"(i32* %".1", i32 %".2", i32 %".3"))rzMdeclare i64 @"___numba_atomic_i64_cas_hack"(i64* %".1", i64 %".2", i64 %".3")) rrrrrrNir_numba_cas_hackrrrllvm100_to_70_irllvm100_to_34_ir)r replacementsdeclfns r llvm_replacerss T (ue L L L N P 'eU K K K M T (ue L L L N H u / / / 1 H u / / / 1 P 'e +0u > > > ? U (u"+0u > > > ? Q 'e +0u > > > ? U (u"+0u > > > ? T 'e5+-E ; ; ; < X (u%E+-E ; ; ; < T 'e5+-E ; ; ; < X (u%E+-E ; ; ; < G$LL 66  -  \  % % % . . 0 \  % % % . . 0  %V,, **bb)) vv*!&))!&)) Mrc t|tr|g}|ddr|dddddt }t }|D]9}t |}||d:| | |j di|S)NfastmathFT)rrrrrr) isinstancestrrupdatersrrr}rrrr)rrcu libdevicemods r llvm_to_ptxr(s&# xx E""           B I**3 cjj(())))y}}''' 2:    rz \!\d+\s*=zmetadata\s*\![{'\"0-9]z\!\d+z,\!{i32 \d, \!\"Debug Info Version\", i32 \d} z\s+z"^attributes #\d+ = \{ ([\w\s]+)\ }> coldminsizeoptiszeoptnonenoinlinenoreturnnounwindreadnonereadonly inlinehint noduplicate alwaysinlinez"\bgetelementptr\s(?:inbounds )?\(?z=\s*\bload\s(?:\bvolatile\s)?z(call\s[^@]+\))(\s@)z\s*!range\s+!\d+z [,{}()[\]]z \bnonnull\bz"\b(local_unnamed_addr|writeonly)\bz\((.*)\)zspFlags: (.*), isDefinition isOptimized)DISPFlagDefinitionDISPFlagOptimizedcg}|jD]}|drt|}|d}dd|D}||d|}||d|S)z, Convert LLVM 10.0 IR for LLVM 7.0. attributes #r r)c3&K|] }|dk|V dS) willreturnNrras r z#llvm100_to_70_ir..s,CC1l1B1BQ1B1B1B1BCCrr) rrre_attributes_defmatchgroupsplitrrr)r buflinemattrss rrrs C  ??> * * 3!''--AGGAJJ$$&&EHHCCCCCCCE<< E22D 4 99S>>rcd}g}|jD]}|dr|dd}|drd|vr|dd}t|rdt |ur|dd }|d d }|d }|d|d z||d zd}}d}d |t ||f}|drJt|t d|}|drt|}|d } d d| D} ||d | }d|vr`t |}|t#d||} |d| ||| dz}d|vrNt&|}|r2|} |d| ||| dz}d|vrmt( d|}t* d|d}d|vr t. t0|}d|vr$d|vr t. t2|}t4 d|}||d |S)z, Convert LLVM 10.0 IR for LLVM 3.4. c8d}d} t||}|td||}|d}|dkr|dkrnn|dvr|dz }n |dvr|dz}z||dS)NrTzfailed parsing leading type: ,z{[(r z)]}) re_type_toksearchrendrClstrip)s par_levelposrGtoks rparse_out_leading_typez0llvm100_to_34_ir..parse_out_leading_types  ""1c**Ay"l#KLLL%%''C''!**Cczz>>"Q Q   w~~rz!numba.llvm.dbg.cuz !llvm.dbg.cuz%tail call void asm sideeffect "// dbgz !numba.dbgz!dbgNz!{z metadata !{z!"z metadata !"=r c2d|dzS)Nz metadata r)rCrGs rfix_metadata_refz*llvm100_to_34_ir..fix_metadata_ref/s&33rr)zsource_filename =cdS)NrrrWs rz"llvm100_to_34_ir..6srr;c3,K|]}|tv |VdSrU)supported_attributesr>s rr@z#llvm100_to_34_ir..<s-KK16J1J1JQ1J1J1J1JKKrzgetelementptr zfailed parsing getelementptr: zload zcall z\1*\2rrKz @llvm.memsetdeclarer)rrrrOre_metadata_defrBre_metadata_correct_usagerMfindrre_metadata_refrre_unsupported_keywordsrArCrDre_getelementptrrrNre_loadre_callre_rangerstripre_parenthesized_list_replace_llvm_memset_usage _replace_llvm_memset_declarationre_annotationsr) r rTrErFassigposlhsrhsrXrGrHrRs rrrs   , C KK ??/ 0 0 F<< 4nEED KKMM $ $%L M M 6 D((<< f55D   & & N077====||D-88||D-8899S>> A .X\]]0CS444xx!0!4!45Es!K!K!MNN ??. / /   " ) )$ / / ;*..||TBBD ??> * * 3!''--AGGAJJ$$&&EHHKKKKKKKE<< E22D t # #!''--Ay"l#OPPP%%''C: 6 6tCDDz B BBD d??t$$A GeeggDSDz$:$:4:$F$FF d??;;x..D<<D))0055D%%,00.   %%,004 !!"d++ 4 99S>>rct|dd}tjd|d}|st d|d}|dd|d|}d |S) zNReplace `llvm.memset` usage for llvm7+. Used as functor for `re.sub. r rKz align (\d+)rz+No alignment attribute found on memset destrzi32 {}r({})) listrCrDrerM ValueErrorinsertrr)rGr align_attralignouts rririis !''!**""3'' ( (F>6!955J $FGGG  ## MM"hooe,,--- ))F  C ==  rct|dd}|ddd|}d|S)zTReplace `llvm.memset` declaration for llvm7+. Used as functor for `re.sub. r rKrrrrp)rqrCrDrtrr)rGrrws rrjrjys` !''!**""3'' ( (F MM"e ))F  C ==  rcD|j}tj|d}tjtjdd}||||f}t j|d}|||j ddS)Nkernel r znvvm.annotationsr.) moduler MetaDataStringConstantIntType add_metadatarget_or_insert_named_metadatar  attributesdiscard)lfuncr'mdstrmdvaluemdnmds rset_cuda_kernelrs ,C  c8 , ,Ek"*R..!,,G  5%1 2 2B  .s4F G GCGGBKKK Z(((((rctjdfdtD}||}|d|dS)zAdd NVVM IR version to moduler{c&g|] }|Srr)rrrs rrz"add_ir_version..s!;;;a33q66;;;rznvvmir.versionN)r rrrAradd_named_metadata)r'rImd_verrs @radd_ir_versionrsk *R..C;;;;466#8#8#:#:;;;K   k * *F+V44444r)drnloggingrrrfrctypesrrrrrrr threadingllvmliter rhr r rlibsrrr numba.corerr getLoggerrkloggerADDRSPACE_GENERICADDRSPACE_GLOBALADDRSPACE_SHAREDADDRSPACE_CONSTANTADDRSPACE_LOCALrprorDrdrrrr7modulesrQrRrLockr,r.rrsrrrrFrrrrrrrrrrrrrrrrrrr(rr^r_radebuginfo_patternrre_metadata_debuginforAr\rcrdrerfrLrkrbrh re_spflags spflagmaprrrirjrrrrrrs/ ;;;;;;;;;;==========&&&&&&&&  8 $ $    EGG I' ( ())DAq GCK !1a((((;7   Y^   BBBBB6BBBJEEEEEfEEEP              < < <2D!!!          #, ( (#@(((<<< <<<JJJJJJ   <<<~."*\**&BJ'@AA"*X&&C" #4#<#