+d+ ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z dZ dZ dejfdZddZdZd Zd Zdd ZddZddZddZdZeeeeeeeeeedd Z dZ!Gdde"Z#dZ$dS)N) Environment Undefinedmetac#Kt|} t|}n#t$rYdSwxYwt|tr|V= t|}t j||}n#t$r|VYnwxYww)z Flatten an arbitrarily deep listTN)iternext StopIteration isinstancestr itertoolschain Exception)iterableitemdatas 4lib/python3.11/site-packages/intake/catalog/utils.pyflattenrsH~~H >>DD    EE  dC  JJJ  ::D tX66HH   JJJJJ s$ 22$A55BBcFtjfd}|S)NcD||g|Ri|SN)reload)selfargskwargsfs rwrapperz!reload_on_change..wrapper*s0 q''''''') functoolswraps)rrs` rreload_on_changer )s8_Q((((( Nrc>t|t||S)zClamp float between given range)maxmin)valueloweruppers rclampr'2s uc%'' ( ((rct|tr|j}t|tr|j}tj||Sr)r r_undefined_nameosgetenv)xdefaults r _j_getenvr/7sH!Y '9%%*) 9Q  rct|tr|j} tj|S#t tf$rYdSwxYw)Nr()r rr* subprocess check_outputdecodeIOErrorOSErrorr-s r _j_getshellr7?se!Y &q))00222 W rrs%AAAcLt|tr|j}d|d|dS)Nz{{(z)}})r rr*)r-funcnames r_j_passthroughr;Hs0!Y #88QQQ ''rct|tr$fd|DSt|ttt fr(t |fd|DSt|trt}rst|j d<n d|j d<rrt|j d<n d|j d<rst|j d<n d|j d<rrt|j d <n d |j d <| |}tj|z||S|S) Nc @i|]\}}|t|S_expand).0kvall_varsclientcontextr,getshells r z_expand..Ps2aaatqRS71gxJJaaarc 3@K|]}t|VdSrr?)rArCrDrErFr,rGs r z_expand..Rs6ZZSTwq'8VVXNNZZZZZZrenvc$t|dS)NrKr:r;r6s rz_expand..Xs^A-N-N-Nr client_envc$t|dS)NrPrMrNr6s rrOz_expand..\sN1|4\4\4\rshellc$t|dS)NrRrMrNr6s rrOz_expand..`s~a'/R/R/Rr client_shellc$t|dS)NrTrMrNr6s rrOz_expand..dsnQQ_6`6`6`r)r dictitemslisttuplesettyper rr/globalsr7parserfind_undeclared_variables from_stringrender)prFrDrEr,rGjinjaasts ````` rr@r@Ns!TaaaaaaaaWXW^W^W`W`aaaa AeS) * *tAwwZZZZZZZZXYZZZZZZ As     O& O#,EM% #N#NEM%  ]f ]*3EM, ' '*\*\EM, '  SF S%0EM' " "%R%REM' "  a a,7EM. ) ),`,`EM. )kk!nnD23777  ##**7333rFTcXt|}t||||||}|r||fS|S)a; Render variables in context into the set of parameters with jinja2. For variables that are not strings, nothing happens. Parameters ---------- pars: dict values are strings containing some jinja2 controls context: dict values to use while rendering return_left: bool whether to return the set of variables in context that were not used in rendering parameters Returns ------- dict with the same keys as ``pars``, but updated values; optionally also return set of unused parameter names. )rZr@)parsrF return_leftrEr,rGrDouts rexpand_templatesrhms=*7||H $668 D DCH} Jrc tjd|}|rY|sW|rU|}tj|dt |dkr|dnd}tjd|}|r<|r:|r8tj|dd}tjd|}|r|s|r tj|d}tj | d}n#tj tf$rd}YnwxYwtjd|}|r|r|r tj|d}tj | d}n#tj tf$rd}YnwxYw|S) aCompile env, client_env, shell and client_shell commands Execution rules: - env() and shell() execute where the cat is loaded, if getenv and getshell are True, respectively - client_env() and client_shell() execute only if client is True and getenv/getshell are also True. If both getenv and getshell are False, this method does nothing. If the environment variable is missing or the shell command fails, the output is an empty string. zenv\((.*),?(.*)\)rr(zclient_env\((.*)\)z shell\((.*)\)utf8zclient_shell\((.*)\))rematchgroupsr+environgetlenshlexsplitr1r2rstripr3CalledProcessErrorr5)r.rEr,rGrgscmds rexpand_defaultsrys %w//AFFFF XXZZ*..AR1(D1"EE &00A4V44*..A33 !7++AH +ahhjjm,,C -c2299;;BB6JJGG-w7   GGG  ('22AV +ahhjjm,,C -c2299;;BB6JJGG-w7   GGG  Ns& A%EE! E!A%G&&HHc | |D]i}||j|j}|Et |t rt ||||}|| |j<j  fd| Dt| d|||\} fd D |D]M}|j vrB| |j |j< |jN||t| d|||}| fd| D| dd| D]4\ } fd |DD] } | | | <!5|S) a Produce open arguments by merging various inputs This function is called in the context of a catalog entry, when finalising the arguments for instantiating the corresponding data source. The three sets of inputs to be considered are: - the arguments section of the original spec (params) - UserParameters associated with the entry (spec_pars) - explicit arguments provided at instantiation time, like entry(arg=value) (user_inputs) Both spec_pars and user_inputs can be considered as template variables and used in expanding string values in params. The default value of a spec_par, if given, may have embedded env and shell functions, which will be evaluated before use, if the default is used and the corresponding getenv/getsgell are set. Similarly, string value params will also have access to these functions within jinja template groups, as well as full jinja processing. Where a key exists in both the spec_pars and the user_inputs, the user_input wins. Where user_inputs contains keys not seen elsewhere, they are regarded as extra kwargs to pass to the data source. Where spec pars have the same name as keys in params, their type, max/min and allowed fields are used to validate the final values of the corresponding arguments. Parameters ---------- params : dict From the entry's original spec user_inputs : dict Provided by the user/calling function spec_pars : list of UserParameters Default and validation instances client : bool Whether this is all running on a client to a remote server - sets which of the env/shell functions are in operation. getenv : bool Whether to allow pulling environment variables. If False, the template blocks will pass through unevaluated getshell : bool Whether or not to allow executing of shell commands. If False, the template blocks will pass through unevaluated Returns ------- Final parameter dict N)r,rGrEc$i|] \}}|v || Sr>r>)rArBrCrFs rrHzmerge_pars..s)MMMTQAWr>rArBrClefts rrHzmerge_pars..s(===119=q!===rFc$i|] \}}|v || Sr>r>r}s rrHzmerge_pars..s(EEEDAq19E1aEEEr CATALOG_DIRc*g|]}|jk |Sr>)name)rArarBs r zmerge_pars..s$77716Q;71777r) copyrprr.r r ryvalidateupdaterWrhremovepop)params user_inputs spec_parsrEr,rGparvalrgrCsprFrBr~s @@@r merge_parsrs)fkkmmG22ooch 44  2#s## \%c&8TZ[[[ # S 1 1GCH  NNMMMM[%6%6%8%8MMMNNN $QQIC==== ===G"" 8w  " # WSX-> ? ?GCH  KK ! ! ! MM#";PXYYK MMEEEEK$5$5$7$7EEEFFF JJ}d### ##%%//17777i777 / /B F1I..F1II / Mrcddl}|sd} t|d|vr|d}n#t$rYnwxYw||S)Nr __datetime__as_str)pandasr TypeError to_datetime)rCrs rcoerce_datetimersyMMM   Q Q  ( A        a  s & 33cddl}t|tr?|ttfvr/ |j|}n#t t tf$rYnwxYw||S)Nr)rcr r r literal_eval ValueErrorr RuntimeError)r$rulercs rwith_str_parsers~JJJ%$sO.D"D $C$U++EEI|4    D  4;;s<AAc|Srr>r6s rrOrO&sqr) booldatetimerVfloatrYmlistintrXr unicodeotherc d|vr.|d\}dnd||St|j|kr|S|dkrt |t t tfrt|St |trF tj |}t|S#t$r}td|d}~wwxYw|St|}| |nt||}t |tr" fd|D}t |t tt fr|fd|D}|S)a# Convert a value to a specific type. If the value is already the given type, then the original value is returned. If the value is None, then the default value given by the type constructor is returned. Otherwise, the type constructor converts and returns the value. []NrzFailed to coerce string to listcBi|]\}}|t|Sr>COERCION_RULES)rArBrCinners rrHzcoerce..Js,CCCtq!q.'**CCCrc3BK|]}t|VdSrr)rArCrs rrJzcoerce..Ls077a&q))777777r)rsrtr[__name__r rYrZrVrXr rcrrrrrW)dtyper$eoprgrs @rcoercer*s e|{{3'' u S!!   E{{u$    eeS$/ 0 0 ;;  eS ! ! K K(//E{{" K K K !BCCJ K  B >""$$$^E2%>%>C#tDDCCCCsyy{{CCC#tS)**8u8b7777377777 Js!"C C#CC#ceZdZdS)RemoteCatalogErrorN)r __module__ __qualname__r>rrrrPsDrrct}|D]5\}}||}tj|}d|vrdS6dS)z:Check is any value in args dict needs CATALOG_DIR variablerTF)rrWr]rr^)rrKrBargparsed_contentvarss r_has_catalog_dirrTsc --C**,,33-n== D  44  5r)r()FFTT)FTTr)%rcrr r+rlrrr1sysjinja2rrrrr maxsizer'r/r7r;r@rhryrrrrrVrrYrXrr rrrrrr>rrrs   //////////* )))) !!!!((( >8####LLLLL^!!!!"            [  ###L        r