\`hP ddlmZddlZddlZddlZddlZddlmZddlm Z ddl Z ddl Z ddl m Z m Z mZddl mZddlmZmZmZmZddlmZdd lmZmZmZ dd lmZn #e$rdZYnwxYw dd lmZm Z n#e$rdZdZ YnwxYw dd l!m"Z"n #e$rdZ"YnwxYwd Z#dZ$dZ%dZ&dZ'dZ(dZ)dZ*dZ+dZ,iZ-de.e DZ/GddeZ0GddeZ1GddeZ2GddeZ3ej j4e j5d ej j6e j5d!ej j7e j5d"ej j8e j5d#ej j9eej j:eej j;e iZ$e j=d%e j>$fej j4e j=d&e j?$e j=d'e j?$fej j8e j=de j@$e j=d(e j@$fej j7e j=d)e jA$e j=d*e jA$fej j;e j=de $e j=d+e $fej j:e j=d,e$e j=d-e$fiZBej j6e j=de j>$e j=d.e j>$fej j4e j=d/e j?$e j=d'e j?$fej j8e j=de j@$e j=d0e j@$fej j7e j=d1e jA$e j=d*e jA$fiZCej j6e j=de j>$e j=d'e j>$fej j4e j=d2e j?$e j=d3e j?$fej j8e j=de j@$e j=d*e j@$fej j7e j=d4e jA$e j=d5e jA$fej j;e j=de$e j=d-e$fej j:e j=d6e$e j=d7e$fiZDd8d9d:ZEdd;ZFdd=ZGd>ZH dddLZI dddNZJ dddWZKddXZLddYZMdd]ZNddaZOGdbdcZPGdddeZQGdfdgZRdhZSdiZTdjZUdkZVddpZWdqZXddsZYdduZZddwZ[ddyZ\dd|Z]dd}Z^dd~Z_ddZ`ddZaddZbddZcddZdddZeddZfddZgddZhddZiddZjddZkdS)) annotationsN)Enum)Path) ModelProto TensorProtoexternal_data_helper)onnx_pb) make_graph make_model make_nodemake_tensor_value_info)ReferenceEvaluator)GraphOptimizationLevelInferenceSessionSessionOptions float8e4m3fn)int4uint4)to_array_extendedz onnx.quantizez0.1.0ai.onnxz com.microsoftQuantizeLinear_QuantizeLinear_InputDequantizeLinear_DequantizeLinear_Output _quantizedlci|]@}ttt|t*tt||AS) isinstancegetattrrint).0ks j/var/www/html/movieo_spanner_bot/venv/lib/python3.11/site-packages/onnxruntime/quantization/quant_utils.py r%9sAqqqq SZ[fhiSjSjloHpHpq Q''qqqc2eZdZdZdZdZedZdS)QuantizationModerc|jSNnameselfs r$__str__zQuantizationMode.__str__D yr&cV t|S#t$rtwxYwr+)r(KeyError ValueError)modes r$ from_stringzQuantizationMode.from_stringGs7 #D) )   ,,   (N)__name__ __module__ __qualname__ IntegerOps QLinearOpsr0 staticmethodr6rr&r$r(r(@sHJJ\r&r(c2eZdZdZdZdZedZdS)QuantizedValueTyperr)c|jSr+r,r.s r$r0zQuantizedValueType.__str__Sr1r&cV t|S#t$rtwxYwr+)r?r3r4)vs r$r6zQuantizedValueType.from_stringVs7 %a( (   ,,  r7N)r8r9r:Input Initializerr0r=r6rr&r$r?r?OsH EK\r&r?c\eZdZdZdZdZdZdZdZdZ dZ e d Z e d Zd S) QuantTyperr)c|jSr+r,r.s r$r0zQuantType.__str__gr1r&cV t|S#t$rtwxYwr+)rFr3r4)ts r$r6zQuantType.from_stringjs6 Q<    ,,  r7c|tjkr tjS|tjkr tjS|tjkr tjS|tjkr tj S|tj kr tj S|tj kr tj S|tjkr tjSt!d|d)NzUnexpected value qtype=.)rFQInt8rINT8QUInt8UINT8QUInt16UINT16QInt16INT16 QFLOAT8E4M3FN FLOAT8E4M3FNQUInt4UINT4QInt4INT4r4r.s r$ tensor_typezQuantType.tensor_typeqs 9? " "# # 9# # #$ $ 9$ $ $% % 9# # #$ $ 9* * *+ + 9# # #$ $ 9? " "# #<4<<<===r&N)r8r9r:rQrSrYrWrUr]r[r0r=r6propertyr_rr&r$rFrF^s| E FM FG E F\ >>X>>>r&rFc2eZdZdZdZdZedZdS) QuantFormatrr)c|jSr+r,r.s r$r0zQuantFormat.__str__r1r&cV t|S#t$rtwxYwr+)rbr3r4)formats r$r6zQuantFormat.from_strings7 v& &   ,,  r7N)r8r9r: QOperatorQDQr0r=r6rr&r$rbrbsHI C\r&rbint8uint8int16uint16dtypeiiiiiiiii@ii@rHzero_point_indexcTg}t|D]\}}tjt|tjr(|tj|nEt|tjr||ntd|d|||krI|d}|j tj ks|j tj krtd|j t|dkrt|n|dS)Nzarg z is not an array: ruzzero_point cannot be r)r) enumeratenumpy issubdtypetypenumberappendarrayrndarray TypeErrorrmfloat32float16lentuple)rwargsnew_argsiarBs r$ _check_typersH$ C C1  DGGU\ 2 2 = OOEKNN + + + + 5= ) ) = OOA    ;1;;;;<< <   Aw%-''17em+C+C A A ABBB!(mma//5???Xa[@r&c|tvsJd|d|tjjtjjtjjtjjfvrF|dkrtd|d|jtj kr tj }n:|jtj kr tj }ntd|jdtt!t#dgdgt$jd|gdg t#d gd d ggd t+d|dt+d|dgt+d |dg}t-|}t/|d||ddSt|} t3|dd\} } |t5| |n| } |t7| |n| } tj|tj |z |z}tj|| | |t/|| S)NUnexpected data type > requested. Only INT8, UINT8, INT16, and UINT16 are supported.rz2zero_point is expected to be null for float 8 not rPzUnexpected dtype Constant zero_point)valuer)XscalerYqurr)rrF) reduce_range symmetric)out) ONNX_TYPE_TO_NP_TYPE onnx_protorrZFLOAT8E4M3FNUZ FLOAT8E5M2FLOAT8E5M2FNUZNotImplementedErrorrmrzrFLOATrFLOAT16r4r r r onnxhelper make_tensorr rrrunget_qmin_qmax_for_qTypemaxminasarrayastyperoundclip)qTypearrrrlowhigh onnx_type onnx_modelrefrmqminqmaxcliplowcliphigharr_fp32s r$quantize_nparrayrsv ( ( ( (eeee ) ( ( +-)-  ??%&j[e&j&j&jkk k 9 % %#)II Y%- ' '#+II====>> > "Bdk>U>UVbdikmpqor>s>s.0L0L0LseTT  *3 4@@*7ItDD(UD99:    !,,3774sU)C)CDDQGHHH%U+,URWXXX d$'O#dC...&*&63tT???D=#**U]";";e"C!J!J!L!Lz!YZZ 8WhH====8??511222r&Fc |dks|dkrtd|d|tj|tjd|j}tj|tjd|j}|,t ||tj||jz}|r?tjtj|tj|}| }| }||ksJd|d|tj||z tj }tj|tj tj|tj z }tj||z } | dks Jd| tj |jj kr7tjd |j} tjd|j} n|rRtjtj ||ztjd tj z |j} n3tjtj ||| z z |j} | |j} | | gS) aCalculate the scale s and zero point z for the quantization relation r = s(q-z), where r are the original values and q are the corresponding quantized values. r and z are calculated such that every value within [rmin,rmax] has an approximate representation within [qmin,qmax]. In addition, qmin <= z <= qmax is enforced. If the symmetric flag is set to True, the interval [rmin,rmax] is symmetrized to [-absmax, +absmax], where absmax = max(abs(rmin), abs(rmax)). :parameter rmin: minimum value of r :parameter rmax: maximum value of r :parameter qmin: minimum value representable by the target quantization data type :parameter qmax: maximum value representable by the target quantization data type :parameter symmetric: True if the floating-point range should be made symmetric. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :return: zero and scale [z, s] rBqmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:, qmmax:rlNzqmin=z > qmax=z scale issue?g@)r4rzminimumrrmmaximumrrabsfloat64finfotinyrr) rminrmaxrrrmin_real_rangeabsmaxdrdqrrs r$compute_scale_zprsQ( axx4!88r^brrlprrsss =u{1DJ??? @ @D =u{1DJ??? @ @D!4 nDJ O O OOPPuy $@@ww 4<<<555t55<<< TD[ 6 6 6B T / / /%+d%-2X2X2X XB KR E A:::}::: u{4:&&+++ Ctz222[$*555  Y  TD[EK5=,Q,Q,QQRRZ^ZdJJU[u 1D%E%ETZXXXJ TZ((  r&c& d}|tvr|tjkrTddlm ddlm}|} fdtdD}tj d|Dtj }ntd |d |t|<n|tjkrddlm}|}|td |d tj t|}tj d|}tj ||z |j}||gS) arCalculate the scale s for a float8 type (E4M3FN). The function assumes the coefficient distribution and the float 8 distribution are similar to two gaussian laws. :return: zero and scale [z, s] More details in notebook `quantization_fp8.ipynb `_. Nr)float8e4m3_to_float32rc&g|] }|Srr)r"rrs r$ z+compute_scale_zp_float8..Ks%GGGq//22GGGr&cbg|],}tj|tj|*|-Sr)rzisnanisinf)r"fs r$rz+compute_scale_zp_float8..Ms3TTTqek!nnTU[QR^^TTTTr&rlzQuantization to element_type=z not implemented.zUnexpected element_type rP)FLOAT8_DISTRIBUTIONSrrZonnx.numpy_helperr#onnx.reference.custom_element_typesrrangerzrrr4rstdrm) element_typerzp_dtyper all_valuesvaluesstd_f8zerorrs @r$compute_scale_zp_float8r:sWH/// ;3 3 3 ? ? ? ? ? ? H H H H H H#HGGGGE#JJGGGJ[TTJTTT\a\iFF\\\\\]] ]-3\** 1 1 1DDDDDDB<BBBCCC Y+L9 : :F ;q ) ) )D Kf CI 6 6 6E %=r&data numpy.ndarray quant_typeonnx.TensorProto.DataTyperboolrr float | None rmin_override rmax_overridereturn#tuple[numpy.ndarray, numpy.ndarray]ct|tjs tdt |d||}n%t |r|nd}||}n%t |r|nd}tj||j }tj||j }tjd|j } |tj krJ|rtdtj |} t|| \} } t| | d S|tjtjtjtjtjtjfvrit-||| \} } t |rt/||| | ||\} } ntjd| j } t| | d St1d |d) a Returns the zero_point and scale for the given data. :param data: The data for which to compute quantization parameters. :param quant_type: The quantization data type. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: zero point and scale z%Weight must be given as an array not rPNgrlrz1Unsupported option reduce_range=True for float 8.rrvrz Unexpected value for quant_type=)rrzrrr|rrrrrmrrZ RuntimeErrorrrrrRrTrXrVr^r\rrr4)rrrrrrrrrrrrrrs r$compute_data_quant_paramsr_s* dEM * *OMT MMMNNN  YY/txxzzzC  YY/txxzzzC ;t4: . . .D ;t4: . . .D K4: . . .E[---  TRSS Sioo3JDD E:uqAAAA -ZQZ[[[ d t99 : 0tT4Tb c c JQdj999J:uqAAAA E EEE F FFr&2tuple[numpy.ndarray, numpy.ndarray, numpy.ndarray]c t|||||||\}}|tjkrt||||} t | t jdzdkrxt j |} td| d| d| d| d ||| fS|tj tjtjtjtjtjfvrt||||} ||| fSt'd|d)al :param data: data to quantize :param qType: data type to quantize to. :param symmetric: whether symmetric quantization is used or not. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation - when data `type == uint8` mode, from `[rmin, rmax]` -> :math:`[0, 2^{b-1}]` and - when data `type == int8`, from `[-m , m]` -> :math:`[-(2^{b-1}-1), 2^{b-1}-1]` where `m = max(abs(rmin), abs(rmax))` and add necessary intermediate nodes to transform quantized weight to full weight using the equation :math:`r = S(q-z)`, where - *r*: real original value - *q*: quantized value - *S*: scale - *z*: zero point roz+One of the quantized value is NaN data in [z, z], quantized_data in [z].zUnexpected value for qType=rP)rrrZranyrrzriravelrrrrrRrTrXrVr^r\r4) rrrrrrrrrquantized_datanp_datas r$ quantize_datars82  J ((()%ujII %%ek2288::S@SH I I mD))GWgkkmmWWw{{}}WW&4&8&8&:&:WW>L>P>P>R>RWWW 5.00  *%ujII5.00 ;5;;; < <JJJRRSYS^__ #0;;M=YY r&c|tjjkrtdd}|rt|}n3|r|t vrt |}nt|}|std|d|\}}|dks|dkr&td|d|d|j d |d |d | |S) z Return qmin and qmax, the minimum and maximum value representable by the given qType :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 :return: qmin, qmax z;This function is not implemented for float 8 as not needed.Nrrrrrz, dtype=z, reduce_range=z , symmetric=z, qType=) rrrZrONNX_INT_TYPE_REDUCED_RANGEgetONNX_INT_TYPE_SYMMETRIC_RANGEONNX_INT_TYPE_RANGEr4rm)rrrqrangerrs r$rr)s#  &333!"_``` F0,0077 0u ===.u5$((// xvvvvwwwJD$ axx4!88 4 4 4"& 4 404  4 4KW 4 4" 4 4,1 4 4   Mr&c6t|||\}}||z S)z Helper function to get the quantization range for a type. parameter qType: quantization type. return: quantization range. r)r)rrrrrs r$get_qrange_for_qTyper'Is& ) RRRJD$ $;r&r!ranktuple[bool, int]c<|dkr||zn|}|dko||k}||fS)z Helper function that tries to return a normalized axis in the range [0, rank - 1]. :parameter axis: The axis to normalize. :parameter rank: The tensor rank (number of dimensions). :return (is_valid, axis_norm) rr)rr( axis_normis_valids r$normalize_axisr-Ss7 $axxt TIA~2)d"2H Y r&src_8bitr  bytearrayc"t|}|dkrtS|dzdz}t|}d}d}||dz kr3||dzdzdz||dzz||<|dz }|dz }||dz k3||kr||dz||<|S)aB Copies a source array of 8-bit values into a destination bytearray of packed 4-bit values. Assumes that the source values are already in the appropriate int4 range. :parameter src_8bit: The 8-bit element values to pack. :return A bytearray with every two 8-bit src elements packed into a single byte. rr)rGrprI)rr/)r. num_elemsdst_sizedstsrc_idst_is r$r r _sH IA~~{{A !#H H  CEE )a-   *S0Q68E?S;PQE      )a-    ye_s*E Jr&c eZdZdZggdfdZdS)QuantizedInitializerzJ Represents a linearly quantized weight input from ONNX operators Nc ||_||_||_||_||_||_||_||_| |_dSr+) r- initializerrminsrmaxs zero_pointsscalesrrr) r/r-r9r:r;r<r=rrrs r$__init__zQuantizedInitializer.__init__sJ &  &  , r&r8r9r:__doc__r>rr&r$r7r7}s= r&r7c"eZdZdZ ddZdS)QuantizedValuezI Represents a linearly quantized value (input\output\intializer) Nc ||_||_||_||_||_||_||_||_| |_dSr+) original_nameq_name scale_namezp_name value_typer node_type node_qtype scale_type) r/r-new_quantized_namerFzero_point_namequantized_value_typerrIrJrKs r$r>zQuantizedValue.__init__sH"( $& . "$$r&)NNNNr?rr&r$rBrBs@%%%%%%r&rBceZdZdZdZdS)BiasToQuantizez+ Represents a bias to be quantized c0||_||_||_dSr+) bias_name input_name weight_name)r/rRrSrTs r$r>zBiasToQuantize.__init__s"$&r&Nr?rr&r$rPrPs-'''''r&rPc|jdkrtd|jd|jdkr|j}n|jdkr|j}n|jdkr|j}n|jdkr|j}n|jdkr|j}n|jd kr|j}nl|jd kr|j }nY|jd kr|j }nF|jd kr|j }n3|jd kr|j }n td|jd|jd|j|iS)z Convert attribute to kwarg format for use with onnx.helper.make_node. :parameter attribute: attribute in AttributeProto format. :return: attribute in {key: value} format. rz attribute z does not have type specified.r)rGrHrIrJrKrq rz has unsupported type rP) r|r4r-rrsrNgfloatsintsstringstensorsgraphs) attributers r$attribute_to_kwargr`sB ~TinTTTUUU~  1    1    1    1    1    1   1  ! 1  ! 2   ]in]]IN]]]^^^ NE ""r&cZfd|D}t|dkr|dndS)z Helper function to find item by name in a list. parameter item_name: name of the item. parameter item_list: list of items. return: item if found. None otherwise. c*g|]}|jk |Srr,)r"item item_names r$rz find_by_name..s% B B Bd49 +A+AT+A+A+Ar&rN)r)rd item_listitemss` r$ find_by_namergs; C B B Bi B B BE5zzA~~5884/r&cdd}tt|D]}|||kr|}|S)zC Helper function to return index of an item in a node list ru)rr) elem_name elem_listelem_idxrs r$get_elem_indexrls@H 3y>> " " Q<9 $ $H Or&cHtjd||g|S)z Helper function to create a Mul node. parameter inputs: list of input names. parameter output: output name. parameter name: name of the node. return: Mul node in NodeProto format. Mul)rrr )inputsoutputr-s r$ get_mul_noderqs" ; $ ? ??r&filenamer identifierr cV|j|j|z|jzS)zp Helper function to generate a identifiable filepath by concatenating the given identifier as a suffix. )parentjoinpathstemsuffix)rrrss r$generate_identified_filenamery s( ? # #HMJ$>$P Q QQr&cddl}ddlm}ddl}|j|jt dt |t dt ||||d|d| d| d | dS) Nr) thresholdz Histogram:zHistogram Edges:T)fillz Tensor valueCountszTensor value V.S. Counts) sysmatplotlib.pyplotpyplotrzset_printoptionsmaxsizeprintstairsxlabelylabeltitleshow)hist hist_edgesr~pltrzs r$ apply_plotrsJJJ######LLLES[1111 , $KKK  *JJtZdJ+++JJ~JJxII()))HHJJJJJr&rPc  ddlddl}ddlddlmcmcm}ddlmcmcm}ddl m m m tjd|Gfddj}||}t#t$j|dd 5}||dddn #1swxYwYd}|d } g} t1|D]:} || } | } t7| d |t7| d |g}t=t?|}| | }| |}|!| |"| ||#| ||$| }| %|<|&| tO| | D]}| (|| )}|*| |+| ||,| }| -|| .}t#t$j|d d5}||dddn #1swxYwYt$j/dddvr|j0|d}|1}te|D]c}|3|}tj|4tj|5dt#t$j|dd 5}t1|D]} || } | } t7| d |t7| d |g}| dzt=t?|z}|||d ddddS#1swxYwYdS)z> Helper function to write calibration table to files. rN)CalibrationMethod TensorData TensorsDatazcalibration cache: c"eZdZfdZdS)*write_calibration_table..MyEncoderc\t|fr|St|jr*|t |jddSt|r|jjt |dSj ||S)Nz numpy.array)rrmCLS)rr) rto_dictrtolistr rm __class__r8 JSONEncoderdefault)r/objrrrjsonnps r$rz2write_calibration_table..MyEncoder.default3s# K899 %{{}}$#rz** ] # s39~~m\\\#011 J"}5CIII#++D#66 6r&N)r8r9r:r)rrrrrsr$ MyEncoderr2sB 7 7 7 7 7 7 7 7 7 7 7r&r)clszcalibration.jsonwihighestlowestzcalibration.flatbufferswbQUANTIZATION_DEBUG0)r)1zcalibration.cache  )6r flatbuffersrz5onnxruntime.quantization.CalTableFlatBuffers.KeyValue quantizationCalTableFlatBuffersKeyValue5onnxruntime.quantization.CalTableFlatBuffers.TrtTableTrtTable"onnxruntime.quantization.calibraterrrlogginginfordumpsopenospathjoinwriterBuildersortedkeysrfloatr"rcr r CreateString KeyValueStartKeyValueAddKeyKeyValueAddValue KeyValueEndr~TrtTableStartDictVectorrPrependUOffsetTRelative EndVector TrtTableStartTrtTableAddDict TrtTableEndFinishOutputenvironGetRootAsTrtTable DictLengthrDictKeyValue)calibration_cachedirrrrr json_datafilerbuilderkey_value_listkeyrd_valuesrZrflat_key flat_value key_value main_dict cal_tablebufdict_lenrrrrrrs @@@@@r$write_calibration_tabler"s! KKKLLLLLLLLLLLLLLLLLLLLLLLL]]]]]]]]]] L:'8::;;;77777777777D$777 ,) <>## (,,y$//4466 7 7 (,,x..3355 6 6 CKK  '',,))%00 w'''222!!':666((11 i(((( $$Wc..A.ABBB#33 '' 2222!!##I 7### Wi000$$W--I NN9 ..  C bgll3 9::D A AT 3 z~~*C00H<<%77Q?? ''))x , ,A!q))I L ) ) ) L** + + + + bgll3 344c : : d+002233  C&s+F~~''Hhll9d3388::;;hll8T227799::F#ICKK 0 00E JJu    JJt                       s7)C  CCM))M-0M-&C7U++U/2U/-C6?c|dktj}|dktj}|}|j|z }|sdS|t |zt |z }|dksJd|d|d||tj}|||z| |zzz }|dkdksJ|S)a~Given a discrete distribution (may have not been normalized to 1), smooth it by replacing zeros with eps multiplied by a scaling factor and taking the corresponding amount off the non-zero values. Ref: http://web.engr.illinois.edu/~hanj/cs412/bk3/KL-divergence.pdf https://github.com//apache/incubator-mxnet/blob/master/python/mxnet/contrib/quantization.py rNrzn_zeros=z , n_nonzeros=z, eps1=)rrzrsumsizer)pepsis_zeros is_nonzerosn_zeros n_nonzeroseps1rs r$smooth_distributionr~sQu}--H6//%-00KllnnG'!J t w % "3"3 3D #:::Q'QQ QQ4QQ::: 88EM " "DC(Nte{2 22D AI??   ! ! ! ! Kr& model_pathctj|d}td|jjDS)NF)load_external_datac3>K|]}tj|VdSr+)ruses_external_data)r" intializers r$ z*model_has_external_data..s.mmz#6zBBmmmmmmr&)rloadas_posixrgraphr9)rmodels r$model_has_external_datarsF Ij))++ F F FE mmUZU`Ulmmm m mmr&opt_model_pathct}||_tj|_i}dg|d<t ||fddgi|}dS)z Generate model that applies graph optimization (constant folding, etc.) parameter model_path: path to the original onnx model parameter opt_model_path: path to the optimized onnx model :return: optimized onnx model ConstantSharingdisabled_optimizers providersCPUExecutionProviderN)rroptimized_model_filepathrORT_ENABLE_BASICgraph_optimization_levelr)rr sess_optionkwargs_s r$optimize_modelrsr!""K+9+B+B+D+DK(+A+RK( F%6$7F !,,.. jjH^G_jcijjAAAr&rrcddi}|jr+|jD]#}||j|ji$tj||dS)z>Tag the model that it went through quantization pre-processingonnx.quant.pre_processonnxruntime.quantNmetadata_propsupdaterrrrset_model_props)rrprops r$add_pre_process_metadatar sh.0CDN :( : :D  ! !48TZ"8 9 9 9 9K~66666r&cZ|jr#|jD]}|jdkr|jdkrdSdS)zCCheck the model whether it went through quantization pre-processingrrTFrrr)rr s r$model_has_pre_process_metadatarsG (  Dx333 FY8Y8Ytt 5r&cddi}|jr+|jD]#}||j|ji$tj||dS)N onnx.inferrr)rrrs r$add_infer_metadatarsh"$78N 4% 4 4A  ! !15!'"2 3 3 3 3K~66666r&cZ|jr#|jD]}|jdkr|jdkrdSdS)NrrTFr)rrs r$model_has_infer_metadatarsF %  Au $$4G)G)Gtt 5r&cd|jD}t|dkrtd|dj}|S)Nc6g|]}|jr |jdk|S)r)domain)r"opsets r$rz%get_opset_version..s.mmm5<mSXS_clSlSleSlSlSlr&r)z$Failed to find proper ai.onnx domainr) opset_importrr4version)rai_onnx_domain opset_versions r$get_opset_versionrsMmm);mmmN >a?@@@"1%-M r& weight_typect|}|}t|d|}|dkr0|tjjkrt jd|dd}n?|dkrt jd|dn |dkrt jd|dd}||kr/tj||}t|}|S) Nr_z$The original model opset version is z, which does not support quantization to float 8. Please update the model to opset >= 19. Automatically update the model to opset 19. Please verify the quantized model.rze, which does not support node fusions. Please update the model to opset >= 11 for better performance.z, which does not support quantization. Please update the model to opset >= 11. Automatically update the model to opset 11. Please verify the quantized model. ) rr rrrZrwarningversion_converterconvert_version&save_and_reload_model_with_shape_infer)rrrtarget_opset_versionweight_quant_types r$update_opset_versionr(s-%e,,M( ]KHHr/43C3PPP 1= 1 1 1   " "   M= M M M       1= 1 1 1   "},,&66u>RSS7u== Lr&c,t|d}tjt |t |tj|}t|||S)Nz -inferred) ryrshape_inferenceinfer_shapes_pathr rrrunlink)rinferred_model_pathrs r$load_model_with_shape_inferr.s{6z;OO**3z??C@Ss r$add_quant_input_suffixrBs + ++r&c |dzS)N_QuantizeLinear_Outputrr>s r$add_quant_output_suffixrEs 1 11r&c |dzS)N_DequantizeLinearrr>s r$add_dequant_suffixrH!s , ,,r&c |dzS)N_DequantizeLinear_Inputrr>s r$add_dequant_input_suffixrK%s 2 22r&c|tzSr+)DEQUANT_OUTPUT_SUFFIXr>s r$add_dequant_output_suffixrN)s . ..r&)NN)FN)FNNN)rrrrrrrrrrrrrrrr)rr)rrrrrrrrrrrrrr)FF)rr!r(r!rr))r.r rr/)rrrrsr rr)rP)r)rr)rrrr)rr)rrrr)rrrr!)rrrrFrr)rrrr)rrrr)r9rrr)r;r rr )rr )l __future__rrrrr2enumrpathlibrrzrrrrr r onnx.helperr r r r onnx.referencer onnxruntimerrrrr ImportErrorrronnx.reference.op_runr __producer__ __version__ onnx_domain ms_domain QUANT_OP_NAMErADEQUANT_OP_NAMErMrMODEL_SIZE_THRESHOLDrrr:r(r?rFrbrRrmrTrXrVrZr^r\rrrirhrkrjr$r#r!rrrrrrrrr'r-r r7rBrPr`rgrlrqryrrrrrr rrrrr(r.r%rr?rBrErHrKrNrr&r$r^sv  #"""""   >>>>>>>>>>&&&&&&QQQQQQQQQQQQ------PPPPPPPPPP@@@@@@@LLL ????????? D EEE7777777      ,$2'!qqCC 4D4Dqqq      t           #>#>#>#>#>#>#>#>L     $    V!4!4 +%+g"6"6 +%+g"6"6!;5;x#8#8' % ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+d%*"E"E"E{u{SV^c^hGiGiGi!j!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ;5;q#>#>#>  BV[@\@\@\"]+%+b"="="={u{1TX?Y?Y?Y!Z  ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+d%*"E"E"E{u{SV^c^hGiGiGi!j!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ! ;5;q #D#D#DkekRU]b]hFiFiFi"j+%+c"D"D"DkekRT\a\fFgFgFg!h!KEK$F$F$F  TYafamHnHnHn#o ;5;vU[#I#I#I;5;W\didoKpKpKp"q ;5;q#=#=#={u{1TX?Y?Y?Y"Z+%+b"="="={u{1TX?Y?Y?Y!Z )+ A A A A A 13131313h<<<<~"""R#'"&"&;G;G;G;G;G~hl:=:=:=:=:=D$( L L L L L ^@    <>%%%%%%%%8''''''''"#"#"#J000@@@RRRR$YYYYx2nnnn k k k k 77777777!!!!H7777++++,,,,2222----3333//////s6AA)(A)-A66 BBB BB