zi7fxddlZddlZddlZddlZddlZddlmZmZmZm Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlZddlmZmZmZejejejejedddlm Z ddl!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.ddl/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5dd l6Td Z7d Z8d e9d e9de:fdZ;dZdZ?d%dZ@e jAddeBdeBde9de9d eBd!e9d"e9fd#ZCdS)&N) ProxyErrorConnectionErrorConnectTimeout ReadTimeout)get_speech_timestamps read_audio save_audioz..)app) TELEGRAM_APISUBSOURCE_BASE_URLSUBSOURCE_API_KEYMOVIEO_TELEGRAM_API)(identify_seasons_from_irregular_filename)identify_episodes_from_irregular_filenameidentify_releaseidentify_episodesidentify_seasons run_command log_request send_request)MovieoDatabaseYOUR_REQUEST_CLOSEDdownload_videoget_video_duration_secondsget_random_secondconvert_to_mkv)*ct|d5}|d}dddn #1swxYwY|drdS|drdSdS)NrbsPKzipsRar!rarunknown)openread startswith)pathfsigs E/var/www/html/auto_sub_bot/celery_tasks/tasks/subtitle_sync_engine.pydetect_archive_type_from_filer+s dD  QffQii ~~m$$u  ( (u 9s 377c|d}tjt|d| dtj|fdtjD}tjt|d|tj|rtj |tjrtj dSdS#tj|rtj |tjrtj wwxYw)Nz.rar.ziptemp_extraction)outdircPg|]"}tj|#S)osr'join).0r(temp_dirs r* z&convert_rar_to_zip..-s)IIIqh**III) r2rename DOWNLOAD_DIRpatoolibextract_archivelistdircreate_archiver'existsremoveshutilrmtree) subtitle_idtemp_rarfilesr5s @r*convert_rar_to_ziprE%s;###HI0{000(;;; =$ (;;;;IIIIBJx4H4HIII< B B B BEJJJ 7>>( # #8RYx%8%8%8 7>>( # #>( # #8RYx%8%8%8 7>>( # #http://apiec803a20e08d2684:driS9IZ2@res.proxy-seller.com:10000)httphttps)rHproxiestimeoutzProxy/Connection error (u%) — retrying request with new IP...cbi|],\}}||-Sr1)lower)r4kvs r* z(request_to_subsource..@s.NNNA 17799NNNr7z cf-mitigated connectionz cache-control challengezno-cacheprivatecloseuACloudflare challenge detected — retrying request with new IP...u3 error detected — retrying request with new IP...)requestsgetr rrrrprinttimesleeprequest_to_subsource status_coderHitems) rFrGrHproxyrMrespe resp_headers cf_mitigatedconnection_type cache_control is_challenges r*r`r`4s LEu--G>|1D8DUDDg_fprsss  E>>> QQQQRRR 1 #HeW========>  3NN9K9K9M9MNNN #''77 &**<88$(("== K ' h = ( fY--G fO_fLf   B U V V V'%AA A  !VVVWWW 1 #HeW===s#-B  7BB B cftj|d}d|D}d|D}||fS)Nutf-8encodingc^g|]*}|j|jjdz +S@@)textstripstartordinalr4ss r*r6z!get_sub_times..Vs0 G G G1  Gago& G G Gr7c^g|]*}|j|jjdz +Srp)rrrsendrurvs r*r6z!get_sub_times..Ws0 C C CqAFLLNN CAEMF " C C Cr7)pysrtr$)sub_pathsubsstartsendss r* get_sub_timesrTsF :h 1 1 1D G G G G GF C CD C C CD 4<r7c @t|}tj||f}t|D]}t|dz|D]}t t||dt||d}|dkrd}nwtjtj||dd|tj||dd|z }tj|}|x|||f<|||f<ӌ|S)Nri')lennpzerosrangeminabsarraymean)sub_times_listnsimilarity_matrixijmin_lendiffdiffss r*compare_subs_timingrZs. NA!Q(( 1XX E Eq1ua E EA#nQ/233S9J19M5N5NOOG!||H^A.q1(7(;<<H^A.q1(7(;<<=wu~~@D D ad #&71&=&= E r7@c|d}tj|tj|fdt|D}||fS)Nr)axisc2g|]\}}|zzk|Sr1r1)r4rmavgstd thresholds r*r6z!find_outliers..os0 M M M$!Q1sY_7L3L3LA3L3L3Lr7)rrr enumerate)rr mean_diffskeeprrs ` @@r* find_outliersrksk"''Q'//J '*  C &  C M M M M M M)J// M M MD  r7c. t}tj|}t|D]3tjfdt|D|<4t tj|}n #d}YnxYw|S)NcXg|]&}|kt|z 'Sr1)r)r4rrrs r*r6z$find_central_sub..ys:#c#c#c1\]ab\b\bC 1 1 (E$F$F\b\b\br7r)rrrrrintargmin)rr distances central_idxrs` @r*find_central_subrss   OOHQKK q e eA7#c#c#c#c#cPUVWPXPX#c#c#cddIaLL")I..//  s BB B???>c tj|}n*#t$r}td|Yd}~dSd}~wwxYw g}g} g} t } d} d} |r|D]}|jrt |jdkr|jj dz }|j j dz }|| z }||kr| | | || || ||} | dz } tj|}g}d}t|d}|D]E}|d |z }|d |z }||z }||z }||kr||kr| ||}F|sdSd}d}|D]J}tj||z }tj|}||}|| vr:|dz }||kr|dz }K||z } t%| d ||fSdS#YdSxYw) NzError opening subtitle: )rrrrrrqc|dS)Nrtr1xs r*z.calculate_smart_anchor_score..s AgJr7)keyrtry)rzr$ Exceptionr]setrrrrsrtruryaddappendrrsortedrrround)! subtitle_path vad_segmentssilence_thresholdmin_speech_duration sync_windowsrr|re sub_startssub_endsvalid_sub_objectsisolated_sub_indiceslast_sub_end_sec temp_indexsub start_seconds end_secondsgap_before_sub audio_anchorslast_vad_end_sec sorted_vadseg start_secend_secsilence_durationspeech_durationmatched_anchorsvalid_comparisons anchor_time differences closest_idxmin_diff final_scores! r*calculate_smart_anchor_scorersz-((  ,,,---uuuuuF "uu :  $ $8 $CHNN$4$4 5 5 9 9$'I$5$>M"%'/F":K%25E%EN%):::,00<<<%%m444OOK000%,,S111'2$!OJ*--JM"  2F2FGGGJ! + +L2- e*r/#,/?#? ")I"5#'888_Pc=c=c!((333#*    uO ! , ) )  fZ+%=>>  i 44 &{3&:::!Q&!{**#q(O),==KQ''(9/I I5uus$ >9>E G8 A(G88G=sync_subtitles)queue admin_user_id request_idrequest_type_strimdb_id title_type file_namefile_idc j|dvrd}n|dvrd}d|vrd}nd}dkr+t|} t|\} } } d|d | } n dx} } d|} t|} td |d d | d | d| d dti}t d| |} |ddd}n#t5}|d|d}dddn #1swxYwY|dd}|dkr=t|d\}}}|t||d}ttd|tYnxYwt dd|d|d|d}fd}g}|st5}|d|d}dddn #1swxYwY|dd}|dkr=t|d\}}}|t||d}ttd|t|D]}|dD]}||\}}} |d pd}| |kr| |ksR| |kr|dksFd!|vs0d"|d#d$vs|dkr0td%|d&||d'nt#|d(krn| r*td)t#|d*d} g}|D]}!td+d,|!d-d.d/}t dd0|!d1|}"|"jd2kre|"jd3d4}#d5|#vrtn$ t+t,|!d6d75}$|"d89D]}%|%r|$|% dddn #1swxYwYt3t,|!d6}&|&d:krt5|!t7jt,|!d6d;5}'|'}(t#|(dkrv|(D]r})dkr-||)\} }} | |kr|)}*td<|*d=n?5t<j |)\} }+|+d>vr|)}*td<|*d=nsn|(d}*td<|*d=|'!|*t,dddn #1swxYwYt=j"t,|!d6t<j |*\} },|,d?krBtd@|,dAtGt,|*|,|*|,d?}*t=j$t,|*t,| dB|d?tdC| dB|d?}*tK|*tM|*|dkrtO|*t<j(tR|*rt=j"t,|*n't=j$t,|*tR|*||*n#YnxYw| dz } tdD|dEtUtV||\}-}.|-r1t,|dF}/t,dG|dH}0tdIt,|-}1dJdK|1dLdMdNdOdPdQg }2 tYj-|2tXj.tXj.dRS}3|3/\}4}5tdTtta|4tdUtta|5|41rLte|413d}6tdV|6ntdWdX}6n7#th$r*}7tdYt|7dX}6Yd}7~7nd}7~7wwxYwdZt,|-d[}8tYj5tmj7|881}9te|9}:|6dkrts|6d\z};d]|;d^|;d_}tw|>dgsttdh txj=>didjdkl\}?}@|@\}A}B}C}D}E|C|/dmn}Fdm}G|A|F|?|Gdodpdpdqr}Hn8#th$r+}7tds|7dttYd}7~7nd}7~7wwxYwtyj?|F}I|HD])}J|F|Jdu|Jdv|I|Jdu|Jdv<*|B|0|I|Gntdwg}Kg}L|D]X}*tjAtR|*tRdx|*|Kdx|*dy|0dztR|*dztRd{|*d| }Mtw|Md} tjtRd{|*d~}Nd}O|ND],}P|O|PjC|P_C|O|PjD|P_D-|NEtRd|*d~t=j$tRd|*tR|* #th$rA}7 t=j$tRd{|*tR|*n #YYd}7~7nxYwYd}7~7Rd}7~7wwxYw|KD]}Q||Qtdi}R|D]S}*ttR|*|H\}S}T}Utd|Td|Ud|Sdi|R|*<|S|R|*d<|T|R|*d<Tt|RHddR}V|Vd\}W}X|W}*|Xd}S|Xd}Ttd|*d=d|vr tV}Yd}Zd}[n2|Sdkr|Tdkr dk}[t}Yd}ZntddR}[tV}Y|}Zt=j$tR|*tR|d?|d?}*t|*|t|*tddt,|-dt,d|-}\tw|\dt=j"t,|-t=j$t,d|-t,|-|.dkr#td|.dt|-|.}-tdtt,|-}]t|]}^|dkr3td|dt|*|d}*dR|Y|Z|[|-|*|S|]|^d SdR|Y|Z|[|-|*|S|]dSddkiSdS)N)zPersian SoftSubzPersian HardSub farsi_persian)zEnglish SoftSubzEnglish HardSubenglishSoftSubrrz?searchType=imdb&imdb=z&season=z% ********************** --- imdb id: z --- title type: z --- file release type: z --- file season: z --- file episode: z ********************** X-API-Keyz movies/searchdatarmovieIdidstatus)chat_idrr sendMessage subtitlesz ?movieId=z &language=z &limit=100cdkr4t|}t|\}}}|st|}ndx}}t|}|||fS)Nr)rrrr) release_infoseason episode_found_episode release_typers r* extract_infoz3find_and_synchronize_subtitle..extract_infosh ??=lKKF(9,(G(G %M1g  RCLQQ# #FW' 55 w ,,r7 releaseInforDcompletez all episodes. z% ********************** --- subtitle z added. ********************** subtitleId z ********************** --- zE subtitles download and extraction has begun. **********************zapplication/zipzattachment; filename="z.zip"zZIP file stream)r contentTypecontentDispositionbody/z /downloadrPz Content-TyperWz text/htmlr-wbi ) chunk_sizer"rz- ********************** --- chosen subtitle: z **********************).srt.assz.ssaz.vttrz/ ********************** --- subtitle format is z6. we have to change it to srt. **********************rzc ********************** --- cleaning and preparation of the subtitles began. **********************z% ********************** --- download z# has begun. **********************z _original.wavspeech_z.wavzW ********************** --- extracting audio from video file... **********************ffprobez-iz -show_entrieszstream=start_timez-select_streamsaz-ofzcsv=p=0T)stdoutstderrrrz=== stdout ===z=== stderr ===uزمان شروع صدا: u>ffprobe هیچ خروجی‌ای برای start_time نداد!ru-یک خطای غیرمنتظره رخ داد:z ffprobe -i "z6" -show_entries format=duration -v quiet -of csv="p=0"izadelay=|z,aresample=16000z atrim=start=zaresample=16000z ffmpeg -i "z " -vn -af "z " -ac 1 -t z "z" -yz!Initial conversion to WAV failed.zL ********************** --- speech recognition (VAD) **********************zsnakers4/silero-vad silero_vadF) repo_or_dirmodel force_reloadr) sampling_rateg?i^d)rrmin_speech_duration_msmin_silence_duration_ms speech_pad_mszG ********************** --- error loading or running Silero VAD model: z **********************rtryzP ********************** --- syncing subtitles with Alass ********************** original_z/usr/local/bin/alass "z" "fixed_"zAlass process was unsuccessful.rlrmcx|j}t|dz dz}tj|S)Nr)rurrz SubRipTime from_ordinal)time_objtotal_ms rounded_mss r*round_time_to_100msz:find_and_synchronize_subtitle..round_time_to_100mss7#+#3%*8c>%:%:S%@ $/<.sRSTURVWcRdr7)rreversez/ ********************** --- selected subtitle: Testi)gףp= ?rOlv7za ********************** --- suspicious subtitle, admin review required!!! **********************z[ ********************** --- trying to delete all soft sub tracks... **********************z ffmpeg -i z -map 0 -map -0:s -c copy temp_z9something went wrong when trying to delete all soft subs.z.mkvz+ ********************** --- file format is z6. we have to change it to mkv. **********************zs ********************** --- fetching duration and choosing random timestamp (for hardsubs). **********************z; ********************** --- ass subtitle format needed for zB, trying to convert subtitle to ass format **********************r) rapi_urluser_id admin_reviewserver_file_namesubtitlerduration random_sec)rr$r%r&r'r(rr))Orrrr]r r`jsonrget_users_requestsrrrrexitrRreplacerrrarHr\r$r9 iter_contentwriter+rEzipfileZipFilenamelistr2r'splitextextractr?convert_to_srtr8convert_to_utf8repair_subtitlefind_ads_in_subtitler> SUBTITLE_DIRrr subprocessPopenPIPE communicatereprrsfloat splitlinesr check_outputshlexsplitdecoderrrtorchhubload zeros_liker@copy2rzrtrysaverrrb add_intro add_outrorrrconvert_to_ass)_rrrrrrr subtitle_lang task_type file_seasonr file_episoderGfile_release_typerHmovie subsource_iddbuser_request_dbrequest_statusr movieo_id title_namerrr subtitle_idsrreleases_season s_episode files_countrrBres content_typer(chunk archive_typezip_refsubtitle_namessub_namer(sub_extsubtitle_formatr' file_formatoriginal_audio_wavspeech_audio_wav file_pathcmdprocessrr  audio_startre cmd_durationvideo_duration_outputvideo_durationdelay_ms af_filter trim_startcommand_convertr utilsrr r VADIteratorcollect_chunkswavrspeech_timestamps output_wavsegmentoriginal_subtitlesr command_alassr|rrw original_subsubtitles_with_scorescorerrsorted_subtitlesbest_sub best_sub_data final_api_url final_user_idadmin_review_requiredcommandsecr*s_ ` r*find_and_synchronize_subtitlersW@@@' B B B! $$$  Q&y11 .y991lGGG+GG%)) l222(33 dG d dz d duF d d[f d d|H d d deee-.G % A A F F H HE V}Q' 2      N 33DHMMO N N N N N N N N N N N N N N N(+A. Q  -8Q-G-G *GY "+,22Q660> !2 !2H'1Q:F,x:P:P9a+79+D+D7?H,13Gdl3G3G3G-H-H-H,1E,E 68W5E5Eh5O5O 7+26S+S+S7?H,13Gdl3G3G3G-H-H-H,1E,T ,:!+< %&zX`&z&z&z { { {#OOH\BBB'CCCCCCCCCCCCCCC( \"D;"D"D"DEEE-/W-=-=m-L-L*?*f44!#_Ve#_#_#_```*l+FH+F+FWWW'/'7'7'O'OH \"=8"="=>`q>`>`SZ>`>`>`aaaHIII&'#7#7'#7#7#7'111'111(O;;0:::7>>\*E8*E*EFF_I&Ax&A&ABBBBI&Ax&A&A\B]S[B]B]^^^!((2222 FAA g ggghhh(6|GI(V(V%+ J %$0!H'!H!H!H ".DDwDDD  o p p p';)9;;Ii!4!3y C "$*%?%?  ")!4!4!6!6&'''d6ll###&'''d6ll###<<>>&"' (A(A(C(CA(F"G"GKD{DDEEEEZ[[["%K " " "EFFFa!  " A,A8HAAAL$.$;EK A''BRSZ[`SaBa>b 77+GEN:;; J'2 F F F F h i i i!# N%   8h88\:^:^T\:^:^___"))*@h*@*@AAA!O:J!O!OQ]!O_g!O!Onz!O!OCK!O!O!O M+LMMM :&G&GX&G&GRYZZZDIII ";;"5"5ag">"> 3 3AE : :IIBBBBWIUUUIBBBBlC^T\C^C^____  \"C"C"C"C E`V^E`E`aaaabaaaa !3 / /   .... d e e e#% % X X>,<>><@cQa@c@c d d df$$KkKKKLLL#12BK#P#P L M M M, -P>N-P-PQQC*3//JA~~lVflllmmmx(((%+++",,$9(8 ("' #",   #,,$9(8 ("' #   e$ $_O%O%s-CE"C6* E"6C: :E"=C: >A"E",GGG%[?1O=1 [=P [P A[CT5) [5T9 9[<T9 =F[[=D a a< a77a<)Af88 g-!g((g-Bm n**(nn%nn%n%%n*)r)rrrr)Dr@sysr2r^r[requests.exceptionsrrrrrandomr+r1rFrC torchaudior;r:numpyrrzr rrr r'rabspathr3dirname__file__celery_tasks.celeryr configr r r rbase.utils.funcsrrrrrrrr base.utilsrrrrrrbase.utils.subtitle_editorr+rEstrdictr`rrrrrtaskrrr1r7r*rsI XXXXXXXXXXXX   DDDDDDDDDD RW__X-F-F M MNNOOO######XXXXXXXXXXXXIIIIIIIIIIIIIIIIIIII&&&&&&&&&&&&&&&&(((( = = =>3>s>T>>>>@ "   LLLL\  !!!Y%Y%#Y%Y\Y%gjY%x{Y%ILY%WZY%Y%Y%"!Y%Y%Y%r7