From e27fda0306e3e726b3c96be4bde5c8d9091856fa Mon Sep 17 00:00:00 2001 From: Kumar Mayank <103871423+Iamkrmayank@users.noreply.github.com> Date: Thu, 29 Feb 2024 20:13:54 +0530 Subject: [PATCH] Youtube_Vector_Search_DB_Contribution --- Problem_Statement_Project_New.xlsx | Bin 0 -> 15009 bytes README.md | 176 +++++------------------------ RetrivelCode.py | 78 +++++++++++++ app.py | 158 ++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 147 deletions(-) create mode 100644 Problem_Statement_Project_New.xlsx create mode 100644 RetrivelCode.py create mode 100644 app.py diff --git a/Problem_Statement_Project_New.xlsx b/Problem_Statement_Project_New.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..e007ac39f8b97dcf5e32e286568ea1d47ec7bc74 GIT binary patch literal 15009 zcmeHu1zR0kvNrDS8r*H;1oz`k0>8QpEHiSr@AsB=NU-mm|^?SJtT7*H9v?E|26sGkUkcBw@Kh^VSUAo&nz z7tSHu?;%F~kk%9F-*(#rDKE!}6g(NE8*XRab8#J-7q2iz{9t9}$T0y$52~!8nISf{ zbu@iUC=Gxi3#o^4Yz8%+AH7|4?{e^mRmGps#0+ddB8Y-(?3v&OI4fz@h~-x4+fzjr zX@C$RLbY_%Qu@PMm@}^!|AHyfDEq?8e$%S_U5G^bL)$utW$axkUnYW)nS>J&wy0k%UatGsOk8maKNT^Iw}Dth zB^M(Z19@R0Fa>95qnPLOIe0dqAB+P1izsV86}4#?Y`qmR449LCEhh|6C%$4GEji z=x_>b=5~RA;9sb{59~@`V@(*%d?}wv}h}w>NMQh5x{7lPZAh=DkJc z9ggttc-40_v36o&{Nwn4IQ?G?&VRf0$^=>YegHz~rNpnWk^9BXFDSy&ZbB06Br3lC zk{ifPF$LuKo843dC@OeC;Gg_DeSeLuZ}P^RjF8-YXRC}t!{8xncB=|aeR6PtrloXD z5qGHE9YA$myj{FY|0Lx>?b;bfTk)+lPkMBlOnm-Iv<_vOQ5_E!y@((bi!VJuYe-ge z%kZHZbWvFOyehP|g){dgaXQ0qIi>g*Q6T)I%-LKz=7^J_*=n`-h&9RWBc7_V`A3Ud z!)!-xQV)G&>z<#Y8QthF-VAb?BTA%fXb&vYpN1)NZ~e3y*{;U3JO(&n`zuC%j0Q## zm2SOVyB5D$31A4X`?iSOb~+t`gI4en4cvxbj*Zm= z_VBP0PUx(waehe z>C9<-^Y@nTg@DN|R)!I+SdddqZSt(4h0ZjD1h5uMs-AaW*1=k|$tZ8xs-D;p%}H*QQ=_(2E=f-< zLxC)uLtrX##G5o}3enAFUxh9K8ouX~qP2aWwZs>a+`>v4D{mxNzpIL@#k+-x=YkBX zQKK(_oosj+E8|uj>v2>7~DP z4rbS3{`k2|w64I_F9~pQRvBdJ)$U3#1F^X+jJq*f?|YcUw2r&nVIyBgyNM{nxzL1t z%c}gvm+}S|dfqMGZ@_S#48wStWxkc25yUv9URmhY>4G@NrY})Ms!CcwUM1s1X~8dK z1iVJ{VI^*(ywHGcSaAi2FAZ)@arTWIM>-ej_{T%LYXG0~_R5kCqAh63N+Swc^`w97 z$}Mku${9_=At7kLb(36heg^u(AML!BHwO|cspCak##G5JAR0TI% zjpCp7WbAXHzJ~Eg7L#k|wjM3E_$?yeh_Z405QhU_OC7}&-KFygOcUJysXM0ER|JjB zR82TNXRx2{w4ReL^xcq^Kx7>G6Rg~PPfY(6vI5C3>b=6mLB=UKr$8N;li3CKRV3Dx zSnibW>^H;9R~gkeB?KpdB&FN@P3ATWeZsu9p}XwGkK5)n)6nNJAG*w1=_yBG!SD<3 zjB;qa=>6$JnGd{ospZgb487*-I1x=v@q$_BZ!>42UuHPbr_*D1kkFQYvSXk6NBgn| zBvr>~@w^CZjoL~Q73)rMbe<-_E?nC4Z_C-_Eqx{uIE{;(F`mi3J)V}#2Za*3`ll5d@Zai z5iaM)p6zyLb5RoQcax~25B~x$RL}0+X?sjd$&ezCEb97YiZ;tyagqfY(js9houMy&=$qhPFAbYl1)}f z64aioj4)>bDqb$VG9l1Rsoj8N=I z@;J3$*BN6KY~2cj99v11DqlsZisCWSB^C&GzLH9si~5=_bNZlvVn-v+2{raAKCm9i z9!HPYQBL42pEN^wirAqQiPx})9CG$ot_s?s!`^l;&FQslvYDCgD&U+j8044`5@`(xKu==hDBdG>kc2Z(=Dpz2(-B#2}cDURm zVP4W8IFEf!z_S(WEJOxUNI$zps+nY9(Vw$X+5E;cMFW0!Yvy}P;bLpG&iakI2lfND zH%I4b;Bc{5t^u$mFFdt7#{-IpeDExswT{#O43d6I3ko%er5VYf8cDSl02C=y7rjQY zRdC_cn<7zgkdP{>9qi(3FF|9n8Rq#FA_Ks)Vl0lE{NZj#Eanp4biqr+e_p-q@T=OU zbrZ~6$)<#t@?!*efieMD14(FXd|2%-Bxv7BA$*)~_Z`({&#~~`AGPgIe_zEo9-*JyU~14v zQ+ljtFYY@A%OS{_>hn-<+hSgOwWrQHVh#5QzLeLHL1TzAzXC|~sDOK5zh;Mz#$cfj z5s0vEm}H+f_Td*doBC%%Yx1orY5WMIzI;&0UWqY)OofJyj#?1E_+$z!fZR?-VA4J6 z!I7T@E<_>^38vn*eI*29$HL1&S3K&*Jt3oRYdh&~BHK>&Hp>;5Z{})RIS1pv?6ftA zc=BW8_?`4J2ceoSX6B6svY5BG+ywgL9=W(_Mt@HF0EX>z^cBusRR*oYz_P;^2j}*d zh`cxoqrAp)S^q}16pR#bays1x+Z-i{?(CGqn&#BUsU?Aj<0pW4`7)~vY;0SqU3Xj` zuzK9GU%1{F-ZjT~N-^SloyQT&VGd`cJW{Y+w`DQ{RHr1@V*X^h)iPkSmAZ>U!dS7r zzThD|?QN)1TWLN^S?}FcYni{($}{6Bh~@FV7Ov@Scl+M6#$wY>;A3^znVnDbIjP3F zsb0YeGa90WCm5if0yy=PMtFVCu>AK}q8A&bOAZ91A8(bY3yV^P z?f8=B1i=wIZ{lFd8;^-*sU}y*st(9AtfjPE5u-l6c9xXfXZw2obiSJFO!NGd2H%&E zqTi)(&6gRyl14tC;%CK0&E_pHkra(;0&DvSPyb7LJ;&D()n*iCcm@1X)H*MPNb=)4 zE!H4RX#BSxQ?7t*%IINDIc2Y1{{lMc+aQ66#E#nO6LQSaGJLETcind7u@2 z!NL#X52i#e`F(T{L4F3N{#z9F$DM(7ig_3`yHGSrgKsJ{h*GK#HT(gu#blAq0G7gJ zU4sY>e5E^g!+}wm=3jKVi=4zv*?eBFKaPKfb@Tb%03ZAH>sy;>rEQ@pl$9?Qcb9*EoN4U05K$!M} zB%Yq~|M8fvf?5@IZnfD$up`VtfQlPn-Cp&!zUfZ=FD62(p8>+iJCq z=&F@GHoZo?4sydN+7-u*d#C*C_>Uj<*)p4nKU*W(BWB{QE*PaMLGF)ObJf2qt;W=v z93n=dMHF-O&M9}-i9{1tw2OovEc3B^m#K}oxj?B23ZJJCaWtS%Vr&#K3?J_^Ar4-< zp4_sj=`XcfW!mcReeHKT*-cOX(Oyr`Tku*P_ zjOc@<54po`@r1hJ5cZIsKj;o{BuF~~=USt1SmXIzS7H=+oz`R$Mh1K-kx0bi-GU^s zz2jDSMEOJr6zZ8bT40;|UQ9{9LGD|AO`-~aGMOT*>xfS8B*JzMtc@~R<85APnv7uO z;r>~cyw;AR>>NFWf390(HCa5l?heN|);(Q^hkY7D;x89qfQ;KWd(484?IcXz!Lhje*fL zOZCLM@Yc1b^R;UjHqhVVJMV(P^|r(%zH7OgL{YQ~kZ$WSsWZa}sw3waJlaKm+VO5xteCdG_( zyRe%NM8|A2G@x`{v5l(L5_F36;VXL@{YDZQ!=3}B02Rwx+zf)2h{O9V88B=^k7aT=>Bw-9avtYf|bd6bOjah}0)I2S5hXXi(4JM}h!QGBn=Rugznoy`I zLDG>CW=UguM|3sZRJyolB5PSXsDdFAD{-8&@@kudhoy@}d8b$*16j0lLkwUJ5Qx1? z&oi0wtU1({9=5Jc4?)4%Kv z)k?OYjh(Oj%@tQN+xEfja|$kYeVAwxw@#wm7!-4Snpe^QgFb^_G9ue7LB0*0x<3lP z*4|e^I-(2*6lb0hca~l+-c$CF9sfQ(wq_=88c?*?P$%N>z(ACqFqxT6$Y^V5v8``9C@CuZg8h;Zar zzxL*xGV-RCgWq+^Z*meXjyXYD9GVpm(w;0hCmx6K>?2lOi!9XB7TLbYF*sIRF}5P1 zEqX|G>@N63f%gI!awb@0H9O?`BO4kN*Be>sVG!gOI38?V#JTvcs;LA`#1nEzu zvL2qsHaZqfq1^&_J?;ItN|b;K!gi@-)K7ExBS{Fk> zlJGuvqo@Qa<)JoLgt+3Ujlb0DoVd^;?-;}@>LAUZK7s0do)L4ks2N^nyx2Qnm<&OU zijvWvz;4=o`rf6p>g{leN}ToFG?uGr<7ydV+Qdf<(J5CjbGF1CqF1GTiRigmJ4?X( zqmv8~X>HL~;gl1u$+1Nn#*)Isw?MNLI$4URv<3eq_l20w$l!p+$G8jR-m<;SjR>ZRXHimI)qN};1a6h$W`vHrUc-#-_i0?e%YxS^^*$^0%?r_eiwwEMh=Q&=bS+zrKf--#ze%*m zwJADN99kz(7KeL)l0&>yETTfAQZ@%wWrOHYufjzC3xnkNYEO32&*EPpS*3lNQ15MndX zaK0~XyI)>lU{y8H4CM{6hvSJjP2&6=poKdE_eM&P0^G%_@7)Iwwv0KX!wDniGcKaP zSbP#b<9i7^u@cWe$D{Y0cVAd&@OgXB486U*o7CG|Qkqrl8UQ1tC9vKDcrQCMK)<*_AE|nIyJS%ufoc$wQKbIcWDBfW$aF~5>A_Z(yybQR24bvgPyOg zy3&+0%%Pu!k;d&v#E4r;Wl<}(UyHRL1eNKqi9!GvnbQ6Ty=iOm^=h|#qI1t|$&!0J57VfODLh$-O+%*d5~ z0me;kZH13@ZG|b!C;dmniqVpD*&&3iM#5(B0{{F56S#x|QAGlyCGy3e{bL;^mJm~S z+xl}{|Lrs9rh3slprSZI;nWBs(K;hjB1TgVAhGVzWp^r@F;dM%gVRZb((X`V;4Fy( zXd{Ke(M5-7M-{5YM+^s=LVy<_?_&U-xQOp~6NM@GA4}-LrzCdo=h;xj-so_O?{ecD ze!52Jp+cxI!h)fxL4f+4pw`ieLZw>#GGc+rOd+=41{d1t`V{DYKotmKijifNtsTNv ze>8Ct2imf8>op`ekQyB@tQs9)ZUd7j6x$pPE7KMEBBiieP?oz5Rc=s(gPTq}=}jO( zKL|C(^+S7bLi2^AHAHuwg#h=H1ejTycq|#mXShU2IhQ4(pW_XguR0O`p9mG%S5H08UDvprtLjWU`=sKWqsS}`O->uHp?rt)S2we&2qxEbTR%CsCy-LY-^RS&YVzw!BZQsL1ovZIs?79aR zK4QFzQNy3Va_A}X{i-W~^^qJVS6+q2Jv`h-{|6ra1chjfBHVj5QM9K147Acv>|wt?)hP-6l3A|cOT43DS~Y%JBb-;AFxH?Z+sI?JNzV^w0OWLt60w6HZvXkezZcZ7 zEGQ%ikw8Gck%53<{38N5nSXXPF;;eVw6Hb%GhvjZrK`Llj^*Q){3gPLtf>PTv?p7y z9ZM4OS)w!~t!HJGx0Ow4TBfGRR?HGKQNR0W<6}qQ{aoJ01eFJldG47s!iV^9psYEMhm6R@PaoEdh^Ww$+`g*Xv`4(<_?atlq zz~|)-SURcRHntW~(KKj>Rb9psN0f1(sLjbux0bxrMH{!(;rp_wn2g(A;<&6C=cE7T zB=55HYwNgSGBMGu>Z8YzeDU2D+mzaxr%Q>CdNVp+W%H7$%^(I{@p()H zwFb%to2|69J?r%xnTl4%{YB_K^^>#y#pmO5>&DutPOAlv!v>F~lXLyy8UvN3eY({f zwdWz=bLEEby-@|ei$$j`mS*+hD>O45oxV+{L`j#8Rfkgtbj@ysSErj}md|YYIbPLL zwQ|LoUPVTe{>O@FG50YgY;*{daVEbC*(|-Cu&#{L{(P0}?9bjPZi{_YFs=Lq{8?v! zw^2Ab%-PM8>@B;+e23&~x~h|*E5$rtr%eQ&a_?l#3Ta6BC6i`GM5Pk8J9zqRe#7OB zib}fZb$Zkk0^E4gNcL96hY_BG+X8jn@tz#lEs}GY>I=&%)Fkb!_9FHP&?Y}mw`Vx* z=?nxR%PRr*+fPR+MXz4?0yrH4Yj4$zF{xF9>NG7{4>)T36;r+z$Vu7sRM02*s<`VO zXZa1@>{_%T-8w`)j+tV~>AJ3~N)=<6_Dipei|jTbArms(m%|z9ef&1+sH2IDy)kk>ud05W0nQE;e-sn?&^*ibi{0)v= zP*!GN2Q?}^=u@hb6pmy^=f$*#6a^)rte~4wvQwliH4{FeA@u$zo}>O!KJ*3)ht$kg zK1*SVVXtJ7gIQ`TibAsUrg4N`Y-{(j$)rAX3hoj^xbgT59!lD9`c362bW1Rz_cNWD zBVYYfeYrFPN=h$wAU5VTYeR9B;RWSUKQ27DwijJc7^ZAL>_ABjm>Nq&3p^&s0=pX6 zPj!ON);(4!u>DhC*g?l@%`U=GeVt41vuIG+7V)xU>ufdPQTH^`2Nl;bSdKte``QRDP2qlCpetAZdoVVX2Sy}ou+6oKb=FeWTD^-SuBI) z6v9jpDTyLH7LPU-+$OjgdVOmQ@g?Xa_u0iBaICU@-Du=dPDc~FT&`PD2g?z z-xwH~jS|hK&GM&kNHr7OtK-Rbt-RuPio>d|&%7`OMeoDg!KTY!%-)hzjFZ^p+~-iy z2(d^16$>o^UIq(p+*&;8WrqYQxUjz9aWE2$YhG#{m2zckW@wytM`_T>i%70gMh?a#$Q4J; z?KkAp%|oY;zykrfEB?Y)Fv0JHB}#vj8go#}Z()a8R9k;O(}Ea?MS{!WT??YEu1dXC zITSzZfPoaAI@sBRyQ@~W7qF5=ibI z5Vkiepd~Whlixm)ypTko$;o4(6nD91=*4iJZ^z(_7F$XL4Z|mKh&a@5K3`Zo&byVw z2IjkWg0vXAsDq1`ouUc9h3*AEalxZpTY$J(cYd-)G7fWHS2Qf2|MrDh%oX3InemC>B%Vwt)SxBqXu3$HX7-5s6NUK;tC}sa)axPOhO#C=%;T# zBn@XG&=iJ@Ob+Hixzv*vg4on9K03GQo0RGm3Lgl{?x96v=qtGgMggI7%uJAeFWk?UC5az$KDG4nk`h{!Kw$;?!*?W55W>A`*5kZ*wl8ND7I7eKg7|_DV6JvX;q}PWrw_1CW*g zJ+zc1PawtXX?G)1$k(3!=_{DqbnXtBaH7Ccy@aR70>?)%#*B%e?KVfb4l@dVlB4#I zeMK4*D{uPE_IaJ(G^bpk+a?N17*LS#IM?=he=v1<;(uEPCg;@hlnHMy*8m1Xyfbfk zy>1p8E$sVVwX=7|7$PPL)GwX4U`Hi2$vfR9DouB8+PmbdtT=yu+@l8XXn{+5g(WRG z9*)l^suo`71ICR7uov10C`C)1=z!~Sjem{?Aryag+dM3c3vktzEHtBd{y90M7r@;AH;zd{w@r<=L+5@KW&Tk-_orN-ksa7XkJ-On5efHg9&YWcv^|cMWitVdZXP7t%-Fn&wJl zU6{i;97alD3@^OC5XX;XuD0C!(i6B;D1Phf*?Rua8g2mxlyE^xRKMKGl_cN(4kK;T zTdcsFAy}D%JMkuk>2Pyc{_>;K$oy1$Q(wz{rl%#t5_q`IQ)De|1TRh2vo0>a60Ra2 z-YdZrMH+gP%bck%@T>SSVua` z1L+f2;w_z-754|qcG*^dhh)12?nDs9qI+11RbyCEe2UxvyD*QqMp1!k1j95Zu^MU+ zB@7_^x*0e_Q;-hT^n?0}3g+p05GsT=%R$`%6o<73B$ByyYvm0}%peyu0+XQhvOQsx zFYIKWq%@kBS-W+}WgJrjdcQ-kHFPC3ybloIaV=5yrKjNVs}utZ4CxM;_$LJt^K5H9 z#WFA&E2Y_dWVfbv}z~@IFYe{J_*7_rZo;WzFiEO5wL53LhUTP zKgl791E#9yKJrlkp>IGQ!;lyc+E9&W1ZHN(N_C8{5?!6>9x+1Z|m-C|T^bt>smn=A-fNO^cJU}sfX~rk$I#grEb!1|u;=W)Yy5-wz zbAsQsDQQgv2kDcmwlJ>sLYH~JBGU(-+goxQQt1Bg`PtSv5iRuTxb4mFZwk3PBRuJ+ z-hRK}2a{4LzF{#)7rZu?=L&;oLK##3Y$09$xNTC`4i}hv+CZ>!v4DV=~+_ zU%j@6%P{@rT>R0ED-Ka|Jc!yff@yeHs9q=$CqDQXUJ!&rvgOy}0Ma;VL}wGgbm4B0 z&RLjMoXH|s84bCk(ZE6;bVI@df+>LFx8jaiT%5{*Rl)a`j`zxDR@nlsAS&W0gZ6eS zLPFqEM<%A_bu>Zn@LMd!!I3tZ8<9K%jJ;YwaKQ*z{|G`7hOb}S!weR_$kJ_C=bsSk zLkZE=ck7f}%DCk=2>GL(z@pM0S}AmfHe!_IL1fr|7mg5zO$j4tUP@AdB-fvyx`mTO zeula|nguC)W8q_-{S10tLVR&1MRF$Fn{HDM4-ulDFnESWpSCl75D$qgtU+))ULTEy z4+N*PH^Uz)#C;GSged+IjFA*kfgPe2sQ;0j5j7#6PhrKF4c-!sTWAW@1XeRXE@vD% z&rCDS7r`(+3^cA{paMh>i#>@VY#{upQ8!o;W)OL8kO>TXS6*TdBX!sP*AX}sgx5*w z0%0A>5Ds`JZQ6%mV15vK6_2$#y-rtQ3oj2s>q1D+qdOGhK2nKDZ;#~2imX!j8q{$F zV^m+}hA*@aDjc!4Anjq*N5KT*Y;JG*k$Rh+i*6YItwL^~Aq6?PPS;P@UOp5q#l$pP z=O2Egz4pR;iUp&n0o8#_B+4Mlzu(TaHbUAR1heTg)964mt>zr-oQ{+7j^t+J*mNN_C_( zp^y9mI;Y@RrBqk4icng8Q2W?6jXiD8O7CbEdbU3uOC5XMm%u9h5X?Gm_7QuJ@k&|s z)^_$NT=$a>x*q+f43(q|>yX`&U_n_)6|Lz?6@;u~;ysNjJ7G4>557qTGlfb3sbla97=~ z_Lz+<8C^194(RCV=5_3yvxa?OiMeB<#x!*T?hUg83pH#Y&6q$D4AY^ludlr;z-~!TAl9m=`K6cc z8*G3L1Jz`}V zvaWr;s$tX7c6jjZ$`Xw^NgdqD6ACzrFcjfloGP#eeand*-IjpQ`@pl^K$#$F*<4Lp!` z7!WZ^)dsP=s{7PdT3o=`d*@@9;=zI7_0bc9&`DeS; zsOs(lU?&8QOF!g+*w*$~7_`#XTdH5O70aQQ)uVZQ zjusakR2C2miBg#wigR*+Qdw!6`>Tu?bHwPvKnvQmF<#_uqfPG;$A?}NQmpd~9f)mv z?YoSJ6uU^+Wf_ASz#qzqoBpIUgZ0Ei-IttL4o>Sdd?!cqXjZViVe8JQg6ISzOht&k-ATLB` z(+L5;!*%eF@;b~qz&mcLK#DPBeArbgQ=!nKjVUKkk9Crdi=M@B+++)poz(2UU5=yK ze__O@VR>OOcT#h^MSXlPum3X>TTde)BY!JPFEu?ur#2!fOIN4LGQ+B_L_H=m?O!K- z4d0VrECna|NI~(~3eXd`G?cPL4L7SS{F0KR54HzfUj?QfVVhu^6&9zg{J|Oh?|S}f ziltZHdy~++4?uaRo*LU3DLC5MJ28E>xBmn8?fJxy(datBpSjDj~1X1zu`X z&B65}aG&QBL`1M%6X%o__OG-=Q=&~f&hD1otkH+p3E(;>hsS}lJPB?59Q-_j!q_4t zjM%Ls#48$YwM+y37wd!A@g(UActqeB-yx{0Z-eEz_9WuE!CCstMZt? z?YQp9VJaJi&o`#lm1l;mo3}%OjhbZZOLCzFqm4zm;!zJpRDi!+&akXyBNMrmqy#jz zS!iEYl5`byxp9a^W18F#wF#o0o}JDHsyi|&Pqad=Q+%SzqWdVnxP*sn#V@QrL`MCn zrr>7a`OGmyv2Y?-0j?2ABXXaCs1td%N_=Aw!j-l!Lkx9hV#G4gB_{P`Cc~rwjzT*c zSjQFYbk6IVbL)umZY*ce(Zm~EpEf>T`wNuENZv4g5BR)K=-@jT+h*q8b}WT`l!gu< zIxO=wwA z$6^YZHebHqV z#=tO$-ym9yM5`|%Zgp>-ES>u32?tUO?kpQNYC}rkM>m% zjvjZCIWu&~YhSSc4z0gh8W?2X(Q5I|^F{r4v>Mqtn*1MJ-(U1Ux99|Y>3#sl;5KBp z$f#FF@wzd*4K%ISY=D8-6$81Owq{}qdBbT7=SzcDIx;`y(T4t1E4f~d$*xZThesKj zXdsG5t~-rmll+W!^Hu5yYYI(oq>@k|9>6?nYipn*zhmJ?{y1FYWUqT{A;N)bAkvZ= zhEYEW-a0K_ zM1o5h_0lZ&SeMQ3NwS~weMl*-//` +Tech Skills Needed +Pytorch/ Python, ML -Services are usually provided in either **remote** mode or **local** mode. +Category +Feature -For example, the text_translation service provided by google can be accessed on your local system at this URL: -`http://localhost:8000/text_translation/google/remote` +Mentor(s) +@GautamR-Samagra -### Deploy Single Image Setup -``` -docker build -t . -docker run -p : -``` +Complexity +Medium -### Multiple Image -In Multi Image Setup, each model has its own quart application and docker image. All model APIs are served via NGINX by utilizing proxies to route incoming requests to the appropriate destinations. - -There won't be any difference in accessing the models. The text_translation service provided by google can still be accessed on your local system at this endpoint: -`http://domain_name/text_translation/google/remote` - -### Deploy Multi Image Setup -For mutliple image setup, the docker-compose file and nginx configuration files are generated dynamically based on the config.json file. The following commands might require sudo privileges. - -Run this script to generate the configuration files. -``` -./generate.sh -``` -Setup NginX configuration -``` -sudo cp .conf /etc/nginx/conf.d/ -sudo nginx -t //Checks for configuration errors -sudo nginx -s reload -``` -- For **Windows** copy to `C:\nginx\conf\` and reload the server. -- For **MacOS** copy to `/usr/local/etc/nginx/` and reload the server. - -The .conf file will be present in your project root path after running the `generate.sh` script. - -Build Image -``` -sudo docker-compose -f ./docker-compose-generated.yaml build -sudo docker-compose -f docker-compose-generated.yaml up -``` - -## Adding a new model locally -To add a new model you need to create sub-directory in `src`. The subdirectory will follow `use_case/provider/mode` format. - -To deploy your application, the dockerfile must: -- copy program files -- install requirements -- build and run the application - -After you have developed and dockerized the application, to add a new model, you need to add a key pair inside models object in the config file with the following properties: - -- serviceName : Name of the service. This will also be the docker image's name. -- modelBasePath: Dockerfile location of your model. -- apiBasePath: Endpoint from where you wish to access the model. Should follow this convention: `//`. -- containerPort: Port at which the application will be hosted inside the container. -- environment: Another JSON object with key-value pairs. Should contain any relevant secrets or API_KEYS required to run the application. - -For example, if you want to add a new Text Translation AI Model from OPENAI in remote mode, you need to do as follows: -``` -{ - "serviceName": "text_translation_google", - "modelBasePath": "src/text_translation/google/remote/.", - "apiBasePath": "text_translation/google/remote", - "containerPort": 8000, - "environment": { - "OPENAI_API_KEY": "${OPENAI_API_KEY}", - "GOOGLE_CLOUD_SECRET": "${GCP_SECRET}" - } -} -``` -## Run Ansible Script - -### Requirements -- [HashiCorp Vault](https://www.hashicorp.com/products/vault) -- [Docker Swarm](https://docs.docker.com/engine/swarm/) -- Github Personal Access Token - -### Steps - -1. Create a `inventory.ini` file with target machine configuration. Target Machine is where the ansible script will run. -Here is a sample `inventory.ini` file: -``` -[swarm_manager] -ee01:ac8:1561::2 ansible_connection=ssh ansible_user=github ansible_ssh_private_key_file=/home/github/.ssh/id_rsa -``` -For More Information Refer https://www.digitalocean.com/community/tutorials/how-to-set-up-ansible-inventories and https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html -You may modify it according to your needs. -2. Create a Hashicorp Vault to store your secrets. You need to store the following types of secrets:(In separate Secret Paths mentioned below. This is not optional), -- Github Credentials. (Path = `secret/github`) - - `USERNAME` (Github Username) - - `PAT` (Github Personal Access Token. It should have read and write access to the contents of the repository) -- Target Machine Credentials (Path = `secret/common`) - - `DDIR` (Destination Directory, place where the files will be downloaded and generated) - - `REPONAME` (Github Repository Name. This could be Samagra-Development/ai-tools or the name of your fork.) - Please note: You don't have to add the entire Github URL, only the Repo Name. for ex: If you fork the ai-tools repo to your account. The repo URL would be something like `/ai-tools`. - This setup will only work **if** you have a restructure branch in your fork/repo with `generate.sh` and `config.json` in your root folder. Please be aware that modifying the file structure will break the code. -- Environement Secrets for the Docker Images (Path = `secret/config`) - - `DOCKER_REGISTRY_URL` (ghcr.io if you are pulling from Github Pacakges. Can be Docker Hub as well) - - `GITHUB_REPOSITORY` (ai-tools unless told otherwise) - - `OPENAI_API_KEY`, and other secrets pertaining to the docker images. This list might exapand as the project grows and supports multiple models. Please take a look at the `sample.env` for an exhaustive list of other env keys that are required by the models. - - Refer to the faq to see how to add secrets to hashicorp -3. Set the Vault Credentials in the environment of the target machine so that it can access the remotely hosted Hashicorp Vault. - - `VAULT_ADDR`: (Vault Address) - - `ANSIBLE_HASHI_VAULT_TOKEN`: (Vault Root Login Token) -You can add environment variables as follows: -``` -export VAULT_ADDR=http://x.x.x.x:8200 -export ANSIBLE_HASHI_VAULT_TOKEN=abc12345 -``` -For the Vault Credentials to persist add it to `.bashrc` -``` -echo 'export VAULT_ADDR=http://x.x.x.x:8200' >> ~/.bashrc -echo 'export 'ANSIBLE_HASHI_VAULT_TOKEN=35r3zxcc' >> ~/.bashrc -source ~/.bashrc -``` - -Alternatively, you can pass the variables during run time as Command Line Arguments using `--extra-vars` field. -Here is an example: -``` -ansible-playbook -i inventory.ini swarm.yml --extra-vars "VAULT_ADDR='http://127.0.0.1:8200' VAULT_TOKEN=abc.123" --ask-become-pass -``` - -### Additional Steps -1. Change Hosts in Ansible Playbook to the Node which is acting as the swarm manager. (Make sure a Docker Swarm is running in that machine) - -### To run the Ansible Playbook -``` -ansible-playbook swarm.yml -i inventory.ini -``` -## Contributing -Contributions to AI Toolchain are welcome! To contribute, please follow these guidelines: - -1. Fork the repository and create a new branch for your feature or bug fix. -2. Write tests for your changes. -3. Submit a pull request describing your changes and why they are needed. - -Thank you for considering contributing to AI Toolchain! +Code Setup: Make sure you have Python installed. \ No newline at end of file diff --git a/RetrivelCode.py b/RetrivelCode.py new file mode 100644 index 0000000..347f06a --- /dev/null +++ b/RetrivelCode.py @@ -0,0 +1,78 @@ +from ragatouille import RAGPretrainedModel +import re +import sqlite_utils +import llm +from datetime import datetime + +# Function to strip HTML tags from text +def strip_html_tags(text): + return re.compile(r'<[^>]+>').sub('', text) + +# Function to retrieve data from the SQL database based on user query +def retrieve_data_from_database(query): + db = sqlite_utils.Database("TextSummarisation.db") + entries = db.execute(f"SELECT * FROM blog_entry WHERE title LIKE '%{query}%' OR body LIKE '%{query}%'").fetchall() + + return entries + +# Function to perform document retrieval and summarization +def retrieve_and_summarize(query): + entries = retrieve_data_from_database(query) + + # Initialize Ragoutille model + rag = RAGPretrainedModel.from_pretrained("colbert-ir/colbertv2.0") + + # Extract information and index the documents + entry_texts = [] + entry_ids = [] + entry_metadatas = [] + + for entry in entries: + entry_text = entry["title"] + '\n' + strip_html_tags(entry["body"]) + entry_texts.append(entry_text) + entry_ids.append(str(entry["id"])) + entry_metadatas.append({"slug": entry["slug"], "created": entry["created"]}) + + # Index the documents + rag.index( + collection=entry_texts, + document_ids=entry_ids, + document_metadatas=entry_metadatas, + index_name="blog", + max_document_length=180, + split_documents=True + ) + + # Search for documents based on user query + docs = rag.search(query) + + # Use LLM for summarization + model = llm.get_model("gpt-4-turbo") + model.key = '' + + prompt = ' '.join([d['content'] for d in docs]) + + # Generate the summary using LLM + response = model.prompt(prompt, system=query) + + # Store the results in the database + result_summary = ' '.join(chunk for chunk in response) + youtube_link = "https://www.youtube.com/example" # Replace with the actual YouTube link + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + db = sqlite_utils.Database("TextSummarisation.db") + db["search_results"].insert({ + "query": query, + "summary": result_summary, + "youtube_link": youtube_link, + "timestamp": timestamp + }) + + # Display the result + print("Summary:", result_summary) + print("YouTube Link:", youtube_link) + print("Timestamp:", timestamp) + +# Example usage: +user_query = input("Enter your query: ") +retrieve_and_summarize(user_query) \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000..6807ba3 --- /dev/null +++ b/app.py @@ -0,0 +1,158 @@ +import os +from pydub import AudioSegment +from pytube import Playlist +from openai import OpenAI +import sqlite3 +from datetime import datetime + +# Function to download audio from the playlist URL +def download_audio_from_playlist(playlist_url, download_directory): + # Create a Playlist object + playlist = Playlist(playlist_url) + # Create the directory if it doesn't exist + if not os.path.exists(download_directory): + os.makedirs(download_directory) + + # Iterate through each video in the playlist + for video in playlist.videos: + try: + # Get the highest resolution audio stream + audio_stream = video.streams.filter(only_audio=True, file_extension='wav').first() + + # Download the audio stream + audio_stream.download(output_path=download_directory) + + print(f"Audio downloaded for video: {video.title}") + + except Exception as e: + print(f"Error downloading audio for video {video.title}: {str(e)}") + + print("Audio download complete.") + +# Function to slice audio into 4-minute chunks +def slice_audio(input_dir, file_name, output_dir, target_length): + song = AudioSegment.from_wav("%s/%s.wav" % (input_dir, file_name)) + timestamps = [] + chunks = [] + + i = 0 + while i < len(song)/(float(target_length)*1000): + seconds = float(target_length) * 1000 * i + seconds2 = float(target_length) * 1000 * (i+1) + cut = song[seconds:seconds2] + + # Store timestamp and chunk name + timestamp = datetime.now() + chunk_name = f"{file_name}-{float(target_length)}sec-{i}.wav" + cut.export(f"{output_dir}/{chunk_name}", format="wav") + + timestamps.append(timestamp) + chunks.append(chunk_name) + i += 1 + + print("Audio slicing complete.") + return timestamps, chunks + +# Function to perform speech-to-text using Whisper model +def transcribe_audio_with_whisper(audio_directory): + client = OpenAI(api_key='your_openai_api_key') # Replace with your OpenAI API key + + transcripts = [] + + # Iterate through each audio file in the directory + for audio_file_name in os.listdir(audio_directory): + if audio_file_name.endswith('.wav'): + audio_file_path = os.path.join(audio_directory, audio_file_name) + + # Open the audio file + with open(audio_file_path, "rb") as audio_file: + # Perform transcription using Whisper ASR model + transcript = client.audio.transcriptions.create( + model="whisper-1", + file=audio_file, + response_format="text" + ) + + print(f"Transcription for {audio_file_name}:") + print(transcript) + + transcripts.append(transcript) + + return transcripts + +# Function to perform text summarization using Whisper model +def text_summarization(transcripts): + client = OpenAI(api_key='your_openai_api_key') # Replace with your OpenAI API key + + summarizations = [] + + # Iterate through each transcription + for transcript in transcripts: + # Extract text from the transcription + text = transcript['data'][0]['text'] + + # Perform text summarization using Whisper model + summary = client.completions.create( + model="gpt-3.5-turbo-instruct", + prompt=f"Summarize the text: {text}", + temperature=0.7, + max_tokens=300, + top_p=1, + frequency_penalty=0, + presence_penalty=0 + ) + + if 'choices' in summary and len(summary['choices']) > 0: + summarization = summary['choices'][0]['text'] + print(f"Summarization for the transcription:") + print(summarization) + + summarizations.append(summarization) + + return summarizations + +def save_to_database(link, chunks, timestamps, summarizations): + connection = sqlite3.connect('TextSummarisation.db') + cursor = connection.cursor() + + for i, (chunk, timestamp, summarization) in enumerate(zip(chunks, timestamps, summarizations), start=1): + # Assuming you have a table named 'audio_data' with columns 'link', 'chunk_number', 'timestamp', 'summarization' + query = "INSERT INTO audio_data (link, chunk_number, timestamp, summarization) VALUES (?, ?, ?, ?)" + cursor.execute(query, (link, i, timestamp, summarization)) + + connection.commit() + connection.close() + +def main(): + global input_dir, target_length, file_name, output_dir, playlist_url, chunks, timestamps + + # Get user input for playlist URL + playlist_url = input("Enter the playlist video link: ") + + # Set parameters (you can modify these as needed) + input_dir = 'audio_downloads' + file_name = 'output' + output_dir = 'audio_chunks' + target_length = 240 # in seconds + chunks = [] + timestamps = [] + + # Download audio from the user-provided playlist URL + download_audio_from_playlist(playlist_url, input_dir) + + # Slice the downloaded audio files into 4-minute chunks + timestamps, chunks = slice_audio(input_dir, file_name, output_dir, target_length) + + # Perform speech-to-text using Whisper model + transcripts = transcribe_audio_with_whisper(output_dir) + + # Perform text summarization using Whisper model + summarizations = text_summarization(transcripts) + + save_to_database(playlist_url, chunks, timestamps, summarizations) + + print("Text Summarizations:") + print(summarizations) + +if __name__ == "__main__": + main() \ No newline at end of file