From ddb2f0b32427019c691ce0ecd6f1e9c6901efc2c Mon Sep 17 00:00:00 2001 From: Gang Li Date: Wed, 3 Apr 2024 16:23:11 +0800 Subject: [PATCH] Fix wrong picking of the npm package.json Sometimes the npm tarball will contain more than one package.json, we should always use the root package.json as the first product metadata. --- charon/cmd/command.py | 6 + charon/pkgs/npm.py | 28 +++-- charon/utils/archive.py | 69 +++++++---- tests/__init__.py | 20 ++++ tests/input/code-frame-7.14.5-multi-pkgs.tgz | Bin 0 -> 3040 bytes tests/input/code-frame-7.14.5-no-root-pkg.tgz | Bin 0 -> 3000 bytes tests/test_npm_upload_diff_pkgs.py | 109 ++++++++++++++++++ 7 files changed, 199 insertions(+), 33 deletions(-) create mode 100644 tests/input/code-frame-7.14.5-multi-pkgs.tgz create mode 100644 tests/input/code-frame-7.14.5-no-root-pkg.tgz create mode 100644 tests/test_npm_upload_diff_pkgs.py diff --git a/charon/cmd/command.py b/charon/cmd/command.py index 2ef88aed..01b8ede4 100644 --- a/charon/cmd/command.py +++ b/charon/cmd/command.py @@ -170,12 +170,15 @@ def upload( buckets = __get_buckets(targets, conf) if npm_archive_type != NpmArchiveType.NOT_NPM: logger.info("This is a npm archive") + npm_root_path = root_path\ + if root_path and root_path != "maven-repository" else "package" tmp_dir, succeeded = handle_npm_uploading( archive_path, product_key, buckets=buckets, aws_profile=aws_profile, dir_=work_dir, + root_path=npm_root_path, gen_sign=contain_signature, key=sign_key, dry_run=dryrun, @@ -331,12 +334,15 @@ def delete( buckets = __get_buckets(targets, conf) if npm_archive_type != NpmArchiveType.NOT_NPM: logger.info("This is a npm archive") + npm_root_path = root_path\ + if root_path and root_path != "maven-repository" else "package" tmp_dir, succeeded = handle_npm_del( archive_path, product_key, buckets=buckets, aws_profile=aws_profile, dir_=work_dir, + root_path=npm_root_path, dry_run=dryrun, manifest_bucket_name=manifest_bucket_name ) diff --git a/charon/pkgs/npm.py b/charon/pkgs/npm.py index 684e8457..bbed7a29 100644 --- a/charon/pkgs/npm.py +++ b/charon/pkgs/npm.py @@ -73,9 +73,10 @@ def default(self, o): def handle_npm_uploading( tarball_path: str, product: str, - buckets: List[Tuple[str, str, str, str]] = None, + buckets: List[Tuple[str, str, str, str]], aws_profile=None, dir_=None, + root_path="package", do_index=True, gen_sign=False, key=None, @@ -103,7 +104,7 @@ def handle_npm_uploading( prefix = remove_prefix(bucket[2], "/") registry = bucket[3] target_dir, valid_paths, package_metadata = _scan_metadata_paths_from_archive( - tarball_path, registry, prod=product, dir__=dir_ + tarball_path, registry, prod=product, dir__=dir_, pkg_root=root_path ) if not os.path.isdir(target_dir): logger.error("Error: the extracted target_dir path %s does not exist.", target_dir) @@ -230,9 +231,10 @@ def handle_npm_uploading( def handle_npm_del( tarball_path: str, product: str, - buckets: List[Tuple[str, str, str, str]] = None, + buckets: List[Tuple[str, str, str, str]], aws_profile=None, dir_=None, + root_path="package", do_index=True, dry_run=False, manifest_bucket_name=None @@ -250,7 +252,7 @@ def handle_npm_del( Returns the directory used for archive processing and if the rollback is successful """ target_dir, package_name_path, valid_paths = _scan_paths_from_archive( - tarball_path, prod=product, dir__=dir_ + tarball_path, prod=product, dir__=dir_, pkg_root=root_path ) valid_dirs = __get_path_tree(valid_paths, target_dir) @@ -433,11 +435,15 @@ def _gen_npm_package_metadata_for_del( return meta_files -def _scan_metadata_paths_from_archive(path: str, registry: str, prod="", dir__=None) ->\ - Tuple[str, list, NPMPackageMetadata]: +def _scan_metadata_paths_from_archive( + path: str, registry: str, prod="", dir__=None, pkg_root="pakage" +) -> Tuple[str, list, NPMPackageMetadata]: tmp_root = mkdtemp(prefix=f"npm-charon-{prod}-", dir=dir__) try: - _, valid_paths = extract_npm_tarball(path, tmp_root, True, registry) + _, valid_paths = extract_npm_tarball( + path=path, target_dir=tmp_root, is_for_upload=True, + pkg_root=pkg_root, registry=registry + ) if len(valid_paths) > 1: version = _scan_for_version(valid_paths[1]) package = NPMPackageMetadata(version, True) @@ -447,9 +453,13 @@ def _scan_metadata_paths_from_archive(path: str, registry: str, prod="", dir__=N sys.exit(1) -def _scan_paths_from_archive(path: str, prod="", dir__=None) -> Tuple[str, str, list]: +def _scan_paths_from_archive( + path: str, prod="", dir__=None, pkg_root="package" +) -> Tuple[str, str, list]: tmp_root = mkdtemp(prefix=f"npm-charon-{prod}-", dir=dir__) - package_name_path, valid_paths = extract_npm_tarball(path, tmp_root, False) + package_name_path, valid_paths = extract_npm_tarball( + path=path, target_dir=tmp_root, is_for_upload=False, pkg_root=pkg_root + ) return tmp_root, package_name_path, valid_paths diff --git a/charon/utils/archive.py b/charon/utils/archive.py index 5bcb2777..eca56ebe 100644 --- a/charon/utils/archive.py +++ b/charon/utils/archive.py @@ -46,8 +46,9 @@ def extract_zip_with_files(zf: ZipFile, target_dir: str, file_suffix: str, debug zf.extractall(target_dir, members=filtered) -def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool, registry=DEFAULT_REGISTRY)\ - -> Tuple[str, list]: +def extract_npm_tarball( + path: str, target_dir: str, is_for_upload: bool, pkg_root="package", registry=DEFAULT_REGISTRY +) -> Tuple[str, list]: """ Extract npm tarball will relocate the tgz file and metadata files. * Locate tar path ( e.g.: jquery/-/jquery-7.6.1.tgz or @types/jquery/-/jquery-2.2.3.tgz). * Locate version metadata path (e.g.: jquery/7.6.1 or @types/jquery/2.2.3). @@ -56,30 +57,50 @@ def extract_npm_tarball(path: str, target_dir: str, is_for_upload: bool, registr valid_paths = [] package_name_path = str() tgz = tarfile.open(path) + pkg_file = None + root_pkg_file_exists = True + try: + root_pkg_path = os.path.join(pkg_root, "package.json") + logger.debug(root_pkg_path) + pkg_file = tgz.getmember(root_pkg_path) + root_pkg_file_exists = pkg_file.isfile() + except KeyError: + root_pkg_file_exists = False + pkg_file = None tgz.extractall() - for f in tgz: - if f.name.endswith("package.json"): - version_data, parse_paths = __parse_npm_package_version_paths(f.path) - package_name_path = parse_paths[0] - os.makedirs(os.path.join(target_dir, parse_paths[0])) - tarball_parent_path = os.path.join(target_dir, parse_paths[0], "-") - valid_paths.append(os.path.join(tarball_parent_path, _get_tgz_name(path))) - version_metadata_parent_path = os.path.join( - target_dir, parse_paths[0], parse_paths[1] + if not root_pkg_file_exists: + logger.info( + "Root package.json is not found for archive: %s, will search others", + path + ) + for f in tgz: + if f.name.endswith("package.json"): + logger.info("Found package.json as %s", f.path) + pkg_file = f + break + if pkg_file: + version_data, parse_paths = __parse_npm_package_version_paths(pkg_file.path) + package_name_path = parse_paths[0] + os.makedirs(os.path.join(target_dir, parse_paths[0])) + tarball_parent_path = os.path.join(target_dir, parse_paths[0], "-") + valid_paths.append(os.path.join(tarball_parent_path, _get_tgz_name(path))) + version_metadata_parent_path = os.path.join( + target_dir, parse_paths[0], parse_paths[1] + ) + valid_paths.append(os.path.join(version_metadata_parent_path, "package.json")) + + if is_for_upload: + tgz_relative_path = "/".join([parse_paths[0], "-", _get_tgz_name(path)]) + __write_npm_version_dist( + path, pkg_file.path, version_data, tgz_relative_path, registry ) - valid_paths.append(os.path.join(version_metadata_parent_path, "package.json")) - - if is_for_upload: - tgz_relative_path = "/".join([parse_paths[0], "-", _get_tgz_name(path)]) - __write_npm_version_dist(path, f.path, version_data, tgz_relative_path, registry) - - os.makedirs(tarball_parent_path) - target = os.path.join(tarball_parent_path, os.path.basename(path)) - shutil.copyfile(path, target) - os.makedirs(version_metadata_parent_path) - target = os.path.join(version_metadata_parent_path, os.path.basename(f.path)) - shutil.copyfile(f.path, target) - break + + os.makedirs(tarball_parent_path) + target = os.path.join(tarball_parent_path, os.path.basename(path)) + shutil.copyfile(path, target) + os.makedirs(version_metadata_parent_path) + target = os.path.join(version_metadata_parent_path, os.path.basename(pkg_file.path)) + shutil.copyfile(pkg_file.path, target) return package_name_path, valid_paths diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..e3cdc8ed 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,20 @@ +""" +Copyright (C) 2022 Red Hat, Inc. (https://github.com/Commonjava/charon) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import logging + +logging.basicConfig(level=logging.INFO) +logging.getLogger("charon").setLevel(logging.DEBUG) diff --git a/tests/input/code-frame-7.14.5-multi-pkgs.tgz b/tests/input/code-frame-7.14.5-multi-pkgs.tgz new file mode 100644 index 0000000000000000000000000000000000000000..b9a284a4b81263b7f64e5b92ae16891bf0a52f46 GIT binary patch literal 3040 zcmV<63m^0!iwFP!000001MM1XbK5pDpZO~gwdqI`nR-~Z8|$1_k?5FHJv@?JU&eN0 zii9Ml6sZzq9ABdUe!BqplpUuxaXQy>Cox4}-wW(6NUja@cVj_ohO=H8wI`1Pb%2At zJ-jv!_Uh?t1Wy{R{d%jl+uGmTeNt~U_V*5+kiExICkr7944;rE3&;K}-b3^Mr?Ae% zU*4(jnCE^y85rRGR_ng_H(IS`F8(dJeM0JAPlfy8A0GdY3Lz!eSkclUDg8V%X4I*f zo<(=&J`PrK`aSiT?YW3?pf+0So*1?$Gktp<$f?)VrM?kRLVe%!83~p~KupgK4BI6J zL8gQ=6*Ftkb^{W4gn6NFQsQ`~fvlJ+xEo=x^n7GIT~hK26(dt>tVnqo1Z#F!t8ryj zi@vV~FkTU4OK(Nju!NCt8`1JiR-s9R7X1K8BU%?=vqfFgrYzA?uC1lLSUT`OKTc#h@o*%F(DZPD5*}$_x2dX8{ zo9_)Lq=zK%L#k{lZAGH0STv5YW^_F#T)B)Kk^E4C4C0K?2>|D<{0XYOhp@bj%LUB& z^p6nARC&9UfLz6)0fEW`WsJ8G=Z5dv?qag^Zd{<98xEs@H4j}Ar^$i_JvwJ)UOFn= z6rTp6@5*Yh2!nw7hlJ0o3!mPKqOvml-;;5lTI$SmEI9{78}?Fu66;7C3W);E5DUmq zJMZg54V0=~`j_sfi%S=-VYA-+k^g;hRm&+l;4=GIt;*0QRb|UmCEhw zXXd@9z@V9kWUr{>-N4u`3Wh6MK|XyVA2;osC4dd{kth|Zb0}gdrICb@SsMf&VNH$; z48}YpLc#&p6lV;R10z_fE8{~sQEi8mQ${`~jY^f&D-j`Zi^*)aEMlfQ&~^mNYOL1t zluvLO6JpOvO2*OA5rn8BeHtlKZ}a9vV#g+CaE1Jg*qBP#j*@%!d@fBFCdHx?AFY!0 z2dEe(DL7&fiJW^r41(=8*AXuNI*O&h?epgmW?F?|xSCNPC~WXXO1wlwK~Gb`PFRq% zW)}&oE8v9*sV(s~gu?FUjBM_tdYXq^P25j??xxsnDW2e&##PFV56QnsXqs5 zQiu|=WKh+*yz=Jpq5h+?+49^@a@NYYozat2S0GT1ag#w8<1Q-X=%w_ebZgqwpNK>+ zN7#pwt}Ps!Nsg#Vb+I%z94GCtgT}1h*0JHuryfGImNXZ1Ar%CZ=Q_F zdc`+T$>sa8l<*M)1LMQFZSjph;nrf!sHArH)}n~Aeb#xoD9UE7l3H=1B{_`U7F9#> z^)WYyv1_zEDMU zsxnH6DwkHIM}=RiSrhMOCQtzXw-UFC8bLXycI8Ta=Rr?bDpg`v%Z?};Q`oLNhsu*d z+T|JZo*$ul5bQe@62#RktHe#4xKPO-GVfh{*1DIa%9l8^A`+BL733wULmI$MfTNph z*y*L%t-J=NJab3gl&T?vFOaQ;bDb)t6)FAFEhWu#+HDw-r%7&&8&{c~uD|~X zamYqJ&YjrcgGOm~n@jimkDOSj5x3;2G*`)1qv&kNXi(;q+))`fNlZT5mQNv*avvR$ z(`7#H#Hx76i+<(`KX0V(IU+|&-hYBlS|XV&q(folp5nO>GDfO-G~nvYJdmuam01Lqq$Z~sT{cWJ$yW_(v=-ZC6kSH$7DI)K&9T6dH1Z-s2PdT zHGHW7#=7)|@ZaV9M$99tu2@rd32mpucn(!>V7$D|x{_MjqBC*g0v4y^RKV-01|oW& zf(sz&P!3ZTiiZISc~npMV}T@=i3;tcRsDR=$pyzP&E~Iu##O-m5Iz6x>s@U)(H=#{ z1Mh#Ejb=0V{I{{U*D9X>ehV1rQ_>f2CKaXYt#5sPt}2@q(yTXHJ8P^|Za^k<2Ca>4 zxMX1dLf!w^l;?OaSVA_!x4gbR3j@z*igHT*mCf*tF(!VPr?XqK@C_G|e-)}4r3B6| zPzQa?$xsap_m-@otpWmXhOdhuMJo8lczuhQ!4lXoZyrFF6jGtyc&5#ZiRGE$3KSH% zZf-kpYAJ(KN@*e~u+IS@7Bw6=6#zF%qA7A?2XKB3AghT-;5=>FuIYpp>JUvkHdGTK z0m-?G7&3)wRpoj@3*=e$9Iup{CEx5;70YJAH!#Tf+g$W@744`&MPbx&kb@0+b6XSb zI1Wfr*67ecx`@lYS$eCiCE%htBxVpPDy)Ss*11G{a*JFLeeOAqcZ0PWLT1_gefy!J z;2U;>Kg2ZS72tXSs47&z0IZX!$|(k&j6-Hr5e@^n_@X~$j*sGDd%3V~9DG*vOgh6inJ{?Uo5Z_aVqoF=L9)lEZpbe)g zNCiyA#}jP=(iQ*tuu_cR!Mr73;A^Qx~2QD9WJ-_ZwE z((4R5uQg6J0xn~MXi~fL6O9jnybk>9PW91H0iSe7!|51qt6=4L8k3&u6Rk=*V|{`S zIUbLIJ-QPpMw|)I4K=|9T}?6(0TjGHn`ntHJ*@)=_SX|6P9v*-^Y`U{H;C&0<9zug z;NkDTcH{T|2aSEG|GV}5W>NpY1!TYf@~O2ng5m?P;sdba1F(O_2VhbDA8Vc7KvP%N zBgl9l|JV1Ljr{jN_2xm5|Gx!%xBVua`D;5Xp&X|l7! zjQ4a0a?Ksa1G=JsgpjkiHGDVbfj7jktZgb_3EVjCUmef_3Mim}0tzUgfC36Apnw7j iD4>7>3Mim}0tzUgfC36ApnwAYk?>!EF$ZA)Pyhfljm7W) literal 0 HcmV?d00001 diff --git a/tests/input/code-frame-7.14.5-no-root-pkg.tgz b/tests/input/code-frame-7.14.5-no-root-pkg.tgz new file mode 100644 index 0000000000000000000000000000000000000000..96c85af0020be6f88c0d5ccb930076b3bedb5dc8 GIT binary patch literal 3000 zcmV;p3rF-HiwFP!000001MOOSbK1rh&)@nK8`^YWg2ltwY0M=p0Xf!`hX-(dGqxKG zX#u4|qFTw0Z^C!K=j^VY*hzXPN$0jY(+1c*@7+DWvywN4`Ma^CHN)AgjM}5GLUo{L z2M74scy>@ve@67E(K@WRTKlcTgZ)SKM&t0{*&}lBb==8P$O6MBKhAqlW-`)gr>J@dVZv>Q3-}ih*f|U^v({lsE zc8NhyDd9@R%*M0bfCL_4Ug(>YIG$;sDy9nUMi{I-9~IA5l)Oa8$c!3mQeFkYh8@*v z+*#G4H?;u9D?)7Lt?34~FbeK6TAs;j)UCPnT_A?{^1;r-B})r0?O+woRnuDw0r{`Z zSV-AVVX*kI^F(VtdER^SeE)|(IyQT<-{q4}`B!g4U7^^qP3ki4*ifH|u}wI4Y_{ro z?!sQe46X*oZkm*FFF1o)g5fO}@ae{5cHsGU33qU};ab5{t`&wt-}!V5YD&PeeQHXz zC0Pa7ZiZq%*089^s@5EPUb9_`zJsC>?F)$6qONIEmUt=m*2-S49QZ##>EGmbiq*mz z_3xL)AX&{^8P4xqvZ>b9dgT2B!z_U9xo5b{=5sAR7u%xl(st7}T!<4HUvv)D=aN&| zDu49}{kjy@e_Y0|W#dobfBgXQe}Dg=!2fSS>Gh9}@t3yoar|#I8aTJV-#Bd559lr_B7q-LWm{=0l2t{}I7ZCqc}}=?8R?MxP{Iu6jL->y=Z*XduH3+~youog z=6w1`2ryONg%YrD|6_m z@KAgjguW}$U>ODh^^XXjSC>A$6QHs-{NIytpIYkNb1XRrpbcj!KZ!Wf1|U(O8Daq$ zY3Bodq=8elOaIdSaB=CvXV|Paf8_r@zpCXN9derkM5{8iNmbc0Rf)F=nDAYw#;qI- zJQH-wg)|tmmu^|aWb(Wpk-Bi#0fk3oufY)!`!AKtLoy?M0L1rd1NO+25Cq1Q*!x&L z>(0F!3JRKuN%oRD-Ytynqha{OD9DEo4zwM?sv5iX zG?f$F#)Q}llCsh1bYO`pGN+L<^)_!#BzGKQhF93nn2n`G>}a`fFBUR%5mIbA$>lMF+0V3{RF}-6OOLkEam*kO=c`6XIE^~oT#P0$LgaC^>>Ib7mDgs0 z68zkzp&|xPidCYFJZe1F!@D|EWIhUq+C2??Vt{Hrj=Xt(z<+c$3(xJOX0468IXy{r1%l-m4;geZ?y^EUFJvZVTGOHaL?nAT z;y#jcZIReaaYRp|WU6fA*g;vM_YT^9#;S?1cEH>bOO?yR%3w+76k{_k0N#YXMKUJw ziXWho%gKoVA_e+8CBatr%;r$@lMt z;PzS?-OJ{xl9m?jpp;9i+L8~~X+~H#vBJQcE)qibRhrc?SZX%_|KnW&2@i3_jMXQF z5VN~Zt0|Q-*xB`c<4(01|4b>^#XzBD3RgndAw-~$%nhu2+yKQYF6vhZi`ev3pxfOz z25{Cce!C2=YD>5*kW!_cGa;fHFN6ZV!D4NVE37$<`@jYZYw6M@aOubvfHKw#Rjf`` zu2SO4r4{*6;UCp(i1#ZKEWrO;iAP0^pqz8Na;3iYprP@g#(TjEsu8~B_3e2Wcs^5jA}pXd zPZ2|=alBi^YFIMM=I`5&6b0X~ z8~h=r8Sens3&2(30xrNNSyeg3pp$XPoGKz=U>9HXr^4~kJRC0<_KidEQ9s(thpB@7 z6OByA$FuY9L?il?oKD8S>V2(GO5G{^h6eLopPh`)W&}7U-O=nd86T7G=r#F8AN8w> z_V3e)Hl333gy_T5fv)wdL?87AXMKJ2io685(HOS54x1g6&c*}{N>#cBa>vS0oAgfL zXZNK(&}Xl!5WIT3H+==l#^L3dah5kD(O!2DJJB2 zG6waSPLLRLB_KD_gceLS$*c&l;P11kmiW@wx^Q8CHAUexv-&rGU;cNa2>&Nqw?EX> zwe=Njd;vYkfNON`>4BF1GrsZU$d10-+`Y` zlf6A=+|WIX-rN&Buqz5!2)T+oxR}K{@CPw0YgY