Skip to content

Commit

Permalink
fill in some edge cases for manifest.json (empty manifest, RC's only).
Browse files Browse the repository at this point in the history
  • Loading branch information
@jackietung-redpanda
jackietung-redpanda committed Sep 5, 2024
1 parent 6d8baec commit 527978d
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 52 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/test_plugin_uploader.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
run: pip install -r requirements_test.txt

- working-directory: resources/plugin_uploader
run: pytest .
run: pytest -vv .

ruff-lint:
runs-on: ubuntu-latest
Expand Down
17 changes: 8 additions & 9 deletions resources/plugin_uploader/plugin_uploader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -297,15 +297,14 @@ def create_and_upload_manifest_json(
logging.info(f"Skipping {key}, missing tag: {ke}")
continue

assert (
version_to_artifact_infos
), f"No artifacts found in bucket {bucket} for {plugin_config.plugin_name}"
max_version = get_max_version_str(list(version_to_artifact_infos))

if max_version is None:
raise ValueError(
f"Could not deduce max version from existing artifacts in {bucket}"
)
max_version: str | None = None
if not version_to_artifact_infos:
logging.warning(f"No artifacts found in bucket {bucket} for {plugin_config.plugin_name}")
else:
max_version = get_max_version_str(list(version_to_artifact_infos))
if max_version is None:
logging.warning("No real releases found (may be only RCs?)")
logging.info(f"All versions found: {list(version_to_artifact_infos)}")

for version, artifact_infos in version_to_artifact_infos.items():
artifacts: dict[str, dict[str, str]] = {}
Expand Down
12 changes: 12 additions & 0 deletions resources/plugin_uploader/test_data/dist/metadata_v4_36_0_rc1.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"project_name": "cow",
"tag": "v4.36.0-rc1",
"previous_tag": "v4.34.0-rc2",
"version": "4.36.0-rc1",
"commit": "7eb28f2a994e277f17bf0530097d99208e65cddb",
"date": "2024-08-29T23:53:58.388135715Z",
"runtime": {
"goos": "linux",
"goarch": "arm64"
}
}
178 changes: 136 additions & 42 deletions resources/plugin_uploader/test_plugin_uploader.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import json
import unittest
from typing import Any

import boto3
from moto import mock_aws
from plugin_uploader import S3BucketClient, PluginConfig, cli
Expand Down Expand Up @@ -76,80 +78,172 @@ def _run_and_validate_upload_archives(
)
assert _result.exit_code == 0
found_keys = set(bucket_client.list_dir_recursive())

print(found_keys)
assert found_keys == expected_keys

# upload-archives (first run, for version v4.34.0)
def _run_and_validate_upload_manifests(expected_manifest: dict[str, Any]):
# upload-manifests (verify both versions of archives show up in manifest.json)
result = runner.invoke(
cli,
[
"upload-manifest",
f"--region={TEST_REGION}",
f"--bucket={TEST_BUCKET}",
f"--plugin={TEST_PLUGIN.plugin_name}",
"--repo-hostname=cow.farm.com",
],
catch_exceptions=False,
)
assert result.exit_code == 0
response = s3_client.get_object(Bucket=TEST_BUCKET, Key="cow/manifest.json")
found_manifest = json.load(response["Body"])

# align created_at - that is always different
found_manifest["created_at"] = 1700000000
assert expected_manifest == found_manifest

# upload-manifests before we have ANY archives in S3 (empty manifest.json)
_run_and_validate_upload_manifests(expected_manifest={
"archives": [],
"created_at": 1700000000,
})

# upload-archives (upload an RC)
_run_and_validate_upload_archives(
metadata_file=f"{TEST_DATA_DIR_PATH}/dist/metadata_v4_36_0_rc1.json",
expected_keys={
"cow/manifest.json",
"cow/archives/4.36.0-rc1/redpanda-cow-darwin-arm64.tar.gz",
"cow/archives/4.36.0-rc1/redpanda-cow-linux-amd64.tar.gz",
},
)
# RC's show up in manifest.json but should never be marked "is_latest"
_run_and_validate_upload_manifests(expected_manifest={
"archives": [
{
'artifacts': {
'darwin-arm64': {
'path': 'https://cow.farm.com/cow/archives/4.36.0-rc1/redpanda-cow-darwin-arm64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
'linux-amd64': {
'path': 'https://cow.farm.com/cow/archives/4.36.0-rc1/redpanda-cow-linux-amd64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
},
'version': '4.36.0-rc1',
},

],
"created_at": 1700000000,
})

# upload-archives (upload a real version 4.34.0 that has a lower version number than the RC)
_run_and_validate_upload_archives(
metadata_file=f"{TEST_DATA_DIR_PATH}/dist/metadata_v4_34_0.json",
expected_keys={
"cow/manifest.json",
"cow/archives/4.34.0/redpanda-cow-darwin-arm64.tar.gz",
"cow/archives/4.34.0/redpanda-cow-linux-amd64.tar.gz",
"cow/archives/4.36.0-rc1/redpanda-cow-darwin-arm64.tar.gz",
"cow/archives/4.36.0-rc1/redpanda-cow-linux-amd64.tar.gz",
},
)
# verify that 4.34 marked as latest, NOT the RC.
_run_and_validate_upload_manifests(expected_manifest={
"archives": [
{
'artifacts': {
'darwin-arm64': {
'path': 'https://cow.farm.com/cow/archives/4.34.0/redpanda-cow-darwin-arm64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
'linux-amd64': {
'path': 'https://cow.farm.com/cow/archives/4.34.0/redpanda-cow-linux-amd64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
},
'is_latest': True,
'version': '4.34.0',
},
{
'artifacts': {
'darwin-arm64': {
'path': 'https://cow.farm.com/cow/archives/4.36.0-rc1/redpanda-cow-darwin-arm64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
'linux-amd64': {
'path': 'https://cow.farm.com/cow/archives/4.36.0-rc1/redpanda-cow-linux-amd64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
},
'version': '4.36.0-rc1',
},

# upload-archives (second run, for version v4.35.0)
],
"created_at": 1700000000,
})

# upload-archives (newer release v4.35.0)
_run_and_validate_upload_archives(
metadata_file=f"{TEST_DATA_DIR_PATH}/dist/metadata_v4_35_0.json",
expected_keys={
"cow/manifest.json",
"cow/archives/4.34.0/redpanda-cow-darwin-arm64.tar.gz",
"cow/archives/4.34.0/redpanda-cow-linux-amd64.tar.gz",
"cow/archives/4.35.0/redpanda-cow-darwin-arm64.tar.gz",
"cow/archives/4.35.0/redpanda-cow-linux-amd64.tar.gz",
"cow/archives/4.36.0-rc1/redpanda-cow-darwin-arm64.tar.gz",
"cow/archives/4.36.0-rc1/redpanda-cow-linux-amd64.tar.gz",
},
)

# upload-manifests (verify both versions of archives show up in manifest.json)
result = runner.invoke(
cli,
[
"upload-manifest",
f"--region={TEST_REGION}",
f"--bucket={TEST_BUCKET}",
f"--plugin={TEST_PLUGIN.plugin_name}",
"--repo-hostname=cow.farm.com",
],
catch_exceptions=False,
)
assert result.exit_code == 0
response = s3_client.get_object(Bucket=TEST_BUCKET, Key="cow/manifest.json")
found_manifest = json.load(response["Body"])
expected_manifest = {
# verify that is_latest points to v4.36.0
_run_and_validate_upload_manifests(expected_manifest={
"archives": [
{
"artifacts": {
"darwin-arm64": {
"path": "https://cow.farm.com/cow/archives/4.34.0/redpanda-cow-darwin-arm64.tar.gz",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
'artifacts': {
'darwin-arm64': {
'path': 'https://cow.farm.com/cow/archives/4.34.0/redpanda-cow-darwin-arm64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
"linux-amd64": {
"path": "https://cow.farm.com/cow/archives/4.34.0/redpanda-cow-linux-amd64.tar.gz",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
'linux-amd64': {
'path': 'https://cow.farm.com/cow/archives/4.34.0/redpanda-cow-linux-amd64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
},
"version": "4.34.0",
'version': '4.34.0',
},
{
"artifacts": {
"darwin-arm64": {
"path": "https://cow.farm.com/cow/archives/4.35.0/redpanda-cow-darwin-arm64.tar.gz",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
'artifacts': {
'darwin-arm64': {
'path': 'https://cow.farm.com/cow/archives/4.35.0/redpanda-cow-darwin-arm64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
"linux-amd64": {
"path": "https://cow.farm.com/cow/archives/4.35.0/redpanda-cow-linux-amd64.tar.gz",
"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
'linux-amd64': {
'path': 'https://cow.farm.com/cow/archives/4.35.0/redpanda-cow-linux-amd64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
},
"is_latest": True,
"version": "4.35.0",
'is_latest': True,
'version': '4.35.0',
},
{
'artifacts': {
'darwin-arm64': {
'path': 'https://cow.farm.com/cow/archives/4.36.0-rc1/redpanda-cow-darwin-arm64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
'linux-amd64': {
'path': 'https://cow.farm.com/cow/archives/4.36.0-rc1/redpanda-cow-linux-amd64.tar.gz',
'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
},
},
'version': '4.36.0-rc1',
},

],
"created_at": 1700000000,
}

# align created_at - that is always different
found_manifest["created_at"] = 1700000000
assert expected_manifest == found_manifest
})


if __name__ == "__main__":
Expand Down

0 comments on commit 527978d

Please sign in to comment.