Skip to content

Commit

Permalink
Merge pull request #227 from Vipinofficial11/addPluginUploadSkipFlag
Browse files Browse the repository at this point in the history 
Adding a flag to skip plugin upload in cdap sandbox.
  • Loading branch information
@Vipinofficial11
Vipinofficial11 authored Nov 22, 2023
2 parents aeba9a4 + a864c99 commit 67246c2
Showing 1 changed file with 59 additions and 60 deletions.
119 changes: 59 additions & 60 deletions src/main/scripts/run_e2e_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def run_shell_command(cmd):
parser.add_argument('--framework', help='Pass this param if workflow is triggered from e2e framework repo')
parser.add_argument('--mvnTestRunProfiles', help='Maven build profiles to run the e2e tests')
parser.add_argument('--mvnProjectBuildProfiles', help='Maven project build profiles')
parser.add_argument('--skipPluginUpload', help='Pass this param if plugin upload needs to be skipped')
args=parser.parse_args()

# Start CDAP sandbox
Expand Down Expand Up @@ -66,7 +67,7 @@ def run_shell_command(cmd):
res = requests.put('http://localhost:11015/v3/preferences', headers= {'Content-Type': 'application/json'}, json={'task.executor.system.resources.memory': 4096})
assert res.ok or print(res.text)

# Upload required plugins from CDAP HUb
# Upload required plugins from CDAP Hub
plugin_details_file = open(os.path.join('e2e', 'src', 'main', 'scripts', 'required_plugins.yaml'))
plugin_details = yaml.load(plugin_details_file, Loader=yaml.FullLoader)

Expand All @@ -79,69 +80,67 @@ def run_shell_command(cmd):
if args.module:
module_to_build = args.module

# Build the plugin
os.chdir("plugin")
if module_to_build:
print(f"Building plugin module: {module_to_build}")
if args.mvnProjectBuildProfiles:
run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests -P {args.mvnProjectBuildProfiles}")
else:
run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests")
if args.skipPluginUpload:
if module_to_build:
print(f"Skipping the plugin upload for module: {module_to_build}")
else:
print("Building plugin")
if args.mvnProjectBuildProfiles:
run_shell_command(f"mvn clean package -DskipTests -P {args.mvnProjectBuildProfiles}")
os.chdir("plugin")
if module_to_build:
print(f"Building plugin module: {module_to_build}")
if args.mvnProjectBuildProfiles:
run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests -P {args.mvnProjectBuildProfiles}")
else:
run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests")
else:
run_shell_command("mvn clean package -DskipTests")

# Get plugin artifact name and version from pom.xml.
root = ET.parse('pom.xml').getroot()
plugin_version = root.find('{http://maven.apache.org/POM/4.0.0}version').text
if module_to_build:
os.chdir(f"{module_to_build}")
root = ET.parse('pom.xml').getroot()
plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text
else:
plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text

os.chdir("target")
plugin_properties = {}
plugin_parents = []
# Get plugin properties and parent from plugin json.
with open(f'{plugin_name}-{plugin_version}.json') as f:
obj = json.loads(f.read())
plugin_properties = obj['properties']
plugin_parents = obj['parents']

data = None
with open(f'{plugin_name}-{plugin_version}.jar', 'rb') as f:
data = f.read()

# Install the plugin on the sandbox.
print("Installing plugin")
res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Extends": '/'.join(plugin_parents), "Artifact-Version": plugin_version}, data=data)
assert res.ok or print(res.text)
res=requests.put(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}/versions/{plugin_version}/properties", json=plugin_properties)
assert res.ok or print(res.text)

if module_to_build:
os.chdir("../../..")
driver_details_file = open(os.path.join('e2e', 'src', 'main', 'scripts', 'driver_details.yaml'))
driver_details = yaml.load(driver_details_file, Loader=yaml.FullLoader)
if module_to_build in driver_details['modules']:
url = driver_details['modules'][module_to_build]['url']
driver_prop = str(driver_details['modules'][module_to_build]['driver_prop'])
artifact_name = driver_details['modules'][module_to_build]['artifact_name']
artifact_version = driver_details['modules'][module_to_build]['artifact_version']
get_driver_jar = requests.get(url)
assert get_driver_jar.ok or print(get_driver_jar.text)
driverData = get_driver_jar.content
print(f"Installing {artifact_name} driver")
res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{artifact_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Version": artifact_version, "Artifact-Plugins": driver_prop }, data=driverData)
print("Building plugin")
if args.mvnProjectBuildProfiles:
run_shell_command(f"mvn clean package -DskipTests -P {args.mvnProjectBuildProfiles}")
else:
run_shell_command("mvn clean package -DskipTests")
# Get plugin artifact name and version from pom.xml.
root = ET.parse('pom.xml').getroot()
plugin_version = root.find('{http://maven.apache.org/POM/4.0.0}version').text
if module_to_build:
os.chdir(f"{module_to_build}")
root = ET.parse('pom.xml').getroot()
plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text
else:
plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text
os.chdir("target")
plugin_properties = {}
plugin_parents = []
# Get plugin properties and parent from plugin json.
with open(f'{plugin_name}-{plugin_version}.json') as f:
obj = json.loads(f.read())
plugin_properties = obj['properties']
plugin_parents = obj['parents']
data = None
with open(f'{plugin_name}-{plugin_version}.jar', 'rb') as f:
data = f.read()
# Install the plugin on the sandbox.
print("Installing plugin")
res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Extends": '/'.join(plugin_parents), "Artifact-Version": plugin_version}, data=data)
assert res.ok or print(res.text)
res=requests.put(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}/versions/{plugin_version}/properties", json=plugin_properties)
assert res.ok or print(res.text)
else:
os.chdir("../..")

if module_to_build:
os.chdir("../../..")
driver_details_file = open(os.path.join('e2e', 'src', 'main', 'scripts', 'driver_details.yaml'))
driver_details = yaml.load(driver_details_file, Loader=yaml.FullLoader)
if module_to_build in driver_details['modules']:
url = driver_details['modules'][module_to_build]['url']
driver_prop = str(driver_details['modules'][module_to_build]['driver_prop'])
artifact_name = driver_details['modules'][module_to_build]['artifact_name']
artifact_version = driver_details['modules'][module_to_build]['artifact_version']
get_driver_jar = requests.get(url)
assert get_driver_jar.ok or print(get_driver_jar.text)
driverData = get_driver_jar.content
print(f"Installing {artifact_name} driver")
res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{artifact_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Version": artifact_version, "Artifact-Plugins": driver_prop }, data=driverData)
assert res.ok or print(res.text)
else:
os.chdir("../..")
print("cwd:", os.getcwd())
print("ls:", os.listdir())

Expand Down

0 comments on commit 67246c2

Please sign in to comment.