Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Merge the power checker changes from the inference repository #322

Merged
merged 4 commits into from
Nov 28, 2023
Merged

Merge the power checker changes from the inference repository #322

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
250aa83
Merge the power checker changes from the inference repository
arjunsuresh Nov 27, 2023
3b6b632
Fix formatting
arjunsuresh Nov 27, 2023
5c83b14
Fix formatting
arjunsuresh Nov 27, 2023
05e4ef2
Merge PR 1496 from inference repo
arjunsuresh Nov 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 13 additions & 9 deletions compliance/check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# =============================================================================

from collections import OrderedDict
from datetime import datetime, timezone
from datetime import datetime, timezone, timedelta
from typing import Dict, List, Tuple, Any, Optional, Callable
import argparse
import hashlib
Expand Down Expand Up @@ -84,7 +84,7 @@ class CheckerWarning(Exception):
re.compile(r"Uncertainty \d+.\d+%, which is above 1.00% limit for the last sample!")
]

TIME_DELTA_TOLERANCE = 500 # in milliseconds
TIME_DELTA_TOLERANCE = 800 # in milliseconds


def _normalize(path: str) -> str:
Expand Down Expand Up @@ -395,6 +395,10 @@ def get_avg_power(power_path: str, run_path: str) -> Tuple[float, float]:
os.path.join(path, os.path.basename(run_path)), client_sd
)

# convert to UTC
power_begin = datetime.fromtimestamp(power_begin, tz=timezone.utc)
power_end = datetime.fromtimestamp(power_end, tz=timezone.utc)

detail_log_fname = os.path.join(run_path, "mlperf_log_detail.txt")
datetime_format = "%m-%d-%Y %H:%M:%S.%f"

Expand All @@ -406,7 +410,7 @@ def get_avg_power(power_path: str, run_path: str) -> Tuple[float, float]:
for line in f:
timestamp = (
datetime.strptime(line.split(",")[1], datetime_format)
).timestamp()
).replace(tzinfo=timezone.utc)
if timestamp > power_begin and timestamp < power_end:
cpower = float(line.split(",")[3])
cpf = float(line.split(",")[9])
Expand Down Expand Up @@ -596,13 +600,13 @@ def find_error_or_warning(reg_exp: str, line: str, error: bool) -> None:
problem_line = re.search(reg_exp, line)

if problem_line and problem_line.group(0):
log_time = get_time_from_line(line, date_regexp, file_path, timezone_offset)
log_time = get_time_from_line(line, date_regexp, file_path, 0)
if start_ranging_time is None or stop_ranging_time is None:
assert False, "Can not find ranging time in ptd_logs.txt."
if error:
if problem_line.group(0).strip() in COMMON_ERROR_TESTING:
raise CheckerWarning(
f"{line.strip()!r} in ptd_log.txt during testing stage but it is accepted. Treated as WARNING"
f"{line.strip().replace('ERROR', 'Warning')!r} in ptd_log.txt during testing stage but it is accepted. Treated as WARNING"
)
assert (
start_ranging_time < log_time < stop_ranging_time
Expand All @@ -614,7 +618,7 @@ def find_error_or_warning(reg_exp: str, line: str, error: bool) -> None:
for common_ranging_error in COMMON_ERROR_RANGING
):
raise CheckerWarning(
f"{line.strip()!r} in ptd_log.txt during ranging stage. Treated as WARNING"
f"{line.strip().replace('ERROR', 'Warning')!r} in ptd_log.txt during ranging stage. Treated as WARNING"
)
else:
if (
Expand Down Expand Up @@ -653,12 +657,12 @@ def get_msg_without_time(line: str) -> Optional[str]:
continue
if (not start_ranging_time) and (start_ranging_line == msg):
start_ranging_time = get_time_from_line(
line, date_regexp, file_path, timezone_offset
line, date_regexp, file_path, 0 # timezone_offset
)
if (not stop_ranging_time) and bool(start_ranging_time):
if ": Completed test" == msg:
stop_ranging_time = get_time_from_line(
line, date_regexp, file_path, timezone_offset
line, date_regexp, file_path, 0 # timezone_offset
)
break

Expand All @@ -673,7 +677,7 @@ def get_msg_without_time(line: str) -> Optional[str]:
try:
log_time = None
log_time = get_time_from_line(
line, date_regexp, file_path, timezone_offset
line, date_regexp, file_path, 0 # timezone_offset
)
except LineWithoutTimeStamp:
assert (
Expand Down
Loading