Skip to content

Commit

Permalink
remove code comment and add device id check
Browse files Browse the repository at this point in the history
Signed-off-by: Bangtian Liu <liubangtian@gmail.com>
  • Loading branch information
bangtianliu committed Jan 9, 2025
1 parent e509ac8 commit 4b2165c
Showing 1 changed file with 7 additions and 11 deletions.
18 changes: 7 additions & 11 deletions tuner/tuner/libtuner.py
Original file line number Diff line number Diff line change
Expand Up @@ -731,10 +731,8 @@ def benchmark_candidates(candidate_indices, devices, tuning_client, candidate_tr
"""
Runs the benchmarking for a given list of candidate indices.
"""
# Create worker context queue
worker_context_queue = create_worker_context_queue(devices)

# Prepare task list
task_list = [
BenchmarkPack(
iree_benchmark_module_flags=tuning_client.get_iree_benchmark_module_flags(),
Expand All @@ -745,16 +743,14 @@ def benchmark_candidates(candidate_indices, devices, tuning_client, candidate_tr
]

# Perform benchmarking
benchmark_results = multiprocess_progress_wrapper(
return multiprocess_progress_wrapper(
num_worker=len(devices),
task_list=task_list,
function=run_iree_benchmark_module_command,
initializer=init_worker_context,
initializer_inputs=(worker_context_queue,),
)

return benchmark_results


def compile(
args: argparse.Namespace,
Expand Down Expand Up @@ -886,7 +882,7 @@ def get_speedup(result: BenchmarkResult) -> float:

top_candidates = [result.candidate_id for result in best_results]

# Benchmarking baselines on each involved device again to check performance regression on devices.
# Benchmarking baselines again to check for performance regressions. These may indicate machine instability, overheating, etc.
post_baseline_indices = [0] * len(args.devices)
post_baseline_results = benchmark_candidates(
candidate_indices=post_baseline_indices,
Expand All @@ -901,22 +897,22 @@ def get_speedup(result: BenchmarkResult) -> float:

assert (
baseline_times_by_device.keys() == post_baseline_times_by_device.keys()
), "Error: The device IDs in baseline and post-baseline results do not match."
), "Device ID mismatch between baseline runs."

regression_detected = False
for device_id in baseline_times_by_device:
if device_id not in baseline_times_by_device:
logging.warning(f"Baseline time missing for device {device_id}")
continue

baseline_time = baseline_times_by_device[device_id]
post_time = post_baseline_times_by_device[device_id]

if post_time > baseline_time * 1.03:
regression_detected = True
percentage_slower = ((post_time - baseline_time) / baseline_time) * 100
logging.warning(
f"Performance regression detected on device {device_id}: "
f"Baseline time = {baseline_time}, Post-baseline time = {post_time}, "
f"Slower by {percentage_slower:.3f}%"
)

if not regression_detected:
logging.debug("No performance regressions detected.")
return top_candidates

0 comments on commit 4b2165c

Please sign in to comment.