Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_tpu7x_2x2x1.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "2x2x1"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "2x2x2", ici_size_range: 8, sharding_strategy: "2x2x1", op_dimension: 1, num_runs: 5} # Parallel Replica
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "2x2x2", ici_size_range: 8, sharding_strategy: "2x2x2", op_dimension: 1, num_runs: 5} # Non-Parallel Replica
Expand Down
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_tpu7x_2x2x2.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "2x2x2"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "4x2x2", ici_size_range: 16, sharding_strategy: "4x2x1", op_dimension: 1, num_runs: 5} # Parallel Replica
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "4x2x2", ici_size_range: 16, sharding_strategy: "4x2x2", op_dimension: 1, num_runs: 5} # Non-Parallel Replica
Expand Down
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_tpu7x_2x2x4.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "2x2x4"
benchmark_sweep_params:
- {matrix_dim_range: {start: 16, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "4x4x2", ici_size_range: 32, sharding_strategy: "4x4x1", op_dimension: 1, num_runs: 5} # Parallel Replica
- {matrix_dim_range: {start: 16, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "4x4x2", ici_size_range: 32, sharding_strategy: "4x4x2", op_dimension: 1, num_runs: 5} # Non-Parallel Replica
Expand Down
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_tpu7x_2x4x4.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "2x4x4"
benchmark_sweep_params:
- {matrix_dim_range: {start: 32, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "8x4x2", ici_size_range: 64, sharding_strategy: "8x4x1", op_dimension: 1, num_runs: 5} # Parallel Replica
- {matrix_dim_range: {start: 32, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "8x4x2", ici_size_range: 64, sharding_strategy: "8x4x2", op_dimension: 1, num_runs: 5} # Non-Parallel Replica
Expand Down
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_tpu7x_4x4x4.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "4x4x4"
benchmark_sweep_params:
- {matrix_dim_range: {start: 64, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "16x4x2", ici_size_range: 128, sharding_strategy: "16x4x1", op_dimension: 1, num_runs: 5} # Parallel Replica
- {matrix_dim_range: {start: 64, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "16x4x2", ici_size_range: 128, sharding_strategy: "16x4x2", op_dimension: 1, num_runs: 5} # Non-Parallel Replica
Expand Down
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_tpu7x_4x4x8.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "4x4x8"
benchmark_sweep_params:
- {matrix_dim_range: {start: 128, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "32x4x2", ici_size_range: 256, sharding_strategy: "32x4x1", op_dimension: 1, num_runs: 5} # Parallel Replica
- {matrix_dim_range: {start: 128, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "32x4x2", ici_size_range: 256, sharding_strategy: "32x4x2", op_dimension: 1, num_runs: 5} # Non-Parallel Replica
Expand Down
9 changes: 9 additions & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_16x16.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
benchmarks:
- benchmark_name: all_to_all
topology: "16x16"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "16x16", ici_size_range: 256, sharding_strategy: "16x16", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_16x16"
csv_path: "../microbenchmarks/all_to_all_v6e_16x16"
xlml_metrics_dir: "../microbenchmarks/all_to_all_v6e_16x16"
xla_dump_dir: "../microbenchmarks/all_to_all_v6e_16x16/hlo_graphs"
9 changes: 9 additions & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_2x2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
benchmarks:
- benchmark_name: all_to_all
topology: "2x2"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "2x2", ici_size_range: 4, sharding_strategy: "2x2", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_2x2"
csv_path: "../microbenchmarks/all_to_all_v6e_2x2"
xlml_metrics_dir: "../microbenchmarks/all_to_all_v6e_2x2"
xla_dump_dir: "../microbenchmarks/all_to_all_v6e_2x2/hlo_graphs"
9 changes: 9 additions & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_2x4.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
benchmarks:
- benchmark_name: all_to_all
topology: "2x4"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "4x2", ici_size_range: 8, sharding_strategy: "4x2", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_2x4"
csv_path: "../microbenchmarks/all_to_all_v6e_2x4"
xlml_metrics_dir: "../microbenchmarks/all_to_all_v6e_2x4"
xla_dump_dir: "../microbenchmarks/all_to_all_v6e_2x4/hlo_graphs"
1 change: 1 addition & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_4x4.yaml
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
benchmarks:
- benchmark_name: all_to_all
topology: "4x4"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "4x4", ici_size_range: 16, sharding_strategy: "4x4", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_4x4"
Expand Down
9 changes: 9 additions & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_4x8.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
benchmarks:
- benchmark_name: all_to_all
topology: "4x8"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "8x4", ici_size_range: 32, sharding_strategy: "8x4", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_4x8"
csv_path: "../microbenchmarks/all_to_all_v6e_4x8"
xlml_metrics_dir: "../microbenchmarks/all_to_all_v6e_4x8"
xla_dump_dir: "../microbenchmarks/all_to_all_v6e_4x8/hlo_graphs"
9 changes: 9 additions & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_8x16.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
benchmarks:
- benchmark_name: all_to_all
topology: "8x16"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "16x8", ici_size_range: 128, sharding_strategy: "16x8", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_8x16"
csv_path: "../microbenchmarks/all_to_all_v6e_8x16"
xlml_metrics_dir: "../microbenchmarks/all_to_all_v6e_8x16"
xla_dump_dir: "../microbenchmarks/all_to_all_v6e_8x16/hlo_graphs"
9 changes: 9 additions & 0 deletions Ironwood/configs/collectives/all_to_all_v6e_8x8.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
benchmarks:
- benchmark_name: all_to_all
topology: "8x8"
benchmark_sweep_params:
- {matrix_dim_range: {start: 8, end: 16384, multiplier: 2}, dtype: "float32", mesh_shape: "8x8", ici_size_range: 64, sharding_strategy: "8x8", op_dimension: 0, num_runs: 5} # Full sharding
trace_dir: "../microbenchmarks/all_to_all_v6e_8x8"
csv_path: "../microbenchmarks/all_to_all_v6e_8x8"
xlml_metrics_dir: "../microbenchmarks/all_to_all_v6e_8x8"
xla_dump_dir: "../microbenchmarks/all_to_all_v6e_8x8/hlo_graphs"
112 changes: 103 additions & 9 deletions Ironwood/src/benchmark_collectives.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,65 @@
GLOBAL_PSTATE = 7
LOG_SPARSECORE_USAGE = False
V6E_DEVICE_KINDS = ["TPU v6 lite", "TPU v6e"]
PEAK_ICI_LINK_BW = 100.0 # GB/s unidirectional for both v6e and tpu7x


def calculate_all_to_all_theoretical_bandwidth(
topology: str,
device_kind: str,
rank: int,
participating_ranks: int,
) -> Any:
"""Calculates theoretical peak All-to-All bandwidth in GB/s."""
if (
not topology
or rank is None
or rank <= 0
or participating_ranks is None
or participating_ranks <= 0
):
return None

try:
topo_dims = [
int(x) for x in topology.split("x") if x.strip() and int(x) > 0
]
except (ValueError, AttributeError):
return None

if not topo_dims:
return None

spatial_dims = [d for d in topo_dims if d > 1] or [1]
dim_count = len(spatial_dims)

if device_kind in V6E_DEVICE_KINDS:
# TPU v6e (2D): wrap-around link exists if any dimension >= 16
has_wrap = any(d >= 16 for d in topo_dims)
else:
# TPU v7x (3D): wrap-around torus requires all 3 spatial dimensions >= 4
has_wrap = (len(topo_dims) >= 3) and all(d >= 4 for d in topo_dims[:3])

wrap_factor = 2.0 if has_wrap else 1.0
sorted_dims = sorted(spatial_dims)

if dim_count == 1:
link_multiplier = 1.0 * wrap_factor
elif dim_count == 2:
link_multiplier = sorted_dims[0] * wrap_factor
else:
link_multiplier = sorted_dims[0] * sorted_dims[1] * wrap_factor

# Generic All-To-All Formula :
# 4 * link_bw (100.0) * link_multiplier * participating_ranks / (rank^2)
theo_bw = (
4.0
* PEAK_ICI_LINK_BW
* link_multiplier
* float(participating_ranks)
/ (float(rank) * float(rank))
)
return theo_bw


def create_mesh(ici_size: int, mesh_shape: str) -> Mesh:
Expand Down Expand Up @@ -97,6 +156,7 @@ def unified_ici_collectives_metrics(
iteration: int,
op_type: str,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
"""Calculates the metrics for the ICI collectives benchmark."""

Expand Down Expand Up @@ -172,6 +232,18 @@ def unified_ici_collectives_metrics(
/ rank
)

# Calculate All-to-All Theoretical Peak Bandwidth if topology is provided
theo_bw = None
if op_type == "A2A" and topology:
theo_bw = calculate_all_to_all_theoretical_bandwidth(
topology=topology,
device_kind=device_kind,
rank=rank,
participating_ranks=participating_ranks,
)

has_valid_theo_bw = theo_bw is not None and theo_bw > 0

sparsecore_used = "NA"
if LOG_SPARSECORE_USAGE:
print("trace_dir: ", trace_dir)
Expand All @@ -197,6 +269,10 @@ def unified_ici_collectives_metrics(
"hlo_replica_groups": json.dumps(hlo_replica_groups),
"sparsecore_used": sparsecore_used,
}
if topology:
metadata["topology"] = topology
if has_valid_theo_bw:
metadata["theoretical_bw (GB/s)"] = theo_bw
achieved_bw = [
transferred_data * 1000 / my_time
for my_time in ici_average_time_ms_list
Expand All @@ -208,6 +284,13 @@ def unified_ici_collectives_metrics(
metrics.update(average_time_ms_statistics.serialize_statistics())
metrics.update(achieved_bw_statistics.serialize_statistics())

if has_valid_theo_bw:
utilization_list = [(bw / theo_bw) * 100.0 for bw in achieved_bw]
util_stats = MetricsStatistics(
metrics_list=utilization_list, metrics_name="utilization_pct"
)
metrics.update(util_stats.serialize_statistics())

print("metadata: ", metadata)
print("metrics: ", metrics)
return metadata, metrics
Expand All @@ -222,6 +305,7 @@ def psum_benchmark(
dtype: jnp.dtype = jax.numpy.bfloat16,
num_runs: int = 1,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Benchmarks the psum collective operation.
Expand Down Expand Up @@ -345,7 +429,8 @@ def psum_benchmark_calculate_metrics(
matrix_shape: tuple[int, int, int],
xla_output: str,
op_type: str,
trace_dir: str,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Calculates the metrics for the psum benchmark."""
Expand All @@ -361,7 +446,8 @@ def psum_benchmark_calculate_metrics(
ici_average_time_ms_list,
matrix_dim,
op_type,
trace_dir,
trace_dir=trace_dir,
topology=topology,
)


Expand All @@ -374,6 +460,7 @@ def psum_scatter_benchmark(
op_dimension: int = 1,
num_runs: int = 1,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Benchmarks the psum_scatter collective operation.
Expand Down Expand Up @@ -462,7 +549,8 @@ def psum_scatter_benchmark_calculate_metrics(
matrix_shape: tuple[int, int, int],
xla_output: str,
op_type: str,
trace_dir: str,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Calculates the metrics for the psum_scatter benchmark."""
Expand All @@ -478,7 +566,8 @@ def psum_scatter_benchmark_calculate_metrics(
ici_average_time_ms_list,
matrix_dim,
op_type,
trace_dir,
trace_dir=trace_dir,
topology=topology,
)


Expand All @@ -491,6 +580,7 @@ def all_gather_benchmark(
op_dimension: int = 1,
num_runs: int = 1,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Benchmarks the all_gather collective operation.
Expand Down Expand Up @@ -523,7 +613,6 @@ def all_gather_benchmark(
f"--xla_tpu_dvfs_p_state={GLOBAL_PSTATE}",
"--xla_tpu_scoped_vmem_limit_kib=65536",
]
# libtpu_init_args=[ ]
os.environ["LIBTPU_INIT_ARGS"] = " ".join(libtpu_init_args)
mesh = create_mesh(ici_size, mesh_shape)

Expand Down Expand Up @@ -579,7 +668,8 @@ def all_gather_benchmark_calculate_metrics(
matrix_shape: tuple[int, int, int],
xla_output: str,
op_type: str,
trace_dir: str,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Calculates the metrics for the all_gather benchmark."""
Expand All @@ -595,7 +685,8 @@ def all_gather_benchmark_calculate_metrics(
ici_average_time_ms_list,
matrix_dim,
op_type,
trace_dir,
trace_dir=trace_dir,
topology=topology,
)


Expand All @@ -608,6 +699,7 @@ def all_to_all_benchmark(
op_dimension: int = 1,
num_runs: int = 1,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Benchmarks the all_to_all collective operation.
Expand Down Expand Up @@ -691,7 +783,8 @@ def all_to_all_benchmark_calculate_metrics(
matrix_shape: tuple[int, int, int],
xla_output: str,
op_type: str,
trace_dir: str,
trace_dir: str = None,
topology: str = None,
) -> Dict[str, Any]:
# pylint: disable=unused-argument
"""Calculates the metrics for the all_to_all benchmark."""
Expand All @@ -707,5 +800,6 @@ def all_to_all_benchmark_calculate_metrics(
ici_average_time_ms_list,
matrix_dim,
op_type,
trace_dir,
trace_dir=trace_dir,
topology=topology,
)
12 changes: 12 additions & 0 deletions Ironwood/src/run_benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,12 @@ def run_single_benchmark(benchmark_config: Dict[str, Any], output_path: str):
for param in benchmark_params:
if "num_runs" not in param:
param["num_runs"] = global_num_runs
# Inject topology from config if present in benchmark_config
global_topology = benchmark_config.get("topology")
if global_topology is not None:
for param in benchmark_params:
if "topology" not in param:
param["topology"] = global_topology

if not benchmark_name:
raise ValueError("Each benchmark must have a benchmark_name.")
Expand Down Expand Up @@ -518,6 +524,12 @@ def run_benchmark_multithreaded(benchmark_config, output_path):
for param in benchmark_params:
if "num_runs" not in param:
param["num_runs"] = global_num_runs
# Inject topology from config if present in benchmark_config
global_topology = benchmark_config.get("topology")
if global_topology is not None:
for param in benchmark_params:
if "topology" not in param:
param["topology"] = global_topology

# Get the benchmark function
benchmark_func, calculate_metrics_func = get_benchmark_functions(
Expand Down
Loading