Fix

joaquinmartis · pytorchmergebot · commit da4b2c087736 · 2025-05-28T19:11:01.000Z
diff --git a/benchmarks/inductor_backends/cutlass.py b/benchmarks/inductor_backends/cutlass.py
@@ -80,7 +80,6 @@ def benchmark_torch_function_in_microseconds(func: Callable, *args, **kwargs) ->
 
 @dataclass(frozen=True, kw_only=True)
 class ExperimentConfig:
-    autotune_fallback_to_aten: bool = False
     max_autotune: bool = True
     coordinate_descent_tuning: bool = True
     max_autotune_gemm_backends: str = "ATEN"
@@ -91,7 +90,6 @@ def name(self) -> str:
 
     def to_options(self) -> dict[str, Any]:
         return {
-            "autotune_fallback_to_aten": self.autotune_fallback_to_aten,
             "max_autotune": self.max_autotune,
             "coordinate_descent_tuning": self.coordinate_descent_tuning,
             "max_autotune_gemm_backends": self.max_autotune_gemm_backends,
diff --git a/test/inductor/test_cutlass_backend.py b/test/inductor/test_cutlass_backend.py
@@ -129,7 +129,6 @@ def gen_args(op, shape, dtype=torch.float16):
         "max_autotune": True,
         "max_autotune_gemm_backends": "CUTLASS",
         "cuda.cutlass_max_profiling_configs": 1,
-        "autotune_fallback_to_aten": False,
         "benchmark_epilogue_fusion": False,  # EVT doesn't support benchmark fusion yet
         "cuda.cutlass_tma_only": True,
         "cuda.cutlass_epilogue_fusion_enabled": True,
@@ -141,7 +140,6 @@ def gen_args(op, shape, dtype=torch.float16):
         "max_autotune": True,
         "max_autotune_gemm_backends": "CUTLASS",
         "cuda.cutlass_max_profiling_configs": 1,
-        "autotune_fallback_to_aten": False,
         "benchmark_epilogue_fusion": False,  # EVT doesn't support benchmark fusion yet
         "cuda.cutlass_tma_only": True,
     }
@@ -269,7 +267,6 @@ def test_cutlass_backend_subproc_mm(self):
                 "max_autotune_gemm_backends": "CUTLASS",
                 "compile_threads": 4,
                 "cuda.cutlass_max_profiling_configs": 4,
-                "autotune_fallback_to_aten": False,
             }
         ):
             Y_compiled = torch.compile(torch.mm)(a, b)
@@ -308,7 +305,6 @@ def test_cutlass_backend_subproc_addmm(self, shape_combo):
                 "max_autotune_gemm_backends": "CUTLASS",
                 "compile_threads": 4,
                 "cuda.cutlass_max_profiling_configs": 4,
-                "autotune_fallback_to_aten": False,
             }
         ):
             for x_shape in x_shapes:
@@ -336,7 +332,6 @@ def test_cutlass_backend_subproc_bmm(self):
                 "max_autotune_gemm_backends": "CUTLASS",
                 "compile_threads": 4,
                 "cuda.cutlass_max_profiling_configs": 4,
-                "autotune_fallback_to_aten": False,
             }
         ):
             Y_compiled = torch.compile(torch.bmm)(a, b)
@@ -369,7 +364,6 @@ def forward(self, a, b, c):
                 "autotune_in_subproc": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 1,
-                "autotune_fallback_to_aten": False,
             }
         ):
             from torch._inductor.utils import run_and_get_code
@@ -412,7 +406,6 @@ def forward(self, a, b, c):
                 "autotune_in_subproc": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 1,
-                "autotune_fallback_to_aten": False,
                 "cuda.cutlass_max_profiling_swizzle_options": [
                     1,
                     2,
@@ -482,7 +475,6 @@ def forward(self, a, b):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
             }
         ), dynamo_config.patch({"error_on_recompile": dynamic}):
             expected = [model(*input) for input in inputs]
@@ -560,7 +552,6 @@ def forward(self, x, a, b):
                     "max_autotune": True,
                     "max_autotune_gemm_backends": max_autotune_gemm_backends,
                     "cuda.cutlass_max_profiling_configs": 2,
-                    "autotune_fallback_to_aten": False,
                 }
             ), dynamo_config.patch({"error_on_recompile": dynamic}):
                 expected = [model(*input) for input in inputs]
@@ -622,7 +613,6 @@ def forward(self, a, b):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
             }
         ):
             expected = [model(*input) for input in inputs]
@@ -657,7 +647,6 @@ def mm(a, b):
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 2,
                 "cuda.cutlass_op_allowlist_regex": "stream_k",  # only stream-k GEMM Kernels
-                "autotune_fallback_to_aten": False,
             }
         ):
             for M, K, N in (
@@ -713,7 +702,6 @@ def _test_max_autotune_cutlass_backend_epilogue_fusion(
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 4,
                 "cuda.version": "12.2",  # required to enable the Kernels we need
-                "autotune_fallback_to_aten": False,
             }
         ):
             counters["inductor"]["cuda_epilogue_fusion_counter"] = 0
@@ -812,7 +800,6 @@ def mm(a, b):
                 "autotune_in_subproc": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
             }
         ):
             Y_compiled = torch.compile(mm, dynamic=dynamic)(a, b)
@@ -831,7 +818,6 @@ def forward(self, x, w):
                 "max_autotune": True,
                 "autotune_in_subproc": False,
                 "max_autotune_gemm_backends": "CUTLASS",
-                "autotune_fallback_to_aten": False,
                 "cuda.cutlass_max_profiling_configs": 2,
             }
         ):
@@ -870,7 +856,6 @@ def forward(self, x, w):
                 "max_autotune": True,
                 "autotune_in_subproc": False,
                 "max_autotune_gemm_backends": "CUTLASS",
-                "autotune_fallback_to_aten": False,
                 "cuda.cutlass_max_profiling_configs": 2,
             }
         ):
@@ -904,7 +889,6 @@ def forward(self, x, w):
                 "max_autotune": True,
                 "autotune_in_subproc": False,
                 "max_autotune_gemm_backends": "CUTLASS",
-                "autotune_fallback_to_aten": False,
                 "cuda.cutlass_op_allowlist_regex": "128x256x64.*stream_k_warpspecialized_cooperative_epi_nosmem",
                 "cuda.cutlass_max_profiling_configs": 1,
             }
@@ -951,7 +935,6 @@ def mm(a, b):
                 "max_autotune_gemm_backends": "CUTLASS",
                 "cuda.cutlass_max_profiling_configs": 2,
                 "autotune_local_cache": True,
-                "autotune_fallback_to_aten": False,
             }
         ):
             Y_compiled = torch.compile(mm, dynamic=dynamic)(a_sparse, b)
@@ -1107,7 +1090,6 @@ def select_no_algorithm(*args, **kwargs):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": "CUTLASS",
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
             }
         ), mock.patch(
             "torch._inductor.kernel.mm.autotune_select_algorithm",
@@ -1166,7 +1148,6 @@ def select_no_algorithm(*args, **kwargs):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": "CUTLASS",
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
                 "cuda.cutlass_presets": presets,
             }
         ), mock.patch(
@@ -1272,7 +1253,6 @@ def test_standalone_runner(self):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
                 "cuda.generate_test_runner": True,  # put standalone runner in the generated code
             }
         ):
@@ -1389,7 +1369,6 @@ def test_cutlass_backend_matmul_same_tensor(self):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": max_autotune_gemm_backends,
                 "cuda.cutlass_max_profiling_configs": 2,
-                "autotune_fallback_to_aten": False,
             }
         ):
             compiled = torch.compile(torch.mm)
@@ -1413,7 +1392,6 @@ def forward(self, B):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": "CUTLASS",
                 "cuda.cutlass_max_profiling_configs": 1,
-                "autotune_fallback_to_aten": False,
             }
         ):
             _ = torch.compile(model)(B)
@@ -1436,7 +1414,6 @@ def forward(self, B):
                 "max_autotune": True,
                 "max_autotune_gemm_backends": "CUTLASS",
                 "cuda.cutlass_max_profiling_configs": 1,
-                "autotune_fallback_to_aten": False,
             }
         ):
             _ = torch.compile(model)(B)
diff --git a/test/inductor/test_max_autotune.py b/test/inductor/test_max_autotune.py
@@ -139,7 +139,6 @@ def mm(a, b):
         with config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": "1",
                 "test_configs.autotune_choice_name_regex": "mm_persistent_tma",
             }
@@ -164,7 +163,6 @@ def mm(a, b):
         with self.assertRaises(BackendCompilerFailed) as context, config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": "1",
                 "test_configs.autotune_choice_name_regex": "mm_persistent_tma",
             }
@@ -198,7 +196,6 @@ def mm(a, b):
         with config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": "1",
                 "test_configs.autotune_choice_name_regex": "mm_persistent_tma",
             }
@@ -259,7 +256,6 @@ def addmm(x, a, b):
         with config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": "1",
                 "test_configs.autotune_choice_name_regex": "mm_persistent_tma",
             }
@@ -285,7 +281,6 @@ def addmm(x, a, b):
         with self.assertRaises(BackendCompilerFailed) as context, config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": "1",
                 "test_configs.autotune_choice_name_regex": "mm_persistent_tma",
             }
@@ -321,7 +316,6 @@ def addmm(x, a, b):
         with config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": "1",
                 "test_configs.autotune_choice_name_regex": "mm_persistent_tma",
             }
@@ -380,7 +374,6 @@ def scaled_mm(
         with config.patch(
             {
                 "max_autotune": True,
-                "autotune_fallback_to_aten": False,
                 "triton.enable_persistent_tma_matmul": True,
                 "max_autotune_gemm_backends": "TRITON",
                 "test_configs.autotune_choice_name_regex": "tma",
@@ -868,7 +861,6 @@ def f(x1, y1, x2, y2):
     @config.patch(
         max_autotune=True,
         max_autotune_gemm_backends="",
-        autotune_fallback_to_aten=False,
     )
     def test_no_valid_choices(self):
         a = torch.zeros([2, 2], device=GPU_TYPE)
@@ -881,7 +873,6 @@ def test_no_valid_choices(self):
     @config.patch(
         max_autotune=True,
         max_autotune_gemm_backends="TRITON",
-        autotune_fallback_to_aten=False,
     )
     def test_inf_timing(self, multi_template):
         from unittest.mock import patch
@@ -955,7 +946,6 @@ def f(x, y):
     @config.patch(
         max_autotune=True,
         max_autotune_gemm_backends="TRITON",
-        autotune_fallback_to_aten=False,
     )
     def test_max_autotune_decompose_k(self, sizes, dtype, dynamic):
         fp16_red_setting = (
@@ -1058,7 +1048,6 @@ def check_divisors(code):
     @config.patch(
         max_autotune=True,
         max_autotune_gemm_backends="TRITON",
-        autotune_fallback_to_aten=False,
     )
     def test_max_autotune_decompose_k_dynamic_input(self):
         def f(a, b):
@@ -1106,7 +1095,6 @@ def f(a, b):
     @config.patch(
         max_autotune=True,
         max_autotune_gemm_backends="TRITON",
-        autotune_fallback_to_aten=False,
     )
     def test_max_autotune_decompose_k_output_stride(self):
         def f(a, b):
diff --git a/torch/_inductor/config.py b/torch/_inductor/config.py
@@ -436,11 +436,8 @@ def prologue_fusion_enabled() -> bool:
     "TORCHINDUCTOR_MAX_AUTOTUNE_GEMM_SEARCH_SPACE", "DEFAULT"
 ).upper()  # type: ignore[assignment]
 
-# NOTE: This feature is deprecated and will be defauled to False in the future.
-# Whether we fall back to ATen or hard error when no matches are found during autotuning
-autotune_fallback_to_aten = (
-    os.environ.get("TORCHINDUCTOR_AUTOTUNE_FALLBACK_TO_ATEN", "0") == "1"
-)
+# DEPRECATED. This setting is ignored.
+autotune_fallback_to_aten = False
 
 # the value used as a fallback for the unbacked SymInts
 # that can appear in the input shapes (e.g., in autotuning)
diff --git a/torch/_inductor/kernel/bmm.py b/torch/_inductor/kernel/bmm.py
@@ -27,7 +27,6 @@
     mm_args,
     mm_config_kwargs,
     mm_options,
-    should_fallback_to_aten,
 )
 
 
@@ -233,9 +232,6 @@ def may_require_contiguous(t, meta_t):
     if use_ck_gemm_template(layout, m, n, k):
         CKGemmTemplate.add_ck_gemm_choices(choices, layout, [mat1, mat2])
 
-    if should_fallback_to_aten(choices):
-        choices.append(aten_bmm.bind((mat1, mat2), layout))
-
     return autotune_select_algorithm("bmm", choices, [mat1, mat2], layout)
 
 
diff --git a/torch/_inductor/kernel/mm.py b/torch/_inductor/kernel/mm.py
@@ -60,7 +60,6 @@
     persistent_mm_options,
     scale_mm_epilogue,
     scaled_mm_options,
-    should_fallback_to_aten,
 )
 
 
@@ -783,8 +782,6 @@ def tuned_mm(mat1, mat2, *, layout=None):
 
     for k in inductor_config.external_matmul:
         choices.append(lazy_register_extern_choice(k).bind((mat1, mat2), layout))
-    if should_fallback_to_aten(choices):
-        return aten_mm.bind((mat1, mat2), aten_layout).output_node()
 
     return autotune_select_algorithm(name, choices, [mat1, mat2], layout)
 
@@ -834,9 +831,6 @@ def tuned_int_mm(mat1, mat2, *, layout=None):
                 **mm_options(config, m, n, k, layout),
             )
 
-    if should_fallback_to_aten(choices):
-        return aten__int_mm.bind((mat1, mat2), layout).output_node()
-
     return autotune_select_algorithm("int_mm", choices, [mat1, mat2], layout)
 
 
@@ -973,30 +967,6 @@ def tuned_addmm(inp, mat1, mat2, *, alpha=1, beta=1, layout=None):
             has_bias=True,
         )
 
-    if should_fallback_to_aten(choices):
-        choices.append(
-            aten_addmm.bind(
-                (inp_expanded, mat1, mat2),
-                layout,
-                ordered_kwargs_for_cpp_kernel,
-                alpha=alpha,
-                beta=beta,
-            )
-        )
-
-        if (
-            inp_expanded.get_stride()[0] == 0
-            and inp_expanded.get_device().type == "cuda"
-            and inductor_config.triton.autotune_cublasLt
-        ):
-            # unexpand inp to make sure fused addmm from cublasLt is used
-            choices.insert(
-                0,
-                aten_bias_addmm.bind(
-                    (inp_expanded, mat1, mat2), layout, alpha=alpha, beta=beta
-                ),
-            )
-
     return autotune_select_algorithm(
         "addmm", choices, [inp_expanded, mat1, mat2], layout
     )
@@ -1198,9 +1168,6 @@ def tuned_scaled_mm(
     if is_nonzero and use_ck_gemm_template(layout, m, n, k):
         CKGemmTemplate.add_ck_gemm_choices(choices, layout, input_nodes)
 
-    if should_fallback_to_aten(choices):
-        return aten_choice.output_node()
-
     return autotune_select_algorithm("scaled_mm", choices, input_nodes, layout)
 
 
diff --git a/torch/_inductor/kernel/mm_common.py b/torch/_inductor/kernel/mm_common.py
diff --git a/torch/_inductor/quantized_lowerings.py b/torch/_inductor/quantized_lowerings.py
