xehpc: jit: gemm: remove C transposition from TT f16/bf16 kernels

petercad · petercad · commit e45e1aa4fe44 · 2025-05-16T08:15:25.000-07:00
diff --git a/src/gpu/intel/jit/gemm/selector/db/kernel.db b/src/gpu/intel/jit/gemm/selector/db/kernel.db
@@ -535,9 +535,9 @@ auto _CATALOG_ = kcatalog::toArray({
 {{'F', "gemm", {"B", "B", "S"}, {"T", "N", "N"}}, {-1, -1, {2048, -1, 2048}, {-1, 16, 5799}, {-1, -1, -1}, {-1, 16, -1}, {8, 4, 1}, "IAB"}, "am32+C32@64 at32 aS wg 1x1x16 ikr af vav sr sb256 bk0 bm0 sys rr", {16, (LoopType) 255, 128, {(LoopType) 0, (LoopType) 1, (LoopType) 2}, {16777216, 8192, 16777216}, {8192, 8192, 16777216}, {4, 16, 32}, {1, 1, 16}, 1, (WGType) 1, 4357, 0, 256, {8, 4, 4}, {true, true, true}}, {'E', 17, {1.12642e+06, 163000, 18949.6, 4807.2, 0, 0, 0.620253, 0.594119, 4.7783, 20.2378, 0.099453, 0.164053, 0.0490596, 1, 1.10688, 0.999502, 3.75419e-13}}},
 {{'F', "gemm", {"B", "B", "S"}, {"T", "N", "N"}}, {-1, -1, {1536, -1, 5800}, {-1, 16, -1}, {-1, -1, -1}, {-1, 16, -1}, {8, 4, 1}, "IAB"}, "am32+m64@64 at32+m16@64 aS wg 1x1x16 kr af vav sr sb64 bm0 bk0 sm sn grf256 sys ikr", {16, (LoopType) 255, 256, {(LoopType) 0, (LoopType) 1, (LoopType) 2}, {16777216, 8192, 16777216}, {8192, 8192, 16777216}, {4, 16, 64}, {1, 1, 16}, 1, (WGType) 1, 4357, 0, 256, {8, 4, 4}, {true, true, true}}, {'E', 17, {981931, 255034, 13270.4, 8465.73, 0, 0, 0.839094, 0.47734, 12.6109, 43.9669, 0.0831258, 0.0199484, 0.123368, 0.98723, 1.28194, 0.802064, 5.50004e-12}}},
 {{'F', "gemm", {"B", "B", "S"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "i"}, "aS16+m16@12 aS8+m16@12 aB wg 8x4 kc8 nse hi pt sr br sb32 bk0 sm sn grf256 kv afb l4", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {524288, 1048576, 16777216}, {524288, 1048576, 32}, {32, 64, 16}, {8, 4, 1}, 1, (WGType) 1, 441, 0, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {874480, 814710, 0, 0, 3.14573e+06, 1.04694e+07, 1.59572, 1.73308, 0.796063, 1.18049, 0.0626559, 0.0626559, 0, 1, 1.00457, 1.00258, -3.94402e-14}}},
-{{'F', "gemm", {"B", "B", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABqI"}, "am32+S1,32@64 av32+B32@64 aS cs sys grf256 af wg 8x4 bo sb256 sm bk0 sr", {16, (LoopType) 255, 256, {(LoopType) 128, (LoopType) 255, (LoopType) 255}, {524288, 1048576, 16777216}, {524288, 1048576, 16777216}, {32, 64, 32}, {8, 4, 1}, 1, (WGType) 1, 257, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {705599, 1.02394e+06, 0, 0, 0, 0, 0.719208, 0.662079, 1.08787, 2.05052, 0.00435156, 0.00435156, 0, 0.998842, 1.73144, 1.10326, 2.79532e-12}}},
 {{'F', "gemm", {"B", "B", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "Is"}, "aS16 aS16 aB sys grf256 cab2 wg 4x4 l4 sr", {16, (LoopType) 255, 256, {(LoopType) 0, (LoopType) 1, (LoopType) 255}, {8192, 8192, 2048}, {8192, 8192, 2048}, {48, 32, 16}, {4, 4, 1}, 1, (WGType) 1, 257, 20480, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {1.2896e+06, 327092, 0, 0, 0, 0, 1.64753, 1.69722, 1.01172, 1.48812, 0.0145767, 0.000763122, 0.0157325, 0.871871, 1.01157, 1.00431, 1.22757e-13}}},
 {{'F', "gemm", {"B", "B", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "i"}, "aS8x2+m16@20 aB8+m16@20 aS wg 8x4 kc8 nse hi pt sr sb32 bk0 sm sn grf256 kv afb l4", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {262144, 1048576, 16777216}, {262144, 1048576, 32}, {16, 64, 8}, {8, 4, 1}, 1, (WGType) 1, 441, 0, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {888082, 626619, 0, 0, 1.5319e+07, 2.15941e+07, 3.12178, 1.38894, 4.6861, 5.03272, 0.0626843, 0.0626843, 0, 1, 1.00285, 1.00112, -2.5704e-15}}},
+{{'F', "gemm", {"B", "B", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "at16x2+m32@48 at16+m32@48 aB wg 4x8 af vav hi pt ar sb64 sm grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {1048576, 524288, 16777216}, {1048576, 524288, 16777216}, {64, 32, 16}, {4, 8, 1}, 1, (WGType) 1, 257, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {887529, 538289, 0, 0, 0, 0, 1.07403, 1.17992, 0.847137, 1.27305, 0.00495505, 0.00495505, 0, 0.897026, 1.67662, 1.14552, 3.20439e-12}}},
 {{'F', "gemm", {"B", "F", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, 16, -1}, {1, 1, 1}, "#I"}, "aB16+m32@32 aB32 aB wg 2x8 af vav li nmk pt sr br ca3 bk0 sys kv dm afb l4", {16, (LoopType) 255, 128, {(LoopType) 225, (LoopType) 255, (LoopType) 255}, {1048576, 32768, 16777216}, {1048576, 32768, 32}, {64, 2, 32}, {2, 8, 1}, 1, (WGType) 1, 409, 24576, 0, {4, 2, 4}, {true, true, true}}, {'E', 17, {1.32162e+06, 161954, 0, 0, 2.32817e+06, 0, 0.71806, 4.15517, 0.786689, 1.40778, 0.0341164, 0.0131941, 0.0256486, 0.947188, 1.39057, 0.987284, 5.0128e-12}}},
 {{'F', "gemm", {"B", "F", "S"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {4, 8, 1}, "ABI"}, "at16x2+m32@48 am32+m16@64 aB wg 4x2x4 kr xaf st vav hi pt sr br sb64 bk0 sm sn grf256 sys kv afb", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 2}, {262144, 262144, 16777216}, {262144, 262144, 32}, {16, 16, 32}, {4, 2, 4}, 1, (WGType) 1, 445, 0, 8192, {4, 8, 4}, {true, true, true}}, {'E', 17, {1.15757e+06, -134939, -26216.3, 226647, 2.94093e+06, 1.99475e+06, 0.459989, 0.701382, 0.869118, 1.51365, 0.0198203, 0.0153374, 0.00626234, 0.558687, 1.30133, 0.935668, 6.0334e-12}}},
 {{'F', "gemm", {"B", "F", "S"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "I"}, "aS32+S32@48 aB32+S16@48 aB wg 4x1 af vav nmk li pt br sr sb256 bk0 sm grf256 sys l4 kd", {16, (LoopType) 255, 256, {(LoopType) 225, (LoopType) 255, (LoopType) 255}, {262144, 16384, 16777216}, {262144, 16384, 16777216}, {16, 1, 32}, {4, 1, 1}, 1, (WGType) 1, 257, 0, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {1.20164e+06, 54541, 0, 0, 0, 0, 0.422505, 7.31873, 4.22063, 10.9759, 0.192755, 0.0407408, 0.156727, 0.693649, 1.00294, 0, 0}}},
@@ -745,8 +745,8 @@ auto _CATALOG_ = kcatalog::toArray({
 {{'F', "gemm", {"H", "H", "S"}, {"T", "N", "N"}}, {-1, -1, {1536, -1, 5800}, {-1, 16, -1}, {-1, -1, -1}, {-1, 16, -1}, {8, 4, 1}, "IAB"}, "am32+m64@64 at32+m16@64 aS wg 1x1x16 kr af vav sr sb64 bm0 bk0 sm sn grf256 sys ikr", {16, (LoopType) 255, 256, {(LoopType) 0, (LoopType) 1, (LoopType) 2}, {16777216, 8192, 16777216}, {8192, 8192, 16777216}, {4, 16, 64}, {1, 1, 16}, 1, (WGType) 1, 4357, 0, 256, {8, 4, 4}, {true, true, true}}, {'E', 17, {981931, 255034, 13270.4, 8465.73, 0, 0, 0.839094, 0.47734, 12.6109, 43.9669, 0.0831258, 0.0199484, 0.123368, 0.98723, 1.28194, 0.802064, 5.50004e-12}}},
 {{'F', "gemm", {"H", "H", "S"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "i"}, "aS16+m16@12 aS8+m16@12 aB wg 8x4 kc8 nse hi pt sb32 bk0 sm sn grf256 kv afb l4 sr", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {524288, 1048576, 16777216}, {524288, 1048576, 32}, {32, 64, 16}, {8, 4, 1}, 1, (WGType) 1, 441, 0, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {878230, 813623, 0, 0, 3.31776e+06, 9.39622e+06, 1.5936, 1.75404, 0.798399, 1.185, 0.0656902, 0.0656902, 0, 1, 1.00344, 1.00135, -9.21479e-15}}},
 {{'F', "gemm", {"H", "H", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "Is"}, "aS16 aS16 aB sys grf256 cab2 wg 4x4 l4 sr", {16, (LoopType) 255, 256, {(LoopType) 0, (LoopType) 1, (LoopType) 255}, {8192, 8192, 2048}, {8192, 8192, 2048}, {48, 32, 16}, {4, 4, 1}, 1, (WGType) 1, 257, 20480, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {1.17895e+06, 343529, 0, 0, 0, 0, 1.63411, 1.77325, 1.00531, 1.48275, 0.0145617, 0.000936039, 0.0155971, 0.877282, 1.01034, 1.0048, 9.67486e-14}}},
-{{'F', "gemm", {"H", "H", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABqI"}, "am32+S1,32@64 av32+B32@64 aS cs sys grf256 af wg 8x4 bo sb256 sm bk0 sr", {16, (LoopType) 255, 256, {(LoopType) 128, (LoopType) 255, (LoopType) 255}, {524288, 1048576, 16777216}, {524288, 1048576, 16777216}, {32, 64, 32}, {8, 4, 1}, 1, (WGType) 1, 257, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {765291, 1.00234e+06, 0, 0, 0, 0, 0.723688, 0.663141, 1.08538, 2.05438, 0.00434664, 0.00434664, 0, 1, 1.8693, 1.21785, 3.96104e-12}}},
 {{'F', "gemm", {"H", "H", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "i"}, "aS8x2+m16@20 aB8+m16@20 aS wg 8x4 kc8 nse hi pt sb32 bk0 sm sn grf256 kv afb l4 sr", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {262144, 1048576, 16777216}, {262144, 1048576, 32}, {16, 32, 8}, {8, 4, 1}, 1, (WGType) 1, 441, 0, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {971283, 550381, 0, 0, 1.25911e+07, 8.13466e+06, 2.33425, 1.5484, 4.45361, 4.80392, 0.0689395, 0.0689395, 0, 1, 1.00725, 0.817916, 1.37423e-12}}},
+{{'F', "gemm", {"H", "H", "S"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "at16x2+m32@48 at16+m32@48 aB wg 4x8 af vav hi pt ar sb64 sm grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {1048576, 524288, 16777216}, {1048576, 524288, 16777216}, {64, 32, 16}, {4, 8, 1}, 1, (WGType) 1, 257, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {887529, 538289, 0, 0, 0, 0, 1.07403, 1.17992, 0.847137, 1.27305, 0.00495505, 0.00495505, 0, 0.897026, 1.67662, 1.14552, 3.20439e-12}}},
 {{'F', "gemm", {"H", "O", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, 8, -1}, {8, 8, 1}, "ABI"}, "av16+B16@48 am16x2 aB wg 8x2 af vav li sr br sb256 bk0 sys ska rr wx4 kv afb pt", {16, (LoopType) 255, 128, {(LoopType) 224, (LoopType) 255, (LoopType) 255}, {1048576, 65536, 16777216}, {1048576, 65536, 32}, {64, 4, 16}, {8, 2, 1}, 4, (WGType) 1, 441, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {879483, 350341, 0, 0, 0, 0, 0.458466, 18.3391, 0.732099, 2.00944, 0.0538243, 0.0538243, 0, 0.0465505, 1.08817, 1.00259, 3.38512e-15}}},
 {{'F', "gemm", {"H", "O", "S"}, {"N", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "#I"}, "aB16 at16 aB wg 1x2x16 kr cab3x2 ks16 af vav hi pt sr br bk0 grf256 kv afb sys sn l4 l2d", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 2}, {524288, 262144, 16777216}, {524288, 262144, 32}, {32, 16, 16}, {1, 2, 16}, 1, (WGType) 1, 445, 6144, 6144, {2, 2, 4}, {true, true, true}}, {'E', 17, {1.43314e+06, -85436.7, -21480.5, 122461, 4.34176e+06, 2.37568e+06, 0.844572, 0.876066, 0.58704, 1.33341, 0.0317939, 0.0288168, 0.00938048, 0.965671, 1.25852, 0.881513, 1.24288e-11}}},
 {{'F', "gemm", {"H", "O", "S"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {4, 1, 2}, "ABI"}, "at16+m32@64 aB16 aB wg 16x2 cb3x2 ks32 af vav hi pt sr br bk0 sm sn grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {262144, 1048576, 16777216}, {262144, 1048576, 16777216}, {16, 64, 32}, {16, 2, 1}, 1, (WGType) 1, 257, 24576, 0, {4, 8, 4}, {true, true, true}}, {'E', 17, {1.01114e+06, 521171, 0, 0, 0, 0, 0.680408, 1.20311, 1.01139, 1.54855, 0.00724061, 0.00724061, 0, 0.952894, 1.46407, 1.15995, 1.81761e-12}}},
