From e3bb9f5a1f66a7328b7cdec25e0cd69bdb358d4e Mon Sep 17 00:00:00 2001 From: Federico Izzo Date: Sun, 9 Aug 2026 18:39:36 +0200 Subject: [PATCH 1/2] Optimize Qwen3-TTS ROCm inference on Strix Halo Expand framework HIP fast-path support for FastPackedProjection4Module and ConvTranspose1d to accept HIP as a CUDA-family backend. Add opt-in gfx1151 decoder graph reuse on rocBLAS for Strix Halo, restore retained graph inputs before execution, and add fast-path unit tests. --- .devops/nix/package.nix | 4 +- CMakeLists.txt | 58 ++++++++++ flake.nix | 11 ++ include/engine/framework/core/module.h | 6 + .../qwen3_tts/tokenizer_speech_decoder.h | 4 + src/framework/modules/conv_modules.cpp | 5 +- .../optimizations/fast_projection_modules.cpp | 4 +- .../qwen3_tts/tokenizer_speech_decoder.cpp | 107 ++++++++++++++---- .../test_conv_transpose_fast_path.cpp | 23 +++- .../test_qwen_decoder_packed_projections.cpp | 68 +++++++++++ 10 files changed, 258 insertions(+), 32 deletions(-) diff --git a/.devops/nix/package.nix b/.devops/nix/package.nix index 4374bb30..8956cba2 100644 --- a/.devops/nix/package.nix +++ b/.devops/nix/package.nix @@ -21,6 +21,7 @@ metalSupport ? stdenv.isDarwin, rocmSupport ? config.rocmSupport or false, rocmGpuTargets ? (lib.optionals rocmSupport rocmPackages.clr.gpuTargets), + strixHaloOptimizations ? (rocmSupport && rocmGpuTargets == [ "gfx1151" ]), # Model selection: if non-empty, only these model targets are built. # See CMakeLists.txt AUDIOCPP_MODEL_SET / AUDIOCPP_MODELS. models ? [ ], @@ -78,7 +79,8 @@ stdenv.mkDerivation (finalAttrs: { ++ lib.optional metalSupport "-DENGINE_ENABLE_METAL=ON" ++ lib.optional rocmSupport "-DENGINE_ENABLE_HIP=ON" ++ lib.optional rocmSupport "-DCMAKE_HIP_COMPILER=${rocmPackages.llvm.clang}/bin/clang" - ++ lib.optional rocmSupport "-DGPU_TARGETS=${lib.concatStringsSep ";" rocmGpuTargets}"; + ++ lib.optional rocmSupport "-DGPU_TARGETS=${lib.concatStringsSep ";" rocmGpuTargets}" + ++ lib.optional strixHaloOptimizations "-DENGINE_HIP_STRIX_HALO_OPTIMIZATIONS=ON"; env = lib.optionalAttrs rocmSupport { ROCM_PATH = "${rocmPackages.clr}"; diff --git a/CMakeLists.txt b/CMakeLists.txt index 664cde5c..1fd940b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,6 +84,9 @@ option(ENGINE_ENABLE_VULKAN "Build ggml with Vulkan backend support" ${ENGINE_DE option(ENGINE_ENABLE_METAL "Build ggml with Metal backend support" ${ENGINE_DEFAULT_ENABLE_METAL}) option(ENGINE_ENABLE_LLAMAFILE "Build ggml with llamafile SGEMM support" ${ENGINE_DEFAULT_ENABLE_LLAMAFILE}) option(ENGINE_ENABLE_CUDA_GRAPHS "Enable ggml CUDA graphs support" ${ENGINE_DEFAULT_ENABLE_CUDA_GRAPHS}) +option(ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS + "Enable opt-in HIP optimizations validated only for Strix Halo (gfx1151)" + OFF) option(ENGINE_ENABLE_NATIVE_CPU "Build ggml CPU kernels with native host ISA flags" ${ENGINE_DEFAULT_ENABLE_NATIVE_CPU}) option(ENGINE_ENABLE_OPENMP "Build host code with OpenMP support" ON) option(ENGINE_ENABLE_CPU_ALL_VARIANTS @@ -105,6 +108,47 @@ if (ENGINE_ENABLE_CUDA AND ENGINE_ENABLE_HIP) "configure with exactly one of them (e.g. -DENGINE_ENABLE_CUDA=OFF -DENGINE_ENABLE_HIP=ON)") endif() +set(AUDIOCPP_HIP_STRIX_HALO_OPTIMIZATIONS_ACTIVE OFF) +if (ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS) + if (NOT ENGINE_ENABLE_HIP) + message(FATAL_ERROR + "ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS=ON requires ENGINE_ENABLE_HIP=ON") + endif() + + # Match ggml HIP target precedence: an explicit CMake HIP architecture wins, + # followed by GPU_TARGETS and then the legacy AMDGPU_TARGETS spelling. + if (CMAKE_HIP_ARCHITECTURES) + set(AUDIOCPP_EFFECTIVE_HIP_TARGETS ${CMAKE_HIP_ARCHITECTURES}) + set(AUDIOCPP_EFFECTIVE_HIP_TARGET_SOURCE "CMAKE_HIP_ARCHITECTURES") + elseif (GPU_TARGETS) + set(AUDIOCPP_EFFECTIVE_HIP_TARGETS ${GPU_TARGETS}) + set(AUDIOCPP_EFFECTIVE_HIP_TARGET_SOURCE "GPU_TARGETS") + elseif (AMDGPU_TARGETS) + set(AUDIOCPP_EFFECTIVE_HIP_TARGETS ${AMDGPU_TARGETS}) + set(AUDIOCPP_EFFECTIVE_HIP_TARGET_SOURCE "AMDGPU_TARGETS") + else() + message(FATAL_ERROR + "ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS=ON requires exactly one explicit HIP target, gfx1151; " + "set -DGPU_TARGETS=gfx1151 (no project-wide gfx1151 default is provided)") + endif() + + list(LENGTH AUDIOCPP_EFFECTIVE_HIP_TARGETS AUDIOCPP_EFFECTIVE_HIP_TARGET_COUNT) + if (NOT AUDIOCPP_EFFECTIVE_HIP_TARGET_COUNT EQUAL 1 OR + NOT "${AUDIOCPP_EFFECTIVE_HIP_TARGETS}" STREQUAL "gfx1151") + message(FATAL_ERROR + "ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS=ON requires the effective HIP target list to be exactly " + "gfx1151; ${AUDIOCPP_EFFECTIVE_HIP_TARGET_SOURCE}='${AUDIOCPP_EFFECTIVE_HIP_TARGETS}'") + endif() + + # hipBLASLt regressed decoder compute by roughly 3 seconds on gfx1151. + # Keep the validated retained-graph path on rocBLAS; the generic HIP build + # remains unchanged because this option is both target- and backend-gated. + set(GGML_HIP_HIPBLASLT OFF CACHE BOOL + "Use hipBLASLt instead of hipBLAS (rocBLAS) for HIP GEMM" FORCE) + set(AUDIOCPP_HIP_STRIX_HALO_OPTIMIZATIONS_ACTIVE ON) + message(STATUS "Strix Halo HIP optimizations enabled for gfx1151 with rocBLAS") +endif() + if (ENGINE_ENABLE_CUDA AND NOT ENGINE_ENABLE_HIP) enable_language(CUDA) # 12.0 is a floor, not a target: any 12.x or 13.x works (the Dockerfile pins @@ -198,6 +242,9 @@ function(audiocpp_configure_runtime_object target_name) ${CMAKE_CURRENT_SOURCE_DIR}/external/libyaml/include ) target_compile_definitions(${target_name} PRIVATE ${AUDIOCPP_LIBYAML_COMPILE_DEFINITIONS}) + if (AUDIOCPP_HIP_STRIX_HALO_OPTIMIZATIONS_ACTIVE) + target_compile_definitions(${target_name} PRIVATE ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS=1) + endif() target_link_libraries(${target_name} PRIVATE ggml sentencepiece) if (ENGINE_ENABLE_OPENMP) target_link_libraries(${target_name} PRIVATE OpenMP::OpenMP_CXX) @@ -1166,6 +1213,9 @@ target_include_directories(engine_runtime PRIVATE target_link_libraries(engine_runtime PUBLIC ggml) target_link_libraries(engine_runtime PRIVATE sentencepiece cjson_vendor yaml_vendor) +if (AUDIOCPP_HIP_STRIX_HALO_OPTIMIZATIONS_ACTIVE) + target_compile_definitions(engine_runtime PRIVATE ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS=1) +endif() if (ENGINE_ENABLE_OPENMP) target_link_libraries(engine_runtime PRIVATE OpenMP::OpenMP_CXX) if (MSVC) @@ -1522,6 +1572,14 @@ if (ENGINE_BUILD_TESTS) COMMAND outetts_generation_budget_test ) + if (qwen3_tts IN_LIST AUDIOCPP_LINKED_MODELS) + add_engine_unittest(qwen3_tts_options_config_test tests/unittests/test_qwen3_tts_options_config.cpp) + add_test( + NAME qwen3_tts_options_config_test + COMMAND qwen3_tts_options_config_test + ) + endif() + add_engine_unittest(subtitle_formatter_test tests/unittests/test_subtitle_formatter.cpp) add_test( diff --git a/flake.nix b/flake.nix index 72c83f7f..f455e76b 100644 --- a/flake.nix +++ b/flake.nix @@ -77,6 +77,7 @@ rocm-gfx1151 = base.override { rocmSupport = true; rocmGpuTargets = [ "gfx1151" ]; + strixHaloOptimizations = true; }; } // nixpkgs.lib.optionalAttrs pkgs.${system}.stdenv.isDarwin { @@ -96,6 +97,16 @@ inputsFrom = [ self.packages.${system}.cuda ]; }; } + # ROCm's Nix toolchain is currently supported on x86_64 Linux only. + # Keep the shell off aarch64 rather than exposing an unevaluable output. + // nixpkgs.lib.optionalAttrs (system == "x86_64-linux") { + rocm = pkgs.${system}.mkShell { + inputsFrom = [ self.packages.${system}.rocm ]; + }; + rocm-gfx1151 = pkgs.${system}.mkShell { + inputsFrom = [ self.packages.${system}.rocm-gfx1151 ]; + }; + } ); }; } diff --git a/include/engine/framework/core/module.h b/include/engine/framework/core/module.h index 19949b84..147775f5 100644 --- a/include/engine/framework/core/module.h +++ b/include/engine/framework/core/module.h @@ -19,6 +19,12 @@ enum class BackendType { BestAvailable, }; +// CUDA and HIP share ggml's ggml-cuda implementation for a small set of +// explicitly verified operators. This is intentionally not a generic GPU test. +constexpr bool uses_ggml_cuda_family_backend(BackendType type) noexcept { + return type == BackendType::Cuda || type == BackendType::Hip; +} + constexpr size_t kMaxTensorRank = 4; struct TensorShape { diff --git a/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h b/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h index 3515ba6e..d98de413 100644 --- a/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h +++ b/include/engine/models/qwen3_tts/tokenizer_speech_decoder.h @@ -6,6 +6,7 @@ #include "engine/models/qwen3_tts/assets.h" #include "engine/models/qwen3_tts/types.h" +#include #include #include @@ -45,6 +46,9 @@ class Qwen3SpeechTokenizerDecoderRuntime { Qwen3TTSPerfMode perf_mode_ = Qwen3TTSPerfMode::Standard; std::unique_ptr constants_; mutable std::unique_ptr graph_; + // Always present to keep this public class layout identical when the private + // Strix Halo compile definition differs between translation units. + mutable std::array, 2> optimized_graphs_; }; } // namespace qwen3_tts diff --git a/src/framework/modules/conv_modules.cpp b/src/framework/modules/conv_modules.cpp index ef8a1b61..f52e604d 100644 --- a/src/framework/modules/conv_modules.cpp +++ b/src/framework/modules/conv_modules.cpp @@ -304,10 +304,7 @@ core::TensorValue view_batch_matrix( bool is_conv_transpose1d_col2im_fast_path_eligible( const core::ModuleBuildContext & ctx, const ConvTranspose1dConfig & config) noexcept { - return (ctx.backend_type == core::BackendType::Cuda || - ctx.backend_type == core::BackendType::Hip || - ctx.backend_type == core::BackendType::Metal) && - config.dilation == 1; + return core::uses_ggml_cuda_family_backend(ctx.backend_type) && config.dilation == 1; } Conv1dModule::Conv1dModule(Conv1dConfig config) : config_(config) { diff --git a/src/framework/modules/optimizations/fast_projection_modules.cpp b/src/framework/modules/optimizations/fast_projection_modules.cpp index d00880bd..1bdb7533 100644 --- a/src/framework/modules/optimizations/fast_projection_modules.cpp +++ b/src/framework/modules/optimizations/fast_projection_modules.cpp @@ -59,8 +59,8 @@ core::TensorValue FastPackedProjection4Module::build( if (ctx.ggml == nullptr) { throw std::runtime_error("ModuleBuildContext.ggml is null"); } - if (ctx.backend_type != core::BackendType::Cuda) { - throw std::runtime_error("FastPackedProjection4Module is CUDA-only"); + if (!core::uses_ggml_cuda_family_backend(ctx.backend_type)) { + throw std::runtime_error("FastPackedProjection4Module requires a ggml CUDA-family backend"); } core::validate_rank_between(input, 1, core::kMaxTensorRank, "input"); diff --git a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp index 4101dd28..3a1b8947 100644 --- a/src/models/qwen3_tts/tokenizer_speech_decoder.cpp +++ b/src/models/qwen3_tts/tokenizer_speech_decoder.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,12 @@ constexpr int64_t kSampleRate = 24000; constexpr int64_t kDecodeSamplesPerCode = 1920; constexpr int64_t kChunkCodes = 300; constexpr int64_t kLeftContextCodes = 25; +constexpr std::array kStrixHaloCachedChunkFrames{300, 105}; +#if defined(ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS) +constexpr bool kStrixHaloGraphCacheEnabled = true; +#else +constexpr bool kStrixHaloGraphCacheEnabled = false; +#endif constexpr float kCodebookEps = 1.0e-5F; constexpr float kMaskNegInf = -1.0e9F; @@ -1018,21 +1025,20 @@ class Qwen3SpeechTokenizerDecoderGraph { if (gallocr_ == nullptr || !ggml_gallocr_alloc_graph(gallocr_, graph_)) { throw std::runtime_error("failed to allocate Qwen3 speech decoder graph"); } - std::vector positions(static_cast(code_frames_)); + positions_data_.resize(static_cast(code_frames_)); for (int64_t i = 0; i < code_frames_; ++i) { - positions[static_cast(i)] = static_cast(i); + positions_data_[static_cast(i)] = static_cast(i); } const auto mask = make_mask(code_frames_, config.sliding_window); - ggml_backend_tensor_set(positions_, positions.data(), 0, positions.size() * sizeof(int32_t)); if (perf_mode_ == Qwen3TTSPerfMode::FlashAttention) { - std::vector mask_f16(mask.size()); + mask_f16_data_.resize(mask.size()); for (size_t index = 0; index < mask.size(); ++index) { - mask_f16[index] = ggml_fp32_to_fp16(mask[index]); + mask_f16_data_[index] = ggml_fp32_to_fp16(mask[index]); } - ggml_backend_tensor_set(mask_, mask_f16.data(), 0, mask_f16.size() * sizeof(ggml_fp16_t)); } else { - ggml_backend_tensor_set(mask_, mask.data(), 0, mask.size() * sizeof(float)); + mask_f32_data_ = mask; } + upload_static_inputs(); } ~Qwen3SpeechTokenizerDecoderGraph() { @@ -1059,6 +1065,10 @@ class Qwen3SpeechTokenizerDecoderGraph { throw std::runtime_error("Qwen3 speech decoder code count exceeds graph capacity"); } const auto upload_start = Clock::now(); + // Cached GGML graphs may reuse backend allocations whose input contents are + // not guaranteed to survive a prior execution. Restore every declared input, + // not only the request-varying codes, before replaying a retained graph. + upload_static_inputs(); std::vector tensor_codes(expected, 0); for (int64_t frame = 0; frame < input_frames; ++frame) { for (int64_t group = 0; group < weights_->config.num_quantizers; ++group) { @@ -1071,7 +1081,6 @@ class Qwen3SpeechTokenizerDecoderGraph { const auto compute_start = Clock::now(); core::set_backend_threads(backend_, compute_threads_); const ggml_status status = engine::core::compute_backend_graph(backend_, graph_); - ggml_backend_synchronize(backend_); last_graph_compute_ms_ = engine::debug::elapsed_ms(compute_start, Clock::now()); if (status != GGML_STATUS_SUCCESS) { throw std::runtime_error("Qwen3 speech decoder graph compute failed"); @@ -1096,6 +1105,27 @@ class Qwen3SpeechTokenizerDecoderGraph { } private: + void upload_static_inputs() { + ggml_backend_tensor_set( + positions_, + positions_data_.data(), + 0, + positions_data_.size() * sizeof(int32_t)); + if (perf_mode_ == Qwen3TTSPerfMode::FlashAttention) { + ggml_backend_tensor_set( + mask_, + mask_f16_data_.data(), + 0, + mask_f16_data_.size() * sizeof(ggml_fp16_t)); + } else { + ggml_backend_tensor_set( + mask_, + mask_f32_data_.data(), + 0, + mask_f32_data_.size() * sizeof(float)); + } + } + std::shared_ptr weights_; int64_t code_frames_ = 0; int64_t waveform_frames_ = 0; @@ -1106,6 +1136,9 @@ class Qwen3SpeechTokenizerDecoderGraph { ggml_tensor * positions_ = nullptr; ggml_tensor * mask_ = nullptr; ggml_tensor * output_ = nullptr; + std::vector positions_data_; + std::vector mask_f32_data_; + std::vector mask_f16_data_; ggml_cgraph * graph_ = nullptr; ggml_gallocr_t gallocr_ = nullptr; Qwen3TTSPerfMode perf_mode_ = Qwen3TTSPerfMode::Standard; @@ -1161,6 +1194,8 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode(const Qwen3Speec int64_t chunks = 0; int64_t graph_rebuilds = 0; int64_t max_chunk_frames = 0; + const bool optimized_cache_enabled = + kStrixHaloGraphCacheEnabled && execution_context_->backend_type() == core::BackendType::Hip; for (int64_t start = 0; start < codec_codes.frames; start += kChunkCodes) { const int64_t end = std::min(start + kChunkCodes, codec_codes.frames); const int64_t context = start > kLeftContextCodes ? kLeftContextCodes : start; @@ -1175,10 +1210,23 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode(const Qwen3Speec std::copy(src, src + codec_codes.code_groups, dst); } const int threads = std::max(1, execution_context_->config().threads); - if (graph_ == nullptr || !graph_->matches(*weights_, chunk_frames, execution_context_->backend(), threads)) { + auto * graph_slot = &graph_; +#if defined(ENGINE_HIP_STRIX_HALO_OPTIMIZATIONS) + if (optimized_cache_enabled) { + for (size_t index = 0; index < kStrixHaloCachedChunkFrames.size(); ++index) { + if (chunk_frames == kStrixHaloCachedChunkFrames[index]) { + graph_slot = &optimized_graphs_[index]; + break; + } + } + } +#endif + auto & graph = *graph_slot; + const bool graph_rebuilt = + graph == nullptr || !graph->matches(*weights_, chunk_frames, execution_context_->backend(), threads); + if (graph_rebuilt) { const auto build_start = Clock::now(); - graph_.reset(); - graph_ = std::make_unique( + auto replacement = std::make_unique( weights_, chunk_frames, *execution_context_, @@ -1186,12 +1234,13 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode(const Qwen3Speec graph_arena_bytes_, perf_mode_); graph_build_ms += engine::debug::elapsed_ms(build_start, Clock::now()); + graph = std::move(replacement); ++graph_rebuilds; } - auto decoded = graph_->run(chunk.data(), chunk.size()); - input_upload_ms += graph_->last_input_upload_ms(); - graph_compute_ms += graph_->last_graph_compute_ms(); - output_read_ms += graph_->last_output_read_ms(); + auto decoded = graph->run(chunk.data(), chunk.size()); + input_upload_ms += graph->last_input_upload_ms(); + graph_compute_ms += graph->last_graph_compute_ms(); + output_read_ms += graph->last_output_read_ms(); ++chunks; const int64_t drop = context * kDecodeSamplesPerCode; if (drop > static_cast(decoded.size())) { @@ -1221,22 +1270,34 @@ runtime::AudioBuffer Qwen3SpeechTokenizerDecoderRuntime::decode_and_trim_referen if (reference_codes.code_groups != generated_codes.code_groups) { throw std::runtime_error("Qwen3 speech decoder reference/generated code group mismatch"); } + if (reference_codes.frames < 0 || generated_codes.frames < 0 || reference_codes.code_groups <= 0) { + throw std::runtime_error("Qwen3 speech decoder reference/generated code shape is invalid"); + } + if (reference_codes.frames > std::numeric_limits::max() - generated_codes.frames) { + throw std::runtime_error("Qwen3 speech decoder combined frame count is too large"); + } Qwen3SpeechCodes combined; combined.frames = reference_codes.frames + generated_codes.frames; combined.code_groups = reference_codes.code_groups; - combined.codes.reserve(static_cast(combined.frames * combined.code_groups)); + if (combined.frames > std::numeric_limits::max() / combined.code_groups) { + throw std::runtime_error("Qwen3 speech decoder combined code count is too large"); + } + const int64_t combined_code_count = combined.frames * combined.code_groups; + if (static_cast(combined_code_count) > std::numeric_limits::max()) { + throw std::runtime_error("Qwen3 speech decoder combined code count exceeds host size limits"); + } + combined.codes.reserve(static_cast(combined_code_count)); combined.codes.insert(combined.codes.end(), reference_codes.codes.begin(), reference_codes.codes.end()); combined.codes.insert(combined.codes.end(), generated_codes.codes.begin(), generated_codes.codes.end()); auto audio = decode(combined); - const int64_t cut = combined.frames > 0 - ? static_cast( - static_cast(reference_codes.frames) / static_cast(combined.frames) * - static_cast(audio.samples.size())) - : 0; - if (cut < 0 || cut > static_cast(audio.samples.size())) { + if (reference_codes.frames > std::numeric_limits::max() / kDecodeSamplesPerCode) { + throw std::runtime_error("Qwen3 speech decoder reference sample count is too large"); + } + const int64_t cut = reference_codes.frames * kDecodeSamplesPerCode; + if (static_cast(cut) > audio.samples.size()) { throw std::runtime_error("Qwen3 speech decoder reference trim is out of range"); } - audio.samples.erase(audio.samples.begin(), audio.samples.begin() + cut); + audio.samples.erase(audio.samples.begin(), audio.samples.begin() + static_cast(cut)); return audio; } diff --git a/tests/unittests/test_conv_transpose_fast_path.cpp b/tests/unittests/test_conv_transpose_fast_path.cpp index 683f182d..d3f4c94b 100644 --- a/tests/unittests/test_conv_transpose_fast_path.cpp +++ b/tests/unittests/test_conv_transpose_fast_path.cpp @@ -337,6 +337,22 @@ RunResult run_conv_transpose_ab_case( return runner.run_f32(output); } +void require_cuda_family_backend_classification() { + static_assert(engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Cuda)); + static_assert(engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Hip)); + static_assert(!engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Cpu)); + static_assert(!engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Vulkan)); + static_assert(!engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Metal)); + + const ConvTransposeCase test_case{"hip_trigger_condition_probe", 1, 256, 128, 96, 10, 5, true}; + const auto eligible_config = make_config(test_case); + engine::core::ModuleBuildContext hip_context{}; + hip_context.backend_type = engine::core::BackendType::Hip; + if (!engine::modules::is_conv_transpose1d_col2im_fast_path_eligible(hip_context, eligible_config)) { + throw std::runtime_error("expected HIP conv-transpose config to be col2im fast-path eligible"); + } +} + void require_fast_path_trigger_conditions(engine::core::BackendType backend_type) { BackendModuleRunner cuda_runner("conv_transpose_fast_path_test.cuda_trigger", backend_type); BackendModuleRunner cpu_runner("conv_transpose_fast_path_test.cpu_trigger", engine::core::BackendType::Cpu); @@ -459,10 +475,13 @@ int main() { try { const ConvTransposeCase qwen3_case{"qwen3_decoder_mid_block_biased", 1, 256, 128, 96, 10, 5, true}; const ConvTransposeCase batched_case{"batched_decoder_block_no_bias", 2, 192, 192, 48, 2, 2, false}; + require_cuda_family_backend_classification(); - constexpr auto kBackend = engine::core::BackendType::Cuda; + const auto kBackend = backend_is_available(engine::core::BackendType::Hip) + ? engine::core::BackendType::Hip + : engine::core::BackendType::Cuda; if (!backend_is_available(kBackend)) { - std::cout << "[SKIP] CUDA backend is not available for conv transpose fast-path parity test\n"; + std::cout << "[SKIP] CUDA/HIP backend is not available for conv transpose fast-path parity test\n"; } else { require_fast_path_trigger_conditions(kBackend); run_case(qwen3_case, kBackend); diff --git a/tests/unittests/test_qwen_decoder_packed_projections.cpp b/tests/unittests/test_qwen_decoder_packed_projections.cpp index 494590de..6ed90267 100644 --- a/tests/unittests/test_qwen_decoder_packed_projections.cpp +++ b/tests/unittests/test_qwen_decoder_packed_projections.cpp @@ -2,6 +2,7 @@ #include "engine/framework/modules/transformers/qwen_causal_decoder.h" #include "engine/framework/modules/transformers/qwen_decoder.h" #include "engine/framework/modules/optimizations/fast_kv_modules.h" +#include "engine/framework/modules/optimizations/fast_projection_modules.h" #include #include @@ -398,6 +399,72 @@ std::string graph_ops(ggml_cgraph * graph) { return out.str(); } +void test_fast_projection_accepts_cuda_family_backends() { + constexpr int64_t in_features = 8; + constexpr int64_t out_features = 12; + for (const auto backend_type : { + engine::core::BackendType::Cuda, + engine::core::BackendType::Hip, + }) { + ggml_init_params params{kGraphBytes, nullptr, true}; + ggml_context * ggml = ggml_init(params); + if (ggml == nullptr) { + throw std::runtime_error("failed to initialize fast projection graph test context"); + } + try { + engine::core::ModuleBuildContext ctx{ggml, "fast_projection_cuda_family_test", backend_type}; + const auto input = engine::core::make_tensor( + ctx, + GGML_TYPE_F32, + engine::core::TensorShape::from_dims({1, in_features})); + const auto weight = engine::core::make_tensor( + ctx, + GGML_TYPE_F32, + engine::core::TensorShape::from_dims({out_features, in_features})); + const auto output = engine::modules::FastPackedProjection4Module({in_features, out_features}) + .build(ctx, input, {weight, std::nullopt}); + if (output.shape.rank != 2 || output.shape.dims[0] != 1 || output.shape.dims[1] != out_features) { + throw std::runtime_error("fast projection CUDA-family output shape mismatch"); + } + ggml_free(ggml); + } catch (...) { + ggml_free(ggml); + throw; + } + } + + ggml_init_params params{kGraphBytes, nullptr, true}; + ggml_context * ggml = ggml_init(params); + if (ggml == nullptr) { + throw std::runtime_error("failed to initialize fast projection rejection test context"); + } + bool rejected = false; + try { + engine::core::ModuleBuildContext ctx{ggml, "fast_projection_cpu_rejection_test", engine::core::BackendType::Cpu}; + const auto input = engine::core::make_tensor( + ctx, + GGML_TYPE_F32, + engine::core::TensorShape::from_dims({1, in_features})); + const auto weight = engine::core::make_tensor( + ctx, + GGML_TYPE_F32, + engine::core::TensorShape::from_dims({out_features, in_features})); + try { + (void) engine::modules::FastPackedProjection4Module({in_features, out_features}) + .build(ctx, input, {weight, std::nullopt}); + } catch (const std::runtime_error &) { + rejected = true; + } + ggml_free(ggml); + } catch (...) { + ggml_free(ggml); + throw; + } + if (!rejected) { + throw std::runtime_error("fast projection must reject non-CUDA-family backends"); + } +} + void test_higgs_decode_graph_exposes_cuda_fast_paths() { constexpr int64_t hidden = 8; constexpr int64_t heads = 2; @@ -504,6 +571,7 @@ int main() { test_suffix_causal_mask(); test_f16_kv_set_rows(); test_f16_kv_set_rows_batched(); + test_fast_projection_accepts_cuda_family_backends(); test_higgs_decode_graph_exposes_cuda_fast_paths(); std::cout << "qwen_decoder_packed_projection_test: ok\n"; return 0; From 2eeaeec6421d1795b8f0bb77a1256a5b8225a0ad Mon Sep 17 00:00:00 2001 From: Federico Izzo Date: Tue, 11 Aug 2026 18:05:43 +0200 Subject: [PATCH 2/2] Address reviewer feedback for ROCm fast paths and Metal eligibility - Fix CMakeLists.txt build failure by removing dangling reference to test_qwen3_tts_options_config.cpp. - Restore Metal eligibility in is_conv_transpose1d_col2im_fast_path_eligible while keeping FastPackedProjection4Module restricted to CUDA/HIP. - Rename uses_ggml_cuda_family_backend to uses_ggml_cuda_or_hip_backend for explicit naming clarity. --- CMakeLists.txt | 7 -- include/engine/framework/core/module.h | 2 +- src/framework/modules/conv_modules.cpp | 4 +- .../optimizations/fast_projection_modules.cpp | 4 +- .../test_conv_transpose_fast_path.cpp | 24 +++---- .../test_qwen_decoder_packed_projections.cpp | 64 ++++++++++--------- 6 files changed, 52 insertions(+), 53 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fd940b3..d257c8ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1572,13 +1572,6 @@ if (ENGINE_BUILD_TESTS) COMMAND outetts_generation_budget_test ) - if (qwen3_tts IN_LIST AUDIOCPP_LINKED_MODELS) - add_engine_unittest(qwen3_tts_options_config_test tests/unittests/test_qwen3_tts_options_config.cpp) - add_test( - NAME qwen3_tts_options_config_test - COMMAND qwen3_tts_options_config_test - ) - endif() add_engine_unittest(subtitle_formatter_test tests/unittests/test_subtitle_formatter.cpp) diff --git a/include/engine/framework/core/module.h b/include/engine/framework/core/module.h index 147775f5..3454c70b 100644 --- a/include/engine/framework/core/module.h +++ b/include/engine/framework/core/module.h @@ -21,7 +21,7 @@ enum class BackendType { // CUDA and HIP share ggml's ggml-cuda implementation for a small set of // explicitly verified operators. This is intentionally not a generic GPU test. -constexpr bool uses_ggml_cuda_family_backend(BackendType type) noexcept { +constexpr bool uses_ggml_cuda_or_hip_backend(BackendType type) noexcept { return type == BackendType::Cuda || type == BackendType::Hip; } diff --git a/src/framework/modules/conv_modules.cpp b/src/framework/modules/conv_modules.cpp index f52e604d..d44a23fe 100644 --- a/src/framework/modules/conv_modules.cpp +++ b/src/framework/modules/conv_modules.cpp @@ -304,7 +304,9 @@ core::TensorValue view_batch_matrix( bool is_conv_transpose1d_col2im_fast_path_eligible( const core::ModuleBuildContext & ctx, const ConvTranspose1dConfig & config) noexcept { - return core::uses_ggml_cuda_family_backend(ctx.backend_type) && config.dilation == 1; + return (core::uses_ggml_cuda_or_hip_backend(ctx.backend_type) || + ctx.backend_type == core::BackendType::Metal) && + config.dilation == 1; } Conv1dModule::Conv1dModule(Conv1dConfig config) : config_(config) { diff --git a/src/framework/modules/optimizations/fast_projection_modules.cpp b/src/framework/modules/optimizations/fast_projection_modules.cpp index 1bdb7533..d7b92a41 100644 --- a/src/framework/modules/optimizations/fast_projection_modules.cpp +++ b/src/framework/modules/optimizations/fast_projection_modules.cpp @@ -59,8 +59,8 @@ core::TensorValue FastPackedProjection4Module::build( if (ctx.ggml == nullptr) { throw std::runtime_error("ModuleBuildContext.ggml is null"); } - if (!core::uses_ggml_cuda_family_backend(ctx.backend_type)) { - throw std::runtime_error("FastPackedProjection4Module requires a ggml CUDA-family backend"); + if (!core::uses_ggml_cuda_or_hip_backend(ctx.backend_type)) { + throw std::runtime_error("FastPackedProjection4Module requires CUDA or HIP backend"); } core::validate_rank_between(input, 1, core::kMaxTensorRank, "input"); diff --git a/tests/unittests/test_conv_transpose_fast_path.cpp b/tests/unittests/test_conv_transpose_fast_path.cpp index d3f4c94b..bd5455f8 100644 --- a/tests/unittests/test_conv_transpose_fast_path.cpp +++ b/tests/unittests/test_conv_transpose_fast_path.cpp @@ -337,19 +337,21 @@ RunResult run_conv_transpose_ab_case( return runner.run_f32(output); } -void require_cuda_family_backend_classification() { - static_assert(engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Cuda)); - static_assert(engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Hip)); - static_assert(!engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Cpu)); - static_assert(!engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Vulkan)); - static_assert(!engine::core::uses_ggml_cuda_family_backend(engine::core::BackendType::Metal)); +void require_cuda_or_hip_backend_classification() { + static_assert(engine::core::uses_ggml_cuda_or_hip_backend(engine::core::BackendType::Cuda)); + static_assert(engine::core::uses_ggml_cuda_or_hip_backend(engine::core::BackendType::Hip)); + static_assert(!engine::core::uses_ggml_cuda_or_hip_backend(engine::core::BackendType::Cpu)); + static_assert(!engine::core::uses_ggml_cuda_or_hip_backend(engine::core::BackendType::Vulkan)); + static_assert(!engine::core::uses_ggml_cuda_or_hip_backend(engine::core::BackendType::Metal)); const ConvTransposeCase test_case{"hip_trigger_condition_probe", 1, 256, 128, 96, 10, 5, true}; const auto eligible_config = make_config(test_case); - engine::core::ModuleBuildContext hip_context{}; - hip_context.backend_type = engine::core::BackendType::Hip; - if (!engine::modules::is_conv_transpose1d_col2im_fast_path_eligible(hip_context, eligible_config)) { - throw std::runtime_error("expected HIP conv-transpose config to be col2im fast-path eligible"); + for (const auto backend : {engine::core::BackendType::Hip, engine::core::BackendType::Metal}) { + engine::core::ModuleBuildContext ctx{}; + ctx.backend_type = backend; + if (!engine::modules::is_conv_transpose1d_col2im_fast_path_eligible(ctx, eligible_config)) { + throw std::runtime_error("expected HIP and Metal conv-transpose config to be col2im fast-path eligible"); + } } } @@ -475,7 +477,7 @@ int main() { try { const ConvTransposeCase qwen3_case{"qwen3_decoder_mid_block_biased", 1, 256, 128, 96, 10, 5, true}; const ConvTransposeCase batched_case{"batched_decoder_block_no_bias", 2, 192, 192, 48, 2, 2, false}; - require_cuda_family_backend_classification(); + require_cuda_or_hip_backend_classification(); const auto kBackend = backend_is_available(engine::core::BackendType::Hip) ? engine::core::BackendType::Hip diff --git a/tests/unittests/test_qwen_decoder_packed_projections.cpp b/tests/unittests/test_qwen_decoder_packed_projections.cpp index 6ed90267..981063d4 100644 --- a/tests/unittests/test_qwen_decoder_packed_projections.cpp +++ b/tests/unittests/test_qwen_decoder_packed_projections.cpp @@ -399,7 +399,7 @@ std::string graph_ops(ggml_cgraph * graph) { return out.str(); } -void test_fast_projection_accepts_cuda_family_backends() { +void test_fast_projection_accepts_cuda_or_hip_backends() { constexpr int64_t in_features = 8; constexpr int64_t out_features = 12; for (const auto backend_type : { @@ -412,7 +412,7 @@ void test_fast_projection_accepts_cuda_family_backends() { throw std::runtime_error("failed to initialize fast projection graph test context"); } try { - engine::core::ModuleBuildContext ctx{ggml, "fast_projection_cuda_family_test", backend_type}; + engine::core::ModuleBuildContext ctx{ggml, "fast_projection_cuda_or_hip_test", backend_type}; const auto input = engine::core::make_tensor( ctx, GGML_TYPE_F32, @@ -424,7 +424,7 @@ void test_fast_projection_accepts_cuda_family_backends() { const auto output = engine::modules::FastPackedProjection4Module({in_features, out_features}) .build(ctx, input, {weight, std::nullopt}); if (output.shape.rank != 2 || output.shape.dims[0] != 1 || output.shape.dims[1] != out_features) { - throw std::runtime_error("fast projection CUDA-family output shape mismatch"); + throw std::runtime_error("fast projection CUDA/HIP output shape mismatch"); } ggml_free(ggml); } catch (...) { @@ -433,35 +433,37 @@ void test_fast_projection_accepts_cuda_family_backends() { } } - ggml_init_params params{kGraphBytes, nullptr, true}; - ggml_context * ggml = ggml_init(params); - if (ggml == nullptr) { - throw std::runtime_error("failed to initialize fast projection rejection test context"); - } - bool rejected = false; - try { - engine::core::ModuleBuildContext ctx{ggml, "fast_projection_cpu_rejection_test", engine::core::BackendType::Cpu}; - const auto input = engine::core::make_tensor( - ctx, - GGML_TYPE_F32, - engine::core::TensorShape::from_dims({1, in_features})); - const auto weight = engine::core::make_tensor( - ctx, - GGML_TYPE_F32, - engine::core::TensorShape::from_dims({out_features, in_features})); + for (const auto rejected_backend : {engine::core::BackendType::Cpu, engine::core::BackendType::Metal}) { + ggml_init_params params{kGraphBytes, nullptr, true}; + ggml_context * ggml = ggml_init(params); + if (ggml == nullptr) { + throw std::runtime_error("failed to initialize fast projection rejection test context"); + } + bool rejected = false; try { - (void) engine::modules::FastPackedProjection4Module({in_features, out_features}) - .build(ctx, input, {weight, std::nullopt}); - } catch (const std::runtime_error &) { - rejected = true; + engine::core::ModuleBuildContext ctx{ggml, "fast_projection_rejection_test", rejected_backend}; + const auto input = engine::core::make_tensor( + ctx, + GGML_TYPE_F32, + engine::core::TensorShape::from_dims({1, in_features})); + const auto weight = engine::core::make_tensor( + ctx, + GGML_TYPE_F32, + engine::core::TensorShape::from_dims({out_features, in_features})); + try { + (void) engine::modules::FastPackedProjection4Module({in_features, out_features}) + .build(ctx, input, {weight, std::nullopt}); + } catch (const std::runtime_error &) { + rejected = true; + } + ggml_free(ggml); + } catch (...) { + ggml_free(ggml); + throw; + } + if (!rejected) { + throw std::runtime_error("fast projection must reject non-CUDA/HIP backends"); } - ggml_free(ggml); - } catch (...) { - ggml_free(ggml); - throw; - } - if (!rejected) { - throw std::runtime_error("fast projection must reject non-CUDA-family backends"); } } @@ -571,7 +573,7 @@ int main() { test_suffix_causal_mask(); test_f16_kv_set_rows(); test_f16_kv_set_rows_batched(); - test_fast_projection_accepts_cuda_family_backends(); + test_fast_projection_accepts_cuda_or_hip_backends(); test_higgs_decode_graph_exposes_cuda_fast_paths(); std::cout << "qwen_decoder_packed_projection_test: ok\n"; return 0;