From 3d6bee9fab2ea88acb892fa77ad232f1bc57cdb4 Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Thu, 13 Aug 2026 13:04:18 +1200 Subject: [PATCH 1/6] [HLSL] Add MatVec interpretation and bias coverage for LinAlg Exercise non-uniform F16 row-major and column-major layouts, packed SInt8 and UInt8 interpreted inputs, unsigned UInt32 output, and a separate non-uniform bias resource. High-bit UInt8 lanes distinguish unsigned from signed decoding of the same bytes. Derive every expected result with overflow-checked host dot products plus optional bias, encode packed lanes least-significant-byte first, and compare the complete poisoned output including padding and guard bytes. Query the exact vector, matrix, bias, and result capability tuple, requiring both mandatory F16 layouts and gating only optional output cases. Column-major is required rather than capability gated because a thread scope matrix load permits row-major, column-major and optimal layouts, and only transposed loads are implementation specific and need a driver query. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 829 ++++++++++++++++++ 1 file changed, 829 insertions(+) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index f98022ec88..a1ab58c7c9 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -2322,8 +2322,14 @@ class DxilConf_SM610_LinAlg { // Matrix Vector Arithmetic TEST_METHOD(MatVecMul_Thread_16x16_F16); TEST_METHOD(MatVecMul_Thread_4x8_F32); + TEST_METHOD(MatVecMul_Thread_4x8_F16_NonUniform); + TEST_METHOD(MatVecMul_Thread_4x8_F16_ColumnMajor); + TEST_METHOD(MatVecMul_Thread_4x8_I8_Interpreted); + TEST_METHOD(MatVecMul_Thread_4x8_U8_Interpreted); + TEST_METHOD(MatVecMul_Thread_4x8_U32_UnsignedOutput); TEST_METHOD(MatVecMulAdd_Thread_16x16_F16); TEST_METHOD(MatVecMulAdd_Thread_4x8_F32); + TEST_METHOD(MatVecMulAdd_Thread_4x8_F16_IndependentBias); TEST_METHOD(OuterProduct_Thread_16x16_F16); // Query Accumulator Layout @@ -5272,6 +5278,829 @@ void DxilConf_SM610_LinAlg::VectorAccumulateDescriptor_Thread_F16() { runVectorAccumulateDescriptor(D3DDevice, DxcSupport, VerboseLogging); } +namespace matvec_interpretation { + +static constexpr size_t OutputGuardBytes = 16; + +struct CaseData { + ComponentType MatrixType = ComponentType::Invalid; + MatrixDim M = 0; + MatrixDim N = 0; + MatrixLayout Layout = MatrixLayout::RowMajor; + ComponentType VectorInputType = ComponentType::Invalid; + ComponentType InputInterpretation = ComponentType::Invalid; + ComponentType BiasInputType = ComponentType::Invalid; + ComponentType ResultType = ComponentType::Invalid; + bool OutputSigned = true; + std::vector MatrixValues; + std::vector InterpretedVectorValues; + std::vector BiasValues; + std::wstring PublicRule; + + bool hasBias() const { return BiasInputType != ComponentType::Invalid; } +}; + +static std::optional componentByteSize(ComponentType Type) { + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: + return 1; + case ComponentType::F16: + case ComponentType::I16: + case ComponentType::U16: + return 2; + case ComponentType::F32: + case ComponentType::I32: + case ComponentType::U32: + return 4; + default: + return std::nullopt; + } +} + +static bool isPackedByteVector(ComponentType Type) { + return Type == ComponentType::I8 || Type == ComponentType::U8; +} + +static const char *storageTypeName(ComponentType Type) { + if (isPackedByteVector(Type)) + return "uint"; + + switch (Type) { + case ComponentType::F16: + return "half"; + case ComponentType::F32: + return "float"; + case ComponentType::I32: + return "int"; + case ComponentType::U32: + return "uint"; + default: + return nullptr; + } +} + +static MatrixDim storageElementCount(ComponentType Type, + MatrixDim LogicalCount) { + return isPackedByteVector(Type) ? (LogicalCount + 3) / 4 : LogicalCount; +} + +static size_t storageElementByteSize(ComponentType Type) { + return isPackedByteVector(Type) ? sizeof(uint32_t) + : componentByteSize(Type).value_or(0); +} + +static bool checkedMultiplyInt64(int64_t Left, int64_t Right, int64_t &Result) { + if (Left == 0 || Right == 0) { + Result = 0; + return true; + } + if ((Left == -1 && Right == std::numeric_limits::min()) || + (Right == -1 && Left == std::numeric_limits::min())) + return false; + + if (Left > 0) { + if ((Right > 0 && Left > std::numeric_limits::max() / Right) || + (Right < 0 && Right < std::numeric_limits::min() / Left)) + return false; + } else { + if ((Right > 0 && Left < std::numeric_limits::min() / Right) || + (Right < 0 && Left < std::numeric_limits::max() / Right)) + return false; + } + + Result = Left * Right; + return true; +} + +static bool checkedAddInt64(int64_t Left, int64_t Right, int64_t &Result) { + if ((Right > 0 && Left > std::numeric_limits::max() - Right) || + (Right < 0 && Left < std::numeric_limits::min() - Right)) + return false; + Result = Left + Right; + return true; +} + +template +static std::vector encodeNativeVector(const std::vector &Values) { + static_assert(std::is_trivially_copyable::value, + "Vector values must be trivially copyable"); + std::vector Bytes(Values.size() * sizeof(T)); + if (!Bytes.empty()) + std::memcpy(Bytes.data(), Values.data(), Bytes.size()); + return Bytes; +} + +static std::optional encodeByte(ComponentType Type, int64_t Value) { + if (Type == ComponentType::I8) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return std::nullopt; + return static_cast(static_cast(static_cast(Value))); + } + if (Type == ComponentType::U8) { + if (Value < 0 || Value > std::numeric_limits::max()) + return std::nullopt; + return static_cast(Value); + } + return std::nullopt; +} + +static std::optional> +encodeComponents(ComponentType Type, const std::vector &Values) { + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: { + std::vector Bytes; + Bytes.reserve(Values.size()); + for (int64_t Value : Values) { + std::optional Encoded = encodeByte(Type, Value); + if (!Encoded) + return std::nullopt; + Bytes.push_back(*Encoded); + } + return Bytes; + } + case ComponentType::F16: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + const HLSLHalf_t Half(static_cast(Value)); + if (static_cast(Half) != static_cast(Value)) + return std::nullopt; + Native.push_back(Half); + } + return encodeNativeVector(Native); + } + case ComponentType::F32: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + const float FloatValue = static_cast(Value); + if (static_cast(FloatValue) != Value) + return std::nullopt; + Native.push_back(FloatValue); + } + return encodeNativeVector(Native); + } + case ComponentType::I32: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return std::nullopt; + Native.push_back(static_cast(Value)); + } + return encodeNativeVector(Native); + } + case ComponentType::U32: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + if (Value < 0 || + static_cast(Value) > std::numeric_limits::max()) + return std::nullopt; + Native.push_back(static_cast(Value)); + } + return encodeNativeVector(Native); + } + default: + return std::nullopt; + } +} + +static std::optional> +encodePackedVector(ComponentType Type, const std::vector &Values) { + if (!isPackedByteVector(Type)) + return std::nullopt; + + size_t PaddedCount; + if (!cpu_oracle::checkedAdd(Values.size(), size_t(3), PaddedCount)) + return std::nullopt; + PaddedCount &= ~size_t(3); + std::vector Bytes(PaddedCount, 0); + + for (size_t WordIndex = 0; WordIndex < PaddedCount / 4; ++WordIndex) { + uint32_t Word = 0; + for (size_t Lane = 0; Lane < 4; ++Lane) { + const size_t ValueIndex = WordIndex * 4 + Lane; + if (ValueIndex == Values.size()) + break; + std::optional Encoded = encodeByte(Type, Values[ValueIndex]); + if (!Encoded) + return std::nullopt; + // Lane zero occupies the least-significant byte of each uint. + Word |= static_cast(*Encoded) << (Lane * 8); + } + for (size_t ByteIndex = 0; ByteIndex < 4; ++ByteIndex) + Bytes[WordIndex * 4 + ByteIndex] = + static_cast(Word >> (ByteIndex * 8)); + } + return Bytes; +} + +static std::optional matrixStrideBytes(const CaseData &Case) { + const std::optional ComponentSize = + componentByteSize(Case.MatrixType); + if (!ComponentSize) + return std::nullopt; + const size_t MinorCount = + Case.Layout == MatrixLayout::RowMajor ? Case.N : Case.M; + size_t Stride; + if (!cpu_oracle::checkedMultiply(MinorCount, *ComponentSize, Stride)) + return std::nullopt; + return Stride; +} + +static std::optional> +encodeMatrixBuffer(const CaseData &Case) { + const std::optional ComponentSize = + componentByteSize(Case.MatrixType); + const std::optional Stride = matrixStrideBytes(Case); + const std::optional> Logical = + encodeComponents(Case.MatrixType, Case.MatrixValues); + if (!ComponentSize || !Stride || !Logical) + return std::nullopt; + + const size_t MajorCount = + Case.Layout == MatrixLayout::RowMajor ? Case.M : Case.N; + size_t BufferSize; + if (!cpu_oracle::checkedMultiply(MajorCount, *Stride, BufferSize)) + return std::nullopt; + std::vector Buffer(BufferSize, 0); + + for (MatrixDim Row = 0; Row < Case.M; ++Row) { + for (MatrixDim Column = 0; Column < Case.N; ++Column) { + const size_t SourceIndex = static_cast(Row) * Case.N + Column; + const size_t SourceOffset = SourceIndex * *ComponentSize; + const size_t DestinationOffset = + Case.Layout == MatrixLayout::RowMajor + ? static_cast(Row) * *Stride + Column * *ComponentSize + : static_cast(Column) * *Stride + Row * *ComponentSize; + std::memcpy(Buffer.data() + DestinationOffset, + Logical->data() + SourceOffset, *ComponentSize); + } + } + return Buffer; +} + +static std::optional> +calculateExpected(const CaseData &Case) { + size_t MatrixElementCount; + if (!cpu_oracle::checkedMultiply(static_cast(Case.M), + static_cast(Case.N), + MatrixElementCount) || + Case.MatrixValues.size() != MatrixElementCount || + Case.InterpretedVectorValues.size() != Case.N || + (Case.hasBias() && Case.BiasValues.size() != Case.M)) + return std::nullopt; + + std::vector Expected(Case.M, 0); + for (MatrixDim Row = 0; Row < Case.M; ++Row) { + for (MatrixDim Column = 0; Column < Case.N; ++Column) { + int64_t Product; + int64_t Sum; + if (!checkedMultiplyInt64( + Case.MatrixValues[static_cast(Row) * Case.N + Column], + Case.InterpretedVectorValues[Column], Product) || + !checkedAddInt64(Expected[Row], Product, Sum)) + return std::nullopt; + Expected[Row] = Sum; + } + if (Case.hasBias()) { + int64_t Sum; + if (!checkedAddInt64(Expected[Row], Case.BiasValues[Row], Sum)) + return std::nullopt; + Expected[Row] = Sum; + } + } + return Expected; +} + +static bool oracleSelfTest() { + const std::optional> PackedSInt8 = + encodePackedVector(ComponentType::I8, {-1, 2, -3, 4, 5}); + const std::optional> PackedUInt8 = + encodePackedVector(ComponentType::U8, {255, 2, 253, 4, 5}); + const std::vector PackedBytes = {0xff, 0x02, 0xfd, 0x04, + 0x05, 0x00, 0x00, 0x00}; + + CaseData DotCase = {}; + DotCase.M = 2; + DotCase.N = 3; + DotCase.MatrixValues = {1, 2, 3, -1, 4, 0}; + DotCase.InterpretedVectorValues = {4, -2, 5}; + DotCase.BiasInputType = ComponentType::I32; + DotCase.BiasValues = {7, -3}; + const std::optional> Dot = calculateExpected(DotCase); + + int64_t Ignored; + return PackedSInt8 == PackedBytes && PackedUInt8 == PackedBytes && Dot && + *Dot == std::vector({22, -15}) && + !checkedMultiplyInt64(std::numeric_limits::max(), 2, + Ignored) && + !checkedAddInt64(std::numeric_limits::max(), 1, Ignored); +} + +static bool isCaseValid(const CaseData &Case) { + size_t MatrixElementCount; + if (Case.M == 0 || Case.N == 0 || + !cpu_oracle::checkedMultiply(static_cast(Case.M), + static_cast(Case.N), + MatrixElementCount) || + Case.MatrixValues.size() != MatrixElementCount || + Case.InterpretedVectorValues.size() != Case.N || + (Case.Layout != MatrixLayout::RowMajor && + Case.Layout != MatrixLayout::ColumnMajor) || + !componentByteSize(Case.MatrixType) || + !storageTypeName(Case.VectorInputType) || + !storageTypeName(Case.ResultType) || Case.PublicRule.empty()) + return false; + + // A vector is either native or an InterpretedVector, which pairs a packed + // vector with an interpretation type. A native element type paired with a + // narrower interpretation is not a valid form. + if (Case.VectorInputType == ComponentType::F32 && + Case.InputInterpretation != ComponentType::F32) + return false; + if (isPackedByteVector(Case.VectorInputType) && + Case.InputInterpretation != Case.VectorInputType) + return false; + if (Case.hasBias() != !Case.BiasValues.empty() || + (Case.hasBias() && (Case.BiasValues.size() != Case.M || + Case.BiasInputType != Case.ResultType || + !storageTypeName(Case.BiasInputType)))) + return false; + + const bool ExpectedSigned = Case.ResultType != ComponentType::U32; + return Case.OutputSigned == ExpectedSigned; +} + +static std::optional> +encodeVectorBuffer(const CaseData &Case) { + if (isPackedByteVector(Case.VectorInputType)) + return encodePackedVector(Case.VectorInputType, + Case.InterpretedVectorValues); + return encodeComponents(Case.VectorInputType, Case.InterpretedVectorValues); +} + +static std::optional> +encodeExpectedOutput(const CaseData &Case) { + const std::optional> Values = calculateExpected(Case); + if (!Values) + return std::nullopt; + const std::optional> Logical = + encodeComponents(Case.ResultType, *Values); + if (!Logical) + return std::nullopt; + + size_t PaddedSize; + if (!cpu_oracle::checkedAdd(Logical->size(), size_t(3), PaddedSize)) + return std::nullopt; + PaddedSize &= ~size_t(3); + size_t BufferSize; + if (!cpu_oracle::checkedAdd(PaddedSize, OutputGuardBytes, BufferSize)) + return std::nullopt; + + std::vector Buffer(BufferSize); + cpu_oracle::fillPoison(Buffer.data(), Buffer.size()); + std::memcpy(Buffer.data(), Logical->data(), Logical->size()); + return Buffer; +} + +static bool needs16BitTypes(ComponentType Type) { + return Type == ComponentType::F16 || Type == ComponentType::I16 || + Type == ComponentType::U16; +} + +static std::optional buildCompilerArgs(const CaseData &Case) { + const std::optional MatrixStride = matrixStrideBytes(Case); + const char *InputStorageType = storageTypeName(Case.VectorInputType); + const char *OutputType = storageTypeName(Case.ResultType); + const char *BiasStorageType = + Case.hasBias() ? storageTypeName(Case.BiasInputType) : nullptr; + if (!MatrixStride || !InputStorageType || !OutputType || + (Case.hasBias() && !BiasStorageType)) + return std::nullopt; + + std::stringstream Args; + Args << "-HV 202x"; + Args << " -DMATRIX_COMP_TYPE=" << static_cast(Case.MatrixType); + Args << " -DM_DIM=" << Case.M; + Args << " -DN_DIM=" << Case.N; + Args << " -DMATRIX_STRIDE=" << *MatrixStride; + Args << " -DMATRIX_LAYOUT=" << static_cast(Case.Layout); + Args << " -DINPUT_STORAGE_TYPE=" << InputStorageType; + Args << " -DINPUT_STORAGE_COUNT=" + << storageElementCount(Case.VectorInputType, Case.N); + Args << " -DINPUT_STORAGE_SIZE=" + << storageElementByteSize(Case.VectorInputType); + Args << " -DINPUT_INTERP=" << static_cast(Case.InputInterpretation); + Args << " -DOUTPUT_TYPE=" << OutputType; + Args << " -DOUTPUT_SIZE=" << componentByteSize(Case.ResultType).value_or(0); + Args << " -DOUTPUT_SIGNED=" << (Case.OutputSigned ? 1 : 0); + if (Case.hasBias()) { + Args << " -DBIAS_STORAGE_TYPE=" << BiasStorageType; + Args << " -DBIAS_STORAGE_COUNT=" + << storageElementCount(Case.BiasInputType, Case.M); + Args << " -DBIAS_STORAGE_SIZE=" + << storageElementByteSize(Case.BiasInputType); + } + if (needs16BitTypes(Case.MatrixType) || + needs16BitTypes(Case.VectorInputType) || + needs16BitTypes(Case.BiasInputType) || needs16BitTypes(Case.ResultType)) + Args << " -enable-16bit-types"; + return Args.str(); +} + +static bool verifyExactBuffer(const void *ActualBuffer, size_t ActualSize, + const std::vector &Expected, bool Verbose) { + if (ActualSize != Expected.size()) { + hlsl_test::LogErrorFmt( + L"MatVec output size mismatch: actual=%zu, expected=%zu", ActualSize, + Expected.size()); + return false; + } + + const BYTE *Actual = static_cast(ActualBuffer); + size_t MismatchCount = 0; + for (size_t I = 0; I < Expected.size(); ++I) { + if (Actual[I] == Expected[I]) + continue; + if (MismatchCount < 8) + hlsl_test::LogErrorFmt( + L"MatVec output byte %zu mismatch: actual=0x%02x, expected=0x%02x", I, + Actual[I], Expected[I]); + ++MismatchCount; + } + if (MismatchCount != 0) { + hlsl_test::LogErrorFmt(L"%zu MatVec output bytes differed", MismatchCount); + return false; + } + if (Verbose) + hlsl_test::LogCommentFmt( + L"All %zu MatVec output, padding, and guard bytes matched exactly", + Expected.size()); + return true; +} + +static const char MatVecMulShader[] = R"( + #define USE_A 0 + #define SCOPE_THREAD 0 + + ByteAddressBuffer MatrixInput : register(t0); + ByteAddressBuffer VectorInput : register(t1); + RWByteAddressBuffer Output : register(u2); + + [numthreads(1, 1, 1)] + void main() { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes( + MATRIX_COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, 0, MATRIX_STRIDE, MATRIX_LAYOUT, 128); + + vector InVec; + for (uint I = 0; I < INPUT_STORAGE_COUNT; ++I) { + InVec[I] = + VectorInput.Load(I * INPUT_STORAGE_SIZE); + } + + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiply( + OutVec, Mat, OUTPUT_SIGNED, InVec, INPUT_INTERP); + + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(I * OUTPUT_SIZE, OutVec[I]); + } + } +)"; + +static const char MatVecMulAddShader[] = R"( + #define USE_A 0 + #define SCOPE_THREAD 0 + + ByteAddressBuffer MatrixInput : register(t0); + ByteAddressBuffer VectorInput : register(t1); + ByteAddressBuffer BiasInput : register(t2); + RWByteAddressBuffer Output : register(u3); + + [numthreads(1, 1, 1)] + void main() { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes( + MATRIX_COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, 0, MATRIX_STRIDE, MATRIX_LAYOUT, 128); + + vector InVec; + for (uint I = 0; I < INPUT_STORAGE_COUNT; ++I) { + InVec[I] = + VectorInput.Load(I * INPUT_STORAGE_SIZE); + } + + vector BiasVec; + for (uint I = 0; I < BIAS_STORAGE_COUNT; ++I) { + BiasVec[I] = BiasInput.Load(I * BIAS_STORAGE_SIZE); + } + + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiplyAdd( + OutVec, Mat, OUTPUT_SIGNED, InVec, INPUT_INTERP, BiasVec); + + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(I * OUTPUT_SIZE, OutVec[I]); + } + } +)"; + +static HRESULT querySupport(ID3D12Device *Device, const CaseData &Case, + bool &TierSupported, bool &Supported) { + TierSupported = false; + Supported = false; + if (!Device) + return E_INVALIDARG; + + const std::optional VectorType = + toCapabilityDataType(Case.VectorInputType); + const std::optional MatrixType = + toCapabilityDataType(Case.MatrixType); + const std::optional BiasType = + Case.hasBias() ? toCapabilityDataType(Case.BiasInputType) + : std::optional( + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE); + const std::optional ResultType = + toCapabilityDataType(Case.ResultType); + if (!VectorType || !MatrixType || !BiasType || !ResultType) + return E_INVALIDARG; + + linalg_test::TierSupport Tier; + HRESULT HR = linalg_test::queryTierSupport(Device, Tier); + if (FAILED(HR)) + return HR; + TierSupported = Tier.supported(); + if (!TierSupported) + return S_OK; + + linalg_test::ThreadVectorMatrixMultiplySupport Multiply; + HR = linalg_test::queryThreadVectorMatrixMultiply( + Device, {*VectorType, *MatrixType, *BiasType, *ResultType}, Multiply); + if (FAILED(HR)) + return HR; + + Supported = Multiply.supported(); + if (!Supported) + hlsl_test::LogCommentFmt( + L"ThreadVectorMatrixMultiply reports vector=%u matrix=%u bias=%u " + L"result=%u layout=%u is unsupported", + static_cast(*VectorType), static_cast(*MatrixType), + static_cast(*BiasType), static_cast(*ResultType), + static_cast(Case.Layout)); + return S_OK; +} + +static void runCase(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + const CaseData &Case, bool Verbose) { + const bool SelfTestPassed = oracleSelfTest(); + VERIFY_IS_TRUE(SelfTestPassed, "MatVec host oracle self-test failed"); + const bool Valid = isCaseValid(Case); + VERIFY_IS_TRUE(Valid, "Invalid MatVec interpretation case"); + if (!SelfTestPassed || !Valid) + return; + + const std::optional> MatrixBuffer = + encodeMatrixBuffer(Case); + const std::optional> VectorBuffer = + encodeVectorBuffer(Case); + const std::optional> BiasBuffer = + Case.hasBias() ? encodeComponents(Case.BiasInputType, Case.BiasValues) + : std::optional>(); + const std::optional> ExpectedOutput = + encodeExpectedOutput(Case); + const std::optional Args = buildCompilerArgs(Case); + VERIFY_IS_TRUE(MatrixBuffer.has_value()); + VERIFY_IS_TRUE(VectorBuffer.has_value()); + VERIFY_IS_TRUE(!Case.hasBias() || BiasBuffer.has_value()); + VERIFY_IS_TRUE(ExpectedOutput.has_value()); + VERIFY_IS_TRUE(Args.has_value()); + if (!MatrixBuffer || !VectorBuffer || (Case.hasBias() && !BiasBuffer) || + !ExpectedOutput || !Args) + return; + + const char *Shader = Case.hasBias() ? MatVecMulAddShader : MatVecMulShader; + const char *RootSignature = Case.hasBias() + ? "SRV(t0), SRV(t1), SRV(t2), UAV(u3)" + : "SRV(t0), SRV(t1), UAV(u2)"; + compileShader(DxcSupport, Shader, "cs_6_10", *Args, Verbose); + + auto Op = createComputeOp(Shader, "cs_6_10", RootSignature, Args->c_str()); + addSRVBuffer(Op.get(), "MatrixInput", MatrixBuffer->size(), "byname"); + addSRVBuffer(Op.get(), "VectorInput", VectorBuffer->size(), "byname"); + if (Case.hasBias()) + addSRVBuffer(Op.get(), "BiasInput", BiasBuffer->size(), "byname"); + addUAVBuffer(Op.get(), "Output", ExpectedOutput->size(), true, "byname"); + addRootView(Op.get(), 0, "MatrixInput"); + addRootView(Op.get(), 1, "VectorInput"); + if (Case.hasBias()) { + addRootView(Op.get(), 2, "BiasInput"); + addRootView(Op.get(), 3, "Output"); + } else { + addRootView(Op.get(), 2, "Output"); + } + + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [&](LPCSTR Name, std::vector &Data, st::ShaderOp *) { + if (_stricmp(Name, "Output") == 0) { + cpu_oracle::fillPoison(Data.data(), Data.size()); + return; + } + + const std::vector *Source = nullptr; + if (_stricmp(Name, "MatrixInput") == 0) + Source = &*MatrixBuffer; + else if (_stricmp(Name, "VectorInput") == 0) + Source = &*VectorBuffer; + else if (Case.hasBias() && _stricmp(Name, "BiasInput") == 0) + Source = &*BiasBuffer; + VERIFY_IS_TRUE(Source != nullptr, + "Unexpected MatVec resource initializer"); + if (!Source) + return; + VERIFY_IS_TRUE(Data.size() == Source->size(), + "MatVec resource initializer size mismatch"); + if (Data.size() == Source->size()) + std::memcpy(Data.data(), Source->data(), Data.size()); + }); + + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); + VERIFY_IS_TRUE(verifyExactBuffer(OutData.data(), OutData.size(), + *ExpectedOutput, Verbose)); +} + +static void runCapabilityChecked(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const CaseData &Case, + linalg_test::CapabilityRequirement Requirement, + LPCWSTR CaseName, bool Verbose) { + bool TierSupported = false; + bool Supported = false; + const HRESULT QueryResult = + querySupport(Device, Case, TierSupported, Supported); + const linalg_test::CapabilityRequirement Effective = + SUCCEEDED(QueryResult) && !TierSupported + ? linalg_test::CapabilityRequirement::CapabilityGated + : Requirement; + if (!applyApplicability( + linalg_test::classifyApplicability(QueryResult, Supported, Effective), + CaseName)) + return; + runCase(Device, DxcSupport, Case, Verbose); +} + +static CaseData makeNonUniformF16Case(MatrixLayout Layout) { + CaseData Case = {}; + Case.MatrixType = ComponentType::F16; + Case.M = 4; + Case.N = 8; + Case.Layout = Layout; + Case.VectorInputType = ComponentType::F16; + Case.InputInterpretation = ComponentType::F16; + Case.ResultType = ComponentType::F16; + Case.MatrixValues = { + 1, 0, -1, 2, -2, 3, -3, 1, 0, 1, 2, -1, 3, -2, 1, -3, + -1, 2, 0, 1, -2, 1, 3, -1, 2, -1, 1, 0, 1, -3, -2, 3, + }; + Case.InterpretedVectorValues = {1, -2, 3, -1, 2, -3, 1, 2}; + Case.PublicRule = + Layout == MatrixLayout::RowMajor + ? L"Exact non-uniform F16 RowMajor matrix-vector dot products" + : L"Exact non-uniform F16 ColumnMajor matrix-vector dot products"; + return Case; +} + +static CaseData makeSInt8Case() { + CaseData Case = {}; + Case.MatrixType = ComponentType::I8; + Case.M = 4; + Case.N = 8; + Case.Layout = MatrixLayout::RowMajor; + Case.VectorInputType = ComponentType::I8; + Case.InputInterpretation = ComponentType::I8; + Case.ResultType = ComponentType::I32; + Case.MatrixValues = { + 1, -2, 3, -4, 5, -6, 7, -8, -1, 2, -3, 4, -5, 6, -7, 8, + 1, 1, 1, 1, 1, 1, 1, 1, -8, -7, -6, -5, -4, -3, -2, -1, + }; + Case.InterpretedVectorValues = {1, -1, 2, -2, 3, -3, 4, -4}; + Case.PublicRule = + L"Exact packed SInt8 vector times SInt8 matrix dot products"; + return Case; +} + +static CaseData makeUInt8Case() { + CaseData Case = {}; + Case.MatrixType = ComponentType::U8; + Case.M = 4; + Case.N = 8; + Case.Layout = MatrixLayout::RowMajor; + Case.VectorInputType = ComponentType::U8; + Case.InputInterpretation = ComponentType::U8; + Case.ResultType = ComponentType::I32; + Case.MatrixValues = { + 255, 1, 2, 3, 4, 5, 6, 7, 128, 127, 1, 1, 1, 1, 1, 1, + 200, 0, 200, 0, 200, 0, 200, 0, 0, 200, 0, 200, 0, 200, 0, 200, + }; + Case.InterpretedVectorValues = {1, 255, 2, 254, 3, 253, 4, 252}; + Case.PublicRule = + L"Exact packed UInt8 vector times UInt8 matrix dot products"; + return Case; +} + +static CaseData makeUInt32OutputCase() { + CaseData Case = {}; + Case.MatrixType = ComponentType::U32; + Case.M = 4; + Case.N = 8; + Case.Layout = MatrixLayout::RowMajor; + Case.VectorInputType = ComponentType::U32; + Case.InputInterpretation = ComponentType::U32; + Case.ResultType = ComponentType::U32; + Case.OutputSigned = false; + Case.MatrixValues = { + 2147483648LL, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 100, 0, 100, 0, 100, 0, 100, 0, 0, 200, 0, 200, 0, 200, 0, 200, + }; + Case.InterpretedVectorValues = {1, 1, 1, 1, 1, 1, 1, 1}; + Case.PublicRule = + L"Exact native UInt32 matrix-vector results with unsigned output"; + return Case; +} + +} // namespace matvec_interpretation + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F16_NonUniform() { + const matvec_interpretation::CaseData Case = + matvec_interpretation::makeNonUniformF16Case(MatrixLayout::RowMajor); + matvec_interpretation::runCapabilityChecked( + D3DDevice, DxcSupport, Case, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_4x8_F16_NonUniform", VerboseLogging); +} + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F16_ColumnMajor() { + const matvec_interpretation::CaseData Case = + matvec_interpretation::makeNonUniformF16Case(MatrixLayout::ColumnMajor); + matvec_interpretation::runCapabilityChecked( + D3DDevice, DxcSupport, Case, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_4x8_F16_ColumnMajor", VerboseLogging); +} + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_I8_Interpreted() { + const matvec_interpretation::CaseData Case = + matvec_interpretation::makeSInt8Case(); + matvec_interpretation::runCapabilityChecked( + D3DDevice, DxcSupport, Case, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_4x8_I8_Interpreted", VerboseLogging); +} + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_U8_Interpreted() { + const matvec_interpretation::CaseData Case = + matvec_interpretation::makeUInt8Case(); + matvec_interpretation::runCapabilityChecked( + D3DDevice, DxcSupport, Case, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_4x8_U8_Interpreted", VerboseLogging); +} + +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_U32_UnsignedOutput() { + const matvec_interpretation::CaseData Case = + matvec_interpretation::makeUInt32OutputCase(); + matvec_interpretation::runCapabilityChecked( + D3DDevice, DxcSupport, Case, + linalg_test::CapabilityRequirement::CapabilityGated, + L"MatVecMul_Thread_4x8_U32_UnsignedOutput", VerboseLogging); +} + +void DxilConf_SM610_LinAlg::MatVecMulAdd_Thread_4x8_F16_IndependentBias() { + matvec_interpretation::CaseData Case = + matvec_interpretation::makeNonUniformF16Case(MatrixLayout::RowMajor); + Case.BiasInputType = ComponentType::F16; + Case.BiasValues = {-5, 7, 3, -9}; + Case.PublicRule = + L"Exact non-uniform F16 dots plus independent non-uniform bias"; + matvec_interpretation::runCapabilityChecked( + D3DDevice, DxcSupport, Case, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMulAdd_Thread_4x8_F16_IndependentBias", VerboseLogging); +} + struct ConvertThreadVectorMatrixMultiplyEntry { UINT VectorInputType; UINT MatrixInputType; From 9bb22e0e82422cd2fc9019b47c3f127fe0fb7fea Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Sat, 15 Aug 2026 08:29:55 +1200 Subject: [PATCH 2/6] [NFC] Move matvec_interpretation above the LinAlg test classes LinAlgTests.cpp places helper namespaces before the test classes that use them: cpu_oracle sits at the top of the file, ahead of the first test class. matvec_interpretation was appended after the classes instead, which left no way for a LinAlgCPUOracleTests method to call into it without a forward declaration. This is a pure relocation of the namespace block. No line is added, removed or edited: the file has the same 6060 lines before and after, and the sorted set of lines is identical. clang-format reports no drift. Verified with the full HLSLExec LinAlg selection on WARP, compared per test rather than by totals: 45 total, 39 passed, 5 failed, 1 skipped, with the non-passing set unchanged from the parent commit. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 6870 ++++++++--------- 1 file changed, 3435 insertions(+), 3435 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index a1ab58c7c9..105e0cab0d 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -1669,1669 +1669,1505 @@ static VariantCompType makeExpectedVec(ComponentType CompType, false); } -// Harness self-check for the CPU oracle. Deliberately carries no Kits metadata -// so HLK runs never select it; drivers are not certified against this class. -class LinAlgCPUOracleTests { -public: - BEGIN_TEST_CLASS(LinAlgCPUOracleTests) - TEST_METHOD_PROPERTY(L"Priority", L"0") - END_TEST_CLASS() +namespace matvec_interpretation { - TEST_METHOD(TypedMatrixBufferRoundTrip); - TEST_METHOD(UntouchedByteVerification); - TEST_METHOD(ViewBoundedElements); - TEST_METHOD(ViewBoundedStoreBytes); -}; +static constexpr size_t OutputGuardBytes = 16; -void LinAlgCPUOracleTests::TypedMatrixBufferRoundTrip() { - using namespace cpu_oracle; +struct CaseData { + ComponentType MatrixType = ComponentType::Invalid; + MatrixDim M = 0; + MatrixDim N = 0; + MatrixLayout Layout = MatrixLayout::RowMajor; + ComponentType VectorInputType = ComponentType::Invalid; + ComponentType InputInterpretation = ComponentType::Invalid; + ComponentType BiasInputType = ComponentType::Invalid; + ComponentType ResultType = ComponentType::Invalid; + bool OutputSigned = true; + std::vector MatrixValues; + std::vector InterpretedVectorValues; + std::vector BiasValues; + std::wstring PublicRule; - auto VerifyScalarEncoding = [](const std::optional &Matrix, - const std::vector &ExpectedBytes) { - if (!Matrix) - return false; - MatrixBufferLayout Layout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/0, - /*StrideBytes=*/ExpectedBytes.size(), - }; - std::vector ActualBytes(ExpectedBytes.size(), 0); - MatrixResultOracle Oracle = - exactResult(*Matrix, L"Host scalar encoding and decoding"); - return writeMatrixBuffer(*Matrix, Layout, ActualBytes) && - ActualBytes == ExpectedBytes && - verifyMatrixBuffer(ActualBytes.data(), ActualBytes.size(), Layout, - Oracle, /*Verbose=*/false); - }; + bool hasBias() const { return BiasInputType != ComponentType::Invalid; } +}; - VERIFY_IS_TRUE(VerifyScalarEncoding( - makeTypedMatrix(1, 1, {HLSLHalf_t(1.5f)}), {0x00, 0x3e})); - VERIFY_IS_TRUE(VerifyScalarEncoding(makeTypedMatrix(1, 1, {-2.5f}), - {0x00, 0x00, 0x20, 0xc0})); - VERIFY_IS_TRUE(VerifyScalarEncoding(makeTypedMatrix(1, 1, {-7}), - {0xf9, 0xff, 0xff, 0xff})); - VERIFY_IS_TRUE( - VerifyScalarEncoding(makeTypedMatrix(1, 1, {0x89abcdefu}), - {0xef, 0xcd, 0xab, 0x89})); +static std::optional componentByteSize(ComponentType Type) { + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: + return 1; + case ComponentType::F16: + case ComponentType::I16: + case ComponentType::U16: + return 2; + case ComponentType::F32: + case ComponentType::I32: + case ComponentType::U32: + return 4; + default: + return std::nullopt; + } +} - const uint32_t AdjacentFloatBits = 0x3f800001; - float AdjacentFloat; - std::memcpy(&AdjacentFloat, &AdjacentFloatBits, sizeof(AdjacentFloat)); - VERIFY_IS_TRUE( - ComponentTraits::format(AdjacentFloat).find(L"3f800001") != - std::wstring::npos); +static bool isPackedByteVector(ComponentType Type) { + return Type == ComponentType::I8 || Type == ComponentType::U8; +} - std::optional Matrix = - makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); - VERIFY_IS_TRUE(Matrix.has_value()); +static const char *storageTypeName(ComponentType Type) { + if (isPackedByteVector(Type)) + return "uint"; - MatrixBufferLayout RowMajor = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/4, - /*StrideBytes=*/16, - }; - std::optional RowBytes = getMatrixBufferSize(*Matrix, RowMajor); - VERIFY_IS_TRUE(RowBytes.has_value()); - std::vector RowBuffer(*RowBytes, 0xcd); - VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, RowMajor, RowBuffer)); - const std::vector ExpectedRowBuffer = { - 0xcd, 0xcd, 0xcd, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, - 0x00, 0x03, 0x00, 0x00, 0x00, 0xcd, 0xcd, 0xcd, 0xcd, 0x04, 0x00, - 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, - }; - VERIFY_IS_TRUE(RowBuffer == ExpectedRowBuffer); - MatrixResultOracle Exact = - exactResult(*Matrix, L"Host exact row-major matrix encoding"); - VERIFY_IS_TRUE(verifyMatrixBuffer(RowBuffer.data(), RowBuffer.size(), - RowMajor, Exact, /*Verbose=*/false)); + switch (Type) { + case ComponentType::F16: + return "half"; + case ComponentType::F32: + return "float"; + case ComponentType::I32: + return "int"; + case ComponentType::U32: + return "uint"; + default: + return nullptr; + } +} - MatrixBufferLayout ColumnMajor = { - MatrixLayout::ColumnMajor, - /*OffsetBytes=*/4, - /*StrideBytes=*/12, - }; - std::optional ColumnBytes = getMatrixBufferSize(*Matrix, ColumnMajor); - VERIFY_IS_TRUE(ColumnBytes.has_value()); - std::vector ColumnBuffer(*ColumnBytes, 0xcd); - VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, ColumnMajor, ColumnBuffer)); - const std::vector ExpectedColumnBuffer = { - 0xcd, 0xcd, 0xcd, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0xcd, 0xcd, 0xcd, 0xcd, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, - 0xcd, 0xcd, 0xcd, 0xcd, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, - }; - VERIFY_IS_TRUE(ColumnBuffer == ExpectedColumnBuffer); - VERIFY_IS_TRUE(verifyMatrixBuffer(ColumnBuffer.data(), ColumnBuffer.size(), - ColumnMajor, Exact, /*Verbose=*/false)); +static MatrixDim storageElementCount(ComponentType Type, + MatrixDim LogicalCount) { + return isPackedByteVector(Type) ? (LogicalCount + 3) / 4 : LogicalCount; +} - std::optional Transposed = transposeMatrix(*Matrix); - std::optional ExpectedTranspose = - makeTypedMatrix(3, 2, {1, 4, 2, 5, 3, 6}); - VERIFY_IS_TRUE(Transposed.has_value()); - VERIFY_IS_TRUE(ExpectedTranspose.has_value()); - size_t FirstMismatch; - VERIFY_IS_TRUE( - exactMatrixMatch(*Transposed, *ExpectedTranspose, FirstMismatch)); +static size_t storageElementByteSize(ComponentType Type) { + return isPackedByteVector(Type) ? sizeof(uint32_t) + : componentByteSize(Type).value_or(0); +} - std::optional MixedActual = - makeTypedMatrix(1, 2, {1, 4}); - std::optional CandidateA = - makeTypedMatrix(1, 2, {1, 2}); - std::optional CandidateB = - makeTypedMatrix(1, 2, {3, 4}); - VERIFY_IS_TRUE(MixedActual.has_value()); - VERIFY_IS_TRUE(CandidateA.has_value()); - VERIFY_IS_TRUE(CandidateB.has_value()); - MatrixResultOracle Permitted = - permittedResults({*CandidateA, *CandidateB}, - L"Host whole-result permitted candidate semantics"); - VERIFY_IS_FALSE(matchesAnyCompleteCandidate(*MixedActual, Permitted)); - Permitted.Candidates.push_back(*MixedActual); - VERIFY_IS_TRUE(matchesAnyCompleteCandidate(*MixedActual, Permitted)); +static bool checkedMultiplyInt64(int64_t Left, int64_t Right, int64_t &Result) { + if (Left == 0 || Right == 0) { + Result = 0; + return true; + } + if ((Left == -1 && Right == std::numeric_limits::min()) || + (Right == -1 && Left == std::numeric_limits::min())) + return false; - MatrixResultOracle Excluded = - excludedResult(L"Host excluded-oracle classification"); - VERIFY_IS_FALSE(matchesAnyCompleteCandidate(*Matrix, Excluded)); + if (Left > 0) { + if ((Right > 0 && Left > std::numeric_limits::max() / Right) || + (Right < 0 && Right < std::numeric_limits::min() / Left)) + return false; + } else { + if ((Right > 0 && Left < std::numeric_limits::min() / Right) || + (Right < 0 && Left < std::numeric_limits::max() / Right)) + return false; + } - MatrixParams Params = {}; - Params.M = 2; - Params.N = 3; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 4; - Params.CompType = ComponentType::I32; - VERIFY_IS_TRUE(buildCompilerArgs(Params).find(" -DELEM_TYPE=int") != - std::string::npos); - Params.CompType = ComponentType::U32; - VERIFY_IS_TRUE(buildCompilerArgs(Params).find(" -DELEM_TYPE=uint") != - std::string::npos); + Result = Left * Right; + return true; } -// The padding check is verified here rather than only through the execution -// tests because a GPU round trip cannot easily produce a store that places -// every element correctly and still damages the bytes around them, which is -// the single case this check exists to catch. -void LinAlgCPUOracleTests::UntouchedByteVerification() { - using namespace cpu_oracle; +static bool checkedAddInt64(int64_t Left, int64_t Right, int64_t &Result) { + if ((Right > 0 && Left > std::numeric_limits::max() - Right) || + (Right < 0 && Left < std::numeric_limits::min() - Right)) + return false; + Result = Left + Right; + return true; +} - // A 2x3 uint32 matrix at a 4 byte offset with a 16 byte stride occupies - // bytes 4..15 and 20..31, leaving a 4 byte prologue at 0..3 and 4 bytes of - // padding at 16..19. - std::optional Matrix = - makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); - VERIFY_IS_TRUE(Matrix.has_value()); +template +static std::vector encodeNativeVector(const std::vector &Values) { + static_assert(std::is_trivially_copyable::value, + "Vector values must be trivially copyable"); + std::vector Bytes(Values.size() * sizeof(T)); + if (!Bytes.empty()) + std::memcpy(Bytes.data(), Values.data(), Bytes.size()); + return Bytes; +} - const MatrixBufferLayout Layout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/4, - /*StrideBytes=*/16, - }; - std::optional Size = getMatrixBufferSize(*Matrix, Layout); - VERIFY_IS_TRUE(Size.has_value()); - VERIFY_ARE_EQUAL(size_t(32), *Size); +static std::optional encodeByte(ComponentType Type, int64_t Value) { + if (Type == ComponentType::I8) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return std::nullopt; + return static_cast(static_cast(static_cast(Value))); + } + if (Type == ComponentType::U8) { + if (Value < 0 || Value > std::numeric_limits::max()) + return std::nullopt; + return static_cast(Value); + } + return std::nullopt; +} - std::vector Buffer(*Size); - fillPoison(Buffer.data(), Buffer.size()); - VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, Layout, Buffer)); - - auto CountTouched = [&Layout](const std::vector &Bytes) { - return countTouchedBytesOutsideElements(ComponentType::U32, 2, 3, Layout, - Bytes.data(), Bytes.size()); - }; - - // A correctly encoded buffer leaves every non-element byte poisoned. - std::optional Clean = CountTouched(Buffer); - VERIFY_IS_TRUE(Clean.has_value()); - VERIFY_ARE_EQUAL(size_t(0), *Clean); - VERIFY_IS_TRUE(verifyUntouchedBytes(ComponentType::U32, 2, 3, Layout, - Buffer.data(), Buffer.size(), - /*Verbose=*/false)); - - // Damaging an element is the element comparison's job, not this check's, so - // the count must stay at zero. - std::vector ElementTouched = Buffer; - ElementTouched[4] ^= 0xff; - std::optional AfterElement = CountTouched(ElementTouched); - VERIFY_IS_TRUE(AfterElement.has_value()); - VERIFY_ARE_EQUAL(size_t(0), *AfterElement); - - // Damaging the prologue or the inter-row padding is what this check exists - // to catch, so each one must be counted. - for (size_t Offset : {size_t(0), size_t(16)}) { - std::vector PaddingTouched = Buffer; - PaddingTouched[Offset] ^= 0xff; - std::optional AfterPadding = CountTouched(PaddingTouched); - VERIFY_IS_TRUE(AfterPadding.has_value()); - VERIFY_ARE_EQUAL(size_t(1), *AfterPadding); +static std::optional> +encodeComponents(ComponentType Type, const std::vector &Values) { + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: { + std::vector Bytes; + Bytes.reserve(Values.size()); + for (int64_t Value : Values) { + std::optional Encoded = encodeByte(Type, Value); + if (!Encoded) + return std::nullopt; + Bytes.push_back(*Encoded); + } + return Bytes; } - - // Every non-element byte damaged at once is still counted exactly. - std::vector AllTouched(*Size); - fillPoison(AllTouched.data(), AllTouched.size()); - for (BYTE &Byte : AllTouched) - Byte = static_cast(~Byte); - VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, Layout, AllTouched)); - std::optional AfterAll = CountTouched(AllTouched); - VERIFY_IS_TRUE(AfterAll.has_value()); - VERIFY_ARE_EQUAL(size_t(8), *AfterAll); - - // A buffer filled with one repeated value is fully detected, which is the - // reason the pattern varies with the offset. A constant poison would score - // zero here whenever the store happened to pick that same value, and 0xcd in - // particular is what the MSVC debug allocator leaves in memory nobody wrote. - std::vector ConstantFill(*Size, BYTE(0xcd)); - std::optional AfterConstant = CountTouched(ConstantFill); - VERIFY_IS_TRUE(AfterConstant.has_value()); - VERIFY_ARE_EQUAL(size_t(8), *AfterConstant); - - // The case a constant poison cannot survive: a store that writes a value the - // poison pattern itself uses. Because the pattern varies, that value matches - // at exactly one offset, so seven of the eight non-element bytes are still - // caught. A constant poison would match everywhere and report nothing. - std::vector PoisonValuedFill(*Size, poisonByteAt(0)); - std::optional AfterPoisonValued = CountTouched(PoisonValuedFill); - VERIFY_IS_TRUE(AfterPoisonValued.has_value()); - VERIFY_ARE_EQUAL(size_t(7), *AfterPoisonValued); - - // No two adjacent bytes share a poison value, so a constant written over any - // two neighbours cannot hide in both. - for (size_t Offset = 1; Offset < *Size; ++Offset) - VERIFY_ARE_NOT_EQUAL(poisonByteAt(Offset - 1), poisonByteAt(Offset)); - - // The diagnostic list is capped but the count is not, so the two have to be - // checked against a buffer with more offenders than the cap. A 2x3 uint32 - // matrix at a 16 byte offset with a 16 byte stride occupies bytes 16..27 and - // 32..43, leaving twenty bytes outside the elements. - const MatrixBufferLayout PaddedLayout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/16, - /*StrideBytes=*/16, - }; - std::optional PaddedSize = - getMatrixBufferSize(ComponentType::U32, 2, 3, PaddedLayout); - VERIFY_IS_TRUE(PaddedSize.has_value()); - VERIFY_ARE_EQUAL(size_t(44), *PaddedSize); - - std::vector AllPaddingTouched(*PaddedSize); - fillPoison(AllPaddingTouched.data(), AllPaddingTouched.size()); - for (BYTE &Byte : AllPaddingTouched) - Byte = static_cast(~Byte); - std::vector ReportedOffsets; - std::optional AfterPadded = countTouchedBytesOutsideElements( - ComponentType::U32, 2, 3, PaddedLayout, AllPaddingTouched.data(), - AllPaddingTouched.size(), &ReportedOffsets); - VERIFY_IS_TRUE(AfterPadded.has_value()); - VERIFY_ARE_EQUAL(size_t(20), *AfterPadded); - VERIFY_ARE_EQUAL(size_t(8), ReportedOffsets.size()); - for (size_t I = 0; I < ReportedOffsets.size(); ++I) - VERIFY_ARE_EQUAL(I, ReportedOffsets[I]); - - // Below the cap every offender is reported, and by its offset in the buffer - // rather than its position among the offenders. - std::vector TwoPaddingBytes(*PaddedSize); - fillPoison(TwoPaddingBytes.data(), TwoPaddingBytes.size()); - TwoPaddingBytes[28] ^= 0xff; - TwoPaddingBytes[29] ^= 0xff; - std::vector TwoOffsets; - std::optional AfterTwo = countTouchedBytesOutsideElements( - ComponentType::U32, 2, 3, PaddedLayout, TwoPaddingBytes.data(), - TwoPaddingBytes.size(), &TwoOffsets); - VERIFY_IS_TRUE(AfterTwo.has_value()); - VERIFY_ARE_EQUAL(size_t(2), *AfterTwo); - VERIFY_ARE_EQUAL(size_t(2), TwoOffsets.size()); - VERIFY_ARE_EQUAL(size_t(28), TwoOffsets[0]); - VERIFY_ARE_EQUAL(size_t(29), TwoOffsets[1]); - - // A buffer too small for the layout cannot be checked at all. - std::vector TooSmall(*Size - 1); - fillPoison(TooSmall.data(), TooSmall.size()); - VERIFY_IS_FALSE(CountTouched(TooSmall).has_value()); -} - -// The per-element arm of the bounds-checking rule is derived on the host, so -// the boundary it draws is checked here rather than only through a GPU round -// trip, where a wrong boundary and a wrong implementation would be -// indistinguishable. -void LinAlgCPUOracleTests::ViewBoundedElements() { - using namespace cpu_oracle; - - // A 2x3 uint32 matrix at a 4 byte offset with a 16 byte stride puts its - // elements at bytes 4, 8, 12, 20, 24 and 28, each 4 bytes wide, so they end - // at 8, 12, 16, 24, 28 and 32. - std::optional Matrix = - makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); - VERIFY_IS_TRUE(Matrix.has_value()); - - const MatrixBufferLayout Layout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/4, - /*StrideBytes=*/16, - }; - - auto BoundedEquals = [&](size_t ViewBytes, - const std::vector &Expected) { - std::optional Bounded = - zeroElementsOutsideView(*Matrix, Layout, ViewBytes); - std::optional Want = makeTypedMatrix(2, 3, Expected); - if (!Bounded || !Want) - return false; - size_t FirstMismatch; - return exactMatrixMatch(*Bounded, *Want, FirstMismatch); - }; - - // A view covering the whole buffer changes nothing, and an empty view - // zeroes everything. - VERIFY_IS_TRUE(BoundedEquals(32, {1, 2, 3, 4, 5, 6})); - VERIFY_IS_TRUE(BoundedEquals(0, {0, 0, 0, 0, 0, 0})); - - // A view ending at 24 admits the element that ends exactly there and - // excludes the rest, which is the inclusive end the rule requires. - VERIFY_IS_TRUE(BoundedEquals(24, {1, 2, 3, 4, 0, 0})); - - // One byte short of that boundary drops the straddling element whole. An - // element is either wholly inside the view or it is not there at all. - VERIFY_IS_TRUE(BoundedEquals(23, {1, 2, 3, 0, 0, 0})); - - // The gap between the rows is not addressable, so a view that reaches into - // the padding admits no further elements. - VERIFY_IS_TRUE(BoundedEquals(19, {1, 2, 3, 0, 0, 0})); - - // A view sized past the buffer cannot admit more than the buffer holds. - VERIFY_IS_TRUE(BoundedEquals(1024, {1, 2, 3, 4, 5, 6})); -} - -// The store side draws the same boundary but leaves the excluded elements -// holding poison rather than zero, so it is checked here as bytes. -void LinAlgCPUOracleTests::ViewBoundedStoreBytes() { - using namespace cpu_oracle; - - // The layout ViewBoundedElements uses: elements at bytes 4, 8, 12, 20, 24 - // and 28, each 4 bytes wide, in a 32 byte buffer. - std::optional Matrix = - makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); - VERIFY_IS_TRUE(Matrix.has_value()); - - const MatrixBufferLayout Layout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/4, - /*StrideBytes=*/16, - }; - static constexpr size_t ElementOffsets[] = {4, 8, 12, 20, 24, 28}; - - // Bit I is set when element I holds its value. Every element must hold - // either that or the poison a rejected store leaves behind, so an oracle - // that zeroed the rejected elements instead fails here rather than - // reporting them as merely unwritten. - auto WrittenMask = [&](size_t ViewBytes) { - std::optional> Buffer = - storeBufferBoundedByView(*Matrix, Layout, ViewBytes); - VERIFY_IS_TRUE(Buffer.has_value()); - VERIFY_ARE_EQUAL(Buffer->size(), static_cast(32)); - unsigned Mask = 0; - for (unsigned I = 0; I < 6; ++I) { - const size_t Offset = ElementOffsets[I]; - const uint32_t Value = I + 1; - BYTE Written[sizeof(Value)]; - memcpy(Written, &Value, sizeof(Value)); - bool HoldsValue = true; - bool HoldsPoison = true; - for (size_t B = 0; B < sizeof(Value); ++B) { - if ((*Buffer)[Offset + B] != Written[B]) - HoldsValue = false; - if ((*Buffer)[Offset + B] != poisonByteAt(Offset + B)) - HoldsPoison = false; - } - VERIFY_IS_TRUE(HoldsValue || HoldsPoison, - "A view bounded store element held neither its value nor " - "the poison it was seeded with"); - if (HoldsValue) - Mask |= 1u << I; + case ComponentType::F16: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + const HLSLHalf_t Half(static_cast(Value)); + if (static_cast(Half) != static_cast(Value)) + return std::nullopt; + Native.push_back(Half); } - return Mask; - }; - - // A view covering the whole buffer writes everything, and an empty view - // drops the whole store, so it writes nothing. - VERIFY_ARE_EQUAL(WrittenMask(32), 0x3fu); - VERIFY_ARE_EQUAL(WrittenMask(0), 0x00u); - - // A view ending at 24 admits the element that ends exactly there. - VERIFY_ARE_EQUAL(WrittenMask(24), 0x0fu); - - // One byte short of that boundary drops the straddling element whole, - // including the part of it the view does reach. - VERIFY_ARE_EQUAL(WrittenMask(23), 0x07u); - - // The prologue before the offset and the padding between rows belong to no - // element, so a full store must leave both holding poison. - std::optional> Full = - storeBufferBoundedByView(*Matrix, Layout, 32); - VERIFY_IS_TRUE(Full.has_value()); - std::optional Corrupted = countTouchedBytesOutsideElements( - ComponentType::U32, 2, 3, Layout, Full->data(), Full->size()); - VERIFY_IS_TRUE(Corrupted.has_value()); - VERIFY_ARE_EQUAL(*Corrupted, static_cast(0)); -} - -class LinAlgCapabilityTests { -public: - BEGIN_TEST_CLASS(LinAlgCapabilityTests) - TEST_METHOD_PROPERTY(L"Priority", L"0") - END_TEST_CLASS() - - TEST_METHOD(CapabilityPolicyAndPredicates); -}; - -void LinAlgCapabilityTests::CapabilityPolicyAndPredicates() { - using namespace linalg_test; - - VERIFY_IS_TRUE( - classifyApplicability(S_OK, true, CapabilityRequirement::Mandatory) == - Applicability::Execute); - VERIFY_IS_TRUE(classifyApplicability( - S_OK, false, CapabilityRequirement::CapabilityGated) == - Applicability::NotApplicable); - VERIFY_IS_TRUE( - classifyApplicability(S_OK, false, CapabilityRequirement::Mandatory) == - Applicability::Fail); - VERIFY_IS_TRUE( - classifyApplicability(E_UNEXPECTED, true, - CapabilityRequirement::CapabilityGated) == - Applicability::Fail); - - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, - MatrixScope::Wave)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, - MatrixScope::ThreadGroup)); - VERIFY_IS_FALSE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, - MatrixScope::Thread)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_WAVE_MATRIX_MULTIPLY, - MatrixScope::Wave)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREADGROUP_MATRIX_MULTIPLY, - MatrixScope::ThreadGroup)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_VECTOR_MATRIX_MULTIPLY, - MatrixScope::Thread)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_OUTER_PRODUCT, - MatrixScope::Thread)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE, - MatrixScope::Thread)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE, - MatrixScope::Wave)); - VERIFY_IS_TRUE(isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE, - MatrixScope::ThreadGroup)); - - MatrixConstructionSupport Construction = {TRUE}; - VERIFY_IS_TRUE(Construction.valid()); - VERIFY_IS_TRUE(Construction.supported()); - MatrixConstructionSupport UnsupportedConstruction = {FALSE}; - VERIFY_IS_TRUE(UnsupportedConstruction.valid()); - VERIFY_IS_FALSE(UnsupportedConstruction.supported()); - // The runtime contract is a canonical BOOL; anything else is a driver bug. - MatrixConstructionSupport InvalidConstruction = {2}; - VERIFY_IS_FALSE(InvalidConstruction.valid()); - VERIFY_IS_FALSE(InvalidConstruction.supported()); - - WaveMatrixMultiplySupport Wave = { - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED}; - VERIFY_IS_TRUE(Wave.valid()); - VERIFY_IS_TRUE(Wave.supported()); - WaveMatrixMultiplySupport UnsupportedWave = { - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_NONE}; - VERIFY_IS_TRUE(UnsupportedWave.valid()); - VERIFY_IS_FALSE(UnsupportedWave.supported()); - WaveMatrixMultiplySupport InvalidWave = { - static_cast< - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS)), - }; - VERIFY_IS_FALSE(InvalidWave.valid()); - - ThreadGroupMatrixMultiplySupport ThreadGroup = { - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED, - 32, - 128, - 64, - }; - VERIFY_IS_TRUE(ThreadGroup.valid()); - VERIFY_IS_TRUE(ThreadGroup.supportsThreadGroupSize(64)); - VERIFY_IS_FALSE(ThreadGroup.supportsThreadGroupSize(48)); - ThreadGroup.PreferredThreadGroupSize = 48; - VERIFY_IS_FALSE(ThreadGroup.valid()); - ThreadGroup = { - static_cast< - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_TRANSPOSE)), - 32, - 128, - 64, - }; - VERIFY_IS_FALSE(ThreadGroup.valid()); - - ThreadVectorMatrixMultiplySupport ThreadVector = { - static_cast< - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_TRANSPOSE)), - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32, - }; - VERIFY_IS_TRUE(ThreadVector.valid()); - VERIFY_IS_TRUE(ThreadVector.supported()); - ThreadVector.SupportFlags = static_cast< - linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | - static_cast( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS)); - VERIFY_IS_FALSE(ThreadVector.valid()); - ThreadVector.MatrixInputType = - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT8_E4M3FN; - VERIFY_IS_TRUE(ThreadVector.valid()); - ThreadVector.SupportFlags = linalg_abi:: - D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS; - VERIFY_IS_FALSE(ThreadVector.valid()); - - ThreadOuterProductSupport OuterProduct = {true}; - VERIFY_IS_TRUE(OuterProduct.supported()); - AtomicAccumulateStoreSupport Atomic = {true, false}; - VERIFY_IS_TRUE(Atomic.supports(AtomicDestination::RWByteAddressBuffer)); - VERIFY_IS_FALSE(Atomic.supports(AtomicDestination::GroupShared)); - - VERIFY_ARE_EQUAL( - 0u, static_cast(linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE)); - MatrixConstructionQuery ConstructionQuery = { - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32, 32, {8, 8, 8}}; - WaveMatrixMultiplyInputs WaveInputs = { - 32, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32, - }; - WaveMatrixMultiplyQuery WaveQuery = {WaveInputs, {16, 16, 16}}; - ThreadGroupMatrixMultiplyQuery ThreadGroupQuery = { - WaveInputs, - {16, 16, 16}, - }; - ThreadVectorMatrixMultiplyQuery ThreadVectorQuery = { - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - }; - ThreadOuterProductQuery OuterProductQuery = { - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, - }; - AtomicAccumulateStoreQuery AtomicQuery = { - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16}; - VERIFY_ARE_EQUAL(32u, ConstructionQuery.WaveSize); - VERIFY_ARE_EQUAL(8u, ConstructionQuery.Shape.K); - VERIFY_ARE_EQUAL(32u, WaveQuery.Inputs.WaveSize); - VERIFY_ARE_EQUAL(16u, WaveQuery.Shape.M); - VERIFY_ARE_EQUAL(32u, ThreadGroupQuery.WaveInputs.WaveSize); - VERIFY_ARE_EQUAL(16u, ThreadGroupQuery.Shape.M); - VERIFY_IS_TRUE(ThreadVectorQuery.BiasInputType == - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE); - VERIFY_IS_TRUE(OuterProductQuery.InputComponentType == - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16); - VERIFY_IS_TRUE(AtomicQuery.ComponentType == - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16); + return encodeNativeVector(Native); + } + case ComponentType::F32: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + const float FloatValue = static_cast(Value); + if (static_cast(FloatValue) != Value) + return std::nullopt; + Native.push_back(FloatValue); + } + return encodeNativeVector(Native); + } + case ComponentType::I32: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return std::nullopt; + Native.push_back(static_cast(Value)); + } + return encodeNativeVector(Native); + } + case ComponentType::U32: { + std::vector Native; + Native.reserve(Values.size()); + for (int64_t Value : Values) { + if (Value < 0 || + static_cast(Value) > std::numeric_limits::max()) + return std::nullopt; + Native.push_back(static_cast(Value)); + } + return encodeNativeVector(Native); + } + default: + return std::nullopt; + } } -class DxilConf_SM610_LinAlg { -public: - BEGIN_TEST_CLASS(DxilConf_SM610_LinAlg) - TEST_CLASS_PROPERTY("Kits.TestName", - "D3D12 - Shader Model 6.10 - LinAlg Matrix Operations") - TEST_CLASS_PROPERTY("Kits.TestId", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") - TEST_CLASS_PROPERTY( - "Kits.Description", - "Validates SM 6.10 linear algebra matrix operations execute correctly") - TEST_CLASS_PROPERTY( - "Kits.Specification", - "Device.Graphics.D3D12.DXILCore.ShaderModel610.CoreRequirement") - TEST_METHOD_PROPERTY(L"Priority", L"0") - END_TEST_CLASS() - - TEST_CLASS_SETUP(setupClass); - TEST_METHOD_SETUP(setupMethod); - - // Load/Store/Accumulate Descriptor - TEST_METHOD(LoadStoreDescriptor_Wave_16x16_F16); - TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded); - TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor); - TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor); - TEST_METHOD(LoadDescriptorOOB_Wave_16x16_F16_PartialView); - TEST_METHOD(LoadDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView); - TEST_METHOD(StoreDescriptorOOB_Wave_16x16_F16_PartialView); - TEST_METHOD(StoreDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView); - TEST_METHOD(SplatStore_Wave_16x16_F16); - TEST_METHOD(AccumulateDescriptor_Wave_16x16_F16); - - // Load/Store/Accumulate Memory - TEST_METHOD(LoadMemory_Wave_16x16_F16); - TEST_METHOD(StoreMemory_Wave_16x16_F16); - TEST_METHOD(AccumulateMemory_Wave_16x16_F16); - - // Element access - TEST_METHOD(ElementAccess_Wave_16x16_F16); - TEST_METHOD(ElementAccess_Wave_4x8_F32); - TEST_METHOD(ElementSet_Wave_16x16_F16); - TEST_METHOD(ElementGetOOB_Wave_4x8_F32); - TEST_METHOD(ElementSetOOB_Wave_4x8_F32); - TEST_METHOD(ElementGetOOB_Wave_16x16_F16); - TEST_METHOD(ElementSetOOB_Wave_16x16_F16); - - // Cast/Convert - TEST_METHOD(CopyConvert_Wave_16x16_F16); - TEST_METHOD(CopyConvert_Wave_16x16_F16_Transpose); - TEST_METHOD(CopyConvert_Wave_4x8_F32_Transpose); - - // Matrix Matrix Arithmetic - TEST_METHOD(MatMatMul_Wave_16x16x16_F16); - TEST_METHOD(MatMatMulAccum_Wave_16x16x16_F16); - TEST_METHOD(MatAccum_Wave_16x16_F16); - - // Matrix Vector Arithmetic - TEST_METHOD(MatVecMul_Thread_16x16_F16); - TEST_METHOD(MatVecMul_Thread_4x8_F32); - TEST_METHOD(MatVecMul_Thread_4x8_F16_NonUniform); - TEST_METHOD(MatVecMul_Thread_4x8_F16_ColumnMajor); - TEST_METHOD(MatVecMul_Thread_4x8_I8_Interpreted); - TEST_METHOD(MatVecMul_Thread_4x8_U8_Interpreted); - TEST_METHOD(MatVecMul_Thread_4x8_U32_UnsignedOutput); - TEST_METHOD(MatVecMulAdd_Thread_16x16_F16); - TEST_METHOD(MatVecMulAdd_Thread_4x8_F32); - TEST_METHOD(MatVecMulAdd_Thread_4x8_F16_IndependentBias); - TEST_METHOD(OuterProduct_Thread_16x16_F16); - - // Query Accumulator Layout - TEST_METHOD(QueryAccumLayout); - - // Convert - TEST_METHOD(Convert); - - // CopyConvert / Convert coverage - TEST_METHOD(CopyConvert_Wave_4x8_F16_ToF32); - TEST_METHOD(CopyConvert_Wave_4x8_F32_ToF16_Transpose); - TEST_METHOD(Convert_I16_ToI32_Exact); - TEST_METHOD(Convert_F32_ToI16_RTNE_Saturate); - - // Vector Accumulate - TEST_METHOD(VectorAccumulateDescriptor_Thread_F16); - -private: - CComPtr D3DDevice; - dxc::SpecificDllLoader DxcSupport; - bool VerboseLogging = false; - bool Initialized = false; - std::optional D3D12SDK; - - WEX::TestExecution::SetVerifyOutput VerifyOutput{ - WEX::TestExecution::VerifyOutputSettings::LogOnlyFailures}; -}; +static std::optional> +encodePackedVector(ComponentType Type, const std::vector &Values) { + if (!isPackedByteVector(Type)) + return std::nullopt; -bool DxilConf_SM610_LinAlg::setupClass() { - if (!Initialized) { - Initialized = true; - VERIFY_SUCCEEDED( - DxcSupport.InitializeForDll(dxc::kDxCompilerLib, "DxcCreateInstance")); - D3D12SDK = D3D12SDKSelector(); - WEX::TestExecution::RuntimeParameters::TryGetValue(L"VerboseLogging", - VerboseLogging); + size_t PaddedCount; + if (!cpu_oracle::checkedAdd(Values.size(), size_t(3), PaddedCount)) + return std::nullopt; + PaddedCount &= ~size_t(3); + std::vector Bytes(PaddedCount, 0); - if (!D3D12SDK->createDevice(&D3DDevice, D3D_SHADER_MODEL_6_10, false)) { -#ifdef _HLK_CONF - hlsl_test::LogErrorFmt( - L"Device creation failed. Expected a driver supporting SM6.10"); -#else - hlsl_test::LogWarningFmt( - L"Device creation failed. Expected a driver supporting SM6.10"); - WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); -#endif - return false; + for (size_t WordIndex = 0; WordIndex < PaddedCount / 4; ++WordIndex) { + uint32_t Word = 0; + for (size_t Lane = 0; Lane < 4; ++Lane) { + const size_t ValueIndex = WordIndex * 4 + Lane; + if (ValueIndex == Values.size()) + break; + std::optional Encoded = encodeByte(Type, Values[ValueIndex]); + if (!Encoded) + return std::nullopt; + // Lane zero occupies the least-significant byte of each uint. + Word |= static_cast(*Encoded) << (Lane * 8); } + for (size_t ByteIndex = 0; ByteIndex < 4; ++ByteIndex) + Bytes[WordIndex * 4 + ByteIndex] = + static_cast(Word >> (ByteIndex * 8)); } - - return true; + return Bytes; } -bool DxilConf_SM610_LinAlg::setupMethod() { - // If the device is healthy, exit otherwise it's possible a previous test - // case caused a device removal. So we need to try and create a new device. - if (D3DDevice && D3DDevice->GetDeviceRemovedReason() == S_OK) - return true; +static std::optional matrixStrideBytes(const CaseData &Case) { + const std::optional ComponentSize = + componentByteSize(Case.MatrixType); + if (!ComponentSize) + return std::nullopt; + const size_t MinorCount = + Case.Layout == MatrixLayout::RowMajor ? Case.N : Case.M; + size_t Stride; + if (!cpu_oracle::checkedMultiply(MinorCount, *ComponentSize, Stride)) + return std::nullopt; + return Stride; +} - hlsl_test::LogCommentFmt(L"Device was lost!"); - D3DDevice.Release(); +static std::optional> +encodeMatrixBuffer(const CaseData &Case) { + const std::optional ComponentSize = + componentByteSize(Case.MatrixType); + const std::optional Stride = matrixStrideBytes(Case); + const std::optional> Logical = + encodeComponents(Case.MatrixType, Case.MatrixValues); + if (!ComponentSize || !Stride || !Logical) + return std::nullopt; - hlsl_test::LogCommentFmt(L"Recreating device"); + const size_t MajorCount = + Case.Layout == MatrixLayout::RowMajor ? Case.M : Case.N; + size_t BufferSize; + if (!cpu_oracle::checkedMultiply(MajorCount, *Stride, BufferSize)) + return std::nullopt; + std::vector Buffer(BufferSize, 0); - return D3D12SDK->createDevice(&D3DDevice, D3D_SHADER_MODEL_6_10, false); + for (MatrixDim Row = 0; Row < Case.M; ++Row) { + for (MatrixDim Column = 0; Column < Case.N; ++Column) { + const size_t SourceIndex = static_cast(Row) * Case.N + Column; + const size_t SourceOffset = SourceIndex * *ComponentSize; + const size_t DestinationOffset = + Case.Layout == MatrixLayout::RowMajor + ? static_cast(Row) * *Stride + Column * *ComponentSize + : static_cast(Column) * *Stride + Row * *ComponentSize; + std::memcpy(Buffer.data() + DestinationOffset, + Logical->data() + SourceOffset, *ComponentSize); + } + } + return Buffer; } -// The alignment the descriptor shader declares to both builtins. Proposal 0035 -// requires the first element's address -- the resource base plus the offset -- -// to meet it. -static constexpr size_t DescriptorDeclaredAlignment = 128; - -static const char LoadStoreDescriptorShader[] = R"( - RWByteAddressBuffer Input : register(u0); - RWByteAddressBuffer Output : register(u1); - - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main() { - if (GetGroupWaveIndex() != 0) - return; +static std::optional> +calculateExpected(const CaseData &Case) { + size_t MatrixElementCount; + if (!cpu_oracle::checkedMultiply(static_cast(Case.M), + static_cast(Case.N), + MatrixElementCount) || + Case.MatrixValues.size() != MatrixElementCount || + Case.InterpretedVectorValues.size() != Case.N || + (Case.hasBias() && Case.BiasValues.size() != Case.M)) + return std::nullopt; - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, LOAD_OFFSET, LOAD_STRIDE, LOAD_LAYOUT, DECLARED_ALIGN); - __builtin_LinAlg_MatrixStoreToDescriptor( - Mat, Output, STORE_OFFSET, STORE_STRIDE, STORE_LAYOUT, DECLARED_ALIGN); + std::vector Expected(Case.M, 0); + for (MatrixDim Row = 0; Row < Case.M; ++Row) { + for (MatrixDim Column = 0; Column < Case.N; ++Column) { + int64_t Product; + int64_t Sum; + if (!checkedMultiplyInt64( + Case.MatrixValues[static_cast(Row) * Case.N + Column], + Case.InterpretedVectorValues[Column], Product) || + !checkedAddInt64(Expected[Row], Product, Sum)) + return std::nullopt; + Expected[Row] = Sum; + } + if (Case.hasBias()) { + int64_t Sum; + if (!checkedAddInt64(Expected[Row], Case.BiasValues[Row], Sum)) + return std::nullopt; + Expected[Row] = Sum; + } } -)"; + return Expected; +} -// The base is a runtime property that no compile-time check can see, so check -// it against the real GPU address. -static void verifyDescriptorBaseAlignment(st::ShaderOpTest *Test, LPCSTR Name, - size_t OffsetBytes) { - // GetResource hands back a borrowed pointer without an AddRef. - ID3D12Resource *Resource = nullptr; - Test->GetResource(Name, &Resource); - VERIFY_IS_NOT_NULL(Resource); +static bool oracleSelfTest() { + const std::optional> PackedSInt8 = + encodePackedVector(ComponentType::I8, {-1, 2, -3, 4, 5}); + const std::optional> PackedUInt8 = + encodePackedVector(ComponentType::U8, {255, 2, 253, 4, 5}); + const std::vector PackedBytes = {0xff, 0x02, 0xfd, 0x04, + 0x05, 0x00, 0x00, 0x00}; - const UINT64 ElementAddress = Resource->GetGPUVirtualAddress() + OffsetBytes; - VERIFY_IS_TRUE(ElementAddress % DescriptorDeclaredAlignment == 0, - "Descriptor buffer's first element does not meet the " - "alignment the shader declares"); -} + CaseData DotCase = {}; + DotCase.M = 2; + DotCase.N = 3; + DotCase.MatrixValues = {1, 2, 3, -1, 4, 0}; + DotCase.InterpretedVectorValues = {4, -2, 5}; + DotCase.BiasInputType = ComponentType::I32; + DotCase.BiasValues = {7, -3}; + const std::optional> Dot = calculateExpected(DotCase); -static void -runLoadStoreDescriptor(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, - const cpu_oracle::MatrixBufferLayout &LoadLayout, - const cpu_oracle::MatrixBufferLayout &StoreLayout, - bool Verbose, UINT ForcedWaveSize = 0) { - std::optional Input = - cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); - VERIFY_IS_TRUE(Input.has_value(), - "Unable to construct typed LoadStoreDescriptor input"); + int64_t Ignored; + return PackedSInt8 == PackedBytes && PackedUInt8 == PackedBytes && Dot && + *Dot == std::vector({22, -15}) && + !checkedMultiplyInt64(std::numeric_limits::max(), 2, + Ignored) && + !checkedAddInt64(std::numeric_limits::max(), 1, Ignored); +} - std::optional InputSize = - cpu_oracle::getMatrixBufferSize(*Input, LoadLayout); - std::optional OutputSize = - cpu_oracle::getMatrixBufferSize(*Input, StoreLayout); - VERIFY_IS_TRUE(InputSize.has_value() && OutputSize.has_value(), - "Unable to size the LoadStoreDescriptor buffers"); +static bool isCaseValid(const CaseData &Case) { + size_t MatrixElementCount; + if (Case.M == 0 || Case.N == 0 || + !cpu_oracle::checkedMultiply(static_cast(Case.M), + static_cast(Case.N), + MatrixElementCount) || + Case.MatrixValues.size() != MatrixElementCount || + Case.InterpretedVectorValues.size() != Case.N || + (Case.Layout != MatrixLayout::RowMajor && + Case.Layout != MatrixLayout::ColumnMajor) || + !componentByteSize(Case.MatrixType) || + !storageTypeName(Case.VectorInputType) || + !storageTypeName(Case.ResultType) || Case.PublicRule.empty()) + return false; - std::stringstream ExtraDefs; - ExtraDefs << " -DLOAD_OFFSET=" << LoadLayout.OffsetBytes; - ExtraDefs << " -DLOAD_STRIDE=" << LoadLayout.StrideBytes; - ExtraDefs << " -DLOAD_LAYOUT=" << static_cast(LoadLayout.Layout); - ExtraDefs << " -DSTORE_OFFSET=" << StoreLayout.OffsetBytes; - ExtraDefs << " -DSTORE_STRIDE=" << StoreLayout.StrideBytes; - ExtraDefs << " -DSTORE_LAYOUT=" << static_cast(StoreLayout.Layout); - ExtraDefs << " -DDECLARED_ALIGN=" << DescriptorDeclaredAlignment; + // A vector is either native or an InterpretedVector, which pairs a packed + // vector with an interpretation type. A native element type paired with a + // narrower interpretation is not a valid form. + if (Case.VectorInputType == ComponentType::F32 && + Case.InputInterpretation != ComponentType::F32) + return false; + if (isPackedByteVector(Case.VectorInputType) && + Case.InputInterpretation != Case.VectorInputType) + return false; + if (Case.hasBias() != !Case.BiasValues.empty() || + (Case.hasBias() && (Case.BiasValues.size() != Case.M || + Case.BiasInputType != Case.ResultType || + !storageTypeName(Case.BiasInputType)))) + return false; - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + const bool ExpectedSigned = Case.ResultType != ComponentType::U32; + return Case.OutputSigned == ExpectedSigned; +} - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); +static std::optional> +encodeVectorBuffer(const CaseData &Case) { + if (isPackedByteVector(Case.VectorInputType)) + return encodePackedVector(Case.VectorInputType, + Case.InterpretedVectorValues); + return encodeComponents(Case.VectorInputType, Case.InterpretedVectorValues); +} - compileShader(DxcSupport, LoadStoreDescriptorShader, "cs_6_10", Args, - Verbose); +static std::optional> +encodeExpectedOutput(const CaseData &Case) { + const std::optional> Values = calculateExpected(Case); + if (!Values) + return std::nullopt; + const std::optional> Logical = + encodeComponents(Case.ResultType, *Values); + if (!Logical) + return std::nullopt; - const cpu_oracle::TypedMatrix InputMatrix = *Input; - cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::exactResult( - InputMatrix, L"HLSL proposal 0035 MatrixLoadFromDescriptor and " - L"MatrixStoreToDescriptor " - L"round trip at the requested offset, stride and layout"); + size_t PaddedSize; + if (!cpu_oracle::checkedAdd(Logical->size(), size_t(3), PaddedSize)) + return std::nullopt; + PaddedSize &= ~size_t(3); + size_t BufferSize; + if (!cpu_oracle::checkedAdd(PaddedSize, OutputGuardBytes, BufferSize)) + return std::nullopt; - // Two UAV buffers, load from one, store to the other. The destination is - // filled by name rather than zeroed so unowned bytes carry the poison. - // - // Bound through a descriptor table rather than as root views. A root view is - // a bare GPU address, and proposal 0035 exempts root descriptors from bounds - // checking precisely because they carry no dimensions. Binding through a - // heap gives each buffer a view whose extent the implementation can see. - auto Op = createComputeOp(LoadStoreDescriptorShader, "cs_6_10", - "DescriptorTable(UAV(u0), UAV(u1))", Args.c_str()); - addUAVBuffer(Op.get(), "Input", *InputSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", *OutputSize, true, "byname"); - addHeapRawUAV(Op.get(), "ResHeap", "Input", *InputSize); - addHeapRawUAV(Op.get(), "ResHeap", "Output", *OutputSize); - addRootTable(Op.get(), 0, "ResHeap"); + std::vector Buffer(BufferSize); + cpu_oracle::fillPoison(Buffer.data(), Buffer.size()); + std::memcpy(Buffer.data(), Logical->data(), Logical->size()); + return Buffer; +} - auto Result = runShaderOp( - Device, DxcSupport, std::move(Op), - [InputMatrix, LoadLayout](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - cpu_oracle::fillPoison(Data.data(), Data.size()); - if (_stricmp(Name, "Input") != 0) - return; - VERIFY_IS_TRUE( - cpu_oracle::writeMatrixBuffer(InputMatrix, LoadLayout, Data), - "Unable to encode typed LoadStoreDescriptor input"); - }, - [LoadLayout, StoreLayout](ID3D12GraphicsCommandList *, - st::ShaderOpTest *Test) { - verifyDescriptorBaseAlignment(Test, "Input", LoadLayout.OffsetBytes); - verifyDescriptorBaseAlignment(Test, "Output", StoreLayout.OffsetBytes); - }); +static bool needs16BitTypes(ComponentType Type) { + return Type == ComponentType::F16 || Type == ComponentType::I16 || + Type == ComponentType::U16; +} - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); +static std::optional buildCompilerArgs(const CaseData &Case) { + const std::optional MatrixStride = matrixStrideBytes(Case); + const char *InputStorageType = storageTypeName(Case.VectorInputType); + const char *OutputType = storageTypeName(Case.ResultType); + const char *BiasStorageType = + Case.hasBias() ? storageTypeName(Case.BiasInputType) : nullptr; + if (!MatrixStride || !InputStorageType || !OutputType || + (Case.hasBias() && !BiasStorageType)) + return std::nullopt; - VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer(OutData.data(), OutData.size(), - StoreLayout, Oracle, Verbose)); - VERIFY_IS_TRUE(cpu_oracle::verifyUntouchedBytes( - Params.CompType, Params.M, Params.N, StoreLayout, OutData.data(), - OutData.size(), Verbose)); + std::stringstream Args; + Args << "-HV 202x"; + Args << " -DMATRIX_COMP_TYPE=" << static_cast(Case.MatrixType); + Args << " -DM_DIM=" << Case.M; + Args << " -DN_DIM=" << Case.N; + Args << " -DMATRIX_STRIDE=" << *MatrixStride; + Args << " -DMATRIX_LAYOUT=" << static_cast(Case.Layout); + Args << " -DINPUT_STORAGE_TYPE=" << InputStorageType; + Args << " -DINPUT_STORAGE_COUNT=" + << storageElementCount(Case.VectorInputType, Case.N); + Args << " -DINPUT_STORAGE_SIZE=" + << storageElementByteSize(Case.VectorInputType); + Args << " -DINPUT_INTERP=" << static_cast(Case.InputInterpretation); + Args << " -DOUTPUT_TYPE=" << OutputType; + Args << " -DOUTPUT_SIZE=" << componentByteSize(Case.ResultType).value_or(0); + Args << " -DOUTPUT_SIGNED=" << (Case.OutputSigned ? 1 : 0); + if (Case.hasBias()) { + Args << " -DBIAS_STORAGE_TYPE=" << BiasStorageType; + Args << " -DBIAS_STORAGE_COUNT=" + << storageElementCount(Case.BiasInputType, Case.M); + Args << " -DBIAS_STORAGE_SIZE=" + << storageElementByteSize(Case.BiasInputType); + } + if (needs16BitTypes(Case.MatrixType) || + needs16BitTypes(Case.VectorInputType) || + needs16BitTypes(Case.BiasInputType) || needs16BitTypes(Case.ResultType)) + Args << " -enable-16bit-types"; + return Args.str(); } -// Proposal 0035 permits two bounds-checking behaviors. An implementation may -// zero the whole matrix when any element falls outside the view, or zero only -// the elements that do, and both are conformant. A single expected buffer -// would therefore be wrong by construction, so the oracle carries both -// outcomes and accepts a complete match against either one. -// -// What makes the test discriminating is that the source buffer is allocated -// and written in full and only its *view* is shortened, so the bytes past the -// view hold real matrix data rather than zeros. An implementation that does no -// bounds checking at all reads that data back and matches neither candidate. -static void runLoadDescriptorOutOfBounds( - ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, const cpu_oracle::MatrixBufferLayout &Layout, - size_t InputViewBytes, bool Verbose, UINT ForcedWaveSize = 0) { - std::optional Input = - cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); - VERIFY_IS_TRUE(Input.has_value(), - "Unable to construct typed LoadDescriptorOOB input"); +static bool verifyExactBuffer(const void *ActualBuffer, size_t ActualSize, + const std::vector &Expected, bool Verbose) { + if (ActualSize != Expected.size()) { + hlsl_test::LogErrorFmt( + L"MatVec output size mismatch: actual=%zu, expected=%zu", ActualSize, + Expected.size()); + return false; + } + + const BYTE *Actual = static_cast(ActualBuffer); + size_t MismatchCount = 0; + for (size_t I = 0; I < Expected.size(); ++I) { + if (Actual[I] == Expected[I]) + continue; + if (MismatchCount < 8) + hlsl_test::LogErrorFmt( + L"MatVec output byte %zu mismatch: actual=0x%02x, expected=0x%02x", I, + Actual[I], Expected[I]); + ++MismatchCount; + } + if (MismatchCount != 0) { + hlsl_test::LogErrorFmt(L"%zu MatVec output bytes differed", MismatchCount); + return false; + } + if (Verbose) + hlsl_test::LogCommentFmt( + L"All %zu MatVec output, padding, and guard bytes matched exactly", + Expected.size()); + return true; +} - std::optional BufferSize = - cpu_oracle::getMatrixBufferSize(*Input, Layout); - VERIFY_IS_TRUE(BufferSize.has_value(), - "Unable to size the LoadDescriptorOOB buffers"); - VERIFY_IS_TRUE(InputViewBytes < *BufferSize, - "The source view must be shorter than its buffer"); +static const char MatVecMulShader[] = R"( + #define USE_A 0 + #define SCOPE_THREAD 0 - std::optional PerElement = - cpu_oracle::zeroElementsOutsideView(*Input, Layout, InputViewBytes); - // The whole-matrix arm is the per-element arm with nothing in view. - std::optional WholeMatrix = - cpu_oracle::zeroElementsOutsideView(*Input, Layout, 0); - VERIFY_IS_TRUE(PerElement.has_value() && WholeMatrix.has_value(), - "Unable to derive the LoadDescriptorOOB candidates"); + ByteAddressBuffer MatrixInput : register(t0); + ByteAddressBuffer VectorInput : register(t1); + RWByteAddressBuffer Output : register(u2); - // The test requires both in-bounds and out-of-bounds elements. If none are - // in bounds, the two permitted results are identical. If all are in bounds, - // an implementation that performs no bounds checking would still pass. - size_t FirstMismatch; - const bool HasInBoundsElement = - !cpu_oracle::exactMatrixMatch(*PerElement, *WholeMatrix, FirstMismatch); - VERIFY_IS_TRUE(HasInBoundsElement, - "The source view must include at least one complete element"); + [numthreads(1, 1, 1)] + void main() { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes( + MATRIX_COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, 0, MATRIX_STRIDE, MATRIX_LAYOUT, 128); - const bool HasOutOfBoundsElement = - !cpu_oracle::exactMatrixMatch(*PerElement, *Input, FirstMismatch); - VERIFY_IS_TRUE(HasOutOfBoundsElement, - "The source view must exclude at least one element"); + vector InVec; + for (uint I = 0; I < INPUT_STORAGE_COUNT; ++I) { + InVec[I] = + VectorInput.Load(I * INPUT_STORAGE_SIZE); + } - std::stringstream ExtraDefs; - ExtraDefs << " -DLOAD_OFFSET=" << Layout.OffsetBytes; - ExtraDefs << " -DLOAD_STRIDE=" << Layout.StrideBytes; - ExtraDefs << " -DLOAD_LAYOUT=" << static_cast(Layout.Layout); - ExtraDefs << " -DSTORE_OFFSET=" << Layout.OffsetBytes; - ExtraDefs << " -DSTORE_STRIDE=" << Layout.StrideBytes; - ExtraDefs << " -DSTORE_LAYOUT=" << static_cast(Layout.Layout); - ExtraDefs << " -DDECLARED_ALIGN=" << DescriptorDeclaredAlignment; + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiply( + OutVec, Mat, OUTPUT_SIGNED, InVec, INPUT_INTERP); - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(I * OUTPUT_SIZE, OutVec[I]); + } + } +)"; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); +static const char MatVecMulAddShader[] = R"( + #define USE_A 0 + #define SCOPE_THREAD 0 - compileShader(DxcSupport, LoadStoreDescriptorShader, "cs_6_10", Args, - Verbose); + ByteAddressBuffer MatrixInput : register(t0); + ByteAddressBuffer VectorInput : register(t1); + ByteAddressBuffer BiasInput : register(t2); + RWByteAddressBuffer Output : register(u3); - cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::permittedResults( - {*PerElement, *WholeMatrix}, - L"HLSL proposal 0035 bounds checking on MatrixLoadFromDescriptor: " - L"either the whole matrix or only the out-of-view elements read as the " - L"default element value"); + [numthreads(1, 1, 1)] + void main() { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes( + MATRIX_COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, MatrixInput, 0, MATRIX_STRIDE, MATRIX_LAYOUT, 128); - // Only the source view is short. The destination is viewed in full so that - // the store cannot be bounds checked as well, which would leave the observed - // result attributable to either operation. - const cpu_oracle::TypedMatrix InputMatrix = *Input; - auto Op = createComputeOp(LoadStoreDescriptorShader, "cs_6_10", - "DescriptorTable(UAV(u0), UAV(u1))", Args.c_str()); - addUAVBuffer(Op.get(), "Input", *BufferSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", *BufferSize, true, "byname"); - addHeapRawUAV(Op.get(), "ResHeap", "Input", InputViewBytes); - addHeapRawUAV(Op.get(), "ResHeap", "Output", *BufferSize); - addRootTable(Op.get(), 0, "ResHeap"); + vector InVec; + for (uint I = 0; I < INPUT_STORAGE_COUNT; ++I) { + InVec[I] = + VectorInput.Load(I * INPUT_STORAGE_SIZE); + } - auto Result = runShaderOp( - Device, DxcSupport, std::move(Op), - [InputMatrix, Layout](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - cpu_oracle::fillPoison(Data.data(), Data.size()); - if (_stricmp(Name, "Input") != 0) - return; - // Written in full, including the part the view does not cover. - VERIFY_IS_TRUE(cpu_oracle::writeMatrixBuffer(InputMatrix, Layout, Data), - "Unable to encode typed LoadDescriptorOOB input"); - }, - [Layout](ID3D12GraphicsCommandList *, st::ShaderOpTest *Test) { - verifyDescriptorBaseAlignment(Test, "Input", Layout.OffsetBytes); - verifyDescriptorBaseAlignment(Test, "Output", Layout.OffsetBytes); - }); + vector BiasVec; + for (uint I = 0; I < BIAS_STORAGE_COUNT; ++I) { + BiasVec[I] = BiasInput.Load(I * BIAS_STORAGE_SIZE); + } - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiplyAdd( + OutVec, Mat, OUTPUT_SIGNED, InVec, INPUT_INTERP, BiasVec); - VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer(OutData.data(), OutData.size(), - Layout, Oracle, Verbose)); - VERIFY_IS_TRUE(cpu_oracle::verifyUntouchedBytes( - Params.CompType, Params.M, Params.N, Layout, OutData.data(), - OutData.size(), Verbose)); -} + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(I * OUTPUT_SIZE, OutVec[I]); + } + } +)"; -// Stores through a destination view shorter than its buffer. Both permitted -// outcomes leave the bytes past the view holding poison, so the comparison is -// byte level rather than matrix level. This cannot by itself fail an -// implementation that stores nothing, since dropping the whole store is one of -// those outcomes; the LoadStoreDescriptor cases require the store to happen. -static void runStoreDescriptorOutOfBounds( - ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, const cpu_oracle::MatrixBufferLayout &Layout, - size_t OutputViewBytes, bool Verbose, UINT ForcedWaveSize = 0) { - std::optional Input = - cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); - VERIFY_IS_TRUE(Input.has_value(), - "Unable to construct typed StoreDescriptorOOB input"); +static HRESULT querySupport(ID3D12Device *Device, const CaseData &Case, + bool &TierSupported, bool &Supported) { + TierSupported = false; + Supported = false; + if (!Device) + return E_INVALIDARG; - std::optional BufferSize = - cpu_oracle::getMatrixBufferSize(*Input, Layout); - VERIFY_IS_TRUE(BufferSize.has_value(), - "Unable to size the StoreDescriptorOOB buffers"); - VERIFY_IS_TRUE(OutputViewBytes < *BufferSize, - "The destination view must be shorter than its buffer"); + const std::optional VectorType = + toCapabilityDataType(Case.VectorInputType); + const std::optional MatrixType = + toCapabilityDataType(Case.MatrixType); + const std::optional BiasType = + Case.hasBias() ? toCapabilityDataType(Case.BiasInputType) + : std::optional( + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE); + const std::optional ResultType = + toCapabilityDataType(Case.ResultType); + if (!VectorType || !MatrixType || !BiasType || !ResultType) + return E_INVALIDARG; - std::optional> PerElement = - cpu_oracle::storeBufferBoundedByView(*Input, Layout, OutputViewBytes); - std::optional> WholeStore = - cpu_oracle::storeBufferBoundedByView(*Input, Layout, 0); - std::optional> Unbounded = - cpu_oracle::storeBufferBoundedByView(*Input, Layout, *BufferSize); - VERIFY_IS_TRUE(PerElement.has_value() && WholeStore.has_value() && - Unbounded.has_value(), - "Unable to derive the StoreDescriptorOOB candidates"); + linalg_test::TierSupport Tier; + HRESULT HR = linalg_test::queryTierSupport(Device, Tier); + if (FAILED(HR)) + return HR; + TierSupported = Tier.supported(); + if (!TierSupported) + return S_OK; - VERIFY_IS_TRUE(*PerElement != *WholeStore, - "The destination view must admit at least one whole element"); - VERIFY_IS_TRUE(*PerElement != *Unbounded, - "The destination view must exclude at least one element"); + linalg_test::ThreadVectorMatrixMultiplySupport Multiply; + HR = linalg_test::queryThreadVectorMatrixMultiply( + Device, {*VectorType, *MatrixType, *BiasType, *ResultType}, Multiply); + if (FAILED(HR)) + return HR; - std::stringstream ExtraDefs; - ExtraDefs << " -DLOAD_OFFSET=" << Layout.OffsetBytes; - ExtraDefs << " -DLOAD_STRIDE=" << Layout.StrideBytes; - ExtraDefs << " -DLOAD_LAYOUT=" << static_cast(Layout.Layout); - ExtraDefs << " -DSTORE_OFFSET=" << Layout.OffsetBytes; - ExtraDefs << " -DSTORE_STRIDE=" << Layout.StrideBytes; - ExtraDefs << " -DSTORE_LAYOUT=" << static_cast(Layout.Layout); - ExtraDefs << " -DDECLARED_ALIGN=" << DescriptorDeclaredAlignment; + Supported = Multiply.supported(); + if (!Supported) + hlsl_test::LogCommentFmt( + L"ThreadVectorMatrixMultiply reports vector=%u matrix=%u bias=%u " + L"result=%u layout=%u is unsupported", + static_cast(*VectorType), static_cast(*MatrixType), + static_cast(*BiasType), static_cast(*ResultType), + static_cast(Case.Layout)); + return S_OK; +} - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; +static void runCase(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + const CaseData &Case, bool Verbose) { + const bool SelfTestPassed = oracleSelfTest(); + VERIFY_IS_TRUE(SelfTestPassed, "MatVec host oracle self-test failed"); + const bool Valid = isCaseValid(Case); + VERIFY_IS_TRUE(Valid, "Invalid MatVec interpretation case"); + if (!SelfTestPassed || !Valid) + return; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + const std::optional> MatrixBuffer = + encodeMatrixBuffer(Case); + const std::optional> VectorBuffer = + encodeVectorBuffer(Case); + const std::optional> BiasBuffer = + Case.hasBias() ? encodeComponents(Case.BiasInputType, Case.BiasValues) + : std::optional>(); + const std::optional> ExpectedOutput = + encodeExpectedOutput(Case); + const std::optional Args = buildCompilerArgs(Case); + VERIFY_IS_TRUE(MatrixBuffer.has_value()); + VERIFY_IS_TRUE(VectorBuffer.has_value()); + VERIFY_IS_TRUE(!Case.hasBias() || BiasBuffer.has_value()); + VERIFY_IS_TRUE(ExpectedOutput.has_value()); + VERIFY_IS_TRUE(Args.has_value()); + if (!MatrixBuffer || !VectorBuffer || (Case.hasBias() && !BiasBuffer) || + !ExpectedOutput || !Args) + return; - compileShader(DxcSupport, LoadStoreDescriptorShader, "cs_6_10", Args, - Verbose); + const char *Shader = Case.hasBias() ? MatVecMulAddShader : MatVecMulShader; + const char *RootSignature = Case.hasBias() + ? "SRV(t0), SRV(t1), SRV(t2), UAV(u3)" + : "SRV(t0), SRV(t1), UAV(u2)"; + compileShader(DxcSupport, Shader, "cs_6_10", *Args, Verbose); + + auto Op = createComputeOp(Shader, "cs_6_10", RootSignature, Args->c_str()); + addSRVBuffer(Op.get(), "MatrixInput", MatrixBuffer->size(), "byname"); + addSRVBuffer(Op.get(), "VectorInput", VectorBuffer->size(), "byname"); + if (Case.hasBias()) + addSRVBuffer(Op.get(), "BiasInput", BiasBuffer->size(), "byname"); + addUAVBuffer(Op.get(), "Output", ExpectedOutput->size(), true, "byname"); + addRootView(Op.get(), 0, "MatrixInput"); + addRootView(Op.get(), 1, "VectorInput"); + if (Case.hasBias()) { + addRootView(Op.get(), 2, "BiasInput"); + addRootView(Op.get(), 3, "Output"); + } else { + addRootView(Op.get(), 2, "Output"); + } - // Only the destination view is short. The source is viewed in full so the - // load cannot be bounds checked as well, which would leave the observed - // result attributable to either operation. - const cpu_oracle::TypedMatrix InputMatrix = *Input; - auto Op = createComputeOp(LoadStoreDescriptorShader, "cs_6_10", - "DescriptorTable(UAV(u0), UAV(u1))", Args.c_str()); - addUAVBuffer(Op.get(), "Input", *BufferSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", *BufferSize, true, "byname"); - addHeapRawUAV(Op.get(), "ResHeap", "Input", *BufferSize); - addHeapRawUAV(Op.get(), "ResHeap", "Output", OutputViewBytes); - addRootTable(Op.get(), 0, "ResHeap"); + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [&](LPCSTR Name, std::vector &Data, st::ShaderOp *) { + if (_stricmp(Name, "Output") == 0) { + cpu_oracle::fillPoison(Data.data(), Data.size()); + return; + } - auto Result = runShaderOp( - Device, DxcSupport, std::move(Op), - [InputMatrix, Layout](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - cpu_oracle::fillPoison(Data.data(), Data.size()); - if (_stricmp(Name, "Input") != 0) - return; - VERIFY_IS_TRUE(cpu_oracle::writeMatrixBuffer(InputMatrix, Layout, Data), - "Unable to encode typed StoreDescriptorOOB input"); - }, - [Layout](ID3D12GraphicsCommandList *, st::ShaderOpTest *Test) { - verifyDescriptorBaseAlignment(Test, "Input", Layout.OffsetBytes); - verifyDescriptorBaseAlignment(Test, "Output", Layout.OffsetBytes); - }); + const std::vector *Source = nullptr; + if (_stricmp(Name, "MatrixInput") == 0) + Source = &*MatrixBuffer; + else if (_stricmp(Name, "VectorInput") == 0) + Source = &*VectorBuffer; + else if (Case.hasBias() && _stricmp(Name, "BiasInput") == 0) + Source = &*BiasBuffer; + VERIFY_IS_TRUE(Source != nullptr, + "Unexpected MatVec resource initializer"); + if (!Source) + return; + VERIFY_IS_TRUE(Data.size() == Source->size(), + "MatVec resource initializer size mismatch"); + if (Data.size() == Source->size()) + std::memcpy(Data.data(), Source->data(), Data.size()); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); + VERIFY_IS_TRUE(verifyExactBuffer(OutData.data(), OutData.size(), + *ExpectedOutput, Verbose)); +} - VERIFY_IS_TRUE(cpu_oracle::verifyStoreBuffer( - OutData.data(), OutData.size(), {*PerElement, *WholeStore}, - L"HLSL proposal 0035 bounds checking on MatrixStoreToDescriptor: either " - L"the whole store or only the out-of-view element stores become a no-op", - Verbose)); +static void runCapabilityChecked(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const CaseData &Case, + linalg_test::CapabilityRequirement Requirement, + LPCWSTR CaseName, bool Verbose) { + bool TierSupported = false; + bool Supported = false; + const HRESULT QueryResult = + querySupport(Device, Case, TierSupported, Supported); + const linalg_test::CapabilityRequirement Effective = + SUCCEEDED(QueryResult) && !TierSupported + ? linalg_test::CapabilityRequirement::CapabilityGated + : Requirement; + if (!applyApplicability( + linalg_test::classifyApplicability(QueryResult, Supported, Effective), + CaseName)) + return; + runCase(Device, DxcSupport, Case, Verbose); } -// No offset and a tightly packed stride: a matrix occupying the whole buffer. -static cpu_oracle::MatrixBufferLayout packedLayout(const MatrixParams &Params) { - return cpu_oracle::MatrixBufferLayout{ - Params.Layout, - /*OffsetBytes=*/0, - /*StrideBytes=*/Params.strideBytes(), +static CaseData makeNonUniformF16Case(MatrixLayout Layout) { + CaseData Case = {}; + Case.MatrixType = ComponentType::F16; + Case.M = 4; + Case.N = 8; + Case.Layout = Layout; + Case.VectorInputType = ComponentType::F16; + Case.InputInterpretation = ComponentType::F16; + Case.ResultType = ComponentType::F16; + Case.MatrixValues = { + 1, 0, -1, 2, -2, 3, -3, 1, 0, 1, 2, -1, 3, -2, 1, -3, + -1, 2, 0, 1, -2, 1, 3, -1, 2, -1, 1, 0, 1, -3, -2, 3, }; + Case.InterpretedVectorValues = {1, -2, 3, -1, 2, -3, 1, 2}; + Case.PublicRule = + Layout == MatrixLayout::RowMajor + ? L"Exact non-uniform F16 RowMajor matrix-vector dot products" + : L"Exact non-uniform F16 ColumnMajor matrix-vector dot products"; + return Case; } -// Where the padded cases put the matrix. Independent of the alignment above, -// which is the contract rather than a placement, but constrained by it. -static constexpr size_t DescriptorAlignedOffset = 128; -static_assert(DescriptorAlignedOffset % DescriptorDeclaredAlignment == 0, - "descriptor offset must keep the first element aligned"); - -void DxilConf_SM610_LinAlg::LoadStoreDescriptor_Wave_16x16_F16() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"LoadStoreDescriptor_Wave_16x16_F16", - SelectedWaveSize)) - return; - - runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, packedLayout(Params), - packedLayout(Params), VerboseLogging, - SelectedWaveSize); +static CaseData makeSInt8Case() { + CaseData Case = {}; + Case.MatrixType = ComponentType::I8; + Case.M = 4; + Case.N = 8; + Case.Layout = MatrixLayout::RowMajor; + Case.VectorInputType = ComponentType::I8; + Case.InputInterpretation = ComponentType::I8; + Case.ResultType = ComponentType::I32; + Case.MatrixValues = { + 1, -2, 3, -4, 5, -6, 7, -8, -1, 2, -3, 4, -5, 6, -7, 8, + 1, 1, 1, 1, 1, 1, 1, 1, -8, -7, -6, -5, -4, -3, -2, -1, + }; + Case.InterpretedVectorValues = {1, -1, 2, -2, 3, -3, 4, -4}; + Case.PublicRule = + L"Exact packed SInt8 vector times SInt8 matrix dot products"; + return Case; } -// Places the matrix at a non-zero offset and pads the row stride, so the -// destination holds bytes the store must not touch: a 128-byte prologue and -// three 16-byte gaps between its four rows. A store that addresses by element -// index rather than by the supplied stride writes into that padding, which the -// untouched-byte check catches and the element comparison cannot. -void DxilConf_SM610_LinAlg:: - LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable( - D3DDevice, Params, {Params.Use}, - L"LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded", - SelectedWaveSize)) - return; - - // A packed row of 8 F16 values is 16 bytes; 32 leaves a 16-byte gap between - // rows while remaining a legal multiple of 16. - const cpu_oracle::MatrixBufferLayout Layout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/32, +static CaseData makeUInt8Case() { + CaseData Case = {}; + Case.MatrixType = ComponentType::U8; + Case.M = 4; + Case.N = 8; + Case.Layout = MatrixLayout::RowMajor; + Case.VectorInputType = ComponentType::U8; + Case.InputInterpretation = ComponentType::U8; + Case.ResultType = ComponentType::I32; + Case.MatrixValues = { + 255, 1, 2, 3, 4, 5, 6, 7, 128, 127, 1, 1, 1, 1, 1, 1, + 200, 0, 200, 0, 200, 0, 200, 0, 0, 200, 0, 200, 0, 200, 0, 200, }; + Case.InterpretedVectorValues = {1, 255, 2, 254, 3, 253, 4, 252}; + Case.PublicRule = + L"Exact packed UInt8 vector times UInt8 matrix dot products"; + return Case; +} - runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, Layout, Layout, - VerboseLogging, SelectedWaveSize); +static CaseData makeUInt32OutputCase() { + CaseData Case = {}; + Case.MatrixType = ComponentType::U32; + Case.M = 4; + Case.N = 8; + Case.Layout = MatrixLayout::RowMajor; + Case.VectorInputType = ComponentType::U32; + Case.InputInterpretation = ComponentType::U32; + Case.ResultType = ComponentType::U32; + Case.OutputSigned = false; + Case.MatrixValues = { + 2147483648LL, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 100, 0, 100, 0, 100, 0, 100, 0, 0, 200, 0, 200, 0, 200, 0, 200, + }; + Case.InterpretedVectorValues = {1, 1, 1, 1, 1, 1, 1, 1}; + Case.PublicRule = + L"Exact native UInt32 matrix-vector results with unsigned output"; + return Case; } -// Loads RowMajor and stores ColumnMajor, which a shared layout cannot express: -// with the same layout on both sides, an implementation that ignores the -// layout argument entirely still round trips byte-identically, because the -// mapping it applies to the load it applies again to the store. Reading one -// layout and writing the other stops the two from cancelling. -void DxilConf_SM610_LinAlg:: - LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F32; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; +} // namespace matvec_interpretation - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable( - D3DDevice, Params, {Params.Use}, - L"LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor", - SelectedWaveSize)) - return; +// Harness self-check for the CPU oracle. Deliberately carries no Kits metadata +// so HLK runs never select it; drivers are not certified against this class. +class LinAlgCPUOracleTests { +public: + BEGIN_TEST_CLASS(LinAlgCPUOracleTests) + TEST_METHOD_PROPERTY(L"Priority", L"0") + END_TEST_CLASS() + + TEST_METHOD(TypedMatrixBufferRoundTrip); + TEST_METHOD(UntouchedByteVerification); + TEST_METHOD(ViewBoundedElements); + TEST_METHOD(ViewBoundedStoreBytes); +}; - // Source rows of 8 F32 values are 32 bytes packed, padded here to 48. - const cpu_oracle::MatrixBufferLayout LoadLayout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/48, - }; +void LinAlgCPUOracleTests::TypedMatrixBufferRoundTrip() { + using namespace cpu_oracle; - // Destination columns of 4 F32 values are 16 bytes, which is already a legal - // stride, so the column-major side is stored packed. - const cpu_oracle::MatrixBufferLayout StoreLayout = { - MatrixLayout::ColumnMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/16, + auto VerifyScalarEncoding = [](const std::optional &Matrix, + const std::vector &ExpectedBytes) { + if (!Matrix) + return false; + MatrixBufferLayout Layout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/0, + /*StrideBytes=*/ExpectedBytes.size(), + }; + std::vector ActualBytes(ExpectedBytes.size(), 0); + MatrixResultOracle Oracle = + exactResult(*Matrix, L"Host scalar encoding and decoding"); + return writeMatrixBuffer(*Matrix, Layout, ActualBytes) && + ActualBytes == ExpectedBytes && + verifyMatrixBuffer(ActualBytes.data(), ActualBytes.size(), Layout, + Oracle, /*Verbose=*/false); }; - runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, LoadLayout, StoreLayout, - VerboseLogging, SelectedWaveSize); -} + VERIFY_IS_TRUE(VerifyScalarEncoding( + makeTypedMatrix(1, 1, {HLSLHalf_t(1.5f)}), {0x00, 0x3e})); + VERIFY_IS_TRUE(VerifyScalarEncoding(makeTypedMatrix(1, 1, {-2.5f}), + {0x00, 0x00, 0x20, 0xc0})); + VERIFY_IS_TRUE(VerifyScalarEncoding(makeTypedMatrix(1, 1, {-7}), + {0xf9, 0xff, 0xff, 0xff})); + VERIFY_IS_TRUE( + VerifyScalarEncoding(makeTypedMatrix(1, 1, {0x89abcdefu}), + {0xef, 0xcd, 0xab, 0x89})); -// The same cross-layout axis on F16, because no tier is required to support -// Fp32 matrices and the F32 case above can skip in its entirety. The shape -// must stay non-square: swapping the two layouts transposes on load and back -// on store, and for a square matrix those cancel byte for byte whatever -// strides are used. -void DxilConf_SM610_LinAlg:: - LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; + const uint32_t AdjacentFloatBits = 0x3f800001; + float AdjacentFloat; + std::memcpy(&AdjacentFloat, &AdjacentFloatBits, sizeof(AdjacentFloat)); + VERIFY_IS_TRUE( + ComponentTraits::format(AdjacentFloat).find(L"3f800001") != + std::wstring::npos); - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable( - D3DDevice, Params, {Params.Use}, - L"LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor", - SelectedWaveSize)) - return; + std::optional Matrix = + makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); + VERIFY_IS_TRUE(Matrix.has_value()); - // Source rows of 8 F16 values are 16 bytes packed, padded here to 48. - const cpu_oracle::MatrixBufferLayout LoadLayout = { + MatrixBufferLayout RowMajor = { MatrixLayout::RowMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/48, + /*OffsetBytes=*/4, + /*StrideBytes=*/16, + }; + std::optional RowBytes = getMatrixBufferSize(*Matrix, RowMajor); + VERIFY_IS_TRUE(RowBytes.has_value()); + std::vector RowBuffer(*RowBytes, 0xcd); + VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, RowMajor, RowBuffer)); + const std::vector ExpectedRowBuffer = { + 0xcd, 0xcd, 0xcd, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, + 0x00, 0x03, 0x00, 0x00, 0x00, 0xcd, 0xcd, 0xcd, 0xcd, 0x04, 0x00, + 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, }; + VERIFY_IS_TRUE(RowBuffer == ExpectedRowBuffer); + MatrixResultOracle Exact = + exactResult(*Matrix, L"Host exact row-major matrix encoding"); + VERIFY_IS_TRUE(verifyMatrixBuffer(RowBuffer.data(), RowBuffer.size(), + RowMajor, Exact, /*Verbose=*/false)); - // Destination columns of 4 F16 values are 8 bytes, padded here to 16 so the - // column-major side carries a gap of its own rather than sitting packed. - const cpu_oracle::MatrixBufferLayout StoreLayout = { + MatrixBufferLayout ColumnMajor = { MatrixLayout::ColumnMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/16, + /*OffsetBytes=*/4, + /*StrideBytes=*/12, }; + std::optional ColumnBytes = getMatrixBufferSize(*Matrix, ColumnMajor); + VERIFY_IS_TRUE(ColumnBytes.has_value()); + std::vector ColumnBuffer(*ColumnBytes, 0xcd); + VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, ColumnMajor, ColumnBuffer)); + const std::vector ExpectedColumnBuffer = { + 0xcd, 0xcd, 0xcd, 0xcd, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0xcd, 0xcd, 0xcd, 0xcd, 0x02, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, + 0xcd, 0xcd, 0xcd, 0xcd, 0x03, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, + }; + VERIFY_IS_TRUE(ColumnBuffer == ExpectedColumnBuffer); + VERIFY_IS_TRUE(verifyMatrixBuffer(ColumnBuffer.data(), ColumnBuffer.size(), + ColumnMajor, Exact, /*Verbose=*/false)); - runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, LoadLayout, StoreLayout, - VerboseLogging, SelectedWaveSize); -} - -// Half the source matrix lies outside the view the descriptor carries. The -// boundary is deliberately placed mid-row rather than on a row boundary, so an -// implementation that bounds checks a row at a time cannot pass it. -void DxilConf_SM610_LinAlg::LoadDescriptorOOB_Wave_16x16_F16_PartialView() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"LoadDescriptorOOB_Wave_16x16_F16_" - L"PartialView", - SelectedWaveSize)) - return; - - // Packed, so the buffer is 16 rows of 32 bytes. A 264 byte view holds the - // first 132 elements: rows 0 to 7 whole, then four elements of row 8. - runLoadDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, - packedLayout(Params), /*InputViewBytes=*/264, - VerboseLogging, SelectedWaveSize); -} - -// The same behaviour where the matrix is offset and its rows are padded, so -// the view boundary falls in a different place for byte offsets than it does -// for element indices. An implementation that bounds checks by element index -// keeps elements this view does not reach. -void DxilConf_SM610_LinAlg:: - LoadDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"LoadDescriptorOOB_Wave_4x8_F16_" - L"OffsetPaddedPartialView", - SelectedWaveSize)) - return; + std::optional Transposed = transposeMatrix(*Matrix); + std::optional ExpectedTranspose = + makeTypedMatrix(3, 2, {1, 4, 2, 5, 3, 6}); + VERIFY_IS_TRUE(Transposed.has_value()); + VERIFY_IS_TRUE(ExpectedTranspose.has_value()); + size_t FirstMismatch; + VERIFY_IS_TRUE( + exactMatrixMatch(*Transposed, *ExpectedTranspose, FirstMismatch)); - const cpu_oracle::MatrixBufferLayout Layout = { - MatrixLayout::RowMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/32, - }; + std::optional MixedActual = + makeTypedMatrix(1, 2, {1, 4}); + std::optional CandidateA = + makeTypedMatrix(1, 2, {1, 2}); + std::optional CandidateB = + makeTypedMatrix(1, 2, {3, 4}); + VERIFY_IS_TRUE(MixedActual.has_value()); + VERIFY_IS_TRUE(CandidateA.has_value()); + VERIFY_IS_TRUE(CandidateB.has_value()); + MatrixResultOracle Permitted = + permittedResults({*CandidateA, *CandidateB}, + L"Host whole-result permitted candidate semantics"); + VERIFY_IS_FALSE(matchesAnyCompleteCandidate(*MixedActual, Permitted)); + Permitted.Candidates.push_back(*MixedActual); + VERIFY_IS_TRUE(matchesAnyCompleteCandidate(*MixedActual, Permitted)); - // Elements sit at 128 + 32*Row + 2*Column. A 172 byte view holds row 0 - // whole and columns 0 to 5 of row 1, so it cuts within a row and stops - // short of the padding rather than on it. - runLoadDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, Layout, - /*InputViewBytes=*/172, VerboseLogging, - SelectedWaveSize); -} + MatrixResultOracle Excluded = + excludedResult(L"Host excluded-oracle classification"); + VERIFY_IS_FALSE(matchesAnyCompleteCandidate(*Matrix, Excluded)); -// The same two views on the destination instead of the source, so the rule -// being exercised is bounds checking on the store rather than on the load. -void DxilConf_SM610_LinAlg::StoreDescriptorOOB_Wave_16x16_F16_PartialView() { MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; + Params.M = 2; + Params.N = 3; Params.Use = MatrixUse::A; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"StoreDescriptorOOB_Wave_16x16_F16_" - L"PartialView", - SelectedWaveSize)) - return; - - // Packed, so the buffer is 16 rows of 32 bytes. A 260 byte view admits the - // first 130 elements: rows 0 to 7 whole, then two of row 8. Ending two - // elements into the row keeps the boundary off the round multiples a - // coarser-than-per-element bounds check would land on. - runStoreDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, - packedLayout(Params), /*OutputViewBytes=*/260, - VerboseLogging, SelectedWaveSize); + Params.NumThreads = 4; + Params.CompType = ComponentType::I32; + VERIFY_IS_TRUE(buildCompilerArgs(Params).find(" -DELEM_TYPE=int") != + std::string::npos); + Params.CompType = ComponentType::U32; + VERIFY_IS_TRUE(buildCompilerArgs(Params).find(" -DELEM_TYPE=uint") != + std::string::npos); } -void DxilConf_SM610_LinAlg:: - StoreDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; +// The padding check is verified here rather than only through the execution +// tests because a GPU round trip cannot easily produce a store that places +// every element correctly and still damages the bytes around them, which is +// the single case this check exists to catch. +void LinAlgCPUOracleTests::UntouchedByteVerification() { + using namespace cpu_oracle; - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"StoreDescriptorOOB_Wave_4x8_F16_" - L"OffsetPaddedPartialView", - SelectedWaveSize)) - return; + // A 2x3 uint32 matrix at a 4 byte offset with a 16 byte stride occupies + // bytes 4..15 and 20..31, leaving a 4 byte prologue at 0..3 and 4 bytes of + // padding at 16..19. + std::optional Matrix = + makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); + VERIFY_IS_TRUE(Matrix.has_value()); - const cpu_oracle::MatrixBufferLayout Layout = { + const MatrixBufferLayout Layout = { MatrixLayout::RowMajor, - /*OffsetBytes=*/DescriptorAlignedOffset, - /*StrideBytes=*/32, + /*OffsetBytes=*/4, + /*StrideBytes=*/16, }; + std::optional Size = getMatrixBufferSize(*Matrix, Layout); + VERIFY_IS_TRUE(Size.has_value()); + VERIFY_ARE_EQUAL(size_t(32), *Size); - // Elements sit at 128 + 32*Row + 2*Column. A 172 byte view holds row 0 - // whole and columns 0 to 5 of row 1, so it cuts within a row and stops - // short of the padding rather than on it. - runStoreDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, Layout, - /*OutputViewBytes=*/172, VerboseLogging, - SelectedWaveSize); -} + std::vector Buffer(*Size); + fillPoison(Buffer.data(), Buffer.size()); + VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, Layout, Buffer)); -static const char SplatStoreShader[] = R"( - RWByteAddressBuffer Output : register(u0); + auto CountTouched = [&Layout](const std::vector &Bytes) { + return countTouchedBytesOutsideElements(ComponentType::U32, 2, 3, Layout, + Bytes.data(), Bytes.size()); + }; - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main() { - if (GetGroupWaveIndex() != 0) - return; + // A correctly encoded buffer leaves every non-element byte poisoned. + std::optional Clean = CountTouched(Buffer); + VERIFY_IS_TRUE(Clean.has_value()); + VERIFY_ARE_EQUAL(size_t(0), *Clean); + VERIFY_IS_TRUE(verifyUntouchedBytes(ComponentType::U32, 2, 3, Layout, + Buffer.data(), Buffer.size(), + /*Verbose=*/false)); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Mat; - __builtin_LinAlg_FillMatrix(Mat, FILL_VALUE); - __builtin_LinAlg_MatrixStoreToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + // Damaging an element is the element comparison's job, not this check's, so + // the count must stay at zero. + std::vector ElementTouched = Buffer; + ElementTouched[4] ^= 0xff; + std::optional AfterElement = CountTouched(ElementTouched); + VERIFY_IS_TRUE(AfterElement.has_value()); + VERIFY_ARE_EQUAL(size_t(0), *AfterElement); + + // Damaging the prologue or the inter-row padding is what this check exists + // to catch, so each one must be counted. + for (size_t Offset : {size_t(0), size_t(16)}) { + std::vector PaddingTouched = Buffer; + PaddingTouched[Offset] ^= 0xff; + std::optional AfterPadding = CountTouched(PaddingTouched); + VERIFY_IS_TRUE(AfterPadding.has_value()); + VERIFY_ARE_EQUAL(size_t(1), *AfterPadding); } -)"; -static void runSplatStore(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, float FillValue, - bool Verbose, UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); + // Every non-element byte damaged at once is still counted exactly. + std::vector AllTouched(*Size); + fillPoison(AllTouched.data(), AllTouched.size()); + for (BYTE &Byte : AllTouched) + Byte = static_cast(~Byte); + VERIFY_IS_TRUE(writeMatrixBuffer(*Matrix, Layout, AllTouched)); + std::optional AfterAll = CountTouched(AllTouched); + VERIFY_IS_TRUE(AfterAll.has_value()); + VERIFY_ARE_EQUAL(size_t(8), *AfterAll); - std::stringstream ExtraDefs; - STREAM_FLOAT(ExtraDefs, "FILL_VALUE", FillValue); + // A buffer filled with one repeated value is fully detected, which is the + // reason the pattern varies with the offset. A constant poison would score + // zero here whenever the store happened to pick that same value, and 0xcd in + // particular is what the MSVC debug allocator leaves in memory nobody wrote. + std::vector ConstantFill(*Size, BYTE(0xcd)); + std::optional AfterConstant = CountTouched(ConstantFill); + VERIFY_IS_TRUE(AfterConstant.has_value()); + VERIFY_ARE_EQUAL(size_t(8), *AfterConstant); - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + // The case a constant poison cannot survive: a store that writes a value the + // poison pattern itself uses. Because the pattern varies, that value matches + // at exactly one offset, so seven of the eight non-element bytes are still + // caught. A constant poison would match everywhere and report nothing. + std::vector PoisonValuedFill(*Size, poisonByteAt(0)); + std::optional AfterPoisonValued = CountTouched(PoisonValuedFill); + VERIFY_IS_TRUE(AfterPoisonValued.has_value()); + VERIFY_ARE_EQUAL(size_t(7), *AfterPoisonValued); - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + // No two adjacent bytes share a poison value, so a constant written over any + // two neighbours cannot hide in both. + for (size_t Offset = 1; Offset < *Size; ++Offset) + VERIFY_ARE_NOT_EQUAL(poisonByteAt(Offset - 1), poisonByteAt(Offset)); - compileShader(DxcSupport, SplatStoreShader, "cs_6_10", Args, Verbose); + // The diagnostic list is capped but the count is not, so the two have to be + // checked against a buffer with more offenders than the cap. A 2x3 uint32 + // matrix at a 16 byte offset with a 16 byte stride occupies bytes 16..27 and + // 32..43, leaving twenty bytes outside the elements. + const MatrixBufferLayout PaddedLayout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/16, + /*StrideBytes=*/16, + }; + std::optional PaddedSize = + getMatrixBufferSize(ComponentType::U32, 2, 3, PaddedLayout); + VERIFY_IS_TRUE(PaddedSize.has_value()); + VERIFY_ARE_EQUAL(size_t(44), *PaddedSize); - auto Expected = - makeExpectedMat(Params.CompType, Params.M, Params.N, FillValue, false); + std::vector AllPaddingTouched(*PaddedSize); + fillPoison(AllPaddingTouched.data(), AllPaddingTouched.size()); + for (BYTE &Byte : AllPaddingTouched) + Byte = static_cast(~Byte); + std::vector ReportedOffsets; + std::optional AfterPadded = countTouchedBytesOutsideElements( + ComponentType::U32, 2, 3, PaddedLayout, AllPaddingTouched.data(), + AllPaddingTouched.size(), &ReportedOffsets); + VERIFY_IS_TRUE(AfterPadded.has_value()); + VERIFY_ARE_EQUAL(size_t(20), *AfterPadded); + VERIFY_ARE_EQUAL(size_t(8), ReportedOffsets.size()); + for (size_t I = 0; I < ReportedOffsets.size(); ++I) + VERIFY_ARE_EQUAL(I, ReportedOffsets[I]); - auto Op = - createComputeOp(SplatStoreShader, "cs_6_10", "UAV(u0)", Args.c_str()); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); + // Below the cap every offender is reported, and by its offset in the buffer + // rather than its position among the offenders. + std::vector TwoPaddingBytes(*PaddedSize); + fillPoison(TwoPaddingBytes.data(), TwoPaddingBytes.size()); + TwoPaddingBytes[28] ^= 0xff; + TwoPaddingBytes[29] ^= 0xff; + std::vector TwoOffsets; + std::optional AfterTwo = countTouchedBytesOutsideElements( + ComponentType::U32, 2, 3, PaddedLayout, TwoPaddingBytes.data(), + TwoPaddingBytes.size(), &TwoOffsets); + VERIFY_IS_TRUE(AfterTwo.has_value()); + VERIFY_ARE_EQUAL(size_t(2), *AfterTwo); + VERIFY_ARE_EQUAL(size_t(2), TwoOffsets.size()); + VERIFY_ARE_EQUAL(size_t(28), TwoOffsets[0]); + VERIFY_ARE_EQUAL(size_t(29), TwoOffsets[1]); - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + // A buffer too small for the layout cannot be checked at all. + std::vector TooSmall(*Size - 1); + fillPoison(TooSmall.data(), TooSmall.size()); + VERIFY_IS_FALSE(CountTouched(TooSmall).has_value()); +} - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); +// The per-element arm of the bounds-checking rule is derived on the host, so +// the boundary it draws is checked here rather than only through a GPU round +// trip, where a wrong boundary and a wrong implementation would be +// indistinguishable. +void LinAlgCPUOracleTests::ViewBoundedElements() { + using namespace cpu_oracle; - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); -} + // A 2x3 uint32 matrix at a 4 byte offset with a 16 byte stride puts its + // elements at bytes 4, 8, 12, 20, 24 and 28, each 4 bytes wide, so they end + // at 8, 12, 16, 24, 28 and 32. + std::optional Matrix = + makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); + VERIFY_IS_TRUE(Matrix.has_value()); -void DxilConf_SM610_LinAlg::SplatStore_Wave_16x16_F16() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; + const MatrixBufferLayout Layout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/4, + /*StrideBytes=*/16, + }; - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"SplatStore_Wave_16x16_F16", - SelectedWaveSize)) - return; + auto BoundedEquals = [&](size_t ViewBytes, + const std::vector &Expected) { + std::optional Bounded = + zeroElementsOutsideView(*Matrix, Layout, ViewBytes); + std::optional Want = makeTypedMatrix(2, 3, Expected); + if (!Bounded || !Want) + return false; + size_t FirstMismatch; + return exactMatrixMatch(*Bounded, *Want, FirstMismatch); + }; - runSplatStore(D3DDevice, DxcSupport, Params, 42.0f, VerboseLogging, - SelectedWaveSize); + // A view covering the whole buffer changes nothing, and an empty view + // zeroes everything. + VERIFY_IS_TRUE(BoundedEquals(32, {1, 2, 3, 4, 5, 6})); + VERIFY_IS_TRUE(BoundedEquals(0, {0, 0, 0, 0, 0, 0})); + + // A view ending at 24 admits the element that ends exactly there and + // excludes the rest, which is the inclusive end the rule requires. + VERIFY_IS_TRUE(BoundedEquals(24, {1, 2, 3, 4, 0, 0})); + + // One byte short of that boundary drops the straddling element whole. An + // element is either wholly inside the view or it is not there at all. + VERIFY_IS_TRUE(BoundedEquals(23, {1, 2, 3, 0, 0, 0})); + + // The gap between the rows is not addressable, so a view that reaches into + // the padding admits no further elements. + VERIFY_IS_TRUE(BoundedEquals(19, {1, 2, 3, 0, 0, 0})); + + // A view sized past the buffer cannot admit more than the buffer holds. + VERIFY_IS_TRUE(BoundedEquals(1024, {1, 2, 3, 4, 5, 6})); } -static const char AccumulateDescriptorShader[] = R"( - #define USE_ACC 2 +// The store side draws the same boundary but leaves the excluded elements +// holding poison rather than zero, so it is checked here as bytes. +void LinAlgCPUOracleTests::ViewBoundedStoreBytes() { + using namespace cpu_oracle; + + // The layout ViewBoundedElements uses: elements at bytes 4, 8, 12, 20, 24 + // and 28, each 4 bytes wide, in a 32 byte buffer. + std::optional Matrix = + makeTypedMatrix(2, 3, {1, 2, 3, 4, 5, 6}); + VERIFY_IS_TRUE(Matrix.has_value()); + + const MatrixBufferLayout Layout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/4, + /*StrideBytes=*/16, + }; + static constexpr size_t ElementOffsets[] = {4, 8, 12, 20, 24, 28}; - ByteAddressBuffer Input : register(t0); - RWByteAddressBuffer Output : register(u1); + // Bit I is set when element I holds its value. Every element must hold + // either that or the poison a rejected store leaves behind, so an oracle + // that zeroed the rejected elements instead fails here rather than + // reporting them as merely unwritten. + auto WrittenMask = [&](size_t ViewBytes) { + std::optional> Buffer = + storeBufferBoundedByView(*Matrix, Layout, ViewBytes); + VERIFY_IS_TRUE(Buffer.has_value()); + VERIFY_ARE_EQUAL(Buffer->size(), static_cast(32)); + unsigned Mask = 0; + for (unsigned I = 0; I < 6; ++I) { + const size_t Offset = ElementOffsets[I]; + const uint32_t Value = I + 1; + BYTE Written[sizeof(Value)]; + memcpy(Written, &Value, sizeof(Value)); + bool HoldsValue = true; + bool HoldsPoison = true; + for (size_t B = 0; B < sizeof(Value); ++B) { + if ((*Buffer)[Offset + B] != Written[B]) + HoldsValue = false; + if ((*Buffer)[Offset + B] != poisonByteAt(Offset + B)) + HoldsPoison = false; + } + VERIFY_IS_TRUE(HoldsValue || HoldsPoison, + "A view bounded store element held neither its value nor " + "the poison it was seeded with"); + if (HoldsValue) + Mask |= 1u << I; + } + return Mask; + }; - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main() { - if (GetGroupWaveIndex() != 0) - return; + // A view covering the whole buffer writes everything, and an empty view + // drops the whole store, so it writes nothing. + VERIFY_ARE_EQUAL(WrittenMask(32), 0x3fu); + VERIFY_ARE_EQUAL(WrittenMask(0), 0x00u); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); - } -)"; + // A view ending at 24 admits the element that ends exactly there. + VERIFY_ARE_EQUAL(WrittenMask(24), 0x0fu); -static void runAccumulateDescriptor(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, int FillValue, - bool Verbose, UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); + // One byte short of that boundary drops the straddling element whole, + // including the part of it the view does reach. + VERIFY_ARE_EQUAL(WrittenMask(23), 0x07u); - std::stringstream ExtraDefs; - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + // The prologue before the offset and the padding between rows belong to no + // element, so a full store must leave both holding poison. + std::optional> Full = + storeBufferBoundedByView(*Matrix, Layout, 32); + VERIFY_IS_TRUE(Full.has_value()); + std::optional Corrupted = countTouchedBytesOutsideElements( + ComponentType::U32, 2, 3, Layout, Full->data(), Full->size()); + VERIFY_IS_TRUE(Corrupted.has_value()); + VERIFY_ARE_EQUAL(*Corrupted, static_cast(0)); +} - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); +class LinAlgCapabilityTests { +public: + BEGIN_TEST_CLASS(LinAlgCapabilityTests) + TEST_METHOD_PROPERTY(L"Priority", L"0") + END_TEST_CLASS() - compileShader(DxcSupport, AccumulateDescriptorShader, "cs_6_10", Args, - Verbose); + TEST_METHOD(CapabilityPolicyAndPredicates); +}; - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, - static_cast(FillValue) * 2, false); +void LinAlgCapabilityTests::CapabilityPolicyAndPredicates() { + using namespace linalg_test; - auto Op = createComputeOp(AccumulateDescriptorShader, "cs_6_10", - "SRV(t0), UAV(u1)", Args.c_str()); - addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + VERIFY_IS_TRUE( + classifyApplicability(S_OK, true, CapabilityRequirement::Mandatory) == + Applicability::Execute); + VERIFY_IS_TRUE(classifyApplicability( + S_OK, false, CapabilityRequirement::CapabilityGated) == + Applicability::NotApplicable); + VERIFY_IS_TRUE( + classifyApplicability(S_OK, false, CapabilityRequirement::Mandatory) == + Applicability::Fail); + VERIFY_IS_TRUE( + classifyApplicability(E_UNEXPECTED, true, + CapabilityRequirement::CapabilityGated) == + Applicability::Fail); - auto Result = runShaderOp( - Device, DxcSupport, std::move(Op), - [NumElements, Params, FillValue](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, NumElements, - /*StartingVal=*/FillValue, - /*Increment=*/false), - "Saw unsupported component type"); - }); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, + MatrixScope::Wave)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, + MatrixScope::ThreadGroup)); + VERIFY_IS_FALSE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, + MatrixScope::Thread)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_WAVE_MATRIX_MULTIPLY, + MatrixScope::Wave)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREADGROUP_MATRIX_MULTIPLY, + MatrixScope::ThreadGroup)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_VECTOR_MATRIX_MULTIPLY, + MatrixScope::Thread)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_OUTER_PRODUCT, + MatrixScope::Thread)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE, + MatrixScope::Thread)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE, + MatrixScope::Wave)); + VERIFY_IS_TRUE(isLegalScope( + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_ATOMIC_ACCUMULATE_STORE, + MatrixScope::ThreadGroup)); - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); + MatrixConstructionSupport Construction = {TRUE}; + VERIFY_IS_TRUE(Construction.valid()); + VERIFY_IS_TRUE(Construction.supported()); + MatrixConstructionSupport UnsupportedConstruction = {FALSE}; + VERIFY_IS_TRUE(UnsupportedConstruction.valid()); + VERIFY_IS_FALSE(UnsupportedConstruction.supported()); + // The runtime contract is a canonical BOOL; anything else is a driver bug. + MatrixConstructionSupport InvalidConstruction = {2}; + VERIFY_IS_FALSE(InvalidConstruction.valid()); + VERIFY_IS_FALSE(InvalidConstruction.supported()); - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); -} + WaveMatrixMultiplySupport Wave = { + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED}; + VERIFY_IS_TRUE(Wave.valid()); + VERIFY_IS_TRUE(Wave.supported()); + WaveMatrixMultiplySupport UnsupportedWave = { + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_NONE}; + VERIFY_IS_TRUE(UnsupportedWave.valid()); + VERIFY_IS_FALSE(UnsupportedWave.supported()); + WaveMatrixMultiplySupport InvalidWave = { + static_cast< + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS)), + }; + VERIFY_IS_FALSE(InvalidWave.valid()); -void DxilConf_SM610_LinAlg::AccumulateDescriptor_Wave_16x16_F16() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; + ThreadGroupMatrixMultiplySupport ThreadGroup = { + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED, + 32, + 128, + 64, + }; + VERIFY_IS_TRUE(ThreadGroup.valid()); + VERIFY_IS_TRUE(ThreadGroup.supportsThreadGroupSize(64)); + VERIFY_IS_FALSE(ThreadGroup.supportsThreadGroupSize(48)); + ThreadGroup.PreferredThreadGroupSize = 48; + VERIFY_IS_FALSE(ThreadGroup.valid()); + ThreadGroup = { + static_cast< + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_TRANSPOSE)), + 32, + 128, + 64, + }; + VERIFY_IS_FALSE(ThreadGroup.valid()); - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"AccumulateDescriptor_Wave_16x16_F16", - SelectedWaveSize)) - return; - if (!accumulateStoreApplicable( - D3DDevice, Params.CompType, - linalg_test::AtomicDestination::RWByteAddressBuffer, - L"AccumulateDescriptor_Wave_16x16_F16")) - return; + ThreadVectorMatrixMultiplySupport ThreadVector = { + static_cast< + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_TRANSPOSE)), + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32, + }; + VERIFY_IS_TRUE(ThreadVector.valid()); + VERIFY_IS_TRUE(ThreadVector.supported()); + ThreadVector.SupportFlags = static_cast< + linalg_abi::D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAGS>( + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_SUPPORTED) | + static_cast( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS)); + VERIFY_IS_FALSE(ThreadVector.valid()); + ThreadVector.MatrixInputType = + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT8_E4M3FN; + VERIFY_IS_TRUE(ThreadVector.valid()); + ThreadVector.SupportFlags = linalg_abi:: + D3D12_LINEAR_ALGEBRA_MULTIPLICATION_SUPPORT_FLAG_EMULATED_INPUTS; + VERIFY_IS_FALSE(ThreadVector.valid()); - runAccumulateDescriptor(D3DDevice, DxcSupport, Params, 12, VerboseLogging, - SelectedWaveSize); -} + ThreadOuterProductSupport OuterProduct = {true}; + VERIFY_IS_TRUE(OuterProduct.supported()); + AtomicAccumulateStoreSupport Atomic = {true, false}; + VERIFY_IS_TRUE(Atomic.supports(AtomicDestination::RWByteAddressBuffer)); + VERIFY_IS_FALSE(Atomic.supports(AtomicDestination::GroupShared)); -// Element access constructs a wave-scope matrix and then reads or writes its -// components, so applicability is exactly MatrixConstruction for the tile the -// case declares. D3D12LinearAlgebraRuntimeFeatureSupport.md guarantees only -// that some shape whose largest component is 16 or less is reported for a -// supported type, and directs applications wanting smaller shapes to query -// them case by case. Neither Fp32 nor Fp16 matrices are required at Tier 1, so -// every element-access case is capability gated rather than mandatory. -static const char ElementAccessShader[] = R"( - RWByteAddressBuffer Input : register(u0); - RWByteAddressBuffer Output : register(u1); + VERIFY_ARE_EQUAL( + 0u, static_cast(linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE)); + MatrixConstructionQuery ConstructionQuery = { + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32, 32, {8, 8, 8}}; + WaveMatrixMultiplyInputs WaveInputs = { + 32, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32, + }; + WaveMatrixMultiplyQuery WaveQuery = {WaveInputs, {16, 16, 16}}; + ThreadGroupMatrixMultiplyQuery ThreadGroupQuery = { + WaveInputs, + {16, 16, 16}, + }; + ThreadVectorMatrixMultiplyQuery ThreadVectorQuery = { + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + }; + ThreadOuterProductQuery OuterProductQuery = { + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16, + }; + AtomicAccumulateStoreQuery AtomicQuery = { + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16}; + VERIFY_ARE_EQUAL(32u, ConstructionQuery.WaveSize); + VERIFY_ARE_EQUAL(8u, ConstructionQuery.Shape.K); + VERIFY_ARE_EQUAL(32u, WaveQuery.Inputs.WaveSize); + VERIFY_ARE_EQUAL(16u, WaveQuery.Shape.M); + VERIFY_ARE_EQUAL(32u, ThreadGroupQuery.WaveInputs.WaveSize); + VERIFY_ARE_EQUAL(16u, ThreadGroupQuery.Shape.M); + VERIFY_IS_TRUE(ThreadVectorQuery.BiasInputType == + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE); + VERIFY_IS_TRUE(OuterProductQuery.InputComponentType == + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16); + VERIFY_IS_TRUE(AtomicQuery.ComponentType == + linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16); +} - // flatten the 2D index into a 1D index then scale by element size - // Always store row-major and work it out in the test runner - uint coordToByteOffset(uint2 coord) { - return (coord.x * N_DIM + coord.y) * ELEM_SIZE; - } +class DxilConf_SM610_LinAlg { +public: + BEGIN_TEST_CLASS(DxilConf_SM610_LinAlg) + TEST_CLASS_PROPERTY("Kits.TestName", + "D3D12 - Shader Model 6.10 - LinAlg Matrix Operations") + TEST_CLASS_PROPERTY("Kits.TestId", "a1b2c3d4-e5f6-7890-abcd-ef1234567890") + TEST_CLASS_PROPERTY( + "Kits.Description", + "Validates SM 6.10 linear algebra matrix operations execute correctly") + TEST_CLASS_PROPERTY( + "Kits.Specification", + "Device.Graphics.D3D12.DXILCore.ShaderModel610.CoreRequirement") + TEST_METHOD_PROPERTY(L"Priority", L"0") + END_TEST_CLASS() - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main(uint threadID : SV_GroupIndex) { - if (GetGroupWaveIndex() != 0) - return; + TEST_CLASS_SETUP(setupClass); + TEST_METHOD_SETUP(setupMethod); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); + // Load/Store/Accumulate Descriptor + TEST_METHOD(LoadStoreDescriptor_Wave_16x16_F16); + TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded); + TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor); + TEST_METHOD(LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor); + TEST_METHOD(LoadDescriptorOOB_Wave_16x16_F16_PartialView); + TEST_METHOD(LoadDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView); + TEST_METHOD(StoreDescriptorOOB_Wave_16x16_F16_PartialView); + TEST_METHOD(StoreDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView); + TEST_METHOD(SplatStore_Wave_16x16_F16); + TEST_METHOD(AccumulateDescriptor_Wave_16x16_F16); - // Copy Matrix values from input to output without assuming order - for (uint I = 0; I < __builtin_LinAlg_MatrixLength(Mat); ++I) { - uint2 Coord = __builtin_LinAlg_MatrixGetCoordinate(Mat, I); - uint Offset = coordToByteOffset(Coord); - ELEM_TYPE Elem; - __builtin_LinAlg_MatrixGetElement(Elem, Mat, I); - Output.Store(Offset, Elem); - } + // Load/Store/Accumulate Memory + TEST_METHOD(LoadMemory_Wave_16x16_F16); + TEST_METHOD(StoreMemory_Wave_16x16_F16); + TEST_METHOD(AccumulateMemory_Wave_16x16_F16); - // Save the matrix length that this thread saw. The length is written - // to the output right after the matrix, offset by the thread index - uint LenIdx = (M_DIM * N_DIM * ELEM_SIZE) + (threadID * sizeof(uint)); - uint Len = __builtin_LinAlg_MatrixLength(Mat); - Output.Store(LenIdx, Len); - } -)"; + // Element access + TEST_METHOD(ElementAccess_Wave_16x16_F16); + TEST_METHOD(ElementAccess_Wave_4x8_F32); + TEST_METHOD(ElementSet_Wave_16x16_F16); + TEST_METHOD(ElementGetOOB_Wave_4x8_F32); + TEST_METHOD(ElementSetOOB_Wave_4x8_F32); + TEST_METHOD(ElementGetOOB_Wave_16x16_F16); + TEST_METHOD(ElementSetOOB_Wave_16x16_F16); -static void runElementAccess(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); - const size_t NumThreads = Params.NumThreads; - const size_t MatrixSize = Params.totalBytes(); - // OutputBuf needs to fit the Matrix plus one uint per thread - const size_t OutputBufSize = MatrixSize + NumThreads * sizeof(uint32_t); + // Cast/Convert + TEST_METHOD(CopyConvert_Wave_16x16_F16); + TEST_METHOD(CopyConvert_Wave_16x16_F16_Transpose); + TEST_METHOD(CopyConvert_Wave_4x8_F32_Transpose); - std::stringstream ExtraDefs; - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + // Matrix Matrix Arithmetic + TEST_METHOD(MatMatMul_Wave_16x16x16_F16); + TEST_METHOD(MatMatMulAccum_Wave_16x16x16_F16); + TEST_METHOD(MatAccum_Wave_16x16_F16); - compileShader(DxcSupport, ElementAccessShader, "cs_6_10", Args, Verbose); + // Matrix Vector Arithmetic + TEST_METHOD(MatVecMul_Thread_16x16_F16); + TEST_METHOD(MatVecMul_Thread_4x8_F32); + TEST_METHOD(MatVecMul_Thread_4x8_F16_NonUniform); + TEST_METHOD(MatVecMul_Thread_4x8_F16_ColumnMajor); + TEST_METHOD(MatVecMul_Thread_4x8_I8_Interpreted); + TEST_METHOD(MatVecMul_Thread_4x8_U8_Interpreted); + TEST_METHOD(MatVecMul_Thread_4x8_U32_UnsignedOutput); + TEST_METHOD(MatVecMulAdd_Thread_16x16_F16); + TEST_METHOD(MatVecMulAdd_Thread_4x8_F32); + TEST_METHOD(MatVecMulAdd_Thread_4x8_F16_IndependentBias); + TEST_METHOD(OuterProduct_Thread_16x16_F16); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1); + // Query Accumulator Layout + TEST_METHOD(QueryAccumLayout); - auto Op = createComputeOp(ElementAccessShader, "cs_6_10", "UAV(u0), UAV(u1)", - Args.c_str()); - addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", OutputBufSize, true); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + // Convert + TEST_METHOD(Convert); - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [NumElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumElements), - "Saw unsupported component type"); - }); + // CopyConvert / Convert coverage + TEST_METHOD(CopyConvert_Wave_4x8_F16_ToF32); + TEST_METHOD(CopyConvert_Wave_4x8_F32_ToF16_Transpose); + TEST_METHOD(Convert_I16_ToI32_Exact); + TEST_METHOD(Convert_F32_ToI16_RTNE_Saturate); - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); + // Vector Accumulate + TEST_METHOD(VectorAccumulateDescriptor_Thread_F16); - // Verify the front of the buffer is a list of elements of the expected type - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); +private: + CComPtr D3DDevice; + dxc::SpecificDllLoader DxcSupport; + bool VerboseLogging = false; + bool Initialized = false; + std::optional D3D12SDK; - // Verify the end of the buffer is NumThreads number of lengths, whose - // sum is greater than or equal to NumElements - const BYTE *Out = static_cast(OutData.data()); - const uint32_t *Lengths = - reinterpret_cast(Out + MatrixSize); - uint32_t TotalLength = 0; - for (size_t I = 0; I < NumThreads; ++I) - TotalLength += Lengths[I]; - VERIFY_IS_GREATER_THAN_OR_EQUAL( - TotalLength, NumElements, "Sum of all lengths must be gte num elements"); -} + WEX::TestExecution::SetVerifyOutput VerifyOutput{ + WEX::TestExecution::VerifyOutputSettings::LogOnlyFailures}; +}; -void DxilConf_SM610_LinAlg::ElementAccess_Wave_16x16_F16() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; +bool DxilConf_SM610_LinAlg::setupClass() { + if (!Initialized) { + Initialized = true; + VERIFY_SUCCEEDED( + DxcSupport.InitializeForDll(dxc::kDxCompilerLib, "DxcCreateInstance")); + D3D12SDK = D3D12SDKSelector(); + WEX::TestExecution::RuntimeParameters::TryGetValue(L"VerboseLogging", + VerboseLogging); - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementAccess_Wave_16x16_F16", - SelectedWaveSize)) - return; + if (!D3D12SDK->createDevice(&D3DDevice, D3D_SHADER_MODEL_6_10, false)) { +#ifdef _HLK_CONF + hlsl_test::LogErrorFmt( + L"Device creation failed. Expected a driver supporting SM6.10"); +#else + hlsl_test::LogWarningFmt( + L"Device creation failed. Expected a driver supporting SM6.10"); + WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); +#endif + return false; + } + } - runElementAccess(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + return true; } -void DxilConf_SM610_LinAlg::ElementAccess_Wave_4x8_F32() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F32; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = false; +bool DxilConf_SM610_LinAlg::setupMethod() { + // If the device is healthy, exit otherwise it's possible a previous test + // case caused a device removal. So we need to try and create a new device. + if (D3DDevice && D3DDevice->GetDeviceRemovedReason() == S_OK) + return true; - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementAccess_Wave_4x8_F32", - SelectedWaveSize)) - return; + hlsl_test::LogCommentFmt(L"Device was lost!"); + D3DDevice.Release(); - // Non-square dimensions make the row-major coordinate mapping observable: a - // transposed GetCoordinate would land inside the matrix for a square tile - // but out of it here. - runElementAccess(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + hlsl_test::LogCommentFmt(L"Recreating device"); + + return D3D12SDK->createDevice(&D3DDevice, D3D_SHADER_MODEL_6_10, false); } -static const char ElementSetShader[] = R"( +// The alignment the descriptor shader declares to both builtins. Proposal 0035 +// requires the first element's address -- the resource base plus the offset -- +// to meet it. +static constexpr size_t DescriptorDeclaredAlignment = 128; + +static const char LoadStoreDescriptorShader[] = R"( RWByteAddressBuffer Input : register(u0); RWByteAddressBuffer Output : register(u1); @@ -3349,396 +3185,540 @@ static const char ElementSetShader[] = R"( [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] Mat; __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); - - // Increment every element by 5 - for (uint I = 0; I < __builtin_LinAlg_MatrixLength(Mat); ++I) { - ELEM_TYPE Elem; - __builtin_LinAlg_MatrixGetElement(Elem, Mat, I); - Elem = Elem + 5; - __builtin_LinAlg_MatrixSetElement(Mat, Mat, I, Elem); - } - + Mat, Input, LOAD_OFFSET, LOAD_STRIDE, LOAD_LAYOUT, DECLARED_ALIGN); __builtin_LinAlg_MatrixStoreToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + Mat, Output, STORE_OFFSET, STORE_STRIDE, STORE_LAYOUT, DECLARED_ALIGN); } )"; -static void runElementSet(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); - const size_t MatrixSize = Params.totalBytes(); +// The base is a runtime property that no compile-time check can see, so check +// it against the real GPU address. +static void verifyDescriptorBaseAlignment(st::ShaderOpTest *Test, LPCSTR Name, + size_t OffsetBytes) { + // GetResource hands back a borrowed pointer without an AddRef. + ID3D12Resource *Resource = nullptr; + Test->GetResource(Name, &Resource); + VERIFY_IS_NOT_NULL(Resource); + + const UINT64 ElementAddress = Resource->GetGPUVirtualAddress() + OffsetBytes; + VERIFY_IS_TRUE(ElementAddress % DescriptorDeclaredAlignment == 0, + "Descriptor buffer's first element does not meet the " + "alignment the shader declares"); +} + +static void +runLoadStoreDescriptor(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, + const cpu_oracle::MatrixBufferLayout &LoadLayout, + const cpu_oracle::MatrixBufferLayout &StoreLayout, + bool Verbose, UINT ForcedWaveSize = 0) { + std::optional Input = + cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); + VERIFY_IS_TRUE(Input.has_value(), + "Unable to construct typed LoadStoreDescriptor input"); + + std::optional InputSize = + cpu_oracle::getMatrixBufferSize(*Input, LoadLayout); + std::optional OutputSize = + cpu_oracle::getMatrixBufferSize(*Input, StoreLayout); + VERIFY_IS_TRUE(InputSize.has_value() && OutputSize.has_value(), + "Unable to size the LoadStoreDescriptor buffers"); std::stringstream ExtraDefs; + ExtraDefs << " -DLOAD_OFFSET=" << LoadLayout.OffsetBytes; + ExtraDefs << " -DLOAD_STRIDE=" << LoadLayout.StrideBytes; + ExtraDefs << " -DLOAD_LAYOUT=" << static_cast(LoadLayout.Layout); + ExtraDefs << " -DSTORE_OFFSET=" << StoreLayout.OffsetBytes; + ExtraDefs << " -DSTORE_STRIDE=" << StoreLayout.StrideBytes; + ExtraDefs << " -DSTORE_LAYOUT=" << static_cast(StoreLayout.Layout); + ExtraDefs << " -DDECLARED_ALIGN=" << DescriptorDeclaredAlignment; + if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, ElementSetShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, LoadStoreDescriptorShader, "cs_6_10", Args, + Verbose); - // Start counting from 6 since each element was increased by 5 - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 6); + const cpu_oracle::TypedMatrix InputMatrix = *Input; + cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::exactResult( + InputMatrix, L"HLSL proposal 0035 MatrixLoadFromDescriptor and " + L"MatrixStoreToDescriptor " + L"round trip at the requested offset, stride and layout"); - auto Op = createComputeOp(ElementSetShader, "cs_6_10", "UAV(u0), UAV(u1)", - Args.c_str()); - addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", MatrixSize, true); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + // Two UAV buffers, load from one, store to the other. The destination is + // filled by name rather than zeroed so unowned bytes carry the poison. + // + // Bound through a descriptor table rather than as root views. A root view is + // a bare GPU address, and proposal 0035 exempts root descriptors from bounds + // checking precisely because they carry no dimensions. Binding through a + // heap gives each buffer a view whose extent the implementation can see. + auto Op = createComputeOp(LoadStoreDescriptorShader, "cs_6_10", + "DescriptorTable(UAV(u0), UAV(u1))", Args.c_str()); + addUAVBuffer(Op.get(), "Input", *InputSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", *OutputSize, true, "byname"); + addHeapRawUAV(Op.get(), "ResHeap", "Input", *InputSize); + addHeapRawUAV(Op.get(), "ResHeap", "Output", *OutputSize); + addRootTable(Op.get(), 0, "ResHeap"); - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [NumElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumElements), - "Saw unsupported component type"); - }); + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [InputMatrix, LoadLayout](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + cpu_oracle::fillPoison(Data.data(), Data.size()); + if (_stricmp(Name, "Input") != 0) + return; + VERIFY_IS_TRUE( + cpu_oracle::writeMatrixBuffer(InputMatrix, LoadLayout, Data), + "Unable to encode typed LoadStoreDescriptor input"); + }, + [LoadLayout, StoreLayout](ID3D12GraphicsCommandList *, + st::ShaderOpTest *Test) { + verifyDescriptorBaseAlignment(Test, "Input", LoadLayout.OffsetBytes); + verifyDescriptorBaseAlignment(Test, "Output", StoreLayout.OffsetBytes); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - // Verify the front of the buffer is a list of elements of the expected type - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer(OutData.data(), OutData.size(), + StoreLayout, Oracle, Verbose)); + VERIFY_IS_TRUE(cpu_oracle::verifyUntouchedBytes( + Params.CompType, Params.M, Params.N, StoreLayout, OutData.data(), + OutData.size(), Verbose)); } -void DxilConf_SM610_LinAlg::ElementSet_Wave_16x16_F16() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementSet_Wave_16x16_F16", - SelectedWaveSize)) - return; +// Proposal 0035 permits two bounds-checking behaviors. An implementation may +// zero the whole matrix when any element falls outside the view, or zero only +// the elements that do, and both are conformant. A single expected buffer +// would therefore be wrong by construction, so the oracle carries both +// outcomes and accepts a complete match against either one. +// +// What makes the test discriminating is that the source buffer is allocated +// and written in full and only its *view* is shortened, so the bytes past the +// view hold real matrix data rather than zeros. An implementation that does no +// bounds checking at all reads that data back and matches neither candidate. +static void runLoadDescriptorOutOfBounds( + ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, const cpu_oracle::MatrixBufferLayout &Layout, + size_t InputViewBytes, bool Verbose, UINT ForcedWaveSize = 0) { + std::optional Input = + cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); + VERIFY_IS_TRUE(Input.has_value(), + "Unable to construct typed LoadDescriptorOOB input"); - runElementSet(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); -} + std::optional BufferSize = + cpu_oracle::getMatrixBufferSize(*Input, Layout); + VERIFY_IS_TRUE(BufferSize.has_value(), + "Unable to size the LoadDescriptorOOB buffers"); + VERIFY_IS_TRUE(InputViewBytes < *BufferSize, + "The source view must be shorter than its buffer"); -// Length() is thread local, so the first index past a lane's own length is -// already out of bounds even though the wave collectively holds more elements. -// Probing Length() and a index far beyond it covers both a driver that clamps -// only at the wave total and one that wraps a large index back into range. -static constexpr UINT FarOOBOffset = 64; + std::optional PerElement = + cpu_oracle::zeroElementsOutsideView(*Input, Layout, InputViewBytes); + // The whole-matrix arm is the per-element arm with nothing in view. + std::optional WholeMatrix = + cpu_oracle::zeroElementsOutsideView(*Input, Layout, 0); + VERIFY_IS_TRUE(PerElement.has_value() && WholeMatrix.has_value(), + "Unable to derive the LoadDescriptorOOB candidates"); -// Per-lane record: {uint Length, uint Executed, float Just, float Far}. -static constexpr UINT OOBRecordSize = 16; + // The test requires both in-bounds and out-of-bounds elements. If none are + // in bounds, the two permitted results are identical. If all are in bounds, + // an implementation that performs no bounds checking would still pass. + size_t FirstMismatch; + const bool HasInBoundsElement = + !cpu_oracle::exactMatrixMatch(*PerElement, *WholeMatrix, FirstMismatch); + VERIFY_IS_TRUE(HasInBoundsElement, + "The source view must include at least one complete element"); -// Seeds every output byte so a lane that never writes cannot be mistaken for a -// lane that correctly wrote the specified zero. The output buffer must be -// created "byname" for this to run at all: ShaderOpTest only invokes the -// initializer callback for that mode, and the default "zero" mode would leave -// the buffer holding exactly the value the out-of-bounds read is required to -// produce, making the comparison vacuous. -static constexpr BYTE OOBSentinelByte = 0xCD; + const bool HasOutOfBoundsElement = + !cpu_oracle::exactMatrixMatch(*PerElement, *Input, FirstMismatch); + VERIFY_IS_TRUE(HasOutOfBoundsElement, + "The source view must exclude at least one element"); -// Seeds the shader's destination locals. Distinct from zero, so a read that is -// dropped rather than performed cannot masquerade as a correct out-of-bounds -// result, and exactly representable in F32. -static constexpr int OOBGetPoisonValue = 999; + std::stringstream ExtraDefs; + ExtraDefs << " -DLOAD_OFFSET=" << Layout.OffsetBytes; + ExtraDefs << " -DLOAD_STRIDE=" << Layout.StrideBytes; + ExtraDefs << " -DLOAD_LAYOUT=" << static_cast(Layout.Layout); + ExtraDefs << " -DSTORE_OFFSET=" << Layout.OffsetBytes; + ExtraDefs << " -DSTORE_STRIDE=" << Layout.StrideBytes; + ExtraDefs << " -DSTORE_LAYOUT=" << static_cast(Layout.Layout); + ExtraDefs << " -DDECLARED_ALIGN=" << DescriptorDeclaredAlignment; -static const char ElementGetOOBShader[] = R"( - RWByteAddressBuffer Input : register(u0); - RWByteAddressBuffer Output : register(u1); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main(uint threadID : SV_GroupIndex) { - if (GetGroupWaveIndex() != 0) - return; + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); + compileShader(DxcSupport, LoadStoreDescriptorShader, "cs_6_10", Args, + Verbose); - uint Len = __builtin_LinAlg_MatrixLength(Mat); + cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::permittedResults( + {*PerElement, *WholeMatrix}, + L"HLSL proposal 0035 bounds checking on MatrixLoadFromDescriptor: " + L"either the whole matrix or only the out-of-view elements read as the " + L"default element value"); - // Seeded so that a dropped read leaves a value distinguishable from the - // zero a correct out-of-bounds read must produce. - ELEM_TYPE Just = (ELEM_TYPE)POISON_VALUE; - __builtin_LinAlg_MatrixGetElement(Just, Mat, Len); - ELEM_TYPE Far = (ELEM_TYPE)POISON_VALUE; - __builtin_LinAlg_MatrixGetElement(Far, Mat, Len + FAR_OOB_OFFSET); + // Only the source view is short. The destination is viewed in full so that + // the store cannot be bounds checked as well, which would leave the observed + // result attributable to either operation. + const cpu_oracle::TypedMatrix InputMatrix = *Input; + auto Op = createComputeOp(LoadStoreDescriptorShader, "cs_6_10", + "DescriptorTable(UAV(u0), UAV(u1))", Args.c_str()); + addUAVBuffer(Op.get(), "Input", *BufferSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", *BufferSize, true, "byname"); + addHeapRawUAV(Op.get(), "ResHeap", "Input", InputViewBytes); + addHeapRawUAV(Op.get(), "ResHeap", "Output", *BufferSize); + addRootTable(Op.get(), 0, "ResHeap"); - // Record unconditionally so the runner can tell that this lane ran. - uint Base = threadID * OOB_RECORD_SIZE; - Output.Store(Base + 0, Len); - Output.Store(Base + 4, 1); - // Widened to float so the runner can read a fixed-width record whatever - // the element type: a half store would leave the record's upper two bytes - // holding the sentinel. Half to float is lossless, so no value is masked. - Output.Store(Base + 8, (float)Just); - Output.Store(Base + 12, (float)Far); - } -)"; + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [InputMatrix, Layout](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + cpu_oracle::fillPoison(Data.data(), Data.size()); + if (_stricmp(Name, "Input") != 0) + return; + // Written in full, including the part the view does not cover. + VERIFY_IS_TRUE(cpu_oracle::writeMatrixBuffer(InputMatrix, Layout, Data), + "Unable to encode typed LoadDescriptorOOB input"); + }, + [Layout](ID3D12GraphicsCommandList *, st::ShaderOpTest *Test) { + verifyDescriptorBaseAlignment(Test, "Input", Layout.OffsetBytes); + verifyDescriptorBaseAlignment(Test, "Output", Layout.OffsetBytes); + }); -// Reads back the {Length, Executed} half of each lane record and checks the -// wave actually ran. Returns the total element count the wave reported. -static uint32_t verifyOOBLaneRecords(const BYTE *Records, size_t NumThreads, - UINT SelectedWaveSize, UINT RecordStride, - size_t NumElements, bool Verbose) { - uint32_t ExecutedLanes = 0; - uint32_t TotalLength = 0; - for (size_t I = 0; I < NumThreads; ++I) { - const BYTE *Record = Records + I * RecordStride; - uint32_t Length = 0; - uint32_t Executed = 0; - memcpy(&Length, Record, sizeof(Length)); - memcpy(&Executed, Record + 4, sizeof(Executed)); - if (Executed != 1) - continue; - ++ExecutedLanes; - TotalLength += Length; - if (Verbose) - hlsl_test::LogCommentFmt(L"lane %u reported Length=%u", - static_cast(I), Length); - } + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); - // Only wave 0 runs, so exactly the lanes of the wave the capability query - // selected must have written a record. - VERIFY_ARE_EQUAL(ExecutedLanes, SelectedWaveSize, - "Every lane of the selected wave must execute"); - VERIFY_IS_GREATER_THAN_OR_EQUAL( - TotalLength, static_cast(NumElements), - "Sum of all lengths must be gte num elements"); - return TotalLength; + VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer(OutData.data(), OutData.size(), + Layout, Oracle, Verbose)); + VERIFY_IS_TRUE(cpu_oracle::verifyUntouchedBytes( + Params.CompType, Params.M, Params.N, Layout, OutData.data(), + OutData.size(), Verbose)); } -static void runElementGetOOB(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - UINT ForcedWaveSize) { - VERIFY_IS_TRUE(Params.CompType == ComponentType::F32 || - Params.CompType == ComponentType::F16, - "Out-of-bounds Get records widen the element to float"); - const size_t NumElements = Params.totalElements(); - const size_t NumThreads = Params.NumThreads; - const size_t MatrixSize = Params.totalBytes(); - const size_t OutputBufSize = NumThreads * OOBRecordSize; +// Stores through a destination view shorter than its buffer. Both permitted +// outcomes leave the bytes past the view holding poison, so the comparison is +// byte level rather than matrix level. This cannot by itself fail an +// implementation that stores nothing, since dropping the whole store is one of +// those outcomes; the LoadStoreDescriptor cases require the store to happen. +static void runStoreDescriptorOutOfBounds( + ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, const cpu_oracle::MatrixBufferLayout &Layout, + size_t OutputViewBytes, bool Verbose, UINT ForcedWaveSize = 0) { + std::optional Input = + cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); + VERIFY_IS_TRUE(Input.has_value(), + "Unable to construct typed StoreDescriptorOOB input"); + + std::optional BufferSize = + cpu_oracle::getMatrixBufferSize(*Input, Layout); + VERIFY_IS_TRUE(BufferSize.has_value(), + "Unable to size the StoreDescriptorOOB buffers"); + VERIFY_IS_TRUE(OutputViewBytes < *BufferSize, + "The destination view must be shorter than its buffer"); + + std::optional> PerElement = + cpu_oracle::storeBufferBoundedByView(*Input, Layout, OutputViewBytes); + std::optional> WholeStore = + cpu_oracle::storeBufferBoundedByView(*Input, Layout, 0); + std::optional> Unbounded = + cpu_oracle::storeBufferBoundedByView(*Input, Layout, *BufferSize); + VERIFY_IS_TRUE(PerElement.has_value() && WholeStore.has_value() && + Unbounded.has_value(), + "Unable to derive the StoreDescriptorOOB candidates"); + + VERIFY_IS_TRUE(*PerElement != *WholeStore, + "The destination view must admit at least one whole element"); + VERIFY_IS_TRUE(*PerElement != *Unbounded, + "The destination view must exclude at least one element"); std::stringstream ExtraDefs; - ExtraDefs << " -DFAR_OOB_OFFSET=" << FarOOBOffset; - ExtraDefs << " -DOOB_RECORD_SIZE=" << OOBRecordSize; - ExtraDefs << " -DPOISON_VALUE=" << OOBGetPoisonValue; + ExtraDefs << " -DLOAD_OFFSET=" << Layout.OffsetBytes; + ExtraDefs << " -DLOAD_STRIDE=" << Layout.StrideBytes; + ExtraDefs << " -DLOAD_LAYOUT=" << static_cast(Layout.Layout); + ExtraDefs << " -DSTORE_OFFSET=" << Layout.OffsetBytes; + ExtraDefs << " -DSTORE_STRIDE=" << Layout.StrideBytes; + ExtraDefs << " -DSTORE_LAYOUT=" << static_cast(Layout.Layout); + ExtraDefs << " -DDECLARED_ALIGN=" << DescriptorDeclaredAlignment; + if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, ElementGetOOBShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, LoadStoreDescriptorShader, "cs_6_10", Args, + Verbose); - auto Op = createComputeOp(ElementGetOOBShader, "cs_6_10", "UAV(u0), UAV(u1)", - Args.c_str()); - addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", OutputBufSize, true, "byname"); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + // Only the destination view is short. The source is viewed in full so the + // load cannot be bounds checked as well, which would leave the observed + // result attributable to either operation. + const cpu_oracle::TypedMatrix InputMatrix = *Input; + auto Op = createComputeOp(LoadStoreDescriptorShader, "cs_6_10", + "DescriptorTable(UAV(u0), UAV(u1))", Args.c_str()); + addUAVBuffer(Op.get(), "Input", *BufferSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", *BufferSize, true, "byname"); + addHeapRawUAV(Op.get(), "ResHeap", "Input", *BufferSize); + addHeapRawUAV(Op.get(), "ResHeap", "Output", OutputViewBytes); + addRootTable(Op.get(), 0, "ResHeap"); - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [NumElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - if (_stricmp(Name, "Output") == 0) { - std::fill(Data.begin(), Data.end(), OOBSentinelByte); - return; - } - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumElements), - "Saw unsupported component type"); - }); + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [InputMatrix, Layout](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + cpu_oracle::fillPoison(Data.data(), Data.size()); + if (_stricmp(Name, "Input") != 0) + return; + VERIFY_IS_TRUE(cpu_oracle::writeMatrixBuffer(InputMatrix, Layout, Data), + "Unable to encode typed StoreDescriptorOOB input"); + }, + [Layout](ID3D12GraphicsCommandList *, st::ShaderOpTest *Test) { + verifyDescriptorBaseAlignment(Test, "Input", Layout.OffsetBytes); + verifyDescriptorBaseAlignment(Test, "Output", Layout.OffsetBytes); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - const BYTE *Out = static_cast(OutData.data()); - verifyOOBLaneRecords(Out, NumThreads, ForcedWaveSize, OOBRecordSize, - NumElements, Verbose); - - // 0035-linalg-matrix.md: reading an index outside [0, Length()-1] yields - // zero cast to the element type. - for (size_t I = 0; I < NumThreads; ++I) { - const BYTE *Record = Out + I * OOBRecordSize; - uint32_t Executed = 0; - memcpy(&Executed, Record + 4, sizeof(Executed)); - if (Executed != 1) - continue; + VERIFY_IS_TRUE(cpu_oracle::verifyStoreBuffer( + OutData.data(), OutData.size(), {*PerElement, *WholeStore}, + L"HLSL proposal 0035 bounds checking on MatrixStoreToDescriptor: either " + L"the whole store or only the out-of-view element stores become a no-op", + Verbose)); +} - float Just = 0.0f; - float Far = 0.0f; - memcpy(&Just, Record + 8, sizeof(Just)); - memcpy(&Far, Record + 12, sizeof(Far)); - VERIFY_ARE_EQUAL(Just, 0.0f, - "Get at Length() must return zero cast to the element " - "type"); - VERIFY_ARE_EQUAL(Far, 0.0f, - "Get far past Length() must return zero cast to the " - "element type"); - } +// No offset and a tightly packed stride: a matrix occupying the whole buffer. +static cpu_oracle::MatrixBufferLayout packedLayout(const MatrixParams &Params) { + return cpu_oracle::MatrixBufferLayout{ + Params.Layout, + /*OffsetBytes=*/0, + /*StrideBytes=*/Params.strideBytes(), + }; } -void DxilConf_SM610_LinAlg::ElementGetOOB_Wave_4x8_F32() { +// Where the padded cases put the matrix. Independent of the alignment above, +// which is the contract rather than a placement, but constrained by it. +static constexpr size_t DescriptorAlignedOffset = 128; +static_assert(DescriptorAlignedOffset % DescriptorDeclaredAlignment == 0, + "descriptor offset must keep the first element aligned"); + +void DxilConf_SM610_LinAlg::LoadStoreDescriptor_Wave_16x16_F16() { MatrixParams Params = {}; - Params.CompType = ComponentType::F32; - Params.M = 4; - Params.N = 8; - Params.Use = MatrixUse::Accumulator; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::A; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; - Params.Enable16Bit = false; + Params.Enable16Bit = true; UINT SelectedWaveSize = 0; if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementGetOOB_Wave_4x8_F32", + L"LoadStoreDescriptor_Wave_16x16_F16", SelectedWaveSize)) return; - runElementGetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, packedLayout(Params), + packedLayout(Params), VerboseLogging, + SelectedWaveSize); } -static const char ElementSetOOBShader[] = R"( - RWByteAddressBuffer Input : register(u0); - RWByteAddressBuffer Output : register(u1); +// Places the matrix at a non-zero offset and pads the row stride, so the +// destination holds bytes the store must not touch: a 128-byte prologue and +// three 16-byte gaps between its four rows. A store that addresses by element +// index rather than by the supplied stride writes into that padding, which the +// untouched-byte check catches and the element comparison cannot. +void DxilConf_SM610_LinAlg:: + LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main(uint threadID : SV_GroupIndex) { - if (GetGroupWaveIndex() != 0) - return; + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable( + D3DDevice, Params, {Params.Use}, + L"LoadStoreDescriptor_Wave_4x8_F16_RowMajorOffsetPadded", + SelectedWaveSize)) + return; - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); + // A packed row of 8 F16 values is 16 bytes; 32 leaves a 16-byte gap between + // rows while remaining a legal multiple of 16. + const cpu_oracle::MatrixBufferLayout Layout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/32, + }; - uint Len = __builtin_LinAlg_MatrixLength(Mat); + runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, Layout, Layout, + VerboseLogging, SelectedWaveSize); +} - // Both indices are outside this lane's range, so both writes must be - // no-ops and the stored matrix must still equal the loaded one. - __builtin_LinAlg_MatrixSetElement(Mat, Mat, Len, (ELEM_TYPE)POISON_VALUE); - __builtin_LinAlg_MatrixSetElement(Mat, Mat, Len + FAR_OOB_OFFSET, - (ELEM_TYPE)POISON_VALUE); +// Loads RowMajor and stores ColumnMajor, which a shared layout cannot express: +// with the same layout on both sides, an implementation that ignores the +// layout argument entirely still round trips byte-identically, because the +// mapping it applies to the load it applies again to the store. Reading one +// layout and writing the other stops the two from cancelling. +void DxilConf_SM610_LinAlg:: + LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F32; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; - __builtin_LinAlg_MatrixStoreToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable( + D3DDevice, Params, {Params.Use}, + L"LoadStoreDescriptor_Wave_4x8_F32_RowMajorToColumnMajor", + SelectedWaveSize)) + return; - uint Base = MATRIX_BYTES + threadID * OOB_RECORD_SIZE; - Output.Store(Base + 0, Len); - Output.Store(Base + 4, 1); - } -)"; + // Source rows of 8 F32 values are 32 bytes packed, padded here to 48. + const cpu_oracle::MatrixBufferLayout LoadLayout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/48, + }; -static void runElementSetOOB(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - UINT ForcedWaveSize) { - const size_t NumElements = Params.totalElements(); - const size_t NumThreads = Params.NumThreads; - const size_t MatrixSize = Params.totalBytes(); - const size_t OutputBufSize = MatrixSize + NumThreads * OOBRecordSize; + // Destination columns of 4 F32 values are 16 bytes, which is already a legal + // stride, so the column-major side is stored packed. + const cpu_oracle::MatrixBufferLayout StoreLayout = { + MatrixLayout::ColumnMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/16, + }; - // Distinct from every sequential input value, so a stray write is visible. - const int PoisonValue = 999; + runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, LoadLayout, StoreLayout, + VerboseLogging, SelectedWaveSize); +} - std::stringstream ExtraDefs; - ExtraDefs << " -DFAR_OOB_OFFSET=" << FarOOBOffset; - ExtraDefs << " -DOOB_RECORD_SIZE=" << OOBRecordSize; - ExtraDefs << " -DMATRIX_BYTES=" << MatrixSize; - ExtraDefs << " -DPOISON_VALUE=" << PoisonValue; - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); +// The same cross-layout axis on F16, because no tier is required to support +// Fp32 matrices and the F32 case above can skip in its entirety. The shape +// must stay non-square: swapping the two layouts transposes on load and back +// on store, and for a square matrix those cancel byte for byte whatever +// strides are used. +void DxilConf_SM610_LinAlg:: + LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; - compileShader(DxcSupport, ElementSetOOBShader, "cs_6_10", Args, Verbose); + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable( + D3DDevice, Params, {Params.Use}, + L"LoadStoreDescriptor_Wave_4x8_F16_RowMajorToColumnMajor", + SelectedWaveSize)) + return; - // The matrix must come back exactly as it went in. - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1); + // Source rows of 8 F16 values are 16 bytes packed, padded here to 48. + const cpu_oracle::MatrixBufferLayout LoadLayout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/48, + }; - auto Op = createComputeOp(ElementSetOOBShader, "cs_6_10", "UAV(u0), UAV(u1)", - Args.c_str()); - addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", OutputBufSize, true, "byname"); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + // Destination columns of 4 F16 values are 8 bytes, padded here to 16 so the + // column-major side carries a gap of its own rather than sitting packed. + const cpu_oracle::MatrixBufferLayout StoreLayout = { + MatrixLayout::ColumnMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/16, + }; - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [NumElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - if (_stricmp(Name, "Output") == 0) { - std::fill(Data.begin(), Data.end(), OOBSentinelByte); - return; - } - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumElements), - "Saw unsupported component type"); - }); + runLoadStoreDescriptor(D3DDevice, DxcSupport, Params, LoadLayout, StoreLayout, + VerboseLogging, SelectedWaveSize); +} - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); - const BYTE *Out = static_cast(OutData.data()); +// Half the source matrix lies outside the view the descriptor carries. The +// boundary is deliberately placed mid-row rather than on a row boundary, so an +// implementation that bounds checks a row at a time cannot pass it. +void DxilConf_SM610_LinAlg::LoadDescriptorOOB_Wave_16x16_F16_PartialView() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; - verifyOOBLaneRecords(Out + MatrixSize, NumThreads, ForcedWaveSize, - OOBRecordSize, NumElements, Verbose); + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"LoadDescriptorOOB_Wave_16x16_F16_" + L"PartialView", + SelectedWaveSize)) + return; - // 0035-linalg-matrix.md: setting an index outside [0, Length()-1] is a - // no-op, so no poisoned value may appear anywhere in the matrix. - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + // Packed, so the buffer is 16 rows of 32 bytes. A 264 byte view holds the + // first 132 elements: rows 0 to 7 whole, then four elements of row 8. + runLoadDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, + packedLayout(Params), /*InputViewBytes=*/264, + VerboseLogging, SelectedWaveSize); } -void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_4x8_F32() { +// The same behaviour where the matrix is offset and its rows are padded, so +// the view boundary falls in a different place for byte offsets than it does +// for element indices. An implementation that bounds checks by element index +// keeps elements this view does not reach. +void DxilConf_SM610_LinAlg:: + LoadDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView() { MatrixParams Params = {}; - Params.CompType = ComponentType::F32; + Params.CompType = ComponentType::F16; Params.M = 4; Params.N = 8; - Params.Use = MatrixUse::Accumulator; + Params.Use = MatrixUse::A; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; - Params.Enable16Bit = false; + Params.Enable16Bit = true; UINT SelectedWaveSize = 0; if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementSetOOB_Wave_4x8_F32", + L"LoadDescriptorOOB_Wave_4x8_F16_" + L"OffsetPaddedPartialView", SelectedWaveSize)) return; - runElementSetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + const cpu_oracle::MatrixBufferLayout Layout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/32, + }; + + // Elements sit at 128 + 32*Row + 2*Column. A 172 byte view holds row 0 + // whole and columns 0 to 5 of row 1, so it cuts within a row and stops + // short of the padding rather than on it. + runLoadDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, Layout, + /*InputViewBytes=*/172, VerboseLogging, + SelectedWaveSize); } -// Out-of-bounds element access on F16. Both cases above pin the boundary -// behaviour to F32, which no tier is required to support, so a conforming -// F16-only device would exercise neither. -void DxilConf_SM610_LinAlg::ElementGetOOB_Wave_16x16_F16() { +// The same two views on the destination instead of the source, so the rule +// being exercised is bounds checking on the store rather than on the load. +void DxilConf_SM610_LinAlg::StoreDescriptorOOB_Wave_16x16_F16_PartialView() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Use = MatrixUse::Accumulator; + Params.Use = MatrixUse::A; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; @@ -3746,20 +3726,27 @@ void DxilConf_SM610_LinAlg::ElementGetOOB_Wave_16x16_F16() { UINT SelectedWaveSize = 0; if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementGetOOB_Wave_16x16_F16", + L"StoreDescriptorOOB_Wave_16x16_F16_" + L"PartialView", SelectedWaveSize)) return; - runElementGetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + // Packed, so the buffer is 16 rows of 32 bytes. A 260 byte view admits the + // first 130 elements: rows 0 to 7 whole, then two of row 8. Ending two + // elements into the row keeps the boundary off the round multiples a + // coarser-than-per-element bounds check would land on. + runStoreDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, + packedLayout(Params), /*OutputViewBytes=*/260, + VerboseLogging, SelectedWaveSize); } -void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_16x16_F16() { +void DxilConf_SM610_LinAlg:: + StoreDescriptorOOB_Wave_4x8_F16_OffsetPaddedPartialView() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; - Params.Use = MatrixUse::Accumulator; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::A; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; @@ -3767,18 +3754,27 @@ void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_16x16_F16() { UINT SelectedWaveSize = 0; if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"ElementSetOOB_Wave_16x16_F16", + L"StoreDescriptorOOB_Wave_4x8_F16_" + L"OffsetPaddedPartialView", SelectedWaveSize)) return; - runElementSetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + const cpu_oracle::MatrixBufferLayout Layout = { + MatrixLayout::RowMajor, + /*OffsetBytes=*/DescriptorAlignedOffset, + /*StrideBytes=*/32, + }; + + // Elements sit at 128 + 32*Row + 2*Column. A 172 byte view holds row 0 + // whole and columns 0 to 5 of row 1, so it cuts within a row and stops + // short of the padding rather than on it. + runStoreDescriptorOutOfBounds(D3DDevice, DxcSupport, Params, Layout, + /*OutputViewBytes=*/172, VerboseLogging, + SelectedWaveSize); } -static const char CopyConvertShader[] = R"( - RWByteAddressBuffer Input : register(u0); - RWByteAddressBuffer Output : register(u1); - RWByteAddressBuffer SourceAfter : register(u2); +static const char SplatStoreShader[] = R"( + RWByteAddressBuffer Output : register(u0); #ifdef FORCED_WAVE_SIZE [WaveSize(FORCED_WAVE_SIZE)] @@ -3792,301 +3788,317 @@ static const char CopyConvertShader[] = R"( __builtin_LinAlgMatrix [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Src; - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(DST_COMP_TYPE, DST_M_DIM, DST_N_DIM, USE, SCOPE)]] - Dst; - - __builtin_LinAlg_MatrixLoadFromDescriptor( - Src, Input, 0, SRC_STRIDE, LAYOUT, 128); - __builtin_LinAlg_CopyConvertMatrix(Dst, Src, TRANSPOSE); - __builtin_LinAlg_MatrixStoreToDescriptor( - Dst, Output, 0, DST_STRIDE, LAYOUT, 128); + Mat; + __builtin_LinAlg_FillMatrix(Mat, FILL_VALUE); __builtin_LinAlg_MatrixStoreToDescriptor( - Src, SourceAfter, 0, SRC_STRIDE, LAYOUT, 128); + Mat, Output, 0, STRIDE, LAYOUT, 128); } )"; -static HRESULT selectCopyConvertWaveSize(ID3D12Device *Device, - const MatrixParams &Params, - ComponentType DestinationCompType, - bool Transpose, bool &Supported, - UINT &SelectedWaveSize) { - Supported = false; - SelectedWaveSize = 0; - if (!Device || Params.Use != MatrixUse::A || - !linalg_test::isLegalScope( - linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, - Params.Scope)) - return E_INVALIDARG; +static void runSplatStore(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, float FillValue, + bool Verbose, UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); - std::optional SourceType = - toCapabilityDataType(Params.CompType); - std::optional DestinationType = - toCapabilityDataType(DestinationCompType); - if (!SourceType.has_value() || !DestinationType.has_value()) - return E_INVALIDARG; + std::stringstream ExtraDefs; + STREAM_FLOAT(ExtraDefs, "FILL_VALUE", FillValue); - linalg_test::TierSupport Tier; - HRESULT HR = linalg_test::queryTierSupport(Device, Tier); - if (FAILED(HR) || !Tier.supported()) - return HR; + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - UINT MinWaveSize = 0; - UINT MaxWaveSize = 0; - HR = queryLaunchableWaveSizes(Device, MinWaveSize, MaxWaveSize); - if (FAILED(HR)) - return HR; - if (MinWaveSize == 0) { - hlsl_test::LogCommentFmt( - L"Wave operations are unsupported; MatrixConstruction is not " - L"applicable"); - return S_OK; - } + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - MatrixParams Destination = Params; - Destination.CompType = DestinationCompType; - if (Transpose) { - Destination.M = Params.N; - Destination.N = Params.M; - } + compileShader(DxcSupport, SplatStoreShader, "cs_6_10", Args, Verbose); - for (UINT WaveSize = 4; WaveSize <= 128; WaveSize *= 2) { - if (WaveSize < MinWaveSize || WaveSize > MaxWaveSize || - WaveSize > static_cast(Params.NumThreads)) - continue; + auto Expected = + makeExpectedMat(Params.CompType, Params.M, Params.N, FillValue, false); - bool SourceSupported = false; - HR = supportsMatrixShape(Device, *SourceType, WaveSize, MatrixUse::A, - Params.M, Params.N, SourceSupported); - if (FAILED(HR)) - return HR; + auto Op = + createComputeOp(SplatStoreShader, "cs_6_10", "UAV(u0)", Args.c_str()); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Output"); - bool DestinationSupported = false; - HR = - supportsMatrixShape(Device, *DestinationType, WaveSize, MatrixUse::A, - Destination.M, Destination.N, DestinationSupported); - if (FAILED(HR)) - return HR; + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); - if (SourceSupported && DestinationSupported) { - hlsl_test::LogCommentFmt( - L"CopyConvert capability matched wave=%u for source=%ux%u and " - L"destination=%ux%u", - WaveSize, Params.M, Params.N, Destination.M, Destination.N); - Supported = true; - SelectedWaveSize = WaveSize; - return S_OK; - } - } + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); - hlsl_test::LogCommentFmt( - L"No MatrixConstruction query supports CopyConvert source=%ux%u and " - L"destination=%ux%u for any wave size launchable within shader " - L"WaveSize(4,128) and a %d-thread group", - Params.M, Params.N, Destination.M, Destination.N, Params.NumThreads); - return S_OK; + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); } -static bool copyConvertApplicable(ID3D12Device *Device, - const MatrixParams &Params, - ComponentType DestinationCompType, - bool Transpose, LPCWSTR CaseName, - UINT &SelectedWaveSize) { - bool Supported = false; - const HRESULT QueryResult = - selectCopyConvertWaveSize(Device, Params, DestinationCompType, Transpose, - Supported, SelectedWaveSize); - if (!applyApplicability( - linalg_test::classifyApplicability( - QueryResult, Supported, - linalg_test::CapabilityRequirement::CapabilityGated), - CaseName)) - return false; +void DxilConf_SM610_LinAlg::SplatStore_Wave_16x16_F16() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::Accumulator; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; - VERIFY_IS_TRUE(SelectedWaveSize != 0, - "A case cleared to run must have a selected wave size"); - return true; + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"SplatStore_Wave_16x16_F16", + SelectedWaveSize)) + return; + + runSplatStore(D3DDevice, DxcSupport, Params, 42.0f, VerboseLogging, + SelectedWaveSize); } -static void runCopyConvert(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, - ComponentType DestinationCompType, bool Verbose, - bool Transpose, UINT ForcedWaveSize = 0) { - MatrixParams DstParams = Params; - DstParams.CompType = DestinationCompType; - if (Transpose) { - DstParams.M = Params.N; - DstParams.N = Params.M; - } +static const char AccumulateDescriptorShader[] = R"( + #define USE_ACC 2 - std::stringstream ExtraDefs; - ExtraDefs << " -DTRANSPOSE=" << Transpose; - ExtraDefs << " -DDST_COMP_TYPE=" << static_cast(DestinationCompType); - ExtraDefs << " -DDST_M_DIM=" << DstParams.M; - ExtraDefs << " -DDST_N_DIM=" << DstParams.N; - ExtraDefs << " -DSRC_STRIDE=" << Params.strideBytes(); - ExtraDefs << " -DDST_STRIDE=" << DstParams.strideBytes(); - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + ByteAddressBuffer Input : register(t0); + RWByteAddressBuffer Output : register(u1); - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif + [numthreads(NUMTHREADS, 1, 1)] + void main() { + if (GetGroupWaveIndex() != 0) + return; - compileShader(DxcSupport, CopyConvertShader, "cs_6_10", Args, Verbose); + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); + } +)"; - std::optional Input = - cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); - VERIFY_IS_TRUE(Input.has_value(), - "Unable to construct typed CopyConvert input"); - std::optional Converted = - cpu_oracle::makeSequentialMatrix(DstParams.CompType, Params.M, Params.N); - VERIFY_IS_TRUE(Converted.has_value(), - "Unable to construct typed CopyConvert conversion oracle"); - if (!Input.has_value() || !Converted.has_value()) - return; - std::optional Expected = - Transpose ? cpu_oracle::transposeMatrix(*Converted) : Converted; - VERIFY_IS_TRUE(Expected.has_value(), - "Unable to construct independent CopyConvert oracle"); - if (!Expected.has_value()) - return; +static void runAccumulateDescriptor(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, int FillValue, + bool Verbose, UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); - cpu_oracle::MatrixBufferLayout SourceLayout = { - Params.Layout, - /*OffsetBytes=*/0, - /*StrideBytes=*/Params.strideBytes(), - }; - cpu_oracle::MatrixBufferLayout DestinationLayout = { - DstParams.Layout, - /*OffsetBytes=*/0, - /*StrideBytes=*/DstParams.strideBytes(), - }; - std::optional SourceBufferSize = - cpu_oracle::getMatrixBufferSize(*Input, SourceLayout); - std::optional DestinationBufferSize = - cpu_oracle::getMatrixBufferSize(*Expected, DestinationLayout); - VERIFY_IS_TRUE(SourceBufferSize.has_value(), - "Unable to size typed CopyConvert input"); - VERIFY_IS_TRUE(DestinationBufferSize.has_value(), - "Unable to size typed CopyConvert output"); - if (!SourceBufferSize.has_value() || !DestinationBufferSize.has_value()) - return; + std::stringstream ExtraDefs; + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - cpu_oracle::TypedMatrix InputMatrix = *Input; - cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::exactResult( - *Expected, - L"HLSL proposal 0035 CopyConvertMatrix transpose and descriptor layout"); - cpu_oracle::MatrixResultOracle SourceOracle = cpu_oracle::exactResult( - *Input, L"CopyConvertMatrix leaves the source matrix unmodified"); + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - auto Op = createComputeOp(CopyConvertShader, "cs_6_10", - "UAV(u0), UAV(u1), UAV(u2)", Args.c_str()); - addUAVBuffer(Op.get(), "Input", *SourceBufferSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", *DestinationBufferSize, true); - addUAVBuffer(Op.get(), "SourceAfter", *SourceBufferSize, true); + compileShader(DxcSupport, AccumulateDescriptorShader, "cs_6_10", Args, + Verbose); + + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, + static_cast(FillValue) * 2, false); + + auto Op = createComputeOp(AccumulateDescriptorShader, "cs_6_10", + "SRV(t0), UAV(u1)", Args.c_str()); + addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); + addUAVBuffer(Op.get(), "Output", BufferSize, true); addRootView(Op.get(), 0, "Input"); addRootView(Op.get(), 1, "Output"); - addRootView(Op.get(), 2, "SourceAfter"); auto Result = runShaderOp( Device, DxcSupport, std::move(Op), - [InputMatrix, SourceLayout](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - if (_stricmp(Name, "Input") != 0) - return; - VERIFY_IS_TRUE( - cpu_oracle::writeMatrixBuffer(InputMatrix, SourceLayout, Data), - "Unable to encode typed CopyConvert input"); + [NumElements, Params, FillValue](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, NumElements, + /*StartingVal=*/FillValue, + /*Increment=*/false), + "Saw unsupported component type"); }); MappedData OutData; - MappedData SourceAfterData; Result->Test->GetReadBackData("Output", &OutData); - Result->Test->GetReadBackData("SourceAfter", &SourceAfterData); - VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer( - OutData.data(), OutData.size(), DestinationLayout, Oracle, Verbose)); - VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer( - SourceAfterData.data(), SourceAfterData.size(), SourceLayout, - SourceOracle, Verbose)); + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); } -void DxilConf_SM610_LinAlg::CopyConvert_Wave_16x16_F16() { +void DxilConf_SM610_LinAlg::AccumulateDescriptor_Wave_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Use = MatrixUse::A; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; Params.Enable16Bit = true; UINT SelectedWaveSize = 0; - if (!copyConvertApplicable(D3DDevice, Params, ComponentType::F16, - /*Transpose=*/false, L"CopyConvert_Wave_16x16_F16", - SelectedWaveSize)) + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"AccumulateDescriptor_Wave_16x16_F16", + SelectedWaveSize)) + return; + if (!accumulateStoreApplicable( + D3DDevice, Params.CompType, + linalg_test::AtomicDestination::RWByteAddressBuffer, + L"AccumulateDescriptor_Wave_16x16_F16")) return; - runCopyConvert(D3DDevice, DxcSupport, Params, ComponentType::F16, - VerboseLogging, - /*Transpose=*/false, SelectedWaveSize); + runAccumulateDescriptor(D3DDevice, DxcSupport, Params, 12, VerboseLogging, + SelectedWaveSize); } -void DxilConf_SM610_LinAlg::CopyConvert_Wave_16x16_F16_Transpose() { +// Element access constructs a wave-scope matrix and then reads or writes its +// components, so applicability is exactly MatrixConstruction for the tile the +// case declares. D3D12LinearAlgebraRuntimeFeatureSupport.md guarantees only +// that some shape whose largest component is 16 or less is reported for a +// supported type, and directs applications wanting smaller shapes to query +// them case by case. Neither Fp32 nor Fp16 matrices are required at Tier 1, so +// every element-access case is capability gated rather than mandatory. +static const char ElementAccessShader[] = R"( + RWByteAddressBuffer Input : register(u0); + RWByteAddressBuffer Output : register(u1); + + // flatten the 2D index into a 1D index then scale by element size + // Always store row-major and work it out in the test runner + uint coordToByteOffset(uint2 coord) { + return (coord.x * N_DIM + coord.y) * ELEM_SIZE; + } + + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif + [numthreads(NUMTHREADS, 1, 1)] + void main(uint threadID : SV_GroupIndex) { + if (GetGroupWaveIndex() != 0) + return; + + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); + + // Copy Matrix values from input to output without assuming order + for (uint I = 0; I < __builtin_LinAlg_MatrixLength(Mat); ++I) { + uint2 Coord = __builtin_LinAlg_MatrixGetCoordinate(Mat, I); + uint Offset = coordToByteOffset(Coord); + ELEM_TYPE Elem; + __builtin_LinAlg_MatrixGetElement(Elem, Mat, I); + Output.Store(Offset, Elem); + } + + // Save the matrix length that this thread saw. The length is written + // to the output right after the matrix, offset by the thread index + uint LenIdx = (M_DIM * N_DIM * ELEM_SIZE) + (threadID * sizeof(uint)); + uint Len = __builtin_LinAlg_MatrixLength(Mat); + Output.Store(LenIdx, Len); + } +)"; + +static void runElementAccess(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t NumThreads = Params.NumThreads; + const size_t MatrixSize = Params.totalBytes(); + // OutputBuf needs to fit the Matrix plus one uint per thread + const size_t OutputBufSize = MatrixSize + NumThreads * sizeof(uint32_t); + + std::stringstream ExtraDefs; + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + + compileShader(DxcSupport, ElementAccessShader, "cs_6_10", Args, Verbose); + + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1); + + auto Op = createComputeOp(ElementAccessShader, "cs_6_10", "UAV(u0), UAV(u1)", + Args.c_str()); + addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", OutputBufSize, true); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); + + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [NumElements, Params](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, + NumElements), + "Saw unsupported component type"); + }); + + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); + + // Verify the front of the buffer is a list of elements of the expected type + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); + + // Verify the end of the buffer is NumThreads number of lengths, whose + // sum is greater than or equal to NumElements + const BYTE *Out = static_cast(OutData.data()); + const uint32_t *Lengths = + reinterpret_cast(Out + MatrixSize); + uint32_t TotalLength = 0; + for (size_t I = 0; I < NumThreads; ++I) + TotalLength += Lengths[I]; + VERIFY_IS_GREATER_THAN_OR_EQUAL( + TotalLength, NumElements, "Sum of all lengths must be gte num elements"); +} + +void DxilConf_SM610_LinAlg::ElementAccess_Wave_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Use = MatrixUse::A; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; Params.Enable16Bit = true; UINT SelectedWaveSize = 0; - if (!copyConvertApplicable(D3DDevice, Params, ComponentType::F16, - /*Transpose=*/true, - L"CopyConvert_Wave_16x16_F16_Transpose", - SelectedWaveSize)) + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementAccess_Wave_16x16_F16", + SelectedWaveSize)) return; - runCopyConvert(D3DDevice, DxcSupport, Params, ComponentType::F16, - VerboseLogging, - /*Transpose=*/true, SelectedWaveSize); + runElementAccess(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); } -void DxilConf_SM610_LinAlg::CopyConvert_Wave_4x8_F32_Transpose() { +void DxilConf_SM610_LinAlg::ElementAccess_Wave_4x8_F32() { MatrixParams Params = {}; Params.CompType = ComponentType::F32; Params.M = 4; Params.N = 8; - Params.Use = MatrixUse::A; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; Params.Enable16Bit = false; UINT SelectedWaveSize = 0; - if (!copyConvertApplicable(D3DDevice, Params, ComponentType::F32, - /*Transpose=*/true, - L"CopyConvert_Wave_4x8_F32_Transpose", - SelectedWaveSize)) + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementAccess_Wave_4x8_F32", + SelectedWaveSize)) return; - // Non-square dimensions make the destination shape and row stride observable. - runCopyConvert(D3DDevice, DxcSupport, Params, ComponentType::F32, - VerboseLogging, - /*Transpose=*/true, SelectedWaveSize); + // Non-square dimensions make the row-major coordinate mapping observable: a + // transposed GetCoordinate would land inside the matrix for a square tile + // but out of it here. + runElementAccess(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); } -static const char MatMatMulShader[] = R"( - #define USE_A 0 - #define USE_B 1 - #define USE_ACC 2 - - RWByteAddressBuffer Output : register(u0); +static const char ElementSetShader[] = R"( + RWByteAddressBuffer Input : register(u0); + RWByteAddressBuffer Output : register(u1); #ifdef FORCED_WAVE_SIZE [WaveSize(FORCED_WAVE_SIZE)] @@ -4099,86 +4111,111 @@ static const char MatMatMulShader[] = R"( return; __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, K_DIM, USE_A, SCOPE)]] - MatA; - __builtin_LinAlg_FillMatrix(MatA, A_FILL); - - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, K_DIM, N_DIM, USE_B, SCOPE)]] - MatB; - __builtin_LinAlg_FillMatrix(MatB, B_FILL); + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] - MatC; - __builtin_LinAlg_MatrixMatrixMultiply(MatC, MatA, MatB); + // Increment every element by 5 + for (uint I = 0; I < __builtin_LinAlg_MatrixLength(Mat); ++I) { + ELEM_TYPE Elem; + __builtin_LinAlg_MatrixGetElement(Elem, Mat, I); + Elem = Elem + 5; + __builtin_LinAlg_MatrixSetElement(Mat, Mat, I, Elem); + } __builtin_LinAlg_MatrixStoreToDescriptor( - MatC, Output, 0, STRIDE, LAYOUT, 128); + Mat, Output, 0, STRIDE, LAYOUT, 128); } )"; -static void runMatMatMul(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, MatrixDim K, - float AFill, float BFill, UINT ForcedWaveSize = 0) { +static void runElementSet(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + UINT ForcedWaveSize = 0) { const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); + const size_t MatrixSize = Params.totalBytes(); std::stringstream ExtraDefs; - ExtraDefs << " -DK_DIM=" << K; - STREAM_FLOAT(ExtraDefs, "A_FILL", AFill); - STREAM_FLOAT(ExtraDefs, "B_FILL", BFill); - if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, MatMatMulShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, ElementSetShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, - AFill * BFill * K, /*Increment=*/false); + // Start counting from 6 since each element was increased by 5 + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 6); - auto Op = - createComputeOp(MatMatMulShader, "cs_6_10", "UAV(u0)", Args.c_str()); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); + auto Op = createComputeOp(ElementSetShader, "cs_6_10", "UAV(u0), UAV(u1)", + Args.c_str()); + addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", MatrixSize, true); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [NumElements, Params](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, + NumElements), + "Saw unsupported component type"); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); + // Verify the front of the buffer is a list of elements of the expected type VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), Expected, NumElements, Verbose)); } -void DxilConf_SM610_LinAlg::MatMatMul_Wave_16x16x16_F16() { +void DxilConf_SM610_LinAlg::ElementSet_Wave_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; Params.Enable16Bit = true; UINT SelectedWaveSize = 0; - if (!waveMatMulApplicable(D3DDevice, Params, /*K=*/16, - L"MatMatMul_Wave_16x16x16_F16", SelectedWaveSize)) + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementSet_Wave_16x16_F16", + SelectedWaveSize)) return; - runMatMatMul(D3DDevice, DxcSupport, Params, VerboseLogging, /*K=*/16, - /*AFill=*/2.0f, /*BFill=*/3.0f, SelectedWaveSize); + runElementSet(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); } -static const char MatMatMulAccumShader[] = R"( - #define USE_A 0 - #define USE_B 1 - #define USE_ACC 2 +// Length() is thread local, so the first index past a lane's own length is +// already out of bounds even though the wave collectively holds more elements. +// Probing Length() and a index far beyond it covers both a driver that clamps +// only at the wave total and one that wraps a large index back into range. +static constexpr UINT FarOOBOffset = 64; - RWByteAddressBuffer Output : register(u0); +// Per-lane record: {uint Length, uint Executed, float Just, float Far}. +static constexpr UINT OOBRecordSize = 16; + +// Seeds every output byte so a lane that never writes cannot be mistaken for a +// lane that correctly wrote the specified zero. The output buffer must be +// created "byname" for this to run at all: ShaderOpTest only invokes the +// initializer callback for that mode, and the default "zero" mode would leave +// the buffer holding exactly the value the out-of-bounds read is required to +// produce, making the comparison vacuous. +static constexpr BYTE OOBSentinelByte = 0xCD; + +// Seeds the shader's destination locals. Distinct from zero, so a read that is +// dropped rather than performed cannot masquerade as a correct out-of-bounds +// result, and exactly representable in F32. +static constexpr int OOBGetPoisonValue = 999; + +static const char ElementGetOOBShader[] = R"( + RWByteAddressBuffer Input : register(u0); + RWByteAddressBuffer Output : register(u1); #ifdef FORCED_WAVE_SIZE [WaveSize(FORCED_WAVE_SIZE)] @@ -4186,97 +4223,164 @@ static const char MatMatMulAccumShader[] = R"( [WaveSize(4, 128)] #endif [numthreads(NUMTHREADS, 1, 1)] - void main() { + void main(uint threadID : SV_GroupIndex) { if (GetGroupWaveIndex() != 0) return; __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, K_DIM, USE_A, SCOPE)]] - MatA; - __builtin_LinAlg_FillMatrix(MatA, A_FILL); - - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, K_DIM, N_DIM, USE_B, SCOPE)]] - MatB; - __builtin_LinAlg_FillMatrix(MatB, B_FILL); + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] - MatC; - __builtin_LinAlg_FillMatrix(MatC, C_FILL); + uint Len = __builtin_LinAlg_MatrixLength(Mat); - __builtin_LinAlg_MatrixMatrixMultiplyAccumulate(MatC, MatA, MatB, MatC); + // Seeded so that a dropped read leaves a value distinguishable from the + // zero a correct out-of-bounds read must produce. + ELEM_TYPE Just = (ELEM_TYPE)POISON_VALUE; + __builtin_LinAlg_MatrixGetElement(Just, Mat, Len); + ELEM_TYPE Far = (ELEM_TYPE)POISON_VALUE; + __builtin_LinAlg_MatrixGetElement(Far, Mat, Len + FAR_OOB_OFFSET); - __builtin_LinAlg_MatrixStoreToDescriptor( - MatC, Output, 0, STRIDE, LAYOUT, 128); + // Record unconditionally so the runner can tell that this lane ran. + uint Base = threadID * OOB_RECORD_SIZE; + Output.Store(Base + 0, Len); + Output.Store(Base + 4, 1); + // Widened to float so the runner can read a fixed-width record whatever + // the element type: a half store would leave the record's upper two bytes + // holding the sentinel. Half to float is lossless, so no value is masked. + Output.Store(Base + 8, (float)Just); + Output.Store(Base + 12, (float)Far); } )"; -static void runMatMatMulAccum(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - MatrixDim K, float AFill, float BFill, - float CFill, UINT ForcedWaveSize = 0) { +// Reads back the {Length, Executed} half of each lane record and checks the +// wave actually ran. Returns the total element count the wave reported. +static uint32_t verifyOOBLaneRecords(const BYTE *Records, size_t NumThreads, + UINT SelectedWaveSize, UINT RecordStride, + size_t NumElements, bool Verbose) { + uint32_t ExecutedLanes = 0; + uint32_t TotalLength = 0; + for (size_t I = 0; I < NumThreads; ++I) { + const BYTE *Record = Records + I * RecordStride; + uint32_t Length = 0; + uint32_t Executed = 0; + memcpy(&Length, Record, sizeof(Length)); + memcpy(&Executed, Record + 4, sizeof(Executed)); + if (Executed != 1) + continue; + ++ExecutedLanes; + TotalLength += Length; + if (Verbose) + hlsl_test::LogCommentFmt(L"lane %u reported Length=%u", + static_cast(I), Length); + } + + // Only wave 0 runs, so exactly the lanes of the wave the capability query + // selected must have written a record. + VERIFY_ARE_EQUAL(ExecutedLanes, SelectedWaveSize, + "Every lane of the selected wave must execute"); + VERIFY_IS_GREATER_THAN_OR_EQUAL( + TotalLength, static_cast(NumElements), + "Sum of all lengths must be gte num elements"); + return TotalLength; +} + +static void runElementGetOOB(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + UINT ForcedWaveSize) { + VERIFY_IS_TRUE(Params.CompType == ComponentType::F32 || + Params.CompType == ComponentType::F16, + "Out-of-bounds Get records widen the element to float"); const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); + const size_t NumThreads = Params.NumThreads; + const size_t MatrixSize = Params.totalBytes(); + const size_t OutputBufSize = NumThreads * OOBRecordSize; std::stringstream ExtraDefs; - ExtraDefs << " -DK_DIM=" << K; - STREAM_FLOAT(ExtraDefs, "A_FILL", AFill); - STREAM_FLOAT(ExtraDefs, "B_FILL", BFill); - STREAM_FLOAT(ExtraDefs, "C_FILL", CFill); - + ExtraDefs << " -DFAR_OOB_OFFSET=" << FarOOBOffset; + ExtraDefs << " -DOOB_RECORD_SIZE=" << OOBRecordSize; + ExtraDefs << " -DPOISON_VALUE=" << OOBGetPoisonValue; if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, MatMatMulAccumShader, "cs_6_10", Args, Verbose); - - auto Expected = - makeExpectedMat(Params.CompType, Params.M, Params.N, - AFill * BFill * K + CFill, /*Increment=*/false); + compileShader(DxcSupport, ElementGetOOBShader, "cs_6_10", Args, Verbose); - auto Op = - createComputeOp(MatMatMulAccumShader, "cs_6_10", "UAV(u0)", Args.c_str()); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); + auto Op = createComputeOp(ElementGetOOBShader, "cs_6_10", "UAV(u0), UAV(u1)", + Args.c_str()); + addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", OutputBufSize, true, "byname"); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [NumElements, Params](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + if (_stricmp(Name, "Output") == 0) { + std::fill(Data.begin(), Data.end(), OOBSentinelByte); + return; + } + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, + NumElements), + "Saw unsupported component type"); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); + const BYTE *Out = static_cast(OutData.data()); - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + verifyOOBLaneRecords(Out, NumThreads, ForcedWaveSize, OOBRecordSize, + NumElements, Verbose); + + // 0035-linalg-matrix.md: reading an index outside [0, Length()-1] yields + // zero cast to the element type. + for (size_t I = 0; I < NumThreads; ++I) { + const BYTE *Record = Out + I * OOBRecordSize; + uint32_t Executed = 0; + memcpy(&Executed, Record + 4, sizeof(Executed)); + if (Executed != 1) + continue; + + float Just = 0.0f; + float Far = 0.0f; + memcpy(&Just, Record + 8, sizeof(Just)); + memcpy(&Far, Record + 12, sizeof(Far)); + VERIFY_ARE_EQUAL(Just, 0.0f, + "Get at Length() must return zero cast to the element " + "type"); + VERIFY_ARE_EQUAL(Far, 0.0f, + "Get far past Length() must return zero cast to the " + "element type"); + } } -void DxilConf_SM610_LinAlg::MatMatMulAccum_Wave_16x16x16_F16() { +void DxilConf_SM610_LinAlg::ElementGetOOB_Wave_4x8_F32() { MatrixParams Params = {}; - Params.CompType = ComponentType::F16; - Params.M = 16; - Params.N = 16; + Params.CompType = ComponentType::F32; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; - Params.Enable16Bit = true; - - UINT SelectedWaveSize = 0; - if (!waveMatMulApplicable(D3DDevice, Params, /*K=*/16, - L"MatMatMulAccum_Wave_16x16x16_F16", - SelectedWaveSize)) + Params.Enable16Bit = false; + + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementGetOOB_Wave_4x8_F32", + SelectedWaveSize)) return; - runMatMatMulAccum(D3DDevice, DxcSupport, Params, VerboseLogging, /*K=*/16, - /*AFill=*/2.0f, /*BFill=*/3.0f, /*CFill=*/4.0f, - SelectedWaveSize); + runElementGetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); } -static const char MatAccumShader[] = R"( - #define USE_A 0 - #define USE_ACC 2 - - RWByteAddressBuffer Output : register(u0); +static const char ElementSetOOBShader[] = R"( + RWByteAddressBuffer Input : register(u0); + RWByteAddressBuffer Output : register(u1); #ifdef FORCED_WAVE_SIZE [WaveSize(FORCED_WAVE_SIZE)] @@ -4284,600 +4388,710 @@ static const char MatAccumShader[] = R"( [WaveSize(4, 128)] #endif [numthreads(NUMTHREADS, 1, 1)] - void main() { + void main(uint threadID : SV_GroupIndex) { if (GetGroupWaveIndex() != 0) return; __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] - MatLHS; - __builtin_LinAlg_FillMatrix(MatLHS, LHS_FILL); + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE)]] - MatRHS; - __builtin_LinAlg_FillMatrix(MatRHS, RHS_FILL); + uint Len = __builtin_LinAlg_MatrixLength(Mat); - __builtin_LinAlg_MatrixAccumulate(MatLHS, MatLHS, MatRHS); + // Both indices are outside this lane's range, so both writes must be + // no-ops and the stored matrix must still equal the loaded one. + __builtin_LinAlg_MatrixSetElement(Mat, Mat, Len, (ELEM_TYPE)POISON_VALUE); + __builtin_LinAlg_MatrixSetElement(Mat, Mat, Len + FAR_OOB_OFFSET, + (ELEM_TYPE)POISON_VALUE); __builtin_LinAlg_MatrixStoreToDescriptor( - MatLHS, Output, 0, STRIDE, LAYOUT, 128); + Mat, Output, 0, STRIDE, LAYOUT, 128); + + uint Base = MATRIX_BYTES + threadID * OOB_RECORD_SIZE; + Output.Store(Base + 0, Len); + Output.Store(Base + 4, 1); } )"; -static void runMatAccum(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, float LHSFill, - float RHSFill, UINT ForcedWaveSize = 0) { +static void runElementSetOOB(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + UINT ForcedWaveSize) { const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); + const size_t NumThreads = Params.NumThreads; + const size_t MatrixSize = Params.totalBytes(); + const size_t OutputBufSize = MatrixSize + NumThreads * OOBRecordSize; - std::stringstream ExtraDefs; - STREAM_FLOAT(ExtraDefs, "LHS_FILL", LHSFill); - STREAM_FLOAT(ExtraDefs, "RHS_FILL", RHSFill); + // Distinct from every sequential input value, so a stray write is visible. + const int PoisonValue = 999; + std::stringstream ExtraDefs; + ExtraDefs << " -DFAR_OOB_OFFSET=" << FarOOBOffset; + ExtraDefs << " -DOOB_RECORD_SIZE=" << OOBRecordSize; + ExtraDefs << " -DMATRIX_BYTES=" << MatrixSize; + ExtraDefs << " -DPOISON_VALUE=" << PoisonValue; if (ForcedWaveSize != 0) ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, MatAccumShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, ElementSetOOBShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, - LHSFill + RHSFill, /*Increment=*/false); + // The matrix must come back exactly as it went in. + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1); - auto Op = createComputeOp(MatAccumShader, "cs_6_10", "UAV(u0)", Args.c_str()); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); + auto Op = createComputeOp(ElementSetOOBShader, "cs_6_10", "UAV(u0), UAV(u1)", + Args.c_str()); + addUAVBuffer(Op.get(), "Input", MatrixSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", OutputBufSize, true, "byname"); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [NumElements, Params](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + if (_stricmp(Name, "Output") == 0) { + std::fill(Data.begin(), Data.end(), OOBSentinelByte); + return; + } + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, + NumElements), + "Saw unsupported component type"); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); + const BYTE *Out = static_cast(OutData.data()); + verifyOOBLaneRecords(Out + MatrixSize, NumThreads, ForcedWaveSize, + OOBRecordSize, NumElements, Verbose); + + // 0035-linalg-matrix.md: setting an index outside [0, Length()-1] is a + // no-op, so no poisoned value may appear anywhere in the matrix. VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), Expected, NumElements, Verbose)); } -void DxilConf_SM610_LinAlg::MatAccum_Wave_16x16_F16() { +void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_4x8_F32() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F32; + Params.M = 4; + Params.N = 8; + Params.Use = MatrixUse::Accumulator; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = false; + + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementSetOOB_Wave_4x8_F32", + SelectedWaveSize)) + return; + + runElementSetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); +} + +// Out-of-bounds element access on F16. Both cases above pin the boundary +// behaviour to F32, which no tier is required to support, so a conforming +// F16-only device would exercise neither. +void DxilConf_SM610_LinAlg::ElementGetOOB_Wave_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; + Params.Use = MatrixUse::Accumulator; Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; Params.NumThreads = 128; Params.Enable16Bit = true; - // MatAccum builds both an accumulator and an A matrix, and the two roles pin - // different extents of the same shape, so both must be constructible. UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable( - D3DDevice, Params, {MatrixUse::Accumulator, MatrixUse::A}, - L"MatAccum_Wave_16x16_F16", SelectedWaveSize)) + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementGetOOB_Wave_16x16_F16", + SelectedWaveSize)) return; - runMatAccum(D3DDevice, DxcSupport, Params, VerboseLogging, - /*LHSFill=*/2.0f, /*RHSFill=*/3.0f, SelectedWaveSize); + runElementGetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); } -static const char MatVecMulShader[] = R"( - #define USE_A 0 - #define SCOPE_THREAD 0 +void DxilConf_SM610_LinAlg::ElementSetOOB_Wave_16x16_F16() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::Accumulator; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; - ByteAddressBuffer Input : register(t0); + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"ElementSetOOB_Wave_16x16_F16", + SelectedWaveSize)) + return; + + runElementSetOOB(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); +} + +static const char CopyConvertShader[] = R"( + RWByteAddressBuffer Input : register(u0); RWByteAddressBuffer Output : register(u1); + RWByteAddressBuffer SourceAfter : register(u2); + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif [numthreads(NUMTHREADS, 1, 1)] void main() { - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); - - vector InVec; - for (uint I = 0; I < N_DIM; ++I) { - InVec[I] = Input.Load(I * ELEM_SIZE); - } + if (GetGroupWaveIndex() != 0) + return; - vector OutVec; - __builtin_LinAlg_MatrixVectorMultiply( - OutVec, Mat, OUTPUT_SIGNED, InVec, IN_INTERP); + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Src; + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(DST_COMP_TYPE, DST_M_DIM, DST_N_DIM, USE, SCOPE)]] + Dst; - for (uint I = 0; I < M_DIM; ++I) { - Output.Store(I * ELEM_SIZE, OutVec[I]); - } + __builtin_LinAlg_MatrixLoadFromDescriptor( + Src, Input, 0, SRC_STRIDE, LAYOUT, 128); + __builtin_LinAlg_CopyConvertMatrix(Dst, Src, TRANSPOSE); + __builtin_LinAlg_MatrixStoreToDescriptor( + Dst, Output, 0, DST_STRIDE, LAYOUT, 128); + __builtin_LinAlg_MatrixStoreToDescriptor( + Src, SourceAfter, 0, SRC_STRIDE, LAYOUT, 128); } )"; -// Thread-scope vector-matrix multiplication is described entirely by its type -// combination. D3D12LinearAlgebraRuntimeFeatureSupport.md scopes -// MatrixConstruction to "wave-scope and group-scope matrices" and states there -// is no requirement around thread-scope vector-matrix multiplication -// dimensions, which is why neither the support struct nor the enumeration -// entry for this operation carries a shape. Applicability therefore rests on -// ThreadVectorMatrixMultiply alone. -static HRESULT queryMatVecMulSupport(ID3D12Device *Device, - const MatrixParams &Params, - ComponentType InputInterp, bool HasBias, - bool &TierSupported, bool &Supported) { - TierSupported = false; +static HRESULT selectCopyConvertWaveSize(ID3D12Device *Device, + const MatrixParams &Params, + ComponentType DestinationCompType, + bool Transpose, bool &Supported, + UINT &SelectedWaveSize) { Supported = false; - if (!Device || + SelectedWaveSize = 0; + if (!Device || Params.Use != MatrixUse::A || !linalg_test::isLegalScope( - linalg_abi:: - D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_VECTOR_MATRIX_MULTIPLY, + linalg_abi::D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_MATRIX_CONSTRUCTION, Params.Scope)) return E_INVALIDARG; - const std::optional MatrixType = + std::optional SourceType = toCapabilityDataType(Params.CompType); - const std::optional VectorType = - toCapabilityDataType(InputInterp); - if (!MatrixType.has_value() || !VectorType.has_value()) + std::optional DestinationType = + toCapabilityDataType(DestinationCompType); + if (!SourceType.has_value() || !DestinationType.has_value()) return E_INVALIDARG; linalg_test::TierSupport Tier; HRESULT HR = linalg_test::queryTierSupport(Device, Tier); + if (FAILED(HR) || !Tier.supported()) + return HR; + + UINT MinWaveSize = 0; + UINT MaxWaveSize = 0; + HR = queryLaunchableWaveSizes(Device, MinWaveSize, MaxWaveSize); if (FAILED(HR)) return HR; - TierSupported = Tier.supported(); - if (!TierSupported) + if (MinWaveSize == 0) { + hlsl_test::LogCommentFmt( + L"Wave operations are unsupported; MatrixConstruction is not " + L"applicable"); return S_OK; + } + + MatrixParams Destination = Params; + Destination.CompType = DestinationCompType; + if (Transpose) { + Destination.M = Params.N; + Destination.N = Params.M; + } + + for (UINT WaveSize = 4; WaveSize <= 128; WaveSize *= 2) { + if (WaveSize < MinWaveSize || WaveSize > MaxWaveSize || + WaveSize > static_cast(Params.NumThreads)) + continue; - // The shaders declare the bias and result vectors with the matrix component - // type. A multiply with no bias is expressed as DATATYPE_NONE, which Tier 1 - // requires alongside a bias type matching the result type. - const linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE BiasType = - HasBias ? *MatrixType : linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE; + bool SourceSupported = false; + HR = supportsMatrixShape(Device, *SourceType, WaveSize, MatrixUse::A, + Params.M, Params.N, SourceSupported); + if (FAILED(HR)) + return HR; - linalg_test::ThreadVectorMatrixMultiplySupport Multiply; - HR = linalg_test::queryThreadVectorMatrixMultiply( - Device, {*VectorType, *MatrixType, BiasType, *MatrixType}, Multiply); - if (FAILED(HR)) - return HR; - if (!Multiply.supported()) { - hlsl_test::LogCommentFmt( - L"ThreadVectorMatrixMultiply reports vector=%u matrix=%u bias=%u " - L"result=%u is unsupported", - static_cast(*VectorType), static_cast(*MatrixType), - static_cast(BiasType), static_cast(*MatrixType)); - return S_OK; + bool DestinationSupported = false; + HR = + supportsMatrixShape(Device, *DestinationType, WaveSize, MatrixUse::A, + Destination.M, Destination.N, DestinationSupported); + if (FAILED(HR)) + return HR; + + if (SourceSupported && DestinationSupported) { + hlsl_test::LogCommentFmt( + L"CopyConvert capability matched wave=%u for source=%ux%u and " + L"destination=%ux%u", + WaveSize, Params.M, Params.N, Destination.M, Destination.N); + Supported = true; + SelectedWaveSize = WaveSize; + return S_OK; + } } - Supported = true; + hlsl_test::LogCommentFmt( + L"No MatrixConstruction query supports CopyConvert source=%ux%u and " + L"destination=%ux%u for any wave size launchable within shader " + L"WaveSize(4,128) and a %d-thread group", + Params.M, Params.N, Destination.M, Destination.N, Params.NumThreads); return S_OK; } -static bool matVecMulApplicable(ID3D12Device *Device, - const MatrixParams &Params, - ComponentType InputInterp, bool HasBias, - linalg_test::CapabilityRequirement Requirement, - LPCWSTR CaseName) { - bool TierSupported = false; +static bool copyConvertApplicable(ID3D12Device *Device, + const MatrixParams &Params, + ComponentType DestinationCompType, + bool Transpose, LPCWSTR CaseName, + UINT &SelectedWaveSize) { bool Supported = false; - const HRESULT QueryResult = queryMatVecMulSupport( - Device, Params, InputInterp, HasBias, TierSupported, Supported); - - // A device that does not implement linear algebra at all is outside the - // Tier 1 requirements, so it skips rather than failing even where the - // configuration is mandatory. - const linalg_test::CapabilityRequirement Effective = - SUCCEEDED(QueryResult) && !TierSupported - ? linalg_test::CapabilityRequirement::CapabilityGated - : Requirement; + const HRESULT QueryResult = + selectCopyConvertWaveSize(Device, Params, DestinationCompType, Transpose, + Supported, SelectedWaveSize); + if (!applyApplicability( + linalg_test::classifyApplicability( + QueryResult, Supported, + linalg_test::CapabilityRequirement::CapabilityGated), + CaseName)) + return false; - return applyApplicability( - linalg_test::classifyApplicability(QueryResult, Supported, Effective), - CaseName); + VERIFY_IS_TRUE(SelectedWaveSize != 0, + "A case cleared to run must have a selected wave size"); + return true; } -static void runMatVecMul(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - int FillValue, bool OutputSigned, - ComponentType InputInterp) { - const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); +static void runCopyConvert(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, + ComponentType DestinationCompType, bool Verbose, + bool Transpose, UINT ForcedWaveSize = 0) { + MatrixParams DstParams = Params; + DstParams.CompType = DestinationCompType; + if (Transpose) { + DstParams.M = Params.N; + DstParams.N = Params.M; + } std::stringstream ExtraDefs; - ExtraDefs << " -DOUTPUT_SIGNED=" << OutputSigned; - ExtraDefs << " -DIN_INTERP=" << static_cast(InputInterp); + ExtraDefs << " -DTRANSPOSE=" << Transpose; + ExtraDefs << " -DDST_COMP_TYPE=" << static_cast(DestinationCompType); + ExtraDefs << " -DDST_M_DIM=" << DstParams.M; + ExtraDefs << " -DDST_N_DIM=" << DstParams.N; + ExtraDefs << " -DSRC_STRIDE=" << Params.strideBytes(); + ExtraDefs << " -DDST_STRIDE=" << DstParams.strideBytes(); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, MatVecMulShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, CopyConvertShader, "cs_6_10", Args, Verbose); - auto Expected = - makeExpectedVec(Params.CompType, Params.M, - static_cast(FillValue * FillValue * Params.N), - /*Increment=*/false); + std::optional Input = + cpu_oracle::makeSequentialMatrix(Params.CompType, Params.M, Params.N); + VERIFY_IS_TRUE(Input.has_value(), + "Unable to construct typed CopyConvert input"); + std::optional Converted = + cpu_oracle::makeSequentialMatrix(DstParams.CompType, Params.M, Params.N); + VERIFY_IS_TRUE(Converted.has_value(), + "Unable to construct typed CopyConvert conversion oracle"); + if (!Input.has_value() || !Converted.has_value()) + return; + std::optional Expected = + Transpose ? cpu_oracle::transposeMatrix(*Converted) : Converted; + VERIFY_IS_TRUE(Expected.has_value(), + "Unable to construct independent CopyConvert oracle"); + if (!Expected.has_value()) + return; - auto Op = createComputeOp(MatVecMulShader, "cs_6_10", "SRV(t0), UAV(u1)", - Args.c_str()); - addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); - addUAVBuffer(Op.get(), "Output", BufferSize, true); + cpu_oracle::MatrixBufferLayout SourceLayout = { + Params.Layout, + /*OffsetBytes=*/0, + /*StrideBytes=*/Params.strideBytes(), + }; + cpu_oracle::MatrixBufferLayout DestinationLayout = { + DstParams.Layout, + /*OffsetBytes=*/0, + /*StrideBytes=*/DstParams.strideBytes(), + }; + std::optional SourceBufferSize = + cpu_oracle::getMatrixBufferSize(*Input, SourceLayout); + std::optional DestinationBufferSize = + cpu_oracle::getMatrixBufferSize(*Expected, DestinationLayout); + VERIFY_IS_TRUE(SourceBufferSize.has_value(), + "Unable to size typed CopyConvert input"); + VERIFY_IS_TRUE(DestinationBufferSize.has_value(), + "Unable to size typed CopyConvert output"); + if (!SourceBufferSize.has_value() || !DestinationBufferSize.has_value()) + return; + + cpu_oracle::TypedMatrix InputMatrix = *Input; + cpu_oracle::MatrixResultOracle Oracle = cpu_oracle::exactResult( + *Expected, + L"HLSL proposal 0035 CopyConvertMatrix transpose and descriptor layout"); + cpu_oracle::MatrixResultOracle SourceOracle = cpu_oracle::exactResult( + *Input, L"CopyConvertMatrix leaves the source matrix unmodified"); + + auto Op = createComputeOp(CopyConvertShader, "cs_6_10", + "UAV(u0), UAV(u1), UAV(u2)", Args.c_str()); + addUAVBuffer(Op.get(), "Input", *SourceBufferSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", *DestinationBufferSize, true); + addUAVBuffer(Op.get(), "SourceAfter", *SourceBufferSize, true); addRootView(Op.get(), 0, "Input"); addRootView(Op.get(), 1, "Output"); + addRootView(Op.get(), 2, "SourceAfter"); auto Result = runShaderOp( Device, DxcSupport, std::move(Op), - [NumElements, Params, FillValue](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, NumElements, - /*StartingVal=*/FillValue, - /*Increment=*/false), - "Saw unsupported component type"); + [InputMatrix, SourceLayout](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + if (_stricmp(Name, "Input") != 0) + return; + VERIFY_IS_TRUE( + cpu_oracle::writeMatrixBuffer(InputMatrix, SourceLayout, Data), + "Unable to encode typed CopyConvert input"); }); MappedData OutData; + MappedData SourceAfterData; Result->Test->GetReadBackData("Output", &OutData); + Result->Test->GetReadBackData("SourceAfter", &SourceAfterData); - VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, Params.M, Verbose)); + VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer( + OutData.data(), OutData.size(), DestinationLayout, Oracle, Verbose)); + VERIFY_IS_TRUE(cpu_oracle::verifyMatrixBuffer( + SourceAfterData.data(), SourceAfterData.size(), SourceLayout, + SourceOracle, Verbose)); } -void DxilConf_SM610_LinAlg::MatVecMul_Thread_16x16_F16() { +void DxilConf_SM610_LinAlg::CopyConvert_Wave_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Scope = MatrixScope::Thread; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 1; + Params.NumThreads = 128; Params.Enable16Bit = true; - // Tier 1 requires Fp16 vector x Fp16 matrix -> Fp16, and requires a bias - // matching the result type as well as no bias at all, so a Tier 1 device - // reporting this unsupported is a conformance failure rather than a skip. - if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F16, - /*HasBias=*/false, - linalg_test::CapabilityRequirement::Mandatory, - L"MatVecMul_Thread_16x16_F16")) + UINT SelectedWaveSize = 0; + if (!copyConvertApplicable(D3DDevice, Params, ComponentType::F16, + /*Transpose=*/false, L"CopyConvert_Wave_16x16_F16", + SelectedWaveSize)) return; - runMatVecMul(D3DDevice, DxcSupport, Params, VerboseLogging, - /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F16); + runCopyConvert(D3DDevice, DxcSupport, Params, ComponentType::F16, + VerboseLogging, + /*Transpose=*/false, SelectedWaveSize); } -void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F32() { +void DxilConf_SM610_LinAlg::CopyConvert_Wave_16x16_F16_Transpose() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; + + UINT SelectedWaveSize = 0; + if (!copyConvertApplicable(D3DDevice, Params, ComponentType::F16, + /*Transpose=*/true, + L"CopyConvert_Wave_16x16_F16_Transpose", + SelectedWaveSize)) + return; + + runCopyConvert(D3DDevice, DxcSupport, Params, ComponentType::F16, + VerboseLogging, + /*Transpose=*/true, SelectedWaveSize); +} + +void DxilConf_SM610_LinAlg::CopyConvert_Wave_4x8_F32_Transpose() { MatrixParams Params = {}; Params.CompType = ComponentType::F32; Params.M = 4; Params.N = 8; - Params.Scope = MatrixScope::Thread; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 1; + Params.NumThreads = 128; + Params.Enable16Bit = false; - // Fp32 vector x Fp32 matrix -> Fp32 is absent from the Tier 1 table, so it - // is optional and a device reporting it unsupported skips. - if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F32, - /*HasBias=*/false, - linalg_test::CapabilityRequirement::CapabilityGated, - L"MatVecMul_Thread_4x8_F32")) + UINT SelectedWaveSize = 0; + if (!copyConvertApplicable(D3DDevice, Params, ComponentType::F32, + /*Transpose=*/true, + L"CopyConvert_Wave_4x8_F32_Transpose", + SelectedWaveSize)) return; - runMatVecMul(D3DDevice, DxcSupport, Params, VerboseLogging, - /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F32); + // Non-square dimensions make the destination shape and row stride observable. + runCopyConvert(D3DDevice, DxcSupport, Params, ComponentType::F32, + VerboseLogging, + /*Transpose=*/true, SelectedWaveSize); } -static const char MatVecMulAddShader[] = R"( +static const char MatMatMulShader[] = R"( #define USE_A 0 - #define SCOPE_THREAD 0 + #define USE_B 1 + #define USE_ACC 2 - ByteAddressBuffer Input : register(t0); - RWByteAddressBuffer Output : register(u1); + RWByteAddressBuffer Output : register(u0); + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif [numthreads(NUMTHREADS, 1, 1)] void main() { - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, Input, 0, STRIDE, LAYOUT, 128); + if (GetGroupWaveIndex() != 0) + return; - vector InVec; - for (uint I = 0; I < N_DIM; ++I) { - InVec[I] = Input.Load(I * ELEM_SIZE); - } + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, K_DIM, USE_A, SCOPE)]] + MatA; + __builtin_LinAlg_FillMatrix(MatA, A_FILL); - vector BiasVec; - for (uint I = 0; I < M_DIM; ++I) { - BiasVec[I] = Input.Load(I * ELEM_SIZE); - } + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, K_DIM, N_DIM, USE_B, SCOPE)]] + MatB; + __builtin_LinAlg_FillMatrix(MatB, B_FILL); - vector OutVec; - __builtin_LinAlg_MatrixVectorMultiplyAdd( - OutVec, Mat, OUTPUT_SIGNED, InVec, IN_INTERP, BiasVec); + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] + MatC; + __builtin_LinAlg_MatrixMatrixMultiply(MatC, MatA, MatB); - for (uint I = 0; I < M_DIM; ++I) { - Output.Store(I * ELEM_SIZE, OutVec[I]); - } + __builtin_LinAlg_MatrixStoreToDescriptor( + MatC, Output, 0, STRIDE, LAYOUT, 128); } )"; -static void runMatVecMulAdd(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - int FillValue, bool OutputSigned, - ComponentType InputInterp) { +static void runMatMatMul(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, MatrixDim K, + float AFill, float BFill, UINT ForcedWaveSize = 0) { const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); std::stringstream ExtraDefs; - ExtraDefs << " -DOUTPUT_SIGNED=" << OutputSigned; - ExtraDefs << " -DIN_INTERP=" << static_cast(InputInterp); + ExtraDefs << " -DK_DIM=" << K; + STREAM_FLOAT(ExtraDefs, "A_FILL", AFill); + STREAM_FLOAT(ExtraDefs, "B_FILL", BFill); + + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, MatVecMulAddShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, MatMatMulShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedVec( - Params.CompType, Params.M, - static_cast(FillValue * FillValue * Params.N + FillValue), - /*Increment=*/false); + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, + AFill * BFill * K, /*Increment=*/false); - auto Op = createComputeOp(MatVecMulAddShader, "cs_6_10", "SRV(t0), UAV(u1)", - Args.c_str()); - addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); + auto Op = + createComputeOp(MatMatMulShader, "cs_6_10", "UAV(u0)", Args.c_str()); addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + addRootView(Op.get(), 0, "Output"); - auto Result = runShaderOp( - Device, DxcSupport, std::move(Op), - [NumElements, Params, FillValue](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, NumElements, - /*StartingVal=*/FillValue, - /*Increment=*/false), - "Saw unsupported component type"); - }); + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, Params.M, Verbose)); + Expected, NumElements, Verbose)); } -void DxilConf_SM610_LinAlg::MatVecMulAdd_Thread_16x16_F16() { +void DxilConf_SM610_LinAlg::MatMatMul_Wave_16x16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Scope = MatrixScope::Thread; + Params.Scope = MatrixScope::Wave; Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 1; + Params.NumThreads = 128; Params.Enable16Bit = true; - // Required by Tier 1: Fp16 throughout, with a bias matching the result type. - if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F16, - /*HasBias=*/true, - linalg_test::CapabilityRequirement::Mandatory, - L"MatVecMulAdd_Thread_16x16_F16")) - return; - - runMatVecMulAdd(D3DDevice, DxcSupport, Params, VerboseLogging, - /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F16); -} - -void DxilConf_SM610_LinAlg::MatVecMulAdd_Thread_4x8_F32() { - MatrixParams Params = {}; - Params.CompType = ComponentType::F32; - Params.M = 4; - Params.N = 8; - Params.Scope = MatrixScope::Thread; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 1; - - // Optional: see MatVecMul_Thread_4x8_F32. - if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F32, - /*HasBias=*/true, - linalg_test::CapabilityRequirement::CapabilityGated, - L"MatVecMulAdd_Thread_4x8_F32")) + UINT SelectedWaveSize = 0; + if (!waveMatMulApplicable(D3DDevice, Params, /*K=*/16, + L"MatMatMul_Wave_16x16x16_F16", SelectedWaveSize)) return; - runMatVecMulAdd(D3DDevice, DxcSupport, Params, VerboseLogging, - /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F32); -} - -// Map a DXIL ComponentType to the D3D12 linear-algebra datatype used by the -// host-side matrix conversion API. -#if defined(DIRECT3D_LINEAR_ALGEBRA) -static D3D12_LINEAR_ALGEBRA_DATATYPE toLinAlgDataType(ComponentType CT) { - switch (CT) { - case ComponentType::F16: - return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16; - case ComponentType::F32: - return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32; - case ComponentType::I16: - return D3D12_LINEAR_ALGEBRA_DATATYPE_SINT16; - case ComponentType::U16: - return D3D12_LINEAR_ALGEBRA_DATATYPE_UINT16; - case ComponentType::I32: - return D3D12_LINEAR_ALGEBRA_DATATYPE_SINT32; - case ComponentType::U32: - return D3D12_LINEAR_ALGEBRA_DATATYPE_UINT32; - default: - VERIFY_IS_TRUE(false, "Unsupported component type for linalg conversion"); - return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16; - } + runMatMatMul(D3DDevice, DxcSupport, Params, VerboseLogging, /*K=*/16, + /*AFill=*/2.0f, /*BFill=*/3.0f, SelectedWaveSize); } -static const char OuterProductShader[] = R"( - #define SCOPE_THREAD 0 +static const char MatMatMulAccumShader[] = R"( + #define USE_A 0 + #define USE_B 1 + #define USE_ACC 2 - RWByteAddressBuffer Input : register(u0); - RWByteAddressBuffer Output : register(u1); + RWByteAddressBuffer Output : register(u0); + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif [numthreads(NUMTHREADS, 1, 1)] void main() { - vector VecA; - for (uint I = 0; I < M_DIM; ++I) { - VecA[I] = Input.Load(I * ELEM_SIZE); - } + if (GetGroupWaveIndex() != 0) + return; - uint EndVecA = M_DIM * ELEM_SIZE; + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, K_DIM, USE_A, SCOPE)]] + MatA; + __builtin_LinAlg_FillMatrix(MatA, A_FILL); - vector VecB; - for (uint I = 0; I < N_DIM; ++I) { - VecB[I] = Input.Load(EndVecA + I * ELEM_SIZE); - } + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, K_DIM, N_DIM, USE_B, SCOPE)]] + MatB; + __builtin_LinAlg_FillMatrix(MatB, B_FILL); __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE_THREAD)]] - Mat; - __builtin_LinAlg_MatrixOuterProduct(Mat, VecA, VecB); + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] + MatC; + __builtin_LinAlg_FillMatrix(MatC, C_FILL); - // Outer product accumulators are stored in the OuterProductOptimal layout - // Matching the dx::linalg header's thread-scoped - // InterlockedAccumulate. The alignment argument must be a non-zero - // multiple of 128; the matrix starts at offset 0 in a buffer D3D12 aligns - // far more strongly than that. - __builtin_LinAlg_MatrixAccumulateToDescriptor( - Mat, Output, 0, STRIDE, LAYOUT, 128); + __builtin_LinAlg_MatrixMatrixMultiplyAccumulate(MatC, MatA, MatB, MatC); + + __builtin_LinAlg_MatrixStoreToDescriptor( + MatC, Output, 0, STRIDE, LAYOUT, 128); } )"; -static void runOuterProduct(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose) { - VERIFY_IS_TRUE( - Params.Layout == MatrixLayout::OuterProductOptimal, - "Outer product must output its matrix in OuterProductOptimal layout"); - VERIFY_IS_TRUE(Params.Use == MatrixUse::Accumulator, - "Outer product must output an accumulator matrix"); - const size_t NumVecElements = Params.M + Params.N; - const size_t InBuffSize = NumVecElements * elementSize(Params.CompType); - const size_t NumMatElements = Params.totalElements(); - const D3D12_LINEAR_ALGEBRA_DATATYPE DataType = - toLinAlgDataType(Params.CompType); +static void runMatMatMulAccum(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + MatrixDim K, float AFill, float BFill, + float CFill, UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); - const UINT OutBufferSize = getLinAlgMatrixByteSize( - Device, Params.M, Params.N, DataType, - D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_OUTER_PRODUCT_OPTIMAL, /*Stride=*/0); + std::stringstream ExtraDefs; + ExtraDefs << " -DK_DIM=" << K; + STREAM_FLOAT(ExtraDefs, "A_FILL", AFill); + STREAM_FLOAT(ExtraDefs, "B_FILL", BFill); + STREAM_FLOAT(ExtraDefs, "C_FILL", CFill); - const UINT RowMajorStride = - static_cast(Params.N * elementSize(Params.CompType)); - const UINT RowMajorSize = getLinAlgMatrixByteSize( - Device, Params.M, Params.N, DataType, - D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_ROW_MAJOR, RowMajorStride); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - std::string Args = buildCompilerArgs(Params); + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, OuterProductShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, MatMatMulAccumShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 4, - /*Increment=*/false); + auto Expected = + makeExpectedMat(Params.CompType, Params.M, Params.N, + AFill * BFill * K + CFill, /*Increment=*/false); - auto Op = createComputeOp(OuterProductShader, "cs_6_10", "UAV(u0), UAV(u1)", - Args.c_str()); - addUAVBuffer(Op.get(), "Input", InBuffSize, false, "byname"); - addUAVBuffer(Op.get(), "Output", OutBufferSize, /*ReadBack=*/false); - addUAVBuffer(Op.get(), "OutputRowMajor", RowMajorSize, /*ReadBack=*/true); - addRootView(Op.get(), 0, "Input"); - addRootView(Op.get(), 1, "Output"); + auto Op = + createComputeOp(MatMatMulAccumShader, "cs_6_10", "UAV(u0)", Args.c_str()); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Output"); - auto Result = runShaderOp( - Device, DxcSupport, std::move(Op), - [NumVecElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumVecElements, - /*StartingVal=*/2, /*Increment=*/false), - "Saw unsupported component type"); - }, - [OutBufferSize, RowMajorSize, RowMajorStride, DataType, - Params](ID3D12GraphicsCommandList *List, st::ShaderOpTest *Test) { - ID3D12Resource *OptimalBuffer = nullptr; - ID3D12Resource *RowMajorBuffer = nullptr; - Test->GetResource("Output", &OptimalBuffer); - Test->GetResource("OutputRowMajor", &RowMajorBuffer); - recordLinAlgMatrixConversion( - List, OptimalBuffer, OutBufferSize, RowMajorBuffer, RowMajorSize, - Params.M, Params.N, DataType, - D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_OUTER_PRODUCT_OPTIMAL, - /*SrcStride=*/0, D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_ROW_MAJOR, - RowMajorStride); - }); + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); MappedData OutData; - Result->Test->GetReadBackData("OutputRowMajor", &OutData); + Result->Test->GetReadBackData("Output", &OutData); VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumMatElements, Verbose)); + Expected, NumElements, Verbose)); } -#endif // defined(DIRECT3D_LINEAR_ALGEBRA) -void DxilConf_SM610_LinAlg::OuterProduct_Thread_16x16_F16() { -#if defined(DIRECT3D_LINEAR_ALGEBRA) +void DxilConf_SM610_LinAlg::MatMatMulAccum_Wave_16x16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Thread; - Params.Layout = MatrixLayout::OuterProductOptimal; - Params.NumThreads = 1; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; Params.Enable16Bit = true; - // Tier 1 requires no outer product formats at all, so this is gated. - if (!outerProductApplicable(D3DDevice, Params.CompType, Params.CompType, - L"OuterProduct_Thread_16x16_F16")) - return; - - // The shader accumulates its result into an RWByteAddressBuffer, which is - // reported independently of the outer product itself. A device may produce - // the outer product yet not support accumulating this component type into a - // buffer, so the destination has to be gated too or that device fails to - // create the pipeline instead of skipping. - if (!accumulateStoreApplicable( - D3DDevice, Params.CompType, - linalg_test::AtomicDestination::RWByteAddressBuffer, - L"OuterProduct_Thread_16x16_F16")) + UINT SelectedWaveSize = 0; + if (!waveMatMulApplicable(D3DDevice, Params, /*K=*/16, + L"MatMatMulAccum_Wave_16x16x16_F16", + SelectedWaveSize)) return; - runOuterProduct(D3DDevice, DxcSupport, Params, VerboseLogging); -#else -#ifdef _HLK_CONF - // HLK forbids skipping, so treat the missing linear-algebra matrix-conversion - // API as a failure rather than emitting a (compiled-out) skip. - hlsl_test::LogErrorFmt(L"OuterProduct_Thread_16x16_F16 requires the " - L"linear-algebra matrix-conversion API " - L"(DIRECT3D_LINEAR_ALGEBRA), which this build lacks"); -#else - WEX::Logging::Log::Comment( - L"Skipping OuterProduct_Thread_16x16_F16: built against a D3D12 SDK " - L"without the linear-algebra matrix-conversion API " - L"(DIRECT3D_LINEAR_ALGEBRA undefined); the host-side conversion helpers " - L"are compiled out."); - WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); -#endif // _HLK_CONF -#endif // defined(DIRECT3D_LINEAR_ALGEBRA) + runMatMatMulAccum(D3DDevice, DxcSupport, Params, VerboseLogging, /*K=*/16, + /*AFill=*/2.0f, /*BFill=*/3.0f, /*CFill=*/4.0f, + SelectedWaveSize); } -static const char QueryAccumLayoutShader[] = R"( +static const char MatAccumShader[] = R"( + #define USE_A 0 + #define USE_ACC 2 + RWByteAddressBuffer Output : register(u0); - [numthreads(1, 1, 1)] + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif + [numthreads(NUMTHREADS, 1, 1)] void main() { - uint Layout = __builtin_LinAlg_MatrixQueryAccumulatorLayout(); - Output.Store(0, Layout); + if (GetGroupWaveIndex() != 0) + return; + + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_ACC, SCOPE)]] + MatLHS; + __builtin_LinAlg_FillMatrix(MatLHS, LHS_FILL); + + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE)]] + MatRHS; + __builtin_LinAlg_FillMatrix(MatRHS, RHS_FILL); + + __builtin_LinAlg_MatrixAccumulate(MatLHS, MatLHS, MatRHS); + + __builtin_LinAlg_MatrixStoreToDescriptor( + MatLHS, Output, 0, STRIDE, LAYOUT, 128); } )"; -static void runQueryAccumLayout(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - bool Verbose) { - std::string Args = "-HV 202x"; - size_t BufferSize = elementSize(ComponentType::I32); +static void runMatAccum(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, float LHSFill, + float RHSFill, UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); - compileShader(DxcSupport, QueryAccumLayoutShader, "cs_6_10", Args, Verbose); + std::stringstream ExtraDefs; + STREAM_FLOAT(ExtraDefs, "LHS_FILL", LHSFill); + STREAM_FLOAT(ExtraDefs, "RHS_FILL", RHSFill); - auto Op = createComputeOp(QueryAccumLayoutShader, "cs_6_10", "UAV(u0)", - Args.c_str()); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + + compileShader(DxcSupport, MatAccumShader, "cs_6_10", Args, Verbose); + + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, + LHSFill + RHSFill, /*Increment=*/false); + + auto Op = createComputeOp(MatAccumShader, "cs_6_10", "UAV(u0)", Args.c_str()); addUAVBuffer(Op.get(), "Output", BufferSize, true); addRootView(Op.get(), 0, "Output"); @@ -4885,376 +5099,550 @@ static void runQueryAccumLayout(ID3D12Device *Device, MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); - const uint32_t *Out = static_cast(OutData.data()); - // Accum Layout must be A or B - VERIFY_IS_TRUE(Out[0] == static_cast(MatrixUse::A) || - Out[0] == static_cast(MatrixUse::B)); - if (Verbose) - hlsl_test::LogCommentFmt(L"AccumulatorLayout = %u", Out[0]); + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); } -void DxilConf_SM610_LinAlg::QueryAccumLayout() { - // Constructs no matrix, so tier support is the only capability it needs. - if (!linAlgTierApplicable(D3DDevice, L"QueryAccumLayout")) +void DxilConf_SM610_LinAlg::MatAccum_Wave_16x16_F16() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; + + // MatAccum builds both an accumulator and an A matrix, and the two roles pin + // different extents of the same shape, so both must be constructible. + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable( + D3DDevice, Params, {MatrixUse::Accumulator, MatrixUse::A}, + L"MatAccum_Wave_16x16_F16", SelectedWaveSize)) return; - runQueryAccumLayout(D3DDevice, DxcSupport, VerboseLogging); + runMatAccum(D3DDevice, DxcSupport, Params, VerboseLogging, + /*LHSFill=*/2.0f, /*RHSFill=*/3.0f, SelectedWaveSize); } -static const char LoadMemoryShader[] = R"( - RWByteAddressBuffer Input : register(u0); +static const char MatVecMulShader[] = R"( + #define USE_A 0 + #define SCOPE_THREAD 0 + + ByteAddressBuffer Input : register(t0); RWByteAddressBuffer Output : register(u1); - groupshared ELEM_TYPE GsData[M_DIM * N_DIM]; - #define ELEM_PER_THREAD (M_DIM * N_DIM / NUMTHREADS) + [numthreads(NUMTHREADS, 1, 1)] + void main() { + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + Mat; + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); + + vector InVec; + for (uint I = 0; I < N_DIM; ++I) { + InVec[I] = Input.Load(I * ELEM_SIZE); + } + + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiply( + OutVec, Mat, OUTPUT_SIGNED, InVec, IN_INTERP); + + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(I * ELEM_SIZE, OutVec[I]); + } + } +)"; + +// Thread-scope vector-matrix multiplication is described entirely by its type +// combination. D3D12LinearAlgebraRuntimeFeatureSupport.md scopes +// MatrixConstruction to "wave-scope and group-scope matrices" and states there +// is no requirement around thread-scope vector-matrix multiplication +// dimensions, which is why neither the support struct nor the enumeration +// entry for this operation carries a shape. Applicability therefore rests on +// ThreadVectorMatrixMultiply alone. +static HRESULT queryMatVecMulSupport(ID3D12Device *Device, + const MatrixParams &Params, + ComponentType InputInterp, bool HasBias, + bool &TierSupported, bool &Supported) { + TierSupported = false; + Supported = false; + if (!Device || + !linalg_test::isLegalScope( + linalg_abi:: + D3D12_LINEAR_ALGEBRA_OPERATION_TYPE_THREAD_VECTOR_MATRIX_MULTIPLY, + Params.Scope)) + return E_INVALIDARG; + + const std::optional MatrixType = + toCapabilityDataType(Params.CompType); + const std::optional VectorType = + toCapabilityDataType(InputInterp); + if (!MatrixType.has_value() || !VectorType.has_value()) + return E_INVALIDARG; + + linalg_test::TierSupport Tier; + HRESULT HR = linalg_test::queryTierSupport(Device, Tier); + if (FAILED(HR)) + return HR; + TierSupported = Tier.supported(); + if (!TierSupported) + return S_OK; + + // The shaders declare the bias and result vectors with the matrix component + // type. A multiply with no bias is expressed as DATATYPE_NONE, which Tier 1 + // requires alongside a bias type matching the result type. + const linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE BiasType = + HasBias ? *MatrixType : linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE; + + linalg_test::ThreadVectorMatrixMultiplySupport Multiply; + HR = linalg_test::queryThreadVectorMatrixMultiply( + Device, {*VectorType, *MatrixType, BiasType, *MatrixType}, Multiply); + if (FAILED(HR)) + return HR; + if (!Multiply.supported()) { + hlsl_test::LogCommentFmt( + L"ThreadVectorMatrixMultiply reports vector=%u matrix=%u bias=%u " + L"result=%u is unsupported", + static_cast(*VectorType), static_cast(*MatrixType), + static_cast(BiasType), static_cast(*MatrixType)); + return S_OK; + } - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif - [numthreads(NUMTHREADS, 1, 1)] - void main(uint threadID : SV_GroupIndex) { - for (uint I = 0; I < ELEM_PER_THREAD; ++I) { - uint Index = threadID * ELEM_PER_THREAD + I; - GsData[Index] = Input.Load(Index * ELEM_SIZE); - } + Supported = true; + return S_OK; +} - GroupMemoryBarrierWithGroupSync(); +static bool matVecMulApplicable(ID3D12Device *Device, + const MatrixParams &Params, + ComponentType InputInterp, bool HasBias, + linalg_test::CapabilityRequirement Requirement, + LPCWSTR CaseName) { + bool TierSupported = false; + bool Supported = false; + const HRESULT QueryResult = queryMatVecMulSupport( + Device, Params, InputInterp, HasBias, TierSupported, Supported); - if (GetGroupWaveIndex() != 0) - return; + // A device that does not implement linear algebra at all is outside the + // Tier 1 requirements, so it skips rather than failing even where the + // configuration is mandatory. + const linalg_test::CapabilityRequirement Effective = + SUCCEEDED(QueryResult) && !TierSupported + ? linalg_test::CapabilityRequirement::CapabilityGated + : Requirement; - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] - Mat; - __builtin_LinAlg_MatrixLoadFromMemory( - Mat, GsData, OFFSET / ELEM_SIZE, STRIDE / ELEM_SIZE, LAYOUT); - __builtin_LinAlg_MatrixStoreToDescriptor( - Mat, Output, OFFSET, STRIDE, LAYOUT, 128); - } -)"; + return applyApplicability( + linalg_test::classifyApplicability(QueryResult, Supported, Effective), + CaseName); +} -static void runLoadMemory(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - UINT ForcedWaveSize = 0) { +static void runMatVecMul(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + int FillValue, bool OutputSigned, + ComponentType InputInterp) { const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); std::stringstream ExtraDefs; - ExtraDefs << " -DOFFSET=" << 0; - - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + ExtraDefs << " -DOUTPUT_SIGNED=" << OutputSigned; + ExtraDefs << " -DIN_INTERP=" << static_cast(InputInterp); std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, LoadMemoryShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, MatVecMulShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1); + auto Expected = + makeExpectedVec(Params.CompType, Params.M, + static_cast(FillValue * FillValue * Params.N), + /*Increment=*/false); - auto Op = createComputeOp(LoadMemoryShader, "cs_6_10", "UAV(u0), UAV(u1)", + auto Op = createComputeOp(MatVecMulShader, "cs_6_10", "SRV(t0), UAV(u1)", Args.c_str()); - addUAVBuffer(Op.get(), "Input", BufferSize, false, "byname"); + addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); addUAVBuffer(Op.get(), "Output", BufferSize, true); addRootView(Op.get(), 0, "Input"); addRootView(Op.get(), 1, "Output"); - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [NumElements, Params](LPCSTR Name, std::vector &Data, - st::ShaderOp *) { - VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, - NumElements), - "Saw unsupported component type"); - }); + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [NumElements, Params, FillValue](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, NumElements, + /*StartingVal=*/FillValue, + /*Increment=*/false), + "Saw unsupported component type"); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + Expected, Params.M, Verbose)); } -void DxilConf_SM610_LinAlg::LoadMemory_Wave_16x16_F16() { +void DxilConf_SM610_LinAlg::MatVecMul_Thread_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; + Params.Scope = MatrixScope::Thread; Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; + Params.NumThreads = 1; Params.Enable16Bit = true; - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"LoadMemory_Wave_16x16_F16", - SelectedWaveSize)) + // Tier 1 requires Fp16 vector x Fp16 matrix -> Fp16, and requires a bias + // matching the result type as well as no bias at all, so a Tier 1 device + // reporting this unsupported is a conformance failure rather than a skip. + if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F16, + /*HasBias=*/false, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMul_Thread_16x16_F16")) return; - runLoadMemory(D3DDevice, DxcSupport, Params, VerboseLogging, - SelectedWaveSize); + runMatVecMul(D3DDevice, DxcSupport, Params, VerboseLogging, + /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F16); } -static const char StoreMemoryShader[] = R"( - RWByteAddressBuffer Output : register(u0); - groupshared ELEM_TYPE GsData[M_DIM * N_DIM]; +void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F32() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F32; + Params.M = 4; + Params.N = 8; + Params.Scope = MatrixScope::Thread; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 1; + + // Fp32 vector x Fp32 matrix -> Fp32 is absent from the Tier 1 table, so it + // is optional and a device reporting it unsupported skips. + if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F32, + /*HasBias=*/false, + linalg_test::CapabilityRequirement::CapabilityGated, + L"MatVecMul_Thread_4x8_F32")) + return; + + runMatVecMul(D3DDevice, DxcSupport, Params, VerboseLogging, + /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F32); +} + +static const char MatVecMulAddShader[] = R"( + #define USE_A 0 + #define SCOPE_THREAD 0 + + ByteAddressBuffer Input : register(t0); + RWByteAddressBuffer Output : register(u1); - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif [numthreads(NUMTHREADS, 1, 1)] void main() { - if (GetGroupWaveIndex() != 0) - return; - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] Mat; - __builtin_LinAlg_FillMatrix(Mat, FILL_VALUE); + __builtin_LinAlg_MatrixLoadFromDescriptor( + Mat, Input, 0, STRIDE, LAYOUT, 128); - __builtin_LinAlg_MatrixStoreToMemory( - Mat, GsData, OFFSET / ELEM_SIZE, STRIDE / ELEM_SIZE, LAYOUT); + vector InVec; + for (uint I = 0; I < N_DIM; ++I) { + InVec[I] = Input.Load(I * ELEM_SIZE); + } - for (uint I = 0; I < M_DIM*N_DIM; ++I) { - Output.Store(I*ELEM_SIZE, GsData[I]); + vector BiasVec; + for (uint I = 0; I < M_DIM; ++I) { + BiasVec[I] = Input.Load(I * ELEM_SIZE); + } + + vector OutVec; + __builtin_LinAlg_MatrixVectorMultiplyAdd( + OutVec, Mat, OUTPUT_SIGNED, InVec, IN_INTERP, BiasVec); + + for (uint I = 0; I < M_DIM; ++I) { + Output.Store(I * ELEM_SIZE, OutVec[I]); } } )"; -static void runStoreMemory(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - float FillValue, UINT ForcedWaveSize = 0) { +static void runMatVecMulAdd(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + int FillValue, bool OutputSigned, + ComponentType InputInterp) { const size_t NumElements = Params.totalElements(); const size_t BufferSize = Params.totalBytes(); std::stringstream ExtraDefs; - ExtraDefs << " -DOFFSET=" << 0; - STREAM_FLOAT(ExtraDefs, "FILL_VALUE", FillValue); - - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + ExtraDefs << " -DOUTPUT_SIGNED=" << OutputSigned; + ExtraDefs << " -DIN_INTERP=" << static_cast(InputInterp); std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - compileShader(DxcSupport, StoreMemoryShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, MatVecMulAddShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, - FillValue, /*Increment=*/false); + auto Expected = makeExpectedVec( + Params.CompType, Params.M, + static_cast(FillValue * FillValue * Params.N + FillValue), + /*Increment=*/false); - auto Op = - createComputeOp(StoreMemoryShader, "cs_6_10", "UAV(u0)", Args.c_str()); + auto Op = createComputeOp(MatVecMulAddShader, "cs_6_10", "SRV(t0), UAV(u1)", + Args.c_str()); + addSRVBuffer(Op.get(), "Input", BufferSize, "byname"); addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [NumElements, Params, FillValue](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, NumElements, + /*StartingVal=*/FillValue, + /*Increment=*/false), + "Saw unsupported component type"); + }); MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + Expected, Params.M, Verbose)); } -void DxilConf_SM610_LinAlg::StoreMemory_Wave_16x16_F16() { +void DxilConf_SM610_LinAlg::MatVecMulAdd_Thread_16x16_F16() { MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; - Params.Use = MatrixUse::A; - Params.Scope = MatrixScope::Wave; + Params.Scope = MatrixScope::Thread; Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; + Params.NumThreads = 1; Params.Enable16Bit = true; - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"StoreMemory_Wave_16x16_F16", - SelectedWaveSize)) + // Required by Tier 1: Fp16 throughout, with a bias matching the result type. + if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F16, + /*HasBias=*/true, + linalg_test::CapabilityRequirement::Mandatory, + L"MatVecMulAdd_Thread_16x16_F16")) + return; + + runMatVecMulAdd(D3DDevice, DxcSupport, Params, VerboseLogging, + /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F16); +} + +void DxilConf_SM610_LinAlg::MatVecMulAdd_Thread_4x8_F32() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F32; + Params.M = 4; + Params.N = 8; + Params.Scope = MatrixScope::Thread; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 1; + + // Optional: see MatVecMul_Thread_4x8_F32. + if (!matVecMulApplicable(D3DDevice, Params, ComponentType::F32, + /*HasBias=*/true, + linalg_test::CapabilityRequirement::CapabilityGated, + L"MatVecMulAdd_Thread_4x8_F32")) return; - runStoreMemory(D3DDevice, DxcSupport, Params, VerboseLogging, - /*FillValue=*/7.0f, SelectedWaveSize); + runMatVecMulAdd(D3DDevice, DxcSupport, Params, VerboseLogging, + /*FillValue=*/2, /*OutputSigned=*/true, ComponentType::F32); } -static const char AccumulateMemoryShader[] = R"( - RWByteAddressBuffer Output : register(u0); - groupshared ELEM_TYPE GsData[M_DIM * N_DIM]; +// Map a DXIL ComponentType to the D3D12 linear-algebra datatype used by the +// host-side matrix conversion API. +#if defined(DIRECT3D_LINEAR_ALGEBRA) +static D3D12_LINEAR_ALGEBRA_DATATYPE toLinAlgDataType(ComponentType CT) { + switch (CT) { + case ComponentType::F16: + return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16; + case ComponentType::F32: + return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT32; + case ComponentType::I16: + return D3D12_LINEAR_ALGEBRA_DATATYPE_SINT16; + case ComponentType::U16: + return D3D12_LINEAR_ALGEBRA_DATATYPE_UINT16; + case ComponentType::I32: + return D3D12_LINEAR_ALGEBRA_DATATYPE_SINT32; + case ComponentType::U32: + return D3D12_LINEAR_ALGEBRA_DATATYPE_UINT32; + default: + VERIFY_IS_TRUE(false, "Unsupported component type for linalg conversion"); + return D3D12_LINEAR_ALGEBRA_DATATYPE_FLOAT16; + } +} - #define ELEM_PER_THREAD (M_DIM * N_DIM / NUMTHREADS) +static const char OuterProductShader[] = R"( + #define SCOPE_THREAD 0 + + RWByteAddressBuffer Input : register(u0); + RWByteAddressBuffer Output : register(u1); - #ifdef FORCED_WAVE_SIZE - [WaveSize(FORCED_WAVE_SIZE)] - #else - [WaveSize(4, 128)] - #endif [numthreads(NUMTHREADS, 1, 1)] - void main(uint threadID : SV_GroupIndex) { - ELEM_TYPE fill = FILL_VALUE; - for (uint I = 0; I < ELEM_PER_THREAD; ++I) { - uint Index = threadID * ELEM_PER_THREAD + I; - GsData[Index] = fill; + void main() { + vector VecA; + for (uint I = 0; I < M_DIM; ++I) { + VecA[I] = Input.Load(I * ELEM_SIZE); } - GroupMemoryBarrierWithGroupSync(); + uint EndVecA = M_DIM * ELEM_SIZE; - if (GetGroupWaveIndex() != 0) - return; + vector VecB; + for (uint I = 0; I < N_DIM; ++I) { + VecB[I] = Input.Load(EndVecA + I * ELEM_SIZE); + } __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE_THREAD)]] Mat; - __builtin_LinAlg_FillMatrix(Mat, FILL_VALUE); - - __builtin_LinAlg_MatrixAccumulateToMemory( - Mat, GsData, COMP_TYPE, OFFSET / ELEM_SIZE, STRIDE / ELEM_SIZE, LAYOUT); + __builtin_LinAlg_MatrixOuterProduct(Mat, VecA, VecB); - for (uint I = 0; I < M_DIM*N_DIM; ++I) { - Output.Store(I*ELEM_SIZE, GsData[I]); - } + // Outer product accumulators are stored in the OuterProductOptimal layout + // Matching the dx::linalg header's thread-scoped + // InterlockedAccumulate. The alignment argument must be a non-zero + // multiple of 128; the matrix starts at offset 0 in a buffer D3D12 aligns + // far more strongly than that. + __builtin_LinAlg_MatrixAccumulateToDescriptor( + Mat, Output, 0, STRIDE, LAYOUT, 128); } )"; -static void runAccumulateMemory(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const MatrixParams &Params, bool Verbose, - float FillValue, UINT ForcedWaveSize = 0) { - const size_t NumElements = Params.totalElements(); - const size_t BufferSize = Params.totalBytes(); +static void runOuterProduct(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose) { + VERIFY_IS_TRUE( + Params.Layout == MatrixLayout::OuterProductOptimal, + "Outer product must output its matrix in OuterProductOptimal layout"); + VERIFY_IS_TRUE(Params.Use == MatrixUse::Accumulator, + "Outer product must output an accumulator matrix"); + const size_t NumVecElements = Params.M + Params.N; + const size_t InBuffSize = NumVecElements * elementSize(Params.CompType); + const size_t NumMatElements = Params.totalElements(); + const D3D12_LINEAR_ALGEBRA_DATATYPE DataType = + toLinAlgDataType(Params.CompType); - std::stringstream ExtraDefs; - ExtraDefs << " -DOFFSET=" << 0; - STREAM_FLOAT(ExtraDefs, "FILL_VALUE", FillValue); + const UINT OutBufferSize = getLinAlgMatrixByteSize( + Device, Params.M, Params.N, DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_OUTER_PRODUCT_OPTIMAL, /*Stride=*/0); - if (ForcedWaveSize != 0) - ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; + const UINT RowMajorStride = + static_cast(Params.N * elementSize(Params.CompType)); + const UINT RowMajorSize = getLinAlgMatrixByteSize( + Device, Params.M, Params.N, DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_ROW_MAJOR, RowMajorStride); - std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); + std::string Args = buildCompilerArgs(Params); - compileShader(DxcSupport, AccumulateMemoryShader, "cs_6_10", Args, Verbose); + compileShader(DxcSupport, OuterProductShader, "cs_6_10", Args, Verbose); - auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, - FillValue * 2, /*Increment=*/false); + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 4, + /*Increment=*/false); - auto Op = createComputeOp(AccumulateMemoryShader, "cs_6_10", "UAV(u0)", + auto Op = createComputeOp(OuterProductShader, "cs_6_10", "UAV(u0), UAV(u1)", Args.c_str()); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); + addUAVBuffer(Op.get(), "Input", InBuffSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", OutBufferSize, /*ReadBack=*/false); + addUAVBuffer(Op.get(), "OutputRowMajor", RowMajorSize, /*ReadBack=*/true); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + auto Result = runShaderOp( + Device, DxcSupport, std::move(Op), + [NumVecElements, Params](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, + NumVecElements, + /*StartingVal=*/2, /*Increment=*/false), + "Saw unsupported component type"); + }, + [OutBufferSize, RowMajorSize, RowMajorStride, DataType, + Params](ID3D12GraphicsCommandList *List, st::ShaderOpTest *Test) { + ID3D12Resource *OptimalBuffer = nullptr; + ID3D12Resource *RowMajorBuffer = nullptr; + Test->GetResource("Output", &OptimalBuffer); + Test->GetResource("OutputRowMajor", &RowMajorBuffer); + recordLinAlgMatrixConversion( + List, OptimalBuffer, OutBufferSize, RowMajorBuffer, RowMajorSize, + Params.M, Params.N, DataType, + D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_OUTER_PRODUCT_OPTIMAL, + /*SrcStride=*/0, D3D12_LINEAR_ALGEBRA_MATRIX_LAYOUT_ROW_MAJOR, + RowMajorStride); + }); MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); + Result->Test->GetReadBackData("OutputRowMajor", &OutData); VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), - Expected, NumElements, Verbose)); + Expected, NumMatElements, Verbose)); } +#endif // defined(DIRECT3D_LINEAR_ALGEBRA) -void DxilConf_SM610_LinAlg::AccumulateMemory_Wave_16x16_F16() { +void DxilConf_SM610_LinAlg::OuterProduct_Thread_16x16_F16() { +#if defined(DIRECT3D_LINEAR_ALGEBRA) MatrixParams Params = {}; Params.CompType = ComponentType::F16; Params.M = 16; Params.N = 16; Params.Use = MatrixUse::Accumulator; - Params.Scope = MatrixScope::Wave; - Params.Layout = MatrixLayout::RowMajor; - Params.NumThreads = 128; + Params.Scope = MatrixScope::Thread; + Params.Layout = MatrixLayout::OuterProductOptimal; + Params.NumThreads = 1; Params.Enable16Bit = true; - UINT SelectedWaveSize = 0; - if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, - L"AccumulateMemory_Wave_16x16_F16", - SelectedWaveSize)) - return; - if (!accumulateStoreApplicable(D3DDevice, Params.CompType, - linalg_test::AtomicDestination::GroupShared, - L"AccumulateMemory_Wave_16x16_F16")) + // Tier 1 requires no outer product formats at all, so this is gated. + if (!outerProductApplicable(D3DDevice, Params.CompType, Params.CompType, + L"OuterProduct_Thread_16x16_F16")) return; - runAccumulateMemory(D3DDevice, DxcSupport, Params, VerboseLogging, - /*FillValue=*/7.0f, SelectedWaveSize); -} - -static const char ConvertShader[] = R"( - #define CT_F16 8 - #define CT_F32 9 - - RWByteAddressBuffer Output : register(u0); - - [numthreads(1, 1, 1)] - void main() { - vector InVec = {1.0, 2.0, 3.0, 4.0}; - vector OutVec; - __builtin_LinAlg_Convert(OutVec, InVec, CT_F16, CT_F32); - Output.Store(0, OutVec.x); - Output.Store(4, OutVec.y); - Output.Store(8, OutVec.z); - Output.Store(12, OutVec.w); - } -)"; - -static void runConvert(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, - bool Verbose) { - std::string Args = "-HV 202x -enable-16bit-types"; - MatrixDim NumElements = 4; - size_t BufferSize = elementSize(ComponentType::F32) * NumElements; - - compileShader(DxcSupport, ConvertShader, "cs_6_10", Args, Verbose); - - auto Expected = makeExpectedVec(ComponentType::F32, NumElements, 1.0); - - auto Op = createComputeOp(ConvertShader, "cs_6_10", "UAV(u0)", Args.c_str()); - addUAVBuffer(Op.get(), "Output", BufferSize, true); - addRootView(Op.get(), 0, "Output"); - - auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); - - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); - - VERIFY_IS_TRUE(verifyComponentBuffer(ComponentType::F32, OutData.data(), - Expected, NumElements, Verbose)); -} - -void DxilConf_SM610_LinAlg::Convert() { - // Operates on vectors rather than matrices, so tier support is the only - // capability it needs. - if (!linAlgTierApplicable(D3DDevice, L"Convert")) + // The shader accumulates its result into an RWByteAddressBuffer, which is + // reported independently of the outer product itself. A device may produce + // the outer product yet not support accumulating this component type into a + // buffer, so the destination has to be gated too or that device fails to + // create the pipeline instead of skipping. + if (!accumulateStoreApplicable( + D3DDevice, Params.CompType, + linalg_test::AtomicDestination::RWByteAddressBuffer, + L"OuterProduct_Thread_16x16_F16")) return; - runConvert(D3DDevice, DxcSupport, VerboseLogging); + runOuterProduct(D3DDevice, DxcSupport, Params, VerboseLogging); +#else +#ifdef _HLK_CONF + // HLK forbids skipping, so treat the missing linear-algebra matrix-conversion + // API as a failure rather than emitting a (compiled-out) skip. + hlsl_test::LogErrorFmt(L"OuterProduct_Thread_16x16_F16 requires the " + L"linear-algebra matrix-conversion API " + L"(DIRECT3D_LINEAR_ALGEBRA), which this build lacks"); +#else + WEX::Logging::Log::Comment( + L"Skipping OuterProduct_Thread_16x16_F16: built against a D3D12 SDK " + L"without the linear-algebra matrix-conversion API " + L"(DIRECT3D_LINEAR_ALGEBRA undefined); the host-side conversion helpers " + L"are compiled out."); + WEX::Logging::Log::Result(WEX::Logging::TestResults::Skipped); +#endif // _HLK_CONF +#endif // defined(DIRECT3D_LINEAR_ALGEBRA) } -static const char VectorAccumulateDescriptorShader[] = R"( +static const char QueryAccumLayoutShader[] = R"( RWByteAddressBuffer Output : register(u0); [numthreads(1, 1, 1)] void main() { - vector InVec = {1.0, 2.0, 3.0, 4.0}; - __builtin_LinAlg_VectorAccumulateToDescriptor(Output, 0, 64, InVec); + uint Layout = __builtin_LinAlg_MatrixQueryAccumulatorLayout(); + Output.Store(0, Layout); } )"; -static void runVectorAccumulateDescriptor(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - bool Verbose) { - std::string Args = "-HV 202x -enable-16bit-types"; - MatrixDim NumElements = 4; - size_t BufferSize = elementSize(ComponentType::F16) * NumElements; - - compileShader(DxcSupport, VectorAccumulateDescriptorShader, "cs_6_10", Args, - Verbose); +static void runQueryAccumLayout(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + bool Verbose) { + std::string Args = "-HV 202x"; + size_t BufferSize = elementSize(ComponentType::I32); - auto Expected = makeExpectedVec(ComponentType::F16, NumElements, 1.0); + compileShader(DxcSupport, QueryAccumLayoutShader, "cs_6_10", Args, Verbose); - auto Op = createComputeOp(VectorAccumulateDescriptorShader, "cs_6_10", - "UAV(u0)", Args.c_str()); + auto Op = createComputeOp(QueryAccumLayoutShader, "cs_6_10", "UAV(u0)", + Args.c_str()); addUAVBuffer(Op.get(), "Output", BufferSize, true); addRootView(Op.get(), 0, "Output"); @@ -5262,786 +5650,398 @@ static void runVectorAccumulateDescriptor(ID3D12Device *Device, MappedData OutData; Result->Test->GetReadBackData("Output", &OutData); + const uint32_t *Out = static_cast(OutData.data()); - VERIFY_IS_TRUE(verifyComponentBuffer(ComponentType::F16, OutData.data(), - Expected, NumElements, Verbose)); + // Accum Layout must be A or B + VERIFY_IS_TRUE(Out[0] == static_cast(MatrixUse::A) || + Out[0] == static_cast(MatrixUse::B)); + if (Verbose) + hlsl_test::LogCommentFmt(L"AccumulatorLayout = %u", Out[0]); } -void DxilConf_SM610_LinAlg::VectorAccumulateDescriptor_Thread_F16() { - // Tier 1 requires no accumulation store formats, so this is gated. - if (!accumulateStoreApplicable( - D3DDevice, ComponentType::F16, - linalg_test::AtomicDestination::RWByteAddressBuffer, - L"VectorAccumulateDescriptor_Thread_F16")) +void DxilConf_SM610_LinAlg::QueryAccumLayout() { + // Constructs no matrix, so tier support is the only capability it needs. + if (!linAlgTierApplicable(D3DDevice, L"QueryAccumLayout")) return; - runVectorAccumulateDescriptor(D3DDevice, DxcSupport, VerboseLogging); -} - -namespace matvec_interpretation { - -static constexpr size_t OutputGuardBytes = 16; - -struct CaseData { - ComponentType MatrixType = ComponentType::Invalid; - MatrixDim M = 0; - MatrixDim N = 0; - MatrixLayout Layout = MatrixLayout::RowMajor; - ComponentType VectorInputType = ComponentType::Invalid; - ComponentType InputInterpretation = ComponentType::Invalid; - ComponentType BiasInputType = ComponentType::Invalid; - ComponentType ResultType = ComponentType::Invalid; - bool OutputSigned = true; - std::vector MatrixValues; - std::vector InterpretedVectorValues; - std::vector BiasValues; - std::wstring PublicRule; - - bool hasBias() const { return BiasInputType != ComponentType::Invalid; } -}; - -static std::optional componentByteSize(ComponentType Type) { - switch (Type) { - case ComponentType::I8: - case ComponentType::U8: - return 1; - case ComponentType::F16: - case ComponentType::I16: - case ComponentType::U16: - return 2; - case ComponentType::F32: - case ComponentType::I32: - case ComponentType::U32: - return 4; - default: - return std::nullopt; - } -} - -static bool isPackedByteVector(ComponentType Type) { - return Type == ComponentType::I8 || Type == ComponentType::U8; -} - -static const char *storageTypeName(ComponentType Type) { - if (isPackedByteVector(Type)) - return "uint"; - - switch (Type) { - case ComponentType::F16: - return "half"; - case ComponentType::F32: - return "float"; - case ComponentType::I32: - return "int"; - case ComponentType::U32: - return "uint"; - default: - return nullptr; - } -} - -static MatrixDim storageElementCount(ComponentType Type, - MatrixDim LogicalCount) { - return isPackedByteVector(Type) ? (LogicalCount + 3) / 4 : LogicalCount; -} - -static size_t storageElementByteSize(ComponentType Type) { - return isPackedByteVector(Type) ? sizeof(uint32_t) - : componentByteSize(Type).value_or(0); + runQueryAccumLayout(D3DDevice, DxcSupport, VerboseLogging); } -static bool checkedMultiplyInt64(int64_t Left, int64_t Right, int64_t &Result) { - if (Left == 0 || Right == 0) { - Result = 0; - return true; - } - if ((Left == -1 && Right == std::numeric_limits::min()) || - (Right == -1 && Left == std::numeric_limits::min())) - return false; +static const char LoadMemoryShader[] = R"( + RWByteAddressBuffer Input : register(u0); + RWByteAddressBuffer Output : register(u1); + groupshared ELEM_TYPE GsData[M_DIM * N_DIM]; - if (Left > 0) { - if ((Right > 0 && Left > std::numeric_limits::max() / Right) || - (Right < 0 && Right < std::numeric_limits::min() / Left)) - return false; - } else { - if ((Right > 0 && Left < std::numeric_limits::min() / Right) || - (Right < 0 && Left < std::numeric_limits::max() / Right)) - return false; - } + #define ELEM_PER_THREAD (M_DIM * N_DIM / NUMTHREADS) - Result = Left * Right; - return true; -} + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif + [numthreads(NUMTHREADS, 1, 1)] + void main(uint threadID : SV_GroupIndex) { + for (uint I = 0; I < ELEM_PER_THREAD; ++I) { + uint Index = threadID * ELEM_PER_THREAD + I; + GsData[Index] = Input.Load(Index * ELEM_SIZE); + } -static bool checkedAddInt64(int64_t Left, int64_t Right, int64_t &Result) { - if ((Right > 0 && Left > std::numeric_limits::max() - Right) || - (Right < 0 && Left < std::numeric_limits::min() - Right)) - return false; - Result = Left + Right; - return true; -} + GroupMemoryBarrierWithGroupSync(); -template -static std::vector encodeNativeVector(const std::vector &Values) { - static_assert(std::is_trivially_copyable::value, - "Vector values must be trivially copyable"); - std::vector Bytes(Values.size() * sizeof(T)); - if (!Bytes.empty()) - std::memcpy(Bytes.data(), Values.data(), Bytes.size()); - return Bytes; -} + if (GetGroupWaveIndex() != 0) + return; -static std::optional encodeByte(ComponentType Type, int64_t Value) { - if (Type == ComponentType::I8) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return std::nullopt; - return static_cast(static_cast(static_cast(Value))); - } - if (Type == ComponentType::U8) { - if (Value < 0 || Value > std::numeric_limits::max()) - return std::nullopt; - return static_cast(Value); + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Mat; + __builtin_LinAlg_MatrixLoadFromMemory( + Mat, GsData, OFFSET / ELEM_SIZE, STRIDE / ELEM_SIZE, LAYOUT); + __builtin_LinAlg_MatrixStoreToDescriptor( + Mat, Output, OFFSET, STRIDE, LAYOUT, 128); } - return std::nullopt; -} +)"; -static std::optional> -encodeComponents(ComponentType Type, const std::vector &Values) { - switch (Type) { - case ComponentType::I8: - case ComponentType::U8: { - std::vector Bytes; - Bytes.reserve(Values.size()); - for (int64_t Value : Values) { - std::optional Encoded = encodeByte(Type, Value); - if (!Encoded) - return std::nullopt; - Bytes.push_back(*Encoded); - } - return Bytes; - } - case ComponentType::F16: { - std::vector Native; - Native.reserve(Values.size()); - for (int64_t Value : Values) { - const HLSLHalf_t Half(static_cast(Value)); - if (static_cast(Half) != static_cast(Value)) - return std::nullopt; - Native.push_back(Half); - } - return encodeNativeVector(Native); - } - case ComponentType::F32: { - std::vector Native; - Native.reserve(Values.size()); - for (int64_t Value : Values) { - const float FloatValue = static_cast(Value); - if (static_cast(FloatValue) != Value) - return std::nullopt; - Native.push_back(FloatValue); - } - return encodeNativeVector(Native); - } - case ComponentType::I32: { - std::vector Native; - Native.reserve(Values.size()); - for (int64_t Value : Values) { - if (Value < std::numeric_limits::min() || - Value > std::numeric_limits::max()) - return std::nullopt; - Native.push_back(static_cast(Value)); - } - return encodeNativeVector(Native); - } - case ComponentType::U32: { - std::vector Native; - Native.reserve(Values.size()); - for (int64_t Value : Values) { - if (Value < 0 || - static_cast(Value) > std::numeric_limits::max()) - return std::nullopt; - Native.push_back(static_cast(Value)); - } - return encodeNativeVector(Native); - } - default: - return std::nullopt; - } -} +static void runLoadMemory(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); -static std::optional> -encodePackedVector(ComponentType Type, const std::vector &Values) { - if (!isPackedByteVector(Type)) - return std::nullopt; + std::stringstream ExtraDefs; + ExtraDefs << " -DOFFSET=" << 0; - size_t PaddedCount; - if (!cpu_oracle::checkedAdd(Values.size(), size_t(3), PaddedCount)) - return std::nullopt; - PaddedCount &= ~size_t(3); - std::vector Bytes(PaddedCount, 0); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - for (size_t WordIndex = 0; WordIndex < PaddedCount / 4; ++WordIndex) { - uint32_t Word = 0; - for (size_t Lane = 0; Lane < 4; ++Lane) { - const size_t ValueIndex = WordIndex * 4 + Lane; - if (ValueIndex == Values.size()) - break; - std::optional Encoded = encodeByte(Type, Values[ValueIndex]); - if (!Encoded) - return std::nullopt; - // Lane zero occupies the least-significant byte of each uint. - Word |= static_cast(*Encoded) << (Lane * 8); - } - for (size_t ByteIndex = 0; ByteIndex < 4; ++ByteIndex) - Bytes[WordIndex * 4 + ByteIndex] = - static_cast(Word >> (ByteIndex * 8)); - } - return Bytes; -} + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); -static std::optional matrixStrideBytes(const CaseData &Case) { - const std::optional ComponentSize = - componentByteSize(Case.MatrixType); - if (!ComponentSize) - return std::nullopt; - const size_t MinorCount = - Case.Layout == MatrixLayout::RowMajor ? Case.N : Case.M; - size_t Stride; - if (!cpu_oracle::checkedMultiply(MinorCount, *ComponentSize, Stride)) - return std::nullopt; - return Stride; + compileShader(DxcSupport, LoadMemoryShader, "cs_6_10", Args, Verbose); + + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, 1); + + auto Op = createComputeOp(LoadMemoryShader, "cs_6_10", "UAV(u0), UAV(u1)", + Args.c_str()); + addUAVBuffer(Op.get(), "Input", BufferSize, false, "byname"); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Input"); + addRootView(Op.get(), 1, "Output"); + + auto Result = + runShaderOp(Device, DxcSupport, std::move(Op), + [NumElements, Params](LPCSTR Name, std::vector &Data, + st::ShaderOp *) { + VERIFY_IS_TRUE(fillInputBuffer(Name, Data, Params.CompType, + NumElements), + "Saw unsupported component type"); + }); + + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); + + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); } -static std::optional> -encodeMatrixBuffer(const CaseData &Case) { - const std::optional ComponentSize = - componentByteSize(Case.MatrixType); - const std::optional Stride = matrixStrideBytes(Case); - const std::optional> Logical = - encodeComponents(Case.MatrixType, Case.MatrixValues); - if (!ComponentSize || !Stride || !Logical) - return std::nullopt; +void DxilConf_SM610_LinAlg::LoadMemory_Wave_16x16_F16() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; - const size_t MajorCount = - Case.Layout == MatrixLayout::RowMajor ? Case.M : Case.N; - size_t BufferSize; - if (!cpu_oracle::checkedMultiply(MajorCount, *Stride, BufferSize)) - return std::nullopt; - std::vector Buffer(BufferSize, 0); + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"LoadMemory_Wave_16x16_F16", + SelectedWaveSize)) + return; - for (MatrixDim Row = 0; Row < Case.M; ++Row) { - for (MatrixDim Column = 0; Column < Case.N; ++Column) { - const size_t SourceIndex = static_cast(Row) * Case.N + Column; - const size_t SourceOffset = SourceIndex * *ComponentSize; - const size_t DestinationOffset = - Case.Layout == MatrixLayout::RowMajor - ? static_cast(Row) * *Stride + Column * *ComponentSize - : static_cast(Column) * *Stride + Row * *ComponentSize; - std::memcpy(Buffer.data() + DestinationOffset, - Logical->data() + SourceOffset, *ComponentSize); - } - } - return Buffer; + runLoadMemory(D3DDevice, DxcSupport, Params, VerboseLogging, + SelectedWaveSize); } -static std::optional> -calculateExpected(const CaseData &Case) { - size_t MatrixElementCount; - if (!cpu_oracle::checkedMultiply(static_cast(Case.M), - static_cast(Case.N), - MatrixElementCount) || - Case.MatrixValues.size() != MatrixElementCount || - Case.InterpretedVectorValues.size() != Case.N || - (Case.hasBias() && Case.BiasValues.size() != Case.M)) - return std::nullopt; +static const char StoreMemoryShader[] = R"( + RWByteAddressBuffer Output : register(u0); + groupshared ELEM_TYPE GsData[M_DIM * N_DIM]; - std::vector Expected(Case.M, 0); - for (MatrixDim Row = 0; Row < Case.M; ++Row) { - for (MatrixDim Column = 0; Column < Case.N; ++Column) { - int64_t Product; - int64_t Sum; - if (!checkedMultiplyInt64( - Case.MatrixValues[static_cast(Row) * Case.N + Column], - Case.InterpretedVectorValues[Column], Product) || - !checkedAddInt64(Expected[Row], Product, Sum)) - return std::nullopt; - Expected[Row] = Sum; - } - if (Case.hasBias()) { - int64_t Sum; - if (!checkedAddInt64(Expected[Row], Case.BiasValues[Row], Sum)) - return std::nullopt; - Expected[Row] = Sum; + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif + [numthreads(NUMTHREADS, 1, 1)] + void main() { + if (GetGroupWaveIndex() != 0) + return; + + __builtin_LinAlgMatrix + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] + Mat; + __builtin_LinAlg_FillMatrix(Mat, FILL_VALUE); + + __builtin_LinAlg_MatrixStoreToMemory( + Mat, GsData, OFFSET / ELEM_SIZE, STRIDE / ELEM_SIZE, LAYOUT); + + for (uint I = 0; I < M_DIM*N_DIM; ++I) { + Output.Store(I*ELEM_SIZE, GsData[I]); } } - return Expected; -} +)"; -static bool oracleSelfTest() { - const std::optional> PackedSInt8 = - encodePackedVector(ComponentType::I8, {-1, 2, -3, 4, 5}); - const std::optional> PackedUInt8 = - encodePackedVector(ComponentType::U8, {255, 2, 253, 4, 5}); - const std::vector PackedBytes = {0xff, 0x02, 0xfd, 0x04, - 0x05, 0x00, 0x00, 0x00}; +static void runStoreMemory(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + float FillValue, UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); - CaseData DotCase = {}; - DotCase.M = 2; - DotCase.N = 3; - DotCase.MatrixValues = {1, 2, 3, -1, 4, 0}; - DotCase.InterpretedVectorValues = {4, -2, 5}; - DotCase.BiasInputType = ComponentType::I32; - DotCase.BiasValues = {7, -3}; - const std::optional> Dot = calculateExpected(DotCase); + std::stringstream ExtraDefs; + ExtraDefs << " -DOFFSET=" << 0; + STREAM_FLOAT(ExtraDefs, "FILL_VALUE", FillValue); - int64_t Ignored; - return PackedSInt8 == PackedBytes && PackedUInt8 == PackedBytes && Dot && - *Dot == std::vector({22, -15}) && - !checkedMultiplyInt64(std::numeric_limits::max(), 2, - Ignored) && - !checkedAddInt64(std::numeric_limits::max(), 1, Ignored); -} + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; -static bool isCaseValid(const CaseData &Case) { - size_t MatrixElementCount; - if (Case.M == 0 || Case.N == 0 || - !cpu_oracle::checkedMultiply(static_cast(Case.M), - static_cast(Case.N), - MatrixElementCount) || - Case.MatrixValues.size() != MatrixElementCount || - Case.InterpretedVectorValues.size() != Case.N || - (Case.Layout != MatrixLayout::RowMajor && - Case.Layout != MatrixLayout::ColumnMajor) || - !componentByteSize(Case.MatrixType) || - !storageTypeName(Case.VectorInputType) || - !storageTypeName(Case.ResultType) || Case.PublicRule.empty()) - return false; + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - // A vector is either native or an InterpretedVector, which pairs a packed - // vector with an interpretation type. A native element type paired with a - // narrower interpretation is not a valid form. - if (Case.VectorInputType == ComponentType::F32 && - Case.InputInterpretation != ComponentType::F32) - return false; - if (isPackedByteVector(Case.VectorInputType) && - Case.InputInterpretation != Case.VectorInputType) - return false; - if (Case.hasBias() != !Case.BiasValues.empty() || - (Case.hasBias() && (Case.BiasValues.size() != Case.M || - Case.BiasInputType != Case.ResultType || - !storageTypeName(Case.BiasInputType)))) - return false; + compileShader(DxcSupport, StoreMemoryShader, "cs_6_10", Args, Verbose); - const bool ExpectedSigned = Case.ResultType != ComponentType::U32; - return Case.OutputSigned == ExpectedSigned; -} + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, + FillValue, /*Increment=*/false); -static std::optional> -encodeVectorBuffer(const CaseData &Case) { - if (isPackedByteVector(Case.VectorInputType)) - return encodePackedVector(Case.VectorInputType, - Case.InterpretedVectorValues); - return encodeComponents(Case.VectorInputType, Case.InterpretedVectorValues); -} + auto Op = + createComputeOp(StoreMemoryShader, "cs_6_10", "UAV(u0)", Args.c_str()); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Output"); -static std::optional> -encodeExpectedOutput(const CaseData &Case) { - const std::optional> Values = calculateExpected(Case); - if (!Values) - return std::nullopt; - const std::optional> Logical = - encodeComponents(Case.ResultType, *Values); - if (!Logical) - return std::nullopt; + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); - size_t PaddedSize; - if (!cpu_oracle::checkedAdd(Logical->size(), size_t(3), PaddedSize)) - return std::nullopt; - PaddedSize &= ~size_t(3); - size_t BufferSize; - if (!cpu_oracle::checkedAdd(PaddedSize, OutputGuardBytes, BufferSize)) - return std::nullopt; + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); - std::vector Buffer(BufferSize); - cpu_oracle::fillPoison(Buffer.data(), Buffer.size()); - std::memcpy(Buffer.data(), Logical->data(), Logical->size()); - return Buffer; + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); } -static bool needs16BitTypes(ComponentType Type) { - return Type == ComponentType::F16 || Type == ComponentType::I16 || - Type == ComponentType::U16; -} +void DxilConf_SM610_LinAlg::StoreMemory_Wave_16x16_F16() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::A; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; -static std::optional buildCompilerArgs(const CaseData &Case) { - const std::optional MatrixStride = matrixStrideBytes(Case); - const char *InputStorageType = storageTypeName(Case.VectorInputType); - const char *OutputType = storageTypeName(Case.ResultType); - const char *BiasStorageType = - Case.hasBias() ? storageTypeName(Case.BiasInputType) : nullptr; - if (!MatrixStride || !InputStorageType || !OutputType || - (Case.hasBias() && !BiasStorageType)) - return std::nullopt; + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"StoreMemory_Wave_16x16_F16", + SelectedWaveSize)) + return; - std::stringstream Args; - Args << "-HV 202x"; - Args << " -DMATRIX_COMP_TYPE=" << static_cast(Case.MatrixType); - Args << " -DM_DIM=" << Case.M; - Args << " -DN_DIM=" << Case.N; - Args << " -DMATRIX_STRIDE=" << *MatrixStride; - Args << " -DMATRIX_LAYOUT=" << static_cast(Case.Layout); - Args << " -DINPUT_STORAGE_TYPE=" << InputStorageType; - Args << " -DINPUT_STORAGE_COUNT=" - << storageElementCount(Case.VectorInputType, Case.N); - Args << " -DINPUT_STORAGE_SIZE=" - << storageElementByteSize(Case.VectorInputType); - Args << " -DINPUT_INTERP=" << static_cast(Case.InputInterpretation); - Args << " -DOUTPUT_TYPE=" << OutputType; - Args << " -DOUTPUT_SIZE=" << componentByteSize(Case.ResultType).value_or(0); - Args << " -DOUTPUT_SIGNED=" << (Case.OutputSigned ? 1 : 0); - if (Case.hasBias()) { - Args << " -DBIAS_STORAGE_TYPE=" << BiasStorageType; - Args << " -DBIAS_STORAGE_COUNT=" - << storageElementCount(Case.BiasInputType, Case.M); - Args << " -DBIAS_STORAGE_SIZE=" - << storageElementByteSize(Case.BiasInputType); - } - if (needs16BitTypes(Case.MatrixType) || - needs16BitTypes(Case.VectorInputType) || - needs16BitTypes(Case.BiasInputType) || needs16BitTypes(Case.ResultType)) - Args << " -enable-16bit-types"; - return Args.str(); + runStoreMemory(D3DDevice, DxcSupport, Params, VerboseLogging, + /*FillValue=*/7.0f, SelectedWaveSize); } -static bool verifyExactBuffer(const void *ActualBuffer, size_t ActualSize, - const std::vector &Expected, bool Verbose) { - if (ActualSize != Expected.size()) { - hlsl_test::LogErrorFmt( - L"MatVec output size mismatch: actual=%zu, expected=%zu", ActualSize, - Expected.size()); - return false; - } +static const char AccumulateMemoryShader[] = R"( + RWByteAddressBuffer Output : register(u0); + groupshared ELEM_TYPE GsData[M_DIM * N_DIM]; - const BYTE *Actual = static_cast(ActualBuffer); - size_t MismatchCount = 0; - for (size_t I = 0; I < Expected.size(); ++I) { - if (Actual[I] == Expected[I]) - continue; - if (MismatchCount < 8) - hlsl_test::LogErrorFmt( - L"MatVec output byte %zu mismatch: actual=0x%02x, expected=0x%02x", I, - Actual[I], Expected[I]); - ++MismatchCount; - } - if (MismatchCount != 0) { - hlsl_test::LogErrorFmt(L"%zu MatVec output bytes differed", MismatchCount); - return false; - } - if (Verbose) - hlsl_test::LogCommentFmt( - L"All %zu MatVec output, padding, and guard bytes matched exactly", - Expected.size()); - return true; -} + #define ELEM_PER_THREAD (M_DIM * N_DIM / NUMTHREADS) -static const char MatVecMulShader[] = R"( - #define USE_A 0 - #define SCOPE_THREAD 0 + #ifdef FORCED_WAVE_SIZE + [WaveSize(FORCED_WAVE_SIZE)] + #else + [WaveSize(4, 128)] + #endif + [numthreads(NUMTHREADS, 1, 1)] + void main(uint threadID : SV_GroupIndex) { + ELEM_TYPE fill = FILL_VALUE; + for (uint I = 0; I < ELEM_PER_THREAD; ++I) { + uint Index = threadID * ELEM_PER_THREAD + I; + GsData[Index] = fill; + } - ByteAddressBuffer MatrixInput : register(t0); - ByteAddressBuffer VectorInput : register(t1); - RWByteAddressBuffer Output : register(u2); + GroupMemoryBarrierWithGroupSync(); + + if (GetGroupWaveIndex() != 0) + return; - [numthreads(1, 1, 1)] - void main() { __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes( - MATRIX_COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] + [[__LinAlgMatrix_Attributes(COMP_TYPE, M_DIM, N_DIM, USE, SCOPE)]] Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, MatrixInput, 0, MATRIX_STRIDE, MATRIX_LAYOUT, 128); - - vector InVec; - for (uint I = 0; I < INPUT_STORAGE_COUNT; ++I) { - InVec[I] = - VectorInput.Load(I * INPUT_STORAGE_SIZE); - } + __builtin_LinAlg_FillMatrix(Mat, FILL_VALUE); - vector OutVec; - __builtin_LinAlg_MatrixVectorMultiply( - OutVec, Mat, OUTPUT_SIGNED, InVec, INPUT_INTERP); + __builtin_LinAlg_MatrixAccumulateToMemory( + Mat, GsData, COMP_TYPE, OFFSET / ELEM_SIZE, STRIDE / ELEM_SIZE, LAYOUT); - for (uint I = 0; I < M_DIM; ++I) { - Output.Store(I * OUTPUT_SIZE, OutVec[I]); + for (uint I = 0; I < M_DIM*N_DIM; ++I) { + Output.Store(I*ELEM_SIZE, GsData[I]); } } )"; -static const char MatVecMulAddShader[] = R"( - #define USE_A 0 - #define SCOPE_THREAD 0 +static void runAccumulateMemory(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + const MatrixParams &Params, bool Verbose, + float FillValue, UINT ForcedWaveSize = 0) { + const size_t NumElements = Params.totalElements(); + const size_t BufferSize = Params.totalBytes(); - ByteAddressBuffer MatrixInput : register(t0); - ByteAddressBuffer VectorInput : register(t1); - ByteAddressBuffer BiasInput : register(t2); - RWByteAddressBuffer Output : register(u3); + std::stringstream ExtraDefs; + ExtraDefs << " -DOFFSET=" << 0; + STREAM_FLOAT(ExtraDefs, "FILL_VALUE", FillValue); - [numthreads(1, 1, 1)] - void main() { - __builtin_LinAlgMatrix - [[__LinAlgMatrix_Attributes( - MATRIX_COMP_TYPE, M_DIM, N_DIM, USE_A, SCOPE_THREAD)]] - Mat; - __builtin_LinAlg_MatrixLoadFromDescriptor( - Mat, MatrixInput, 0, MATRIX_STRIDE, MATRIX_LAYOUT, 128); + if (ForcedWaveSize != 0) + ExtraDefs << " -DFORCED_WAVE_SIZE=" << ForcedWaveSize; - vector InVec; - for (uint I = 0; I < INPUT_STORAGE_COUNT; ++I) { - InVec[I] = - VectorInput.Load(I * INPUT_STORAGE_SIZE); - } + std::string Args = buildCompilerArgs(Params, ExtraDefs.str().c_str()); - vector BiasVec; - for (uint I = 0; I < BIAS_STORAGE_COUNT; ++I) { - BiasVec[I] = BiasInput.Load(I * BIAS_STORAGE_SIZE); - } + compileShader(DxcSupport, AccumulateMemoryShader, "cs_6_10", Args, Verbose); - vector OutVec; - __builtin_LinAlg_MatrixVectorMultiplyAdd( - OutVec, Mat, OUTPUT_SIGNED, InVec, INPUT_INTERP, BiasVec); + auto Expected = makeExpectedMat(Params.CompType, Params.M, Params.N, + FillValue * 2, /*Increment=*/false); - for (uint I = 0; I < M_DIM; ++I) { - Output.Store(I * OUTPUT_SIZE, OutVec[I]); - } + auto Op = createComputeOp(AccumulateMemoryShader, "cs_6_10", "UAV(u0)", + Args.c_str()); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Output"); + + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); + + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); + + VERIFY_IS_TRUE(verifyComponentBuffer(Params.CompType, OutData.data(), + Expected, NumElements, Verbose)); +} + +void DxilConf_SM610_LinAlg::AccumulateMemory_Wave_16x16_F16() { + MatrixParams Params = {}; + Params.CompType = ComponentType::F16; + Params.M = 16; + Params.N = 16; + Params.Use = MatrixUse::Accumulator; + Params.Scope = MatrixScope::Wave; + Params.Layout = MatrixLayout::RowMajor; + Params.NumThreads = 128; + Params.Enable16Bit = true; + + UINT SelectedWaveSize = 0; + if (!matrixConstructionApplicable(D3DDevice, Params, {Params.Use}, + L"AccumulateMemory_Wave_16x16_F16", + SelectedWaveSize)) + return; + if (!accumulateStoreApplicable(D3DDevice, Params.CompType, + linalg_test::AtomicDestination::GroupShared, + L"AccumulateMemory_Wave_16x16_F16")) + return; + + runAccumulateMemory(D3DDevice, DxcSupport, Params, VerboseLogging, + /*FillValue=*/7.0f, SelectedWaveSize); +} + +static const char ConvertShader[] = R"( + #define CT_F16 8 + #define CT_F32 9 + + RWByteAddressBuffer Output : register(u0); + + [numthreads(1, 1, 1)] + void main() { + vector InVec = {1.0, 2.0, 3.0, 4.0}; + vector OutVec; + __builtin_LinAlg_Convert(OutVec, InVec, CT_F16, CT_F32); + Output.Store(0, OutVec.x); + Output.Store(4, OutVec.y); + Output.Store(8, OutVec.z); + Output.Store(12, OutVec.w); } )"; -static HRESULT querySupport(ID3D12Device *Device, const CaseData &Case, - bool &TierSupported, bool &Supported) { - TierSupported = false; - Supported = false; - if (!Device) - return E_INVALIDARG; +static void runConvert(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, + bool Verbose) { + std::string Args = "-HV 202x -enable-16bit-types"; + MatrixDim NumElements = 4; + size_t BufferSize = elementSize(ComponentType::F32) * NumElements; + + compileShader(DxcSupport, ConvertShader, "cs_6_10", Args, Verbose); - const std::optional VectorType = - toCapabilityDataType(Case.VectorInputType); - const std::optional MatrixType = - toCapabilityDataType(Case.MatrixType); - const std::optional BiasType = - Case.hasBias() ? toCapabilityDataType(Case.BiasInputType) - : std::optional( - linalg_abi::D3D12_LINEAR_ALGEBRA_DATATYPE_NONE); - const std::optional ResultType = - toCapabilityDataType(Case.ResultType); - if (!VectorType || !MatrixType || !BiasType || !ResultType) - return E_INVALIDARG; + auto Expected = makeExpectedVec(ComponentType::F32, NumElements, 1.0); - linalg_test::TierSupport Tier; - HRESULT HR = linalg_test::queryTierSupport(Device, Tier); - if (FAILED(HR)) - return HR; - TierSupported = Tier.supported(); - if (!TierSupported) - return S_OK; + auto Op = createComputeOp(ConvertShader, "cs_6_10", "UAV(u0)", Args.c_str()); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Output"); - linalg_test::ThreadVectorMatrixMultiplySupport Multiply; - HR = linalg_test::queryThreadVectorMatrixMultiply( - Device, {*VectorType, *MatrixType, *BiasType, *ResultType}, Multiply); - if (FAILED(HR)) - return HR; + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); - Supported = Multiply.supported(); - if (!Supported) - hlsl_test::LogCommentFmt( - L"ThreadVectorMatrixMultiply reports vector=%u matrix=%u bias=%u " - L"result=%u layout=%u is unsupported", - static_cast(*VectorType), static_cast(*MatrixType), - static_cast(*BiasType), static_cast(*ResultType), - static_cast(Case.Layout)); - return S_OK; + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); + + VERIFY_IS_TRUE(verifyComponentBuffer(ComponentType::F32, OutData.data(), + Expected, NumElements, Verbose)); } -static void runCase(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, - const CaseData &Case, bool Verbose) { - const bool SelfTestPassed = oracleSelfTest(); - VERIFY_IS_TRUE(SelfTestPassed, "MatVec host oracle self-test failed"); - const bool Valid = isCaseValid(Case); - VERIFY_IS_TRUE(Valid, "Invalid MatVec interpretation case"); - if (!SelfTestPassed || !Valid) +void DxilConf_SM610_LinAlg::Convert() { + // Operates on vectors rather than matrices, so tier support is the only + // capability it needs. + if (!linAlgTierApplicable(D3DDevice, L"Convert")) return; - const std::optional> MatrixBuffer = - encodeMatrixBuffer(Case); - const std::optional> VectorBuffer = - encodeVectorBuffer(Case); - const std::optional> BiasBuffer = - Case.hasBias() ? encodeComponents(Case.BiasInputType, Case.BiasValues) - : std::optional>(); - const std::optional> ExpectedOutput = - encodeExpectedOutput(Case); - const std::optional Args = buildCompilerArgs(Case); - VERIFY_IS_TRUE(MatrixBuffer.has_value()); - VERIFY_IS_TRUE(VectorBuffer.has_value()); - VERIFY_IS_TRUE(!Case.hasBias() || BiasBuffer.has_value()); - VERIFY_IS_TRUE(ExpectedOutput.has_value()); - VERIFY_IS_TRUE(Args.has_value()); - if (!MatrixBuffer || !VectorBuffer || (Case.hasBias() && !BiasBuffer) || - !ExpectedOutput || !Args) - return; + runConvert(D3DDevice, DxcSupport, VerboseLogging); +} - const char *Shader = Case.hasBias() ? MatVecMulAddShader : MatVecMulShader; - const char *RootSignature = Case.hasBias() - ? "SRV(t0), SRV(t1), SRV(t2), UAV(u3)" - : "SRV(t0), SRV(t1), UAV(u2)"; - compileShader(DxcSupport, Shader, "cs_6_10", *Args, Verbose); +static const char VectorAccumulateDescriptorShader[] = R"( + RWByteAddressBuffer Output : register(u0); - auto Op = createComputeOp(Shader, "cs_6_10", RootSignature, Args->c_str()); - addSRVBuffer(Op.get(), "MatrixInput", MatrixBuffer->size(), "byname"); - addSRVBuffer(Op.get(), "VectorInput", VectorBuffer->size(), "byname"); - if (Case.hasBias()) - addSRVBuffer(Op.get(), "BiasInput", BiasBuffer->size(), "byname"); - addUAVBuffer(Op.get(), "Output", ExpectedOutput->size(), true, "byname"); - addRootView(Op.get(), 0, "MatrixInput"); - addRootView(Op.get(), 1, "VectorInput"); - if (Case.hasBias()) { - addRootView(Op.get(), 2, "BiasInput"); - addRootView(Op.get(), 3, "Output"); - } else { - addRootView(Op.get(), 2, "Output"); + [numthreads(1, 1, 1)] + void main() { + vector InVec = {1.0, 2.0, 3.0, 4.0}; + __builtin_LinAlg_VectorAccumulateToDescriptor(Output, 0, 64, InVec); } +)"; - auto Result = - runShaderOp(Device, DxcSupport, std::move(Op), - [&](LPCSTR Name, std::vector &Data, st::ShaderOp *) { - if (_stricmp(Name, "Output") == 0) { - cpu_oracle::fillPoison(Data.data(), Data.size()); - return; - } +static void runVectorAccumulateDescriptor(ID3D12Device *Device, + dxc::SpecificDllLoader &DxcSupport, + bool Verbose) { + std::string Args = "-HV 202x -enable-16bit-types"; + MatrixDim NumElements = 4; + size_t BufferSize = elementSize(ComponentType::F16) * NumElements; - const std::vector *Source = nullptr; - if (_stricmp(Name, "MatrixInput") == 0) - Source = &*MatrixBuffer; - else if (_stricmp(Name, "VectorInput") == 0) - Source = &*VectorBuffer; - else if (Case.hasBias() && _stricmp(Name, "BiasInput") == 0) - Source = &*BiasBuffer; - VERIFY_IS_TRUE(Source != nullptr, - "Unexpected MatVec resource initializer"); - if (!Source) - return; - VERIFY_IS_TRUE(Data.size() == Source->size(), - "MatVec resource initializer size mismatch"); - if (Data.size() == Source->size()) - std::memcpy(Data.data(), Source->data(), Data.size()); - }); + compileShader(DxcSupport, VectorAccumulateDescriptorShader, "cs_6_10", Args, + Verbose); - MappedData OutData; - Result->Test->GetReadBackData("Output", &OutData); - VERIFY_IS_TRUE(verifyExactBuffer(OutData.data(), OutData.size(), - *ExpectedOutput, Verbose)); -} + auto Expected = makeExpectedVec(ComponentType::F16, NumElements, 1.0); -static void runCapabilityChecked(ID3D12Device *Device, - dxc::SpecificDllLoader &DxcSupport, - const CaseData &Case, - linalg_test::CapabilityRequirement Requirement, - LPCWSTR CaseName, bool Verbose) { - bool TierSupported = false; - bool Supported = false; - const HRESULT QueryResult = - querySupport(Device, Case, TierSupported, Supported); - const linalg_test::CapabilityRequirement Effective = - SUCCEEDED(QueryResult) && !TierSupported - ? linalg_test::CapabilityRequirement::CapabilityGated - : Requirement; - if (!applyApplicability( - linalg_test::classifyApplicability(QueryResult, Supported, Effective), - CaseName)) - return; - runCase(Device, DxcSupport, Case, Verbose); -} + auto Op = createComputeOp(VectorAccumulateDescriptorShader, "cs_6_10", + "UAV(u0)", Args.c_str()); + addUAVBuffer(Op.get(), "Output", BufferSize, true); + addRootView(Op.get(), 0, "Output"); -static CaseData makeNonUniformF16Case(MatrixLayout Layout) { - CaseData Case = {}; - Case.MatrixType = ComponentType::F16; - Case.M = 4; - Case.N = 8; - Case.Layout = Layout; - Case.VectorInputType = ComponentType::F16; - Case.InputInterpretation = ComponentType::F16; - Case.ResultType = ComponentType::F16; - Case.MatrixValues = { - 1, 0, -1, 2, -2, 3, -3, 1, 0, 1, 2, -1, 3, -2, 1, -3, - -1, 2, 0, 1, -2, 1, 3, -1, 2, -1, 1, 0, 1, -3, -2, 3, - }; - Case.InterpretedVectorValues = {1, -2, 3, -1, 2, -3, 1, 2}; - Case.PublicRule = - Layout == MatrixLayout::RowMajor - ? L"Exact non-uniform F16 RowMajor matrix-vector dot products" - : L"Exact non-uniform F16 ColumnMajor matrix-vector dot products"; - return Case; -} + auto Result = runShaderOp(Device, DxcSupport, std::move(Op)); -static CaseData makeSInt8Case() { - CaseData Case = {}; - Case.MatrixType = ComponentType::I8; - Case.M = 4; - Case.N = 8; - Case.Layout = MatrixLayout::RowMajor; - Case.VectorInputType = ComponentType::I8; - Case.InputInterpretation = ComponentType::I8; - Case.ResultType = ComponentType::I32; - Case.MatrixValues = { - 1, -2, 3, -4, 5, -6, 7, -8, -1, 2, -3, 4, -5, 6, -7, 8, - 1, 1, 1, 1, 1, 1, 1, 1, -8, -7, -6, -5, -4, -3, -2, -1, - }; - Case.InterpretedVectorValues = {1, -1, 2, -2, 3, -3, 4, -4}; - Case.PublicRule = - L"Exact packed SInt8 vector times SInt8 matrix dot products"; - return Case; -} + MappedData OutData; + Result->Test->GetReadBackData("Output", &OutData); -static CaseData makeUInt8Case() { - CaseData Case = {}; - Case.MatrixType = ComponentType::U8; - Case.M = 4; - Case.N = 8; - Case.Layout = MatrixLayout::RowMajor; - Case.VectorInputType = ComponentType::U8; - Case.InputInterpretation = ComponentType::U8; - Case.ResultType = ComponentType::I32; - Case.MatrixValues = { - 255, 1, 2, 3, 4, 5, 6, 7, 128, 127, 1, 1, 1, 1, 1, 1, - 200, 0, 200, 0, 200, 0, 200, 0, 0, 200, 0, 200, 0, 200, 0, 200, - }; - Case.InterpretedVectorValues = {1, 255, 2, 254, 3, 253, 4, 252}; - Case.PublicRule = - L"Exact packed UInt8 vector times UInt8 matrix dot products"; - return Case; + VERIFY_IS_TRUE(verifyComponentBuffer(ComponentType::F16, OutData.data(), + Expected, NumElements, Verbose)); } -static CaseData makeUInt32OutputCase() { - CaseData Case = {}; - Case.MatrixType = ComponentType::U32; - Case.M = 4; - Case.N = 8; - Case.Layout = MatrixLayout::RowMajor; - Case.VectorInputType = ComponentType::U32; - Case.InputInterpretation = ComponentType::U32; - Case.ResultType = ComponentType::U32; - Case.OutputSigned = false; - Case.MatrixValues = { - 2147483648LL, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, - 100, 0, 100, 0, 100, 0, 100, 0, 0, 200, 0, 200, 0, 200, 0, 200, - }; - Case.InterpretedVectorValues = {1, 1, 1, 1, 1, 1, 1, 1}; - Case.PublicRule = - L"Exact native UInt32 matrix-vector results with unsigned output"; - return Case; -} +void DxilConf_SM610_LinAlg::VectorAccumulateDescriptor_Thread_F16() { + // Tier 1 requires no accumulation store formats, so this is gated. + if (!accumulateStoreApplicable( + D3DDevice, ComponentType::F16, + linalg_test::AtomicDestination::RWByteAddressBuffer, + L"VectorAccumulateDescriptor_Thread_F16")) + return; -} // namespace matvec_interpretation + runVectorAccumulateDescriptor(D3DDevice, DxcSupport, VerboseLogging); +} void DxilConf_SM610_LinAlg::MatVecMul_Thread_4x8_F16_NonUniform() { const matvec_interpretation::CaseData Case = From 5f25de91cf6ea5a4d5915880e68061104ebe82ac Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Sat, 15 Aug 2026 08:31:32 +1200 Subject: [PATCH 3/6] [HLSL] Address review feedback on the LinAlg MatVec interpretation tests Move the checked int64 helpers into cpu_oracle, next to the existing checkedMultiply and checkedAdd for size_t. That namespace is already the file's home for arithmetic the oracle relies on, so matvec_interpretation no longer carries its own copies. The checks stay: encodeComponents accepts UInt32 values up to 4294967295, so a UInt32 matrix times a UInt32 vector overflows int64 by construction. An overflow in the oracle yields a wrong expected value, which can pass a broken implementation rather than fail a correct one, so this is the direction worth guarding. Report the rejected value when a component cannot be represented in the target type. All six rejection sites now name the type and the value instead of failing with no context, and componentTypeName covers SInt8 and UInt8 rather than returning "Unsupported" for them. Promote the oracle self-test out of runCase into its own method on LinAlgCPUOracleTests, so it runs once instead of once per case. That class already exists for exactly this purpose and deliberately carries no Kits metadata, so HLK runs never select it. Each assertion gets its own VERIFY, replacing a single six-term conjunction that could not say which part failed. Verified with the full HLSLExec LinAlg selection on WARP, compared per test rather than by totals: 46 total, 40 passed, 5 failed, 1 skipped. The only difference from the parent commit is the added MatVecHostOracle passing; the non-passing set is unchanged. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 159 ++++++++++-------- 1 file changed, 91 insertions(+), 68 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index 105e0cab0d..ddae2d7ee8 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -655,6 +655,37 @@ static bool checkedAdd(size_t Left, size_t Right, size_t &Result) { return true; } +static bool checkedMultiplyInt64(int64_t Left, int64_t Right, int64_t &Result) { + if (Left == 0 || Right == 0) { + Result = 0; + return true; + } + if ((Left == -1 && Right == std::numeric_limits::min()) || + (Right == -1 && Left == std::numeric_limits::min())) + return false; + + if (Left > 0) { + if ((Right > 0 && Left > std::numeric_limits::max() / Right) || + (Right < 0 && Right < std::numeric_limits::min() / Left)) + return false; + } else { + if ((Right > 0 && Left < std::numeric_limits::min() / Right) || + (Right < 0 && Left < std::numeric_limits::max() / Right)) + return false; + } + + Result = Left * Right; + return true; +} + +static bool checkedAddInt64(int64_t Left, int64_t Right, int64_t &Result) { + if ((Right > 0 && Left > std::numeric_limits::max() - Right) || + (Right < 0 && Left < std::numeric_limits::min() - Right)) + return false; + Result = Left + Right; + return true; +} + static bool isSupportedComponentType(ComponentType CompType) { switch (CompType) { case ComponentType::F16: @@ -671,6 +702,10 @@ static LPCWSTR componentTypeName(ComponentType CompType) { switch (CompType) { case ComponentType::I16: return L"I16"; + case ComponentType::I8: + return L"I8"; + case ComponentType::U8: + return L"U8"; case ComponentType::F16: return L"F16"; case ComponentType::F32: @@ -1741,37 +1776,6 @@ static size_t storageElementByteSize(ComponentType Type) { : componentByteSize(Type).value_or(0); } -static bool checkedMultiplyInt64(int64_t Left, int64_t Right, int64_t &Result) { - if (Left == 0 || Right == 0) { - Result = 0; - return true; - } - if ((Left == -1 && Right == std::numeric_limits::min()) || - (Right == -1 && Left == std::numeric_limits::min())) - return false; - - if (Left > 0) { - if ((Right > 0 && Left > std::numeric_limits::max() / Right) || - (Right < 0 && Right < std::numeric_limits::min() / Left)) - return false; - } else { - if ((Right > 0 && Left < std::numeric_limits::min() / Right) || - (Right < 0 && Left < std::numeric_limits::max() / Right)) - return false; - } - - Result = Left * Right; - return true; -} - -static bool checkedAddInt64(int64_t Left, int64_t Right, int64_t &Result) { - if ((Right > 0 && Left > std::numeric_limits::max() - Right) || - (Right < 0 && Left < std::numeric_limits::min() - Right)) - return false; - Result = Left + Right; - return true; -} - template static std::vector encodeNativeVector(const std::vector &Values) { static_assert(std::is_trivially_copyable::value, @@ -1782,16 +1786,22 @@ static std::vector encodeNativeVector(const std::vector &Values) { return Bytes; } +static std::nullopt_t reportUnrepresentable(ComponentType Type, int64_t Value) { + hlsl_test::LogErrorFmt(L"MatVec case value %lld is not representable as %s", + Value, cpu_oracle::componentTypeName(Type)); + return std::nullopt; +} + static std::optional encodeByte(ComponentType Type, int64_t Value) { if (Type == ComponentType::I8) { if (Value < std::numeric_limits::min() || Value > std::numeric_limits::max()) - return std::nullopt; + return reportUnrepresentable(Type, Value); return static_cast(static_cast(static_cast(Value))); } if (Type == ComponentType::U8) { if (Value < 0 || Value > std::numeric_limits::max()) - return std::nullopt; + return reportUnrepresentable(Type, Value); return static_cast(Value); } return std::nullopt; @@ -1818,7 +1828,7 @@ encodeComponents(ComponentType Type, const std::vector &Values) { for (int64_t Value : Values) { const HLSLHalf_t Half(static_cast(Value)); if (static_cast(Half) != static_cast(Value)) - return std::nullopt; + return reportUnrepresentable(Type, Value); Native.push_back(Half); } return encodeNativeVector(Native); @@ -1829,7 +1839,7 @@ encodeComponents(ComponentType Type, const std::vector &Values) { for (int64_t Value : Values) { const float FloatValue = static_cast(Value); if (static_cast(FloatValue) != Value) - return std::nullopt; + return reportUnrepresentable(Type, Value); Native.push_back(FloatValue); } return encodeNativeVector(Native); @@ -1840,7 +1850,7 @@ encodeComponents(ComponentType Type, const std::vector &Values) { for (int64_t Value : Values) { if (Value < std::numeric_limits::min() || Value > std::numeric_limits::max()) - return std::nullopt; + return reportUnrepresentable(Type, Value); Native.push_back(static_cast(Value)); } return encodeNativeVector(Native); @@ -1851,7 +1861,7 @@ encodeComponents(ComponentType Type, const std::vector &Values) { for (int64_t Value : Values) { if (Value < 0 || static_cast(Value) > std::numeric_limits::max()) - return std::nullopt; + return reportUnrepresentable(Type, Value); Native.push_back(static_cast(Value)); } return encodeNativeVector(Native); @@ -1952,16 +1962,17 @@ calculateExpected(const CaseData &Case) { for (MatrixDim Column = 0; Column < Case.N; ++Column) { int64_t Product; int64_t Sum; - if (!checkedMultiplyInt64( + if (!cpu_oracle::checkedMultiplyInt64( Case.MatrixValues[static_cast(Row) * Case.N + Column], Case.InterpretedVectorValues[Column], Product) || - !checkedAddInt64(Expected[Row], Product, Sum)) + !cpu_oracle::checkedAddInt64(Expected[Row], Product, Sum)) return std::nullopt; Expected[Row] = Sum; } if (Case.hasBias()) { int64_t Sum; - if (!checkedAddInt64(Expected[Row], Case.BiasValues[Row], Sum)) + if (!cpu_oracle::checkedAddInt64(Expected[Row], Case.BiasValues[Row], + Sum)) return std::nullopt; Expected[Row] = Sum; } @@ -1969,31 +1980,6 @@ calculateExpected(const CaseData &Case) { return Expected; } -static bool oracleSelfTest() { - const std::optional> PackedSInt8 = - encodePackedVector(ComponentType::I8, {-1, 2, -3, 4, 5}); - const std::optional> PackedUInt8 = - encodePackedVector(ComponentType::U8, {255, 2, 253, 4, 5}); - const std::vector PackedBytes = {0xff, 0x02, 0xfd, 0x04, - 0x05, 0x00, 0x00, 0x00}; - - CaseData DotCase = {}; - DotCase.M = 2; - DotCase.N = 3; - DotCase.MatrixValues = {1, 2, 3, -1, 4, 0}; - DotCase.InterpretedVectorValues = {4, -2, 5}; - DotCase.BiasInputType = ComponentType::I32; - DotCase.BiasValues = {7, -3}; - const std::optional> Dot = calculateExpected(DotCase); - - int64_t Ignored; - return PackedSInt8 == PackedBytes && PackedUInt8 == PackedBytes && Dot && - *Dot == std::vector({22, -15}) && - !checkedMultiplyInt64(std::numeric_limits::max(), 2, - Ignored) && - !checkedAddInt64(std::numeric_limits::max(), 1, Ignored); -} - static bool isCaseValid(const CaseData &Case) { size_t MatrixElementCount; if (Case.M == 0 || Case.N == 0 || @@ -2255,11 +2241,9 @@ static HRESULT querySupport(ID3D12Device *Device, const CaseData &Case, static void runCase(ID3D12Device *Device, dxc::SpecificDllLoader &DxcSupport, const CaseData &Case, bool Verbose) { - const bool SelfTestPassed = oracleSelfTest(); - VERIFY_IS_TRUE(SelfTestPassed, "MatVec host oracle self-test failed"); const bool Valid = isCaseValid(Case); VERIFY_IS_TRUE(Valid, "Invalid MatVec interpretation case"); - if (!SelfTestPassed || !Valid) + if (!Valid) return; const std::optional> MatrixBuffer = @@ -2446,6 +2430,7 @@ class LinAlgCPUOracleTests { TEST_METHOD(UntouchedByteVerification); TEST_METHOD(ViewBoundedElements); TEST_METHOD(ViewBoundedStoreBytes); + TEST_METHOD(MatVecHostOracle); }; void LinAlgCPUOracleTests::TypedMatrixBufferRoundTrip() { @@ -2838,6 +2823,44 @@ void LinAlgCPUOracleTests::ViewBoundedStoreBytes() { VERIFY_ARE_EQUAL(*Corrupted, static_cast(0)); } +void LinAlgCPUOracleTests::MatVecHostOracle() { + using namespace matvec_interpretation; + + const std::vector PackedBytes = {0xff, 0x02, 0xfd, 0x04, + 0x05, 0x00, 0x00, 0x00}; + const std::optional> PackedSInt8 = + encodePackedVector(ComponentType::I8, {-1, 2, -3, 4, 5}); + VERIFY_IS_TRUE(PackedSInt8.has_value(), "SInt8 packing failed"); + VERIFY_IS_TRUE(PackedSInt8 == PackedBytes, + "SInt8 packing produced unexpected bytes"); + + const std::optional> PackedUInt8 = + encodePackedVector(ComponentType::U8, {255, 2, 253, 4, 5}); + VERIFY_IS_TRUE(PackedUInt8.has_value(), "UInt8 packing failed"); + VERIFY_IS_TRUE(PackedUInt8 == PackedBytes, + "UInt8 packing produced unexpected bytes"); + + CaseData DotCase = {}; + DotCase.M = 2; + DotCase.N = 3; + DotCase.MatrixValues = {1, 2, 3, -1, 4, 0}; + DotCase.InterpretedVectorValues = {4, -2, 5}; + DotCase.BiasInputType = ComponentType::I32; + DotCase.BiasValues = {7, -3}; + const std::optional> Dot = calculateExpected(DotCase); + VERIFY_IS_TRUE(Dot.has_value(), "Biased dot product oracle returned nothing"); + VERIFY_IS_TRUE(*Dot == std::vector({22, -15}), + "Biased dot product oracle returned the wrong values"); + + int64_t Ignored; + VERIFY_IS_FALSE(cpu_oracle::checkedMultiplyInt64( + std::numeric_limits::max(), 2, Ignored), + "checkedMultiplyInt64 missed an overflow"); + VERIFY_IS_FALSE(cpu_oracle::checkedAddInt64( + std::numeric_limits::max(), 1, Ignored), + "checkedAddInt64 missed an overflow"); +} + class LinAlgCapabilityTests { public: BEGIN_TEST_CLASS(LinAlgCapabilityTests) From 9b8b5b58c9416a85a57f606897af292b0ca64bd3 Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Sat, 15 Aug 2026 10:06:48 +1200 Subject: [PATCH 4/6] [HLSL] Drop the checked integer arithmetic from the MatVec oracle Follows the same review feedback Damyan and Chris gave on #8774, applied here so the two pull requests stay consistent and so neither lands this code. Every value these helpers guarded is authored by the test: dimensions are literals of at most M=16 and N=16, and the largest integer literal in the file is 65504. The widest accumulation the oracle can perform is far below the int64 range, so the overflow branches were unreachable, and an overflow would have indicated a bug in the test rather than a driver failing conformance. checkedAddInt64 and checkedMultiplyInt64 are removed together with the two self-test assertions that existed only to exercise them. calculateExpected returns its result directly rather than an optional and reads as ordinary arithmetic. Its size preconditions are already established by isCaseValid, which runCase verifies before any of this is reached. This also removes a collision that neither pull request shows in its own diff. #8774 defines the same two helpers in the same cpu_oracle namespace but in a different region of the file, so git would have merged both without conflict and left main with a duplicate definition. Verified with the full HLSLExec LinAlg selection on WARP, compared per test rather than by totals: 46 total, 40 passed, 5 failed, 1 skipped, identical to the parent commit. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 85 +++---------------- 1 file changed, 11 insertions(+), 74 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index ddae2d7ee8..985b988484 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -655,37 +655,6 @@ static bool checkedAdd(size_t Left, size_t Right, size_t &Result) { return true; } -static bool checkedMultiplyInt64(int64_t Left, int64_t Right, int64_t &Result) { - if (Left == 0 || Right == 0) { - Result = 0; - return true; - } - if ((Left == -1 && Right == std::numeric_limits::min()) || - (Right == -1 && Left == std::numeric_limits::min())) - return false; - - if (Left > 0) { - if ((Right > 0 && Left > std::numeric_limits::max() / Right) || - (Right < 0 && Right < std::numeric_limits::min() / Left)) - return false; - } else { - if ((Right > 0 && Left < std::numeric_limits::min() / Right) || - (Right < 0 && Left < std::numeric_limits::max() / Right)) - return false; - } - - Result = Left * Right; - return true; -} - -static bool checkedAddInt64(int64_t Left, int64_t Right, int64_t &Result) { - if ((Right > 0 && Left > std::numeric_limits::max() - Right) || - (Right < 0 && Left < std::numeric_limits::min() - Right)) - return false; - Result = Left + Right; - return true; -} - static bool isSupportedComponentType(ComponentType CompType) { switch (CompType) { case ComponentType::F16: @@ -1946,36 +1915,15 @@ encodeMatrixBuffer(const CaseData &Case) { return Buffer; } -static std::optional> -calculateExpected(const CaseData &Case) { - size_t MatrixElementCount; - if (!cpu_oracle::checkedMultiply(static_cast(Case.M), - static_cast(Case.N), - MatrixElementCount) || - Case.MatrixValues.size() != MatrixElementCount || - Case.InterpretedVectorValues.size() != Case.N || - (Case.hasBias() && Case.BiasValues.size() != Case.M)) - return std::nullopt; - +static std::vector calculateExpected(const CaseData &Case) { std::vector Expected(Case.M, 0); for (MatrixDim Row = 0; Row < Case.M; ++Row) { - for (MatrixDim Column = 0; Column < Case.N; ++Column) { - int64_t Product; - int64_t Sum; - if (!cpu_oracle::checkedMultiplyInt64( - Case.MatrixValues[static_cast(Row) * Case.N + Column], - Case.InterpretedVectorValues[Column], Product) || - !cpu_oracle::checkedAddInt64(Expected[Row], Product, Sum)) - return std::nullopt; - Expected[Row] = Sum; - } - if (Case.hasBias()) { - int64_t Sum; - if (!cpu_oracle::checkedAddInt64(Expected[Row], Case.BiasValues[Row], - Sum)) - return std::nullopt; - Expected[Row] = Sum; - } + for (MatrixDim Column = 0; Column < Case.N; ++Column) + Expected[Row] += + Case.MatrixValues[static_cast(Row) * Case.N + Column] * + Case.InterpretedVectorValues[Column]; + if (Case.hasBias()) + Expected[Row] += Case.BiasValues[Row]; } return Expected; } @@ -2024,11 +1972,9 @@ encodeVectorBuffer(const CaseData &Case) { static std::optional> encodeExpectedOutput(const CaseData &Case) { - const std::optional> Values = calculateExpected(Case); - if (!Values) - return std::nullopt; + const std::vector Values = calculateExpected(Case); const std::optional> Logical = - encodeComponents(Case.ResultType, *Values); + encodeComponents(Case.ResultType, Values); if (!Logical) return std::nullopt; @@ -2847,18 +2793,9 @@ void LinAlgCPUOracleTests::MatVecHostOracle() { DotCase.InterpretedVectorValues = {4, -2, 5}; DotCase.BiasInputType = ComponentType::I32; DotCase.BiasValues = {7, -3}; - const std::optional> Dot = calculateExpected(DotCase); - VERIFY_IS_TRUE(Dot.has_value(), "Biased dot product oracle returned nothing"); - VERIFY_IS_TRUE(*Dot == std::vector({22, -15}), + const std::vector Dot = calculateExpected(DotCase); + VERIFY_IS_TRUE(Dot == std::vector({22, -15}), "Biased dot product oracle returned the wrong values"); - - int64_t Ignored; - VERIFY_IS_FALSE(cpu_oracle::checkedMultiplyInt64( - std::numeric_limits::max(), 2, Ignored), - "checkedMultiplyInt64 missed an overflow"); - VERIFY_IS_FALSE(cpu_oracle::checkedAddInt64( - std::numeric_limits::max(), 1, Ignored), - "checkedAddInt64 missed an overflow"); } class LinAlgCapabilityTests { From 00b20a159fad1aa13e2fcecf58583f2e3fe64a1a Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Tue, 18 Aug 2026 12:05:26 +1200 Subject: [PATCH 5/6] [HLSL] Make the MatVec component type switches exhaustive and loud Ashley asked that anything switching over ComponentType list every case we care about rather than lean on a default arm. The authoritative set is the fourteen types ComponentTypeTraits declares in hlsl/dx/linalg.h, which is narrower than the twenty four in DxilConstants.h: it excludes I1, the SNorm and UNorm variants, PackedS8x32 and PackedU8x32. componentByteSize, storageTypeName and isPackedByteVector now enumerate those fourteen, and componentByteSize answers the question the review opened with by returning 8 for I64, U64 and F64. storageElementCount no longer hardcodes 3 and 4. The packing factor comes from a new elementsPerScalar that mirrors ComponentTypeTraits::ElementsPerScalar, and Ashley's point about BFloat16 holds: it is two elements per scalar, not four. It is not derivable from the byte size either, since F16 and BFloat16 are both two bytes but only F16 has a native HLSL scalar. isPackedByteVector stays byte only for that same reason, because encodeVectorBuffer routes on it and encodePackedVector packs four bytes to a uint, which would corrupt BFloat16. isCaseValid is now a sequence of grouped guards covering dimensions, input counts, layout, component types, vector form and bias, rather than one nine term conjunction. Its checkedMultiply is gone: MatrixDim is uint32_t, so widening the row by column product to uint64_t cannot overflow. The remaining checked arithmetic this branch introduced is removed on the same grounds as the parent commit, which leaves the file free of qualified cpu_oracle::checked calls. The size_t helpers those calls reached into are pre-existing in main and are a separate cleanup. Unknown types no longer fail quietly. componentByteSize, elementsPerScalar and storageTypeName name the offending type and fail, because each maps a type onto a value its caller needs, so a missing answer is a defect rather than a result. The report includes the numeric enum value, since componentTypeName only knows seven types and prints Unsupported for the rest. isPackedByteVector and isEncodableComponentType stay silent because false is a legitimate answer for them. storageTypeName separates the two failures: a valid matrix type the encoder cannot drive yet reports as unsupported, while anything outside the fourteen reports as unexpected. This matters because isCaseValid has a single caller, so the previous behaviour was a generic invalid case failure that did not say which field was at fault. Verified with the full HLSLExec LinAlg selection on WARP, compared per test rather than by totals: 50 total, 44 passed, 5 failed, 1 skipped, identical to the parent commit in every entry. Both the baseline and this change were measured on the same experimental tier D3D12 runtime, because a default tier runtime cannot enable SM 6.10 and blocks the whole selection. The assertions were confirmed to fire by temporarily calling the helpers with PackedS8x32 and SNormF32 and observing the logged type and the failed test. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- .../clang/unittests/HLSLExec/LinAlgTests.cpp | 179 ++++++++++++++---- 1 file changed, 138 insertions(+), 41 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index 985b988484..8981e0b460 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -1695,26 +1695,86 @@ struct CaseData { bool hasBias() const { return BiasInputType != ComponentType::Invalid; } }; +static void reportUnexpectedComponentType(LPCWSTR Function, + ComponentType Type) { + hlsl_test::LogErrorFmt(L"%s received an unexpected ComponentType: %s (%u)", + Function, cpu_oracle::componentTypeName(Type), + static_cast(Type)); + VERIFY_FAIL(L"Unexpected ComponentType"); +} + +static void reportUnsupportedComponentType(LPCWSTR Function, + ComponentType Type) { + hlsl_test::LogErrorFmt(L"%s does not yet support ComponentType: %s (%u)", + Function, cpu_oracle::componentTypeName(Type), + static_cast(Type)); + VERIFY_FAIL(L"Unsupported ComponentType"); +} + static std::optional componentByteSize(ComponentType Type) { switch (Type) { case ComponentType::I8: case ComponentType::U8: + case ComponentType::F8_E4M3FN: + case ComponentType::F8_E5M2: return 1; - case ComponentType::F16: case ComponentType::I16: case ComponentType::U16: + case ComponentType::F16: + case ComponentType::BFloat16: return 2; - case ComponentType::F32: case ComponentType::I32: case ComponentType::U32: + case ComponentType::F32: return 4; + case ComponentType::I64: + case ComponentType::U64: + case ComponentType::F64: + return 8; default: + reportUnexpectedComponentType(L"componentByteSize", Type); return std::nullopt; } } +static MatrixDim elementsPerScalar(ComponentType Type) { + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: + case ComponentType::F8_E4M3FN: + case ComponentType::F8_E5M2: + return 4; + case ComponentType::BFloat16: + return 2; + case ComponentType::I16: + case ComponentType::U16: + case ComponentType::F16: + case ComponentType::I32: + case ComponentType::U32: + case ComponentType::F32: + case ComponentType::I64: + case ComponentType::U64: + case ComponentType::F64: + return 1; + default: + reportUnexpectedComponentType(L"elementsPerScalar", Type); + return 1; + } +} + +// True only for the byte-sized types that encodePackedVector can pack four to a +// uint. BFloat16 is also packed, but two to a uint, so it must not be routed +// through the byte packer. static bool isPackedByteVector(ComponentType Type) { - return Type == ComponentType::I8 || Type == ComponentType::U8; + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: + case ComponentType::F8_E4M3FN: + case ComponentType::F8_E5M2: + return true; + default: + return false; + } } static const char *storageTypeName(ComponentType Type) { @@ -1730,19 +1790,45 @@ static const char *storageTypeName(ComponentType Type) { return "int"; case ComponentType::U32: return "uint"; + // Valid matrix component types that the host encoder cannot yet produce + // values for. + case ComponentType::I16: + case ComponentType::U16: + case ComponentType::I64: + case ComponentType::U64: + case ComponentType::F64: + case ComponentType::BFloat16: + reportUnsupportedComponentType(L"storageTypeName", Type); + return nullptr; default: + reportUnexpectedComponentType(L"storageTypeName", Type); return nullptr; } } +static bool isEncodableComponentType(ComponentType Type) { + switch (Type) { + case ComponentType::I8: + case ComponentType::U8: + case ComponentType::F16: + case ComponentType::F32: + case ComponentType::I32: + case ComponentType::U32: + return true; + default: + return false; + } +} + static MatrixDim storageElementCount(ComponentType Type, MatrixDim LogicalCount) { - return isPackedByteVector(Type) ? (LogicalCount + 3) / 4 : LogicalCount; + const MatrixDim PerScalar = elementsPerScalar(Type); + return (LogicalCount + PerScalar - 1) / PerScalar; } static size_t storageElementByteSize(ComponentType Type) { - return isPackedByteVector(Type) ? sizeof(uint32_t) - : componentByteSize(Type).value_or(0); + return elementsPerScalar(Type) > 1 ? sizeof(uint32_t) + : componentByteSize(Type).value_or(0); } template @@ -1845,10 +1931,9 @@ encodePackedVector(ComponentType Type, const std::vector &Values) { if (!isPackedByteVector(Type)) return std::nullopt; - size_t PaddedCount; - if (!cpu_oracle::checkedAdd(Values.size(), size_t(3), PaddedCount)) - return std::nullopt; - PaddedCount &= ~size_t(3); + // Round the element count up to a whole number of 4-byte words. The count is + // bounded by the matrix dimensions, so the addition cannot overflow. + const size_t PaddedCount = (Values.size() + 3) & ~size_t(3); std::vector Bytes(PaddedCount, 0); for (size_t WordIndex = 0; WordIndex < PaddedCount / 4; ++WordIndex) { @@ -1877,10 +1962,7 @@ static std::optional matrixStrideBytes(const CaseData &Case) { return std::nullopt; const size_t MinorCount = Case.Layout == MatrixLayout::RowMajor ? Case.N : Case.M; - size_t Stride; - if (!cpu_oracle::checkedMultiply(MinorCount, *ComponentSize, Stride)) - return std::nullopt; - return Stride; + return MinorCount * *ComponentSize; } static std::optional> @@ -1895,10 +1977,7 @@ encodeMatrixBuffer(const CaseData &Case) { const size_t MajorCount = Case.Layout == MatrixLayout::RowMajor ? Case.M : Case.N; - size_t BufferSize; - if (!cpu_oracle::checkedMultiply(MajorCount, *Stride, BufferSize)) - return std::nullopt; - std::vector Buffer(BufferSize, 0); + std::vector Buffer(MajorCount * *Stride, 0); for (MatrixDim Row = 0; Row < Case.M; ++Row) { for (MatrixDim Column = 0; Column < Case.N; ++Column) { @@ -1929,18 +2008,31 @@ static std::vector calculateExpected(const CaseData &Case) { } static bool isCaseValid(const CaseData &Case) { - size_t MatrixElementCount; - if (Case.M == 0 || Case.N == 0 || - !cpu_oracle::checkedMultiply(static_cast(Case.M), - static_cast(Case.N), - MatrixElementCount) || - Case.MatrixValues.size() != MatrixElementCount || - Case.InterpretedVectorValues.size() != Case.N || - (Case.Layout != MatrixLayout::RowMajor && - Case.Layout != MatrixLayout::ColumnMajor) || - !componentByteSize(Case.MatrixType) || - !storageTypeName(Case.VectorInputType) || - !storageTypeName(Case.ResultType) || Case.PublicRule.empty()) + // Dimensions. + if (Case.M == 0 || Case.N == 0) + return false; + + // Input counts must match the declared dimensions. MatrixDim is 32 bits, so + // the row-by-column product cannot overflow a 64-bit comparison. + if (Case.MatrixValues.size() != static_cast(Case.M) * Case.N) + return false; + if (Case.InterpretedVectorValues.size() != Case.N) + return false; + + // Layout. + if (Case.Layout != MatrixLayout::RowMajor && + Case.Layout != MatrixLayout::ColumnMajor) + return false; + + // Component types the host and the shader can both express. + if (!isEncodableComponentType(Case.MatrixType)) + return false; + if (!storageTypeName(Case.VectorInputType)) + return false; + if (!storageTypeName(Case.ResultType)) + return false; + + if (Case.PublicRule.empty()) return false; // A vector is either native or an InterpretedVector, which pairs a packed @@ -1952,11 +2044,18 @@ static bool isCaseValid(const CaseData &Case) { if (isPackedByteVector(Case.VectorInputType) && Case.InputInterpretation != Case.VectorInputType) return false; - if (Case.hasBias() != !Case.BiasValues.empty() || - (Case.hasBias() && (Case.BiasValues.size() != Case.M || - Case.BiasInputType != Case.ResultType || - !storageTypeName(Case.BiasInputType)))) + + // Bias values are present exactly when a bias type is declared. + if (Case.hasBias() != !Case.BiasValues.empty()) return false; + if (Case.hasBias()) { + if (Case.BiasValues.size() != Case.M) + return false; + if (Case.BiasInputType != Case.ResultType) + return false; + if (!storageTypeName(Case.BiasInputType)) + return false; + } const bool ExpectedSigned = Case.ResultType != ComponentType::U32; return Case.OutputSigned == ExpectedSigned; @@ -1978,13 +2077,11 @@ encodeExpectedOutput(const CaseData &Case) { if (!Logical) return std::nullopt; - size_t PaddedSize; - if (!cpu_oracle::checkedAdd(Logical->size(), size_t(3), PaddedSize)) - return std::nullopt; - PaddedSize &= ~size_t(3); - size_t BufferSize; - if (!cpu_oracle::checkedAdd(PaddedSize, OutputGuardBytes, BufferSize)) - return std::nullopt; + // Round the byte count up to a whole number of 4-byte words so that the four + // guard bytes start on a word boundary. Both sizes are bounded by the matrix + // dimensions, so neither addition can overflow. + const size_t PaddedSize = (Logical->size() + 3) & ~size_t(3); + const size_t BufferSize = PaddedSize + OutputGuardBytes; std::vector Buffer(BufferSize); cpu_oracle::fillPoison(Buffer.data(), Buffer.size()); From 3fc0ed5e59dc5826ee1666d00b6e279308f3b70a Mon Sep 17 00:00:00 2001 From: Jack Elliott Date: Wed, 19 Aug 2026 06:56:46 +1200 Subject: [PATCH 6/6] [NFC] Correct the guard region comment in encodeExpectedOutput The comment above the output buffer sizing said the rounding aligned "the four guard bytes", but OutputGuardBytes is 16 and has been since this change was first written. The wording conflated the 4-byte word alignment being applied with the size of the guard region itself. Refer to the guard region rather than a byte count so the comment stays accurate if OutputGuardBytes ever changes. Comment only, no functional change. The full HLSLExec LinAlg suite is unchanged per test on a current D3D12 runtime (50 total, 49 passed, 0 failed, 1 skipped). Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 83725f5d-8e98-4c1d-91ee-ad47629e007b --- tools/clang/unittests/HLSLExec/LinAlgTests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp index 8981e0b460..e06162cf72 100644 --- a/tools/clang/unittests/HLSLExec/LinAlgTests.cpp +++ b/tools/clang/unittests/HLSLExec/LinAlgTests.cpp @@ -2077,9 +2077,9 @@ encodeExpectedOutput(const CaseData &Case) { if (!Logical) return std::nullopt; - // Round the byte count up to a whole number of 4-byte words so that the four - // guard bytes start on a word boundary. Both sizes are bounded by the matrix - // dimensions, so neither addition can overflow. + // Round the byte count up to a whole number of 4-byte words so that the + // guard region starts on a word boundary. Both sizes are bounded by the + // matrix dimensions, so neither addition can overflow. const size_t PaddedSize = (Logical->size() + 3) & ~size_t(3); const size_t BufferSize = PaddedSize + OutputGuardBytes;