diff options
Diffstat (limited to 'thirdparty/glslang/SPIRV/GlslangToSpv.cpp')
| -rw-r--r-- | thirdparty/glslang/SPIRV/GlslangToSpv.cpp | 343 |
1 files changed, 271 insertions, 72 deletions
diff --git a/thirdparty/glslang/SPIRV/GlslangToSpv.cpp b/thirdparty/glslang/SPIRV/GlslangToSpv.cpp index ec40f663a7..1674c55036 100644 --- a/thirdparty/glslang/SPIRV/GlslangToSpv.cpp +++ b/thirdparty/glslang/SPIRV/GlslangToSpv.cpp @@ -1,7 +1,7 @@ // // Copyright (C) 2014-2016 LunarG, Inc. // Copyright (C) 2015-2020 Google, Inc. -// Copyright (C) 2017 ARM Limited. +// Copyright (C) 2017, 2022-2024 Arm Limited. // Modifications Copyright (C) 2020 Advanced Micro Devices, Inc. All rights reserved. // // All rights reserved. @@ -204,7 +204,8 @@ protected: spv::Id createBinaryMatrixOperation(spv::Op, OpDecorations&, spv::Id typeId, spv::Id left, spv::Id right); spv::Id createUnaryOperation(glslang::TOperator op, OpDecorations&, spv::Id typeId, spv::Id operand, glslang::TBasicType typeProxy, - const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags); + const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, + const glslang::TType &opType); spv::Id createUnaryMatrixOperation(spv::Op op, OpDecorations&, spv::Id typeId, spv::Id operand, glslang::TBasicType typeProxy); spv::Id createConversion(glslang::TOperator op, OpDecorations&, spv::Id destTypeId, spv::Id operand, @@ -213,7 +214,8 @@ protected: spv::Id makeSmearedConstant(spv::Id constant, int vectorSize); spv::Id createAtomicOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy, - const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags); + const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, + const glslang::TType &opType); spv::Id createInvocationsOperation(glslang::TOperator op, spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy); spv::Id CreateInvocationsVectorOperation(spv::Op op, spv::GroupOperation groupOperation, @@ -225,7 +227,9 @@ protected: spv::Id createNoArgOperation(glslang::TOperator op, spv::Decoration precision, spv::Id typeId); spv::Id getSymbolId(const glslang::TIntermSymbol* node); void addMeshNVDecoration(spv::Id id, int member, const glslang::TQualifier & qualifier); + bool hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor); void addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor); + void addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather); spv::Id createSpvConstant(const glslang::TIntermTyped&); spv::Id createSpvConstantFromConstUnionArray(const glslang::TType& type, const glslang::TConstUnionArray&, int& nextConst, bool specConstant); @@ -280,6 +284,7 @@ protected: spv::Id taskPayloadID; // Used later for generating OpTraceKHR/OpExecuteCallableKHR/OpHitObjectRecordHit*/OpHitObjectGetShaderBindingTableData std::unordered_map<unsigned int, glslang::TIntermSymbol *> locationToSymbol[4]; + std::unordered_map<spv::Id, std::vector<spv::Decoration> > idToQCOMDecorations; }; // @@ -395,11 +400,11 @@ void TranslateMemoryDecoration(const glslang::TQualifier& qualifier, std::vector bool useVulkanMemoryModel) { if (!useVulkanMemoryModel) { - if (qualifier.isCoherent()) - memory.push_back(spv::DecorationCoherent); if (qualifier.isVolatile()) { memory.push_back(spv::DecorationVolatile); memory.push_back(spv::DecorationCoherent); + } else if (qualifier.isCoherent()) { + memory.push_back(spv::DecorationCoherent); } } if (qualifier.isRestrict()) @@ -1163,6 +1168,7 @@ spv::ImageFormat TGlslangToSpvTraverser::TranslateImageFormat(const glslang::TTy case glslang::ElfR64i: builder.addExtension(spv::E_SPV_EXT_shader_image_int64); builder.addCapability(spv::CapabilityInt64ImageEXT); + break; default: break; } @@ -1558,8 +1564,13 @@ TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion, this->options.generateDebugInfo = true; if (this->options.generateDebugInfo) { - builder.setEmitOpLines(); - builder.setSourceFile(glslangIntermediate->getSourceFile()); + if (this->options.emitNonSemanticShaderDebugInfo) { + builder.setEmitNonSemanticShaderDebugInfo(this->options.emitNonSemanticShaderDebugSource); + } + else { + builder.setEmitSpirvDebugInfo(); + } + builder.setDebugSourceFile(glslangIntermediate->getSourceFile()); // Set the source shader's text. If for SPV version 1.0, include // a preamble in comments stating the OpModuleProcessed instructions. @@ -1584,9 +1595,6 @@ TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion, builder.addInclude(iItr->first, iItr->second); } - builder.setEmitNonSemanticShaderDebugInfo(this->options.emitNonSemanticShaderDebugInfo); - builder.setEmitNonSemanticShaderDebugSource(this->options.emitNonSemanticShaderDebugSource); - stdBuiltins = builder.import("GLSL.std.450"); spv::AddressingModel addressingModel = spv::AddressingModelLogical; @@ -1635,6 +1643,24 @@ TGlslangToSpvTraverser::TGlslangToSpvTraverser(unsigned int spvVersion, builder.addExtension(spv::E_SPV_KHR_subgroup_uniform_control_flow); builder.addExecutionMode(shaderEntry, spv::ExecutionModeSubgroupUniformControlFlowKHR); } + if (glslangIntermediate->getMaximallyReconverges()) { + builder.addExtension(spv::E_SPV_KHR_maximal_reconvergence); + builder.addExecutionMode(shaderEntry, spv::ExecutionModeMaximallyReconvergesKHR); + } + + if (glslangIntermediate->getQuadDerivMode()) + { + builder.addCapability(spv::CapabilityQuadControlKHR); + builder.addExtension(spv::E_SPV_KHR_quad_control); + builder.addExecutionMode(shaderEntry, spv::ExecutionModeQuadDerivativesKHR); + } + + if (glslangIntermediate->getReqFullQuadsMode()) + { + builder.addCapability(spv::CapabilityQuadControlKHR); + builder.addExtension(spv::E_SPV_KHR_quad_control); + builder.addExecutionMode(shaderEntry, spv::ExecutionModeRequireFullQuadsKHR); + } unsigned int mode; switch (glslangIntermediate->getStage()) { @@ -1983,8 +2009,9 @@ void TGlslangToSpvTraverser::finishSpv(bool compileOnly) } // finish off the entry-point SPV instruction by adding the Input/Output <id> - for (auto it = iOSet.cbegin(); it != iOSet.cend(); ++it) - entryPoint->addIdOperand(*it); + entryPoint->reserveOperands(iOSet.size()); + for (auto id : iOSet) + entryPoint->addIdOperand(id); } // Add capabilities, extensions, remove unneeded decorations, etc., @@ -2019,7 +2046,9 @@ void TGlslangToSpvTraverser::visitSymbol(glslang::TIntermSymbol* symbol) { // We update the line information even though no code might be generated here // This is helpful to yield correct lines for control flow instructions - builder.setLine(symbol->getLoc().line, symbol->getLoc().getFilename()); + if (!linkageOnly) { + builder.setDebugSourceLocation(symbol->getLoc().line, symbol->getLoc().getFilename()); + } SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder); if (symbol->getType().isStruct()) @@ -2128,7 +2157,7 @@ void TGlslangToSpvTraverser::visitSymbol(glslang::TIntermSymbol* symbol) bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::TIntermBinary* node) { - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); if (node->getLeft()->getAsSymbolNode() != nullptr && node->getLeft()->getType().isStruct()) { glslangTypeToIdMap[node->getLeft()->getType().getStruct()] = node->getLeft()->getAsSymbolNode()->getId(); } @@ -2173,7 +2202,7 @@ bool TGlslangToSpvTraverser::visitBinary(glslang::TVisit /* visit */, glslang::T spv::Id rValue = accessChainLoad(node->getRight()->getType()); // reset line number for assignment - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); if (node->getOp() != glslang::EOpAssign) { // the left is also an r-value @@ -2506,7 +2535,7 @@ spv::Id TGlslangToSpvTraverser::translateForcedType(spv::Id object) bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TIntermUnary* node) { - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); SpecConstantOpModeGuard spec_constant_op_mode_setter(&builder); if (node->getType().getQualifier().isSpecConstant()) @@ -2670,7 +2699,7 @@ bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TI // if not, then possibly an operation if (! result) result = createUnaryOperation(node->getOp(), decorations, resultType(), operand, - node->getOperand()->getBasicType(), lvalueCoherentFlags); + node->getOperand()->getBasicType(), lvalueCoherentFlags, node->getType()); // it could be attached to a SPIR-V intruction if (!result) { @@ -2763,6 +2792,11 @@ bool TGlslangToSpvTraverser::visitUnary(glslang::TVisit /* visit */, glslang::TI return false; + case glslang::EOpAssumeEXT: + builder.addCapability(spv::CapabilityExpectAssumeKHR); + builder.addExtension(spv::E_SPV_KHR_expect_assume); + builder.createNoResultOp(spv::OpAssumeTrueKHR, operand); + return false; case glslang::EOpEmitStreamVertex: builder.createNoResultOp(spv::OpEmitStreamVertex, operand); return false; @@ -2894,11 +2928,11 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt return false; } else { if (node->getOp() == glslang::EOpScope) - builder.enterScope(0); + builder.enterLexicalBlock(0); } } else { if (sequenceDepth > 1 && node->getOp() == glslang::EOpScope) - builder.leaveScope(); + builder.leaveLexicalBlock(); --sequenceDepth; } @@ -2925,6 +2959,9 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt } case glslang::EOpFunction: if (visit == glslang::EvPreVisit) { + if (options.generateDebugInfo) { + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); + } if (isShaderEntryPoint(node)) { inEntryPoint = true; builder.setBuildPoint(shaderEntry->getLastBlock()); @@ -2933,10 +2970,10 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt } else { handleFunctionEntry(node); } - if (options.generateDebugInfo) { + if (options.generateDebugInfo && !options.emitNonSemanticShaderDebugInfo) { const auto& loc = node->getLoc(); const char* sourceFileName = loc.getFilename(); - spv::Id sourceFileId = sourceFileName ? builder.getStringId(sourceFileName) : builder.getSourceFile(); + spv::Id sourceFileId = sourceFileName ? builder.getStringId(sourceFileName) : builder.getMainFileId(); currentFunction->setDebugLineInfo(sourceFileId, loc.line, loc.column); } } else { @@ -2954,7 +2991,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt return false; case glslang::EOpFunctionCall: { - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); if (node->isUserDefined()) result = handleUserFunctionCall(node); if (result) { @@ -3020,7 +3057,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt case glslang::EOpConstructF16Mat4x3: case glslang::EOpConstructF16Mat4x4: isMatrix = true; - // fall through + [[fallthrough]]; case glslang::EOpConstructFloat: case glslang::EOpConstructVec2: case glslang::EOpConstructVec3: @@ -3075,7 +3112,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt case glslang::EOpConstructCooperativeMatrixNV: case glslang::EOpConstructCooperativeMatrixKHR: { - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); std::vector<spv::Id> arguments; translateArguments(*node, arguments, lvalueCoherentFlags); spv::Id constructed; @@ -3191,7 +3228,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt case glslang::EOpAtomicStore: noReturnValue = true; - // fallthrough + [[fallthrough]]; case glslang::EOpAtomicLoad: atomic = true; break; @@ -3221,6 +3258,12 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt binOp = node->getOp(); break; + case glslang::EOpExpectEXT: + builder.addCapability(spv::CapabilityExpectAssumeKHR); + builder.addExtension(spv::E_SPV_KHR_expect_assume); + binOp = node->getOp(); + break; + case glslang::EOpIgnoreIntersectionNV: case glslang::EOpTerminateRayNV: case glslang::EOpTraceNV: @@ -3288,7 +3331,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt case glslang::EOpHitObjectRecordHitWithIndexMotionNV: case glslang::EOpReorderThreadNV: noReturnValue = true; - //Fallthrough + [[fallthrough]]; case glslang::EOpHitObjectIsEmptyNV: case glslang::EOpHitObjectIsMissNV: case glslang::EOpHitObjectIsHitNV: @@ -3331,6 +3374,22 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt builder.addExtension(spv::E_SPV_QCOM_image_processing); break; + case glslang::EOpImageBlockMatchWindowSSDQCOM: + case glslang::EOpImageBlockMatchWindowSADQCOM: + builder.addCapability(spv::CapabilityTextureBlockMatchQCOM); + builder.addExtension(spv::E_SPV_QCOM_image_processing); + builder.addCapability(spv::CapabilityTextureBlockMatch2QCOM); + builder.addExtension(spv::E_SPV_QCOM_image_processing2); + break; + + case glslang::EOpImageBlockMatchGatherSSDQCOM: + case glslang::EOpImageBlockMatchGatherSADQCOM: + builder.addCapability(spv::CapabilityTextureBlockMatchQCOM); + builder.addExtension(spv::E_SPV_QCOM_image_processing); + builder.addCapability(spv::CapabilityTextureBlockMatch2QCOM); + builder.addExtension(spv::E_SPV_QCOM_image_processing2); + break; + case glslang::EOpFetchMicroTriangleVertexPositionNV: case glslang::EOpFetchMicroTriangleVertexBarycentricNV: builder.addExtension(spv::E_SPV_NV_displacement_micromap); @@ -3361,7 +3420,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt right->traverse(this); spv::Id rightId = accessChainLoad(right->getType()); - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); OpDecorations decorations = { precision, TranslateNoContractionDecoration(node->getType().getQualifier()), TranslateNonUniformDecoration(node->getType().getQualifier()) }; @@ -3596,7 +3655,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt lvalueCoherentFlags = builder.getAccessChain().coherentFlags; lvalueCoherentFlags |= TranslateCoherent(glslangOperands[arg]->getAsTyped()->getType()); } else { - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); glslang::TOperator glslangOp = node->getOp(); if (arg == 1 && (glslangOp == glslang::EOpRayQueryGetIntersectionType || @@ -3648,7 +3707,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt } } - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); if (node->getOp() == glslang::EOpCooperativeMatrixLoad || node->getOp() == glslang::EOpCooperativeMatrixLoadNV) { std::vector<spv::IdImmediate> idImmOps; @@ -3739,7 +3798,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt glslang::TBasicType typeProxy = (node->getOp() == glslang::EOpAtomicStore) ? node->getSequence()[0]->getAsTyped()->getBasicType() : node->getBasicType(); result = createAtomicOperation(node->getOp(), precision, resultType(), operands, typeProxy, - lvalueCoherentFlags); + lvalueCoherentFlags, node->getType()); } else if (node->getOp() == glslang::EOpSpirvInst) { const auto& spirvInst = node->getSpirvInstruction(); if (spirvInst.set == "") { @@ -3786,7 +3845,7 @@ bool TGlslangToSpvTraverser::visitAggregate(glslang::TVisit visit, glslang::TInt result = createUnaryOperation( node->getOp(), decorations, resultType(), operands.front(), - glslangOperands[0]->getAsTyped()->getBasicType(), lvalueCoherentFlags); + glslangOperands[0]->getAsTyped()->getBasicType(), lvalueCoherentFlags, node->getType()); } break; default: @@ -3888,7 +3947,7 @@ bool TGlslangToSpvTraverser::visitSelection(glslang::TVisit /* visit */, glslang node->getFalseBlock()->traverse(this); spv::Id falseValue = accessChainLoad(node->getFalseBlock()->getAsTyped()->getType()); - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); // done if void if (node->getBasicType() == glslang::EbtVoid) @@ -4096,7 +4155,7 @@ bool TGlslangToSpvTraverser::visitLoop(glslang::TVisit /* visit */, glslang::TIn // instructions in it, since the body/test may have arbitrary instructions, // including merges of its own. builder.setBuildPoint(&blocks.head); - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); builder.createLoopMerge(&blocks.merge, &blocks.continue_target, control, operands); if (node->testFirst() && node->getTest()) { spv::Block& test = builder.makeNewBlock(); @@ -4119,7 +4178,7 @@ bool TGlslangToSpvTraverser::visitLoop(glslang::TVisit /* visit */, glslang::TIn node->getTerminal()->traverse(this); builder.createBranch(&blocks.head); } else { - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); builder.createBranch(&blocks.body); breakForLoop.push(true); @@ -4154,7 +4213,7 @@ bool TGlslangToSpvTraverser::visitBranch(glslang::TVisit /* visit */, glslang::T if (node->getExpression()) node->getExpression()->traverse(this); - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); switch (node->getFlowOp()) { case glslang::EOpKill: @@ -5322,17 +5381,34 @@ void TGlslangToSpvTraverser::updateMemberOffset(const glslang::TType& structType int memberAlignment = glslangIntermediate->getMemberAlignment(memberType, memberSize, dummyStride, explicitLayout, matrixLayout == glslang::ElmRowMajor); + bool isVectorLike = memberType.isVector(); + if (memberType.isMatrix()) { + if (matrixLayout == glslang::ElmRowMajor) + isVectorLike = memberType.getMatrixRows() == 1; + else + isVectorLike = memberType.getMatrixCols() == 1; + } + // Adjust alignment for HLSL rules // TODO: make this consistent in early phases of code: // adjusting this late means inconsistencies with earlier code, which for reflection is an issue // Until reflection is brought in sync with these adjustments, don't apply to $Global, // which is the most likely to rely on reflection, and least likely to rely implicit layouts if (glslangIntermediate->usingHlslOffsets() && - ! memberType.isArray() && memberType.isVector() && structType.getTypeName().compare("$Global") != 0) { - int dummySize; - int componentAlignment = glslangIntermediate->getBaseAlignmentScalar(memberType, dummySize); - if (componentAlignment <= 4) + ! memberType.isStruct() && structType.getTypeName().compare("$Global") != 0) { + int componentSize; + int componentAlignment = glslangIntermediate->getBaseAlignmentScalar(memberType, componentSize); + if (! memberType.isArray() && isVectorLike && componentAlignment <= 4) memberAlignment = componentAlignment; + + // Don't add unnecessary padding after this member + if (memberType.isMatrix()) { + if (matrixLayout == glslang::ElmRowMajor) + memberSize -= componentSize * (4 - memberType.getMatrixCols()); + else + memberSize -= componentSize * (4 - memberType.getMatrixRows()); + } else if (memberType.isArray()) + memberSize -= componentSize * (4 - memberType.getVectorSize()); } // Bump up to member alignment @@ -5340,7 +5416,7 @@ void TGlslangToSpvTraverser::updateMemberOffset(const glslang::TType& structType // Bump up to vec4 if there is a bad straddle if (explicitLayout != glslang::ElpScalar && glslangIntermediate->improperStraddle(memberType, memberSize, - currentOffset)) + currentOffset, isVectorLike)) glslang::RoundToPow2(currentOffset, 16); nextOffset = currentOffset + memberSize; @@ -5425,8 +5501,10 @@ void TGlslangToSpvTraverser::makeFunctions(const glslang::TIntermSequence& glslF // memory and use RestrictPointer/AliasedPointer. if (originalParam(type.getQualifier().storage, type, false) || !writableParam(type.getQualifier().storage)) { - decorations.push_back(type.getQualifier().isRestrict() ? spv::DecorationRestrict : - spv::DecorationAliased); + // TranslateMemoryDecoration added Restrict decoration already. + if (!type.getQualifier().isRestrict()) { + decorations.push_back(spv::DecorationAliased); + } } else { decorations.push_back(type.getQualifier().isRestrict() ? spv::DecorationRestrictPointerEXT : spv::DecorationAliasedPointerEXT); @@ -5738,7 +5816,7 @@ spv::Id TGlslangToSpvTraverser::createImageTextureFunctionCall(glslang::TIntermO if (! node->isImage() && ! node->isTexture()) return spv::NoResult; - builder.setLine(node->getLoc().line, node->getLoc().getFilename()); + builder.setDebugSourceLocation(node->getLoc().line, node->getLoc().getFilename()); // Process a GLSL texturing op (will be SPV image) @@ -6041,7 +6119,7 @@ spv::Id TGlslangToSpvTraverser::createImageTextureFunctionCall(glslang::TIntermO operands.push_back(*opIt); return createAtomicOperation(node->getOp(), precision, resultType(), operands, typeProxy, - lvalueCoherentFlags); + lvalueCoherentFlags, node->getType()); } } @@ -6566,6 +6644,10 @@ spv::Id TGlslangToSpvTraverser::createBinaryOperation(glslang::TOperator op, OpD binOp = isUnsigned ? spv::OpUMul32x16INTEL : spv::OpIMul32x16INTEL; break; + case glslang::EOpExpectEXT: + binOp = spv::OpExpectKHR; + break; + case glslang::EOpLessThan: case glslang::EOpGreaterThan: case glslang::EOpLessThanEqual: @@ -6788,7 +6870,8 @@ spv::Id TGlslangToSpvTraverser::createBinaryMatrixOperation(spv::Op op, OpDecora } spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDecorations& decorations, spv::Id typeId, - spv::Id operand, glslang::TBasicType typeProxy, const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags) + spv::Id operand, glslang::TBasicType typeProxy, const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, + const glslang::TType &opType) { spv::Op unaryOp = spv::OpNop; int extBuiltins = -1; @@ -7076,7 +7159,7 @@ spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDe // Handle all of the atomics in one place, in createAtomicOperation() std::vector<spv::Id> operands; operands.push_back(operand); - return createAtomicOperation(op, decorations.precision, typeId, operands, typeProxy, lvalueCoherentFlags); + return createAtomicOperation(op, decorations.precision, typeId, operands, typeProxy, lvalueCoherentFlags, opType); } case glslang::EOpBitFieldReverse: @@ -7169,7 +7252,9 @@ spv::Id TGlslangToSpvTraverser::createUnaryOperation(glslang::TOperator op, OpDe case glslang::EOpSubgroupExclusiveXor: case glslang::EOpSubgroupQuadSwapHorizontal: case glslang::EOpSubgroupQuadSwapVertical: - case glslang::EOpSubgroupQuadSwapDiagonal: { + case glslang::EOpSubgroupQuadSwapDiagonal: + case glslang::EOpSubgroupQuadAll: + case glslang::EOpSubgroupQuadAny: { std::vector<spv::Id> operands; operands.push_back(operand); return createSubgroupOperation(op, typeId, operands, typeProxy); @@ -7792,7 +7877,7 @@ spv::Id TGlslangToSpvTraverser::makeSmearedConstant(spv::Id constant, int vector // For glslang ops that map to SPV atomic opCodes spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv::Decoration /*precision*/, spv::Id typeId, std::vector<spv::Id>& operands, glslang::TBasicType typeProxy, - const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags) + const spv::Builder::AccessChain::CoherentFlags &lvalueCoherentFlags, const glslang::TType &opType) { spv::Op opCode = spv::OpNop; @@ -7803,14 +7888,20 @@ spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv opCode = spv::OpAtomicIAdd; if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) { opCode = spv::OpAtomicFAddEXT; - builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_add); - if (typeProxy == glslang::EbtFloat16) { - builder.addExtension(spv::E_SPV_EXT_shader_atomic_float16_add); - builder.addCapability(spv::CapabilityAtomicFloat16AddEXT); - } else if (typeProxy == glslang::EbtFloat) { - builder.addCapability(spv::CapabilityAtomicFloat32AddEXT); + if (typeProxy == glslang::EbtFloat16 && + (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) { + builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector); + builder.addCapability(spv::CapabilityAtomicFloat16VectorNV); } else { - builder.addCapability(spv::CapabilityAtomicFloat64AddEXT); + builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_add); + if (typeProxy == glslang::EbtFloat16) { + builder.addExtension(spv::E_SPV_EXT_shader_atomic_float16_add); + builder.addCapability(spv::CapabilityAtomicFloat16AddEXT); + } else if (typeProxy == glslang::EbtFloat) { + builder.addCapability(spv::CapabilityAtomicFloat32AddEXT); + } else { + builder.addCapability(spv::CapabilityAtomicFloat64AddEXT); + } } } break; @@ -7823,13 +7914,19 @@ spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv case glslang::EOpAtomicCounterMin: if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) { opCode = spv::OpAtomicFMinEXT; - builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_min_max); - if (typeProxy == glslang::EbtFloat16) - builder.addCapability(spv::CapabilityAtomicFloat16MinMaxEXT); - else if (typeProxy == glslang::EbtFloat) - builder.addCapability(spv::CapabilityAtomicFloat32MinMaxEXT); - else - builder.addCapability(spv::CapabilityAtomicFloat64MinMaxEXT); + if (typeProxy == glslang::EbtFloat16 && + (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) { + builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector); + builder.addCapability(spv::CapabilityAtomicFloat16VectorNV); + } else { + builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_min_max); + if (typeProxy == glslang::EbtFloat16) + builder.addCapability(spv::CapabilityAtomicFloat16MinMaxEXT); + else if (typeProxy == glslang::EbtFloat) + builder.addCapability(spv::CapabilityAtomicFloat32MinMaxEXT); + else + builder.addCapability(spv::CapabilityAtomicFloat64MinMaxEXT); + } } else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) { opCode = spv::OpAtomicUMin; } else { @@ -7841,13 +7938,19 @@ spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv case glslang::EOpAtomicCounterMax: if (typeProxy == glslang::EbtFloat16 || typeProxy == glslang::EbtFloat || typeProxy == glslang::EbtDouble) { opCode = spv::OpAtomicFMaxEXT; - builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_min_max); - if (typeProxy == glslang::EbtFloat16) - builder.addCapability(spv::CapabilityAtomicFloat16MinMaxEXT); - else if (typeProxy == glslang::EbtFloat) - builder.addCapability(spv::CapabilityAtomicFloat32MinMaxEXT); - else - builder.addCapability(spv::CapabilityAtomicFloat64MinMaxEXT); + if (typeProxy == glslang::EbtFloat16 && + (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) { + builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector); + builder.addCapability(spv::CapabilityAtomicFloat16VectorNV); + } else { + builder.addExtension(spv::E_SPV_EXT_shader_atomic_float_min_max); + if (typeProxy == glslang::EbtFloat16) + builder.addCapability(spv::CapabilityAtomicFloat16MinMaxEXT); + else if (typeProxy == glslang::EbtFloat) + builder.addCapability(spv::CapabilityAtomicFloat32MinMaxEXT); + else + builder.addCapability(spv::CapabilityAtomicFloat64MinMaxEXT); + } } else if (typeProxy == glslang::EbtUint || typeProxy == glslang::EbtUint64) { opCode = spv::OpAtomicUMax; } else { @@ -7872,6 +7975,12 @@ spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv case glslang::EOpAtomicExchange: case glslang::EOpImageAtomicExchange: case glslang::EOpAtomicCounterExchange: + if ((typeProxy == glslang::EbtFloat16) && + (opType.getVectorSize() == 2 || opType.getVectorSize() == 4)) { + builder.addExtension(spv::E_SPV_NV_shader_atomic_fp16_vector); + builder.addCapability(spv::CapabilityAtomicFloat16VectorNV); + } + opCode = spv::OpAtomicExchange; break; case glslang::EOpAtomicCompSwap: @@ -7970,6 +8079,7 @@ spv::Id TGlslangToSpvTraverser::createAtomicOperation(glslang::TOperator op, spv } std::vector<spv::Id> spvAtomicOperands; // hold the spv operands + spvAtomicOperands.reserve(6); spvAtomicOperands.push_back(pointerId); spvAtomicOperands.push_back(scopeId); spvAtomicOperands.push_back(semanticsId); @@ -8274,6 +8384,11 @@ spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, s case glslang::EOpSubgroupElect: builder.addCapability(spv::CapabilityGroupNonUniform); break; + case glslang::EOpSubgroupQuadAll: + case glslang::EOpSubgroupQuadAny: + builder.addExtension(spv::E_SPV_KHR_quad_control); + builder.addCapability(spv::CapabilityQuadControlKHR); + [[fallthrough]]; case glslang::EOpSubgroupAll: case glslang::EOpSubgroupAny: case glslang::EOpSubgroupAllEqual: @@ -8293,6 +8408,11 @@ spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, s builder.addCapability(spv::CapabilityGroupNonUniform); builder.addCapability(spv::CapabilityGroupNonUniformBallot); break; + case glslang::EOpSubgroupRotate: + case glslang::EOpSubgroupClusteredRotate: + builder.addExtension(spv::E_SPV_KHR_subgroup_rotate); + builder.addCapability(spv::CapabilityGroupNonUniformRotateKHR); + break; case glslang::EOpSubgroupShuffle: case glslang::EOpSubgroupShuffleXor: builder.addCapability(spv::CapabilityGroupNonUniform); @@ -8381,7 +8501,9 @@ spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, s // Figure out which opcode to use. switch (op) { case glslang::EOpSubgroupElect: opCode = spv::OpGroupNonUniformElect; break; + case glslang::EOpSubgroupQuadAll: opCode = spv::OpGroupNonUniformQuadAllKHR; break; case glslang::EOpSubgroupAll: opCode = spv::OpGroupNonUniformAll; break; + case glslang::EOpSubgroupQuadAny: opCode = spv::OpGroupNonUniformQuadAnyKHR; break; case glslang::EOpSubgroupAny: opCode = spv::OpGroupNonUniformAny; break; case glslang::EOpSubgroupAllEqual: opCode = spv::OpGroupNonUniformAllEqual; break; case glslang::EOpSubgroupBroadcast: opCode = spv::OpGroupNonUniformBroadcast; break; @@ -8398,6 +8520,8 @@ spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, s case glslang::EOpSubgroupShuffleXor: opCode = spv::OpGroupNonUniformShuffleXor; break; case glslang::EOpSubgroupShuffleUp: opCode = spv::OpGroupNonUniformShuffleUp; break; case glslang::EOpSubgroupShuffleDown: opCode = spv::OpGroupNonUniformShuffleDown; break; + case glslang::EOpSubgroupRotate: + case glslang::EOpSubgroupClusteredRotate: opCode = spv::OpGroupNonUniformRotateKHR; break; case glslang::EOpSubgroupAdd: case glslang::EOpSubgroupInclusiveAdd: case glslang::EOpSubgroupExclusiveAdd: @@ -8578,7 +8702,10 @@ spv::Id TGlslangToSpvTraverser::createSubgroupOperation(glslang::TOperator op, s // Every operation begins with the Execution Scope operand. spv::IdImmediate executionScope = { true, builder.makeUintConstant(spv::ScopeSubgroup) }; - spvGroupOperands.push_back(executionScope); + // All other ops need the execution scope. Quad Control Ops don't need scope, it's always Quad. + if (opCode != spv::OpGroupNonUniformQuadAllKHR && opCode != spv::OpGroupNonUniformQuadAnyKHR) { + spvGroupOperands.push_back(executionScope); + } // Next, for all operations that use a Group Operation, push that as an operand. if (groupOperation != spv::GroupOperationMax) { @@ -8821,6 +8948,8 @@ spv::Id TGlslangToSpvTraverser::createMiscOperation(glslang::TOperator op, spv:: case glslang::EOpSubgroupShuffleXor: case glslang::EOpSubgroupShuffleUp: case glslang::EOpSubgroupShuffleDown: + case glslang::EOpSubgroupRotate: + case glslang::EOpSubgroupClusteredRotate: case glslang::EOpSubgroupClusteredAdd: case glslang::EOpSubgroupClusteredMul: case glslang::EOpSubgroupClusteredMin: @@ -9179,6 +9308,30 @@ spv::Id TGlslangToSpvTraverser::createMiscOperation(glslang::TOperator op, spv:: opCode = spv::OpFetchMicroTriangleVertexPositionNV; break; + case glslang::EOpImageBlockMatchWindowSSDQCOM: + typeId = builder.makeVectorType(builder.makeFloatType(32), 4); + opCode = spv::OpImageBlockMatchWindowSSDQCOM; + addImageProcessing2QCOMDecoration(operands[0], false); + addImageProcessing2QCOMDecoration(operands[2], false); + break; + case glslang::EOpImageBlockMatchWindowSADQCOM: + typeId = builder.makeVectorType(builder.makeFloatType(32), 4); + opCode = spv::OpImageBlockMatchWindowSADQCOM; + addImageProcessing2QCOMDecoration(operands[0], false); + addImageProcessing2QCOMDecoration(operands[2], false); + break; + case glslang::EOpImageBlockMatchGatherSSDQCOM: + typeId = builder.makeVectorType(builder.makeFloatType(32), 4); + opCode = spv::OpImageBlockMatchGatherSSDQCOM; + addImageProcessing2QCOMDecoration(operands[0], true); + addImageProcessing2QCOMDecoration(operands[2], true); + break; + case glslang::EOpImageBlockMatchGatherSADQCOM: + typeId = builder.makeVectorType(builder.makeFloatType(32), 4); + opCode = spv::OpImageBlockMatchGatherSADQCOM; + addImageProcessing2QCOMDecoration(operands[0], true); + addImageProcessing2QCOMDecoration(operands[2], true); + break; default: return 0; } @@ -9685,6 +9838,16 @@ void TGlslangToSpvTraverser::addMeshNVDecoration(spv::Id id, int member, const g } } +bool TGlslangToSpvTraverser::hasQCOMImageProceessingDecoration(spv::Id id, spv::Decoration decor) +{ + std::vector<spv::Decoration> &decoVec = idToQCOMDecorations[id]; + for ( auto d : decoVec ) { + if ( d == decor ) + return true; + } + return false; +} + void TGlslangToSpvTraverser::addImageProcessingQCOMDecoration(spv::Id id, spv::Decoration decor) { spv::Op opc = builder.getOpCode(id); @@ -9695,7 +9858,43 @@ void TGlslangToSpvTraverser::addImageProcessingQCOMDecoration(spv::Id id, spv::D if (opc == spv::OpLoad) { spv::Id texid = builder.getIdOperand(id, 0); - builder.addDecoration(texid, decor); + if (!hasQCOMImageProceessingDecoration(texid, decor)) {// + builder.addDecoration(texid, decor); + idToQCOMDecorations[texid].push_back(decor); + } + } +} + +void TGlslangToSpvTraverser::addImageProcessing2QCOMDecoration(spv::Id id, bool isForGather) +{ + if (isForGather) { + return addImageProcessingQCOMDecoration(id, spv::DecorationBlockMatchTextureQCOM); + } + + auto addDecor = + [this](spv::Id id, spv::Decoration decor) { + spv::Id tsopc = this->builder.getOpCode(id); + if (tsopc == spv::OpLoad) { + spv::Id tsid = this->builder.getIdOperand(id, 0); + if (this->glslangIntermediate->getSpv().spv >= glslang::EShTargetSpv_1_4) { + assert(iOSet.count(tsid) > 0); + } + if (!hasQCOMImageProceessingDecoration(tsid, decor)) { + this->builder.addDecoration(tsid, decor); + idToQCOMDecorations[tsid].push_back(decor); + } + } + }; + + spv::Id opc = builder.getOpCode(id); + bool isInterfaceObject = (opc != spv::OpSampledImage); + + if (!isInterfaceObject) { + addDecor(builder.getIdOperand(id, 0), spv::DecorationBlockMatchTextureQCOM); + addDecor(builder.getIdOperand(id, 1), spv::DecorationBlockMatchSamplerQCOM); + } else { + addDecor(id, spv::DecorationBlockMatchTextureQCOM); + addDecor(id, spv::DecorationBlockMatchSamplerQCOM); } } @@ -10025,6 +10224,7 @@ spv::Id TGlslangToSpvTraverser::createShortCircuit(glslang::TOperator op, glslan // Operands to accumulate OpPhi operands std::vector<spv::Id> phiOperands; + phiOperands.reserve(4); // accumulate left operand's phi information phiOperands.push_back(leftId); phiOperands.push_back(builder.getBuildPoint()->getId()); @@ -10064,7 +10264,6 @@ spv::Id TGlslangToSpvTraverser::getExtBuiltins(const char* name) if (extBuiltinMap.find(name) != extBuiltinMap.end()) return extBuiltinMap[name]; else { - builder.addExtension(name); spv::Id extBuiltins = builder.import(name); extBuiltinMap[name] = extBuiltins; return extBuiltins; |