CGHLSLBuiltins.cpp

//===------- CGHLSLBuiltins.cpp - Emit LLVM Code for HLSL builtins --------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code to emit HLSL Builtin calls as LLVM code.
//
//===----------------------------------------------------------------------===//

#include "CGBuiltin.h"
#include "CGHLSLRuntime.h"

using namespace clang;
using namespace CodeGen;
using namespace llvm;

static Value *handleAsDoubleBuiltin(CodeGenFunction &CGF, const CallExpr *E) {
  assert((E->getArg(0)->getType()->hasUnsignedIntegerRepresentation() &&
          E->getArg(1)->getType()->hasUnsignedIntegerRepresentation()) &&
         "asdouble operands types mismatch");
  Value *OpLowBits = CGF.EmitScalarExpr(E->getArg(0));
  Value *OpHighBits = CGF.EmitScalarExpr(E->getArg(1));

  llvm::Type *ResultType = CGF.DoubleTy;
  int N = 1;
  if (auto *VTy = E->getArg(0)->getType()->getAs<clang::VectorType>()) {
    N = VTy->getNumElements();
    ResultType = llvm::FixedVectorType::get(CGF.DoubleTy, N);
  }

  if (CGF.CGM.getTarget().getTriple().isDXIL())
    return CGF.Builder.CreateIntrinsic(
        /*ReturnType=*/ResultType, Intrinsic::dx_asdouble,
        {OpLowBits, OpHighBits}, nullptr, "hlsl.asdouble");

  if (!E->getArg(0)->getType()->isVectorType()) {
    OpLowBits = CGF.Builder.CreateVectorSplat(1, OpLowBits);
    OpHighBits = CGF.Builder.CreateVectorSplat(1, OpHighBits);
  }

  llvm::SmallVector<int> Mask;
  for (int i = 0; i < N; i++) {
    Mask.push_back(i);
    Mask.push_back(i + N);
  }

  Value *BitVec = CGF.Builder.CreateShuffleVector(OpLowBits, OpHighBits, Mask);

  return CGF.Builder.CreateBitCast(BitVec, ResultType);
}

static Value *handleHlslClip(const CallExpr *E, CodeGenFunction *CGF) {
  Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));

  Constant *FZeroConst = ConstantFP::getZero(CGF->FloatTy);
  Value *CMP;
  Value *LastInstr;

  if (const auto *VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>()) {
    FZeroConst = ConstantVector::getSplat(
        ElementCount::getFixed(VecTy->getNumElements()), FZeroConst);
    auto *FCompInst = CGF->Builder.CreateFCmpOLT(Op0, FZeroConst);
    CMP = CGF->Builder.CreateIntrinsic(
        CGF->Builder.getInt1Ty(), CGF->CGM.getHLSLRuntime().getAnyIntrinsic(),
        {FCompInst});
  } else
    CMP = CGF->Builder.CreateFCmpOLT(Op0, FZeroConst);

  if (CGF->CGM.getTarget().getTriple().isDXIL())
    LastInstr =
        CGF->Builder.CreateIntrinsic(CGF->VoidTy, Intrinsic::dx_discard, {CMP});
  else if (CGF->CGM.getTarget().getTriple().isSPIRV()) {
    BasicBlock *LT0 = CGF->createBasicBlock("lt0", CGF->CurFn);
    BasicBlock *End = CGF->createBasicBlock("end", CGF->CurFn);

    CGF->Builder.CreateCondBr(CMP, LT0, End);

    CGF->Builder.SetInsertPoint(LT0);

    CGF->Builder.CreateIntrinsic(CGF->VoidTy, Intrinsic::spv_discard, {});

    LastInstr = CGF->Builder.CreateBr(End);
    CGF->Builder.SetInsertPoint(End);
  } else {
    llvm_unreachable("Backend Codegen not supported.");
  }

  return LastInstr;
}

static Value *handleHlslSplitdouble(const CallExpr *E, CodeGenFunction *CGF) {
  Value *Op0 = CGF->EmitScalarExpr(E->getArg(0));
  const auto *OutArg1 = dyn_cast<HLSLOutArgExpr>(E->getArg(1));
  const auto *OutArg2 = dyn_cast<HLSLOutArgExpr>(E->getArg(2));

  CallArgList Args;
  LValue Op1TmpLValue =
      CGF->EmitHLSLOutArgExpr(OutArg1, Args, OutArg1->getType());
  LValue Op2TmpLValue =
      CGF->EmitHLSLOutArgExpr(OutArg2, Args, OutArg2->getType());

  if (CGF->getTarget().getCXXABI().areArgsDestroyedLeftToRightInCallee())
    Args.reverseWritebacks();

  Value *LowBits = nullptr;
  Value *HighBits = nullptr;

  if (CGF->CGM.getTarget().getTriple().isDXIL()) {

    llvm::Type *RetElementTy = CGF->Int32Ty;
    if (auto *Op0VecTy = E->getArg(0)->getType()->getAs<clang::VectorType>())
      RetElementTy = llvm::VectorType::get(
          CGF->Int32Ty, ElementCount::getFixed(Op0VecTy->getNumElements()));
    auto *RetTy = llvm::StructType::get(RetElementTy, RetElementTy);

    CallInst *CI = CGF->Builder.CreateIntrinsic(
        RetTy, Intrinsic::dx_splitdouble, {Op0}, nullptr, "hlsl.splitdouble");

    LowBits = CGF->Builder.CreateExtractValue(CI, 0);
    HighBits = CGF->Builder.CreateExtractValue(CI, 1);

  } else {
    // For Non DXIL targets we generate the instructions.

    if (!Op0->getType()->isVectorTy()) {
      FixedVectorType *DestTy = FixedVectorType::get(CGF->Int32Ty, 2);
      Value *Bitcast = CGF->Builder.CreateBitCast(Op0, DestTy);

      LowBits = CGF->Builder.CreateExtractElement(Bitcast, (uint64_t)0);
      HighBits = CGF->Builder.CreateExtractElement(Bitcast, 1);
    } else {
      int NumElements = 1;
      if (const auto *VecTy =
              E->getArg(0)->getType()->getAs<clang::VectorType>())
        NumElements = VecTy->getNumElements();

      FixedVectorType *Uint32VecTy =
          FixedVectorType::get(CGF->Int32Ty, NumElements * 2);
      Value *Uint32Vec = CGF->Builder.CreateBitCast(Op0, Uint32VecTy);
      if (NumElements == 1) {
        LowBits = CGF->Builder.CreateExtractElement(Uint32Vec, (uint64_t)0);
        HighBits = CGF->Builder.CreateExtractElement(Uint32Vec, 1);
      } else {
        SmallVector<int> EvenMask, OddMask;
        for (int I = 0, E = NumElements; I != E; ++I) {
          EvenMask.push_back(I * 2);
          OddMask.push_back(I * 2 + 1);
        }
        LowBits = CGF->Builder.CreateShuffleVector(Uint32Vec, EvenMask);
        HighBits = CGF->Builder.CreateShuffleVector(Uint32Vec, OddMask);
      }
    }
  }
  CGF->Builder.CreateStore(LowBits, Op1TmpLValue.getAddress());
  auto *LastInst =
      CGF->Builder.CreateStore(HighBits, Op2TmpLValue.getAddress());
  CGF->EmitWritebacks(Args);
  return LastInst;
}

// Return dot product intrinsic that corresponds to the QT scalar type
static Intrinsic::ID getDotProductIntrinsic(CGHLSLRuntime &RT, QualType QT) {
  if (QT->isFloatingType())
    return RT.getFDotIntrinsic();
  if (QT->isSignedIntegerType())
    return RT.getSDotIntrinsic();
  assert(QT->isUnsignedIntegerType());
  return RT.getUDotIntrinsic();
}

static Intrinsic::ID getFirstBitHighIntrinsic(CGHLSLRuntime &RT, QualType QT) {
  if (QT->hasSignedIntegerRepresentation()) {
    return RT.getFirstBitSHighIntrinsic();
  }

  assert(QT->hasUnsignedIntegerRepresentation());
  return RT.getFirstBitUHighIntrinsic();
}

// Return wave active sum that corresponds to the QT scalar type
static Intrinsic::ID getWaveActiveSumIntrinsic(llvm::Triple::ArchType Arch,
                                               CGHLSLRuntime &RT, QualType QT) {
  switch (Arch) {
  case llvm::Triple::spirv:
    return Intrinsic::spv_wave_reduce_sum;
  case llvm::Triple::dxil: {
    if (QT->isUnsignedIntegerType())
      return Intrinsic::dx_wave_reduce_usum;
    return Intrinsic::dx_wave_reduce_sum;
  }
  default:
    llvm_unreachable("Intrinsic WaveActiveSum"
                     " not supported by target architecture");
  }
}

// Return wave active sum that corresponds to the QT scalar type
static Intrinsic::ID getWaveActiveMaxIntrinsic(llvm::Triple::ArchType Arch,
                                               CGHLSLRuntime &RT, QualType QT) {
  switch (Arch) {
  case llvm::Triple::spirv:
    if (QT->isUnsignedIntegerType())
      return Intrinsic::spv_wave_reduce_umax;
    return Intrinsic::spv_wave_reduce_max;
  case llvm::Triple::dxil: {
    if (QT->isUnsignedIntegerType())
      return Intrinsic::dx_wave_reduce_umax;
    return Intrinsic::dx_wave_reduce_max;
  }
  default:
    llvm_unreachable("Intrinsic WaveActiveMax"
                     " not supported by target architecture");
  }
}

Value *CodeGenFunction::EmitHLSLBuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E,
                                            ReturnValueSlot ReturnValue) {
  if (!getLangOpts().HLSL)
    return nullptr;

  switch (BuiltinID) {
  case Builtin::BI__builtin_hlsl_adduint64: {
    Value *OpA = EmitScalarExpr(E->getArg(0));
    Value *OpB = EmitScalarExpr(E->getArg(1));
    QualType Arg0Ty = E->getArg(0)->getType();
    uint64_t NumElements = Arg0Ty->castAs<VectorType>()->getNumElements();
    assert(Arg0Ty == E->getArg(1)->getType() &&
           "AddUint64 operand types must match");
    assert(Arg0Ty->hasIntegerRepresentation() &&
           "AddUint64 operands must have an integer representation");
    assert((NumElements == 2 || NumElements == 4) &&
           "AddUint64 operands must have 2 or 4 elements");

    llvm::Value *LowA;
    llvm::Value *HighA;
    llvm::Value *LowB;
    llvm::Value *HighB;

    // Obtain low and high words of inputs A and B
    if (NumElements == 2) {
      LowA = Builder.CreateExtractElement(OpA, (uint64_t)0, "LowA");
      HighA = Builder.CreateExtractElement(OpA, (uint64_t)1, "HighA");
      LowB = Builder.CreateExtractElement(OpB, (uint64_t)0, "LowB");
      HighB = Builder.CreateExtractElement(OpB, (uint64_t)1, "HighB");
    } else {
      LowA = Builder.CreateShuffleVector(OpA, {0, 2}, "LowA");
      HighA = Builder.CreateShuffleVector(OpA, {1, 3}, "HighA");
      LowB = Builder.CreateShuffleVector(OpB, {0, 2}, "LowB");
      HighB = Builder.CreateShuffleVector(OpB, {1, 3}, "HighB");
    }

    // Use an uadd_with_overflow to compute the sum of low words and obtain a
    // carry value
    llvm::Value *Carry;
    llvm::Value *LowSum = EmitOverflowIntrinsic(
        *this, Intrinsic::uadd_with_overflow, LowA, LowB, Carry);
    llvm::Value *ZExtCarry =
        Builder.CreateZExt(Carry, HighA->getType(), "CarryZExt");

    // Sum the high words and the carry
    llvm::Value *HighSum = Builder.CreateAdd(HighA, HighB, "HighSum");
    llvm::Value *HighSumPlusCarry =
        Builder.CreateAdd(HighSum, ZExtCarry, "HighSumPlusCarry");

    if (NumElements == 4) {
      return Builder.CreateShuffleVector(LowSum, HighSumPlusCarry, {0, 2, 1, 3},
                                         "hlsl.AddUint64");
    }

    llvm::Value *Result = PoisonValue::get(OpA->getType());
    Result = Builder.CreateInsertElement(Result, LowSum, (uint64_t)0,
                                         "hlsl.AddUint64.upto0");
    Result = Builder.CreateInsertElement(Result, HighSumPlusCarry, (uint64_t)1,
                                         "hlsl.AddUint64");
    return Result;
  }
  case Builtin::BI__builtin_hlsl_resource_getpointer: {
    Value *HandleOp = EmitScalarExpr(E->getArg(0));
    Value *IndexOp = EmitScalarExpr(E->getArg(1));

    // TODO: Map to an hlsl_device address space.
    llvm::Type *RetTy = llvm::PointerType::getUnqual(getLLVMContext());

    return Builder.CreateIntrinsic(
        RetTy, CGM.getHLSLRuntime().getCreateResourceGetPointerIntrinsic(),
        ArrayRef<Value *>{HandleOp, IndexOp});
  }
  case Builtin::BI__builtin_hlsl_all: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    return Builder.CreateIntrinsic(
        /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()),
        CGM.getHLSLRuntime().getAllIntrinsic(), ArrayRef<Value *>{Op0}, nullptr,
        "hlsl.all");
  }
  case Builtin::BI__builtin_hlsl_and: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    Value *Op1 = EmitScalarExpr(E->getArg(1));
    return Builder.CreateAnd(Op0, Op1, "hlsl.and");
  }
  case Builtin::BI__builtin_hlsl_or: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    Value *Op1 = EmitScalarExpr(E->getArg(1));
    return Builder.CreateOr(Op0, Op1, "hlsl.or");
  }
  case Builtin::BI__builtin_hlsl_any: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    return Builder.CreateIntrinsic(
        /*ReturnType=*/llvm::Type::getInt1Ty(getLLVMContext()),
        CGM.getHLSLRuntime().getAnyIntrinsic(), ArrayRef<Value *>{Op0}, nullptr,
        "hlsl.any");
  }
  case Builtin::BI__builtin_hlsl_asdouble:
    return handleAsDoubleBuiltin(*this, E);
  case Builtin::BI__builtin_hlsl_elementwise_clamp: {
    Value *OpX = EmitScalarExpr(E->getArg(0));
    Value *OpMin = EmitScalarExpr(E->getArg(1));
    Value *OpMax = EmitScalarExpr(E->getArg(2));

    QualType Ty = E->getArg(0)->getType();
    if (auto *VecTy = Ty->getAs<VectorType>())
      Ty = VecTy->getElementType();

    Intrinsic::ID Intr;
    if (Ty->isFloatingType()) {
      Intr = CGM.getHLSLRuntime().getNClampIntrinsic();
    } else if (Ty->isUnsignedIntegerType()) {
      Intr = CGM.getHLSLRuntime().getUClampIntrinsic();
    } else {
      assert(Ty->isSignedIntegerType());
      Intr = CGM.getHLSLRuntime().getSClampIntrinsic();
    }
    return Builder.CreateIntrinsic(
        /*ReturnType=*/OpX->getType(), Intr,
        ArrayRef<Value *>{OpX, OpMin, OpMax}, nullptr, "hlsl.clamp");
  }
  case Builtin::BI__builtin_hlsl_cross: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    Value *Op1 = EmitScalarExpr(E->getArg(1));
    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           E->getArg(1)->getType()->hasFloatingRepresentation() &&
           "cross operands must have a float representation");
    // make sure each vector has exactly 3 elements
    assert(
        E->getArg(0)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
        E->getArg(1)->getType()->castAs<VectorType>()->getNumElements() == 3 &&
        "input vectors must have 3 elements each");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getCrossIntrinsic(),
        ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.cross");
  }
  case Builtin::BI__builtin_hlsl_dot: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    Value *Op1 = EmitScalarExpr(E->getArg(1));
    llvm::Type *T0 = Op0->getType();
    llvm::Type *T1 = Op1->getType();

    // If the arguments are scalars, just emit a multiply
    if (!T0->isVectorTy() && !T1->isVectorTy()) {
      if (T0->isFloatingPointTy())
        return Builder.CreateFMul(Op0, Op1, "hlsl.dot");

      if (T0->isIntegerTy())
        return Builder.CreateMul(Op0, Op1, "hlsl.dot");

      llvm_unreachable(
          "Scalar dot product is only supported on ints and floats.");
    }
    // For vectors, validate types and emit the appropriate intrinsic

    // A VectorSplat should have happened
    assert(T0->isVectorTy() && T1->isVectorTy() &&
           "Dot product of vector and scalar is not supported.");

    auto *VecTy0 = E->getArg(0)->getType()->castAs<VectorType>();
    [[maybe_unused]] auto *VecTy1 =
        E->getArg(1)->getType()->castAs<VectorType>();

    assert(VecTy0->getElementType() == VecTy1->getElementType() &&
           "Dot product of vectors need the same element types.");

    assert(VecTy0->getNumElements() == VecTy1->getNumElements() &&
           "Dot product requires vectors to be of the same size.");

    return Builder.CreateIntrinsic(
        /*ReturnType=*/T0->getScalarType(),
        getDotProductIntrinsic(CGM.getHLSLRuntime(), VecTy0->getElementType()),
        ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.dot");
  }
  case Builtin::BI__builtin_hlsl_dot4add_i8packed: {
    Value *A = EmitScalarExpr(E->getArg(0));
    Value *B = EmitScalarExpr(E->getArg(1));
    Value *C = EmitScalarExpr(E->getArg(2));

    Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddI8PackedIntrinsic();
    return Builder.CreateIntrinsic(
        /*ReturnType=*/C->getType(), ID, ArrayRef<Value *>{A, B, C}, nullptr,
        "hlsl.dot4add.i8packed");
  }
  case Builtin::BI__builtin_hlsl_dot4add_u8packed: {
    Value *A = EmitScalarExpr(E->getArg(0));
    Value *B = EmitScalarExpr(E->getArg(1));
    Value *C = EmitScalarExpr(E->getArg(2));

    Intrinsic::ID ID = CGM.getHLSLRuntime().getDot4AddU8PackedIntrinsic();
    return Builder.CreateIntrinsic(
        /*ReturnType=*/C->getType(), ID, ArrayRef<Value *>{A, B, C}, nullptr,
        "hlsl.dot4add.u8packed");
  }
  case Builtin::BI__builtin_hlsl_elementwise_firstbithigh: {
    Value *X = EmitScalarExpr(E->getArg(0));

    return Builder.CreateIntrinsic(
        /*ReturnType=*/ConvertType(E->getType()),
        getFirstBitHighIntrinsic(CGM.getHLSLRuntime(), E->getArg(0)->getType()),
        ArrayRef<Value *>{X}, nullptr, "hlsl.firstbithigh");
  }
  case Builtin::BI__builtin_hlsl_elementwise_firstbitlow: {
    Value *X = EmitScalarExpr(E->getArg(0));

    return Builder.CreateIntrinsic(
        /*ReturnType=*/ConvertType(E->getType()),
        CGM.getHLSLRuntime().getFirstBitLowIntrinsic(), ArrayRef<Value *>{X},
        nullptr, "hlsl.firstbitlow");
  }
  case Builtin::BI__builtin_hlsl_lerp: {
    Value *X = EmitScalarExpr(E->getArg(0));
    Value *Y = EmitScalarExpr(E->getArg(1));
    Value *S = EmitScalarExpr(E->getArg(2));
    if (!E->getArg(0)->getType()->hasFloatingRepresentation())
      llvm_unreachable("lerp operand must have a float representation");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/X->getType(), CGM.getHLSLRuntime().getLerpIntrinsic(),
        ArrayRef<Value *>{X, Y, S}, nullptr, "hlsl.lerp");
  }
  case Builtin::BI__builtin_hlsl_normalize: {
    Value *X = EmitScalarExpr(E->getArg(0));

    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           "normalize operand must have a float representation");

    return Builder.CreateIntrinsic(
        /*ReturnType=*/X->getType(),
        CGM.getHLSLRuntime().getNormalizeIntrinsic(), ArrayRef<Value *>{X},
        nullptr, "hlsl.normalize");
  }
  case Builtin::BI__builtin_hlsl_elementwise_degrees: {
    Value *X = EmitScalarExpr(E->getArg(0));

    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           "degree operand must have a float representation");

    return Builder.CreateIntrinsic(
        /*ReturnType=*/X->getType(), CGM.getHLSLRuntime().getDegreesIntrinsic(),
        ArrayRef<Value *>{X}, nullptr, "hlsl.degrees");
  }
  case Builtin::BI__builtin_hlsl_elementwise_frac: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    if (!E->getArg(0)->getType()->hasFloatingRepresentation())
      llvm_unreachable("frac operand must have a float representation");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getFracIntrinsic(),
        ArrayRef<Value *>{Op0}, nullptr, "hlsl.frac");
  }
  case Builtin::BI__builtin_hlsl_elementwise_isinf: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    llvm::Type *Xty = Op0->getType();
    llvm::Type *retType = llvm::Type::getInt1Ty(this->getLLVMContext());
    if (Xty->isVectorTy()) {
      auto *XVecTy = E->getArg(0)->getType()->castAs<VectorType>();
      retType = llvm::VectorType::get(
          retType, ElementCount::getFixed(XVecTy->getNumElements()));
    }
    if (!E->getArg(0)->getType()->hasFloatingRepresentation())
      llvm_unreachable("isinf operand must have a float representation");
    return Builder.CreateIntrinsic(retType, Intrinsic::dx_isinf,
                                   ArrayRef<Value *>{Op0}, nullptr, "dx.isinf");
  }
  case Builtin::BI__builtin_hlsl_mad: {
    Value *M = EmitScalarExpr(E->getArg(0));
    Value *A = EmitScalarExpr(E->getArg(1));
    Value *B = EmitScalarExpr(E->getArg(2));
    if (E->getArg(0)->getType()->hasFloatingRepresentation())
      return Builder.CreateIntrinsic(
          /*ReturnType*/ M->getType(), Intrinsic::fmuladd,
          ArrayRef<Value *>{M, A, B}, nullptr, "hlsl.fmad");

    if (E->getArg(0)->getType()->hasSignedIntegerRepresentation()) {
      if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
        return Builder.CreateIntrinsic(
            /*ReturnType*/ M->getType(), Intrinsic::dx_imad,
            ArrayRef<Value *>{M, A, B}, nullptr, "dx.imad");

      Value *Mul = Builder.CreateNSWMul(M, A);
      return Builder.CreateNSWAdd(Mul, B);
    }
    assert(E->getArg(0)->getType()->hasUnsignedIntegerRepresentation());
    if (CGM.getTarget().getTriple().getArch() == llvm::Triple::dxil)
      return Builder.CreateIntrinsic(
          /*ReturnType=*/M->getType(), Intrinsic::dx_umad,
          ArrayRef<Value *>{M, A, B}, nullptr, "dx.umad");

    Value *Mul = Builder.CreateNUWMul(M, A);
    return Builder.CreateNUWAdd(Mul, B);
  }
  case Builtin::BI__builtin_hlsl_elementwise_rcp: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    if (!E->getArg(0)->getType()->hasFloatingRepresentation())
      llvm_unreachable("rcp operand must have a float representation");
    llvm::Type *Ty = Op0->getType();
    llvm::Type *EltTy = Ty->getScalarType();
    Constant *One = Ty->isVectorTy()
                        ? ConstantVector::getSplat(
                              ElementCount::getFixed(
                                  cast<FixedVectorType>(Ty)->getNumElements()),
                              ConstantFP::get(EltTy, 1.0))
                        : ConstantFP::get(EltTy, 1.0);
    return Builder.CreateFDiv(One, Op0, "hlsl.rcp");
  }
  case Builtin::BI__builtin_hlsl_elementwise_rsqrt: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    if (!E->getArg(0)->getType()->hasFloatingRepresentation())
      llvm_unreachable("rsqrt operand must have a float representation");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getRsqrtIntrinsic(),
        ArrayRef<Value *>{Op0}, nullptr, "hlsl.rsqrt");
  }
  case Builtin::BI__builtin_hlsl_elementwise_saturate: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           "saturate operand must have a float representation");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Op0->getType(),
        CGM.getHLSLRuntime().getSaturateIntrinsic(), ArrayRef<Value *>{Op0},
        nullptr, "hlsl.saturate");
  }
  case Builtin::BI__builtin_hlsl_select: {
    Value *OpCond = EmitScalarExpr(E->getArg(0));
    RValue RValTrue = EmitAnyExpr(E->getArg(1));
    Value *OpTrue =
        RValTrue.isScalar()
            ? RValTrue.getScalarVal()
            : RValTrue.getAggregatePointer(E->getArg(1)->getType(), *this);
    RValue RValFalse = EmitAnyExpr(E->getArg(2));
    Value *OpFalse =
        RValFalse.isScalar()
            ? RValFalse.getScalarVal()
            : RValFalse.getAggregatePointer(E->getArg(2)->getType(), *this);
    if (auto *VTy = E->getType()->getAs<VectorType>()) {
      if (!OpTrue->getType()->isVectorTy())
        OpTrue =
            Builder.CreateVectorSplat(VTy->getNumElements(), OpTrue, "splat");
      if (!OpFalse->getType()->isVectorTy())
        OpFalse =
            Builder.CreateVectorSplat(VTy->getNumElements(), OpFalse, "splat");
    }

    Value *SelectVal =
        Builder.CreateSelect(OpCond, OpTrue, OpFalse, "hlsl.select");
    if (!RValTrue.isScalar())
      Builder.CreateStore(SelectVal, ReturnValue.getAddress(),
                          ReturnValue.isVolatile());

    return SelectVal;
  }
  case Builtin::BI__builtin_hlsl_step: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    Value *Op1 = EmitScalarExpr(E->getArg(1));
    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           E->getArg(1)->getType()->hasFloatingRepresentation() &&
           "step operands must have a float representation");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Op0->getType(), CGM.getHLSLRuntime().getStepIntrinsic(),
        ArrayRef<Value *>{Op0, Op1}, nullptr, "hlsl.step");
  }
  case Builtin::BI__builtin_hlsl_wave_active_all_true: {
    Value *Op = EmitScalarExpr(E->getArg(0));
    assert(Op->getType()->isIntegerTy(1) &&
           "Intrinsic WaveActiveAllTrue operand must be a bool");

    Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAllTrueIntrinsic();
    return EmitRuntimeCall(
        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});
  }
  case Builtin::BI__builtin_hlsl_wave_active_any_true: {
    Value *Op = EmitScalarExpr(E->getArg(0));
    assert(Op->getType()->isIntegerTy(1) &&
           "Intrinsic WaveActiveAnyTrue operand must be a bool");

    Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveAnyTrueIntrinsic();
    return EmitRuntimeCall(
        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID), {Op});
  }
  case Builtin::BI__builtin_hlsl_wave_active_count_bits: {
    Value *OpExpr = EmitScalarExpr(E->getArg(0));
    Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveActiveCountBitsIntrinsic();
    return EmitRuntimeCall(
        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID),
        ArrayRef{OpExpr});
  }
  case Builtin::BI__builtin_hlsl_wave_active_sum: {
    // Due to the use of variadic arguments, explicitly retreive argument
    Value *OpExpr = EmitScalarExpr(E->getArg(0));
    llvm::FunctionType *FT = llvm::FunctionType::get(
        OpExpr->getType(), ArrayRef{OpExpr->getType()}, false);
    Intrinsic::ID IID = getWaveActiveSumIntrinsic(
        getTarget().getTriple().getArch(), CGM.getHLSLRuntime(),
        E->getArg(0)->getType());

    // Get overloaded name
    std::string Name =
        Intrinsic::getName(IID, ArrayRef{OpExpr->getType()}, &CGM.getModule());
    return EmitRuntimeCall(CGM.CreateRuntimeFunction(FT, Name, {},
                                                     /*Local=*/false,
                                                     /*AssumeConvergent=*/true),
                           ArrayRef{OpExpr}, "hlsl.wave.active.sum");
  }
  case Builtin::BI__builtin_hlsl_wave_active_max: {
    // Due to the use of variadic arguments, explicitly retreive argument
    Value *OpExpr = EmitScalarExpr(E->getArg(0));
    llvm::FunctionType *FT = llvm::FunctionType::get(
        OpExpr->getType(), ArrayRef{OpExpr->getType()}, false);
    Intrinsic::ID IID = getWaveActiveMaxIntrinsic(
        getTarget().getTriple().getArch(), CGM.getHLSLRuntime(),
        E->getArg(0)->getType());

    // Get overloaded name
    std::string Name =
        Intrinsic::getName(IID, ArrayRef{OpExpr->getType()}, &CGM.getModule());
    return EmitRuntimeCall(CGM.CreateRuntimeFunction(FT, Name, {},
                                                     /*Local=*/false,
                                                     /*AssumeConvergent=*/true),
                           ArrayRef{OpExpr}, "hlsl.wave.active.max");
  }
  case Builtin::BI__builtin_hlsl_wave_get_lane_index: {
    // We don't define a SPIR-V intrinsic, instead it is a SPIR-V built-in
    // defined in SPIRVBuiltins.td. So instead we manually get the matching name
    // for the DirectX intrinsic and the demangled builtin name
    switch (CGM.getTarget().getTriple().getArch()) {
    case llvm::Triple::dxil:
      return EmitRuntimeCall(Intrinsic::getOrInsertDeclaration(
          &CGM.getModule(), Intrinsic::dx_wave_getlaneindex));
    case llvm::Triple::spirv:
      return EmitRuntimeCall(CGM.CreateRuntimeFunction(
          llvm::FunctionType::get(IntTy, {}, false),
          "__hlsl_wave_get_lane_index", {}, false, true));
    default:
      llvm_unreachable(
          "Intrinsic WaveGetLaneIndex not supported by target architecture");
    }
  }
  case Builtin::BI__builtin_hlsl_wave_is_first_lane: {
    Intrinsic::ID ID = CGM.getHLSLRuntime().getWaveIsFirstLaneIntrinsic();
    return EmitRuntimeCall(
        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
  }
  case Builtin::BI__builtin_hlsl_wave_read_lane_at: {
    // Due to the use of variadic arguments we must explicitly retreive them and
    // create our function type.
    Value *OpExpr = EmitScalarExpr(E->getArg(0));
    Value *OpIndex = EmitScalarExpr(E->getArg(1));
    llvm::FunctionType *FT = llvm::FunctionType::get(
        OpExpr->getType(), ArrayRef{OpExpr->getType(), OpIndex->getType()},
        false);

    // Get overloaded name
    std::string Name =
        Intrinsic::getName(CGM.getHLSLRuntime().getWaveReadLaneAtIntrinsic(),
                           ArrayRef{OpExpr->getType()}, &CGM.getModule());
    return EmitRuntimeCall(CGM.CreateRuntimeFunction(FT, Name, {},
                                                     /*Local=*/false,
                                                     /*AssumeConvergent=*/true),
                           ArrayRef{OpExpr, OpIndex}, "hlsl.wave.readlane");
  }
  case Builtin::BI__builtin_hlsl_elementwise_sign: {
    auto *Arg0 = E->getArg(0);
    Value *Op0 = EmitScalarExpr(Arg0);
    llvm::Type *Xty = Op0->getType();
    llvm::Type *retType = llvm::Type::getInt32Ty(this->getLLVMContext());
    if (Xty->isVectorTy()) {
      auto *XVecTy = Arg0->getType()->castAs<VectorType>();
      retType = llvm::VectorType::get(
          retType, ElementCount::getFixed(XVecTy->getNumElements()));
    }
    assert((Arg0->getType()->hasFloatingRepresentation() ||
            Arg0->getType()->hasIntegerRepresentation()) &&
           "sign operand must have a float or int representation");

    if (Arg0->getType()->hasUnsignedIntegerRepresentation()) {
      Value *Cmp = Builder.CreateICmpEQ(Op0, ConstantInt::get(Xty, 0));
      return Builder.CreateSelect(Cmp, ConstantInt::get(retType, 0),
                                  ConstantInt::get(retType, 1), "hlsl.sign");
    }

    return Builder.CreateIntrinsic(
        retType, CGM.getHLSLRuntime().getSignIntrinsic(),
        ArrayRef<Value *>{Op0}, nullptr, "hlsl.sign");
  }
  case Builtin::BI__builtin_hlsl_elementwise_radians: {
    Value *Op0 = EmitScalarExpr(E->getArg(0));
    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           "radians operand must have a float representation");
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Op0->getType(),
        CGM.getHLSLRuntime().getRadiansIntrinsic(), ArrayRef<Value *>{Op0},
        nullptr, "hlsl.radians");
  }
  case Builtin::BI__builtin_hlsl_buffer_update_counter: {
    Value *ResHandle = EmitScalarExpr(E->getArg(0));
    Value *Offset = EmitScalarExpr(E->getArg(1));
    Value *OffsetI8 = Builder.CreateIntCast(Offset, Int8Ty, true);
    return Builder.CreateIntrinsic(
        /*ReturnType=*/Offset->getType(),
        CGM.getHLSLRuntime().getBufferUpdateCounterIntrinsic(),
        ArrayRef<Value *>{ResHandle, OffsetI8}, nullptr);
  }
  case Builtin::BI__builtin_hlsl_elementwise_splitdouble: {

    assert((E->getArg(0)->getType()->hasFloatingRepresentation() &&
            E->getArg(1)->getType()->hasUnsignedIntegerRepresentation() &&
            E->getArg(2)->getType()->hasUnsignedIntegerRepresentation()) &&
           "asuint operands types mismatch");
    return handleHlslSplitdouble(E, this);
  }
  case Builtin::BI__builtin_hlsl_elementwise_clip:
    assert(E->getArg(0)->getType()->hasFloatingRepresentation() &&
           "clip operands types mismatch");
    return handleHlslClip(E, this);
  case Builtin::BI__builtin_hlsl_group_memory_barrier_with_group_sync: {
    Intrinsic::ID ID =
        CGM.getHLSLRuntime().getGroupMemoryBarrierWithGroupSyncIntrinsic();
    return EmitRuntimeCall(
        Intrinsic::getOrInsertDeclaration(&CGM.getModule(), ID));
  }
  }
  return nullptr;
}