New pass to inline single call site functions

* Adds a pass that inlines functions with a single call site
 * Added -no-inline-single option to disable the pass
 * Due to some unexpected interactions, found some crashes in customer
   kernels due to another pass. For now, this pass is only enabled if the
   function has pointer to local arguments
* Pass is disabled in many tests to avoid substantial updates
* Added new lit tests for the pass
* Delete dead functions at the end of the pass

include/clspv/Option.h

@@ -70,5 +70,8 @@ bool PodArgsInUniformBuffer();
 // code generation.
 bool ShowIDs();
 
+// Returns true if functions with single call sites should be inlined.
+bool InlineSingleCallSite();
+
 } // namespace Option
 } // namespace clspv

include/clspv/Passes.h

@@ -75,6 +75,12 @@ llvm::ModulePass *createInlineFuncWithPointerBitCastArgPass();
 /// the compiler will not normally optimize away.
 llvm::ModulePass *createInlineFuncWithPointerToFunctionArgPass();
 
+/// Inline call instructions to functions having a single call site.
+/// @return An LLVM module pass.
+///
+/// Inline functions that have a single call site.
+llvm::ModulePass *createInlineFuncWithSingleCallSitePass();
+
 /// Zero-initialize allocas.
 /// @return An LLVM module pass.
 ///
@@ -325,10 +331,10 @@ llvm::ModulePass *createSignedCompareFixupPass();
 /// Share Module Scope Variables
 /// @return An LLVM module pass.
 ///
-/// Attempts to de-duplicate module scope Workgroup scope variables between kernels.
-/// Module scope variables of the same type can be merged if they are used by an
-/// exclusive set of kernels. This pass should run before direct resource access
-/// to enable more opportunities for that pass.
+/// Attempts to de-duplicate module scope Workgroup scope variables between
+/// kernels. Module scope variables of the same type can be merged if they are
+/// used by an exclusive set of kernels. This pass should run before direct
+/// resource access to enable more opportunities for that pass.
 llvm::ModulePass *createShareModuleScopeVariablesPass();
 
 } // namespace clspv

lib/CMakeLists.txt

@@ -25,6 +25,7 @@ add_library(clspv_core STATIC
   ${CMAKE_CURRENT_SOURCE_DIR}/HideConstantLoadsPass.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/InlineFuncWithPointerBitCastArgPass.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/InlineFuncWithPointerToFunctionArgPass.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/InlineFuncWithSingleCallSitePass.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/OpenCLInlinerPass.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/Option.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/SPIRVProducerPass.cpp

lib/InlineFuncWithSingleCallSitePass.cpp

@@ -0,0 +1,107 @@
+// Copyright 2018 The Clspv Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "llvm/IR/CallingConv.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+
+#include "ArgKind.h"
+#include "clspv/Option.h"
+
+using namespace llvm;
+
+namespace {
+class InlineFuncWithSingleCallSitePass : public ModulePass {
+public:
+  static char ID;
+  InlineFuncWithSingleCallSitePass() : ModulePass(ID) {}
+
+  bool runOnModule(Module &M) override;
+
+private:
+  bool InlineFunctions(Module &M);
+};
+} // namespace
+
+namespace clspv {
+ModulePass *createInlineFuncWithSingleCallSitePass() {
+  return new InlineFuncWithSingleCallSitePass();
+}
+} // namespace clspv
+
+char InlineFuncWithSingleCallSitePass::ID = 0;
+static RegisterPass<InlineFuncWithSingleCallSitePass>
+    X("InlineFuncWithSingleCallSite",
+      "Inline functions with a single call site pass");
+
+bool InlineFuncWithSingleCallSitePass::runOnModule(Module &M) {
+  if (!clspv::Option::InlineSingleCallSite())
+    return false;
+
+  bool Changed = false;
+  for (bool local_changed = true; local_changed; Changed |= local_changed) {
+    local_changed = InlineFunctions(M);
+  }
+
+  // Clean up dead functions. This done here to avoid ordering requirements on
+  // inlining.
+  std::vector<Function *> to_delete;
+  for (auto &F : M) {
+    if (F.isDeclaration() || F.getCallingConv() == CallingConv::SPIR_KERNEL)
+      continue;
+
+    if (F.user_empty())
+      to_delete.push_back(&F);
+  }
+  for (auto func : to_delete) {
+    func->eraseFromParent();
+  }
+
+  return Changed;
+}
+
+bool InlineFuncWithSingleCallSitePass::InlineFunctions(Module &M) {
+  bool Changed = false;
+  std::vector<CallInst *> to_inline;
+  for (auto &F : M) {
+    if (F.isDeclaration() || F.getCallingConv() == CallingConv::SPIR_KERNEL)
+      continue;
+
+    bool has_local_ptr_arg = false;
+    for (auto &Arg : F.args()) {
+      if (clspv::IsLocalPtr(Arg.getType()))
+        has_local_ptr_arg = true;
+    }
+
+    // Only inline if the function has a local address space parameter.
+    if (!has_local_ptr_arg) continue;
+
+    if (F.getNumUses() == 1) {
+      if (auto *call = dyn_cast<CallInst>(*F.user_begin()))
+        to_inline.push_back(call);
+    }
+  }
+
+  for (auto call : to_inline) {
+    InlineFunctionInfo IFI;
+    CallSite CS(call);
+    // Disable generation of lifetime intrinsic.
+    Changed |= InlineFunction(CS, IFI, nullptr, false);
+  }
+
+  return Changed;
+}

lib/Option.cpp

@@ -18,6 +18,10 @@
 
 namespace {
 
+llvm::cl::opt<bool> no_inline_single_call_site(
+    "no-inline-single", llvm::cl::init(false),
+    llvm::cl::desc("Disable inlining functions with single call sites."));
+
 // Should the compiler try to use direct resource accesses within helper
 // functions instead of passing pointers via function arguments?
 llvm::cl::opt<bool> no_direct_resource_access(
@@ -106,6 +110,7 @@ llvm::cl::opt<bool> show_ids("show-ids", llvm::cl::init(false),
 namespace clspv {
 namespace Option {
 
+bool InlineSingleCallSite() { return !no_inline_single_call_site; }
 bool DirectResourceAccess() {
   return !(no_direct_resource_access || distinct_kernel_descriptor_sets);
 }

test/DirectResourceAccess/constant_arg.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/global_arg.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/global_subobject_base.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/local_arg.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/local_arg_one_entry_point.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/local_variable.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/ro_image2_sampler_args.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/ro_image3_sampler_args.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/wo_image2_args.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/DirectResourceAccess/wo_image3_args.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/ImageBuiltins/read_only_image2d_passed_to_other_function.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/ImageBuiltins/read_only_image3d_passed_to_other_function.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/ImageBuiltins/write_only_image2d_passed_to_other_function.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/ImageBuiltins/write_only_image3d_passed_to_other_function.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/InlineFuncWithSingleCallSite/function_chain.cl

@@ -0,0 +1,19 @@
+// RUN: clspv %s -S -o %t.spvasm
+// RUN: FileCheck %s < %t.spvasm
+// RUN: clspv %s -o %t.spv
+// RUN: spirv-dis -o %t2.spvasm %t.spv
+// RUN: FileCheck %s < %t2.spvasm
+// RUN: spirv-val --target-env vulkan1.0 %t.spv
+
+// All functions should be inlined into the kernel.
+// CHECK: OpEntryPoint GLCompute [[entry:%[0-9a-zA-Z_]+]] "func_0"
+// CHECK-NOT: OpFunctionCall
+// CHECK: [[entry]] = OpFunction
+// CHECK-NOT: OpFunctionCall
+
+int func_3(local int *in, int n) { return in[n]; }
+int func_2(local int *in, int n) { return func_3(in, n); }
+int func_1(local int *in, int n) { return func_2(in, n); }
+kernel void func_0(local int *in, global int *out, int n) {
+  out[n] = func_1(in, n);
+}

test/InlineFuncWithSingleCallSite/non_local_parameter.cl

@@ -0,0 +1,15 @@
+// RUN: clspv %s -S -o %t.spvasm
+// RUN: FileCheck %s < %t.spvasm
+// RUN: clspv %s -o %t.spv
+// RUN: spirv-dis -o %t2.spvasm %t.spv
+// RUN: FileCheck %s < %t2.spvasm
+// RUN: spirv-val --target-env vulkan1.0 %t.spv
+
+// CHECK: OpEntryPoint GLCompute [[entry:%[0-9a-zA-Z_]+]] "func_0"
+// CHECK: [[entry]] = OpFunction
+// CHECK: OpFunctionCall
+
+int func_1(global int *in, int n) { return in[n]; }
+kernel void func_0(global int *in, global int *out, int n) {
+  out[n] = func_1(in, n);
+}

test/InlineFuncWithSingleCallSite/two_kernels_partial_chain_inline.cl

@@ -0,0 +1,31 @@
+// RUN: clspv %s -S -o %t.spvasm
+// RUN: FileCheck %s < %t.spvasm
+// RUN: clspv %s -o %t.spv
+// RUN: spirv-dis -o %t2.spvasm %t.spv
+// RUN: FileCheck %s < %t2.spvasm
+// RUN: spirv-val --target-env vulkan1.0 %t.spv
+
+// func_1 is called by both kernel_1 and kernel_2, so it will not be inlined,
+// but func_2 and func_3 will both be inlined into func_1.
+// CHECK: OpEntryPoint GLCompute [[k1:%[0-9a-zA-Z_]+]] "kernel_1"
+// CHECK: OpEntryPoint GLCompute [[k2:%[0-9a-zA-Z_]+]] "kernel_2"
+// CHECK: [[func:%[0-9a-zA-Z_]+]] = OpFunction
+// CHECK-NOT: OpFunctionCall
+// CHECK: OpFunctionEnd
+// CHECK: [[k1]] = OpFunction
+// CHECK: OpFunctionCall {{%[0-9a-zA-Z_]+}} [[func]]
+// CHECK-NOT: OpFunctionCall
+// CHECK: [[k2]] = OpFunction
+// CHECK: OpFunctionCall {{%[0-9a-zA-Z_]+}} [[func]]
+// CHECK-NOT: OpFunctionCall
+
+int func_3(local int *in, int n) { return in[n]; }
+int func_2(local int *in, int n) { return func_3(in, n); }
+int func_1(local int *in, int n) { return func_2(in, n); }
+kernel void kernel_1(local int *in, global int *out, int n) {
+  out[n] = func_1(in, n);
+}
+kernel void kernel_2(local int *in, global int *out, int n) {
+  out[n] = func_1(in, n);
+}
+

test/PointerAccessChains/pointer_deref.cl

@@ -1,6 +1,6 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv

test/PointerAccessChains/pointer_index_in_called_function.cl

@@ -1,13 +1,13 @@
-// RUN: clspv %s -S -o %t.spvasm
+// RUN: clspv %s -S -o %t.spvasm -no-inline-single
 // RUN: FileCheck %s < %t.spvasm
-// RUN: clspv %s -o %t.spv
+// RUN: clspv %s -o %t.spv -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm
 // RUN: spirv-val --target-env vulkan1.0 %t.spv
 
-// RUN: clspv %s -S -o %t.spvasm -no-dra
+// RUN: clspv %s -S -o %t.spvasm -no-dra -no-inline-single
 // RUN: FileCheck %s < %t.spvasm -check-prefix=NODRA
-// RUN: clspv %s -o %t.spv -no-dra
+// RUN: clspv %s -o %t.spv -no-dra -no-inline-single
 // RUN: spirv-dis -o %t2.spvasm %t.spv
 // RUN: FileCheck %s < %t2.spvasm -check-prefix=NODRA
 // RUN: spirv-val --target-env vulkan1.0 %t.spv