meson: Add architecture for LLVM bitcode emission

This commit adds suport for bitcode emission for both normal and
generated source files (processed by bison, flex, etc). These bitcode
files are installed into $pkglibdir/bitcode/ directory if the LLVM is
found.

New variable `bitcode_modules` is introduced to generate bitcode files.
All required information is gathered in this variable. Then, this
variable is processed by the main meson LLVM bitcode emission scripts:
src/backend/jit/llvm/bitcode/meson.build -> src/tools/irlink.

An example of a possible structure of bitcode_modules is:
```
bitcode_modules = [
  {
    'name': '...',
    'target': ...,
    'srcfiles': [
      '...',
      '...',
    ],
    'additional_flags': [
      '-I...',
      '-I...',
    ],
    'gen_srcfiles': [
      {
        'srcfiles': [
          <custom_target for ...>,
          <custom_target for ...>,
        ],
        'additional_flags': [
          '-I...',
          '-I...',
        ]
      }
    ]
  }
]
```

Author: Andres Freund <andres@anarazel.de>
Author: Nazir Bilal Yavuz <byavuz81@gmail.com>
Author: Diego Fronza <diego.fronza@percona.com>
Reviewed-by: Diego Fronza <diego.fronza@percona.com>
Discussion: https://postgr.es/m/206b001d-1884-4081-bd02-bed5c92f02ba%40eisentraut.org

Author: nbyavuz

meson.build

@@ -815,6 +815,8 @@ if add_languages('cpp', required: llvmopt, native: false)
     # Some distros put LLVM and clang in different paths, so fallback to
     # find via PATH, too.
     clang = find_program(llvm_binpath / 'clang', 'clang', required: true)
+    llvm_lto = find_program(llvm_binpath / 'llvm-lto', required: true)
+    irlink = find_program('src/tools/irlink', native: true)
   endif
 elif llvmopt.auto()
   message('llvm requires a C++ compiler')
@@ -3044,6 +3046,11 @@ test_deps = []
 tests = []
 meson_extension_tests = []
 
+# List of object files + source files to generated LLVM IR for inlining.
+# Each element is a hash of:
+# {'target': target, 'srcfiles': ..., 'additional_flags': ...}.
+bitcode_modules = []
+
 
 # Default options for targets
 
@@ -3366,6 +3373,11 @@ subdir('src/interfaces/ecpg/test')
 
 subdir('doc/src/sgml')
 
+# generate bitcode for JIT inlining after giving contrib modules etc a chance
+# to add themselves to bitcode_modules[]
+subdir('src/backend/jit/llvm/bitcode', if_found: llvm)
+
+
 generated_sources_ac += {'': ['GNUmakefile']}
 
 # After processing src/test, add test_install_libs to the testprep_targets
@@ -4001,6 +4013,15 @@ if meson.version().version_compare('>=0.57')
     section: 'Programs',
   )
 
+  if llvm.found()
+    summary(
+      {
+        'clang': clang,
+      },
+      section: 'Programs',
+    )
+  endif
+
   summary(
     {
       'bonjour': bonjour,

src/backend/jit/llvm/bitcode/meson.build

@@ -0,0 +1,69 @@
+# Copyright (c) 2022-2024, PostgreSQL Global Development Group
+#
+# emit LLVM bitcode for JIT inlining
+
+assert(llvm.found())
+
+foreach bitcode_module : bitcode_modules
+  bitcode_targets = []
+  bitcode_obj = bitcode_module['target']
+  bitcode_cflags_local = bitcode_cflags + bitcode_module.get('additional_flags', [])
+  bitcode_name = bitcode_module.get('name', bitcode_obj.name())
+
+  foreach srcfile : bitcode_module['srcfiles']
+    if meson.version().version_compare('>=0.59')
+      srcfilename = fs.parent(srcfile) / fs.name(srcfile)
+    else
+      srcfilename = '@0@'.format(srcfile)
+    endif
+
+    targetname = '@0@_@1@.bc'.format(
+      bitcode_name,
+      srcfilename.underscorify(),
+    )
+    bitcode_targets += custom_target(
+      targetname,
+      depends: [bitcode_obj],
+      input: [srcfile],
+      output: targetname,
+      command: [llvm_irgen_command, bitcode_cflags_local],
+      install: true,
+      install_dir: dir_bitcode,
+    )
+  endforeach
+
+  # Process generated sources, which may include custom compilation flags.
+  foreach gen_sources: bitcode_module.get('gen_sources', [])
+    bitcode_cflags_gen_local = bitcode_cflags_local + gen_sources.get('additional_flags', [])
+
+    foreach srcfile: gen_sources['srcfiles']
+      # Generated sources are stored in some folder under meson.build_root()/**,
+      # remove the build prefix from the string.
+      srcfilename = srcfile.full_path().split(meson.build_root() + '/')[1]
+
+      targetname = '@0@_@1@.bc'.format(
+        bitcode_name,
+        srcfilename.underscorify(),
+      )
+      bitcode_targets += custom_target(
+        targetname,
+        depends: [bitcode_obj],
+        input: [srcfile],
+        output: targetname,
+        command: [llvm_irgen_command, bitcode_cflags_gen_local],
+        install: true,
+        install_dir: dir_bitcode,
+      )
+    endforeach
+  endforeach
+
+  index_name = '@0@.index.bc'.format(bitcode_name)
+  bitcode_index = custom_target('@0@'.format(bitcode_name),
+    output: index_name,
+    input: bitcode_targets,
+    command: [irlink, '--lto', llvm_lto, '--outdir', '@OUTDIR@', '--index', index_name, '@INPUT@'],
+    install: true,
+    install_dir: dir_bitcode,
+  )
+  backend_targets += bitcode_index
+endforeach

src/backend/jit/llvm/meson.build

@@ -42,21 +42,22 @@ backend_targets += llvmjit
 
 # Define a few bits and pieces used here and elsewhere to generate bitcode
 
-llvm_irgen_args = [
-  '-c', '-o', '@OUTPUT@', '@INPUT@',
+llvm_irgen_command = []
+if ccache.found()
+  llvm_irgen_command += ccache
+endif
+
+llvm_irgen_command += [
+  clang,
+  '-c', '-o', '@OUTPUT0@', '@INPUT0@',
   '-flto=thin', '-emit-llvm',
-  '-MD', '-MQ', '@OUTPUT@', '-MF', '@DEPFILE@',
   '-O2',
   '-Wno-ignored-attributes',
   '-Wno-empty-body',
+  '-Wno-unknown-warning-option',
+  '-Wno-compound-token-split-by-macro',
 ]
-
-if ccache.found()
-  llvm_irgen_command = ccache
-  llvm_irgen_args = [clang.path()] + llvm_irgen_args
-else
-  llvm_irgen_command = clang
-endif
+llvm_irgen_dep_args = ['-MD', '-MQ', '@OUTPUT0@', '-MF', '@DEPFILE@']
 
 
 # XXX: Need to determine proper version of the function cflags for clang
@@ -73,7 +74,7 @@ bitcode_cflags += '-I@SOURCE_ROOT@/src/include'
 # Note this is intentionally not installed to bitcodedir, as it's not for
 # inlining
 llvmjit_types = custom_target('llvmjit_types.bc',
-  command: [llvm_irgen_command] + llvm_irgen_args + bitcode_cflags,
+  command: llvm_irgen_command + llvm_irgen_dep_args + bitcode_cflags,
   input: 'llvmjit_types.c',
   output: 'llvmjit_types.bc',
   depends: [postgres],
@@ -82,3 +83,11 @@ llvmjit_types = custom_target('llvmjit_types.bc',
   depfile: '@BASENAME@.c.bc.d',
 )
 backend_targets += llvmjit_types
+
+# Figure out -I's needed to build all postgres code, including all its
+# dependencies
+pkg_config = find_program(['pkg-config', 'pkgconf'], required: true)
+r = run_command(pkg_config,
+  ['--cflags-only-I', meson.build_root() / 'meson-uninstalled/postgresql-extension-uninstalled.pc'],
+  check: true)
+bitcode_cflags += r.stdout().split()

src/backend/meson.build

@@ -140,6 +140,12 @@ postgres = executable('postgres',
 
 backend_targets += postgres
 
+bitcode_modules += {
+  'name': 'postgres',
+  'target': postgres_lib,
+  'srcfiles': backend_sources,
+}
+
 pg_mod_c_args = cflags_mod
 pg_mod_cpp_args = cxxflags_mod
 pg_mod_link_args = ldflags_sl + ldflags_mod

src/tools/irlink

@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import shutil
+import subprocess
+import sys
+
+parser = argparse.ArgumentParser(
+    description='generate PostgreSQL JIT IR module')
+
+parser.add_argument('--index', type=str, required=True)
+parser.add_argument('--lto', type=str, required=True)
+parser.add_argument('--outdir', type=str, required=True)
+parser.add_argument('INPUT', type=str, nargs='+')
+
+args = parser.parse_args()
+
+file_names = [os.path.basename(f) for f in args.INPUT]
+command = [args.lto,
+           '-thinlto', '-thinlto-action=thinlink',
+           '-o', args.index] + file_names
+res = subprocess.run(command, cwd=args.outdir)
+
+exit(res.returncode)