Dehackify cxx and llvm building (BinaryAnalysisPlatform#557)

* split setup.ml.in and myocamlbuild.ml.in

The above specified files contained definitions that were not generic,
but belonged to specific modules, thus breaking module abstraction
boundaries. The llvm flags were hardcoded in the rule for building C++
files.

This PR makes everything in a proper way. We use tags properly in our
rules, so that later we can inject proper flags. Now, all packages,
that need special building rules, have their own files in the oasis
directory.

For example, llvm package defines its building rules in three files:

    - oasis/llvm.setup.ml.in -- define configuration parameters;
    - oasis/llvm.tags.in -- tag files, that need specific handling;
    - oasis/llvm.ocamlbuild.ml.in - inject llvm flags, when building
                                    tagged files.

There are still few things, that should be fixed. For example,
both piqi and cxx rules have a hardcoded paths for -I option.

* fix configuration on mac os x

1. don't panic if there is no `llvm-config`
2. look at `llvm-confi-mp-$ver` where `$ver` doesn't include patch version

lib/bap_build/bap_build.ml

@@ -153,6 +153,7 @@ module Plugin_rules = struct
                    make_list_option "-provides" in
     Cmd (S [
         A "bapbundle"; A "pack";
+        T (Tags.of_list ["bundle"; "library"; "plugin"]);
         A "-name"; A (dashify (env "%"));
         A "-main"; A (env "%.cmxs");
         A "-main"; A (env "%.cma");

myocamlbuild.ml.in

@@ -1,46 +1,20 @@
-let nonempty = function (A s) -> String.length s <> 0 | _ -> true
-
-(* the piqi support is rather fragile *)
-let piqic_rule () : unit =
-  rule "piqic: piqi -> .ml & _ext.ml"
-    ~prods:["%_piqi.ml"; "%_piqi_ext.ml"]
-    ~deps:["%.piqi"]
-    (fun env _ ->
-       Cmd (S (List.filter nonempty [
-           A (expand "${piqic}");
-           A (expand "${piqic_flags}");
-           A "-C";
-           A "lib/bap_piqi";
-           A "-I";
-           A "../lib/bap_piqi";
-           A (env "%.piqi");
-           A"--multi-format"])));;
-
 let cxx_rule () =
   let deps = ["%.hpp"; "%.cpp"; "%.h"] and prod = "%.o" in
-  let action env _ = Cmd (S [
-      Sh (expand "${cxx} ${cxxflags} ${llvm_cxxflags}");
-      A "-c"; P (env "%.cpp"); A "-o"; P (env "%.o")]) in
+  let action env _ =
+    let src = env "%.cpp" and obj = env "%.o" in
+    let cxx = expand "${cxx} ${cc_optimization} ${cxxflags}" in
+    let tags = tags_of_pathname src ++ "c++" ++ "compile" in
+    Cmd (S [Sh cxx; T tags; A "-c"; P src; A "-o"; Px obj]) in
   rule "cxx: hpp & cpp & h -> o" ~deps ~prod action
 
-let dispatch = function
-  | Before_rules ->
-    piqic_rule ();
-    cxx_rule ();
-  | After_rules ->
-    List.iter
-      (fun tag ->
-         pflag ["ocaml"; tag] "pa_ounit_lib"
-           (fun s -> S[A"-ppopt"; A"-pa-ounit-lib"; A"-ppopt"; A s]))
-      ["ocamldep"; "compile"; "doc"];
-  | _ -> ()
+let () = Rules.add cxx_rule
 
-module Ocamlbuild_compat = struct
-  let mark_tag_used = ignore
-  include Ocamlbuild_plugin
-end
 
 let mark_tags () =
+  let module Ocamlbuild_compat = struct
+    let mark_tag_used = ignore
+    include Ocamlbuild_plugin
+  end in
   let open Ocamlbuild_compat in
   List.iter mark_tag_used [
     "pkg_core_bench";
@@ -54,17 +28,24 @@ let pr_6184_hack = function
   | After_rules ->
     (* Pass -predicates to ocamldep *)
     pflag ["ocaml"; "ocamldep"] "predicate" (fun s -> S [A "-predicates"; A s]);
+  | _ -> ()
+
+
+let predicate_used_libs = function
+  | After_rules ->
     package_default.MyOCamlbuildBase.lib_ocaml |>
     List.iter (fun (lib,_,_) ->
         flag ["ocaml"; "link"; "use_"^lib]
-          (S [A "-predicates"; A ("used_"^lib)]));
+          (S [A "-predicates"; A ("used_"^lib)]))
   | _ -> ()
 
 
+
 let () =
   mark_tags ();
-  Ocamlbuild_plugin.dispatch (fun hook ->
-      dispatch hook;
-      dispatch_default hook;
-      pr_6184_hack hook;
-      Ppx_driver_ocamlbuild.dispatch hook)
+  Ocamlbuild_plugin.dispatch (fun stage ->
+      Rules.dispatch stage;
+      dispatch_default stage;
+      pr_6184_hack stage;
+      predicate_used_libs stage;
+      Ppx_driver_ocamlbuild.dispatch stage)

oasis/common.ocamlbuild.ml.in

@@ -1,8 +1,21 @@
 (* OASIS_START *)
 (* OASIS_STOP *)
-#4 "oasis/common.ocamlbuild.ml.in"
+
 let oasis_env =
   BaseEnvLight.load
     ~allow_empty:true
     ()
 let expand s = BaseEnvLight.var_expand s oasis_env
+
+type rule = unit -> unit
+
+module Rules : sig
+  val add : rule -> unit
+  val dispatch : hook -> unit
+end = struct
+  let rules : (unit -> unit) list ref  = ref []
+  let add rule = rules := rule :: !rules
+  let dispatch = function
+    | Before_rules -> !rules |> List.iter (fun rule -> rule ())
+    | _ -> ()
+end

oasis/common.setup.ml.in

@@ -3,6 +3,9 @@
 (* OASIS_STOP *)
 #5 "oasis/common.setup.ml.in"
 let definition_end = BaseEnv.var_ignore
+let define definitions =
+  List.iter (fun f -> try f () with exn -> ()) definitions
+
 
 let ctxt = !BaseContext.default
 
@@ -47,3 +50,11 @@ let is_undefined var : bool = not (is_defined var)
 
 let is_set_to var value : bool =
   is_defined var && BaseEnv.var_get var = value
+
+
+let rec find_map xs ~f =
+  match xs with
+  | [] -> None
+  | x :: xs -> match f x with
+    | None -> find_map xs ~f
+    | yay -> yay

oasis/ida.setup.ml.in

@@ -1,3 +1,4 @@
+
 let define_headless = function
   | Some "true" when BaseEnv.var_get "system" <> "linux" ->
     invalid_arg "headless mode is supported only on linux"

oasis/llvm.ocamlbuild.ml.in

@@ -0,0 +1,2 @@
+let () =
+  flag ["c++"; "compile"; "use_libllvm"] (Sh (expand "${llvm_cxxflags}"))

oasis/llvm.setup.ml.in

@@ -0,0 +1,91 @@
+let strip_patch ver =
+  if String.length ver <> 5 then ver
+  else String.sub ver 0 3
+
+
+let llvm var () : unit =
+  BaseEnv.var_define
+    ~hide:true
+    ~dump:true
+    ~short_desc:(fun () -> "llvm-config --"^var)
+    ("llvm_"^var)
+    (fun () ->
+       let llvm_config = BaseEnv.var_get "llvm_config" in
+       let llvm_version = BaseEnv.var_get "llvm_version" in
+       let extract v =
+	 OASISExec.run_read_one_line ~ctxt llvm_config ["--"^v] in
+       if strip_patch llvm_version > "3.4" && var = "ldflags"
+       then extract var ^ " " ^ extract "system-libs"
+       else extract var) |>
+    definition_end
+
+let llvm_version () : unit =
+  BaseEnv.var_define
+    ~hide:false
+    ~dump:true
+    ~cli:BaseEnv.CLIWith
+    ~short_desc:(fun () -> "llvm version (e.g., 3.4)")
+    "llvm_version"
+    (fun () ->
+       try
+         ignore @@ OASISFileUtil.which ~ctxt "llvm-config";
+         OASISExec.run_read_one_line ~ctxt "llvm-config" ["--version"]
+       with Not_found -> "3.4") |>
+  definition_end
+
+let llvm_config () : unit =
+  BaseEnv.var_define
+    ~hide:false
+    ~dump:true
+    ~cli:BaseEnv.CLIWith
+    ~short_desc:(fun () -> "llvm-config executable")
+    "llvm_config"
+    (fun () ->
+       (* default macports if we're on mac os x *)
+       let macosx = BaseEnv.var_get "system" = "macosx" in
+       let vers = match BaseEnv.var_get "llvm_version" with
+         | "" -> []
+         | ver when macosx -> ["-mp-" ^ strip_patch ver; "-" ^ strip_patch ver]
+         | ver -> ["-" ^ ver; "-" ^ strip_patch ver] in
+       find_map vers ~f:(fun ver ->
+           try Some (OASISFileUtil.which ~ctxt ("llvm-config" ^ ver))
+           with Not_found -> None) |> function
+       | Some path -> path
+       | None -> raise Not_found) |>
+  definition_end
+
+let llvm_mainlib () : unit =
+  BaseEnv.var_define
+    ~hide:true
+    ~dump:true
+    ~short_desc:(fun () -> "main LLVM library")
+    "llvm_mainlib"
+    (fun () -> "-lLLVM-"^BaseEnv.var_get "llvm_version") |>
+  definition_end
+
+
+let llvm_lib () : unit =
+  BaseEnv.var_define
+    ~hide:true
+    ~dump:true
+    ~short_desc:(fun () -> "LLVM library(ies) to link with")
+    "llvm_lib"
+    (fun () ->
+       let llvm_static = BaseEnv.var_get "llvm_static" in
+       let lib = if llvm_static = "true"
+         then "llvm_libs"
+         else "llvm_mainlib" in
+       BaseEnv.var_get lib) |>
+  definition_end
+
+let () =
+  define [
+    llvm_version;
+    llvm_config;
+    llvm_mainlib;
+    llvm "cxxflags";
+    llvm "ldflags";
+    llvm "cflags";
+    llvm "libs";
+    llvm_lib
+  ]

oasis/llvm.tags.in

@@ -0,0 +1 @@
+<plugins/llvm/llvm_*.cpp>: use_libllvm

oasis/objdump.setup.ml.in

@@ -1,5 +1,3 @@
-
-
 let () =
   add_variable  ~doc:"A list (OCaml syntax) of supported targets" "objdump_targets"
     ~define:(function

oasis/piqi.ocamlbuild.ml.in

@@ -0,0 +1,19 @@
+let nonempty = function (A s) -> String.length s <> 0 | _ -> true
+
+(* the piqi support is rather fragile *)
+let piqic_rule () : unit =
+  rule "piqic: piqi -> .ml & _ext.ml"
+    ~prods:["%_piqi.ml"; "%_piqi_ext.ml"]
+    ~deps:["%.piqi"]
+    (fun env _ ->
+       Cmd (S (List.filter nonempty [
+           A (expand "${piqic}");
+           A (expand "${piqic_flags}");
+           A "-C";
+           A "lib/bap_piqi";
+           A "-I";
+           A "../lib/bap_piqi";
+           A (env "%.piqi");
+           A"--multi-format"])));;
+
+let () = Rules.add piqic_rule

oasis/piqi.setup.ml.in

@@ -0,0 +1,22 @@
+let piqic () : unit =
+  let chop str = try
+      Filename.chop_extension str
+    with _ -> str in
+  let piqic_path = BaseCheck.prog_best "piqic_path" ["piqic-ocaml"; "piqic"] () in
+  BaseEnv.var_define "piqic_flags" (fun () ->
+      if chop (Filename.basename (BaseEnv.var_get "piqic")) = "piqic"
+      then "ocaml"
+      else "") |>
+  definition_end;
+  BaseEnv.var_define "piqic" (fun () ->
+      if List.mem (BaseStandardVar.os_type ()) ["Cygwin"; "Win32"]
+      then
+        String.concat " "
+          (OASISExec.run_read_output ~ctxt "cygpath" [piqic_path])
+      else piqic_path) |>
+  definition_end
+
+
+
+
+let () = define [piqic]