Merge pull request #5969 from edwintorok/private/edvint/buildopt

Optimize incremental builds

dune

@@ -13,3 +13,7 @@
 (executable
   (name configure)
   (libraries dune-configurator findlib cmdliner unix))
+
+; Can still be used for dependencies, but dune won't scan these dirs
+; for dune files
+(data_only_dirs doc scripts python3 .vscode)

ocaml/doc/dune

@@ -35,3 +35,4 @@
   (action (run %{x}))
 )
 
+(data_only_dirs templates)

ocaml/dune

@@ -0,0 +1 @@
+(data_only_dirs xe)

ocaml/gencert/dune

@@ -67,3 +67,5 @@
   )
   (action (run %{test} --color=always))
 )
+
+(data_only_dirs test_data)

ocaml/idl/dune

@@ -81,3 +81,5 @@
   (alias update-dm-lifecycle)
   (action
     (diff datamodel_lifecycle.ml datamodel_lifecycle.ml.generated)))
+
+(data_only_dirs templates)

ocaml/idl/ocaml_backend/dune

@@ -11,3 +11,4 @@
   )
 )
 
+(data_only_dirs python)

ocaml/libs/clock/test_timer.ml

@@ -7,7 +7,7 @@ let spans =
 
 let test_timer_remaining =
   let print = Fmt.to_to_string Mtime.Span.pp in
-  Test.make ~name:"Timer.remaining" ~print spans @@ fun duration ->
+  Test.make ~count:20 ~name:"Timer.remaining" ~print spans @@ fun duration ->
   let timer = Timer.start ~duration in
   let half = Timer.span_to_s duration /. 2. in
   let elapsed = Mtime_clock.counter () in

ocaml/libs/vhd/vhd_format/f.ml

@@ -31,11 +31,7 @@ exception Cstruct_differ
 let cstruct_equal a b =
   let check_contents a b =
     try
-      for i = 0 to Cstruct.length a - 1 do
-        let a' = Cstruct.get_char a i in
-        let b' = Cstruct.get_char b i in
-        if a' <> b' then raise Cstruct_differ
-      done ;
+      if Cstruct.compare a b <> 0 then raise Cstruct_differ ;
       true
     with _ -> false
   in

ocaml/libs/vhd/vhd_format_lwt_test/lib.ml

@@ -99,7 +99,7 @@ let check_written_sectors t expected =
         | false ->
             fail (Failure "read empty sector, expected data")
         | true ->
-            Alcotest.check cstruct __LOC__ data y ;
+            Alcotest.check cstruct ~pos:__POS__ "" data y ;
             return ()
         )
         >>= fun () -> loop xs
@@ -139,10 +139,10 @@ let check_raw_stream_contents t expected =
               let actual = Cstruct.sub data (i * 512) 512 in
 
               ( if not (List.mem_assoc sector expected) then
-                  Alcotest.check cstruct __LOC__ empty_sector actual
+                  Alcotest.check cstruct ~pos:__POS__ "" empty_sector actual
                 else
                   let expected = List.assoc sector expected in
-                  Alcotest.check cstruct __LOC__ expected actual
+                  Alcotest.check cstruct ~pos:__POS__ "" expected actual
               ) ;
               check (i + 1)
           in
@@ -163,7 +163,7 @@ let check_raw_stream_contents t expected =
             else
               let expected = List.assoc offset expected in
               let actual = Cstruct.sub remaining 0 F.sector_size in
-              Alcotest.check cstruct __LOC__ expected actual ;
+              Alcotest.check cstruct ~pos:__POS__ "" expected actual ;
               loop Int64.(add offset 1L) (Cstruct.shift remaining F.sector_size)
           in
           loop offset data

ocaml/libs/vhd/vhd_format_lwt_test/parse_test.ml

@@ -51,9 +51,9 @@ let check_empty_disk size =
   let filename = make_new_filename () in
   Vhd_IO.create_dynamic ~filename ~size () >>= fun vhd ->
   Vhd_IO.openchain filename false >>= fun vhd' ->
-  Alcotest.check Lib.header __LOC__ vhd.Vhd.header vhd'.Vhd.header ;
-  Alcotest.check Lib.footer __LOC__ vhd.Vhd.footer vhd'.Vhd.footer ;
-  Alcotest.check Lib.bat __LOC__ vhd.Vhd.bat vhd'.Vhd.bat ;
+  Alcotest.check Lib.header ~pos:__POS__ "" vhd.Vhd.header vhd'.Vhd.header ;
+  Alcotest.check Lib.footer ~pos:__POS__ "" vhd.Vhd.footer vhd'.Vhd.footer ;
+  Alcotest.check Lib.bat ~pos:__POS__ "" vhd.Vhd.bat vhd'.Vhd.bat ;
   Vhd_IO.close vhd' >>= fun () -> Vhd_IO.close vhd
 
 (* Create a disk, resize it, check headers *)
@@ -64,7 +64,8 @@ let check_resize size =
   let vhd = Vhd.resize vhd newsize in
   Vhd_IO.close vhd >>= fun () ->
   Vhd_IO.openchain filename false >>= fun vhd' ->
-  Alcotest.(check int64) __LOC__ newsize vhd.Vhd.footer.Footer.current_size ;
+  Alcotest.(check int64 ~pos:__POS__)
+    "" newsize vhd.Vhd.footer.Footer.current_size ;
   Vhd_IO.close vhd'
 
 (* Create a snapshot, check headers *)
@@ -74,9 +75,9 @@ let check_empty_snapshot size =
   let filename = make_new_filename () in
   Vhd_IO.create_difference ~filename ~parent:vhd () >>= fun vhd' ->
   Vhd_IO.openchain filename false >>= fun vhd'' ->
-  Alcotest.check Lib.header __LOC__ vhd'.Vhd.header vhd''.Vhd.header ;
-  Alcotest.check Lib.footer __LOC__ vhd'.Vhd.footer vhd''.Vhd.footer ;
-  Alcotest.check Lib.bat __LOC__ vhd'.Vhd.bat vhd''.Vhd.bat ;
+  Alcotest.check Lib.header ~pos:__POS__ "" vhd'.Vhd.header vhd''.Vhd.header ;
+  Alcotest.check Lib.footer ~pos:__POS__ "" vhd'.Vhd.footer vhd''.Vhd.footer ;
+  Alcotest.check Lib.bat ~pos:__POS__ "" vhd'.Vhd.bat vhd''.Vhd.bat ;
   Vhd_IO.close vhd'' >>= fun () ->
   Vhd_IO.close vhd' >>= fun () -> Vhd_IO.close vhd
 

ocaml/libs/xapi-rrd/lib_test/dune

@@ -25,3 +25,5 @@
     xapi-rrd
   )
 )
+
+(data_only_dirs test_data)

ocaml/libs/xapi-stdext/lib/xapi-stdext-unix/test/unixext_test.ml

@@ -115,7 +115,8 @@ let test_time_limited_write =
 let test_time_limited_read =
   let gen = Gen.tup2 Generate.t Generate.timeouts
   and print = Print.tup2 Generate.print Print.float in
-  Test.make ~name:__FUNCTION__ ~print gen @@ fun (behaviour, timeout) ->
+  Test.make ~count:20 ~name:__FUNCTION__ ~print gen
+  @@ fun (behaviour, timeout) ->
   skip_blk behaviour.kind ;
   skip_dirlnk behaviour.kind ;
   skip_blk_timed behaviour ;
@@ -166,7 +167,7 @@ let test_time_limited_read =
 
 let test_proxy =
   let gen = Generate.t and print = Generate.print in
-  Test.make ~name:__FUNCTION__ ~print gen @@ fun behaviour ->
+  Test.make ~count:20 ~name:__FUNCTION__ ~print gen @@ fun behaviour ->
   if behaviour.kind <> Unix.S_SOCK then
     QCheck2.assume_fail () ;
   let test wrapped_fd =

ocaml/message-switch/core_test/concur-rpc-test.sh

@@ -9,37 +9,36 @@ trap "cleanup" TERM INT
 function cleanup {
   rm -rf "${SWITCHPATH}"
 }
+SECS=${SECS:-0.1}
 
 rm -rf "${SWITCHPATH}" && mkdir -p "${SWITCHPATH}"
 
 echo Test message switch concurrent processing
 
 echo Checking the switch can start late
-test -x ./server_unix_main.exe || exit 1
-./server_unix_main.exe -path "$SPATH" &
-sleep 1
-test -x ../switch/switch_main.exe && test -x ./client_unix_main.exe || exit 1
-../switch/switch_main.exe --path "$SPATH" --statedir "${SWITCHPATH}" &
-./client_unix_main.exe -path "$SPATH" -secs 5
-sleep 2
+./server_unix_main.exe -path "${SPATH}" &
+SERVER=$!
+sleep "${SECS}"
+../switch/switch_main.exe --path "${SPATH}" --statedir "${SWITCHPATH}" &
+./client_unix_main.exe -path "${SPATH}" -secs "${SECS}"
+wait "${SERVER}"
 
 echo Performance test of Lwt to Lwt
-test -x lwt/server_main.exe && test -x lwt/client_main.exe || exit 1
-lwt/server_main.exe -path "$SPATH" -concurrent &
-lwt/client_main.exe -path "$SPATH" -secs 5
-sleep 2
+lwt/server_main.exe -path "${SPATH}" -concurrent &
+SERVER=$!
+lwt/client_main.exe -path "${SPATH}" -secs "${SECS}"
+wait "${SERVER}"
 
 echo Performance test of Async to Lwt
-test -x lwt/server_main.exe && test -x async/client_async_main.exe || exit 1
-lwt/server_main.exe -path "$SPATH" -concurrent &
-async/client_async_main.exe -path "$SPATH" -secs 5
-sleep 2
+lwt/server_main.exe -path "${SPATH}" -concurrent &
+SERVER=$!
+async/client_async_main.exe -path "${SPATH}" -secs "${SECS}"
+wait "${SERVER}"
 
 echo Performance test of Async to Async
-test -x async/server_async_main.exe && test -x async/client_async_main.exe || exit 1
-async/server_async_main.exe -path "$SPATH" -concurrent &
-async/client_async_main.exe -path "$SPATH" -secs 5
-sleep 2
+async/server_async_main.exe -path "${SPATH}" -concurrent &
+SERVER=$!
+async/client_async_main.exe -path "${SPATH}" -secs "${SECS}"
+wait "${SERVER}"
 
-../cli/main.exe shutdown --path "$SPATH"
-sleep 2
+../cli/main.exe shutdown --path "${SPATH}"

ocaml/message-switch/dune

@@ -1,3 +1,5 @@
 (executable
   (name configure)
   (libraries dune-configurator findlib))
+
+(data_only_dirs www python)

ocaml/networkd/test/dune

@@ -25,3 +25,5 @@
   )
  )
 )
+
+(data_only_dirs jsonrpc_files)

ocaml/quicktest/dune

@@ -40,7 +40,7 @@
     xenctrl
     xml-light2
   )
-  (preprocess (pps ppx_deriving_rpc ppx_sexp_conv))
+  (preprocess (per_module ((pps ppx_deriving_rpc) Quicktest_vm_lifecycle)))
 )
 
 

ocaml/rrd2csv/dune

@@ -0,0 +1 @@
+(data_only_dirs man)

ocaml/sdk-gen/c/autogen/dune

@@ -23,3 +23,4 @@
   )
 )
 
+(data_only_dirs src include)

ocaml/sdk-gen/c/dune

@@ -19,3 +19,4 @@
   (action (run %{x}))
 )
 
+(data_only_dirs templates)

ocaml/sdk-gen/csharp/autogen/dune

@@ -12,4 +12,6 @@
     LICENSE
     (source_tree .)
   )
-)
+)
+
+(data_only_dirs src)

ocaml/sdk-gen/csharp/dune

@@ -46,3 +46,4 @@
   (action (run %{x} -s %{y}))
 )
 
+(data_only_dirs templates)

ocaml/sdk-gen/dune

@@ -0,0 +1 @@
+(data_only_dirs component-test)

ocaml/sdk-gen/go/autogen/dune

@@ -22,3 +22,5 @@
     (source_tree .)
   )
 )
+
+(data_only_dirs src)

ocaml/sdk-gen/go/dune

@@ -44,3 +44,5 @@
     (source_tree templates)
   )
 )
+
+(data_only_dirs test_data templates)

ocaml/sdk-gen/java/autogen/dune

@@ -5,3 +5,4 @@
   )
 )
 
+(data_only_dirs xen-api)

ocaml/sdk-gen/java/dune

@@ -29,3 +29,4 @@
   (action (run %{x}))
 )
 
+(data_only_dirs templates)

ocaml/sdk-gen/powershell/autogen/dune

@@ -13,3 +13,5 @@
     (source_tree .)
   )
 )
+
+(data_only_dirs src)

ocaml/sdk-gen/powershell/dune

@@ -19,3 +19,4 @@
   (action (run %{x}))
 )
 
+(data_only_dirs templates)

ocaml/squeezed/dune

@@ -0,0 +1 @@
+(data_only_dirs scripts)

ocaml/tests/dune

@@ -122,7 +122,7 @@
     xml-light2
     yojson
   )
-  (preprocess (pps ppx_deriving_rpc ppx_sexp_conv))
+  (preprocess (per_module ((pps ppx_deriving_rpc) Test_cluster_host)))
 )
 (test
 (name test_storage_smapiv1_wrapper)
@@ -168,3 +168,5 @@
 )
 
 (env (_ (env-vars (XAPI_TEST 1))))
+
+(data_only_dirs test_data tests)

ocaml/xapi-cli-server/dune

@@ -45,7 +45,7 @@
     xmlm
     xml-light2
   )
-  (preprocess (pps ppx_deriving_rpc))
+  (preprocess (per_module ((pps ppx_deriving_rpc) Cli_operations)))
   (wrapped false)
 )
 

ocaml/xapi-idl/dune

@@ -0,0 +1 @@
+(data_only_dirs designs xen-api-plugin)

ocaml/xapi-idl/gpumon/dune

@@ -11,7 +11,7 @@
    xapi-log
  )
  (wrapped false)
- (preprocess (pps ppx_deriving_rpc)))
+ (preprocess (per_module ((pps ppx_deriving_rpc) Gpumon_interface))))
 
 (executable
  (name gpumon_cli)

ocaml/xapi-idl/lib_test/device_number_test.ml

@@ -133,10 +133,13 @@ let test_2_way_convert =
     let original = of_disk_number hvm disk_number |> Option.get in
     let of_linux = of_linux_device (to_linux_device original) |> Option.get in
     let of_xenstore = of_xenstore_key (to_xenstore_key original) in
-    Alcotest.check device_number_equal_linux
-      "of_linux must be equal to original" original of_linux ;
-    Alcotest.check device_number "of_xenstore must be equal to original"
-      original of_xenstore
+    (* use ~pos instead of msg: a non-empty msg causes the formatter to be flushed,
+       and messages printed on stdout, which is very slow if we do this in a loop a million times
+    *)
+    Alcotest.check' ~pos:__POS__ ~msg:"" device_number_equal_linux
+      ~expected:original ~actual:of_linux ;
+    Alcotest.check' ~pos:__POS__ ~msg:"" device_number ~expected:original
+      ~actual:of_xenstore
   in
 
   let max_d = (1 lsl 20) - 1 in

ocaml/xapi-idl/lib_test/dune

@@ -1,3 +1,5 @@
+(data_only_dirs test_data)
+
 (library
   (name test_lib)
   (modules idl_test_common)
@@ -6,18 +8,23 @@
   (wrapped false)
 )
 
-(test
+(executable
  (name guard_interfaces_test)
- (package xapi-idl)
  (modules guard_interfaces_test)
- (deps (source_tree test_data))
  (libraries
    test_lib
    xapi-idl.guard.privileged
    xapi-idl.guard.varstored
  )
 )
 
+(rule
+  (alias runtest)
+  (package xapi-idl)
+  (deps (:exe ./guard_interfaces_test.exe) (source_tree test_data/guard))
+  (action (run %{exe}))
+)
+
 (test
  (name device_number_test)
  (package xapi-idl)
@@ -34,7 +41,6 @@
  (modes exe)
  (package xapi-idl)
  (modules (:standard \ idl_test_common guard_interfaces_test device_number_test))
- (deps (source_tree test_data))
  (libraries
    alcotest
    cohttp_posix
@@ -61,3 +67,10 @@
    xapi-log
  )
  (preprocess (per_module ((pps ppx_deriving_rpc) Task_server_test Updates_test))))
+
+(rule
+  (alias runtest)
+  (package xapi-idl)
+  (deps (:exe ./test.exe) (source_tree test_data/guard))
+  (action (run %{exe}))
+)