Merge branch 'master' of github.com:realworldocaml/mdx into jupyter

Author: avsm

.travis.yml

@@ -7,10 +7,11 @@ env:
   - TO_TEST=tests OPAMBUILDTEST=true OCAML_VERSION=4.04.2
   - TO_TEST=tests OPAMBUILDTEST=true OCAML_VERSION=4.05.0
   - TO_TEST=tests OPAMBUILDTEST=true OCAML_VERSION=4.06.1
-  - TO_TEST=tests OPAMBUILDTEST=true OCAML_VERSION=4.07.0
+  - TO_TEST=tests OPAMBUILDTEST=true OCAML_VERSION=4.07.1
+  - TO_TEST=tests OPAMBUILDTEST=true OCAML_VERSION=4.08.0
 before_install:
   # Download and use opam2
-  - wget -O ${HOME}/opam https://github.com/ocaml/opam/releases/download/2.0.2/opam-2.0.2-x86_64-linux
+  - wget -O ${HOME}/opam https://github.com/ocaml/opam/releases/download/2.0.4/opam-2.0.4-x86_64-linux
   - chmod +x ${HOME}/opam
   # Some opam boilerplate
   - export OPAMYES=1

CHANGES.md

@@ -1,3 +1,9 @@
+### 1.4.0 (2019-06-11)
+- Add `--force-output` option to force generation of diff file (#118 @clecat)
+- Support OCaml 4.08.0 (#121 @xclerc)
+- README and documentation fixes (#122 #118 @andreypopp @clecat @samoht)
+- Use latest ocaml-migrate-parsetree interfaces (@avsm)
+
 ### 1.3.0 (2019-03-01)
 - Updated readme file with the new features: dune rules, named environment and
   ocaml versions, Some grammar correction too (@gpetiot, #101, aantron, #102)

README.md

@@ -4,16 +4,16 @@
 
 `mdx` allows to execute code blocks inside markdown files.
 There are (currently) two sub-commands, corresponding
-to two modes of operations: pre-processing (`mdx pp`)
-and tests (`mdx test`).
+to two modes of operations: pre-processing (`ocaml-mdx pp`)
+and tests (`ocaml-mdx test`).
 
 The pre-processor mode allows to mix documentation and code,
 and to practice "literate programming" using markdown and OCaml.
 
 The test mode allows to ensure that shell scripts and OCaml fragments
 in the documentation always stays up-to-date.
 
-`mdx` is released as a single binary (called `mdx`) and
+`mdx` is released as a single binary (called `ocaml-mdx`) and
 can be installed using opam:
 
 ```sh
@@ -24,7 +24,7 @@ $ opam install mdx
 
 #### Shell Scripts
 
-`mdx` interprets shell scripts inside `sh` code blocks as cram-like tests. The
+`ocaml-mdx` interprets shell scripts inside `sh` code blocks as cram-like tests. The
 syntax is the following:
 
 - Lines beginning with a dollar sign and a space are
@@ -63,7 +63,7 @@ with a padding of 3:
        10
     ```
 
-`mdx` will also consider exit codes when the syntax `[<exit code>]`is used:
+`ocaml-mdx` will also consider exit codes when the syntax `[<exit code>]`is used:
 
     ```sh
     $ exit 1
@@ -75,7 +75,7 @@ of success).
 
 #### OCaml Code
 
-`mdx` interprets OCaml fragments. It understands _normal_ code fragments and
+`ocaml-mdx` interprets OCaml fragments. It understands _normal_ code fragments and
 _toplevel_ code fragments (starting with a `#` sign and optionally ending with
 `;;`). Arbitrary whitespace padding is supported, at long as it stays
 consistent within a code block.
@@ -97,7 +97,7 @@ Here is an examples of toplevel OCaml code:
 
 ### Pre-processing
 
-`mdx pp` allows to transform a markdown file into a valid
+`ocaml-mdx pp` allows to transform a markdown file into a valid
 OCaml file, which can be passed to OCaml using the `-pp`
 option.
 
@@ -111,7 +111,7 @@ For instance, given the following `file.md` document:
 Can be compiled and executed using:
 
 ```sh
-$ ocamlc -pp 'mdx pp' -impl file.md -o file.exe
+$ ocamlc -pp 'ocaml-mdx pp' -impl file.md -o file.exe
 $ ./file.exe
 42
 ```
@@ -122,7 +122,7 @@ This can be automated using `dune`:
 (rule
  ((targets (file.ml))
   (deps    (file.md))
-  (action  (with-stdout-to ${@} (run mdx pp ${<})))))
+  (action  (with-stdout-to ${@} (run ocaml-mdx pp ${<})))))
 
 (executable ((name file)))
 ```
@@ -131,7 +131,7 @@ This can be automated using `dune`:
 
 #### Cram Tests
 
-Cram tests can be executed and checked with `mdx test <file.md>`.
+Cram tests can be executed and checked with `ocaml-mdx test <file.md>`.
 
     ```sh
      $ for i in `seq 1 10`; do echo $i; done
@@ -145,7 +145,7 @@ If the output is not consistent with what is expected,
 
 #### OCaml
 
-To execute OCaml code and toplevel fragments, uses `mdx test <file.md>`.
+To execute OCaml code and toplevel fragments, uses `ocaml-mdx test <file.md>`.
 
     ```ocaml
     # print_endline "42"
@@ -162,11 +162,11 @@ dune's `diff?` stanza:
 
 ```
 (alias
- ((name runtest)
-  (deps (file.md))
+  (name runtest)
+  (deps (:test file.md))
   (action (progn
-           (run mdx test ${<})
-           (diff? ${<} ${<}.corrected)))))
+           (run ocaml-mdx test %{test})
+           (diff? %{test} %{test}.corrected))))
 ```
 
 This allows to test the consistency of a markdown file using the normal dev
@@ -204,7 +204,7 @@ $ dune promote
 
 **Non-deterministic Outputs**
 
-`mdx test` supports non-deterministic outputs:
+`ocaml-mdx test` supports non-deterministic outputs:
 
     ```sh non-deterministic=output
     $ <command>
@@ -213,19 +213,19 @@ $ dune promote
 
 In that case, `ppx test <file>` will run the command but will not
 generate `<file>.corrected` if the new output differs from the one
-described in the file. Use `mdx test --non-deterministic <file>` to come
+described in the file. Use `ocaml-mdx test --non-deterministic <file>` to come
 back to the default behaviour.
 
 **Non-deterministic Commands**
 
-`mdx test` supports non-deterministic commands:
+`ocaml-mdx test` supports non-deterministic commands:
 
     ```ocaml non-deterministic=command
     # Random.int 10;;
     - : int = 5
     ```
 
-In that case, `mdx test <file>` will *not* run the command. Use `mdx test
+In that case, `ocaml-mdx test <file>` will *not* run the command. Use `ocaml-mdx test
 --non-deterministic <file>` to come back to the default behaviour.
 
 #### Named execution environments (since mdx 1.1.0)
@@ -294,39 +294,39 @@ The version number can be of the following forms:
 
 Environment variables can be declared at the beginning of a block:
 
-```ocaml set-FOO=bar,set-BAR=foo
-  # print_endline (Sys.getenv "FOO")
-  bar
-  - : unit = ()
-  # print_endline (Sys.getenv "BAR")
-  foo
-  - : unit = ()
-```
+    ```ocaml set-FOO=bar,set-BAR=foo
+    # print_endline (Sys.getenv "FOO")
+    bar
+    - : unit = ()
+    # print_endline (Sys.getenv "BAR")
+    foo
+    - : unit = ()
+    ```
 
 Those variables are then available in the subsequent blocks
 
-```ocaml
-  # print_endline (Sys.getenv "FOO")
-  bar
-  - : unit = ()
-```
+    ```ocaml
+    # print_endline (Sys.getenv "FOO")
+    bar
+    - : unit = ()
+    ```
 
 ### Sections
 
 It is possible to test or execute only a subset of the file using
 sections using the `--section` option (short name is `-s`). For
-instance `mdx pp -s foo` will only consider the section matching the
+instance `ocaml-mdx pp -s foo` will only consider the section matching the
 perl regular expression `foo`.
 
 ### Dune rules (since mdx 1.1.0)
 
-`mdx` can generate `dune` rules to synchronize .md files with .ml files.
+`ocaml-mdx` can generate `dune` rules to synchronize .md files with .ml files.
 
 Consider the test/dune_rules.md file that contains blocks referring to files
 dune_rules_1.ml and dune_rules_2.ml, running:
 
 ```
-$ mdx rule test/dune_rules.md
+$ ocaml-mdx rule test/dune_rules.md
 ```
 
 generates the following `dune` rules on the standard output:
@@ -338,7 +338,7 @@ generates the following `dune` rules on the standard output:
          (:y0 dune_rules_2.ml)
          (source_tree foo))
  (action (progn
-           (run mdx test --direction=infer-timestamp %{x})
+           (run ocaml-mdx test --direction=infer-timestamp %{x})
            (diff? %{x} %{x}.corrected)
            (diff? %{y1} %{y1}.corrected)
            (diff? %{y0} %{y0}.corrected))))

bin/rule.ml

@@ -79,10 +79,11 @@ let print_rule ~nd ~prelude ~md_file ~ml_files ~dirs ~root options =
       "\
 (alias\n\
 \ (name   %s)\n\
-\ (deps   (:x %s)%s)\n\
+\ (deps   (:x %s)\n\
+\         (package mdx)%s)\n\
 \ (action (progn\n\
-\           (run ocaml-mdx test %a %s%s%%{x})\n\
-\           (diff? %%{x} %%{x}.corrected)\n%a)))\n"
+\           (run ocaml-mdx test %a %s%s%%{x})\n%a\n\
+\           (diff? %%{x} %%{x}.corrected))))\n"
       name
       md_file
       deps

bin/test/main.ml

@@ -74,12 +74,24 @@ let with_dir root f =
       Sys.chdir old_d;
       raise e
 
-let run_test ?root temp_file t =
+let get_env blacklist =
+  let blacklist = "INSIDE_DUNE"::blacklist in
+  let env = Array.to_list (Unix.environment ()) in
+  let env = List.map (String.split_on_char '=') env in
+  let f env var =
+    let g l = String.compare (List.nth l 0) var <> 0 in
+    List.filter g env
+  in
+  let env = List.fold_left f env blacklist in
+  Array.of_list (List.map (String.concat "=") env)
+
+let run_test ?root blacklist temp_file t =
   let cmd = Cram.command_line t in
+  let env = get_env blacklist in
   Log.info (fun l -> l "exec: %S" cmd);
   let fd = Unix.openfile temp_file [O_WRONLY; O_TRUNC] 0 in
   let pid = with_dir root (fun () ->
-      Unix.create_process "sh" [| "sh"; "-c"; cmd |] Unix.stdin fd fd
+      Unix.create_process_env "sh" [| "sh"; "-c"; cmd |] env Unix.stdin fd fd
     ) in
   Unix.close fd;
   match snd (Unix.waitpid [] pid) with
@@ -102,7 +114,8 @@ let run_cram_tests ?syntax t ?root ppf temp_file pad tests =
   in
   List.iter (fun test ->
       let root = root_dir ?root t in
-      let n = run_test ?root temp_file test in
+      let blacklist = Block.unset_variables t in
+      let n = run_test ?root blacklist temp_file test in
       let lines = read_lines temp_file in
       let output =
         let output = List.map (fun x -> `Output x) lines in
@@ -182,9 +195,9 @@ type file = { first: Mdx_top.Part.file; current: Mdx_top.Part.file }
 
 let files: (string, file) Hashtbl.t = Hashtbl.create 8
 
-let has_changed { first; current } =
+let has_changed ~force_output { first; current } =
   let contents = Mdx_top.Part.contents current in
-  if contents = Mdx_top.Part.contents first
+  if contents = Mdx_top.Part.contents first && force_output = false
   then None
   else Some contents
 
@@ -196,9 +209,9 @@ let read_parts file =
     Hashtbl.add files file f;
     f
 
-let write_parts file parts =
+let write_parts ~force_output file parts =
   let output_file = file ^ ".corrected" in
-  match has_changed parts with
+  match has_changed ~force_output parts with
   | None   -> if Sys.file_exists output_file then Sys.remove output_file
   | Some c ->
     let oc = open_out output_file in
@@ -306,10 +319,12 @@ let run_exn ()
           | Section _
           | Text _ as t -> Mdx.pp_line ?syntax ppf t
           | Block t ->
-            List.iter (fun (k, v) -> Unix.putenv k v) (Block.variables t);
+            List.iter (fun (k, v) -> Unix.putenv k v) (Block.set_variables t);
               Mdx_top.in_env (Block.environment t)
               (fun () ->
-                 let active = active t && (not (Block.skip t)) in
+                 let active =
+                   active t && Block.version_enabled t && (not (Block.skip t))
+                 in
                  match active, non_deterministic, Block.mode t, Block.value t with
                  (* Print errors *)
                  | _, _, _, Error _ -> Block.pp ?syntax ppf t
@@ -323,8 +338,9 @@ let run_exn ()
                     non-deterministic; run it but keep the old output. *)
                  | true, false, `Non_det `Output, Cram { tests; _ } ->
                    Block.pp ?syntax ppf t;
+                   let blacklist = Block.unset_variables t in
                    List.iter (fun t ->
-                       let _: int = run_test ?root temp_file t in ()
+                       let _: int = run_test ?root blacklist temp_file t in ()
                      ) tests
                  | true, false, `Non_det `Output, Toplevel tests ->
                    assert (syntax <> Some Cram);
@@ -340,32 +356,28 @@ let run_exn ()
                  (* Run raw OCaml code *)
                  | true, _, _, OCaml ->
                    assert (syntax <> Some Cram);
-                   let version_enabled = Block.version_enabled t in
                    (match Block.file t with
-                    | Some ml_file when version_enabled ->
+                    | Some ml_file ->
                       update_file_or_block ?root ppf file ml_file t direction
-                    | None when version_enabled ->
+                    | None ->
                       eval_raw t ?root c ~line:t.line t.contents;
-                      Block.pp ppf t
-                    | _ -> Block.pp ppf t )
+                      Block.pp ppf t )
                  (* Cram tests. *)
                  | true, _, _, Cram { tests; pad } ->
                    run_cram_tests ?syntax t ?root ppf temp_file pad tests
                  (* Top-level tests. *)
                  | true, _, _, Toplevel tests ->
                    assert (syntax <> Some Cram);
-                   let version_enabled = Block.version_enabled t in
                    match Block.file t with
-                   | Some ml_file when version_enabled ->
+                   | Some ml_file ->
                      update_file_or_block ?root ppf file ml_file t direction
-                   | None when version_enabled ->
+                   | None ->
                      run_toplevel_tests ?root c ppf tests t
-                   | _ -> Block.pp ppf t
               )
         ) items;
       Format.pp_print_flush ppf ();
       Buffer.contents buf);
-  Hashtbl.iter write_parts files;
+  Hashtbl.iter (write_parts ~force_output) files;
   0
 
 let run ()