Pass stubs via the shared memory when parsing models and queries

Summary:
As opposed to copying them to each process. This should improve the
performance.

Reviewed By: arthaud

Differential Revision: D49426887

fbshipit-source-id: f479606d1ed7aece4e7f90cfac43814eafee1ae8

Author: Tianhan Lu

source/interprocedural_analyses/taint/modelParser.ml

@@ -3095,7 +3095,7 @@ let create_model_from_attribute
 let is_obscure ~definitions ~stubs call_target =
   (* The callable is obscure if and only if it is a type stub or it is not in the set of known
      definitions. *)
-  Hash_set.mem stubs call_target
+  Interprocedural.Target.HashsetSharedMemory.ReadOnly.mem stubs call_target
   || definitions >>| Core.Fn.flip Hash_set.mem call_target >>| not |> Option.value ~default:false
 
 

source/interprocedural_analyses/taint/modelParser.mli

@@ -28,7 +28,7 @@ val parse
   taint_configuration:TaintConfiguration.Heap.t ->
   source_sink_filter:SourceSinkFilter.t option ->
   definitions:Interprocedural.Target.HashSet.t option ->
-  stubs:Interprocedural.Target.HashSet.t ->
+  stubs:Interprocedural.Target.HashsetSharedMemory.ReadOnly.t ->
   python_version:PythonVersion.t ->
   unit ->
   ModelParseResult.t

source/interprocedural_analyses/taint/taintAnalysis.ml

@@ -165,7 +165,7 @@ let parse_models_and_queries_from_sources
     ~python_version
     sources
   =
-  (* TODO(T117715045): Do not pass all definitions and stubs explicitly to map_reduce,
+  (* TODO(T117715045): Do not pass all definitions explicitly to map_reduce,
    * since this will marshal-ed between processes and hence is costly. *)
   let map sources =
     let taint_configuration = TaintConfiguration.SharedMemory.get taint_configuration in
@@ -250,7 +250,7 @@ let initialize_models
       ~resolution
       ~source_sink_filter:taint_configuration.source_sink_filter
       ~definitions:(Some definitions_hashset)
-      ~stubs:stubs_hashset
+      ~stubs:(Interprocedural.Target.HashsetSharedMemory.read_only stubs_shared_memory)
   in
   Statistics.performance
     ~name:"Parsed taint models"

source/interprocedural_analyses/taint/test/forwardAnalysisTest.ml

@@ -45,7 +45,10 @@ let assert_taint ?models ?models_source ~context source expect =
         ~taint_configuration:TaintConfiguration.Heap.default
         ~source_sink_filter:None
         ~definitions:None
-        ~stubs:(Target.HashSet.create ())
+        ~stubs:
+          ([]
+          |> Interprocedural.Target.HashsetSharedMemory.from_heap
+          |> Interprocedural.Target.HashsetSharedMemory.read_only)
         ~python_version:ModelParser.PythonVersion.default
         ()
     in

source/interprocedural_analyses/taint/test/modelTest.ml

@@ -101,7 +101,10 @@ let set_up_environment
       ~taint_configuration
       ~source_sink_filter:(Some taint_configuration.source_sink_filter)
       ~definitions:None
-      ~stubs:(Target.HashSet.create ())
+      ~stubs:
+        ([]
+        |> Interprocedural.Target.HashsetSharedMemory.from_heap
+        |> Interprocedural.Target.HashsetSharedMemory.read_only)
       ~python_version:ModelParser.PythonVersion.default
       ()
   in
@@ -211,7 +214,10 @@ let assert_invalid_model ?path ?source ?(sources = []) ~context ~model_source ~e
       ?path
       ~source:(Test.trim_extra_indentation model_source)
       ~definitions:None
-      ~stubs:(Target.HashSet.create ())
+      ~stubs:
+        ([]
+        |> Interprocedural.Target.HashsetSharedMemory.from_heap
+        |> Interprocedural.Target.HashsetSharedMemory.read_only)
       ~python_version:ModelParser.PythonVersion.default
       ()
     |> fun { ModelParseResult.errors; _ } ->

source/interprocedural_analyses/taint/test/testHelper.ml

@@ -427,7 +427,10 @@ let get_initial_models ~context =
       ~taint_configuration:TaintConfiguration.Heap.default
       ~source_sink_filter:None
       ~definitions:None
-      ~stubs:(Target.HashSet.create ())
+      ~stubs:
+        ([]
+        |> Interprocedural.Target.HashsetSharedMemory.from_heap
+        |> Interprocedural.Target.HashsetSharedMemory.read_only)
       ~python_version:ModelParser.PythonVersion.default
       ()
   in
@@ -546,6 +549,7 @@ let initialize
     match models_source with
     | None -> Registry.empty, ModelQueryExecution.ModelQueryRegistryMap.empty
     | Some source ->
+        let stubs_shared_memory = Target.HashsetSharedMemory.from_heap stubs in
         let { ModelParseResult.models; errors; queries } =
           ModelParser.parse
             ~resolution:global_resolution
@@ -554,7 +558,7 @@ let initialize
             ~taint_configuration
             ~source_sink_filter:(Some taint_configuration.source_sink_filter)
             ~definitions:(Some (Target.HashSet.of_list definitions))
-            ~stubs:(Target.HashSet.of_list stubs)
+            ~stubs:(Target.HashsetSharedMemory.read_only stubs_shared_memory)
             ~python_version:ModelParser.PythonVersion.default
             ()
         in

source/server/query.ml

@@ -797,7 +797,10 @@ let rec process_request_exn ~type_environment ~build_system request =
                   ~taint_configuration
                   ~source_sink_filter:None
                   ~definitions:None
-                  ~stubs:(Interprocedural.Target.HashSet.create ())
+                  ~stubs:
+                    ([]
+                    |> Interprocedural.Target.HashsetSharedMemory.from_heap
+                    |> Interprocedural.Target.HashsetSharedMemory.read_only)
                   ~python_version
                   ()
                 |> fun { Taint.ModelParseResult.queries; errors; _ } ->
@@ -1019,7 +1022,10 @@ let rec process_request_exn ~type_environment ~build_system request =
               ~taint_configuration
               ~source_sink_filter:None
               ~definitions:None
-              ~stubs:(Interprocedural.Target.HashSet.create ())
+              ~stubs:
+                ([]
+                |> Interprocedural.Target.HashsetSharedMemory.from_heap
+                |> Interprocedural.Target.HashsetSharedMemory.read_only)
               ~python_version
               ()
             |> fun { Taint.ModelParseResult.errors; queries; _ } -> errors, queries