[AutoDiff upstream] Add differential operators and some utilities.

* Add all [differential operators](https://github.com/apple/swift/blob/master/docs/DifferentiableProgramming.md#list-of-differential-operators).
* Add `withoutDerivative(at:)`, used for efficiently stopping the derivative propagation at a value and causing the derivative at the value to be zero.
* Add utility `differentiableFunction(from:)`, used for creating a `@differentiable` function from an original function and a derivative function.

Mostly work done by @marcrasi and @dan-zheng.
Partially resolves TF-843.

TODO:
* Add `AnyDerivative`.
* Add `Array.differentiableMap(_:)` and `differentiableReduce(_:_:)`.

Author: rxwei

lib/SILGen/SILGenBuiltin.cpp

@@ -1106,14 +1106,9 @@ static ManagedValue emitBuiltinAutoDiffApplyTransposeFunction(
     origFnArgVals.push_back(arg.getValue());
 
   // Get the transpose function.
-  // TODO(TF-1142): Create a linear_function_extract instead of an undef.
-  auto fnTy = origFnVal->getType().castTo<SILFunctionType>();
-  auto transposeFnType =
-      fnTy->getWithoutDifferentiability()->getAutoDiffTransposeFunctionType(
-      fnTy->getDifferentiabilityParameterIndices(), SGF.SGM.M.Types,
-      LookUpConformanceInModule(SGF.SGM.M.getSwiftModule()));
-  SILValue transposeFn =
-      SILUndef::get(SILType::getPrimitiveObjectType(transposeFnType), SGF.F);
+  SILValue transposeFn = SGF.B.createLinearFunctionExtract(
+      loc, LinearDifferentiableFunctionTypeComponent::Transpose, origFnVal);
+  auto transposeFnType = transposeFn->getType().castTo<SILFunctionType>();
   auto transposeFnUnsubstType =
       transposeFnType->getUnsubstitutedType(SGF.getModule());
   if (transposeFnType != transposeFnUnsubstType) {
@@ -1204,19 +1199,16 @@ static ManagedValue emitBuiltinLinearFunction(
   assert(args.size() == 2);
   auto origFn = args.front();
   auto origType = origFn.getType().castTo<SILFunctionType>();
-  // TODO(TF-1142): Create a linear_function instead of an undef.
-  auto linearFnTy = origType->getWithDifferentiability(
-      DifferentiabilityKind::Linear,
+  auto linearFn = SGF.B.createLinearFunction(
+      loc,
       IndexSubset::getDefault(
-          SGF.getASTContext(), origType->getNumParameters(),
-          /*includeAll*/ true));
-  SILValue linearFn = SILUndef::get(
-      SILType::getPrimitiveObjectType(linearFnTy), SGF.F);
+          SGF.getASTContext(),
+          origType->getNumParameters(),
+          /*includeAll*/ true),
+      origFn.forward(SGF), args[1].forward(SGF));
   return SGF.emitManagedRValueWithCleanup(linearFn);
 }
 
-
-
 /// Emit SIL for the named builtin: globalStringTablePointer. Unlike the default
 /// ownership convention for named builtins, which is to take (non-trivial)
 /// arguments as Owned, this builtin accepts owned as well as guaranteed

lib/SILOptimizer/Transforms/SemanticARCOpts.cpp

@@ -776,6 +776,10 @@ struct SemanticARCOptVisitor
   FORWARDING_INST(OpenExistentialBoxValue)
   FORWARDING_INST(MarkDependence)
   FORWARDING_INST(InitExistentialRef)
+  FORWARDING_INST(DifferentiableFunction)
+  FORWARDING_INST(LinearFunction)
+  FORWARDING_INST(DifferentiableFunctionExtract)
+  FORWARDING_INST(LinearFunctionExtract)
 #undef FORWARDING_INST
 
 #define FORWARDING_TERM(NAME)                                                  \

stdlib/public/Differentiation/CMakeLists.txt

@@ -12,7 +12,12 @@
 
 add_swift_target_library(swift_Differentiation ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
   Differentiable.swift
+  DifferentialOperators.swift
+  DifferentiationUtilities.swift
 
-  SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+  SWIFT_COMPILE_FLAGS
+    ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+    -parse-stdlib
+    -Xfrontend -enable-experimental-differentiable-programming
   LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
   INSTALL_IN_COMPONENT stdlib)

stdlib/public/Differentiation/Differentiable.swift

@@ -20,6 +20,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+import Swift
+
 /// A type that mathematically represents a differentiable manifold whose
 /// tangent spaces are finite-dimensional.
 public protocol Differentiable {