From 17454b96f1e101b28480b553051120daaabefed1 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Sun, 19 May 2024 22:34:23 -0400
Subject: [PATCH 1/9] ADTypes v1.0

---
 ext/OptimizationMTKExt.jl            |  4 +-
 ext/OptimizationSparseFiniteDiff.jl  | 52 +++++++++++-----------
 ext/OptimizationSparseForwardDiff.jl | 66 ++++++++++++++--------------
 ext/OptimizationSparseReverseDiff.jl |  8 ++--
 4 files changed, 66 insertions(+), 64 deletions(-)

diff --git a/ext/OptimizationMTKExt.jl b/ext/OptimizationMTKExt.jl
index 7dab595..91f2b35 100644
--- a/ext/OptimizationMTKExt.jl
+++ b/ext/OptimizationMTKExt.jl
@@ -6,7 +6,7 @@ import OptimizationBase.SciMLBase: OptimizationFunction
 import OptimizationBase.ADTypes: AutoModelingToolkit
 isdefined(Base, :get_extension) ? (using ModelingToolkit) : (using ..ModelingToolkit)
 
-function OptimizationBase.instantiate_function(f, x, adtype::AutoModelingToolkit, p,
+function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSymbolics}, p,
         num_cons = 0)
     p = isnothing(p) ? SciMLBase.NullParameters() : p
 
@@ -53,7 +53,7 @@ function OptimizationBase.instantiate_function(f, x, adtype::AutoModelingToolkit
 end
 
 function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInitCache,
-        adtype::AutoModelingToolkit, num_cons = 0)
+        adtype::AutoSparse{<:AutoSymbolics}, num_cons = 0)
     p = isnothing(cache.p) ? SciMLBase.NullParameters() : cache.p
 
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, cache.u0,
diff --git a/ext/OptimizationSparseFiniteDiff.jl b/ext/OptimizationSparseFiniteDiff.jl
index 9963248..9f78292 100644
--- a/ext/OptimizationSparseFiniteDiff.jl
+++ b/ext/OptimizationSparseFiniteDiff.jl
@@ -1,6 +1,6 @@
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
-        adtype::AutoSparseFiniteDiff, p,
+        adtype::AutoSparse(<:AutoFiniteDiff), p,
         num_cons = 0)
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
@@ -19,8 +19,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = Symbolics.hessian_sparsity(_f, x)
-        hess_colors = matrix_colors(tril(hess_sparsity))
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
             ForwardColorHesCache(_f, x,
                 hess_colors,
@@ -70,8 +70,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
 
     conshess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
-        function gen_conshess_cache(_f, x)
-            conshess_sparsity = Symbolics.hessian_sparsity(_f, x)
+        function gen_conshess_cache(_f, x, i)
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors, conshess_sparsity)
             return hesscache
@@ -80,7 +80,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
         cons(_res, x);
         _res[i]) for i in 1:num_cons]
-        conshess_caches = gen_conshess_cache.(fcons, Ref(x))
+        conshess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
             for i in 1:num_cons
                 numauto_color_hessian!(res[i], fcons[i], θ, conshess_caches[i])
@@ -127,12 +127,14 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparseFiniteDiff, num_cons = 0)
+        adtype::AutoSparse(<:AutoFiniteDiff), num_cons = 0)
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
     _f = (θ, args...) -> first(f.f(θ, cache.p, args...))
-
+    x = cache.u0
+    p = cache.p
+    
     if f.grad === nothing
         gradcache = FD.GradientCache(cache.u0, cache.u0)
         grad = (res, θ, args...) -> FD.finite_difference_gradient!(res, x -> _f(x, args...),
@@ -144,17 +146,17 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = Symbolics.hessian_sparsity(_f, cache.u0)
-        hess_colors = matrix_colors(tril(hess_sparsity))
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
-            ForwardColorHesCache(_f, θ,
+            ForwardColorHesCache(_f, x,
                 hess_colors,
                 hess_sparsity,
                 (res, θ) -> grad(res,
                     θ,
                     args...)))
     else
-        hess = (H, θ, args...) -> f.hess(H, θ, cache.p, args...)
+        hess = (H, θ, args...) -> f.hess(H, θ, p, args...)
     end
 
     if f.hv === nothing
@@ -168,7 +170,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     if f.cons === nothing
         cons = nothing
     else
-        cons = (res, θ) -> f.cons(res, θ, cache.p)
+        cons = (res, θ) -> f.cons(res, θ, p)
     end
 
     cons_jac_prototype = f.cons_jac_prototype
@@ -176,45 +178,45 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     if cons !== nothing && f.cons_j === nothing
         cons_jac_prototype = f.cons_jac_prototype === nothing ?
                              Symbolics.jacobian_sparsity(cons,
-            zeros(eltype(cache.u0), num_cons),
-            cache.u0) :
+            zeros(eltype(x), num_cons),
+            x) :
                              f.cons_jac_prototype
         cons_jac_colorvec = f.cons_jac_colorvec === nothing ?
                             matrix_colors(cons_jac_prototype) :
                             f.cons_jac_colorvec
         cons_j = function (J, θ)
             y0 = zeros(num_cons)
-            jaccache = FD.JacobianCache(copy(θ), copy(y0), copy(y0);
+            jaccache = FD.JacobianCache(copy(x), copy(y0), copy(y0);
                 colorvec = cons_jac_colorvec,
                 sparsity = cons_jac_prototype)
             FD.finite_difference_jacobian!(J, cons, θ, jaccache)
         end
     else
-        cons_j = (J, θ) -> f.cons_j(J, θ, cache.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, p)
     end
 
     conshess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
-        function gen_conshess_cache(_f, x)
-            conshess_sparsity = copy(Symbolics.hessian_sparsity(_f, x))
+        function gen_conshess_cache(_f, x, i)
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
-            hesscache = ForwardColorHesCache(_f, x, conshess_colors,
-                conshess_sparsity)
+            hesscache = ForwardColorHesCache(_f, x, conshess_colors, conshess_sparsity)
             return hesscache
         end
 
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
         cons(_res, x);
         _res[i]) for i in 1:num_cons]
-        conshess_caches = [gen_conshess_cache(fcons[i], cache.u0) for i in 1:num_cons]
+        conshess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
             for i in 1:num_cons
                 numauto_color_hessian!(res[i], fcons[i], θ, conshess_caches[i])
             end
         end
     else
-        cons_h = (res, θ) -> f.cons_h(res, θ, cache.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, p)
     end
+
     if f.lag_h === nothing
         # lag_hess_cache = FD.HessianCache(copy(cache.u0))
         # c = zeros(num_cons)
@@ -256,7 +258,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
 end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x,
-        adtype::AutoSparseFiniteDiff, p,
+        adtype::AutoSparse(<:AutoFiniteDiff), p,
         num_cons = 0)
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
@@ -389,7 +391,7 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparseFiniteDiff, num_cons = 0)
+        adtype::AutoSparse(<:AutoFiniteDiff), num_cons = 0)
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
diff --git a/ext/OptimizationSparseForwardDiff.jl b/ext/OptimizationSparseForwardDiff.jl
index 97cde29..573e18f 100644
--- a/ext/OptimizationSparseForwardDiff.jl
+++ b/ext/OptimizationSparseForwardDiff.jl
@@ -1,5 +1,5 @@
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
-        adtype::AutoSparseForwardDiff{_chunksize}, p,
+        adtype::AutoSparse{<:AutoForwardDiff{_chunksize}}, p,
         num_cons = 0) where {_chunksize}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
@@ -19,8 +19,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = Symbolics.hessian_sparsity(_f, x)
-        hess_colors = matrix_colors(tril(hess_sparsity))
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
             ForwardColorHesCache(_f, x,
                 hess_colors,
@@ -50,8 +50,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     cons_jac_prototype = f.cons_jac_prototype
     cons_jac_colorvec = f.cons_jac_colorvec
     if cons !== nothing && f.cons_j === nothing
-        cons_jac_prototype = Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
-            x)
+        cons_jac_prototype = isnothing(f.cons_jac_prototype) ? Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
+            x) : f.cons_jac_prototype
         cons_jac_colorvec = matrix_colors(cons_jac_prototype)
         jaccache = ForwardColorJacCache(cons,
             x,
@@ -68,8 +68,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
 
     cons_hess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
-        function gen_conshess_cache(_f, x)
-            conshess_sparsity = copy(Symbolics.hessian_sparsity(_f, x))
+        function gen_conshess_cache(_f, x, i)
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors,
                 conshess_sparsity)
@@ -79,11 +79,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
         cons(_res, x);
         _res[i]) for i in 1:num_cons]
-        cons_hess_caches = gen_conshess_cache.(fcons, Ref(x))
+        cons_hess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
-            for i in 1:num_cons
-                numauto_color_hessian!(res[i], fcons[i], θ, cons_hess_caches[i])
-            end
+            fetch.([Threads.@spawn numauto_color_hessian!(res[i], fcons[i], θ, cons_hess_caches[i]) for i in 1:num_cons])
         end
     else
         cons_h = (res, θ) -> f.cons_h(res, θ, p)
@@ -107,12 +105,15 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparseForwardDiff{_chunksize},
+        adtype::AutoSparse{<:AutoForwardDiff{_chunksize}},
         num_cons = 0) where {_chunksize}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
     chunksize = _chunksize === nothing ? default_chunk_size(length(cache.u0)) : _chunksize
+    
+    x = cache.u0
+    p = cache.p
 
     _f = (θ, args...) -> first(f.f(θ, cache.p, args...))
 
@@ -127,17 +128,17 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = Symbolics.hessian_sparsity(_f, cache.u0)
-        hess_colors = matrix_colors(tril(hess_sparsity))
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
-            ForwardColorHesCache(_f, θ,
+            ForwardColorHesCache(_f, x,
                 hess_colors,
                 hess_sparsity,
                 (res, θ) -> grad(res,
                     θ,
                     args...)))
     else
-        hess = (H, θ, args...) -> f.hess(H, θ, cache.p, args...)
+        hess = (H, θ, args...) -> f.hess(H, θ, p, args...)
     end
 
     if f.hv === nothing
@@ -151,33 +152,34 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     if f.cons === nothing
         cons = nothing
     else
-        cons = (res, θ) -> f.cons(res, θ, cache.p)
+        cons = (res, θ) -> f.cons(res, θ, p)
         cons_oop = (x) -> (_res = zeros(eltype(x), num_cons); cons(_res, x); _res)
     end
 
     cons_jac_prototype = f.cons_jac_prototype
     cons_jac_colorvec = f.cons_jac_colorvec
     if cons !== nothing && f.cons_j === nothing
-        cons_jac_prototype = Symbolics.jacobian_sparsity(cons,
-            zeros(eltype(cache.u0), num_cons),
-            cache.u0)
+        cons_jac_prototype = isnothing(f.cons_jac_prototype) ? Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
+            x) : f.cons_jac_prototype
         cons_jac_colorvec = matrix_colors(cons_jac_prototype)
-        jaccache = ForwardColorJacCache(cons, cache.u0, chunksize;
+        jaccache = ForwardColorJacCache(cons,
+            x,
+            chunksize;
             colorvec = cons_jac_colorvec,
             sparsity = cons_jac_prototype,
-            dx = zeros(eltype(cache.u0), num_cons))
+            dx = zeros(eltype(x), num_cons))
         cons_j = function (J, θ)
             forwarddiff_color_jacobian!(J, cons, θ, jaccache)
         end
     else
-        cons_j = (J, θ) -> f.cons_j(J, θ, cache.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, p)
     end
 
     cons_hess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
-        function gen_conshess_cache(_f, x)
-            conshess_sparsity = copy(Symbolics.hessian_sparsity(_f, x))
-            conshess_colors = matrix_colors(tril(conshess_sparsity))
+        function gen_conshess_cache(_f, x, i)
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
+            conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors,
                 conshess_sparsity)
             return hesscache
@@ -186,14 +188,12 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
         cons(_res, x);
         _res[i]) for i in 1:num_cons]
-        cons_hess_caches = gen_conshess_cache.(fcons, Ref(cache.u0))
+        cons_hess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
-            for i in 1:num_cons
-                numauto_color_hessian!(res[i], fcons[i], θ, cons_hess_caches[i])
-            end
+            fetch.([Threads.@spawn numauto_color_hessian!(res[i], fcons[i], θ, cons_hess_caches[i]) for i in 1:num_cons])
         end
     else
-        cons_h = (res, θ) -> f.cons_h(res, θ, cache.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, p)
     end
 
     if f.lag_h === nothing
@@ -214,7 +214,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
 end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x,
-        adtype::AutoSparseForwardDiff{_chunksize}, p,
+        adtype::AutoSparse{<:AutoForwardDiff{_chunksize}}, p,
         num_cons = 0) where {_chunksize}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
@@ -323,7 +323,7 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparseForwardDiff{_chunksize},
+        adtype::AutoSparse{<:AutoForwardDiff{_chunksize}},
         num_cons = 0) where {_chunksize}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
diff --git a/ext/OptimizationSparseReverseDiff.jl b/ext/OptimizationSparseReverseDiff.jl
index 1f32574..20dced8 100644
--- a/ext/OptimizationSparseReverseDiff.jl
+++ b/ext/OptimizationSparseReverseDiff.jl
@@ -1,5 +1,5 @@
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
-        adtype::AutoSparseReverseDiff,
+        adtype::AutoSparse{<:AutoReverseDiff},
         p = SciMLBase.NullParameters(),
         num_cons = 0)
     _f = (θ, args...) -> first(f.f(θ, p, args...))
@@ -176,7 +176,7 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparseReverseDiff, num_cons = 0)
+        adtype::AutoSparse{<:AutoReverseDiff}, num_cons = 0)
     _f = (θ, args...) -> first(f.f(θ, cache.p, args...))
 
     chunksize = default_chunk_size(length(cache.u0))
@@ -374,7 +374,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
 end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x,
-        adtype::AutoSparseReverseDiff,
+        adtype::AutoSparse{<:AutoReverseDiff},
         p = SciMLBase.NullParameters(),
         num_cons = 0)
     _f = (θ, args...) -> first(f.f(θ, p, args...))
@@ -551,7 +551,7 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparseReverseDiff, num_cons = 0)
+        adtype::AutoSparse{<:AutoReverseDiff}, num_cons = 0)
     _f = (θ, args...) -> first(f.f(θ, cache.p, args...))
 
     chunksize = default_chunk_size(length(cache.u0))

From 33d80d2ee476b85583cefa8e63e9adc00c6ea5ae Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 10:42:25 -0400
Subject: [PATCH 2/9] Actual ADTypes v1.0 updates

---
 ext/OptimizationMTKExt.jl           | 16 ++++++++--------
 ext/OptimizationSparseDiffExt.jl    |  3 +--
 ext/OptimizationSparseFiniteDiff.jl | 12 ++++++------
 3 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/ext/OptimizationMTKExt.jl b/ext/OptimizationMTKExt.jl
index 91f2b35..27ef948 100644
--- a/ext/OptimizationMTKExt.jl
+++ b/ext/OptimizationMTKExt.jl
@@ -3,11 +3,11 @@ module OptimizationMTKExt
 import OptimizationBase, OptimizationBase.ArrayInterface
 import OptimizationBase.SciMLBase
 import OptimizationBase.SciMLBase: OptimizationFunction
-import OptimizationBase.ADTypes: AutoModelingToolkit
+import OptimizationBase.ADTypes: AutoModelingToolkit, AutoSymbolics, AutoSparse
 isdefined(Base, :get_extension) ? (using ModelingToolkit) : (using ..ModelingToolkit)
 
-function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSymbolics}, p,
-        num_cons = 0)
+function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSymbolics, S, C}, p,
+        num_cons = 0) where {S, C}
     p = isnothing(p) ? SciMLBase.NullParameters() : p
 
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, x, p;
@@ -17,8 +17,8 @@ function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSy
             num_cons))))
     #sys = ModelingToolkit.structural_simplify(sys)
     f = OptimizationProblem(sys, x, p, grad = true, hess = true,
-        sparse = adtype.obj_sparse, cons_j = true, cons_h = true,
-        cons_sparse = adtype.cons_sparse).f
+        sparse = false, cons_j = true, cons_h = true,
+        cons_sparse = false).f
 
     grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
 
@@ -53,7 +53,7 @@ function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSy
 end
 
 function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparse{<:AutoSymbolics}, num_cons = 0)
+        adtype::AutoSparse{<:AutoSymbolics, S, C}, num_cons = 0) where {S, C}
     p = isnothing(cache.p) ? SciMLBase.NullParameters() : cache.p
 
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, cache.u0,
@@ -64,8 +64,8 @@ function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInit
             num_cons))))
     #sys = ModelingToolkit.structural_simplify(sys)
     f = OptimizationProblem(sys, cache.u0, cache.p, grad = true, hess = true,
-        sparse = adtype.obj_sparse, cons_j = true, cons_h = true,
-        cons_sparse = adtype.cons_sparse).f
+        sparse = false, cons_j = true, cons_h = true,
+        cons_sparse = false).f
 
     grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
 
diff --git a/ext/OptimizationSparseDiffExt.jl b/ext/OptimizationSparseDiffExt.jl
index 299b7e5..d9955e4 100644
--- a/ext/OptimizationSparseDiffExt.jl
+++ b/ext/OptimizationSparseDiffExt.jl
@@ -2,8 +2,7 @@ module OptimizationSparseDiffExt
 
 import OptimizationBase, OptimizationBase.ArrayInterface
 import OptimizationBase.SciMLBase: OptimizationFunction
-import OptimizationBase.ADTypes: AutoSparseForwardDiff,
-    AutoSparseFiniteDiff, AutoSparseReverseDiff
+import OptimizationBase.ADTypes: AutoSparse, AutoFiniteDiff, AutoForwardDiff, AutoReverseDiff
 using OptimizationBase.LinearAlgebra, ReverseDiff
 isdefined(Base, :get_extension) ?
 (using SparseDiffTools,
diff --git a/ext/OptimizationSparseFiniteDiff.jl b/ext/OptimizationSparseFiniteDiff.jl
index 9f78292..4370467 100644
--- a/ext/OptimizationSparseFiniteDiff.jl
+++ b/ext/OptimizationSparseFiniteDiff.jl
@@ -1,7 +1,7 @@
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
-        adtype::AutoSparse(<:AutoFiniteDiff), p,
-        num_cons = 0)
+        adtype::AutoSparse{<:AutoFiniteDiff, S, C}, p,
+        num_cons = 0) where {S, C}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
@@ -127,7 +127,7 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparse(<:AutoFiniteDiff), num_cons = 0)
+        adtype::AutoSparse{<:AutoFiniteDiff, S, C}, num_cons = 0) where {S, C}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
@@ -258,8 +258,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
 end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x,
-        adtype::AutoSparse(<:AutoFiniteDiff), p,
-        num_cons = 0)
+        adtype::AutoSparse{<:AutoFiniteDiff, S, C}, p,
+        num_cons = 0) where {S, C}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
@@ -391,7 +391,7 @@ end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
         cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparse(<:AutoFiniteDiff), num_cons = 0)
+        adtype::AutoSparse{<:AutoFiniteDiff, S, C}, num_cons = 0) where {S, C}
     if maximum(getfield.(methods(f.f), :nargs)) > 3
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end

From a6b8f88c097b20eeaeba5d3605e261db9a8878e1 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 10:55:02 -0400
Subject: [PATCH 3/9] Downstream CI

---
 .github/workflows/Downstream.yml | 54 ++++++++++++++++++++++++++++++++
 1 file changed, 54 insertions(+)
 create mode 100644 .github/workflows/Downstream.yml

diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml
new file mode 100644
index 0000000..1052b99
--- /dev/null
+++ b/.github/workflows/Downstream.yml
@@ -0,0 +1,54 @@
+name: IntegrationTest
+on:
+  push:
+    branches: [master]
+    tags: [v*]
+  pull_request:
+
+jobs:
+  test:
+    name: ${{ matrix.package.repo }}/${{ matrix.package.group }}/${{ matrix.julia-version }}
+    runs-on: ${{ matrix.os }}
+    env:
+      GROUP: ${{ matrix.package.group }}
+    strategy:
+      fail-fast: false
+      matrix:
+        julia-version: [1]
+        os: [ubuntu-latest]
+        package:
+          - {user: SciML, repo: Optimization.jl, group: Optimization}
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
+        with:
+          version: ${{ matrix.julia-version }}
+          arch: x64
+      - uses: julia-actions/julia-buildpkg@latest
+      - name: Clone Downstream
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.package.user }}/${{ matrix.package.repo }}
+          path: downstream
+      - name: Load this and run the downstream tests
+        shell: julia --color=yes --project=downstream {0}
+        run: |
+          using Pkg
+          try
+            # force it to use this PR's version of the package
+            Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
+            Pkg.update()
+            Pkg.test(coverage=true)  # resolver may fail with test time deps
+          catch err
+            err isa Pkg.Resolve.ResolverError || rethrow()
+            # If we can't resolve that means this is incompatible by SemVer and this is fine
+            # It means we marked this as a breaking change, so we don't need to worry about
+            # Mistakenly introducing a breaking change, as we have intentionally made one
+            @info "Not compatible with this release. No problem." exception=err
+            exit(0)  # Exit immediately, as a success
+          end
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v4
+        with:
+          file: lcov.info

From 935e89c0a7c635750a950f2b73d6618cbc1a7453 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 12:44:15 -0400
Subject: [PATCH 4/9] Some methods for constructors

---
 ext/OptimizationMTKExt.jl            | 103 ++++++++++++++++++++++++++-
 ext/OptimizationSparseReverseDiff.jl |  28 ++++----
 2 files changed, 118 insertions(+), 13 deletions(-)

diff --git a/ext/OptimizationMTKExt.jl b/ext/OptimizationMTKExt.jl
index 27ef948..4611d87 100644
--- a/ext/OptimizationMTKExt.jl
+++ b/ext/OptimizationMTKExt.jl
@@ -6,10 +6,111 @@ import OptimizationBase.SciMLBase: OptimizationFunction
 import OptimizationBase.ADTypes: AutoModelingToolkit, AutoSymbolics, AutoSparse
 isdefined(Base, :get_extension) ? (using ModelingToolkit) : (using ..ModelingToolkit)
 
+function OptimizationBase.ADTypes.AutoModelingToolkit(sparse = false, cons_sparse = false) 
+    if sparse || cons_sparse
+        return AutoSparse(AutoSymbolics())
+    else
+        return AutoSymbolics()
+    end
+end
+
 function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSymbolics, S, C}, p,
         num_cons = 0) where {S, C}
     p = isnothing(p) ? SciMLBase.NullParameters() : p
 
+    sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, x, p;
+        lcons = fill(0.0,
+            num_cons),
+        ucons = fill(0.0,
+            num_cons))))
+    #sys = ModelingToolkit.structural_simplify(sys)
+    f = OptimizationProblem(sys, x, p, grad = true, hess = true,
+        sparse = true, cons_j = true, cons_h = true,
+        cons_sparse = true).f
+
+    grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
+
+    hess = (H, θ, args...) -> f.hess(H, θ, p, args...)
+
+    hv = function (H, θ, v, args...)
+        res = adtype.obj_sparse ? (eltype(θ)).(f.hess_prototype) :
+              ArrayInterface.zeromatrix(θ)
+        hess(res, θ, args...)
+        H .= res * v
+    end
+
+    if !isnothing(f.cons)
+        cons = (res, θ) -> f.cons(res, θ, p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, p)
+    else
+        cons = nothing
+        cons_j = nothing
+        cons_h = nothing
+    end
+
+    return OptimizationFunction{true}(f.f, adtype; grad = grad, hess = hess, hv = hv,
+        cons = cons, cons_j = cons_j, cons_h = cons_h,
+        hess_prototype = f.hess_prototype,
+        cons_jac_prototype = f.cons_jac_prototype,
+        cons_hess_prototype = f.cons_hess_prototype,
+        expr = OptimizationBase.symbolify(f.expr),
+        cons_expr = OptimizationBase.symbolify.(f.cons_expr),
+        sys = sys,
+        observed = f.observed)
+end
+
+function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInitCache,
+        adtype::AutoSparse{<:AutoSymbolics, S, C}, num_cons = 0) where {S, C}
+    p = isnothing(cache.p) ? SciMLBase.NullParameters() : cache.p
+
+    sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, cache.u0,
+        cache.p;
+        lcons = fill(0.0,
+            num_cons),
+        ucons = fill(0.0,
+            num_cons))))
+    #sys = ModelingToolkit.structural_simplify(sys)
+    f = OptimizationProblem(sys, cache.u0, cache.p, grad = true, hess = true,
+        sparse = true, cons_j = true, cons_h = true,
+        cons_sparse = true).f
+
+    grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
+
+    hess = (H, θ, args...) -> f.hess(H, θ, cache.p, args...)
+
+    hv = function (H, θ, v, args...)
+        res = adtype.obj_sparse ? (eltype(θ)).(f.hess_prototype) :
+              ArrayInterface.zeromatrix(θ)
+        hess(res, θ, args...)
+        H .= res * v
+    end
+
+    if !isnothing(f.cons)
+        cons = (res, θ) -> f.cons(res, θ, cache.p)
+        cons_j = (J, θ) -> f.cons_j(J, θ, cache.p)
+        cons_h = (res, θ) -> f.cons_h(res, θ, cache.p)
+    else
+        cons = nothing
+        cons_j = nothing
+        cons_h = nothing
+    end
+
+    return OptimizationFunction{true}(f.f, adtype; grad = grad, hess = hess, hv = hv,
+        cons = cons, cons_j = cons_j, cons_h = cons_h,
+        hess_prototype = f.hess_prototype,
+        cons_jac_prototype = f.cons_jac_prototype,
+        cons_hess_prototype = f.cons_hess_prototype,
+        expr = OptimizationBase.symbolify(f.expr),
+        cons_expr = OptimizationBase.symbolify.(f.cons_expr),
+        sys = sys,
+        observed = f.observed)
+end
+
+function OptimizationBase.instantiate_function(f, x, adtype::AutoSymbolics, p,
+    num_cons = 0)
+    p = isnothing(p) ? SciMLBase.NullParameters() : p
+
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, x, p;
         lcons = fill(0.0,
             num_cons),
@@ -53,7 +154,7 @@ function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSy
 end
 
 function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInitCache,
-        adtype::AutoSparse{<:AutoSymbolics, S, C}, num_cons = 0) where {S, C}
+    adtype::AutoSymbolics, num_cons = 0)
     p = isnothing(cache.p) ? SciMLBase.NullParameters() : cache.p
 
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, cache.u0,
diff --git a/ext/OptimizationSparseReverseDiff.jl b/ext/OptimizationSparseReverseDiff.jl
index 20dced8..b21b1af 100644
--- a/ext/OptimizationSparseReverseDiff.jl
+++ b/ext/OptimizationSparseReverseDiff.jl
@@ -1,3 +1,7 @@
+function OptimizationBase.ADTypes.AutoSparseReverseDiff(compile::Bool)
+    return AutoSparse(AutoReverseDiff(; compile)) 
+end
+
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
         adtype::AutoSparse{<:AutoReverseDiff},
         p = SciMLBase.NullParameters(),
@@ -7,7 +11,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     chunksize = default_chunk_size(length(x))
 
     if f.grad === nothing
-        if adtype.compile
+        if adtype.dense_ad.compile
             _tape = ReverseDiff.GradientTape(_f, x)
             tape = ReverseDiff.compile(_tape)
             grad = function (res, θ, args...)
@@ -29,7 +33,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     if f.hess === nothing
         hess_sparsity = Symbolics.hessian_sparsity(_f, x)
         hess_colors = SparseDiffTools.matrix_colors(tril(hess_sparsity))
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xdual = ForwardDiff.Dual{
                 typeof(T),
@@ -115,7 +119,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         conshess_sparsity = Symbolics.hessian_sparsity.(fncs, Ref(x))
         conshess_colors = SparseDiffTools.matrix_colors.(conshess_sparsity)
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xduals = [ForwardDiff.Dual{
                 typeof(T),
@@ -182,7 +186,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     chunksize = default_chunk_size(length(cache.u0))
 
     if f.grad === nothing
-        if adtype.compile
+        if adtype.dense_ad.compile
             _tape = ReverseDiff.GradientTape(_f, cache.u0)
             tape = ReverseDiff.compile(_tape)
             grad = function (res, θ, args...)
@@ -204,7 +208,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     if f.hess === nothing
         hess_sparsity = Symbolics.hessian_sparsity(_f, cache.u0)
         hess_colors = SparseDiffTools.matrix_colors(tril(hess_sparsity))
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(cache.u0))
             xdual = ForwardDiff.Dual{
                 typeof(T),
@@ -307,7 +311,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
             end
         end
         conshess_colors = SparseDiffTools.matrix_colors.(conshess_sparsity)
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(cache.u0))
             xduals = [ForwardDiff.Dual{
                 typeof(T),
@@ -382,7 +386,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
     chunksize = default_chunk_size(length(x))
 
     if f.grad === nothing
-        if adtype.compile
+        if adtype.dense_ad.compile
             _tape = ReverseDiff.GradientTape(_f, x)
             tape = ReverseDiff.compile(_tape)
             grad = function (θ, args...)
@@ -403,7 +407,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
     if f.hess === nothing
         hess_sparsity = Symbolics.hessian_sparsity(_f, x)
         hess_colors = SparseDiffTools.matrix_colors(tril(hess_sparsity))
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xdual = ForwardDiff.Dual{
                 typeof(T),
@@ -491,7 +495,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         conshess_sparsity = Symbolics.hessian_sparsity.(fncs, Ref(x))
         conshess_colors = SparseDiffTools.matrix_colors.(conshess_sparsity)
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xduals = [ForwardDiff.Dual{
                 typeof(T),
@@ -559,7 +563,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
     x = cache.u0
 
     if f.grad === nothing
-        if adtype.compile
+        if adtype.dense_ad.compile
             _tape = ReverseDiff.GradientTape(_f, x)
             tape = ReverseDiff.compile(_tape)
             grad = function (θ, args...)
@@ -580,7 +584,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
     if f.hess === nothing
         hess_sparsity = Symbolics.hessian_sparsity(_f, x)
         hess_colors = SparseDiffTools.matrix_colors(tril(hess_sparsity))
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xdual = ForwardDiff.Dual{
                 typeof(T),
@@ -668,7 +672,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         conshess_sparsity = Symbolics.hessian_sparsity.(fncs, Ref(x))
         conshess_colors = SparseDiffTools.matrix_colors.(conshess_sparsity)
-        if adtype.compile
+        if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xduals = [ForwardDiff.Dual{
                 typeof(T),

From 057992b0711938b7c3eda682bc35df8e136da049 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 12:48:38 -0400
Subject: [PATCH 5/9] Only 1

---
 Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Project.toml b/Project.toml
index 1d9b764..47bbb59 100644
--- a/Project.toml
+++ b/Project.toml
@@ -35,7 +35,7 @@ OptimizationTrackerExt = "Tracker"
 OptimizationZygoteExt = "Zygote"
 
 [compat]
-ADTypes = "0.2.5, 1"
+ADTypes = "1"
 ArrayInterface = "7.6"
 DocStringExtensions = "0.9"
 Enzyme = "0.11.11, 0.12"

From 07cb4dd6c5c5ca2e14609ac551a3b8d5e10dd7d4 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 13:36:17 -0400
Subject: [PATCH 6/9] Enzyme bound

---
 Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Project.toml b/Project.toml
index 47bbb59..32d328e 100644
--- a/Project.toml
+++ b/Project.toml
@@ -38,7 +38,7 @@ OptimizationZygoteExt = "Zygote"
 ADTypes = "1"
 ArrayInterface = "7.6"
 DocStringExtensions = "0.9"
-Enzyme = "0.11.11, 0.12"
+Enzyme = "0.11.11, =0.12.5"
 FiniteDiff = "2.12"
 ForwardDiff = "0.10.26"
 LinearAlgebra = "1.9, 1.10"

From fa0f12904a5df5938cc160b9038a3923c62575e6 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 13:48:46 -0400
Subject: [PATCH 7/9] format

---
 ext/OptimizationFiniteDiffExt.jl     |  6 +-
 ext/OptimizationForwardDiffExt.jl    |  8 +--
 ext/OptimizationMTKExt.jl            |  9 +--
 ext/OptimizationReverseDiffExt.jl    | 32 +++++-----
 ext/OptimizationSparseDiffExt.jl     |  7 ++-
 ext/OptimizationSparseFiniteDiff.jl  | 30 +++++----
 ext/OptimizationSparseForwardDiff.jl | 36 +++++++----
 ext/OptimizationSparseReverseDiff.jl | 94 +++++++++++++++-------------
 ext/OptimizationTrackerExt.jl        |  6 +-
 src/OptimizationBase.jl              |  6 +-
 test/adtests.jl                      | 35 +++++++----
 11 files changed, 154 insertions(+), 115 deletions(-)

diff --git a/ext/OptimizationFiniteDiffExt.jl b/ext/OptimizationFiniteDiffExt.jl
index 6dc6f48..707e608 100644
--- a/ext/OptimizationFiniteDiffExt.jl
+++ b/ext/OptimizationFiniteDiffExt.jl
@@ -16,7 +16,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
 
     if f.grad === nothing
         gradcache = FD.GradientCache(x, x, adtype.fdtype)
-        grad = (res, θ, args...) -> FD.finite_difference_gradient!(res, x -> _f(x, args...),
+        grad = (res, θ, args...) -> FD.finite_difference_gradient!(
+            res, x -> _f(x, args...),
             θ, gradcache)
     else
         grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
@@ -125,7 +126,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
 
     if f.grad === nothing
         gradcache = FD.GradientCache(cache.u0, cache.u0, adtype.fdtype)
-        grad = (res, θ, args...) -> FD.finite_difference_gradient!(res, x -> _f(x, args...),
+        grad = (res, θ, args...) -> FD.finite_difference_gradient!(
+            res, x -> _f(x, args...),
             θ, gradcache)
     else
         grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
diff --git a/ext/OptimizationForwardDiffExt.jl b/ext/OptimizationForwardDiffExt.jl
index 256cebc..7d75cc9 100644
--- a/ext/OptimizationForwardDiffExt.jl
+++ b/ext/OptimizationForwardDiffExt.jl
@@ -65,7 +65,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     if cons !== nothing && f.cons_h === nothing
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         hess_config_cache = [ForwardDiff.HessianConfig(fncs[i], x,
-            ForwardDiff.Chunk{chunksize}())
+                                 ForwardDiff.Chunk{chunksize}())
                              for i in 1:num_cons]
         cons_h = function (res, θ)
             for i in 1:num_cons
@@ -143,7 +143,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     if cons !== nothing && f.cons_h === nothing
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         hess_config_cache = [ForwardDiff.HessianConfig(fncs[i], cache.u0,
-            ForwardDiff.Chunk{chunksize}())
+                                 ForwardDiff.Chunk{chunksize}())
                              for i in 1:num_cons]
         cons_h = function (res, θ)
             for i in 1:num_cons
@@ -224,7 +224,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
     if cons !== nothing && f.cons_h === nothing
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         hess_config_cache = [ForwardDiff.HessianConfig(fncs[i], x,
-            ForwardDiff.Chunk{chunksize}())
+                                 ForwardDiff.Chunk{chunksize}())
                              for i in 1:num_cons]
         cons_h = function (θ)
             map(1:num_cons) do i
@@ -306,7 +306,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
     if cons !== nothing && f.cons_h === nothing
         fncs = [(x) -> cons_oop(x)[i] for i in 1:num_cons]
         hess_config_cache = [ForwardDiff.HessianConfig(fncs[i], x,
-            ForwardDiff.Chunk{chunksize}())
+                                 ForwardDiff.Chunk{chunksize}())
                              for i in 1:num_cons]
         cons_h = function (θ)
             map(1:num_cons) do i
diff --git a/ext/OptimizationMTKExt.jl b/ext/OptimizationMTKExt.jl
index 4611d87..2f75933 100644
--- a/ext/OptimizationMTKExt.jl
+++ b/ext/OptimizationMTKExt.jl
@@ -6,7 +6,7 @@ import OptimizationBase.SciMLBase: OptimizationFunction
 import OptimizationBase.ADTypes: AutoModelingToolkit, AutoSymbolics, AutoSparse
 isdefined(Base, :get_extension) ? (using ModelingToolkit) : (using ..ModelingToolkit)
 
-function OptimizationBase.ADTypes.AutoModelingToolkit(sparse = false, cons_sparse = false) 
+function OptimizationBase.ADTypes.AutoModelingToolkit(sparse = false, cons_sparse = false)
     if sparse || cons_sparse
         return AutoSparse(AutoSymbolics())
     else
@@ -14,7 +14,8 @@ function OptimizationBase.ADTypes.AutoModelingToolkit(sparse = false, cons_spars
     end
 end
 
-function OptimizationBase.instantiate_function(f, x, adtype::AutoSparse{<:AutoSymbolics, S, C}, p,
+function OptimizationBase.instantiate_function(
+        f, x, adtype::AutoSparse{<:AutoSymbolics, S, C}, p,
         num_cons = 0) where {S, C}
     p = isnothing(p) ? SciMLBase.NullParameters() : p
 
@@ -108,7 +109,7 @@ function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInit
 end
 
 function OptimizationBase.instantiate_function(f, x, adtype::AutoSymbolics, p,
-    num_cons = 0)
+        num_cons = 0)
     p = isnothing(p) ? SciMLBase.NullParameters() : p
 
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, x, p;
@@ -154,7 +155,7 @@ function OptimizationBase.instantiate_function(f, x, adtype::AutoSymbolics, p,
 end
 
 function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInitCache,
-    adtype::AutoSymbolics, num_cons = 0)
+        adtype::AutoSymbolics, num_cons = 0)
     p = isnothing(cache.p) ? SciMLBase.NullParameters() : cache.p
 
     sys = complete(ModelingToolkit.modelingtoolkitize(OptimizationProblem(f, cache.u0,
diff --git a/ext/OptimizationReverseDiffExt.jl b/ext/OptimizationReverseDiffExt.jl
index 83b6400..d6c3a1a 100644
--- a/ext/OptimizationReverseDiffExt.jl
+++ b/ext/OptimizationReverseDiffExt.jl
@@ -49,7 +49,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(x),
-                chunksize,
+                chunksize
             }.(x, Ref(ForwardDiff.Partials((ones(eltype(x), chunksize)...,))))
             h_tape = ReverseDiff.GradientTape(_f, xdual)
             htape = ReverseDiff.compile(h_tape)
@@ -119,9 +119,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
             end
             gs = [x -> grad_cons(x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardDiff.JacobianConfig(gs[i],
-                x,
-                ForwardDiff.Chunk{chunksize}(),
-                T) for i in 1:num_cons]
+                           x,
+                           ForwardDiff.Chunk{chunksize}(),
+                           T) for i in 1:num_cons]
             cons_h = function (res, θ)
                 for i in 1:num_cons
                     ForwardDiff.jacobian!(res[i], gs[i], θ, jaccfgs[i], Val{false}())
@@ -182,7 +182,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(cache.u0),
-                chunksize,
+                chunksize
             }.(cache.u0, Ref(ForwardDiff.Partials((ones(eltype(cache.u0), chunksize)...,))))
             h_tape = ReverseDiff.GradientTape(_f, xdual)
             htape = ReverseDiff.compile(h_tape)
@@ -255,9 +255,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
             end
             gs = [x -> grad_cons(x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardDiff.JacobianConfig(gs[i],
-                cache.u0,
-                ForwardDiff.Chunk{chunksize}(),
-                T) for i in 1:num_cons]
+                           cache.u0,
+                           ForwardDiff.Chunk{chunksize}(),
+                           T) for i in 1:num_cons]
             cons_h = function (res, θ)
                 for i in 1:num_cons
                     ForwardDiff.jacobian!(res[i], gs[i], θ, jaccfgs[i], Val{false}())
@@ -319,7 +319,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(x),
-                chunksize,
+                chunksize
             }.(x, Ref(ForwardDiff.Partials((ones(eltype(x), chunksize)...,))))
             h_tape = ReverseDiff.GradientTape(_f, xdual)
             htape = ReverseDiff.compile(h_tape)
@@ -393,9 +393,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
             end
             gs = [x -> grad_cons(x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardDiff.JacobianConfig(gs[i],
-                x,
-                ForwardDiff.Chunk{chunksize}(),
-                T) for i in 1:num_cons]
+                           x,
+                           ForwardDiff.Chunk{chunksize}(),
+                           T) for i in 1:num_cons]
             cons_h = function (θ)
                 map(1:num_cons) do i
                     ForwardDiff.jacobian(gs[i], θ, jaccfgs[i], Val{false}())
@@ -456,7 +456,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(x),
-                chunksize,
+                chunksize
             }.(x, Ref(ForwardDiff.Partials((ones(eltype(x), chunksize)...,))))
             h_tape = ReverseDiff.GradientTape(_f, xdual)
             htape = ReverseDiff.compile(h_tape)
@@ -530,9 +530,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
             end
             gs = [x -> grad_cons(x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardDiff.JacobianConfig(gs[i],
-                x,
-                ForwardDiff.Chunk{chunksize}(),
-                T) for i in 1:num_cons]
+                           x,
+                           ForwardDiff.Chunk{chunksize}(),
+                           T) for i in 1:num_cons]
             cons_h = function (θ)
                 map(1:num_cons) do i
                     ForwardDiff.jacobian(gs[i], θ, jaccfgs[i], Val{false}())
diff --git a/ext/OptimizationSparseDiffExt.jl b/ext/OptimizationSparseDiffExt.jl
index d9955e4..cc0ffd6 100644
--- a/ext/OptimizationSparseDiffExt.jl
+++ b/ext/OptimizationSparseDiffExt.jl
@@ -2,13 +2,14 @@ module OptimizationSparseDiffExt
 
 import OptimizationBase, OptimizationBase.ArrayInterface
 import OptimizationBase.SciMLBase: OptimizationFunction
-import OptimizationBase.ADTypes: AutoSparse, AutoFiniteDiff, AutoForwardDiff, AutoReverseDiff
+import OptimizationBase.ADTypes: AutoSparse, AutoFiniteDiff, AutoForwardDiff,
+                                 AutoReverseDiff
 using OptimizationBase.LinearAlgebra, ReverseDiff
 isdefined(Base, :get_extension) ?
 (using SparseDiffTools,
-    SparseDiffTools.ForwardDiff, SparseDiffTools.FiniteDiff, Symbolics) :
+       SparseDiffTools.ForwardDiff, SparseDiffTools.FiniteDiff, Symbolics) :
 (using ..SparseDiffTools,
-    ..SparseDiffTools.ForwardDiff, ..SparseDiffTools.FiniteDiff, ..Symbolics)
+       ..SparseDiffTools.ForwardDiff, ..SparseDiffTools.FiniteDiff, ..Symbolics)
 
 function default_chunk_size(len)
     if len < ForwardDiff.DEFAULT_CHUNK_THRESHOLD
diff --git a/ext/OptimizationSparseFiniteDiff.jl b/ext/OptimizationSparseFiniteDiff.jl
index 4370467..f3f5a2a 100644
--- a/ext/OptimizationSparseFiniteDiff.jl
+++ b/ext/OptimizationSparseFiniteDiff.jl
@@ -10,7 +10,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
 
     if f.grad === nothing
         gradcache = FD.GradientCache(x, x)
-        grad = (res, θ, args...) -> FD.finite_difference_gradient!(res, x -> _f(x, args...),
+        grad = (res, θ, args...) -> FD.finite_difference_gradient!(
+            res, x -> _f(x, args...),
             θ, gradcache)
     else
         grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
@@ -19,7 +20,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) :
+                        f.hess_prototype
         hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
             ForwardColorHesCache(_f, x,
@@ -71,15 +73,17 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     conshess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
         function gen_conshess_cache(_f, x, i)
-            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ?
+                                copy(Symbolics.hessian_sparsity(_f, x)) :
+                                f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors, conshess_sparsity)
             return hesscache
         end
 
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
-        cons(_res, x);
-        _res[i]) for i in 1:num_cons]
+                 cons(_res, x);
+                 _res[i]) for i in 1:num_cons]
         conshess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
             for i in 1:num_cons
@@ -134,10 +138,11 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     _f = (θ, args...) -> first(f.f(θ, cache.p, args...))
     x = cache.u0
     p = cache.p
-    
+
     if f.grad === nothing
         gradcache = FD.GradientCache(cache.u0, cache.u0)
-        grad = (res, θ, args...) -> FD.finite_difference_gradient!(res, x -> _f(x, args...),
+        grad = (res, θ, args...) -> FD.finite_difference_gradient!(
+            res, x -> _f(x, args...),
             θ, gradcache)
     else
         grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
@@ -146,7 +151,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) :
+                        f.hess_prototype
         hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
             ForwardColorHesCache(_f, x,
@@ -198,15 +204,17 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     conshess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
         function gen_conshess_cache(_f, x, i)
-            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ?
+                                copy(Symbolics.hessian_sparsity(_f, x)) :
+                                f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors, conshess_sparsity)
             return hesscache
         end
 
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
-        cons(_res, x);
-        _res[i]) for i in 1:num_cons]
+                 cons(_res, x);
+                 _res[i]) for i in 1:num_cons]
         conshess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
             for i in 1:num_cons
diff --git a/ext/OptimizationSparseForwardDiff.jl b/ext/OptimizationSparseForwardDiff.jl
index 573e18f..12ab242 100644
--- a/ext/OptimizationSparseForwardDiff.jl
+++ b/ext/OptimizationSparseForwardDiff.jl
@@ -19,7 +19,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) :
+                        f.hess_prototype
         hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
             ForwardColorHesCache(_f, x,
@@ -50,7 +51,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     cons_jac_prototype = f.cons_jac_prototype
     cons_jac_colorvec = f.cons_jac_colorvec
     if cons !== nothing && f.cons_j === nothing
-        cons_jac_prototype = isnothing(f.cons_jac_prototype) ? Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
+        cons_jac_prototype = isnothing(f.cons_jac_prototype) ?
+                             Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
             x) : f.cons_jac_prototype
         cons_jac_colorvec = matrix_colors(cons_jac_prototype)
         jaccache = ForwardColorJacCache(cons,
@@ -69,7 +71,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
     cons_hess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
         function gen_conshess_cache(_f, x, i)
-            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ?
+                                copy(Symbolics.hessian_sparsity(_f, x)) :
+                                f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors,
                 conshess_sparsity)
@@ -77,11 +81,12 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
         end
 
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
-        cons(_res, x);
-        _res[i]) for i in 1:num_cons]
+                 cons(_res, x);
+                 _res[i]) for i in 1:num_cons]
         cons_hess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
-            fetch.([Threads.@spawn numauto_color_hessian!(res[i], fcons[i], θ, cons_hess_caches[i]) for i in 1:num_cons])
+            fetch.([Threads.@spawn numauto_color_hessian!(
+                        res[i], fcons[i], θ, cons_hess_caches[i]) for i in 1:num_cons])
         end
     else
         cons_h = (res, θ) -> f.cons_h(res, θ, p)
@@ -111,7 +116,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         error("$(string(adtype)) with SparseDiffTools does not support functions with more than 2 arguments")
     end
     chunksize = _chunksize === nothing ? default_chunk_size(length(cache.u0)) : _chunksize
-    
+
     x = cache.u0
     p = cache.p
 
@@ -128,7 +133,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     hess_sparsity = f.hess_prototype
     hess_colors = f.hess_colorvec
     if f.hess === nothing
-        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) : f.hess_prototype
+        hess_sparsity = isnothing(f.hess_prototype) ? Symbolics.hessian_sparsity(_f, x) :
+                        f.hess_prototype
         hess_colors = matrix_colors(hess_sparsity)
         hess = (res, θ, args...) -> numauto_color_hessian!(res, x -> _f(x, args...), θ,
             ForwardColorHesCache(_f, x,
@@ -159,7 +165,8 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     cons_jac_prototype = f.cons_jac_prototype
     cons_jac_colorvec = f.cons_jac_colorvec
     if cons !== nothing && f.cons_j === nothing
-        cons_jac_prototype = isnothing(f.cons_jac_prototype) ? Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
+        cons_jac_prototype = isnothing(f.cons_jac_prototype) ?
+                             Symbolics.jacobian_sparsity(cons, zeros(eltype(x), num_cons),
             x) : f.cons_jac_prototype
         cons_jac_colorvec = matrix_colors(cons_jac_prototype)
         jaccache = ForwardColorJacCache(cons,
@@ -178,7 +185,9 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
     cons_hess_caches = [(; sparsity = f.cons_hess_prototype, colors = f.cons_hess_colorvec)]
     if cons !== nothing && f.cons_h === nothing
         function gen_conshess_cache(_f, x, i)
-            conshess_sparsity = isnothing(f.cons_hess_prototype) ? copy(Symbolics.hessian_sparsity(_f, x)) : f.cons_hess_prototype[i]
+            conshess_sparsity = isnothing(f.cons_hess_prototype) ?
+                                copy(Symbolics.hessian_sparsity(_f, x)) :
+                                f.cons_hess_prototype[i]
             conshess_colors = matrix_colors(conshess_sparsity)
             hesscache = ForwardColorHesCache(_f, x, conshess_colors,
                 conshess_sparsity)
@@ -186,11 +195,12 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         end
 
         fcons = [(x) -> (_res = zeros(eltype(x), num_cons);
-        cons(_res, x);
-        _res[i]) for i in 1:num_cons]
+                 cons(_res, x);
+                 _res[i]) for i in 1:num_cons]
         cons_hess_caches = [gen_conshess_cache(fcons[i], x, i) for i in 1:num_cons]
         cons_h = function (res, θ)
-            fetch.([Threads.@spawn numauto_color_hessian!(res[i], fcons[i], θ, cons_hess_caches[i]) for i in 1:num_cons])
+            fetch.([Threads.@spawn numauto_color_hessian!(
+                        res[i], fcons[i], θ, cons_hess_caches[i]) for i in 1:num_cons])
         end
     else
         cons_h = (res, θ) -> f.cons_h(res, θ, p)
diff --git a/ext/OptimizationSparseReverseDiff.jl b/ext/OptimizationSparseReverseDiff.jl
index b21b1af..ee2def8 100644
--- a/ext/OptimizationSparseReverseDiff.jl
+++ b/ext/OptimizationSparseReverseDiff.jl
@@ -1,5 +1,5 @@
 function OptimizationBase.ADTypes.AutoSparseReverseDiff(compile::Bool)
-    return AutoSparse(AutoReverseDiff(; compile)) 
+    return AutoSparse(AutoReverseDiff(; compile))
 end
 
 function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
@@ -38,7 +38,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(x),
-                min(chunksize, maximum(hess_colors)),
+                min(chunksize, maximum(hess_colors))
             }.(x,
                 Ref(ForwardDiff.Partials((ones(eltype(x),
                     min(chunksize, maximum(hess_colors)))...,))))
@@ -122,12 +122,13 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
         if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xduals = [ForwardDiff.Dual{
-                typeof(T),
-                eltype(x),
-                min(chunksize, maximum(conshess_colors[i])),
-            }.(x,
-                Ref(ForwardDiff.Partials((ones(eltype(x),
-                    min(chunksize, maximum(conshess_colors[i])))...,)))) for i in 1:num_cons]
+                          typeof(T),
+                          eltype(x),
+                          min(chunksize, maximum(conshess_colors[i]))
+                      }.(x,
+                          Ref(ForwardDiff.Partials((ones(eltype(x),
+                              min(chunksize, maximum(conshess_colors[i])))...,))))
+                      for i in 1:num_cons]
             consh_tapes = [ReverseDiff.GradientTape(fncs[i], xduals[i]) for i in 1:num_cons]
             conshtapes = ReverseDiff.compile.(consh_tapes)
             function grad_cons(res1, θ, htape)
@@ -135,10 +136,10 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true}, x,
             end
             gs = [(res1, x) -> grad_cons(res1, x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardColorJacCache(gs[i],
-                x;
-                tag = typeof(T),
-                colorvec = conshess_colors[i],
-                sparsity = conshess_sparsity[i]) for i in 1:num_cons]
+                           x;
+                           tag = typeof(T),
+                           colorvec = conshess_colors[i],
+                           sparsity = conshess_sparsity[i]) for i in 1:num_cons]
             cons_h = function (res, θ, args...)
                 for i in 1:num_cons
                     SparseDiffTools.forwarddiff_color_jacobian!(res[i],
@@ -213,7 +214,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(cache.u0),
-                min(chunksize, maximum(hess_colors)),
+                min(chunksize, maximum(hess_colors))
             }.(cache.u0,
                 Ref(ForwardDiff.Partials((ones(eltype(cache.u0),
                     min(chunksize, maximum(hess_colors)))...,))))
@@ -314,12 +315,13 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
         if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(cache.u0))
             xduals = [ForwardDiff.Dual{
-                typeof(T),
-                eltype(cache.u0),
-                min(chunksize, maximum(conshess_colors[i])),
-            }.(cache.u0,
-                Ref(ForwardDiff.Partials((ones(eltype(cache.u0),
-                    min(chunksize, maximum(conshess_colors[i])))...,)))) for i in 1:num_cons]
+                          typeof(T),
+                          eltype(cache.u0),
+                          min(chunksize, maximum(conshess_colors[i]))
+                      }.(cache.u0,
+                          Ref(ForwardDiff.Partials((ones(eltype(cache.u0),
+                              min(chunksize, maximum(conshess_colors[i])))...,))))
+                      for i in 1:num_cons]
             consh_tapes = [ReverseDiff.GradientTape(fncs[i], xduals[i]) for i in 1:num_cons]
             conshtapes = ReverseDiff.compile.(consh_tapes)
             function grad_cons(res1, θ, htape)
@@ -333,10 +335,10 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{true},
                 end
             end
             jaccfgs = [ForwardColorJacCache(gs[i],
-                cache.u0;
-                tag = typeof(T),
-                colorvec = conshess_colors[i],
-                sparsity = conshess_sparsity[i]) for i in 1:num_cons]
+                           cache.u0;
+                           tag = typeof(T),
+                           colorvec = conshess_colors[i],
+                           sparsity = conshess_sparsity[i]) for i in 1:num_cons]
             cons_h = function (res, θ)
                 for i in 1:num_cons
                     SparseDiffTools.forwarddiff_color_jacobian!(res[i],
@@ -412,7 +414,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(x),
-                min(chunksize, maximum(hess_colors)),
+                min(chunksize, maximum(hess_colors))
             }.(x,
                 Ref(ForwardDiff.Partials((ones(eltype(x),
                     min(chunksize, maximum(hess_colors)))...,))))
@@ -498,12 +500,13 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
         if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xduals = [ForwardDiff.Dual{
-                typeof(T),
-                eltype(x),
-                min(chunksize, maximum(conshess_colors[i])),
-            }.(x,
-                Ref(ForwardDiff.Partials((ones(eltype(x),
-                    min(chunksize, maximum(conshess_colors[i])))...,)))) for i in 1:num_cons]
+                          typeof(T),
+                          eltype(x),
+                          min(chunksize, maximum(conshess_colors[i]))
+                      }.(x,
+                          Ref(ForwardDiff.Partials((ones(eltype(x),
+                              min(chunksize, maximum(conshess_colors[i])))...,))))
+                      for i in 1:num_cons]
             consh_tapes = [ReverseDiff.GradientTape(fncs[i], xduals[i]) for i in 1:num_cons]
             conshtapes = ReverseDiff.compile.(consh_tapes)
             function grad_cons(θ, htape)
@@ -511,10 +514,10 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false}, x
             end
             gs = [(x) -> grad_cons(x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardColorJacCache(gs[i],
-                x;
-                tag = typeof(T),
-                colorvec = conshess_colors[i],
-                sparsity = conshess_sparsity[i]) for i in 1:num_cons]
+                           x;
+                           tag = typeof(T),
+                           colorvec = conshess_colors[i],
+                           sparsity = conshess_sparsity[i]) for i in 1:num_cons]
             cons_h = function (θ, args...)
                 map(1:num_cons) do i
                     SparseDiffTools.forwarddiff_color_jacobian(gs[i],
@@ -589,7 +592,7 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
             xdual = ForwardDiff.Dual{
                 typeof(T),
                 eltype(x),
-                min(chunksize, maximum(hess_colors)),
+                min(chunksize, maximum(hess_colors))
             }.(x,
                 Ref(ForwardDiff.Partials((ones(eltype(x),
                     min(chunksize, maximum(hess_colors)))...,))))
@@ -675,12 +678,13 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
         if adtype.dense_ad.compile
             T = ForwardDiff.Tag(OptimizationSparseReverseTag(), eltype(x))
             xduals = [ForwardDiff.Dual{
-                typeof(T),
-                eltype(x),
-                min(chunksize, maximum(conshess_colors[i])),
-            }.(x,
-                Ref(ForwardDiff.Partials((ones(eltype(x),
-                    min(chunksize, maximum(conshess_colors[i])))...,)))) for i in 1:num_cons]
+                          typeof(T),
+                          eltype(x),
+                          min(chunksize, maximum(conshess_colors[i]))
+                      }.(x,
+                          Ref(ForwardDiff.Partials((ones(eltype(x),
+                              min(chunksize, maximum(conshess_colors[i])))...,))))
+                      for i in 1:num_cons]
             consh_tapes = [ReverseDiff.GradientTape(fncs[i], xduals[i]) for i in 1:num_cons]
             conshtapes = ReverseDiff.compile.(consh_tapes)
             function grad_cons(θ, htape)
@@ -688,10 +692,10 @@ function OptimizationBase.instantiate_function(f::OptimizationFunction{false},
             end
             gs = [(x) -> grad_cons(x, conshtapes[i]) for i in 1:num_cons]
             jaccfgs = [ForwardColorJacCache(gs[i],
-                x;
-                tag = typeof(T),
-                colorvec = conshess_colors[i],
-                sparsity = conshess_sparsity[i]) for i in 1:num_cons]
+                           x;
+                           tag = typeof(T),
+                           colorvec = conshess_colors[i],
+                           sparsity = conshess_sparsity[i]) for i in 1:num_cons]
             cons_h = function (θ, args...)
                 map(1:num_cons) do i
                     SparseDiffTools.forwarddiff_color_jacobian(gs[i],
diff --git a/ext/OptimizationTrackerExt.jl b/ext/OptimizationTrackerExt.jl
index 9df1605..f315d54 100644
--- a/ext/OptimizationTrackerExt.jl
+++ b/ext/OptimizationTrackerExt.jl
@@ -11,7 +11,8 @@ function OptimizationBase.instantiate_function(f, x, adtype::AutoTracker, p,
     _f = (θ, args...) -> first(f.f(θ, p, args...))
 
     if f.grad === nothing
-        grad = (res, θ, args...) -> res .= Tracker.data(Tracker.gradient(x -> _f(x, args...),
+        grad = (res, θ, args...) -> res .= Tracker.data(Tracker.gradient(
+            x -> _f(x, args...),
             θ)[1])
     else
         grad = (G, θ, args...) -> f.grad(G, θ, p, args...)
@@ -42,7 +43,8 @@ function OptimizationBase.instantiate_function(f, cache::OptimizationBase.ReInit
     _f = (θ, args...) -> first(f.f(θ, cache.p, args...))
 
     if f.grad === nothing
-        grad = (res, θ, args...) -> res .= Tracker.data(Tracker.gradient(x -> _f(x, args...),
+        grad = (res, θ, args...) -> res .= Tracker.data(Tracker.gradient(
+            x -> _f(x, args...),
             θ)[1])
     else
         grad = (G, θ, args...) -> f.grad(G, θ, cache.p, args...)
diff --git a/src/OptimizationBase.jl b/src/OptimizationBase.jl
index cf4cd0c..89bb53e 100644
--- a/src/OptimizationBase.jl
+++ b/src/OptimizationBase.jl
@@ -11,12 +11,12 @@ end
 using ArrayInterface, Base.Iterators, SparseArrays, LinearAlgebra
 
 import SciMLBase: OptimizationProblem,
-    OptimizationFunction, ObjSense,
-    MaxSense, MinSense, OptimizationStats
+                  OptimizationFunction, ObjSense,
+                  MaxSense, MinSense, OptimizationStats
 export ObjSense, MaxSense, MinSense
 
 struct NullCallback end
-(x::NullCallback)(args...) = false;
+(x::NullCallback)(args...) = false
 const DEFAULT_CALLBACK = NullCallback()
 
 struct NullData end
diff --git a/test/adtests.jl b/test/adtests.jl
index dcd68ea..2558eb6 100644
--- a/test/adtests.jl
+++ b/test/adtests.jl
@@ -73,7 +73,8 @@ H2 = Array{Float64}(undef, 2, 2)
 
 if VERSION >= v"1.9"
     optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoEnzyme(), cons = cons)
-    optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoEnzyme(),
+    optprob = OptimizationBase.instantiate_function(
+        optf, x0, OptimizationBase.AutoEnzyme(),
         nothing, 1)
     optprob.grad(G2, x0)
     @test G1 == G2
@@ -93,7 +94,8 @@ if VERSION >= v"1.9"
     H2 = Array{Float64}(undef, 2, 2)
 
     optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoEnzyme(), cons = con2_c)
-    optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoEnzyme(),
+    optprob = OptimizationBase.instantiate_function(
+        optf, x0, OptimizationBase.AutoEnzyme(),
         nothing, 2)
     optprob.grad(G2, x0)
     @test G1 == G2
@@ -235,7 +237,8 @@ optprob.grad(G2, x0)
 prob = OptimizationProblem(optf, x0)
 
 optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoFiniteDiff())
-optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoFiniteDiff(),
+optprob = OptimizationBase.instantiate_function(
+    optf, x0, OptimizationBase.AutoFiniteDiff(),
     nothing)
 optprob.grad(G2, x0)
 @test G1≈G2 rtol=1e-6
@@ -245,7 +248,8 @@ optprob.hess(H2, x0)
 # Test new constraints
 cons = (res, x, p) -> (res .= [x[1]^2 + x[2]^2])
 optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoFiniteDiff(), cons = cons)
-optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoFiniteDiff(),
+optprob = OptimizationBase.instantiate_function(
+    optf, x0, OptimizationBase.AutoFiniteDiff(),
     nothing, 1)
 optprob.grad(G2, x0)
 @test G1≈G2 rtol=1e-6
@@ -274,7 +278,8 @@ cons_jac_colors = 1:2
 optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoFiniteDiff(), cons = cons,
     cons_jac_prototype = cons_jac_proto,
     cons_jac_colorvec = cons_jac_colors)
-optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoFiniteDiff(),
+optprob = OptimizationBase.instantiate_function(
+    optf, x0, OptimizationBase.AutoFiniteDiff(),
     nothing, 1)
 @test optprob.cons_jac_prototype == sparse([1.0 1.0]) # make sure it's still using it
 @test optprob.cons_jac_colorvec == 1:2
@@ -286,7 +291,8 @@ function con2_c(res, x, p)
     res .= [x[1]^2 + x[2]^2, x[2] * sin(x[1]) - x[1]]
 end
 optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoFiniteDiff(), cons = con2_c)
-optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoFiniteDiff(),
+optprob = OptimizationBase.instantiate_function(
+    optf, x0, OptimizationBase.AutoFiniteDiff(),
     nothing, 2)
 optprob.grad(G2, x0)
 @test G1≈G2 rtol=1e-6
@@ -309,7 +315,8 @@ cons_jac_colors = 1:2
 optf = OptimizationFunction(rosenbrock, OptimizationBase.AutoFiniteDiff(), cons = con2_c,
     cons_jac_prototype = cons_jac_proto,
     cons_jac_colorvec = cons_jac_colors)
-optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoFiniteDiff(),
+optprob = OptimizationBase.instantiate_function(
+    optf, x0, OptimizationBase.AutoFiniteDiff(),
     nothing, 2)
 @test optprob.cons_jac_prototype == sparse([1.0 1.0; 1.0 1.0]) # make sure it's still using it
 @test optprob.cons_jac_colorvec == 1:2
@@ -356,7 +363,7 @@ optprob2.cons_j(sJ, [5.0, 3.0])
 optprob2.cons_h(sH3, [5.0, 3.0])
 @test sH3 ≈ [
     [2.0 0.0; 0.0 2.0],
-    [2.8767727327346804 0.2836621681849162; 0.2836621681849162 -6.622738308376736e-9],
+    [2.8767727327346804 0.2836621681849162; 0.2836621681849162 -6.622738308376736e-9]
 ]
 
 using SparseDiffTools
@@ -489,7 +496,8 @@ optprob.hess(H2, x0)
     optf = OptimizationFunction{false}(rosenbrock,
         OptimizationBase.AutoEnzyme(),
         cons = cons)
-    optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoEnzyme(),
+    optprob = OptimizationBase.instantiate_function(
+        optf, x0, OptimizationBase.AutoEnzyme(),
         nothing, 1)
 
     @test optprob.grad(x0) == G1
@@ -505,7 +513,8 @@ optprob.hess(H2, x0)
     optf = OptimizationFunction{false}(rosenbrock,
         OptimizationBase.AutoEnzyme(),
         cons = cons)
-    optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoEnzyme(),
+    optprob = OptimizationBase.instantiate_function(
+        optf, x0, OptimizationBase.AutoEnzyme(),
         nothing, 2)
 
     @test optprob.grad(x0) == G1
@@ -765,7 +774,8 @@ optprob.hess(H2, x0)
     optf = OptimizationFunction{false}(rosenbrock,
         OptimizationBase.AutoZygote(),
         cons = cons)
-    optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoZygote(),
+    optprob = OptimizationBase.instantiate_function(
+        optf, x0, OptimizationBase.AutoZygote(),
         nothing, 1)
 
     @test optprob.grad(x0) == G1
@@ -780,7 +790,8 @@ optprob.hess(H2, x0)
     optf = OptimizationFunction{false}(rosenbrock,
         OptimizationBase.AutoZygote(),
         cons = cons)
-    optprob = OptimizationBase.instantiate_function(optf, x0, OptimizationBase.AutoZygote(),
+    optprob = OptimizationBase.instantiate_function(
+        optf, x0, OptimizationBase.AutoZygote(),
         nothing, 2)
 
     @test optprob.grad(x0) == G1

From 35a3dc345488636eee48db0dd225b2861d39176b Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 14:38:39 -0400
Subject: [PATCH 8/9] Mark broken and fail downstream

---
 .github/workflows/Downstream.yml | 18 +++++-------------
 Project.toml                     |  2 +-
 test/adtests.jl                  |  2 +-
 3 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml
index 1052b99..cfd1eb2 100644
--- a/.github/workflows/Downstream.yml
+++ b/.github/workflows/Downstream.yml
@@ -35,19 +35,11 @@ jobs:
         shell: julia --color=yes --project=downstream {0}
         run: |
           using Pkg
-          try
-            # force it to use this PR's version of the package
-            Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
-            Pkg.update()
-            Pkg.test(coverage=true)  # resolver may fail with test time deps
-          catch err
-            err isa Pkg.Resolve.ResolverError || rethrow()
-            # If we can't resolve that means this is incompatible by SemVer and this is fine
-            # It means we marked this as a breaking change, so we don't need to worry about
-            # Mistakenly introducing a breaking change, as we have intentionally made one
-            @info "Not compatible with this release. No problem." exception=err
-            exit(0)  # Exit immediately, as a success
-          end
+          # force it to use this PR's version of the package
+          Pkg.develop(PackageSpec(path="."))  # resolver may fail with main deps
+          Pkg.update()
+          Pkg.test(coverage=true)  # resolver may fail with test time deps
+          
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v4
         with:
diff --git a/Project.toml b/Project.toml
index 32d328e..8976047 100644
--- a/Project.toml
+++ b/Project.toml
@@ -38,7 +38,7 @@ OptimizationZygoteExt = "Zygote"
 ADTypes = "1"
 ArrayInterface = "7.6"
 DocStringExtensions = "0.9"
-Enzyme = "0.11.11, =0.12.5"
+Enzyme = "0.11.11, =0.12.6"
 FiniteDiff = "2.12"
 ForwardDiff = "0.10.26"
 LinearAlgebra = "1.9, 1.10"
diff --git a/test/adtests.jl b/test/adtests.jl
index 2558eb6..3d7586b 100644
--- a/test/adtests.jl
+++ b/test/adtests.jl
@@ -521,7 +521,7 @@ optprob.hess(H2, x0)
     @test optprob.hess(x0) == H1
     @test optprob.cons(x0) == [0.0, 0.0]
     @test optprob.cons_j([5.0, 3.0])≈[10.0 6.0; -0.149013 -0.958924] rtol=1e-6
-    @test optprob.cons_h(x0) == [[2.0 0.0; 0.0 2.0], [-0.0 1.0; 1.0 0.0]]
+    @test_broken optprob.cons_h(x0) # == [[2.0 0.0; 0.0 2.0], [-0.0 1.0; 1.0 0.0]]
 
     cons = (x, p) -> [x[1]^2 + x[2]^2]
     optf = OptimizationFunction{false}(rosenbrock,

From 3e26da5818db485267921fd45bbaf26417428f34 Mon Sep 17 00:00:00 2001
From: Vaibhav Dixit <vaibhavyashdixit@gmail.com>
Date: Mon, 20 May 2024 14:54:08 -0400
Subject: [PATCH 9/9] bump version

---
 Project.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Project.toml b/Project.toml
index 8976047..c1cbe7c 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "OptimizationBase"
 uuid = "bca83a33-5cc9-4baa-983d-23429ab6bcbb"
 authors = ["Vaibhav Dixit <vaibhavyashdixit@gmail.com> and contributors"]
-version = "1.0.0"
+version = "1.0.1"
 
 [deps]
 ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b"