In [1]:

    

using Plots
using Optim


    

In [35]:

    

# example of Rosenbrock function
function f(x) 
    return (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2
end
function g!(x::Vector, storage::Vector)
storage[1] = -2.0 * (1.0 - x[1]) - 400.0 * (x[2] - x[1]^2) * x[1]
storage[2] = 200.0 * (x[2] - x[1]^2)
end

soln = optimize(f, g!, [0.0, 0.0], LBFGS())


    

    




WARNING: Method definition f(Any) in module Main at In[10]:3 overwritten at In[35]:3.
WARNING: Method definition g!(Array{T<:Any, 1}, Array{T<:Any, 1}) in module Main at In[10]:6 overwritten at In[35]:6.






    
Out[35]:





Results of Optimization Algorithm
 * Algorithm: L-BFGS
 * Starting Point: [0.0,0.0]
 * Minimizer: [1.0000000000000007,1.000000000000001]
 * Minimum: 0.000000
 * Iterations: 21
 * Convergence: true
   * |x - x'| < 1.0e-32: false
   * |f(x) - f(x')| / |f(x)| < 1.0e-08: true
   * |g(x)| < 1.0e-08: true
   * Reached Maximum Number of Iterations: false
 * Objective Function Calls: 90
 * Gradient Calls: 90

In [5]:

    

reshape(randn(5,3),15,1)


    

    
Out[5]:





15×1 Array{Float64,2}:
 -0.961733 
  0.256498 
  1.35844  
  0.176834 
  0.0325291
 -0.494308 
 -1.62881  
  0.586711 
  1.18127  
  2.75914  
  0.697867 
 -0.408102 
 -0.121016 
  0.498591 
 -0.511114 

In [41]:

    

# now, we do the matrix factorization example
# originally from Poblano example2

function matrix_approx_function(x::Vector, A::Matrix, r::Int)
    # unpack U and V from x
    m,n = size(A)
    U = reshape(x[1:m*r],m,r)
    V = reshape(x[(m*r+1):end],n,r)
    return 0.5*vecnorm(A - U*V')
end

function matrix_approx_gradient!(x::Vector, storage::Vector, A::Matrix, r::Int)
    m,n = size(A)
    U = reshape(x[1:m*r],m,r)
    V = reshape(x[(m*r+1):end],n,r)
    D = A - U*V'
    storage[1:(m*r)] = -vec(D*V)
    storage[(m*r+1):end] = -vec(D'*U)
end


    

    




WARNING: Method definition matrix_approx_function(Array{T<:Any, 1}, Array{T<:Any, 2}, Int64) in module Main at In[39]:6 overwritten at In[41]:6.
WARNING: Method definition matrix_approx_gradient!(Array{T<:Any, 1}, Array{T<:Any, 1}, Array{T<:Any, 2}, Int64) in module Main at In[39]:13 overwritten at In[41]:13.






    
Out[41]:





matrix_approx_gradient! (generic function with 1 method)

In [47]:

    

?optimize


    

    




search: optimize optimize_ticks OptimizationTrace OptimizationState







    
Out[47]:




No documentation found.
Optim.optimize is a Function.

# 29 methods for generic function "optimize":
optimize(d::Optim.DifferentiableFunction, initial_x::Array; method, xtol, ftol, grtol, iterations, store_trace, show_trace, extended_trace, show_every, callback) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:85
optimize{T}(d::Optim.DifferentiableFunction, initial_x::Array{T,N<:Any}, mo::Optim.GradientDescent, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/gradient_descent.jl:35
optimize{T}(d::Optim.DifferentiableFunction, initial_x::Array{T,1}, mo::Optim.AcceleratedGradientDescent, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/accelerated_gradient_descent.jl:43
optimize{T}(d::Optim.DifferentiableFunction, initial_x::Array{T,1}, mo::Optim.MomentumGradientDescent, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/momentum_gradient_descent.jl:39
optimize{T}(df::Optim.DifferentiableFunction, initial_x::Array{T,N<:Any}, cg::Optim.ConjugateGradient, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/cg.jl:126
optimize{T}(d::Optim.DifferentiableFunction, initial_x::Array{T,1}, mo::Optim.BFGS, o::Optim.OptimizationOptions; initial_invH) at /opt/julia_packages/.julia/v0.5/Optim/src/bfgs.jl:41
optimize{T}(d::Optim.DifferentiableFunction, initial_x::Array{T,1}, mo::Optim.LBFGS, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/l_bfgs.jl:96
optimize{T<:AbstractFloat}(df::Optim.DifferentiableFunction, initial_x::Array{T,N<:Any}, l::Array{T,N<:Any}, u::Array{T,N<:Any}, ::Optim.Fminbox; xtol, ftol, grtol, iterations, store_trace, show_trace, extended_trace, callback, show_every, linesearch!, eta, mu0, mufactor, precondprep, optimizer, optimizer_o, nargs...) at /opt/julia_packages/.julia/v0.5/Optim/src/fminbox.jl:136
optimize(d::Optim.DifferentiableFunction, initial_x::Array, method::Optim.Optimizer, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:157
optimize{T}(d::Optim.TwiceDifferentiableFunction, initial_x::Array{T,1}, mo::Optim.Newton, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/newton.jl:43
optimize(d::Optim.TwiceDifferentiableFunction, initial_x::Array; method, xtol, ftol, grtol, iterations, store_trace, show_trace, extended_trace, show_every, callback) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:106
optimize(d::Optim.TwiceDifferentiableFunction, initial_x::Array, method::Optim.Optimizer, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:164
optimize(f::Function, initial_x::Array; method, xtol, ftol, grtol, iterations, store_trace, show_trace, extended_trace, show_every, autodiff, callback) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:18
optimize(d::Function, initial_x::Array, method::Optim.Newton, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/newton.jl:35
optimize{T}(f::Function, initial_x::Array{T,1}, nm::Optim.NelderMead, o::Optim.OptimizationOptions; initial_step) at /opt/julia_packages/.julia/v0.5/Optim/src/nelder_mead.jl:58
optimize{T}(cost::Function, initial_x::Array{T,N<:Any}, mo::Optim.SimulatedAnnealing, o::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/simulated_annealing.jl:50
optimize{T<:AbstractFloat}(f::Function, x_lower::T, x_upper::T, ::Optim.GoldenSection; iterations, store_trace, show_trace, show_every, extended_trace, rel_tol, abs_tol, callback, nargs...) at /opt/julia_packages/.julia/v0.5/Optim/src/golden_section.jl:36
optimize{T<:AbstractFloat}(f::Function, x_lower::T, x_upper::T, ::Optim.Brent; iterations, store_trace, show_trace, show_every, extended_trace, rel_tol, abs_tol, callback) at /opt/julia_packages/.julia/v0.5/Optim/src/brent.jl:37
optimize(f::Function, g!::Function, initial_x::Array; method, xtol, ftol, grtol, iterations, store_trace, show_trace, extended_trace, show_every, callback) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:41
optimize(f::Function, g!::Function, h!::Function, initial_x::Array; method, xtol, ftol, grtol, iterations, store_trace, show_trace, extended_trace, show_every, callback) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:64
optimize(f::Function, g!::Function, initial_x::Array, method::Optim.Optimizer) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:127
optimize(f::Function, g!::Function, initial_x::Array, method::Optim.Optimizer, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:127
optimize(f::Function, g!::Function, h!::Function, initial_x::Array, method::Optim.Optimizer) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:137
optimize(f::Function, g!::Function, h!::Function, initial_x::Array, method::Optim.Optimizer, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:137
optimize(f::Function, initial_x::Array, method::Optim.Optimizer, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:145
optimize{T<:AbstractFloat}(f::Function, lower::T, upper::T; method, rel_tol, abs_tol, iterations, store_trace, show_trace, callback, show_every, extended_trace) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:180
optimize(f::Function, lower::Real, upper::Real; kwargs...) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:203
optimize(d, initial_x::Array, method::Optim.Optimizer) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:119
optimize(d, initial_x::Array, method::Optim.Optimizer, options::Optim.OptimizationOptions) at /opt/julia_packages/.julia/v0.5/Optim/src/optimize.jl:119

In [49]:

    

?OptimizationOptions


    

    




search: OptimizationOptions







    
Out[49]:




No documentation found.
Summary:

immutable Optim.OptimizationOptions{TCallback<:Union{Function,Void}} <: Any
Fields:

xtol           :: Float64
ftol           :: Float64
grtol          :: Float64
iterations     :: Int64
store_trace    :: Bool
show_trace     :: Bool
extended_trace :: Bool
autodiff       :: Bool
show_every     :: Int64
callback       :: TCallback<:Union{Function,Void}

In [54]:

    

m = 5
n = 4
A = randn(m,n)
r = 2
myf = x -> matrix_approx_function(x, A, r)
myg! = (x, storage) -> matrix_approx_gradient!(x, storage, A, r)

#soln = optimize(myf, myg!, randn(m*r+n*r), LBFGS(), Optim.Options(f_tol = 1e-8))
soln = optimize(myf, myg!, randn(m*r+n*r), LBFGS(), OptimizationOptions(ftol = 1e-8))
x = Optim.minimizer(soln)
@show soln
Uopt = reshape(x[1:m*r],m,r)
Vopt = reshape(x[(m*r+1):end],n,r)
objval = 2*myf(x)
opterr = vecnorm(A-Uopt*Vopt')

Usvd,Ssvd,Vsvd = svd(A)
svderr = vecnorm(A-Usvd[:,1:r]*diagm(Ssvd[1:r])*Vsvd[:,1:r]')
@show objval
@show opterr
@show svderr
; # hide final output in JuliaBox


    

    




soln = Results of Optimization Algorithm
 * Algorithm: L-BFGS
 * Starting Point: [1.025768020262662,1.7373559569835082, ...]
 * Minimizer: [-0.2692364378685741,1.808908598076151, ...]
 * Minimum: 0.825548
 * Iterations: 49
 * Convergence: true
   * |x - x'| < 1.0e-32: false
   * |f(x) - f(x')| / |f(x)| < 1.0e-08: true
   * |g(x)| < 1.0e-08: false
   * Reached Maximum Number of Iterations: false
 * Objective Function Calls: 183
 * Gradient Calls: 183
objval = 1.6510966810998684
opterr = 1.6510966810998684
svderr = 1.6510966698289107