
\section{Approximations with Quadratic Stabilizers}

Consider the problem of approximation of a function $h(x)$ $x\in [a,
b]$ by another function $u(x)$ with better smoothness or other
favorable properties. For example, we may want to approximate the
noisy experimental curve by a smooth one, or approximate a curve
with a blocky piece-wise constant curve. The following method is
used for approximations: A variational problem is formulated to
minimize a norm of the difference $u-h$:
$$D_2= \hf\int_a^b (u-h)^2 dx \mbox{ or } D_1=\int_a^b |u-h| dx $$
plus a penalty of the form
$$P=\int_a^b p(u, u', u'') dx $$
The {\em stabilizer} $P$ is chosen to penalize  the approximation $u$ for being non-smooth or having
large variation. The approximate $u(x)$ balances the closeness to the
approximating curve and the smoothness properties. The problem of
the best approximate is a variational problem for an unknown
function $u(x)$ that minimizes the integral functional 
    \beq
     \min_{u(x)} (D + \Gg P) = \min_{u(x)}  \int_a^b \left[(u-h)^q +\gamma  p(u, u', u'') \right] dx 
\eeq{81}
where $\Gg$ is a penalty coefficient. Here we consider several
problems of the best approximation.

%\subsection{Quadratic penalties}
\subsection{Penalized magnitude}
A simple penalization $p= u^2$ is 
proportional to the square of the magnitude of the approximate $u$.
The approximation problem has the form
    \beq
    \min_{u(x)} J_1(u), \quad J_1(u)=\int_a^b
    \hf \left( \Gg u^2 + (h-u)^2 \right) dx
    \eeq{122.2.1}
it minimizes the sum of  square $D_2^2$ of $D_2$ and the penaltyThe Lagrangian does not contain derivative $u'$, therefore the
stationarity condition is
$$
\od{}{u}\left( \Gg u^2 + (h-u)^2 \right)=0
$$
We find $u_\Gg (x)=\rec{1+ \Gg} h(x)$ - the approximate $u$ is
proportional to $h$.  The penalty reduces the magnitude of the
function. When $\Gg \to 0$, the approximation coincides with $h(x)$,
and when $\Gg \to \infty$, the approximation $u_\Gg =0$,
$$\lim_{\Gg\to 0} u_\Gg (x)=h(x), \quad \lim_{\Gg\to \infty }
u_\Gg (x)=0
$$
\subsection{Penalized growth rate}
Approximation of the given function $h(x)$ by function $u(x)$ with a
limited growth rate can be formulated as a variational problem
    \beq
    \min_u J_2(u), \quad J_2(u)=\int_a^b
    \hf \left( \Gg u'\,^2 + (h-u)^2 \right) dx
    \eeq{122.2.1}
The first term of the integrant represents the penalty for the
growth rate to be non-constant, and the second term describes the
closeness of $u$ and $h$.

The equation for the approximate (Euler equation of  \eq{122.2.1})
is \beq \Gg u'' -u + h=0, \quad u'(a)=u'(b)=0
 \eeq{123.1.3}
Here, the natural boundary conditions are assumed since there is no
reason to assign special values of the approximation curve at the
ends of the interval.

The approximation depends on the parameter $\Gg$, $u=u_\Gg$. When
$\Gg \to 0$, the approximation coincides with $h(x)$ and when  $\Gg
\to \infty$, the approximation is a constant curve, equal to the
mean value of $h(x)$,
$$\lim_{\Gg\to 0} u_\Gg (x)=h(x), \quad \lim_{\Gg\to \infty }
u_\Gg (x)=\Cnst=\rec{b-a}\int_a^b h(\Gx) d\Gx
$$






\subsection{Penalized smoothness}

The problem of smooth approximation is similar. We choose the
penalization as integral of the square of the second derivative
$u''$. This functional increases with  deviation of $u$ from a
straight line. The corresponding variational problem can be
formulated as
    \beq
    \min_u J(u), \quad J(u)=\int_a^b
    \hf \left( \Gg( u'')^2 + (h-u)^2 \right) dx
    \eeq{122.2.11}
The equation for the best approximate (Euler equation of
\eq{122.2.11}) is
$$
\Gg u^{IV} +u - h=0, \quad u''(a)=u''(b)=0, \quad u'''(a)=u'''(b)=0,
$$
Here, the natural boundary conditions are assumed since there is
no reason to assign special values of the approximation curve at
the ends of the interval.

 When $\Gg \to
0$, the approximation coincides with $h(x)$ and when  $\Gg \to
\infty$, the approximation is a straight line closest to $h$.
\begin{problem}
Find the limit of $u_\Gg$  when $\Gg\to \infty$
\end{problem}



\section{Appendix. Methods for  linear boundary-value problems}

The Euler equation is  a linear boundary value problem of the type
$$\CL_\Gg(u)=h $$
where $\CL$  is the linear operator of the Sturm-Liuville type, such
as $\CL_\Gg=\Gg u''- u$. A linear  differential equation is defined
in the interval $[a,b]$, and the homogeneous boundary conditions
$u'(a)=u'(b)=0$ are posed at the ends. Here we remind an approach
based on the expansion of $u$ into series of {\em eigenfunctions} of
the operator $\CL_\Gg$.
\subsubsection{Spectral method}

\paragraph{Solution of linear boundary value problem}
We briefly comment on solution of the linear Euler equation by
spectral method. To solve the problem, we first find the spectrum of
 operator  $\CL_\Gg$ solving the Sturm-Liouville problem:
 $$
 \CL_\Gg u=\Gl u, \quad u'(0)=u'(1)=0.
 $$
This problem has nontrivial solutions ($u(x)\not\equiv 0$
everywhere) for only some  values of $\Gl$ called the eigenvalues
$\Gl_0, \Gl_1, \Gl_2 , \ldots$. The solutions called the
eigenfunctions $u_n(x)$,  satisfy the equations
$$\CL_\Gg\,u_n= \Gl_n u_n, \quad n=1, 2, \ldots, .
$$
Eigenfunctions $u_n$ are defined up to a multiplier and are mutually
orthogonal,
$$
\langle u_k, u_m \rangle= 0, \quad \If k\neq m
$$
where the inner product $\langle f, g \rangle$ is defined as
$$
\langle f, g \rangle= \int_{a}^{b} f(x) g(x) \, dx.
$$


%%the operation $\langle z \rangle $ means the average
%%$$
%%\langle z \rangle= \f{1}{b-a}\int_{a}^{b} z\, dx.
%%$$

The system of eigenfunctions is complete. This means that a general
solution can be represented by series
$$ u(x)=\sum_{n=0}^\infty \Ga_n u_n(x)
$$ where $\Ga_n$ are arbitrary coefficients. With this
representation, the Euler equation $\CL \, (u)=h$ becomes
$$
\CL(u)= \CL\left(\sum_{n=0}^\infty\Ga_n u_n \right)=
\sum_{n=0}^\infty \Ga_n \Gl_n u_n =h
$$
To find the coefficients $\Ga_n$, we expand $h$ as follows
$$
h=\sum_{n=0}^\infty \Gb_n u_n
$$
where  parameters $\Gb_n$ are found from the orthogonality property
of $u_n$ as
$$
\Gb_n=\f{\langle h\,, u_n \rangle}{\langle u_n, u_n \rangle}
$$
Comparing the expansions, we obtain the unknown coefficients $\Ga_n$
 $$
\Gl_n \Ga_n= \Gb_n \quad \mbox{or }  \Ga_n =\f{ \langle u_n,
h\rangle}{\Gl_n\left\langle u_n, u_n\right\rangle}
$$
and the solution $u$
$$
u(x)=\sum_{n=0}^\infty \f{ \langle u_n, h\rangle}{\Gl_n\left\langle
u_n, u_n\right\rangle} u_n(x).
$$
We observe that the approximate linearly depends on $h$ as expected.
\paragraph{Example}
For the operator \eq{appr-oper} we have
 $$
 a^2 u''-  u=\Gl u, \quad u'(0)=u'(1)=0.
 $$
Particular solutions (the eigenfunctions) are $ u_n=\cos\left( {\pi
n x }\right)$ and the eigenvalues are $\Gl_n= (1+\pi n^2 a^2) $. The
general solution is
$$
u(x)=\sum_{n=0}^\infty \Gg_n\cos(\pi n x)
$$
where $\Gg_n$ are arbitrary coefficients. Solving, we obtain
$$
u(x)=\sum_{n=0}^\infty {c_n \over \Gl_n}\cos(\pi n x), \quad
c_n=\int_0^1 h(x) \cos(\pi n x) dx
$$
\begin{problem}
Let $h(x)$ be  $h(x)=|x|$. Find $u(x)$.  Graph the results.
\end{problem}

\subsection{Green's function}
\input{green.tex}


\section{Approximation with non�quadratic stabilizers}
\subsection{$L_1$-approxmation with quadratic penalty}
The   variational problem has the form
    \beq
    \min_u J_{1}(u), \quad J_{1}(u)=\int_a^b
     \left( \hf \Gg( u')^2 +  |u-h | \right) dx
    \eeq{123.3.1}
The Euler equation is
 \beq
 u''- \rec{\Gg}\sign(u-h)=0, \quad u'(a)=u'(b)=0
 \eeq{123.3.2}
or, equivalently,
$$
u''=\cond{\rec{~\Gg}}{u>h}{-\rec{\Gg}}{u<h}
$$
The remaining case, $u=h$, can be viewed as infinitely often
alteration of the two above regimes. It corresponds to the condition
$\rec{\Gg} > |u''| $. This regime exists only if $\rec{\Gg}
> |h(x)''| $, but not necessarily for all $x$ that satisfy the inequality.
%
 The minimizer $u(x)$ must be combined of these three
regimes. The Weierstrass-Erdmann conditions requests that $u(x)$ and
$u'(x)$ are continuous functions. To summarize, $u(x)$ satisfies the
system of relations:

 \beq
 \bay{ll}
 u=h , & |u''|\leq \rec{\Gg} \\
 u<h, & u''= -\Gg \\
 u>h, & u''= \Gg \\
\eay, \quad u\in C_1(a, b)
 \eeq{122.2.41}
\begin{example}[Discontinuity]
Let $h(x)$ be $h(x)=\sign (x)$. Check that $u(x)$ is
$$
u(x)=\left\{\bay{ll} h(x)=\sign(x) & \If |x| \geq \rec{\Gg}
\\
 \left(\sqrt{\f{2}{\Gg}} x- \f{x^2}{2 \Gg}\sign(x) \right) & \If |x|
<\rec{\Gg}
 \eay \right.
$$

\end{example}
Notice that behavior of $h(x)$ in the proximity of the discontinuity
point does not affect $u(x)$ in that region. Similarly, the value of
penalty coefficient $\Gg$ does not affect $u$ away from the
discontinuity point. This penalization scheme results in function
$u(x)$ that coincides with $h(x)$ away from discountinuities and
zones of large values of $h''$. Near these zones, $h(x)$ is smoothed
-- it is replaced by parabolas.

\begin{problem}
A. Let $h(x)$ be $h(x)=\exp(- \Ga|x|)$. Find $u(x)$.  Graph the results.
\end{problem}
\begin{problem}
B. Let $h(x)$ be $h(x)=\cos( \Ga x)$. Find $u(x)$. Consider different
cases depending on the value of $\Ga\in R$.  Graph the results.

\end{problem}


\subsection{Approximation with penalized total variation} This
approximation penalizes the function for its  variation. The total
variation $TV(f)$ of a function $u(x)$ is defined as
$$
TV(u)=\int_a^b |u'(x)|dx
$$
For a monotonic function $u(x)$ one evaluates the integral and finds
that
$$
TV(u)=\max_{x\in [a, b]} u(x)-\min_{x\in [a, b]} u(x)
$$
If $u(x)$ has $N$ intervals $L_k$ of monotonicity $(N < \infty)$,
the total variation is
$$
TV(u)=\sum_k^N \left(\max_{x\in L_k} u(x)-\min_{x\in L_k}
u(x)\right)
$$
The   variational problem with total-variation penalty has the
form
    \beq
    \min_u J_3(u), \quad J_3(u)=\int_a^b
    \hf \left( \Gg| u'| + (u-h)^2 \right) dx
    \eeq{122.2.3}
Here, $\Gg\geq 0$, the first term of the integrant represents the
 total-variation penalty and the second term describes the
closeness of the original curve and the approximate.  When $\Gg \to
0$, the approximation coincides with $h(x)$, and when  $\Gg \to
\infty$, the approximation becomes constant equal to the mean value
of $h$.

The formal application of the stationarity technique gives:
    \beq
    (\Gg \sign ( u'))' +u = h, \quad \sign(u'(a))=\sign(u'(b))=0
    \eeq{122.2.4}
This formula is not very helpful because it requires the
differentiation of a discontinuous function $\sign$; besides, the
Lagrangian \eq{122.2.3} is not a twice-differential function of $u'$
as it is required in the procedure of derivation  of the Euler
equation.

%\paragraph{Regularization}

%Regularization is a trick to deal with under-determined problems. We
%want to replace the irregular term $w(u')=\Gg| u'|$ in the
%Lagrangian with another one that is (i) close to the replaced term,
%and (ii) is a differentiable function of $x$ so that the formalism
%of stationarity condition holds.


To deal with this problem, we introduce a family $ W(u', \Ge)$ such
that
$$ \lim_{\Ge\to 0} W(u',
0)=w(u'),
$$
and $ W(u', \Ge)$ is twice-differentiable function of $u'$ for all
$\Ge>0$. Particularly, we can use the family
 \beq W_1(u', \Ge)=\sqrt{u'(x)^2 +\Ge^2}
. \eeq{122.3.sqrt} to approximate $w(u')=|u'|$.
\begin{remark}
We could also choose another family of functions which derivarive
converges to $w(u')=\sign(u')$ when parameter $\Ge$ of the family
goes to zero, $\Ge\to 0$, for example
 \beq
 W_2(u'(x), \Ge)=\f{2}{\pi} \int_0 ^x\arctan \left( \f{u'(t)}{
\Ge} \right) dt= \f{2}{\pi} \left( x \arctan \f{x}{\Ge} - \rec{\Ge}
\log \f{x^2 + \Ge^2}{\Ge^2}\right).
 \eeq{122.3.arctan}
 or
 \beq
W_3(u', \Ge)=\cond{|u'|}{|u'| > \Ge}{\f{u'^2}{2 \Ge} + \hf\Ge}{|u'|
< \Ge}.
 \eeq{122.3.quad}

The derivatives of these functions are:
 \beqan
\pd{W_1}{u'} &=& \f{u'}{\sqrt{u'(x)^2 +\Ge^2}}
\\
\pd{W_2}{u'} &=& \f{2}{\pi}\arctan \left( \f{u'(x)}{ \Ge}\right)
\\
\pd{W_3}{u'} &=&\cond{\sign (u')}{|u'| > \Ge}{\f{u'}{ \Ge}}{|u'| <
\Ge}.
 \eeqan
These are continuous functions that tend to the discontinuous
function $sign(u')$ when $\Ge\to 0$.

\end{remark}



It need to be shown that the solutions of minimization problem with
function $ W(u', \Ge)$ converge to the solution of the original
problem when $\Ge \to 0$. If this is true, than the problem is {\em
regularized}. This technique is called {\em regularization method}.
It  allows for construction of solutions even if the original
problem is not well posed. Here, we do not prove but assume the
convergence of the family of solutions.



The {\em regularized} Lagrangian is
$$
L(u, u',\Ge)= \Gg  \sqrt{u'(x)^2 +\Ge^2}+ \hf (u-f)^2
$$
Now we can compute Euler equation. We have
$$
\pd{L(u, u', \Ge)}{u'}=\Gg \f{u'}{(|u'|^2+ \Ge^2)^{\f{1}{2}}}
$$
and the Euler equation is
    \beq
R(u',\Ge) u''=u-  h , \quad\mbox{where }
R(u',\Ge)=\f{\Ge^2}{\left(u'^2+\Ge^2\right)^{\f{3}{2}}}
    \eeq{122.2.5}
The second step of  procedure is the asymptotic analysis of the
stationary approximate when $\Ge\to 0$. Coefficient $R(u',\Ge)$
reaches its maximum at $u'=0$. It is or the order of $\rec{\Ge}$,  $
R(0,\Ge)=\rec{\Ge}$.  $R(u',\Ge)$ quickly and monotonically decays
when $u'$ grows. When $u'=\sqrt{\Ge}$, the coefficient $R(u',\Ge)$
approximately equals $\sqrt{\Ge}$, and it becomes even smaller for
larger values of $u'$. Indeed, one can check that
$R(\sqrt{\Ge},\Ge)=\sqrt{\Ge}+ o(\sqrt{\Ge}))$.
%
%%We compute $R(\Ge,\Ge)=2^{-\f{3}{2}}\rec{\Ge}$ and conclude that
%%outside $\Ge$-neighborhood of  zero, this coefficient is
%%$\Ge$-small, and inside this interval it can be arbitrary large when
%%$\Ge\to 0$.
%
We conclude that the stationary condition \eq{122.2.5} can be
satisfied (up to the order of $\Ge$) in one of two ways.
\begin{itemize}
\item When $|h'|\gg \Ge $ the solution $u=h$ is stationary. Indeed,
in this case $ |u'| =|h'|> \Ge$, $ R(u', \Ge) \ll \Ge $, and  first
term $R(u',\Ge) u''$ does not influence the equation.

\item When $ u(x)$ is approximately constant, $|u'| \leq \Ge$, the
first term is extremely sensitive to the variation of $u'$ and it
can take any value; in particular, it can compensate the second term
$u-h$ of the equality. The (always) constant $u$ is another
stationary solution.
\end{itemize}

This sketch shows that in the limit $\Ge \to 0$, the stationary
condition \eq{122.2.5} is satisfied either when $u(x)$ is a
constant, $ u'= 0$, or when $u(x)$ coincides with $h(x)$.
$$
u(x)=h(x) \quad \mbox{or }u'(x)=0, \quad \forall x\in[a, b]
$$

The approximation cuts the maxima and minima of the approximating
function, which agrees with the expected behavior of $TV$-penalized
approximation. Indeed, the value of $TV(u)$ depends only on its
local maxima and minima and is invariant to the intermediate values
of $u(x)$. Therefore, the equality $u=h$ does not changes the
penalty  $TV(u)$ but decreases the terms $(u-h)^2$. Near the maxima
(minima) points, the cut $u=\Cnst(x)$ decreases the penalty $TV(u)$
and slightly increases the norm of difference of $u$ and $h$.


Let us find the cutting points. For simplicity in notations we
assume that the function $h(x)$ monotonically increases on $[a, b]$.
The approximation $u(x)$ is also a monotonically increasing
function, $ u' \geq 0$ that either coincides with $h(x)$ or stays
constant cutting the maximum and the minimum of $h(x)$:
$$
u(x)=\left\{ \bay{ll} h(\Ga) &\If x\in [a, \Ga] \\
 h(x) &\If x\in [\Ga, \Gb] \\
 h(\Gb) &\If x\in [\Gb, b] \eay \right.
 $$
 The cost of the problem
 $$
J=\f{\Gg}{2} \left[\int_a^\Ga  (h(x)-h(\Ga))^2 dx + \int_\Gb^b
(h(x)-h(\Gb))^2 dx \right] + h(\Gb)- h(\Ga)
$$
depends on two unknown parameters, $\Ga$ and $\Gb$, the
coordinates on the cuts. They are found by straight
differentiation. The equation for $\Ga$ is
 \beqan
 \od{J}{\Ga}= h'(\Ga) \left(\Gg \int_a^\Ga  (h(x)-h(\Ga))dx -1\right)=0\eeqan
 or, noticing the cut point $\Ga$ is not a stationary point,
 $h'(\Ga)\neq 0 $
\beqan  \int_a^\Ga  [h(x)-h(\Ga)]dx =\f{1}{\Gg}
    \eeqan
 the equation for $\Gb$ is similar:
\beqan  \int_\Gb^b [ h(x)-h(\Gb)] dx=\f{1}{\Gg}
    \eeqan
Notice that the extremal is broken; regular variational method based
on the Euler equation is not effective. These irregular problems
will be discussed later in Chapter \ref{irregular}.
\paragraph{Problems}
 \begin{problem}
 Use the regularization \eq{122.3.quad} instead of \eq{122.3sqrt} and find a family of
solutions in a regularized problem.  Graph the results.
 \end{problem}
 \begin{problem}
 Let $h(x) $ be
 \beqan
 h_1(x)&=& \cos(\Go x), x\in [-\pi, \pi] \\
 h_2(x) &=&  \exp(-\Go |x|), x\in [-\pi, \pi]\\
h_3(x) &=&  \sign (x),~ x\in [-\pi, \pi]
 \eeqan
 Find all three
 approximate . Discuss the difference.  Graph the results.
 \end{problem}

 \begin{problem}
Replace term $|u-h| $ in \eq{123.3.1} by a family of smoother
functions \eq{122.3.arctan} or \eq{122.3.quad} and investigate
stationarity conditions.  Graph the results.
 \end{problem}

 \begin{problem}[Project]
 Consider the problem
$$
\min_{u(x)} \int_a^b( \Gg|u'|+ |u-f|) dx $$ Investigate the
properties of $u$. Use regularization by different families
\eq{122.3.sqrt}  \eq{122.3.arctan}  \eq{122.3.quad}
 \end{problem}

\begin{problem}[Project]
Regularize the problem of minimal surface and find both the catenoid
and Goldschmidt solution by a regular variational method.
 \end{problem}
