My eyes bleed, but here it is:
\documentclass{article}
\usepackage{xparse}
\NewDocumentCommand{\dgal}{sO{}m}{%
\IfBooleanTF{#1}
{\dgalext{#3}}
{\dgalx[#2]{#3}}%
}
\NewDocumentCommand{\dgalext}{m}{%
\sbox0{%
\mathsurround=0pt % just for safety
$\left\{\vphantom{#1}\right.\kern-\nulldelimiterspace$%
}%
\sbox2{\{}%
\ifdim\ht0=\ht2
\{\kern-.625\wd2 \{#1\}\kern-.625\wd2 \}%
\else
\left\{\kern-.7\wd0\left\{#1\right\}\kern-.7\wd0\right\}%
\fi
}
\NewDocumentCommand{\dgalx}{om}{%
\sbox0{\mathsurround=0pt$#1\{$}%
\sbox2{\{}%
\ifdim\ht0=\ht2
\{\kern-.625\wd2 \{#2\}\kern-.625\wd2 \}%
\else
\mathopen{#1\{\kern-.7\wd0 #1\{}
#2
\mathclose{#1\}\kern-.7\wd0 #1\}}
\fi
}
\begin{document}
\def\test#1{#1|xxx}
$\dgal*{\test{}}$
$\dgal*{\test{\big}}$
$\dgal*{\test{\Big}}$
$\dgal*{\test{\bigg}}$
$\dgal*{\test{\Bigg}}$
$\dgal{xxx}$
$\dgal[\big]{xxx}$
$\dgal[\Big]{xxx}$
$\dgal[\bigg]{xxx}$
$\dgal[\Bigg]{xxx}$
\end{document}

If I change .625 into .45 and .7 into .5, the result seems better:
\documentclass{article}
\usepackage{xparse}
\NewDocumentCommand{\dgal}{sO{}m}{%
\IfBooleanTF{#1}
{\dgalext{#3}}
{\dgalx[#2]{#3}}%
}
\NewDocumentCommand{\dgalext}{m}{%
\sbox0{%
\mathsurround=0pt % just for safety
$\left\{\vphantom{#1}\right.\kern-\nulldelimiterspace$%
}%
\sbox2{\{}%
\ifdim\ht0=\ht2
\{\kern-.45\wd2 \{#1\}\kern-.45\wd2 \}%
\else
\left\{\kern-.5\wd0\left\{#1\right\}\kern-.5\wd0\right\}%
\fi
}
\NewDocumentCommand{\dgalx}{om}{%
\sbox0{\mathsurround=0pt$#1\{$}%
\sbox2{\{}%
\ifdim\ht0=\ht2
\{\kern-.45\wd2 \{#2\}\kern-.45\wd2 \}%
\else
\mathopen{#1\{\kern-.5\wd0 #1\{}
#2
\mathclose{#1\}\kern-.5\wd0 #1\}}
\fi
}
\begin{document}
\def\test#1{#1|xxx}
$\dgal*{\test{}}$
$\dgal*{\test{\big}}$
$\dgal*{\test{\Big}}$
$\dgal*{\test{\bigg}}$
$\dgal*{\test{\Bigg}}$
$\dgal{xxx}$
$\dgal[\big]{xxx}$
$\dgal[\Big]{xxx}$
$\dgal[\bigg]{xxx}$
$\dgal[\Bigg]{xxx}$
\end{document}
