If I had to do it on a traditional engine without the extensions of LuaTeX or eTeX and without expl3 programming layer, maybe I would do it like this:
\makeatletter
%%=============================================================================
%% PARAPHERNALIA:
%% \UD@firstoftwo, \UD@secondoftwo, \UD@PassFirstToSecond, \UD@Exchange,
%% \UD@removespace, \UD@stopromannumeral, \UD@CheckWhetherNull,
%% \UD@CheckWhetherBrace, \UD@CheckWhetherLeadingExplicitSpace,
%%=============================================================================
\newcommand\UD@firstoftwo[2]{#1}%
\newcommand\UD@secondoftwo[2]{#2}%
\newcommand\UD@PassFirstToSecond[2]{#2{#1}}%
\newcommand\UD@Exchange[2]{#2#1}%
\@ifdefinable\UD@removespace{\UD@Exchange{ }{\def\UD@removespace}{}}%
\@ifdefinable\UD@stopromannumeral{\chardef\UD@stopromannumeral=`\^^00 }%
%%-----------------------------------------------------------------------------
%% Check whether argument is empty:
%%.............................................................................
%% \UD@CheckWhetherNull{<Argument which is to be checked>}%
%% {<Tokens to be delivered in case that argument
%% which is to be checked is empty>}%
%% {<Tokens to be delivered in case that argument
%% which is to be checked is not empty>}%
%%
%% The gist of this macro comes from Robert R. Schneck's \ifempty-macro:
%% <https://groups.google.com/forum/#!original/comp.text.tex/kuOEIQIrElc/lUg37FmhA74J>
\newcommand\UD@CheckWhetherNull[1]{%
\romannumeral\expandafter\UD@secondoftwo\string{\expandafter
\UD@secondoftwo\expandafter{\expandafter{\string#1}\expandafter
\UD@secondoftwo\string}\expandafter\UD@firstoftwo\expandafter{\expandafter
\UD@secondoftwo\string}\expandafter\UD@stopromannumeral\UD@secondoftwo}{%
\expandafter\UD@stopromannumeral\UD@firstoftwo}%
}%
%%-----------------------------------------------------------------------------
%% Check whether argument's first token is an explicit character token
%% of category 1 (begin group):
%%.............................................................................
%% \UD@CheckWhetherBrace{<Argument which is to be checked>}%
%% {<Tokens to be delivered in case that argument
%% which is to be checked has a leading
%% explicit catcode-1-character-token>}%
%% {<Tokens to be delivered in case that argument
%% which is to be checked does not have a
%% leading explicit catcode-1-character-token>}%
\newcommand\UD@CheckWhetherBrace[1]{%
\romannumeral\expandafter\UD@secondoftwo\expandafter{\expandafter{%
\string#1.}\expandafter\UD@firstoftwo\expandafter{\expandafter
\UD@secondoftwo\string}\expandafter\UD@stopromannumeral\UD@firstoftwo}{%
\expandafter\UD@stopromannumeral\UD@secondoftwo}%
}%
%%-----------------------------------------------------------------------------
%% Check whether brace-balanced argument's first token is an explicit
%% space-token:
%%.............................................................................
%% \UD@CheckWhetherLeadingExplicitSpace{<Argument which is to be checked>}%
%% {<Tokens to be delivered in case <argument
%% which is to be checked> does have a
%% leading explicit space-token>}%
%% {<Tokens to be delivered in case <argument
%% which is to be checked> does not have a
%% a leading explicit space-token>}%
\newcommand\UD@CheckWhetherLeadingExplicitSpace[1]{%
\romannumeral\UD@CheckWhetherNull{#1}%
{\expandafter\UD@stopromannumeral\UD@secondoftwo}%
{%
% Let's nest things into \UD@firstoftwo{...}{} to make sure they are nested in braces
% and thus do not disturb when the test is carried out within \halign/\valign:
\expandafter\UD@firstoftwo\expandafter{%
\expandafter\expandafter\expandafter\UD@stopromannumeral
\romannumeral\expandafter\UD@secondoftwo
\string{\UD@CheckWhetherLeadingExplicitSpaceB.#1 }{}%
}{}%
}%
}%
\@ifdefinable\UD@CheckWhetherLeadingExplicitSpaceB{%
\long\def\UD@CheckWhetherLeadingExplicitSpaceB#1 {%
\expandafter\UD@CheckWhetherNull\expandafter{\UD@firstoftwo{}#1}%
{\UD@Exchange{\UD@firstoftwo}}{\UD@Exchange{\UD@secondoftwo}}%
{\expandafter\expandafter\expandafter\UD@stopromannumeral
\expandafter\expandafter\expandafter}%
\expandafter\UD@secondoftwo\expandafter{\string}%
}%
}%
%%-----------------------------------------------------------------------------
%% Extract first inner undelimited argument:
%%
%% \UD@ExtractFirstArgUnbraced{ABCDE} yields A
%%
%% \UD@ExtractFirstArgUnbraced{{AB}CDE} yields AB
%%
%% Due to \romannumeral-expansion the result is delivered after two
%% expansion-steps/after "hitting" \ExtractFirstArgUnbraced with
%% \expandafter twice.
%%
%% \UD@ExtractFirstArgUnbraced's argument must not be blank.
%%
%% Use frozen-\relax as delimiter for speeding things up.
%% I chose frozen-\relax because David Carlisle pointed out in
%% <https://tex.stackexchange.com/a/578877>
%% that frozen-\relax cannot be (re)defined in terms of \outer and cannot be
%% affected by \uppercase/\lowercase.
%%
%% \ExtractFirstArgUnbraced's argument may contain frozen-\relax:
%% The only effect is that internally more iterations are needed for
%% obtaining the result.
%%
%%.............................................................................
\@ifdefinable\UD@RemoveTillFrozenrelax{%
\expandafter\expandafter\expandafter\UD@Exchange
\expandafter\expandafter\expandafter{%
\expandafter\expandafter\ifnum0=0\fi}%
{\long\def\UD@RemoveTillFrozenrelax#1#2}{{#1}}%
}%
\expandafter\UD@PassFirstToSecond\expandafter{%
\romannumeral\expandafter
\UD@PassFirstToSecond\expandafter{\romannumeral
\expandafter\expandafter\expandafter\UD@Exchange
\expandafter\expandafter\expandafter{%
\expandafter\expandafter\ifnum0=0\fi}{\UD@stopromannumeral#1{}}%
}{%
\UD@stopromannumeral\romannumeral\UD@ExtractFirstArgUnbracedLoop
}%
}{%
\newcommand\UD@ExtractFirstArgUnbraced[1]%
}%
\newcommand\UD@ExtractFirstArgUnbracedLoop[1]{%
\expandafter\UD@CheckWhetherNull\expandafter{\UD@firstoftwo{}#1}%
{\expandafter\UD@stopromannumeral\UD@secondoftwo{}#1}%
{\expandafter\UD@ExtractFirstArgUnbracedLoop
\expandafter{\UD@RemoveTillFrozenrelax#1}}%
}%
%%=============================================================================
%% \UDExtractFirstBraceGroup
%% {<tokens in case there is no brace group>}%
%% {<argument where to extract content of 1st brace group>}%
%%
%% In case <argument where to extract content of 1st brace group> does not
%% contain a brace group you get <tokens in case there is no brace group>.
%% Otherwise you get the content of the 1st brace group of the <argument where
%% to extract content of 1st brace group>.
%%
%% Due to \romannumeral-expansion the result can be obtained by triggering
%% two expansions on \UDExtractFirstBraceGroup
%%
%%=============================================================================
\newcommand\UDExtractFirstBraceGroup[2]{%
\romannumeral\UD@ExtractFirstBraceGroupLoop{#2}{#1}%
}%
\newcommand\UD@ExtractFirstBraceGroupLoop[2]{%
\UD@CheckWhetherNull{#1}{\UD@stopromannumeral#2}{%
\UD@CheckWhetherBrace{#1}{%
\expandafter\UD@firstoftwo\expandafter{\expandafter}%
\UD@ExtractFirstArgUnbraced{#1}%
}{%
\UD@CheckWhetherLeadingExplicitSpace{#1}{%
\expandafter\UD@ExtractFirstBraceGroupLoop\expandafter{\UD@removespace#1}%
}{%
\expandafter\UD@ExtractFirstBraceGroupLoop\expandafter{\UD@firstoftwo{}#1}%
}%
{#2}%
}%
}%
}%
%%=============================================================================
\makeatother
\documentclass{article}
\begin{document}
\ttfamily\frenchspacing
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
This not {This is the string I want} This not {This not} This not {This not}%
}%
\expandafter\expandafter\expandafter\def
\expandafter\expandafter\expandafter\foo
\expandafter\expandafter\expandafter{%
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
This not {This is the string I want} This not {This not} This not {This not}%
}%
}%
\string\foo\space is: \meaning\foo
\edef\bar{%
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
This not {This is the string I want} This not {This not} This not {This not}%
}%
}%
\string\bar\space is: \meaning\bar
\bigskip
\hrule
\bigskip
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
{This is the string I want} This not%
}%
\expandafter\expandafter\expandafter\def
\expandafter\expandafter\expandafter\bas
\expandafter\expandafter\expandafter{%
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
{This is the string I want} This not%
}%
}%
\string\bas\space is: \meaning\bas
\edef\bat{%
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
{This is the string I want} This not%
}%
}%
\string\bat\space is: \meaning\bat
\bigskip
\hrule
\bigskip
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
This is a string without brace group.%
}%
\expandafter\expandafter\expandafter\def
\expandafter\expandafter\expandafter\bau
\expandafter\expandafter\expandafter{%
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
This is a string without brace group.%
}%
}%
\string\bau\space is: \meaning\bau
\edef\bav{%
\UDExtractFirstBraceGroup{%
Hey, there is no brace group in the string!!!%
}{%
This is a string without brace group.%
}%
}%
\string\bav\space is: \meaning\bav
\end{document}

\edef\foo{...}instead of just\def\foo{...}? (None of the tokens of the\edefis expandable. So in this case there is no difference between\defand\edef.)