Annotate a general Sylvester matrix with TikZ

Question

I am trying to annotate the following general Sylvester matrix

with braces on the right side marking "m lines" for the first block and "n lines" for the second. I was trying to accomplish this with the code

\documentclass{article}
\usepackage{tikz}
\usetikzlibrary{matrix}
\usetikzlibrary{decorations.pathreplacing,calc,fit,backgrounds}
\begin{document}
\pgfmathsetmacro{\myscale}{2}
\pgfkeys{tikz/mymatrixbrace/.style={decorate,thick}}
%[
%R_{f,g} =
\begin{tikzpicture}
    \matrix[matrix of math nodes,
            left delimiter=(,
            right delimiter=),
            %nodes={draw},
            row sep=10pt,
            column sep=10pt] (m) {%
    a_0 & a_1    &&         &         && a_{n-1} & a_n     &        && \null  \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        && a_0     & a_1     &&         &         &        &a_{n-1}& a_{n} \
    b_0 & b_1    && b_{m-1} & b_m     &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
\null   &        &&         &         && b_{0}   & b_{1}   &        &b_{m-1}& b_{m}  \
                         };
    \draw[] (m-1-1) -- (m-4-4) -- cycle;
    \draw[] (m-4-11) -- (m-1-8) -- cycle;
    \draw[] (m-5-1) -- (m-13-7) -- cycle;
    \draw[] (m-13-11) -- (m-5-5) -- cycle;
    \draw[dashed] (m-1-7) -- (m-13-7) -- cycle;
    \draw[dotted] (m-1-2) -- (m-1-7) -- cycle;
    \draw[dotted] (m-4-5) -- (m-4-10) -- cycle;
    \draw[dotted] (m-5-2) -- (m-5-4) -- cycle;
    \draw[dotted] (m-13-8) -- (m-13-10) -- cycle;


%    \begin{scope}[every node/.append style={scale=\myscale,transform shape},very thick]
%        \draw[mymatrixbrace] (m.east|-m-1-1.north east) -- node[right=2pt] {$m$ lines} (m.east|-m-4-1.south east);
%        \draw[mymatrixbrace] (m.east|-m-5-1.north east) -- node[right=2pt] {$n$ lines} (m.east|-m-13-1.south east);
%    \end{scope}
\end{tikzpicture}
%]
\end{document}

(line of the annotation code commented out), with part of the code from here, but I am running into problems making it work.

Answer 1

I modified your distances; I think it is better to have relative separation distances among the elements of the matrix. I added baseline=0pt to have the matrix correctly aligned in the equation, and the decoration for the braces.

The code

\documentclass[11pt, margin=1cm]{standalone} %{article}
\usepackage{tikz}
\usetikzlibrary{matrix}
\usetikzlibrary{decorations.pathreplacing, calc, fit, backgrounds}
\begin{document}
$
R_{f,g} =
\begin{tikzpicture}[baseline=0pt]
  \matrix[matrix of math nodes,
  left delimiter=(,
  right delimiter=),
  % nodes={draw},
  row sep=1.25ex,
  column sep=2ex,
  nodes in empty cells] (m) {%
    a_0 & a_1    &&         &         && a_{n-1} & a_n     &        && \null  \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        && a_0     & a_1     &&         &         &        &a_{n-1}& a_{n} \
    b_0 & b_1    && b_{m-1} & b_m     &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        &        &&         &         &&         &         &        &&        \
        \null
        &        &&         &         && b_{0}   & b_{1}   &        &b_{m-1}& b_{m}  \
  };
  \draw[] (m-1-1) -- (m-4-4);
  \draw[] (m-4-11) -- (m-1-8);
  \draw[] (m-5-1) -- (m-13-7);
  \draw[] (m-13-11) -- (m-5-5);
  \draw[dashed] (m-1-7) -- (m-13-7);
  \draw[dotted] (m-1-2) -- (m-1-7);
  \draw[dotted] (m-4-5) -- (m-4-10);
  \draw[dotted] (m-5-2) -- (m-5-4);
  \draw[dotted] (m-13-8) -- (m-13-10);
\draw[decorate, decoration={brace, amplitude=1ex, raise=3em}]
  (m-1-11.north) -- (m-4-11.south)
  node[text=black, midway, right, xshift=3.5em, scale=.9]
  {$m$ rows};

  \draw[decorate, decoration={brace, amplitude=1ex, raise=3em}]
  (m-5-11.north) -- (m-13-11.south)
  node[text=black, midway, right, xshift=3.5em, scale=.9]
  {$n$ rows};

\end{tikzpicture}
$
\end{document}