You can use tikzmark:
\documentclass{article}
\usepackage{mathtools}
\usepackage{tikz}
\usetikzlibrary{tikzmark}
\begin{document}
\begin{equation}
\begingroup % keep the change local
\setlength\arraycolsep{2pt}
\begin{pmatrix}
\tikzmarknode{x}{h_{11}}&h_{12}&h_{13}&h_{14}&\cdots&h_{1n}&\cdots&\cdots&h_{1m}\\
h_{21}&h_{22}&h_{23}&h_{24}&\cdots&h_{2n}&\cdots&\cdots&h_{2m}\\
0 & h_{32} &h_{33}&h_{34}&\cdots&h_{3n}&\cdots&\cdots&h_{3m}\\
0 & 0 & h_{43}&h_{44}&\cdots&h_{4n}&\cdots&\cdots&h_{4m}\\
\vdots & \vdots & & \ddots & \ddots & \vdots & & & \vdots \\
0 & 0 & & & \ddots & h_{n,n}& & & h_{nm} \\
0 & 0 & & & & \tikzmarknode{y}{h_{n+1,n}}& & & h_{n+1,m} \\
\vdots & \vdots & & & & & \ddots & \ddots & \vdots \\
0 & 0 & \cdots & & \cdots & \cdots & 0 & h_{m,m-1} & h_{m,m} \\
\end{pmatrix}
\endgroup
\end{equation}
\tikz[remember picture,overlay] \draw (x.north west) rectangle (y.south east);
\end{document}

If you want better spacing,
\documentclass{article}
\usepackage{mathtools}
\usepackage{tikz}
\usetikzlibrary{tikzmark}
\begin{document}
\begin{equation}
\begingroup % keep the change local
\setlength\arraycolsep{2pt}
\begin{pmatrix}
\tikzmarknode[inner sep=.3em]{x}{h_{11}}&h_{12}&h_{13}&h_{14}&\cdots&h_{1n}&\cdots&\cdots&h_{1m}\\
h_{21}&h_{22}&h_{23}&h_{24}&\cdots&h_{2n}&\cdots&\cdots&h_{2m}\\
0 & h_{32} &h_{33}&h_{34}&\cdots&h_{3n}&\cdots&\cdots&h_{3m}\\
0 & 0 & h_{43}&h_{44}&\cdots&h_{4n}&\cdots&\cdots&h_{4m}\\
\vdots & \vdots & & \ddots & \ddots & \vdots & & & \vdots \\
0 & 0 & & & \ddots & h_{n,n}& & & h_{nm} \\
0 & 0 & & & & \tikzmarknode[inner sep=.3em]{y}{h_{n+1,n}}& & & h_{n+1,m} \\
\vdots & \vdots & & & & & \ddots & \ddots & \vdots \\
0 & 0 & \cdots & & \cdots & \cdots & 0 & h_{m,m-1} & h_{m,m} \\
\end{pmatrix}
\endgroup
\end{equation}
\tikz[remember picture,overlay] \draw[red] (x.north west) rectangle (y.south east);
\end{document}
