You can use your approach if you ask Solve for {a2, b1, b2}
a1 = 1;
soln1 = Solve[{a2, b2} == {{t, -I*κ}, {-I*κ, t}}.{a1, b1} &&
b1 == b2*Exp[-I*(2 π)/λ*n*2 π*R], {a2, b1, b2}][[1]]
(* {a2 -> (E^((4 I n π^2 R)/λ) t - t^2 - κ^2)/(
E^((4 I n π^2 R)/λ) - t),
b1 -> -((I κ)/(E^((4 I n π^2 R)/λ) - t)),
b2 -> -((I E^((4 I n π^2 R)/λ) κ)/(
E^((4 I n π^2 R)/λ) - t))} *)
You can also include a1==1 either directly,
Clear[a1]
soln2 = Solve[
And @@ Thread[{a2, b2} == {{t, -I*κ}, {-I*κ, t}}.{a1, b1}] &&
b1 == b2*Exp[-I*(2 π)/λ*n*2 π*R] && a1 == 1, {a2, b1,
b2}][[1]]
(* {a2 -> (E^((4 I n π^2 R)/λ) t - t^2 - κ^2)/(
E^((4 I n π^2 R)/λ) - t),
b1 -> -((I κ)/(E^((4 I n π^2 R)/λ) - t)),
b2 -> -((I E^((4 I n π^2 R)/λ) κ)/(
E^((4 I n π^2 R)/λ) - t))} *)
or as an assumption to Simplify or FullSimplify
soln3 = Solve[
And @@ Thread[{a2, b2} == {{t, -I*κ}, {-I*κ, t}}.{a1, b1}] &&
b1 == b2*Exp[-I*(2 π)/λ*n*2 π*R], {a2, b1, b2}][[1]] //
Simplify[#, a1 == 1] &
(* {a2 -> -((-E^(((4 I n π^2 R)/λ)) t + t^2 + κ^2)/(
E^((4 I n π^2 R)/λ) - t)),
b1 -> -((I κ)/(E^((4 I n π^2 R)/λ) - t)),
b2 -> -((I E^((4 I n π^2 R)/λ) κ)/(
E^((4 I n π^2 R)/λ) - t))} *)
Verifying that all three approaches are equivalent
({a2, b1, b2} /. soln1) == ({a2, b1, b2} /. soln2) == ({a2, b1, b2} /.
soln3) // Simplify
(* True *)
Solve[{a2, b2} == {{t, -I*\[Kappa]}, {-I*\[Kappa], t}}.{a1, b2*Exp[-I*(2 \[Pi])/\[Lambda]*n*2 \[Pi]*R]}, {a2, b2}]– Henrik Schumacher Sep 24 '17 at 22:19