select distinct rank, letter
  from table1 t2
 where rank in 
         (select top 2 rank
            from table1 t2 
           where t2.letter = t1.letter 
           order by rank)
       order by letter, rank

编辑：（我的第一次尝试将无法在MySql（Quassnoi注释）上工作，例如，我将其修改为可在sql服务器上工作）

第二次尝试：

select t.letter, t.rank
from table1 t
join (
    select t1.letter, min(t1.rank) m
    from table1 t1
    join (select t0.letter, min(t0.rank) m, count(1) c 
           from table1 t0 group by t0.letter) t2
    on t1.letter = t2.letter and ((t2.c = 1) or (t2.c > 1 and t1.rank > m))
    group by t1.letter) t3 
  on t.letter = t3.letter and t.rank <= t3.m