0x1 初步分析

MFC程序，根据字符串"Wrong!"找到验证函数0x401ea0

int __thiscall ctf_check_handler(CWnd *this)
{
  CWnd *v1; // esi
  int i; // eax
  WCHAR input; // [esp+Ch] [ebp-310h]
  char v5; // [esp+Eh] [ebp-30Eh]
  char input_1; // [esp+20Ch] [ebp-110h]
  char v7; // [esp+20Dh] [ebp-10Fh]
  DWORD ignore; // [esp+30Ch] [ebp-10h]
  CWnd *v9; // [esp+310h] [ebp-Ch]
  int ret; // [esp+314h] [ebp-8h]
  DWORD flOldProtect; // [esp+318h] [ebp-4h]

  v1 = this;
  v9 = this;
  input = 0;
  memset(&v5, 0, 0x1FEu);
  input_1 = 0;
  memset(&v7, 0, 0xFFu);
  CWnd::GetDlgItemTextW(v1, 1000, &input, 20);
  if ( wcslen(&input) == 16 )
  {
    i = 0;
    while ( !(*(&input + i) & 0xFF00) )
    {
      *(&input_1 + i) = *((_BYTE *)&input + 2 * i);
      if ( ++i >= 16 )
      {
        ignore = 64;
        flOldProtect = 0;
        VirtualProtect(ctf_check, 0xD17u, 0x40u, &flOldProtect);
        if ( GetLastError() )
          return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
        qmemcpy(ctf_check, &shellcode, 0x330u);
        VirtualProtect(ctf_check, 0xD17u, flOldProtect, &ignore);
        if ( !GetLastError() )
        {
          ret = 0;
          ret = ctf_check(&input_1);
          if ( ret == 1 )
            return CWnd::MessageBoxW(v9, L"Congratulations! You are right!", 0, 0);
        }
        v1 = v9;
        return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
      }
    }
  }
  return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
}

key长度为16

if ( wcslen(&input) == 16 )

对话框初始化时解密shellcode:

signed int __thiscall sub_401D80(CDialog *this)
{
  CDialog *v1; // esi
  unsigned int i; // eax

  v1 = this;
  CDialog::OnInitDialog(this);
  SendMessageW(*((HWND *)v1 + 8), 0x80u, 1u, *((_DWORD *)v1 + 29));
  SendMessageW(*((HWND *)v1 + 8), 0x80u, 0, *((_DWORD *)v1 + 29));
  i = 0;
  do
  {
    shellcode[i] ^= 0xABu;
    ++i;
  }
  while ( i < 0x330 );
  return 1;
}

复制解密完的shellcode到一特定地址，传入input，开始验证，返回为1则成功：

VirtualProtect(ctf_check, 0xD17u, 0x40u, &flOldProtect);
if ( GetLastError() )
  return CWnd::MessageBoxW(v1, L"Wrong!", 0, 0);
qmemcpy(ctf_check, shellcode, 0x330u);
VirtualProtect(ctf_check, 0xD17u, flOldProtect, &ignore);
if ( !GetLastError() )
{
  ret = 0;
  ret = ctf_check(&input_1);
  if ( ret == 1 )
    return CWnd::MessageBoxW(v9, L"Congratulations! You are right!", 0, 0);
}

0x2 验证算法分析

为了便于调试，我nop掉了复制shellcode的步骤，然后写了一段简单的Idapython把解密后的shellcode放到正确的位置上：

origin_code = GetManyBytes(0x5647b8,0x330)

addr = 0x4010e0
for i,c in enumerate(origin_code):
    PatchByte(addr+i,ord(c)^0xab)

F5出来的伪代码：

signed int __cdecl ctf_check(char *p)
{
  signed int i; // eax
  unsigned __int8 v2; // cl
  signed int v3; // ecx
  signed int v4; // eax
  signed int v5; // eax
  signed int i_1; // esi
  signed int i_2; // ecx
  __int16 c; // dx
  unsigned __int8 *v9; // edi
  __int16 v10; // ax
  signed int v11; // eax
  signed int j; // ecx
  unsigned __int16 v13; // bx
  signed int i_3; // esi
  signed int j_2; // ecx
  __int16 c_1; // dx
  unsigned __int8 *v17; // edi
  __int16 v18; // ax
  signed int v19; // eax
  signed int j_1; // ecx
  unsigned __int16 v21; // bx
  unsigned __int8 v22[4]; // eax
  signed int i_4; // ecx
  unsigned __int16 v24; // dx
  char bflag; // dl
  signed int i_5; // eax
  __int16 t; // si
  int v28; // eax
  char xor_data1[8]; // [esp+8h] [ebp-90h]
  char xor_data2[8]; // [esp+10h] [ebp-88h]
  int v32; // [esp+18h] [ebp-80h]
  int v33; // [esp+1Ch] [ebp-7Ch]
  int v34; // [esp+20h] [ebp-78h]
  int v35; // [esp+24h] [ebp-74h]
  unsigned __int8 data1[8]; // [esp+28h] [ebp-70h]
  unsigned __int8 buf[16]; // [esp+30h] [ebp-68h]
  unsigned __int8 data2[8]; // [esp+40h] [ebp-58h]
  unsigned __int8 outbuf3[17]; // [esp+48h] [ebp-50h]
  unsigned __int8 outbuf4[17]; // [esp+5Ch] [ebp-3Ch]
  unsigned __int8 outbuf1[17]; // [esp+70h] [ebp-28h]
  unsigned __int8 outbuf2[17]; // [esp+84h] [ebp-14h]

  xor_data1[4] = 0x81u;
  xor_data2[3] = 0x81u;
  i = 0;
  xor_data1[0] = 0x16;
  xor_data1[1] = 0x96u;
  xor_data1[2] = 0x8Cu;
  xor_data1[3] = 0xE3u;
  xor_data1[5] = 0x98u;
  xor_data1[6] = 0x6E;
  xor_data1[7] = 0x64;
  xor_data2[0] = 0x84u;
  xor_data2[1] = 8;
  xor_data2[2] = 0xDCu;
  xor_data2[4] = 0xBEu;
  xor_data2[5] = 0x4D;
  xor_data2[6] = 0x48;
  xor_data2[7] = 0x4F;
  v32 = 0;
  v33 = 0;
  v34 = 0;
  v35 = 0;
  *(_DWORD *)data1 = 0;
  *(_DWORD *)&data1[4] = 0;
  *(_DWORD *)data2 = 0;
  *(_DWORD *)&data2[4] = 0;
  do
  {
    v2 = xor_data2[i] ^ p[i + 8];
    data1[i] = xor_data1[i] ^ xor_data1[i + p - xor_data1];
    data2[i++] = v2;
  }
  while ( i < 8 );
  *(_DWORD *)xor_data1 = 0;
  *(_DWORD *)outbuf1 = 0;
  *(_DWORD *)&outbuf1[4] = 0;
  *(_DWORD *)&outbuf1[8] = 0;
  *(_DWORD *)&outbuf1[12] = 0;
  outbuf1[16] = 0;
  *(_DWORD *)outbuf2 = 0;
  *(_DWORD *)&outbuf2[4] = 0;
  *(_DWORD *)&outbuf2[8] = 0;
  *(_DWORD *)&outbuf2[12] = 0;
  outbuf2[16] = 0;
  *(_DWORD *)outbuf3 = 0;
  *(_DWORD *)&outbuf3[4] = 0;
  *(_DWORD *)&outbuf3[8] = 0;
  *(_DWORD *)&outbuf3[12] = 0;
  outbuf3[16] = 0;
  *(_DWORD *)outbuf4 = 0;
  *(_DWORD *)&outbuf4[4] = 0;
  *(_DWORD *)&outbuf4[8] = 0;
  *(_DWORD *)&outbuf4[12] = 0;
  outbuf4[16] = 0;
  *(_DWORD *)&xor_data1[4] = 0;
  *(_DWORD *)xor_data2 = 0;
  *(_DWORD *)&xor_data2[4] = 0;
  LOBYTE(v32) = 0;
  v3 = 8;
  xor_data1[0] = 8;
  v4 = 7;
  do
  {
    if ( data1[v4] )
      break;
    --v3;
    --v4;
  }
  while ( v4 >= 0 );
  if ( v3 == 8 )
  {
    v5 = 7;
    do
    {
      if ( data2[v5] )
        break;
      --v3;
      --v5;
    }
    while ( v5 >= 0 );
    if ( v3 == 8 && !(data1[7] & 0xF0) )
    {
      i_1 = 0;
      do
      {
        *(_DWORD *)buf = 0;
        *(_DWORD *)&buf[4] = 0;
        *(_DWORD *)&buf[8] = 0;
        *(_DWORD *)&buf[12] = 0;
        i_2 = 0;
        c = data1[i_1];
        v9 = &buf[i_1];
        do
        {
          v10 = buf[i_1 + 8] + c * data1[i_2];
          v9[i_2] = buf[i_1 + 8] + c * data1[i_2];
          ++i_2;
          buf[i_1 + 8] = HIBYTE(v10);
        }
        while ( i_2 < 8 );
        LOBYTE(v11) = 0;
        j = 0;
        do
        {
          v13 = (char)v11 + outbuf1[j + i_1] + v9[j];
          outbuf1[j++ + i_1] = v13;
          v11 = (signed int)v13 >> 8;
        }
        while ( j < 9 );
        ++i_1;
      }
      while ( i_1 < 8 );
      i_3 = 0;
      do
      {
        *(_DWORD *)buf = 0;
        *(_DWORD *)&buf[4] = 0;
        *(_DWORD *)&buf[8] = 0;
        *(_DWORD *)&buf[12] = 0;
        j_2 = 0;
        c_1 = data2[i_3];
        v17 = &buf[i_3];
        do
        {
          v18 = buf[i_3 + 8] + c_1 * data2[j_2];
          v17[j_2] = buf[i_3 + 8] + c_1 * data2[j_2];
          ++j_2;
          buf[i_3 + 8] = HIBYTE(v18);
        }
        while ( j_2 < 8 );
        LOBYTE(v19) = 0;
        j_1 = 0;
        do
        {
          v21 = (char)v19 + outbuf2[j_1 + i_3] + v17[j_1];
          outbuf2[j_1++ + i_3] = v21;
          v19 = (signed int)v21 >> 8;
        }
        while ( j_1 < 9 );
        ++i_3;
      }
      while ( i_3 < 8 );
      v22[0] = outbuf3[16];
      i_4 = 0;
      do
      {
        v24 = v22[0] + 7 * outbuf2[i_4];
        outbuf3[i_4++] = v24;
        *(_DWORD *)v22 = (unsigned int)v24 >> 8;
      }
      while ( i_4 < 17 );
      outbuf3[16] = HIBYTE(v24);
      bflag = 0;
      i_5 = 0;
      do
      {
        t = outbuf1[i_5] - outbuf3[i_5] - bflag;
        outbuf4[i_5] = t;
        if ( t < 0 )
          bflag = 1;
        ++i_5;
      }
      while ( i_5 < 17 );
      if ( !bflag )
      {
        v28 = 0;
        while ( outbuf4[v28] == xor_data1[v28] )
        {
          if ( ++v28 >= 17 )
            return 1;
        }
      }
    }
  }
  return 0;
}

看起来乱七八糟的，于是我把它等价转成了Python:

xor_data1 = [0x16,0x96,0x8c,0xe3,0x81,0x98,0x6e,0x64]
xor_data2 = [0x84,0x8, 0xdc,0x81,0xbe,0x4d,0x48,0x4f]


input_s = '7654321e87654321'

data1 = [xor_data1[i]^ord(input_s[i]) for i in range(8)]
data2 = [xor_data2[i]^ord(input_s[i+8]) for i in range(8)]

outbuf1 = [0]*17
outbuf2 = [0]*17
outbuf3 = [0]*17
outbuf4 = [0]*17
def loop1():
    for i in range(8):
        buf = [0]*16
        ch = data1[i]

        for j in range(8):
            t = buf[i+8] + ch*data1[j]
            buf[i+j] = t%256
            buf[i+8] = t>>8

        t1 = 0
        for j in range(9):
            t2 = t1 + outbuf1[j+i] + buf[i+j]
            outbuf1[i+j] = t2%256
            t1 = t2>>8

def loop2():
    for i in range(8):
        buf = [0]*16
        ch = data2[i]

        for j in range(8):
            t = buf[i+8] + ch*data2[j]
            buf[i+j] = t%256
            buf[i+8] = t>>8

        t1 = 0
        for j in range(9):
            t2 = t1 + outbuf2[j+i] + buf[i+j]
            outbuf2[i+j] = t2%256
            t1 = t2>>8

def loop3():
    t1 = 0
    for i in range(17):
        t2 = t1 + 7*outbuf2[i]
        outbuf3[i] = t2%256
        t1 = t2>>8
    outbuf3[16] = t2>>8

def loop4():
    bflag = 0
    for i in range(17):
        t = outbuf1[i] - outbuf3[i] - bflag
        outbuf4[i] = t%256
        if t<0:
            bflag = 1

# loop1()
# loop2()
# loop3()
# loop4()
# print(list(map(hex,outbuf1)))
# print(list(map(hex,outbuf2)))
# print(list(map(hex,outbuf3)))
# print(list(map(hex,outbuf4)))

data1 = ['5a', 'a5', 'e1', 'b9', 'b3', 'f3', '57', '05']
data2 = ['de', '38', 'bd', 'b2', '88', '09', '05', '02']
o1 = [chr(int(data1[i],16)^xor_data1[i]) for i in range(8)]
o2 = [chr(int(data2[i],16)^xor_data2[i]) for i in range(8)]

print(''.join(o1+o2))

总结一下：

16字节的输入先和固定值异或，然后分为两部分长度为8的256进制大数x,y
loop1计算x**2
loop2计算y**2
loop3计算7*y**2
loop4计算x**2-7*y**2
最后的结果与8比较，相等就返回1。
另外还有对x,y大小的限制：
(1<<56)<=x<(1<<60)
y>=(1>>56)

直接用wolframalpha解得
x = 385044246406735194, y = 145533045678356702
最后用上面的代码转换为输入，最后的key为L3mZ2k9aZ0a36DMM

loop4的减法实现似乎有点问题，借位为1就一直为1了，但是并不影响最后的答案，因为一旦借位，就返回0了。

[CTF入门培训]顶尖高校博士及硕士团队亲授《30小时教你玩转CTF》，视频+靶场+题目！助力进入CTF世界