CTF-RE-cello_rule (lib库函数的分析)

关注

原创最新推荐文章于 2026-03-03 06:10:41 发布 731 阅读 AI 写同款· GEO 优化 ›

IDA Pro *(_QWORD *)和*(_BYTE *)表达式解释这种类型转换和解引用的组合在反汇编和逆向工程中非常常见，因为原始源代码中的数据类型和变量名在编译过程中通常会丢失，而分析人员需要依赖工具来理解和解释机器代码。作为一个整体，通常用于将一个地址（或其他整数）转换为一个指向64位无符号整数的指针，并获取该地址上的值。时，这通常意味着某个地址或值被转换为一个指向字节的指针，并且随后会解引用该指针以获取该地址上的字节值。允许你在不知道原始数据类型的情况下，以特定的方式（这里是64位无符号整数）解释和访问内存中的数据。类似，但这里涉及的是1字节（8位）的数据。阅读详情

题目概述

题目给出了一个linux下的可执行程序和一个加密过的flag.png。可以知道程序是对这个文件进行加密，然后输出加密后的文件。因此我们的思路就是逆向程序的加密方式和逻辑，从而逆推出flag.enc的源文件。

具体分析

由于我们不清楚程序真正逻辑的入口点，因此第一步是ida查看程序的可以字符串。
在这里插入图片描述
这些字符串显然是值得我们去研究的，在google上面可以查到这些字符串的信息。发现这些字符串来源于libCello.so。而import界面找到read相关函数确定重要的函数入口点会出现很多形式一样并且莫名其妙的函数。实际上这些函数在libCello.so已经封装成一个个函数了，但是ida没有识别出来。一个比较好用的方法是使用ida官方的flare工具自己做符号文件然后导入。但是由于在做这题的时候死活没调对环境，我采用了硬性分析的方法。

__int64 __fastcall Read_and_Print(signed int a1, _QWORD *a2)
{
  _QWORD *v2; // rax
  __int64 v3; // rbx
  _QWORD *v4; // rax
  __int64 *v5; // rax
  _QWORD *v6; // rax
  __int64 v7; // rdx
  _QWORD *v8; // rax
  _QWORD *v9; // rax
  _QWORD *v10; // rax
  __int64 v11; // rax
  _QWORD *v12; // rax
  __int64 v13; // rbx
  _QWORD *v14; // rax
  __int64 v15; // rax
  _QWORD *v16; // rax
  __int64 v17; // rbx
  _QWORD *v18; // rax
  _QWORD *v19; // rax
  _QWORD *v20; // rax
  __int64 v21; // rbx
  _QWORD *v22; // rax
  void *v24; // [rsp+20h] [rbp-320h]
  void **v25; // [rsp+30h] [rbp-310h]
  __int64 *v26; // [rsp+40h] [rbp-300h]
  __int64 v27; // [rsp+50h] [rbp-2F0h]
  __int64 *v28; // [rsp+60h] [rbp-2E0h]
  __int64 (__fastcall *v29)(__int64); // [rsp+70h] [rbp-2D0h]
  __int64 *v30; // [rsp+80h] [rbp-2C0h]
  __int64 (__fastcall *v31)(__int64); // [rsp+90h] [rbp-2B0h]
  __int64 *v32; // [rsp+A0h] [rbp-2A0h]
  const char *v33; // [rsp+B0h] [rbp-290h]
  __int64 (__fastcall *v34)(); // [rsp+C0h] [rbp-280h]
  __int64 *v35; // [rsp+D0h] [rbp-270h]
  __int64 *v36; // [rsp+E0h] [rbp-260h]
  __int64 v37; // [rsp+E8h] [rbp-258h]
  __int64 v38; // [rsp+F0h] [rbp-250h]
  __int64 v39; // [rsp+F8h] [rbp-248h]
  __int64 v40; // [rsp+100h] [rbp-240h]
  __int64 v41; // [rsp+108h] [rbp-238h]
  _QWORD *v42; // [rsp+110h] [rbp-230h]
  void *v43; // [rsp+118h] [rbp-228h]
  _QWORD *v44; // [rsp+120h] [rbp-220h]
  void *v45; // [rsp+128h] [rbp-218h]
  __int64 v46; // [rsp+130h] [rbp-210h]
  void *v47; // [rsp+138h] [rbp-208h]
  __int64 v48; // [rsp+140h] [rbp-200h]
  __int64 v49; // [rsp+148h] [rbp-1F8h]
  void *v50; // [rsp+150h] [rbp-1F0h]
  __int64 v51; // [rsp+160h] [rbp-1E0h]
  __int64 v52; // [rsp+168h] [rbp-1D8h]
  void *v53; // [rsp+170h] [rbp-1D0h]
  __int64 v54; // [rsp+180h] [rbp-1C0h]
  __int64 v55; // [rsp+188h] [rbp-1B8h]
  __int64 v56; // [rsp+190h] [rbp-1B0h]
  __int64 v57; // [rsp+198h] [rbp-1A8h]
  __int64 v58; // [rsp+1A0h] [rbp-1A0h]
  __int64 v59; // [rsp+1A8h] [rbp-198h]
  __int64 v60; // [rsp+1B0h] [rbp-190h]
  __int64 v61; // [rsp+1B8h] [rbp-188h]
  __int64 v62; // [rsp+1C0h] [rbp-180h]
  __int64 v63; // [rsp+1C8h] [rbp-178h]
  __int64 v64; // [rsp+1D0h] [rbp-170h]
  __int64 v65; // [rsp+1D8h] [rbp-168h]
  __int64 v66; // [rsp+1E0h] [rbp-160h]
  __int64 v67; // [rsp+1E8h] [rbp-158h]
  __int64 v68; // [rsp+1F0h] [rbp-150h]
  __int64 v69; // [rsp+1F8h] [rbp-148h]
  __int64 v70; // [rsp+200h] [rbp-140h]
  __int64 v71; // [rsp+208h] [rbp-138h]
  __int64 v72; // [rsp+210h] [rbp-130h]
  __int64 v73; // [rsp+218h] [rbp-128h]
  __int64 v74; // [rsp+220h] [rbp-120h]
  __int64 v75; // [rsp+228h] [rbp-118h]
  __int64 v76; // [rsp+230h] [rbp-110h]
  __int64 v77; // [rsp+238h] [rbp-108h]
  __int64 v78; // [rsp+240h] [rbp-100h]
  __int64 v79; // [rsp+248h] [rbp-F8h]
  __int64 v80; // [rsp+250h] [rbp-F0h]
  __int64 v81; // [rsp+258h] [rbp-E8h]
  __int64 v82; // [rsp+260h] [rbp-E0h]
  __int64 v83; // [rsp+268h] [rbp-D8h]
  __int64 v84; // [rsp+270h] [rbp-D0h]
  __int64 v85; // [rsp+278h] [rbp-C8h]
  __int64 v86; // [rsp+280h] [rbp-C0h]
  __int64 v87; // [rsp+288h] [rbp-B8h]
  __int64 v88; // [rsp+290h] [rbp-B0h]
  __int64 v89; // [rsp+298h] [rbp-A8h]
  __int64 v90; // [rsp+2A0h] [rbp-A0h]
  __int64 v91; // [rsp+2A8h] [rbp-98h]
  __int64 v92; // [rsp+2B0h] [rbp-90h]
  __int64 v93; // [rsp+2B8h] [rbp-88h]
  __int64 v94; // [rsp+2C0h] [rbp-80h]
  __int64 v95; // [rsp+2C8h] [rbp-78h]
  __int64 v96; // [rsp+2D0h] [rbp-70h]
  __int64 v97; // [rsp+2D8h] [rbp-68h]
  __int64 v98; // [rsp+2E0h] [rbp-60h]
  __int64 v99; // [rsp+2E8h] [rbp-58h]
  __int64 v100; // [rsp+2F0h] [rbp-50h]
  __int64 v101; // [rsp+2F8h] [rbp-48h]
  __int64 v102; // [rsp+300h] [rbp-40h]
  __int64 v103; // [rsp+308h] [rbp-38h]
  __int64 v104; // [rsp+310h] [rbp-30h]
  __int64 v105; // [rsp+318h] [rbp-28h]
  __int64 v106; // [rsp+320h] [rbp-20h]
  unsigned __int64 v107; // [rsp+328h] [rbp-18h]

  v107 = __readfsqword(0x28u);
  if ( a1 <= 1 )
  {
    printf("usage: %s file\n", *a2, a2);
    exit(1);
  }
  v24 = off_636F70;
  v25 = &v24;
  v58 = 0LL;
  v59 = 0LL;
  v60 = 0LL;
  v61 = 0LL;
  v2 = Some_fun(&v58, off_63A780, 2);
  v3 = v2;
  *v2 = v25;
  v54 = 0LL;
  v55 = 0LL;
  v56 = 0LL;
  v57 = 0LL;
  v4 = Some_fun(&v54, off_636460, 2);
  *v4 = Rand_place;
  v5 = sub_40A357(v4, v3);
  v36 = v5;
  v40 = *v5;
  v41 = 30LL;
  v102 = 0LL;
  v103 = 0LL;
  v104 = 0LL;
  v105 = 0LL;
  v106 = 0LL;
  v6 = Some_fun(&v102, off_6332D0, 2);
  v7 = v41;
  *v6 = v40;
  v6[1] = v7;
  v42 = v6;
  v43 = off_636F70;
  v26 = &v42;
  v62 = 0LL;
  v63 = 0LL;
  v64 = 0LL;
  v65 = 0LL;
  v8 = Some_fun(&v62, off_63A780, 2);
  *v8 = v26;
  v37 = New(off_6332D0, v8);
  v27 = a2[1];
  v70 = 0LL;
  v71 = 0LL;
  v72 = 0LL;
  v73 = 0LL;
  v9 = Some_fun(&v70, off_639A70, 2);
  *v9 = v27;
  v44 = v9;
  v45 = off_636F70;
  v28 = &v44;
  v66 = 0LL;
  v67 = 0LL;
  v68 = 0LL;
  v69 = 0LL;
  v10 = Some_fun(&v66, off_63A780, 2);
  *v10 = v28;
  v11 = New(off_639A70, v10);
  v38 = v11;
  v46 = v11;
  v47 = off_636F70;
  v30 = &v46;
  v78 = 0LL;
  v79 = 0LL;
  v80 = 0LL;
  v81 = 0LL;
  v12 = Some_fun(&v78, off_63A780, 2);
  v13 = v12;
  *v12 = v30;
  v29 = Next1;
  v74 = 0LL;
  v75 = 0LL;
  v76 = 0LL;
  v77 = 0LL;
  v14 = Some_fun(&v74, off_636460, 2);
  *v14 = v29;
  v15 = sub_40A357(v14, v13);
  v39 = v15;
  v48 = v15;
  v49 = v37;
  v50 = off_636F70;
  v32 = &v48;
  v86 = 0LL;
  v87 = 0LL;
  v88 = 0LL;
  v89 = 0LL;
  v16 = Some_fun(&v86, off_63A780, 2);
  v17 = v16;
  *v16 = v32;
  v31 = Next2;
  v82 = 0LL;
  v83 = 0LL;
  v84 = 0LL;
  v85 = 0LL;
  v18 = Some_fun(&v82, off_636460, 2);
  *v18 = v31;
  sub_40A357(v18, v17);
  v33 = ".enc";
  v90 = 0LL;
  v91 = 0LL;
  v92 = 0LL;
  v93 = 0LL;
  v19 = Some_fun(&v90, off_639A70, 2);
  *v19 = v33;
  sub_406C5A(v38, v19);
  v51 = v39;
  v52 = v38;
  v53 = off_636F70;
  v35 = &v51;
  v98 = 0LL;
  v99 = 0LL;
  v100 = 0LL;
  v101 = 0LL;
  v20 = Some_fun(&v98, off_63A780, 2);
  v21 = v20;
  *v20 = v35;
  v34 = sub_402A9F;
  v94 = 0LL;
  v95 = 0LL;
  v96 = 0LL;
  v97 = 0LL;
  v22 = Some_fun(&v94, off_636460, 2);
  *v22 = v34;
  sub_40A357(v22, v21);
  return 0LL;
}

这个函数便是在程序执行过程中主要进行加密逻辑的部分，包括对文件内容的读取储存，对内容的加密和输出。一些分支函数也已经改了函数名，以便于分析。
每一个函数都比较复杂并且晦涩难懂，但是只要找到了分析的方法和调用特征，就可以发现其实很多地方都可以忽略，然后我们就能精简一下程序内容，得到真正的解密过程。

首先查看Rand_place函数：

__int64 Rand_place()
{
  _QWORD *v0; // rax
  _QWORD *v1; // rax
  _QWORD *v2; // rax
  __int64 v3; // ST30_8
  _QWORD *v4; // rax
  _QWORD *v5; // rax
  __int64 urand; // [rsp+50h] [rbp-100h]
  __int64 Sstream; // [rsp+58h] [rbp-F8h]
  _QWORD *v9; // [rsp+60h] [rbp-F0h]
  void *v10; // [rsp+68h] [rbp-E8h]
  _QWORD *v11; // [rsp+70h] [rbp-E0h]
  _QWORD *v12; // [rsp+78h] [rbp-D8h]
  void *v13; // [rsp+80h] [rbp-D0h]
  __int64 v14; // [rsp+90h] [rbp-C0h]
  __int64 v15; // [rsp+98h] [rbp-B8h]
  __int64 v16; // [rsp+A0h] [rbp-B0h]
  __int64 v17; // [rsp+A8h] [rbp-A8h]
  __int64 v18; // [rsp+B0h] [rbp-A0h]
  __int64 v19; // [rsp+B8h] [rbp-98h]
  __int64 v20; // [rsp+C0h] [rbp-90h]
  __int64 v21; // [rsp+C8h] [rbp-88h]
  __int64 v22; // [rsp+D0h] [rbp-80h]
  __int64 v23; // [rsp+D8h] [rbp-78h]
  __int64 v24; // [rsp+E0h] [rbp-70h]
  __int64 v25; // [rsp+E8h] [rbp-68h]
  __int64 v26; // [rsp+F0h] [rbp-60h]
  __int64 v27; // [rsp+F8h] [rbp-58h]
  __int64 v28; // [rsp+100h] [rbp-50h]
  __int64 v29; // [rsp+108h] [rbp-48h]
  __int64 v30; // [rsp+110h] [rbp-40h]
  __int64 v31; // [rsp+118h] [rbp-38h]
  __int64 v32; // [rsp+120h] [rbp-30h]
  __int64 v33; // [rsp+128h] [rbp-28h]
  unsigned __int64 v34; // [rsp+138h] [rbp-18h]

  v34 = __readfsqword(0x28u);
  v18 = 0LL;
  v19 = 0LL;
  v20 = 0LL;
  v21 = 0LL;
  v0 = Some_fun(&v18, off_639A70, 2);
  *v0 = "/dev/urandom";
  v11 = v0;
  v22 = 0LL;
  v23 = 0LL;
  v24 = 0LL;
  v25 = 0LL;
  v1 = Some_fun(&v22, off_639A70, 2);
  *v1 = &aRB;
  v12 = v1;
  v13 = off_636F70;
  v14 = 0LL;
  v15 = 0LL;
  v16 = 0LL;
  v17 = 0LL;
  v2 = Some_fun(&v14, off_63A780, 2);
  *v2 = &v11;
  Sstream = New(off_635DE8, v2);
  Read(Sstream, &urand, 4LL);
  urand |= ~urand << 32;
  Close(Sstream);
  v3 = urand;
  v30 = 0LL;
  v31 = 0LL;
  v32 = 0LL;
  v33 = 0LL;
  v4 = Some_fun(&v30, off_6333D0, 2);
  *v4 = v3;
  v9 = v4;
  v10 = off_636F70;
  v26 = 0LL;
  v27 = 0LL;
  v28 = 0LL;
  v29 = 0LL;
  v5 = Some_fun(&v26, off_63A780, 2);
  *v5 = &v9;
  return New(off_6333D0, v5);
}

大概是打开了/dev/urandom文件然后得到一个随机数urand，并且urand |= ~urand << 32;之后将urand new到一个数据结构中保存。

__int64 __fastcall Next1(__int64 a1)
{
  _QWORD *v1; // rax
  _QWORD *v2; // rax
  _QWORD *v3; // rax
  _QWORD *v4; // rax
  _QWORD *v5; // rax
  _QWORD *v6; // rbx
  _QWORD *v7; // rax
  signed __int64 v8; // rax
  void *v9; // rsi
  __int64 v10; // ST80_8
  _QWORD *v11; // rax
  __int64 v13; // [rsp+88h] [rbp-1E8h]
  void *i; // [rsp+90h] [rbp-1E0h]
  __int64 v15; // [rsp+98h] [rbp-1D8h]
  __int64 v16; // [rsp+A0h] [rbp-1D0h]
  unsigned __int64 v17; // [rsp+A8h] [rbp-1C8h]
  _QWORD *v18; // [rsp+B0h] [rbp-1C0h]
  __int64 (__fastcall **v19)(_QWORD *, void *); // [rsp+B8h] [rbp-1B8h]
  void *v20; // [rsp+C0h] [rbp-1B0h]
  void *v21; // [rsp+C8h] [rbp-1A8h]
  _QWORD *v22; // [rsp+D0h] [rbp-1A0h]
  void *v23; // [rsp+D8h] [rbp-198h]
  __int64 v24; // [rsp+E0h] [rbp-190h]
  _QWORD *v25; // [rsp+E8h] [rbp-188h]
  void *v26; // [rsp+F0h] [rbp-180h]
  __int128 v27; // [rsp+100h] [rbp-170h]
  __int128 v28; // [rsp+110h] [rbp-160h]
  __int64 v29; // [rsp+120h] [rbp-150h]
  __int64 v30; // [rsp+128h] [rbp-148h]
  __int64 v31; // [rsp+130h] [rbp-140h]
  __int64 v32; // [rsp+138h] [rbp-138h]
  __int64 v33; // [rsp+140h] [rbp-130h]
  __int64 v34; // [rsp+148h] [rbp-128h]
  __int64 v35; // [rsp+150h] [rbp-120h]
  __int64 v36; // [rsp+158h] [rbp-118h]
  __int64 v37; // [rsp+160h] [rbp-110h]
  __int64 v38; // [rsp+168h] [rbp-108h]
  __int64 v39; // [rsp+170h] [rbp-100h]
  __int64 v40; // [rsp+178h] [rbp-F8h]
  __int64 v41; // [rsp+180h] [rbp-F0h]
  __int64 v42; // [rsp+188h] [rbp-E8h]
  __int64 v43; // [rsp+190h] [rbp-E0h]
  __int64 v44; // [rsp+198h] [rbp-D8h]
  __int64 v45; // [rsp+1A0h] [rbp-D0h]
  __int64 v46; // [rsp+1A8h] [rbp-C8h]
  __int64 v47; // [rsp+1B0h] [rbp-C0h]
  __int64 v48; // [rsp+1B8h] [rbp-B8h]
  __int64 v49; // [rsp+1C0h] [rbp-B0h]
  __int64 v50; // [rsp+1C8h] [rbp-A8h]
  __int64 v51; // [rsp+1D0h] [rbp-A0h]
  __int64 v52; // [rsp+1D8h] [rbp-98h]
  __int64 v53; // [rsp+1E0h] [rbp-90h]
  __int64 v54; // [rsp+1E8h] [rbp-88h]
  __int64 v55; // [rsp+1F0h] [rbp-80h]
  __int64 v56; // [rsp+1F8h] [rbp-78h]
  __int64 v57; // [rsp+200h] [rbp-70h]
  __int64 v58; // [rsp+208h] [rbp-68h]
  __int64 v59; // [rsp+210h] [rbp-60h]
  __int64 v60; // [rsp+218h] [rbp-58h]
  char v61; // [rsp+220h] [rbp-50h]
  unsigned __int64 v62; // [rsp+258h] [rbp-18h]

  v62 = __readfsqword(0x28u);
  v20 = off_6333D0;
  v21 = off_636F70;
  v29 = 0LL;
  v30 = 0LL;
  v31 = 0LL;
  v32 = 0LL;
  v1 = Some_fun(&v29, off_63A780, 2);
  *v1 = &v20;
  v15 = New(off_633D20, v1);
  v37 = 0LL;
  v38 = 0LL;
  v39 = 0LL;
  v40 = 0LL;
  v2 = Some_fun(&v37, off_6386C0, 2);
  *v2 = 0LL;
  v24 = sub_40C263(a1, v2);
  v41 = 0LL;
  v42 = 0LL;
  v43 = 0LL;
  v44 = 0LL;
  v3 = Some_fun(&v41, off_639A70, 2);
  *v3 = &aRB;
  v25 = v3;
  v26 = off_636F70;
  v33 = 0LL;
  v34 = 0LL;
  v35 = 0LL;
  v36 = 0LL;
  v4 = Some_fun(&v33, off_63A780, 2);
  *v4 = &v24;
  v16 = New(off_635DE8, v4);
  SSeek(v16, 0LL, 2LL);
  v17 = SStell(v16);
  SSeek(v16, 0LL, 0LL);
  v53 = 0LL;
  v54 = 0LL;
  v55 = 0LL;
  v56 = 0LL;
  v5 = Some_fun(&v53, off_6386C0, 2);
  *v5 = v17 >> 3;
  v22 = v5;
  v23 = off_636F70;
  v49 = 0LL;
  v50 = 0LL;
  v51 = 0LL;
  v52 = 0LL;
  v6 = Some_fun(&v49, off_63A780, 2);
  *v6 = &v22;
  v45 = 0LL;
  v46 = 0LL;
  v47 = 0LL;
  v48 = 0LL;
  v7 = Some_fun(&v45, off_6386C0, 2);
  *v7 = 0LL;
  v27 = v7;
  v28 = 0uLL;
  memset(&v61, 0, 0x38uLL);
  v8 = Some_fun(&v61, off_637370, 2);
  *v8 = v27;
  *(v8 + 16) = v28;
  v18 = sub_40C871(v8, v6);
  v9 = off_6370C8;
  v19 = sub_41B63E(v18, off_6370C8);
  for ( i = (*v19)(v18, v9); i != off_636F70; i = v19[1](v18, i) )
  {
    Read(v16, &v13, 8LL);
    v10 = v13;
    v57 = 0LL;
    v58 = 0LL;
    v59 = 0LL;
    v60 = 0LL;
    v11 = Some_fun(&v57, off_6333D0, 2);
    *v11 = v10;
    Push(v15, v11);
  }
  Close(v16);
  return v15;
}

上面是Next1函数的内容。根据几个主要函数可以大致推断出逻辑为从待加密文件中读入数据，并且按块存入某个数据结构中（Push函数）。

接下来我们查看一下Next2的内容

__int64 __fastcall Next2(__int64 a1)
{
  _QWORD *v1; // rax
  _QWORD *v2; // rax
  __int64 v3; // ST40_8
  _QWORD *v4; // rax
  _QWORD *v5; // rbx
  _QWORD *v6; // rax
  signed __int64 v7; // rax
  void *v8; // rsi
  __int64 v9; // r12
  _QWORD *v10; // rbx
  _QWORD *v11; // rax
  __int64 v12; // ST80_8
  _QWORD *v13; // rax
  void *i; // [rsp+88h] [rbp-1B8h]
  __int64 v16; // [rsp+90h] [rbp-1B0h]
  __int64 v17; // [rsp+98h] [rbp-1A8h]
  _QWORD *v18; // [rsp+A0h] [rbp-1A0h]
  __int64 (__fastcall **v19)(_QWORD *, void *); // [rsp+A8h] [rbp-198h]
  _QWORD *v20; // [rsp+B0h] [rbp-190h]
  void *v21; // [rsp+B8h] [rbp-188h]
  __int64 v22; // [rsp+C0h] [rbp-180h]
  void *v23; // [rsp+C8h] [rbp-178h]
  __int128 v24; // [rsp+D0h] [rbp-170h]
  __int128 v25; // [rsp+E0h] [rbp-160h]
  __int64 v26; // [rsp+F0h] [rbp-150h]
  __int64 v27; // [rsp+F8h] [rbp-148h]
  __int64 v28; // [rsp+100h] [rbp-140h]
  __int64 v29; // [rsp+108h] [rbp-138h]
  __int64 v30; // [rsp+110h] [rbp-130h]
  __int64 v31; // [rsp+118h] [rbp-128h]
  __int64 v32; // [rsp+120h] [rbp-120h]
  __int64 v33; // [rsp+128h] [rbp-118h]
  __int64 v34; // [rsp+130h] [rbp-110h]
  __int64 v35; // [rsp+138h] [rbp-108h]
  __int64 v36; // [rsp+140h] [rbp-100h]
  __int64 v37; // [rsp+148h] [rbp-F8h]
  __int64 v38; // [rsp+150h] [rbp-F0h]
  __int64 v39; // [rsp+158h] [rbp-E8h]
  __int64 v40; // [rsp+160h] [rbp-E0h]
  __int64 v41; // [rsp+168h] [rbp-D8h]
  __int64 v42; // [rsp+170h] [rbp-D0h]
  __int64 v43; // [rsp+178h] [rbp-C8h]
  __int64 v44; // [rsp+180h] [rbp-C0h]
  __int64 v45; // [rsp+188h] [rbp-B8h]
  __int64 v46; // [rsp+190h] [rbp-B0h]
  __int64 v47; // [rsp+198h] [rbp-A8h]
  __int64 v48; // [rsp+1A0h] [rbp-A0h]
  __int64 v49; // [rsp+1A8h] [rbp-98h]
  __int64 v50; // [rsp+1B0h] [rbp-90h]
  __int64 v51; // [rsp+1B8h] [rbp-88h]
  __int64 v52; // [rsp+1C0h] [rbp-80h]
  __int64 v53; // [rsp+1C8h] [rbp-78h]
  __int64 v54; // [rsp+1D0h] [rbp-70h]
  __int64 v55; // [rsp+1D8h] [rbp-68h]
  __int64 v56; // [rsp+1E0h] [rbp-60h]
  __int64 v57; // [rsp+1E8h] [rbp-58h]
  char v58; // [rsp+1F0h] [rbp-50h]
  unsigned __int64 v59; // [rsp+228h] [rbp-18h]

  v59 = __readfsqword(0x28u);
  v26 = 0LL;
  v27 = 0LL;
  v28 = 0LL;
  v29 = 0LL;
  v1 = Some_fun(&v26, off_6386C0, 2);
  *v1 = 0LL;
  v16 = sub_40C263(a1, v1);
  v30 = 0LL;
  v31 = 0LL;
  v32 = 0LL;
  v33 = 0LL;
  v2 = Some_fun(&v30, off_6386C0, 2);
  *v2 = 1LL;
  v17 = sub_40C263(a1, v2);
  v3 = sub_410063(v16);
  v42 = 0LL;
  v43 = 0LL;
  v44 = 0LL;
  v45 = 0LL;
  v4 = Some_fun(&v42, off_6386C0, 2);
  *v4 = v3;
  v20 = v4;
  v21 = off_636F70;
  v38 = 0LL;
  v39 = 0LL;
  v40 = 0LL;
  v41 = 0LL;
  v5 = Some_fun(&v38, off_63A780, 2);
  *v5 = &v20;
  v34 = 0LL;
  v35 = 0LL;
  v36 = 0LL;
  v37 = 0LL;
  v6 = Some_fun(&v34, off_6386C0, 2);
  *v6 = 0LL;
  v24 = v6;
  v25 = 0uLL;
  memset(&v58, 0, 0x38uLL);
  v7 = Some_fun(&v58, off_637370, 2);
  *v7 = v24;
  *(v7 + 16) = v25;
  v18 = sub_40C871(v7, v5);
  v8 = off_6370C8;
  v19 = sub_41B63E(v18, off_6370C8);
  for ( i = (*v19)(v18, v8); i != off_636F70; i = v19[1](v18, i) )
  {
    v9 = *sub_40C263(v16, i);
    v22 = v17;
    v23 = off_636F70;
    v54 = 0LL;
    v55 = 0LL;
    v56 = 0LL;
    v57 = 0LL;
    v10 = Some_fun(&v54, off_63A780, 2);
    *v10 = &v22;
    v50 = 0LL;
    v51 = 0LL;
    v52 = 0LL;
    v53 = 0LL;
    v11 = Some_fun(&v50, off_636460, 2);
    *v11 = Encode_fun;
    v12 = v9 ^ *sub_40A357(v11, v10);
    v46 = 0LL;
    v47 = 0LL;
    v48 = 0LL;
    v49 = 0LL;
    v13 = Some_fun(&v46, off_6333D0, 2);
    *v13 = v12;
    sub_40C2AA(v16, i, v13);
  }
  return 0LL;
}

大概是对所读入的内容进行加密的函数。看到循环题内有一个比较重要的Ecode_fun函数，进入查看。

__int64 __fastcall Encode_fun(__int64 a1)
{
  _QWORD *v1; // rax
  _QWORD *v2; // rax
  _QWORD *v3; // rax
  _QWORD *v4; // rax
  _QWORD *v5; // rbx
  _QWORD *v6; // rax
  signed __int64 v7; // rax
  void *v8; // rsi
  signed __int64 v9; // rbx
  _QWORD *v10; // rax
  _QWORD *v11; // rbx
  _QWORD *v12; // rax
  signed __int64 v13; // rax
  void *v14; // rsi
  unsigned __int64 v15; // rbx
  int v16; // er12
  unsigned __int64 v17; // rbx
  _QWORD *v18; // rax
  _QWORD *v19; // rax
  __int64 key; // [rsp+B8h] [rbp-2B8h]
  void *i; // [rsp+C0h] [rbp-2B0h]
  void *j; // [rsp+C8h] [rbp-2A8h]
  unsigned __int64 *seed; // [rsp+D0h] [rbp-2A0h]
  _QWORD *v25; // [rsp+D8h] [rbp-298h]
  __int64 (__fastcall **v26)(_QWORD *, void *); // [rsp+E0h] [rbp-290h]
  unsigned __int64 orig_seed; // [rsp+E8h] [rbp-288h]
  _QWORD *v28; // [rsp+F0h] [rbp-280h]
  __int64 (__fastcall **v29)(_QWORD *, void *); // [rsp+F8h] [rbp-278h]
  _QWORD *v30; // [rsp+100h] [rbp-270h]
  void *v31; // [rsp+108h] [rbp-268h]
  _QWORD *v32; // [rsp+110h] [rbp-260h]
  void *v33; // [rsp+118h] [rbp-258h]
  __int128 v34; // [rsp+120h] [rbp-250h]
  __int128 v35; // [rsp+130h] [rbp-240h]
  _QWORD *v36; // [rsp+140h] [rbp-230h]
  _QWORD *v37; // [rsp+148h] [rbp-228h]
  _QWORD *v38; // [rsp+150h] [rbp-220h]
  void *v39; // [rsp+158h] [rbp-218h]
  __int128 v40; // [rsp+160h] [rbp-210h]
  __int128 v41; // [rsp+170h] [rbp-200h]
  __int64 v42; // [rsp+180h] [rbp-1F0h]
  __int64 v43; // [rsp+188h] [rbp-1E8h]
  __int64 v44; // [rsp+190h] [rbp-1E0h]
  __int64 v45; // [rsp+198h] [rbp-1D8h]
  __int64 v46; // [rsp+1A0h] [rbp-1D0h]
  __int64 v47; // [rsp+1A8h] [rbp-1C8h]
  __int64 v48; // [rsp+1B0h] [rbp-1C0h]
  __int64 v49; // [rsp+1B8h] [rbp-1B8h]
  __int64 v50; // [rsp+1C0h] [rbp-1B0h]
  __int64 v51; // [rsp+1C8h] [rbp-1A8h]
  __int64 v52; // [rsp+1D0h] [rbp-1A0h]
  __int64 v53; // [rsp+1D8h] [rbp-198h]
  __int64 v54; // [rsp+1E0h] [rbp-190h]
  __int64 v55; // [rsp+1E8h] [rbp-188h]
  __int64 v56; // [rsp+1F0h] [rbp-180h]
  __int64 v57; // [rsp+1F8h] [rbp-178h]
  __int64 v58; // [rsp+200h] [rbp-170h]
  __int64 v59; // [rsp+208h] [rbp-168h]
  __int64 v60; // [rsp+210h] [rbp-160h]
  __int64 v61; // [rsp+218h] [rbp-158h]
  __int64 v62; // [rsp+220h] [rbp-150h]
  __int64 v63; // [rsp+228h] [rbp-148h]
  __int64 v64; // [rsp+230h] [rbp-140h]
  __int64 v65; // [rsp+238h] [rbp-138h]
  __int64 v66; // [rsp+240h] [rbp-130h]
  __int64 v67; // [rsp+248h] [rbp-128h]
  __int64 v68; // [rsp+250h] [rbp-120h]
  __int64 v69; // [rsp+258h] [rbp-118h]
  __int64 v70; // [rsp+260h] [rbp-110h]
  __int64 v71; // [rsp+268h] [rbp-108h]
  __int64 v72; // [rsp+270h] [rbp-100h]
  __int64 v73; // [rsp+278h] [rbp-F8h]
  __int64 v74; // [rsp+280h] [rbp-F0h]
  __int64 v75; // [rsp+288h] [rbp-E8h]
  __int64 v76; // [rsp+290h] [rbp-E0h]
  __int64 v77; // [rsp+298h] [rbp-D8h]
  __int64 v78; // [rsp+2A0h] [rbp-D0h]
  __int64 v79; // [rsp+2A8h] [rbp-C8h]
  __int64 v80; // [rsp+2B0h] [rbp-C0h]
  __int64 v81; // [rsp+2B8h] [rbp-B8h]
  __int64 v82; // [rsp+2C0h] [rbp-B0h]
  __int64 v83; // [rsp+2C8h] [rbp-A8h]
  __int64 v84; // [rsp+2D0h] [rbp-A0h]
  __int64 v85; // [rsp+2D8h] [rbp-98h]
  char v86; // [rsp+2E0h] [rbp-90h]
  char v87; // [rsp+320h] [rbp-50h]
  unsigned __int64 v88; // [rsp+358h] [rbp-18h]

  v88 = __readfsqword(0x28u);
  v42 = 0LL;
  v43 = 0LL;
  v44 = 0LL;
  v45 = 0LL;
  v1 = Some_fun(&v42, off_6386C0, 2);
  *v1 = 0LL;
  seed = sub_40C263(a1, v1);
  key = 0LL;
  v54 = 0LL;
  v55 = 0LL;
  v56 = 0LL;
  v57 = 0LL;
  v2 = Some_fun(&v54, off_6386C0, 2);
  *v2 = 0LL;
  v36 = v2;
  v58 = 0LL;
  v59 = 0LL;
  v60 = 0LL;
  v61 = 0LL;
  v3 = Some_fun(&v58, off_6386C0, 2);
  *v3 = 64LL;
  v37 = v3;
  v62 = 0LL;
  v63 = 0LL;
  v64 = 0LL;
  v65 = 0LL;
  v4 = Some_fun(&v62, off_6386C0, 2);
  *v4 = -1LL;
  v38 = v4;
  v39 = off_636F70;
  v50 = 0LL;
  v51 = 0LL;
  v52 = 0LL;
  v53 = 0LL;
  v5 = Some_fun(&v50, off_63A780, 2);
  *v5 = &v36;
  v46 = 0LL;
  v47 = 0LL;
  v48 = 0LL;
  v49 = 0LL;
  v6 = Some_fun(&v46, off_6386C0, 2);
  *v6 = 0LL;
  v34 = v6;
  v35 = 0uLL;
  memset(&v86, 0, 0x38uLL);
  v7 = Some_fun(&v86, off_637370, 2);
  *v7 = v34;
  *(v7 + 16) = v35;
  v25 = sub_40C871(v7, v5);
  v8 = off_6370C8;
  v26 = sub_41B63E(v25, off_6370C8);
  for ( i = (*v26)(v25, v8); i != off_636F70; i = v26[1](v25, i) )//from 63 to 0
  {
    orig_seed = *seed;
    v9 = *seed & 1;
    key |= v9 << Int(i, v8);                    // key=(rand&1)<<i
    *seed = 0LL;
    v74 = 0LL;
    v75 = 0LL;
    v76 = 0LL;
    v77 = 0LL;
    v10 = Some_fun(&v74, off_6386C0, 2);
    *v10 = 64LL;
    v30 = v10;
    v31 = off_636F70;
    v70 = 0LL;
    v71 = 0LL;
    v72 = 0LL;
    v73 = 0LL;
    v11 = Some_fun(&v70, off_63A780, 2);
    *v11 = &v30;
    v66 = 0LL;
    v67 = 0LL;
    v68 = 0LL;
    v69 = 0LL;
    v12 = Some_fun(&v66, off_6386C0, 2);
    *v12 = 0LL;
    v40 = v12;
    v41 = 0uLL;
    memset(&v87, 0, 0x38uLL);
    v13 = Some_fun(&v87, off_637370, 2);
    *v13 = v40;
    *(v13 + 16) = v41;
    v28 = sub_40C871(v13, v11);
    v14 = off_6370C8;
    v29 = sub_41B63E(v28, off_6370C8);
    for ( j = (*v29)(v28, v14); j != off_636F70; j = v29[1](v28, j) )//from 0 to 63
    {
      v15 = seed[1];
      v16 = orig_seed >> (Int(j, v14) - 1);
      if ( (v15 >> ((v16 | (orig_seed << (65 - Int(j, v14)))) & 7)) & 1 )
      {
        v17 = *seed;
        *seed = (1LL << Int(j, v14)) | v17;
      }
      v14 = j;
    }
    v8 = i;
  }
  v82 = 0LL;
  v83 = 0LL;
  v84 = 0LL;
  v85 = 0LL;
  v18 = Some_fun(&v82, off_6333D0, 2);
  *v18 = key;
  v32 = v18;
  v33 = off_636F70;
  v78 = 0LL;
  v79 = 0LL;
  v80 = 0LL;
  v81 = 0LL;
  v19 = Some_fun(&v78, off_63A780, 2);
  *v19 = &v32;
  return New(off_6333D0, v19);
}

到这里我们终于找到了函数真正执行加密的地方。可以看到非常冗杂且混乱。这个地方只能硬着头皮逆，然后把这一块写成比较好理解的C语言形式：

for( int i=63; i>=0; --i )
    {
        key |= (seed & 1) << i;

        orig_seed = seed;
        seed      = 0;

        for( int j=0; j<64; ++j )
        {
            uint shf = (orig_seed >> (j - 1)) | (orig_seed << (64 - (j - 1)));

            if( (0x1e >> (shf & 7)) & 1 )
                seed |= (ullong)(1) << j;
        }
    }

要注意的是这个seed就是urand |= ~urand << 32得到的urand。数据的每一位都会与这个seed进行异或，但是这个seed会逐次更新，增强了安全性。
比较直观的做法就是暴力破解初始的seed。由于解密前的文件是.png文件，因此这个文件的前8个字节其实是已知的：

00000000  89 50 4e 47 0d 0a 1a 0a                           |.PNG....|

写一个脚本跑一下（得好几个小时，可以分几台设备跑，节约时间）就可以了

script:

#include<cstdio>
#include<iostream>
#include<cstring>
#include<cstdlib>
#include<algorithm>
#include<queue>
#include<stack>
#include<vector>
#include<cmath>
#include<map>
#define LL long long
#define uint unsigned int
#define ulong unsigned long long
using namespace std;
const ulong maxn=0xffffffff;
ulong get_key(ulong &seed){
	ulong key=0,orig_seed;
	for(int i=63;i>=0;i--){
		key|=(seed&1)<<i;
		orig_seed=seed; 
		seed=0;
		for(int j=0;j<64;j++){
			uint sf=(orig_seed>>(j-1))|(orig_seed<<(65-j));
			if((0x1e>>(sf&7))&1)
				seed|=(ulong)(1)<<j;
		}
	}
	return key;
}
int main(){
	ulong seed,key;
	for(uint urand=0;urand<=maxn;urand++){
		seed=(~(ulong)urand<<32)|urand;
		key=get_key(seed);
		if((key^0x7be05d85a22c66a2)==0x0a1a0a0d474e5089){
			printf("0x%X",urand);
			return 0;
		}
	}
}

跑出答案为0x5d53c9a8，因此我们可以根据这个seed的初始值来推出flag.png源文件了

#include<cstdio>
#include<iostream>
#include<cstring>
#include<cstdlib>
#include<algorithm>
#include<queue>
#include<stack>
#include<vector>
#include<cmath>
#include<map>
#define LL long long
#define uint unsigned int
#define ulong unsigned long long
using namespace std;
const ulong maxn=0xffffffff;
ulong get_key(ulong &seed){
	ulong key=0,orig_seed;
	for(int i=63;i>=0;i--){
		key|=(seed&1)<<i;
		orig_seed=seed; 
		seed=0;
		for(int j=0;j<64;j++){
			uint sf=(orig_seed>>(j-1))|(orig_seed<<(65-j));
			if((0x1e>>(sf&7))&1)
				seed|=(ulong)(1)<<j;
		}
	}
	return key;
}
int main(){
	ulong seed,key;
	uint urand=0x5D53C9A8;
	FILE *in=fopen("flag.png.enc","rb");
    FILE *out=fopen("flag.png","wb");
    ulong data=0;
    seed=(~(ulong)urand<<32)|urand;
    for(;fread(&data,8,1,in)==1;data^=get_key(seed),fwrite(&data,8,1,out));
    fclose(in);
    fclose(out);
}