Add statistics and introduce new hook format (#4)

Author: 4z0t

Debug.py

@@ -67,12 +67,21 @@ def load_sect_map(map_path: Path) -> list[tuple[int, str]]:
     return map
 
 
+def can_convert_to_hex(num: str):
+    try:
+        int(num, 16)
+        return True
+    except ValueError:
+        return False
+
+
 def get_stack_trace(data: list[str]) -> list[int]:
     stack_trace = []
     for line in data:
         if line.startswith("Stacktrace:"):
             _, * trace = line.split(" ")
-            stack_trace.extend((int(i, 16) for i in trace))
+            stack_trace.extend((int(i, 16)
+                               for i in trace if can_convert_to_hex(i)))
             break
     return stack_trace
 
@@ -118,7 +127,7 @@ def main(patches_folder_path, args):
         else:
             break
 
-    exe_map = load_exe_map(Path("./exe.map"))
+    exe_map = load_exe_map(Path("./statistics/exe.map"))
     sect = load_sect_map(Path(patches_folder_path)/"build"/"sectmap.txt")
     exe_map.extend(sect)
 

Patterns.md

@@ -0,0 +1,155 @@
+During REing you will notice certain patterns. 
+
+## Object ctor and dtor
+
+These patterns will help with noticing certain structures when REing code.
+
+### std::vector
+
+In ctor you won't notice it at first glance. However if you look closely:
+
+```cpp
+...
+*(_DWORD *)(a + 1) = 0;
+*(_DWORD *)(a + 2) = 0;
+*(_DWORD *)(a + 3) = 0;
+...
+```
+You'll notice that `*a` wasn't set to anything, and if you find dtor:
+```cpp
+...
+if(*(a + 1))
+{
+    ...
+    free(*(_DWORD *)(a + 1)); // or operator delete(*(a + 1));
+}
+*(_DWORD *)(a + 1) = 0;
+*(_DWORD *)(a + 2) = 0;
+*(_DWORD *)(a + 3) = 0;
+...
+```
+This is a definitely std::vector and function before free (if it presents) may suggest you the type of vector.
+
+```cpp
+...
+if(a->v.begin)
+{
+    ...
+    free(a->v.begin); // or operator delete(*(a + 1));
+}
+a->v.begin = 0;
+a->v.end = 0;
+a->v.capacity_end = 0;
+...
+```
+
+**Note: offsets may be different, but overall idea stays**
+
+An example:
+
+```cpp
+ v2 = *(std::string **)(a1 + 8);
+  if ( v2 )
+  {
+    func_FreeStringsRange(v2, *(std::string **)(a1 + 12));
+    operator delete(*(void **)(a1 + 8));
+  }
+  *(_DWORD *)(a1 + 8) = 0;
+  *(_DWORD *)(a1 + 12) = 0;
+  *(_DWORD *)(a1 + 16) = 0;
+```
+
+This is a vector of std::string.
+
+### linked list
+
+ctor
+```cpp
+*(_DWORD *)(a1 + 4) = a1 + 4;
+*(_DWORD *)(a1 + 8) = a1 + 4;
+```
+
+dtor
+```cpp
+*(_DWORD *)(*(_DWORD *)(a1 + 4) + 4) = *(_DWORD *)(a1 + 8);
+**(_DWORD **)(a1 + 8) = *(_DWORD *)(a1 + 4);
+*(_DWORD *)(a1 + 4) = a1 + 4;
+*(_DWORD *)(a1 + 8) = a1 + 4;
+```
+
+This is a linked list
+
+```cpp
+a1->l.prev = &a1->l;
+a1->l.next = &a1->l;
+```
+
+```cpp
+  a1->l.next->prev = a1->l.prev;
+  a1->l.prev->next = a1->l.next;
+  a1->l.prev = &a1->l;
+  a1->l.next = &a1->l;
+```
+
+### std::map (binary tree)
+
+```cpp
+v1 = sub_465480();                                              // this function has weird stuff and call to *new* with size we'll use later
+                                                                // no first field set
+*((_DWORD *)this + 1) = v1;                                     // doing some stuff with the second field
+*(_BYTE *)(v1 + 45/*any offset*/) = 1;                          // setting some value to 1
+*(_DWORD *)(*((_DWORD *)this + 1) + 4) = *((_DWORD *)this + 1);  
+**((_DWORD **)this + 1) = *((_DWORD *)this + 1);
+*(_DWORD *)(*((_DWORD *)this + 1) + 8) = *((_DWORD *)this + 1); 
+*((_DWORD *)this + 2) = 0;                                      // setting third field to zero
+```
+
+This is a map based on binary tree. 
+
+```cpp
+node = create_node();
+this->m.root = node;
+node->is_leaf = 1;
+this->m.root->parent = this->m.root;
+this->m.root->left = this->m.root;
+this->m.root->right = this->m.root;
+this->m.size = 0;
+```
+
+Where dtor will look very fancy and you'll guess it very fast
+
+```cpp
+... // maybe an iteration over map before
+some_function(&this->m, &node, this->m.root->left, this->m.root);
+operator delete(this->m.root);
+this->m.root = 0;
+this->m.size = 0;
+...
+```
+
+### std::shared_ptr
+
+```cpp
+if ( v10 )
+{
+  if ( !_InterlockedExchangeAdd(v10 + 1, 0xFFFFFFFF) )
+  {
+    (*(void (__thiscall **)(volatile signed __int32 *))(*v10 + 4))(v10);
+    if ( !_InterlockedExchangeAdd(v10 + 2, 0xFFFFFFFF) )
+      (*(void (__thiscall **)(volatile signed __int32 *))(*v10 + 8))(v10);
+  }
+}
+```
+You'll see this very frequently. It is inlined dtor of shared pointer. So if `v10` is a counter block, then field before it is pointer to data associated with it.
+
+```cpp
+if ( pi )
+  {
+    if ( !_InterlockedExchangeAdd(&pi->use_count_, 0xFFFFFFFF) )
+    {
+      pi->vtable->dispose(pi);
+      if ( !_InterlockedExchangeAdd(&pi->weak_count_, 0xFFFFFFFF) )
+        pi->vtable->destroy(pi);
+    }
+  }
+```

RE.md

@@ -0,0 +1,26 @@
+We are looking for experienced reverse engineers to improve SCFA's engine and expand it's functions.
+Right now we are having goal to improve and expose some of the UI's functionality to have more control over it in Lua.
+Specifically user's input and what user can see in UI.
+
+There are 2 main fields where we need help:
+
+* Reverse engineering: revesing
+  * structures and classes of engine and their names
+  * functions' logic and their names
+* Engine recreation: with given decompiled parts of engine
+  * recreate functions in C/C++
+  * recreate structures and classes in C/C++
+  * recreate generic structures and algorithms in C/C++
+
+Tooling:
+
+* C/C++
+* GCC inline x86 asm
+* IDA + x64dbg
+* a bit of Lua + its API
+
+Dedicated repositories:
+
+* [C++ patcher](https://github.com/FAForever/FA_Patcher)
+* [Python patcher](https://github.com/4z0t/SCFA-python-patcher)
+* [Game's executable patches](https://github.com/FAForever/FA-Binary-Patches)

main.py

@@ -1,9 +1,9 @@
 import sys
 import time
-import Patcher
+import patcher
 
 if __name__ == "__main__":
     start = time.time()
-    Patcher.patch(*sys.argv)
+    patcher.patch(*sys.argv)
     end = time.time()
     print(f"Patched in {end-start:.2f}s")

BasicBinaryParser.py renamed to patcher/BasicBinaryParser.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Optional, Any
 import struct
-from BasicBinaryParser import BasicBinaryParser
+from .BasicBinaryParser import BasicBinaryParser
 
 
 @dataclass

COFFData.py renamed to patcher/COFFData.py

@@ -0,0 +1,66 @@
+import re
+from pathlib import Path
+
+ADDRESS_RE = re.compile(r"^(0[xX][0-9A-Fa-f]{6,8})\:$")
+FUNCTION_NAME_RE = re.compile(r"@([a-zA-Z\_][a-zA-Z0-9\_]+)")
+
+
+class Section:
+
+    def __init__(self, address: str, lines: list[str]) -> None:
+        self._address: str = address
+        self._lines: list[str] = lines
+
+    def lines_to_cpp(self):
+        s = ""
+        for line in self._lines:
+            line = line.translate(str.maketrans({"\"":  r"\"", "\\": r"\\", }))
+
+            if FUNCTION_NAME_RE.findall(line):
+                line = FUNCTION_NAME_RE.subn(r'"QU(\1)"', line)[0]
+
+            s += f'"{line};"\n'
+        return s
+
+    def to_cpp(self, index: int) -> str:
+        if self._address is None:
+            return self.lines_to_cpp()
+        return f'SECTION({index:X}, {self._address})\n{self.lines_to_cpp()}'
+
+
+class Hook:
+    def __init__(self, sections: list[Section]) -> None:
+        self._sections: list[Section] = sections
+
+    def to_cpp(self):
+        s = '#include "../asm.h"\n#include "../define.h"\n'
+        if len(self._sections) > 0:
+            sections_lines = (section.to_cpp(i).split("\n")
+                              for i, section in enumerate(self._sections))
+            s += f"asm(\n{''.join((f"    {line}\n" for lines in sections_lines for line in lines))});"
+        return s
+
+
+def load_hook(file_path: Path) -> Hook:
+    sections: list[Section] = []
+    lines = []
+    address = None
+    with open(file_path) as f:
+        for line in f.readlines():
+            # remove comments
+            line = line.split("//")[0]
+            line = line.split("#")[0]
+            line = line.strip()
+            if not line:
+                continue
+
+            if match := ADDRESS_RE.match(line):
+                if len(lines) > 0:
+                    sections.append(Section(address, lines))
+                lines = []
+                address = match.group(1)
+                continue
+            lines.append(line)
+        if len(lines) > 0:
+            sections.append(Section(address, lines))
+        return Hook(sections)

patcher/Hook.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from typing import Optional, Any
 import struct
-from BasicBinaryParser import BasicBinaryParser
+from .BasicBinaryParser import BasicBinaryParser
 
 
 @dataclass

PEData.py renamed to patcher/PEData.py

@@ -1,18 +1,21 @@
-from PEData import PEData, PESect
-from COFFData import COFFData, COFFSect
+from .PEData import PEData, PESect
+from .COFFData import COFFData, COFFSect
 import os
 from pathlib import Path
 import re
 from typing import Optional
 import struct
 import itertools
+from patcher import Hook
 
-
-CLANG_FLAGS = " ".join(["-pipe -m32 -Os -nostdlib -nostartfiles -w -masm=intel -std=c++20 -march=core2 -c",
+CLANG_FLAGS = " ".join(["-pipe -m32 -Os -nostdlib -Werror -masm=intel -std=c++20 -march=core2 -c",
                         ])
 
-GCC_FLAGS = " ".join(["-pipe -m32 -Os -fno-exceptions -nostdlib -nostartfiles -w -fpermissive -masm=intel -std=c++20 -march=core2 -mfpmath=both",
+GCC_FLAGS = " ".join(["-pipe -m32 -Os -fno-exceptions -nostdlib -nostartfiles -fpermissive -masm=intel -std=c++20 -march=core2 -mfpmath=both",
                       ])
+
+GCC_FLAGS_ASM = " ".join(["-pipe -m32 -Os -fno-exceptions -nostdlib -nostartfiles -w -fpermissive -masm=intel -std=c++20 -march=core2 -mfpmath=both",
+                          ])
 SECT_SIZE = 0x80000
 
 ASM_RE = re.compile(r"(asm\(\"(0[xX][0-9a-fA-F]{1,8})\"\);)", re.IGNORECASE)
@@ -380,9 +383,19 @@ def create_defines_file(path: Path, addresses: dict[str, str]):
                 f.write(f"#define {name} {address}\n")
     create_defines_file(target_path / "define.h", addresses)
 
+    def generate_hook_files(folder_path: Path):
+        for file_path in list_files_at(folder_path, "**/*.hook"):
+            hook = Hook. load_hook(folder_path/file_path)
+            hook_path = file_path.replace(os.sep, "_") + ".cpp"
+            print(f"Generating {hook_path}")
+            with open(folder_path/hook_path, "w") as f:
+                f.write(hook.to_cpp())
+
+    generate_hook_files(target_path/"hooks")
+
     if run_system(
             f"""cd {build_folder_path} &
-            {gcc_compiler_path} -c {GCC_FLAGS} ../hooks/*.cpp"""):
+            {gcc_compiler_path} -c {GCC_FLAGS_ASM} ../hooks/*.cpp"""):
         raise Exception("Errors occurred during building of hooks files")
 
     hooks: list[COFFData] = []
@@ -417,7 +430,7 @@ def create_defines_file(path: Path, addresses: dict[str, str]):
         for hook in hooks:
             for sect in hook.sects:
                 pld.writelines([
-                    f" .h{hi} 0x{sect.offset:x} : SUBALIGN(1) {{\n",
+                    f" .h{hi:X} 0x{sect.offset:x} : SUBALIGN(1) {{\n",
                     f"     {hook.name}({sect.name})\n",
                     " }\n",
                 ])
@@ -457,7 +470,7 @@ def replace_data(new_data, offset):
             print(f"No hooks in {hook.name}")
             continue
         for sect in hook.sects:
-            psect = patch_pe.find_sect(f".h{hi}")
+            psect = patch_pe.find_sect(f".h{hi:X}")
             size = sect.size
             replace_data(
                 patch_pe.data[psect.f_offset:psect.f_offset + size], psect.v_offset)
@@ -507,3 +520,4 @@ def save_new_base_data(data: bytearray):
     save_new_base_data(base_file_data)
 
     remove_files_at(build_folder_path, "**/*.o")
+    remove_files_at(target_path/"hooks", "*.hook.cpp")

Patcher.py renamed to patcher/Patcher.py

@@ -0,0 +1 @@
+from .Patcher import patch