Misc upgrades including mips3 float mode support, skip overwriting existing files if they're identical to the current recompiled output

This commit is contained in:
Mr-Wiseguy 2023-10-29 20:53:17 -04:00
parent 9321a60f28
commit d249363fe5
6 changed files with 701 additions and 505 deletions

View file

@ -4,6 +4,7 @@
#include <unordered_set> #include <unordered_set>
#include <unordered_map> #include <unordered_map>
#include <cassert> #include <cassert>
#include <filesystem>
#include "rabbitizer.hpp" #include "rabbitizer.hpp"
#include "fmt/format.h" #include "fmt/format.h"
#include "fmt/ostream.h" #include "fmt/ostream.h"
@ -187,7 +188,7 @@ BranchTargets get_branch_targets(const std::vector<rabbitizer::InstructionRsp>&
return ret; return ret;
} }
bool process_instruction(size_t instr_index, const std::vector<rabbitizer::InstructionRsp>& instructions, std::ofstream& output_file, const BranchTargets& branch_targets, bool indent, bool in_delay_slot) { bool process_instruction(size_t instr_index, const std::vector<rabbitizer::InstructionRsp>& instructions, std::ofstream& output_file, const BranchTargets& branch_targets, const std::unordered_set<uint32_t>& unsupported_instructions, bool indent, bool in_delay_slot) {
const auto& instr = instructions[instr_index]; const auto& instr = instructions[instr_index];
uint32_t instr_vram = instr.getVram(); uint32_t instr_vram = instr.getVram();
@ -230,7 +231,7 @@ bool process_instruction(size_t instr_index, const std::vector<rabbitizer::Instr
auto print_unconditional_branch = [&]<typename... Ts>(fmt::format_string<Ts...> fmt_str, Ts ...args) { auto print_unconditional_branch = [&]<typename... Ts>(fmt::format_string<Ts...> fmt_str, Ts ...args) {
if (instr_index < instructions.size() - 1) { if (instr_index < instructions.size() - 1) {
uint32_t next_vram = instr_vram + 4; uint32_t next_vram = instr_vram + 4;
process_instruction(instr_index + 1, instructions, output_file, branch_targets, false, true); process_instruction(instr_index + 1, instructions, output_file, branch_targets, unsupported_instructions, false, true);
} }
print_indent(); print_indent();
fmt::print(output_file, fmt_str, args...); fmt::print(output_file, fmt_str, args...);
@ -241,7 +242,7 @@ bool process_instruction(size_t instr_index, const std::vector<rabbitizer::Instr
fmt::print(output_file, "{{\n "); fmt::print(output_file, "{{\n ");
if (instr_index < instructions.size() - 1) { if (instr_index < instructions.size() - 1) {
uint32_t next_vram = instr_vram + 4; uint32_t next_vram = instr_vram + 4;
process_instruction(instr_index + 1, instructions, output_file, branch_targets, true, true); process_instruction(instr_index + 1, instructions, output_file, branch_targets, unsupported_instructions, true, true);
} }
fmt::print(output_file, " "); fmt::print(output_file, " ");
fmt::print(output_file, fmt_str, args...); fmt::print(output_file, fmt_str, args...);
@ -252,6 +253,14 @@ bool process_instruction(size_t instr_index, const std::vector<rabbitizer::Instr
print_indent(); print_indent();
} }
// Replace unsupported instructions with early returns
if (unsupported_instructions.contains(instr_vram)) {
print_line("return RspExitReason::Unsupported", instr_vram);
if (indent) {
print_indent();
}
}
int rd = (int)instr.GetO32_rd(); int rd = (int)instr.GetO32_rd();
int rs = (int)instr.GetO32_rs(); int rs = (int)instr.GetO32_rs();
int base = rs; int base = rs;
@ -337,6 +346,10 @@ bool process_instruction(size_t instr_index, const std::vector<rabbitizer::Instr
break; break;
case InstrId::rsp_add: case InstrId::rsp_add:
case InstrId::rsp_addu: case InstrId::rsp_addu:
if (rd == 0) {
fmt::print(output_file, "\n");
break;
}
print_line("{}{} = RSP_ADD32({}{}, {}{})", ctx_gpr_prefix(rd), rd, ctx_gpr_prefix(rs), rs, ctx_gpr_prefix(rt), rt); print_line("{}{} = RSP_ADD32({}{}, {}{})", ctx_gpr_prefix(rd), rd, ctx_gpr_prefix(rs), rs, ctx_gpr_prefix(rt), rt);
break; break;
case InstrId::rsp_negu: // pseudo instruction for subu x, 0, y case InstrId::rsp_negu: // pseudo instruction for subu x, 0, y
@ -349,6 +362,10 @@ bool process_instruction(size_t instr_index, const std::vector<rabbitizer::Instr
print_line("{}{} = RSP_ADD32({}{}, {})", ctx_gpr_prefix(rt), rt, ctx_gpr_prefix(rs), rs, signed_imm_string); print_line("{}{} = RSP_ADD32({}{}, {})", ctx_gpr_prefix(rt), rt, ctx_gpr_prefix(rs), rs, signed_imm_string);
break; break;
case InstrId::rsp_and: case InstrId::rsp_and:
if (rd == 0) {
fmt::print(output_file, "\n");
break;
}
print_line("{}{} = {}{} & {}{}", ctx_gpr_prefix(rd), rd, ctx_gpr_prefix(rs), rs, ctx_gpr_prefix(rt), rt); print_line("{}{} = {}{} & {}{}", ctx_gpr_prefix(rd), rd, ctx_gpr_prefix(rs), rs, ctx_gpr_prefix(rt), rt);
break; break;
case InstrId::rsp_andi: case InstrId::rsp_andi:
@ -518,6 +535,7 @@ void write_indirect_jumps(std::ofstream& output_file, const BranchTargets& branc
//std::string output_file_path = "../test/rsp/njpgdspMain.cpp"; //std::string output_file_path = "../test/rsp/njpgdspMain.cpp";
//std::string output_function_name = "njpgdspMain"; //std::string output_function_name = "njpgdspMain";
//const std::vector<uint32_t> extra_indirect_branch_targets{}; //const std::vector<uint32_t> extra_indirect_branch_targets{};
//const std::unordered_set<uint32_t> unsupported_instructions{};
// OoT aspMain // OoT aspMain
//constexpr size_t rsp_text_offset = 0xB89260; //constexpr size_t rsp_text_offset = 0xB89260;
@ -527,17 +545,35 @@ void write_indirect_jumps(std::ofstream& output_file, const BranchTargets& branc
//std::string output_file_path = "../test/rsp/aspMain.cpp"; //std::string output_file_path = "../test/rsp/aspMain.cpp";
//std::string output_function_name = "aspMain"; //std::string output_function_name = "aspMain";
//const std::vector<uint32_t> extra_indirect_branch_targets{ 0x1F68, 0x1230, 0x114C, 0x1F18, 0x1E2C, 0x14F4, 0x1E9C, 0x1CB0, 0x117C, 0x17CC, 0x11E8, 0x1AA4, 0x1B34, 0x1190, 0x1C5C, 0x1220, 0x1784, 0x1830, 0x1A20, 0x1884, 0x1A84, 0x1A94, 0x1A48, 0x1BA0 }; //const std::vector<uint32_t> extra_indirect_branch_targets{ 0x1F68, 0x1230, 0x114C, 0x1F18, 0x1E2C, 0x14F4, 0x1E9C, 0x1CB0, 0x117C, 0x17CC, 0x11E8, 0x1AA4, 0x1B34, 0x1190, 0x1C5C, 0x1220, 0x1784, 0x1830, 0x1A20, 0x1884, 0x1A84, 0x1A94, 0x1A48, 0x1BA0 };
//const std::unordered_set<uint32_t> unsupported_instructions{};
// MM's njpgdspMain is identical to OoT's // MM's njpgdspMain is identical to OoT's
// MM aspMain //// MM aspMain
constexpr size_t rsp_text_offset = 0xC40FF0; //constexpr size_t rsp_text_offset = 0xC40FF0;
constexpr size_t rsp_text_size = 0x1000; //constexpr size_t rsp_text_size = 0x1000;
constexpr size_t rsp_text_address = 0x04001000; //constexpr size_t rsp_text_address = 0x04001000;
std::string rom_file_path = "../../MMRecomp/mm.us.rev1.z64"; // uncompressed rom! //std::string rom_file_path = "../../MMRecomp/mm.us.rev1.z64"; // uncompressed rom!
std::string output_file_path = "../../MMRecomp/rsp/aspMain.cpp"; //std::string output_file_path = "../../MMRecomp/rsp/aspMain.cpp";
std::string output_function_name = "aspMain"; //std::string output_function_name = "aspMain";
const std::vector<uint32_t> extra_indirect_branch_targets{ 0x1F80, 0x1250, 0x1154, 0x1094, 0x1E0C, 0x1514, 0x1E7C, 0x1C90, 0x1180, 0x1808, 0x11E8, 0x1ADC, 0x1B6C, 0x1194, 0x1EF8, 0x1240, 0x17C0, 0x186C, 0x1A58, 0x18BC, 0x1ABC, 0x1ACC, 0x1A80, 0x1BD4 }; //const std::vector<uint32_t> extra_indirect_branch_targets{ 0x1F80, 0x1250, 0x1154, 0x1094, 0x1E0C, 0x1514, 0x1E7C, 0x1C90, 0x1180, 0x1808, 0x11E8, 0x1ADC, 0x1B6C, 0x1194, 0x1EF8, 0x1240, 0x17C0, 0x186C, 0x1A58, 0x18BC, 0x1ABC, 0x1ACC, 0x1A80, 0x1BD4 };
//const std::unordered_set<uint32_t> unsupported_instructions{};
// BT n_aspMain
constexpr size_t rsp_text_offset = 0x1E4F3B0;
constexpr size_t rsp_text_size = 0xF80;
constexpr size_t rsp_text_address = 0x04001080;
std::string rom_file_path = "../../BTRecomp/banjotooie.decompressed.us.z64"; // uncompressed rom!
std::string output_file_path = "../../BTRecomp/rsp/n_aspMain.cpp";
std::string output_function_name = "n_aspMain";
const std::vector<uint32_t> extra_indirect_branch_targets{
// dispatch table
0x1AE8, 0x143C, 0x1240, 0x1D84, 0x126C, 0x1B20, 0x12A8, 0x1214, 0x141C, 0x1310, 0x13CC, 0x12E4, 0x1FB0, 0x1358, 0x16EC, 0x1408
};
const std::unordered_set<uint32_t> unsupported_instructions{
// cmd_MP3
0x00001214
};
#ifdef _MSC_VER #ifdef _MSC_VER
inline uint32_t byteswap(uint32_t val) { inline uint32_t byteswap(uint32_t val) {
@ -589,6 +625,7 @@ int main() {
} }
// Open output file and write beginning // Open output file and write beginning
std::filesystem::create_directories(std::filesystem::path{ output_file_path }.parent_path());
std::ofstream output_file(output_file_path); std::ofstream output_file(output_file_path);
fmt::print(output_file, fmt::print(output_file,
"#include \"rsp.h\"\n" "#include \"rsp.h\"\n"
@ -603,7 +640,7 @@ int main() {
" r1 = 0xFC0;\n", output_function_name); " r1 = 0xFC0;\n", output_function_name);
// Write each instruction // Write each instruction
for (size_t instr_index = 0; instr_index < instrs.size(); instr_index++) { for (size_t instr_index = 0; instr_index < instrs.size(); instr_index++) {
process_instruction(instr_index, instrs, output_file, branch_targets, false, false); process_instruction(instr_index, instrs, output_file, branch_targets, unsupported_instructions, false, false);
} }
// Terminate instruction code with a return to indicate that the microcode has run past its end // Terminate instruction code with a return to indicate that the microcode has run past its end

View file

@ -39,14 +39,23 @@ namespace RecompPort {
uint32_t value; uint32_t value;
}; };
struct FunctionSize {
std::string func_name;
uint32_t size_bytes;
};
struct Config { struct Config {
int32_t entrypoint; int32_t entrypoint;
bool uses_mips3_float_mode;
std::filesystem::path elf_path; std::filesystem::path elf_path;
std::filesystem::path output_func_path; std::filesystem::path output_func_path;
std::filesystem::path relocatable_sections_path; std::filesystem::path relocatable_sections_path;
std::vector<std::string> stubbed_funcs; std::vector<std::string> stubbed_funcs;
std::vector<std::string> ignored_funcs;
DeclaredFunctionMap declared_funcs; DeclaredFunctionMap declared_funcs;
std::vector<InstructionPatch> instruction_patches; std::vector<InstructionPatch> instruction_patches;
std::vector<FunctionSize> manual_func_sizes;
std::string bss_section_suffix;
Config(const char* path); Config(const char* path);
bool good() { return !bad; } bool good() { return !bad; }
@ -107,6 +116,7 @@ namespace RecompPort {
std::vector<uint32_t> function_addrs; std::vector<uint32_t> function_addrs;
std::vector<Reloc> relocs; std::vector<Reloc> relocs;
std::string name; std::string name;
ELFIO::Elf_Half bss_section_index = (ELFIO::Elf_Half)-1;
bool executable = false; bool executable = false;
bool relocatable = false; bool relocatable = false;
}; };
@ -128,6 +138,8 @@ namespace RecompPort {
std::vector<std::vector<size_t>> section_functions; std::vector<std::vector<size_t>> section_functions;
// The section names that were specified as relocatable // The section names that were specified as relocatable
std::unordered_set<std::string> relocatable_sections; std::unordered_set<std::string> relocatable_sections;
// Functions with manual size overrides
std::unordered_map<std::string, size_t> manually_sized_funcs;
int executable_section_count; int executable_section_count;
Context(const ELFIO::elfio& elf_file) { Context(const ELFIO::elfio& elf_file) {
@ -142,7 +154,7 @@ namespace RecompPort {
}; };
bool analyze_function(const Context& context, const Function& function, const std::vector<rabbitizer::InstructionCpu>& instructions, FunctionStats& stats); bool analyze_function(const Context& context, const Function& function, const std::vector<rabbitizer::InstructionCpu>& instructions, FunctionStats& stats);
bool recompile_function(const Context& context, const Function& func, const std::filesystem::path& output_path, std::span<std::vector<uint32_t>> static_funcs); bool recompile_function(const Context& context, const Config& config, const Function& func, const std::filesystem::path& output_path, std::span<std::vector<uint32_t>> static_funcs);
} }
#endif #endif

View file

@ -199,7 +199,9 @@ bool analyze_instruction(const rabbitizer::InstructionCpu& instr, const RecompPo
address, address,
instr.getVram() instr.getVram()
); );
} else { }
// Allow tail calls (TODO account for trailing nops due to bad function splits)
else if (instr.getVram() != func.vram + (func.words.size() - 2) * sizeof(func.words[0])) {
// Inconclusive analysis // Inconclusive analysis
fmt::print(stderr, "Failed to to find jump table for `jr {}` at 0x{:08X} in {}\n", RabbitizerRegister_getNameGpr(rs), instr.getVram(), func.name); fmt::print(stderr, "Failed to to find jump table for `jr {}` at 0x{:08X} in {}\n", RabbitizerRegister_getNameGpr(rs), instr.getVram(), func.name);
return false; return false;

View file

@ -43,6 +43,32 @@ std::vector<std::string> get_stubbed_funcs(const toml::value& patches_data) {
return stubbed_funcs; return stubbed_funcs;
} }
std::vector<std::string> get_ignored_funcs(const toml::value& patches_data) {
std::vector<std::string> ignored_funcs{};
// Check if the ignored funcs array exists.
const auto& ignored_funcs_data = toml::find_or<toml::value>(patches_data, "ignored", toml::value{});
if (ignored_funcs_data.type() == toml::value_t::empty) {
// No stubs, nothing to do here.
return ignored_funcs;
}
// Get the ignored funcs array as an array type.
const toml::array& ignored_funcs_array = ignored_funcs_data.as_array();
// Make room for all the ignored funcs in the array.
ignored_funcs.resize(ignored_funcs_array.size());
// Gather the stubs and place them into the array.
for (size_t stub_idx = 0; stub_idx < ignored_funcs_array.size(); stub_idx++) {
// Copy the entry into the ignored function list.
ignored_funcs[stub_idx] = ignored_funcs_array[stub_idx].as_string();
}
return ignored_funcs;
}
std::unordered_map<std::string, RecompPort::FunctionArgType> arg_type_map{ std::unordered_map<std::string, RecompPort::FunctionArgType> arg_type_map{
{"u32", RecompPort::FunctionArgType::u32}, {"u32", RecompPort::FunctionArgType::u32},
{"s32", RecompPort::FunctionArgType::s32}, {"s32", RecompPort::FunctionArgType::s32},
@ -84,7 +110,7 @@ RecompPort::DeclaredFunctionMap get_declared_funcs(const toml::value& patches_da
const toml::array& funcs_array = funcs_data.as_array(); const toml::array& funcs_array = funcs_data.as_array();
// Reserve room for all the funcs in the map. // Reserve room for all the funcs in the map.
declared_funcs.reserve(funcs_data.size()); declared_funcs.reserve(funcs_array.size());
for (const toml::value& cur_func_val : funcs_array) { for (const toml::value& cur_func_val : funcs_array) {
const std::string& func_name = toml::find<std::string>(cur_func_val, "name"); const std::string& func_name = toml::find<std::string>(cur_func_val, "name");
const toml::array& args_in = toml::find<toml::array>(cur_func_val, "args"); const toml::array& args_in = toml::find<toml::array>(cur_func_val, "args");
@ -95,6 +121,40 @@ RecompPort::DeclaredFunctionMap get_declared_funcs(const toml::value& patches_da
return declared_funcs; return declared_funcs;
} }
std::vector<RecompPort::FunctionSize> get_func_sizes(const toml::value& patches_data) {
std::vector<RecompPort::FunctionSize> func_sizes{};
// Check if the func size array exists.
const toml::value& sizes_data = toml::find_or<toml::value>(patches_data, "function_sizes", toml::value{});
if (sizes_data.type() == toml::value_t::empty) {
// No func size array, nothing to do here
return func_sizes;
}
// Get the funcs array as an array type.
const toml::array& sizes_array = sizes_data.as_array();
// Reserve room for all the funcs in the map.
func_sizes.reserve(sizes_array.size());
for (const toml::value& cur_func_size : sizes_array) {
const std::string& func_name = toml::find<std::string>(cur_func_size, "name");
uint32_t func_size = toml::find<uint32_t>(cur_func_size, "size");
// Make sure the size is divisible by 4
if (func_size & (4 - 1)) {
// It's not, so throw an error (and make it look like a normal toml one).
throw toml::type_error(toml::detail::format_underline(
std::string{ std::source_location::current().function_name() } + ": function size not divisible by 4", {
{cur_func_size.location(), ""}
}), cur_func_size.location());
}
func_sizes.emplace_back(func_name, func_size);
}
return func_sizes;
}
std::vector<RecompPort::InstructionPatch> get_instruction_patches(const toml::value& patches_data) { std::vector<RecompPort::InstructionPatch> get_instruction_patches(const toml::value& patches_data) {
std::vector<RecompPort::InstructionPatch> ret; std::vector<RecompPort::InstructionPatch> ret;
@ -155,6 +215,8 @@ RecompPort::Config::Config(const char* path) {
elf_path = concat_if_not_empty(basedir, toml::find<std::string>(input_data, "elf_path")); elf_path = concat_if_not_empty(basedir, toml::find<std::string>(input_data, "elf_path"));
output_func_path = concat_if_not_empty(basedir, toml::find<std::string>(input_data, "output_func_path")); output_func_path = concat_if_not_empty(basedir, toml::find<std::string>(input_data, "output_func_path"));
relocatable_sections_path = concat_if_not_empty(basedir, toml::find_or<std::string>(input_data, "relocatable_sections_path", "")); relocatable_sections_path = concat_if_not_empty(basedir, toml::find_or<std::string>(input_data, "relocatable_sections_path", ""));
uses_mips3_float_mode = toml::find_or<bool>(input_data, "uses_mips3_float_mode", false);
bss_section_suffix = toml::find_or<std::string>(input_data, "bss_section_suffix", ".bss");
// Patches section (optional) // Patches section (optional)
const toml::value& patches_data = toml::find_or<toml::value>(config_data, "patches", toml::value{}); const toml::value& patches_data = toml::find_or<toml::value>(config_data, "patches", toml::value{});
@ -162,11 +224,17 @@ RecompPort::Config::Config(const char* path) {
// Stubs array (optional) // Stubs array (optional)
stubbed_funcs = get_stubbed_funcs(patches_data); stubbed_funcs = get_stubbed_funcs(patches_data);
// Ignored funcs array (optional)
ignored_funcs = get_ignored_funcs(patches_data);
// Functions (optional) // Functions (optional)
declared_funcs = get_declared_funcs(patches_data); declared_funcs = get_declared_funcs(patches_data);
// Single-instruction patches (optional) // Single-instruction patches (optional)
instruction_patches = get_instruction_patches(patches_data); instruction_patches = get_instruction_patches(patches_data);
// Manual function sizes (optional)
manual_func_sizes = get_func_sizes(patches_data);
} }
} }
catch (const toml::syntax_error& err) { catch (const toml::syntax_error& err) {

View file

@ -540,6 +540,8 @@ std::unordered_set<std::string> renamed_funcs{
"sincosf", "sincosf",
"sinf", "sinf",
"cosf", "cosf",
"__sinf",
"__cosf",
"sqrt", "sqrt",
"sqrtf", "sqrtf",
"memcpy", "memcpy",
@ -567,17 +569,8 @@ std::unordered_set<std::string> renamed_funcs{
"roundf", "roundf",
"trunc", "trunc",
"truncf", "truncf",
"vsprintf" "vsprintf",
}; "__assert",
// Functions that weren't declared properly and thus have no size in the elf
std::unordered_map<std::string, size_t> unsized_funcs{
{ "guMtxF2L", 0x64 },
{ "guScaleF", 0x48 },
{ "guTranslateF", 0x48 },
{ "guMtxIdentF", 0x48 },
{ "sqrtf", 0x8 },
{ "guMtxIdent", 0x4C },
}; };
bool read_symbols(RecompPort::Context& context, const ELFIO::elfio& elf_file, ELFIO::section* symtab_section, uint32_t entrypoint) { bool read_symbols(RecompPort::Context& context, const ELFIO::elfio& elf_file, ELFIO::section* symtab_section, uint32_t entrypoint) {
@ -605,23 +598,22 @@ bool read_symbols(RecompPort::Context& context, const ELFIO::elfio& elf_file, EL
} }
// Check if this symbol is the entrypoint // Check if this symbol is the entrypoint
if (value == entrypoint /*&& type == ELFIO::STT_FUNC*/) { if (value == entrypoint && type == ELFIO::STT_FUNC) {
if (found_entrypoint_func) { if (found_entrypoint_func) {
fmt::print(stderr, "Ambiguous entrypoint\n"); fmt::print(stderr, "Ambiguous entrypoint: {}\n", name);
return false; return false;
} }
found_entrypoint_func = true; found_entrypoint_func = true;
fmt::print("Found entrypoint, original name: {}\n", name);
size = 0x50; // dummy size for entrypoints, should cover them all size = 0x50; // dummy size for entrypoints, should cover them all
name = "recomp_entrypoint"; name = "recomp_entrypoint";
} }
// Check if this symbol is unsized and if so populate its size from the unsized_funcs map // Check if this symbol has a size override
if (size == 0) { auto size_find = context.manually_sized_funcs.find(name);
auto size_find = unsized_funcs.find(name); if (size_find != context.manually_sized_funcs.end()) {
if (size_find != unsized_funcs.end()) { size = size_find->second;
size = size_find->second; type = ELFIO::STT_FUNC;
type = ELFIO::STT_FUNC;
}
} }
if (reimplemented_funcs.contains(name)) { if (reimplemented_funcs.contains(name)) {
@ -653,7 +645,7 @@ bool read_symbols(RecompPort::Context& context, const ELFIO::elfio& elf_file, EL
context.functions_by_vram[vram].push_back(context.functions.size()); context.functions_by_vram[vram].push_back(context.functions.size());
// Find the entrypoint by rom address in case it doesn't have vram as its value // Find the entrypoint by rom address in case it doesn't have vram as its value
if (rom_address == 0x1000) { if (rom_address == 0x1000 && type == ELFIO::STT_FUNC) {
vram = entrypoint; vram = entrypoint;
found_entrypoint_func = true; found_entrypoint_func = true;
name = "recomp_entrypoint"; name = "recomp_entrypoint";
@ -719,7 +711,7 @@ std::optional<size_t> get_segment(const std::vector<SegmentEntry>& segments, ELF
return std::nullopt; return std::nullopt;
} }
ELFIO::section* read_sections(RecompPort::Context& context, const ELFIO::elfio& elf_file) { ELFIO::section* read_sections(RecompPort::Context& context, const RecompPort::Config& config, const ELFIO::elfio& elf_file) {
ELFIO::section* symtab_section = nullptr; ELFIO::section* symtab_section = nullptr;
std::vector<SegmentEntry> segments{}; std::vector<SegmentEntry> segments{};
segments.resize(elf_file.segments.size()); segments.resize(elf_file.segments.size());
@ -740,6 +732,7 @@ ELFIO::section* read_sections(RecompPort::Context& context, const ELFIO::elfio&
//); //);
std::unordered_map<std::string, ELFIO::section*> reloc_sections_by_name; std::unordered_map<std::string, ELFIO::section*> reloc_sections_by_name;
std::unordered_map<std::string, ELFIO::section*> bss_sections_by_name;
// Iterate over every section to record rom addresses and find the symbol table // Iterate over every section to record rom addresses and find the symbol table
fmt::print("Sections\n"); fmt::print("Sections\n");
@ -756,6 +749,10 @@ ELFIO::section* read_sections(RecompPort::Context& context, const ELFIO::elfio&
if (type == ELFIO::SHT_SYMTAB) { if (type == ELFIO::SHT_SYMTAB) {
symtab_section = section.get(); symtab_section = section.get();
} }
if (context.relocatable_sections.contains(section_name)) {
section_out.relocatable = true;
}
// Check if this section is a reloc section // Check if this section is a reloc section
if (type == ELFIO::SHT_REL) { if (type == ELFIO::SHT_REL) {
@ -773,6 +770,16 @@ ELFIO::section* read_sections(RecompPort::Context& context, const ELFIO::elfio&
} }
} }
// If the section is bss (SHT_NOBITS) and ends with the bss suffix, add it to the bss section map
if (type == ELFIO::SHT_NOBITS && section_name.ends_with(config.bss_section_suffix)) {
std::string bss_target_section = section_name.substr(0, section_name.size() - config.bss_section_suffix.size());
// If this bss section is for a section that has been marked as relocatable, record it in the reloc section lookup
if (context.relocatable_sections.contains(bss_target_section)) {
bss_sections_by_name[bss_target_section] = section.get();
}
}
// If this section isn't bss (SHT_NOBITS) and ends up in the rom (SHF_ALLOC), // If this section isn't bss (SHT_NOBITS) and ends up in the rom (SHF_ALLOC),
// find this section's rom address and copy it into the rom // find this section's rom address and copy it into the rom
if (type != ELFIO::SHT_NOBITS && section->get_flags() & ELFIO::SHF_ALLOC && section->get_size() != 0) { if (type != ELFIO::SHT_NOBITS && section->get_flags() & ELFIO::SHF_ALLOC && section->get_size() != 0) {
@ -831,97 +838,103 @@ ELFIO::section* read_sections(RecompPort::Context& context, const ELFIO::elfio&
// TODO make sure that a reloc section was found for every section marked as relocatable // TODO make sure that a reloc section was found for every section marked as relocatable
// Process reloc sections // Process bss and reloc sections
for (RecompPort::Section &section_out : context.sections) { for (RecompPort::Section &section_out : context.sections) {
// Check if a reloc section was found that corresponds with this section // Check if a bss section was found that corresponds with this section
auto reloc_find = reloc_sections_by_name.find(section_out.name); auto bss_find = bss_sections_by_name.find(section_out.name);
if (reloc_find != reloc_sections_by_name.end()) { if (bss_find != bss_sections_by_name.end()) {
// Mark the section as relocatable section_out.bss_section_index = bss_find->second->get_index();
section_out.relocatable = true; }
// Create an accessor for the reloc section
ELFIO::relocation_section_accessor rel_accessor{ elf_file, reloc_find->second };
// Allocate space for the relocs in this section
section_out.relocs.resize(rel_accessor.get_entries_num());
// Track whether the previous reloc was a HI16 and its previous full_immediate
bool prev_hi = false;
uint32_t prev_hi_immediate = 0;
uint32_t prev_hi_symbol = std::numeric_limits<uint32_t>::max();
for (size_t i = 0; i < section_out.relocs.size(); i++) { if (section_out.relocatable) {
// Get the current reloc // Check if a reloc section was found that corresponds with this section
ELFIO::Elf64_Addr rel_offset; auto reloc_find = reloc_sections_by_name.find(section_out.name);
ELFIO::Elf_Word rel_symbol; if (reloc_find != reloc_sections_by_name.end()) {
unsigned int rel_type; // Create an accessor for the reloc section
ELFIO::Elf_Sxword bad_rel_addend; // Addends aren't encoded in the reloc, so ignore this one ELFIO::relocation_section_accessor rel_accessor{ elf_file, reloc_find->second };
rel_accessor.get_entry(i, rel_offset, rel_symbol, rel_type, bad_rel_addend); // Allocate space for the relocs in this section
section_out.relocs.resize(rel_accessor.get_entries_num());
// Track whether the previous reloc was a HI16 and its previous full_immediate
bool prev_hi = false;
uint32_t prev_hi_immediate = 0;
uint32_t prev_hi_symbol = std::numeric_limits<uint32_t>::max();
RecompPort::Reloc& reloc_out = section_out.relocs[i]; for (size_t i = 0; i < section_out.relocs.size(); i++) {
// Get the current reloc
ELFIO::Elf64_Addr rel_offset;
ELFIO::Elf_Word rel_symbol;
unsigned int rel_type;
ELFIO::Elf_Sxword bad_rel_addend; // Addends aren't encoded in the reloc, so ignore this one
rel_accessor.get_entry(i, rel_offset, rel_symbol, rel_type, bad_rel_addend);
// Get the real full_immediate by extracting the immediate from the instruction RecompPort::Reloc& reloc_out = section_out.relocs[i];
uint32_t instr_word = byteswap(*reinterpret_cast<const uint32_t*>(context.rom.data() + section_out.rom_addr + rel_offset - section_out.ram_addr));
rabbitizer::InstructionCpu instr{ instr_word, static_cast<uint32_t>(rel_offset) };
//context.rom section_out.rom_addr;
reloc_out.address = rel_offset; // Get the real full_immediate by extracting the immediate from the instruction
reloc_out.symbol_index = rel_symbol; uint32_t instr_word = byteswap(*reinterpret_cast<const uint32_t*>(context.rom.data() + section_out.rom_addr + rel_offset - section_out.ram_addr));
reloc_out.type = static_cast<RecompPort::RelocType>(rel_type); rabbitizer::InstructionCpu instr{ instr_word, static_cast<uint32_t>(rel_offset) };
reloc_out.needs_relocation = false; //context.rom section_out.rom_addr;
std::string rel_symbol_name; reloc_out.address = rel_offset;
ELFIO::Elf64_Addr rel_symbol_value; reloc_out.symbol_index = rel_symbol;
ELFIO::Elf_Xword rel_symbol_size; reloc_out.type = static_cast<RecompPort::RelocType>(rel_type);
unsigned char rel_symbol_bind; reloc_out.needs_relocation = false;
unsigned char rel_symbol_type;
ELFIO::Elf_Half rel_symbol_section_index;
unsigned char rel_symbol_other;
bool found_rel_symbol = symbol_accessor.get_symbol( std::string rel_symbol_name;
rel_symbol, rel_symbol_name, rel_symbol_value, rel_symbol_size, rel_symbol_bind, rel_symbol_type, rel_symbol_section_index, rel_symbol_other); ELFIO::Elf64_Addr rel_symbol_value;
ELFIO::Elf_Xword rel_symbol_size;
unsigned char rel_symbol_bind;
unsigned char rel_symbol_type;
ELFIO::Elf_Half rel_symbol_section_index;
unsigned char rel_symbol_other;
reloc_out.target_section = rel_symbol_section_index; bool found_rel_symbol = symbol_accessor.get_symbol(
rel_symbol, rel_symbol_name, rel_symbol_value, rel_symbol_size, rel_symbol_bind, rel_symbol_type, rel_symbol_section_index, rel_symbol_other);
bool rel_needs_relocation = false; reloc_out.target_section = rel_symbol_section_index;
if (rel_symbol_section_index < context.sections.size()) { bool rel_needs_relocation = false;
rel_needs_relocation = context.sections[rel_symbol_section_index].relocatable;
}
// Reloc pairing, see MIPS System V ABI documentation page 4-18 (https://refspecs.linuxfoundation.org/elf/mipsabi.pdf) if (rel_symbol_section_index < context.sections.size()) {
if (reloc_out.type == RecompPort::RelocType::R_MIPS_LO16) { rel_needs_relocation = context.sections[rel_symbol_section_index].relocatable;
if (prev_hi) { }
if (prev_hi_symbol != rel_symbol) {
fmt::print(stderr, "[WARN] Paired HI16 and LO16 relocations have different symbols\n" // Reloc pairing, see MIPS System V ABI documentation page 4-18 (https://refspecs.linuxfoundation.org/elf/mipsabi.pdf)
" LO16 reloc index {} in section {} referencing symbol {} with offset 0x{:08X}\n", if (reloc_out.type == RecompPort::RelocType::R_MIPS_LO16) {
i, section_out.name, reloc_out.symbol_index, reloc_out.address); if (prev_hi) {
if (prev_hi_symbol != rel_symbol) {
fmt::print(stderr, "[WARN] Paired HI16 and LO16 relocations have different symbols\n"
" LO16 reloc index {} in section {} referencing symbol {} with offset 0x{:08X}\n",
i, section_out.name, reloc_out.symbol_index, reloc_out.address);
}
uint32_t rel_immediate = instr.getProcessedImmediate();
uint32_t full_immediate = (prev_hi_immediate << 16) + (int16_t)rel_immediate;
// Set this and the previous HI16 relocs' relocated addresses
section_out.relocs[i - 1].target_address = full_immediate;
reloc_out.target_address = full_immediate;
} }
} else {
if (prev_hi) {
fmt::print(stderr, "Unpaired HI16 reloc index {} in section {} referencing symbol {} with offset 0x{:08X}\n",
i - 1, section_out.name, section_out.relocs[i - 1].symbol_index, section_out.relocs[i - 1].address);
return nullptr;
}
}
if (reloc_out.type == RecompPort::RelocType::R_MIPS_HI16) {
uint32_t rel_immediate = instr.getProcessedImmediate(); uint32_t rel_immediate = instr.getProcessedImmediate();
uint32_t full_immediate = (prev_hi_immediate << 16) + (int16_t)rel_immediate; prev_hi = true;
prev_hi_immediate = rel_immediate;
prev_hi_symbol = rel_symbol;
// Set this and the previous HI16 relocs' relocated addresses } else {
section_out.relocs[i - 1].target_address = full_immediate; prev_hi = false;
reloc_out.target_address = full_immediate;
} }
} else {
if (prev_hi) { if (reloc_out.type == RecompPort::RelocType::R_MIPS_32) {
fmt::print(stderr, "Unpaired HI16 reloc index {} in section {} referencing symbol {} with offset 0x{:08X}\n", // Nothing to do here
i - 1, section_out.name, section_out.relocs[i - 1].symbol_index, section_out.relocs[i - 1].address);
return nullptr;
} }
} }
if (reloc_out.type == RecompPort::RelocType::R_MIPS_HI16) {
uint32_t rel_immediate = instr.getProcessedImmediate();
prev_hi = true;
prev_hi_immediate = rel_immediate;
prev_hi_symbol = rel_symbol;
} else {
prev_hi = false;
}
if (reloc_out.type == RecompPort::RelocType::R_MIPS_32) {
// Nothing to do here
}
} }
// Sort this section's relocs by address, which allows for binary searching and more efficient iteration during recompilation. // Sort this section's relocs by address, which allows for binary searching and more efficient iteration during recompilation.
@ -968,7 +981,7 @@ void analyze_sections(RecompPort::Context& context, const ELFIO::elfio& elf_file
); );
} }
bool read_list_file(const std::filesystem::path& filename, std::unordered_set<std::string>& entries_out) { bool read_list_file(const std::filesystem::path& filename, std::vector<std::string>& entries_out) {
std::ifstream input_file{ filename }; std::ifstream input_file{ filename };
if (!input_file.good()) { if (!input_file.good()) {
return false; return false;
@ -977,7 +990,7 @@ bool read_list_file(const std::filesystem::path& filename, std::unordered_set<st
std::string entry; std::string entry;
while (input_file >> entry) { while (input_file >> entry) {
entries_out.emplace(std::move(entry)); entries_out.emplace_back(std::move(entry));
} }
return true; return true;
@ -1007,14 +1020,17 @@ int main(int argc, char** argv) {
RabbitizerConfig_Cfg.pseudos.pseudoBnez = false; RabbitizerConfig_Cfg.pseudos.pseudoBnez = false;
RabbitizerConfig_Cfg.pseudos.pseudoNot = false; RabbitizerConfig_Cfg.pseudos.pseudoNot = false;
std::unordered_set<std::string> relocatable_sections{}; std::vector<std::string> relocatable_sections_ordered{};
if (!config.relocatable_sections_path.empty()) { if (!config.relocatable_sections_path.empty()) {
if (!read_list_file(config.relocatable_sections_path, relocatable_sections)) { if (!read_list_file(config.relocatable_sections_path, relocatable_sections_ordered)) {
exit_failure("Failed to load the relocatable section list file: " + std::string(argv[4]) + "\n"); exit_failure("Failed to load the relocatable section list file: " + std::string(argv[4]) + "\n");
} }
} }
std::unordered_set<std::string> relocatable_sections{};
relocatable_sections.insert(relocatable_sections_ordered.begin(), relocatable_sections_ordered.end());
if (!elf_file.load(config.elf_path.string())) { if (!elf_file.load(config.elf_path.string())) {
exit_failure("Failed to load provided elf file\n"); exit_failure("Failed to load provided elf file\n");
} }
@ -1031,7 +1047,7 @@ int main(int argc, char** argv) {
context.relocatable_sections = std::move(relocatable_sections); context.relocatable_sections = std::move(relocatable_sections);
// Read all of the sections in the elf and look for the symbol table section // Read all of the sections in the elf and look for the symbol table section
ELFIO::section* symtab_section = read_sections(context, elf_file); ELFIO::section* symtab_section = read_sections(context, config, elf_file);
// Search the sections to see if any are overlays or TLB-mapped // Search the sections to see if any are overlays or TLB-mapped
analyze_sections(context, elf_file); analyze_sections(context, elf_file);
@ -1041,6 +1057,17 @@ int main(int argc, char** argv) {
exit_failure("No symbol table section found\n"); exit_failure("No symbol table section found\n");
} }
// Functions that weren't declared properly and thus have no size in the elf
//context.manually_sized_funcs.emplace("guMtxF2L", 0x64);
//context.manually_sized_funcs.emplace("guScaleF", 0x48);
//context.manually_sized_funcs.emplace("guTranslateF", 0x48);
//context.manually_sized_funcs.emplace("guMtxIdentF", 0x48);
//context.manually_sized_funcs.emplace("sqrtf", 0x8);
//context.manually_sized_funcs.emplace("guMtxIdent", 0x4C);
for (const auto& func_size : config.manual_func_sizes) {
context.manually_sized_funcs.emplace(func_size.func_name, func_size.size_bytes);
}
// Read all of the symbols in the elf and look for the entrypoint function // Read all of the symbols in the elf and look for the entrypoint function
bool found_entrypoint_func = read_symbols(context, elf_file, symtab_section, config.entrypoint); bool found_entrypoint_func = read_symbols(context, elf_file, symtab_section, config.entrypoint);
@ -1050,6 +1077,8 @@ int main(int argc, char** argv) {
fmt::print("Function count: {}\n", context.functions.size()); fmt::print("Function count: {}\n", context.functions.size());
std::filesystem::create_directories(config.output_func_path);
std::ofstream lookup_file{ config.output_func_path / "lookup.cpp" }; std::ofstream lookup_file{ config.output_func_path / "lookup.cpp" };
std::ofstream func_header_file{ config.output_func_path / "funcs.h" }; std::ofstream func_header_file{ config.output_func_path / "funcs.h" };
@ -1087,6 +1116,19 @@ int main(int argc, char** argv) {
context.functions[func_find->second].stubbed = true; context.functions[func_find->second].stubbed = true;
} }
// Ignore any functions specified in the config file.
for (const std::string& ignored_func : config.ignored_funcs) {
// Check if the specified function exists.
auto func_find = context.functions_by_name.find(ignored_func);
if (func_find == context.functions_by_name.end()) {
// Function doesn't exist, present an error to the user instead of silently failing to mark it as ignored.
// This helps prevent typos in the config file or functions renamed between versions from causing issues.
exit_failure(fmt::format("Function {} is set as ignored in the config file but does not exist!", ignored_func));
}
// Mark the function as .
context.functions[func_find->second].ignored = true;
}
// Apply any single-instruction patches. // Apply any single-instruction patches.
for (const RecompPort::InstructionPatch& patch : config.instruction_patches) { for (const RecompPort::InstructionPatch& patch : config.instruction_patches) {
// Check if the specified function exists. // Check if the specified function exists.
@ -1102,7 +1144,7 @@ int main(int argc, char** argv) {
// Check that the function actually contains this vram address. // Check that the function actually contains this vram address.
if (patch.vram < func_vram || patch.vram >= func_vram + func.words.size() * sizeof(func.words[0])) { if (patch.vram < func_vram || patch.vram >= func_vram + func.words.size() * sizeof(func.words[0])) {
exit_failure(fmt::vformat("Function {} has an instruction patch for vram 0x{:08X} but doesn't contain that vram address!", fmt::make_format_args(patch.vram))); exit_failure(fmt::format("Function {} has an instruction patch for vram 0x{:08X} but doesn't contain that vram address!", patch.func_name, (uint32_t)patch.vram));
} }
// Calculate the instruction index and modify the instruction. // Calculate the instruction index and modify the instruction.
@ -1119,7 +1161,7 @@ int main(int argc, char** argv) {
"void {}(uint8_t* rdram, recomp_context* ctx);\n", func.name); "void {}(uint8_t* rdram, recomp_context* ctx);\n", func.name);
//fmt::print(lookup_file, //fmt::print(lookup_file,
// " {{ 0x{:08X}u, {} }},\n", func.vram, func.name); // " {{ 0x{:08X}u, {} }},\n", func.vram, func.name);
if (RecompPort::recompile_function(context, func, config.output_func_path / (func.name + ".c"), static_funcs_by_section) == false) { if (RecompPort::recompile_function(context, config, func, config.output_func_path / (func.name + ".c"), static_funcs_by_section) == false) {
//lookup_file.clear(); //lookup_file.clear();
fmt::print(stderr, "Error recompiling {}\n", func.name); fmt::print(stderr, "Error recompiling {}\n", func.name);
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
@ -1188,7 +1230,7 @@ int main(int argc, char** argv) {
"void {}(uint8_t* rdram, recomp_context* ctx);\n", func.name); "void {}(uint8_t* rdram, recomp_context* ctx);\n", func.name);
//fmt::print(lookup_file, //fmt::print(lookup_file,
// " {{ 0x{:08X}u, {} }},\n", func.vram, func.name); // " {{ 0x{:08X}u, {} }},\n", func.vram, func.name);
if (RecompPort::recompile_function(context, func, config.output_func_path / (func.name + ".c"), static_funcs_by_section) == false) { if (RecompPort::recompile_function(context, config, func, config.output_func_path / (func.name + ".c"), static_funcs_by_section) == false) {
//lookup_file.clear(); //lookup_file.clear();
fmt::print(stderr, "Error recompiling {}\n", func.name); fmt::print(stderr, "Error recompiling {}\n", func.name);
std::exit(EXIT_FAILURE); std::exit(EXIT_FAILURE);
@ -1226,13 +1268,24 @@ int main(int argc, char** argv) {
"\n" "\n"
); );
std::unordered_map<std::string, size_t> relocatable_section_indices{};
size_t written_sections = 0;
for (size_t section_index = 0; section_index < context.sections.size(); section_index++) { for (size_t section_index = 0; section_index < context.sections.size(); section_index++) {
const auto& section = context.sections[section_index]; const auto& section = context.sections[section_index];
const auto& section_funcs = context.section_functions[section_index]; const auto& section_funcs = context.section_functions[section_index];
if (section.name == ".cosection") {
fmt::print("");
}
if (!section_funcs.empty()) { if (!section_funcs.empty()) {
std::string_view section_name_trimmed{ section.name }; std::string_view section_name_trimmed{ section.name };
if (section.relocatable) {
relocatable_section_indices.emplace(section.name, written_sections);
}
while (section_name_trimmed[0] == '.') { while (section_name_trimmed[0] == '.') {
section_name_trimmed.remove_prefix(1); section_name_trimmed.remove_prefix(1);
} }
@ -1253,6 +1306,7 @@ int main(int argc, char** argv) {
} }
fmt::print(overlay_file, "}};\n"); fmt::print(overlay_file, "}};\n");
written_sections++;
} }
} }
section_load_table += "};\n"; section_load_table += "};\n";
@ -1260,6 +1314,24 @@ int main(int argc, char** argv) {
fmt::print(overlay_file, "{}", section_load_table); fmt::print(overlay_file, "{}", section_load_table);
fmt::print(overlay_file, "const size_t num_sections = {};\n", context.sections.size()); fmt::print(overlay_file, "const size_t num_sections = {};\n", context.sections.size());
fmt::print(overlay_file, "static int overlay_sections_by_index[] = {{\n");
for (const std::string& section : relocatable_sections_ordered) {
// Check if this is an empty overlay
if (section == "*") {
fmt::print(overlay_file, " -1,\n");
}
else {
auto find_it = relocatable_section_indices.find(section);
if (find_it == relocatable_section_indices.end()) {
fmt::print(stderr, "Failed to find written section index of relocatable section: {}\n", section);
std::exit(EXIT_FAILURE);
}
fmt::print(overlay_file, " {},\n", relocatable_section_indices[section]);
}
}
fmt::print(overlay_file, "}};\n");
} }
return 0; return 0;

File diff suppressed because it is too large Load diff