Enable more IDO warnings and apply fixes (#1264)

* Update asm-processor and fix includes with EARLY

* Enable more IDO warnings and disable unwanted warning 516

* Fix most new and remaining warnings

* Improve skelanime comment

* Improve asmproc pragma comment

Co-authored-by: Tharo <17233964+Thar0@users.noreply.github.com>

* Add suggested comment for a wrong prototype

* Update asm-processor with the latest fix

Co-authored-by: Tharo <17233964+Thar0@users.noreply.github.com>

Makefile

@@ -123,7 +123,7 @@ ifeq ($(COMPILER),gcc)
   MIPS_VERSION := -mips3
 else 
   # we support Microsoft extensions such as anonymous structs, which the compiler does support but warns for their usage. Surpress the warnings with -woff.
-  CFLAGS += -G 0 -non_shared -Xfullwarn -Xcpluscomm $(INC) -Wab,-r4300_mul -woff 649,838,712 
+  CFLAGS += -G 0 -non_shared -fullwarn -verbose -Xcpluscomm $(INC) -Wab,-r4300_mul -woff 516,649,838,712
   MIPS_VERSION := -mips2
 endif
 

include/z64animation.h

@@ -207,7 +207,10 @@ typedef struct SkelAnime {
     /* 0x24 */ Vec3s* morphTable; // Table of values used to morph between animations
     /* 0x28 */ f32 morphWeight;   // Weight of the current animation morph as a fraction in [0,1]
     /* 0x2C */ f32 morphRate;     // Reciprocal of the number of frames in the morph
-    /* 0x30 */ s32 (*update)();   // Can be Loop, Partial loop, Play once, Morph, or Tapered morph. Link only has Loop, Play once, and Morph.
+    /* 0x30 */ union {
+                    s32 (*normal)(struct SkelAnime*); // Can be Loop, Partial loop, Play once, Morph, or Tapered morph
+                    s32 (*link)(struct PlayState*, struct SkelAnime*); // Can be Loop, Play once, or Morph
+                } update;
     /* 0x34 */ s8 initFlags;      // Flags used when initializing Link's skeleton
     /* 0x35 */ u8 moveFlags;      // Flags used for animations that move the actor in worldspace.
     /* 0x36 */ s16 prevRot;       // Previous rotation in worldspace.

include/z64cutscene.h

@@ -144,10 +144,12 @@ typedef enum {
  * on its own line.
  *
  * Files that contain this type that are included in other C files
- * must include an 'EARLY' qualifier to inform asm-processor that it
+ * must be preceded by a '#pragma asmproc recurse' qualifier to
- * must recursively process that include.
+ * inform asm-processor that it must recursively process that include.
  *
- * Example: #include "file.c" EARLY
+ * Example:
+ * #pragma asmproc recurse
+ * #include "file.c"
  */
 typedef union CutsceneData {
     s32 i;

src/code/code_800F9280.c

@@ -96,9 +96,10 @@ typedef enum {
 } SeqCmdType;
 
 void Audio_ProcessSeqCmd(u32 cmd) {
-    s32 pad[2];
+    s32 pad;
     u16 fadeTimer;
     u16 channelMask;
+    u32 channelMaskReversed;
     u16 val;
     u8 oldSpec;
     u8 spec;
@@ -305,9 +306,10 @@ void Audio_ProcessSeqCmd(u32 cmd) {
                 // stop channels
                 Audio_QueueCmdS8(0x08000000 | _SHIFTL(playerIdx, 16, 8) | 0xFF00, 1);
             }
-            if ((channelMask ^ 0xFFFF) != 0) {
+            channelMaskReversed = channelMask ^ 0xFFFF;
+            if (channelMaskReversed != 0) {
                 // with channel mask ~channelMask...
-                Audio_QueueCmdU16(0x90000000 | _SHIFTL(playerIdx, 16, 8), (channelMask ^ 0xFFFF));
+                Audio_QueueCmdU16(0x90000000 | _SHIFTL(playerIdx, 16, 8), channelMaskReversed);
                 // unstop channels
                 Audio_QueueCmdS8(0x08000000 | _SHIFTL(playerIdx, 16, 8) | 0xFF00, 0);
             }

src/code/graph.c

@@ -31,6 +31,9 @@ void Graph_FaultClient(void) {
     osViSwapBuffer(nextFb);
 }
 
+// TODO: merge Gfx and GfxMod to make this function's arguments consistent
+void UCodeDisas_Disassemble(UCodeDisas*, Gfx*);
+
 void Graph_DisassembleUCode(Gfx* workBuf) {
     UCodeDisas disassembler;
 

src/code/sys_cfb.c

@@ -36,7 +36,7 @@ void SysCfb_Init(s32 n64dd) {
     osSyncPrintf("フレームバッファのアドレスは %08x と %08x です\n", sSysCfbFbPtr[0], sSysCfbFbPtr[1]);
 }
 
-void SysCfb_Reset() {
+void SysCfb_Reset(void) {
     sSysCfbFbPtr[0] = 0;
     sSysCfbFbPtr[1] = 0;
     sSysCfbEnd = 0;

src/code/z_camera.c

@@ -8142,7 +8142,7 @@ s32 Camera_Copy(Camera* dstCamera, Camera* srcCamera) {
     return true;
 }
 
-s32 Camera_GetDbgCamEnabled() {
+s32 Camera_GetDbgCamEnabled(void) {
     return gDbgCamEnabled;
 }
 

src/code/z_lights.c

@@ -161,7 +161,7 @@ void Lights_BindAll(Lights* lights, LightNode* listHead, Vec3f* vec) {
     }
 }
 
-LightNode* Lights_FindBufSlot() {
+LightNode* Lights_FindBufSlot(void) {
     LightNode* node;
 
     if (sLightsBuffer.numOccupied >= LIGHTS_BUFFER_SIZE) {

src/code/z_play.c

@@ -307,7 +307,7 @@ void Play_Init(GameState* thisx) {
     if ((gEntranceTable[((void)0, gSaveContext.entranceIndex)].scene == SCENE_SPOT09) &&
         gSaveContext.sceneSetupIndex == 6) {
         osSyncPrintf("エンディングはじまるよー\n"); // "The ending starts"
-        ((void (*)())0x81000000)();
+        ((void (*)(void))0x81000000)();
         osSyncPrintf("出戻り？\n"); // "Return?"
     }
 

src/code/z_skelanime.c

@@ -1086,9 +1086,9 @@ void SkelAnime_InitLink(PlayState* play, SkelAnime* skelAnime, FlexSkeletonHeade
  */
 void LinkAnimation_SetUpdateFunction(SkelAnime* skelAnime) {
     if (skelAnime->mode <= ANIMMODE_LOOP_INTERP) {
-        skelAnime->update = LinkAnimation_Loop;
+        skelAnime->update.link = LinkAnimation_Loop;
     } else {
-        skelAnime->update = LinkAnimation_Once;
+        skelAnime->update.link = LinkAnimation_Once;
     }
     skelAnime->morphWeight = 0.0f;
 }
@@ -1098,7 +1098,7 @@ void LinkAnimation_SetUpdateFunction(SkelAnime* skelAnime) {
  * finishes.
  */
 s32 LinkAnimation_Update(PlayState* play, SkelAnime* skelAnime) {
-    return skelAnime->update(play, skelAnime);
+    return skelAnime->update.link(play, skelAnime);
 }
 
 /**
@@ -1201,7 +1201,7 @@ void LinkAnimation_Change(PlayState* play, SkelAnime* skelAnime, LinkAnimationHe
             SkelAnime_CopyFrameTable(skelAnime, skelAnime->morphTable, skelAnime->jointTable);
             morphFrames = -morphFrames;
         } else {
-            skelAnime->update = LinkAnimation_Morph;
+            skelAnime->update.link = LinkAnimation_Morph;
             AnimationContext_SetLoadFrame(play, animation, (s32)startFrame, skelAnime->limbCount,
                                           skelAnime->morphTable);
         }
@@ -1463,11 +1463,11 @@ SkelAnime_InitSkin(PlayState* play, SkelAnime* skelAnime, SkeletonHeader* skelet
  */
 void SkelAnime_SetUpdate(SkelAnime* skelAnime) {
     if (skelAnime->mode <= ANIMMODE_LOOP_INTERP) {
-        skelAnime->update = SkelAnime_LoopFull;
+        skelAnime->update.normal = SkelAnime_LoopFull;
     } else if (skelAnime->mode <= ANIMMODE_ONCE_INTERP) {
-        skelAnime->update = SkelAnime_Once;
+        skelAnime->update.normal = SkelAnime_Once;
     } else {
-        skelAnime->update = SkelAnime_LoopPartial;
+        skelAnime->update.normal = SkelAnime_LoopPartial;
     }
 }
 
@@ -1476,7 +1476,7 @@ void SkelAnime_SetUpdate(SkelAnime* skelAnime) {
  * finishes.
  */
 s32 SkelAnime_Update(SkelAnime* skelAnime) {
-    return skelAnime->update(skelAnime);
+    return skelAnime->update.normal(skelAnime);
 }
 
 /**
@@ -1636,10 +1636,10 @@ void Animation_ChangeImpl(SkelAnime* skelAnime, AnimationHeader* animation, f32
             morphFrames = -morphFrames;
         } else {
             if (taper != ANIMTAPER_NONE) {
-                skelAnime->update = SkelAnime_MorphTaper;
+                skelAnime->update.normal = SkelAnime_MorphTaper;
                 skelAnime->taper = taper;
             } else {
-                skelAnime->update = SkelAnime_Morph;
+                skelAnime->update.normal = SkelAnime_Morph;
             }
             SkelAnime_GetFrameData(animation, startFrame, skelAnime->limbCount, skelAnime->morphTable);
         }

src/code/z_vr_box.c

@@ -266,7 +266,7 @@ s32 func_800AE2C0(SkyboxContext* skyboxCtx, Vtx* roomVtx, s32 arg2, s32 arg3, s3
             }
             break;
     }
-    skyboxCtx->unk_138 = &skyboxCtx->dListBuf[2 * arg8];
+    skyboxCtx->unk_138 = &skyboxCtx->dListBuf[2 * arg8][0];
 
     for (i = 0; i < 0x20; i++) {
         index = D_8012ADD8[i];

src/overlays/actors/ovl_Boss_Va/z_boss_va.c

@@ -2038,7 +2038,7 @@ void BossVa_ZapperAttack(BossVa* this, PlayState* play) {
 
             this->skelAnime.playSpeed = 0.0f;
             if (Math_SmoothStepToF(&this->skelAnime.curFrame, 0.0f, 1.0f, 2.0f, 0.0f) == 0.0f) {
-                if (sp88 < sp90) {
+                if (sp88 < (u32)sp90) {
                     this->timer2 = 0;
                     this->burst++;
                     this->unk_1D8 = sp7C;

src/overlays/actors/ovl_Demo_Du/z_demo_du.c

@@ -16,7 +16,8 @@ void DemoDu_Draw(Actor* thisx, PlayState* play);
 
 static s32 sUnused = 0;
 
-#include "z_demo_du_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_demo_du_cutscene_data.c"
 
 static void* sEyeTextures[] = { gDaruniaEyeOpenTex, gDaruniaEyeOpeningTex, gDaruniaEyeShutTex, gDaruniaEyeClosingTex };
 static void* sMouthTextures[] = { gDaruniaMouthSeriousTex, gDaruniaMouthGrinningTex, gDaruniaMouthOpenTex,
@@ -707,7 +708,7 @@ void DemoDu_InitCs_AfterGanon(DemoDu* this, PlayState* play) {
     this->actor.shape.shadowAlpha = 0;
 }
 
-void DemoDu_CsPlaySfx_WhiteOut() {
+void DemoDu_CsPlaySfx_WhiteOut(void) {
     func_800788CC(NA_SE_SY_WHITE_OUT_T);
 }
 

src/overlays/actors/ovl_Demo_Gt/z_demo_gt.c

@@ -19,7 +19,7 @@ void DemoGt_Destroy(Actor* thisx, PlayState* play) {
     }
 }
 
-void DemoGt_PlayEarthquakeSfx() {
+void DemoGt_PlayEarthquakeSfx(void) {
     func_800788CC(NA_SE_EV_EARTHQUAKE - SFX_FLAG);
 }
 
@@ -435,7 +435,7 @@ void func_8097ED64(DemoGt* this, PlayState* play, s32 actionIdx) {
     func_8097E824(this, actionIdx);
 }
 
-u8 func_8097ED94() {
+u8 func_8097ED94(void) {
     if (kREG(2) != 0) {
         return true;
     } else if (gSaveContext.sceneSetupIndex < 4) {

src/overlays/actors/ovl_Demo_Im/z_demo_im.c

@@ -72,7 +72,8 @@ static ColliderCylinderInitType1 sCylinderInit = {
     { 25, 80, 0, { 0, 0, 0 } },
 };
 
-#include "z_demo_im_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_demo_im_cutscene_data.c"
 
 static DemoImActionFunc sActionFuncs[] = {
     func_809856F8, func_80985718, func_80985738, func_80985770, func_809857B0, func_809857F0, func_80985830,

src/overlays/actors/ovl_Demo_Sa/z_demo_sa.c

@@ -71,7 +71,8 @@ static void* sMouthTextures[] = {
 
 static u32 D_80990108 = 0;
 
-#include "z_demo_sa_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_demo_sa_cutscene_data.c"
 
 static DemoSaActionFunc sActionFuncs[] = {
     func_8098EBB8, func_8098EBD8, func_8098EBF8, func_8098EC28, func_8098EC60, func_8098EC94, func_8098ECCC,

src/overlays/actors/ovl_En_Ds/z_en_ds.c

@@ -155,7 +155,7 @@ void EnDs_OfferOddPotion(EnDs* this, PlayState* play) {
     }
 }
 
-s32 EnDs_CheckRupeesAndBottle() {
+s32 EnDs_CheckRupeesAndBottle(void) {
     if (gSaveContext.rupees < 100) {
         return 0;
     } else if (Inventory_HasEmptyBottle() == 0) {

src/overlays/actors/ovl_En_Eg/z_en_eg.c

@@ -34,7 +34,7 @@ const ActorInit En_Eg_InitVars = {
     (ActorFunc)EnEg_Draw,
 };
 
-void EnEg_PlayVoidOutSFX() {
+void EnEg_PlayVoidOutSFX(void) {
     func_800788CC(NA_SE_OC_ABYSS);
 }
 

src/overlays/actors/ovl_En_Fr/z_en_fr.c

@@ -798,7 +798,7 @@ void EnFr_CheckOcarinaInputFrogSong(u8 ocarinaNote) {
     }
 }
 
-void EnFr_DeactivateButterfly() {
+void EnFr_DeactivateButterfly(void) {
     s32 frogIndex;
     EnFr* frog;
 

src/overlays/actors/ovl_En_Holl/z_en_holl.c

@@ -75,7 +75,7 @@ void EnHoll_SetupAction(EnHoll* this, EnHollActionFunc func) {
     this->actionFunc = func;
 }
 
-s32 EnHoll_IsKokiriSetup8() {
+s32 EnHoll_IsKokiriSetup8(void) {
     return gSaveContext.entranceIndex == ENTR_SPOT04_0 && gSaveContext.sceneSetupIndex == 8;
 }
 

src/overlays/actors/ovl_En_Jj/z_en_jj.c

@@ -42,7 +42,8 @@ const ActorInit En_Jj_InitVars = {
 
 static s32 sUnused = 0;
 
-#include "z_en_jj_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_en_jj_cutscene_data.c"
 
 static s32 sUnused2[] = { 0, 0 };
 

src/overlays/actors/ovl_En_Md/z_en_md.c

@@ -69,7 +69,7 @@ typedef enum {
     /* 10 */ ENMD_ANIM_10,
     /* 11 */ ENMD_ANIM_11,
     /* 12 */ ENMD_ANIM_12,
-    /* 13 */ ENMD_ANIM_13,
+    /* 13 */ ENMD_ANIM_13
 } EnMdAnimation;
 
 static AnimationInfo sAnimationInfo[] = {

src/overlays/actors/ovl_En_Nb/z_en_nb.c

@@ -85,7 +85,8 @@ static void* sEyeTextures[] = {
 
 static s32 D_80AB4318 = 0;
 
-#include "z_en_nb_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_en_nb_cutscene_data.c"
 
 s32 EnNb_GetPath(EnNb* this) {
     s32 path = this->actor.params >> 8;

src/overlays/actors/ovl_En_Ru1/z_en_ru1.c

@@ -105,7 +105,8 @@ static void* sMouthTextures[] = {
 
 static s32 sUnused = 0;
 
-#include "z_en_ru1_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_en_ru1_cutscene_data.c"
 
 static u32 D_80AF1938 = 0;
 

src/overlays/actors/ovl_En_Ru2/z_en_ru2.c

@@ -63,7 +63,8 @@ static void* sEyeTextures[] = {
 
 static UNK_TYPE D_80AF4118 = 0;
 
-#include "z_en_ru2_cutscene_data.c" EARLY
+#pragma asmproc recurse
+#include "z_en_ru2_cutscene_data.c"
 
 static EnRu2ActionFunc sActionFuncs[] = {
     func_80AF2CB4, func_80AF2CD4, func_80AF2CF4, func_80AF2D2C, func_80AF2D6C, func_80AF2DAC, func_80AF2DEC,
@@ -379,7 +380,7 @@ void func_80AF2E1C(EnRu2* this, PlayState* play) {
     this->actor.shape.shadowAlpha = 0;
 }
 
-void func_80AF2E64() {
+void func_80AF2E64(void) {
     func_800788CC(NA_SE_SY_WHITE_OUT_T);
 }
 

src/overlays/actors/ovl_En_Tp/z_en_tp.c

@@ -160,7 +160,8 @@ void EnTp_Init(Actor* thisx, PlayState* play2) {
             next = (EnTp*)Actor_Spawn(&play->actorCtx, play, ACTOR_EN_TP, this->actor.world.pos.x,
                                       this->actor.world.pos.y, this->actor.world.pos.z, 0, 0, 0, 0 * i);
 
-            if (0 * i) {} // Very fake, but needed to get the s registers right
+            if ((0 * i) != 0) {} // Very fake, but needed to get the s registers right
+
             if (next != NULL) {
                 now->actor.child = &next->actor;
                 next->actor.parent = &now->actor;
@@ -176,7 +177,8 @@ void EnTp_Init(Actor* thisx, PlayState* play2) {
                 next->timer = next->unk_15C = i * -5;
                 next->horizontalVariation = 6.0f - (i * 0.75f);
                 now = next;
-                if (0 * i) {}
+
+                if ((0 * i) != 0) {}
             }
         }
     } else if (this->actor.params == TAILPASARAN_TAIL) {

src/overlays/actors/ovl_En_Xc/z_en_xc.c

@@ -1360,7 +1360,7 @@ void func_80B3F3C8(EnXc* this, PlayState* play) {
     this->action = SHEIK_ACTION_45;
 }
 
-void func_80B3F3D8() {
+void func_80B3F3D8(void) {
     func_800788CC(NA_SE_PL_SKIP);
 }
 

src/overlays/misc/ovl_kaleido_scope/z_kaleido_equipment.c

@@ -113,6 +113,9 @@ void KaleidoScope_DrawPlayerWork(PlayState* play) {
                      BOOTS_EQUIP_TO_PLAYER(CUR_EQUIP_VALUE(EQUIP_TYPE_BOOTS)));
 }
 
+// Wrong prototype; this function is called with `play` even though it has no arguments
+void KaleidoScope_ProcessPlayerPreRender(PlayState* play);
+
 void KaleidoScope_DrawEquipment(PlayState* play) {
     PauseContext* pauseCtx = &play->pauseCtx;
     Input* input = &play->state.input[0];

tools/asm_processor/asm_processor.py

@@ -85,6 +85,18 @@ MIPS_DEBUG_ST_STATIC = 2
 MIPS_DEBUG_ST_STATIC_PROC = 14
 
 
+class ElfFormat:
+    def __init__(self, is_big_endian):
+        self.is_big_endian = is_big_endian
+        self.struct_char = ">" if is_big_endian else "<"
+
+    def pack(self, fmt, *args):
+        return struct.pack(self.struct_char + fmt, *args)
+
+    def unpack(self, fmt, data):
+        return struct.unpack(self.struct_char + fmt, data)
+
+
 class ElfHeader:
     """
     typedef struct {
@@ -107,9 +119,9 @@ class ElfHeader:
 
     def __init__(self, data):
         self.e_ident = data[:EI_NIDENT]
-        self.e_type, self.e_machine, self.e_version, self.e_entry, self.e_phoff, self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx = struct.unpack('>HHIIIIIHHHHHH', data[EI_NIDENT:])
         assert self.e_ident[EI_CLASS] == 1 # 32-bit
-        assert self.e_ident[EI_DATA] == 2 # big-endian
+        self.fmt = ElfFormat(is_big_endian=(self.e_ident[EI_DATA] == 2))
+        self.e_type, self.e_machine, self.e_version, self.e_entry, self.e_phoff, self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize, self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx = self.fmt.unpack('HHIIIIIHHHHHH', data[EI_NIDENT:])
         assert self.e_type == 1 # relocatable
         assert self.e_machine == 8 # MIPS I Architecture
         assert self.e_phoff == 0 # no program header
@@ -117,7 +129,7 @@ class ElfHeader:
         assert self.e_shstrndx != SHN_UNDEF
 
     def to_bin(self):
-        return self.e_ident + struct.pack('>HHIIIIIHHHHHH', self.e_type,
+        return self.e_ident + self.fmt.pack('HHIIIIIHHHHHH', self.e_type,
                 self.e_machine, self.e_version, self.e_entry, self.e_phoff,
                 self.e_shoff, self.e_flags, self.e_ehsize, self.e_phentsize,
                 self.e_phnum, self.e_shentsize, self.e_shnum, self.e_shstrndx)
@@ -135,8 +147,9 @@ class Symbol:
     } Elf32_Sym;
     """
 
-    def __init__(self, data, strtab, name=None):
+    def __init__(self, fmt, data, strtab, name=None):
-        self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = struct.unpack('>IIIBBH', data)
+        self.fmt = fmt
+        self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx = fmt.unpack('IIIBBH', data)
         assert self.st_shndx != SHN_XINDEX, "too many sections (SHN_XINDEX not supported)"
         self.bind = st_info >> 4
         self.type = st_info & 15
@@ -144,31 +157,32 @@ class Symbol:
         self.visibility = self.st_other & 3
 
     @staticmethod
-    def from_parts(st_name, st_value, st_size, st_info, st_other, st_shndx, strtab, name):
+    def from_parts(fmt, st_name, st_value, st_size, st_info, st_other, st_shndx, strtab, name):
-        header = struct.pack('>IIIBBH', st_name, st_value, st_size, st_info, st_other, st_shndx)
+        header = fmt.pack('IIIBBH', st_name, st_value, st_size, st_info, st_other, st_shndx)
-        return Symbol(header, strtab, name)
+        return Symbol(fmt, header, strtab, name)
 
     def to_bin(self):
         st_info = (self.bind << 4) | self.type
-        return struct.pack('>IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx)
+        return self.fmt.pack('IIIBBH', self.st_name, self.st_value, self.st_size, st_info, self.st_other, self.st_shndx)
 
 
 class Relocation:
-    def __init__(self, data, sh_type):
+    def __init__(self, fmt, data, sh_type):
+        self.fmt = fmt
         self.sh_type = sh_type
         if sh_type == SHT_REL:
-            self.r_offset, self.r_info = struct.unpack('>II', data)
+            self.r_offset, self.r_info = fmt.unpack('II', data)
         else:
-            self.r_offset, self.r_info, self.r_addend = struct.unpack('>III', data)
+            self.r_offset, self.r_info, self.r_addend = fmt.unpack('III', data)
         self.sym_index = self.r_info >> 8
         self.rel_type = self.r_info & 0xff
 
     def to_bin(self):
         self.r_info = (self.sym_index << 8) | self.rel_type
         if self.sh_type == SHT_REL:
-            return struct.pack('>II', self.r_offset, self.r_info)
+            return self.fmt.pack('II', self.r_offset, self.r_info)
         else:
-            return struct.pack('>III', self.r_offset, self.r_info, self.r_addend)
+            return self.fmt.pack('III', self.r_offset, self.r_info, self.r_addend)
 
 
 class Section:
@@ -187,8 +201,9 @@ class Section:
     } Elf32_Shdr;
     """
 
-    def __init__(self, header, data, index):
+    def __init__(self, fmt, header, data, index):
-        self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize = struct.unpack('>IIIIIIIIII', header)
+        self.fmt = fmt
+        self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize = fmt.unpack('IIIIIIIIII', header)
         assert not self.sh_flags & SHF_LINK_ORDER
         if self.sh_entsize != 0:
             assert self.sh_size % self.sh_entsize == 0
@@ -200,9 +215,9 @@ class Section:
         self.relocated_by = []
 
     @staticmethod
-    def from_parts(sh_name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data, index):
+    def from_parts(fmt, sh_name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data, index):
-        header = struct.pack('>IIIIIIIIII', sh_name, sh_type, sh_flags, 0, 0, len(data), sh_link, sh_info, sh_addralign, sh_entsize)
+        header = fmt.pack('IIIIIIIIII', sh_name, sh_type, sh_flags, 0, 0, len(data), sh_link, sh_info, sh_addralign, sh_entsize)
-        return Section(header, data, index)
+        return Section(fmt, header, data, index)
 
     def lookup_str(self, index):
         assert self.sh_type == SHT_STRTAB
@@ -222,7 +237,7 @@ class Section:
     def header_to_bin(self):
         if self.sh_type != SHT_NOBITS:
             self.sh_size = len(self.data)
-        return struct.pack('>IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize)
+        return self.fmt.pack('IIIIIIIIII', self.sh_name, self.sh_type, self.sh_flags, self.sh_addr, self.sh_offset, self.sh_size, self.sh_link, self.sh_info, self.sh_addralign, self.sh_entsize)
 
     def late_init(self, sections):
         if self.sh_type == SHT_SYMTAB:
@@ -251,14 +266,14 @@ class Section:
         self.strtab = sections[self.sh_link]
         entries = []
         for i in range(0, self.sh_size, self.sh_entsize):
-            entries.append(Symbol(self.data[i:i+self.sh_entsize], self.strtab))
+            entries.append(Symbol(self.fmt, self.data[i:i+self.sh_entsize], self.strtab))
         self.symbol_entries = entries
 
     def init_relocs(self):
         assert self.is_rel()
         entries = []
         for i in range(0, self.sh_size, self.sh_entsize):
-            entries.append(Relocation(self.data[i:i+self.sh_entsize], self.sh_type))
+            entries.append(Relocation(self.fmt, self.data[i:i+self.sh_entsize], self.sh_type))
         self.relocations = entries
 
     def local_symbols(self):
@@ -281,9 +296,9 @@ class Section:
             hdrr_cbOptOffset, hdrr_iauxMax, hdrr_cbAuxOffset, hdrr_issMax, \
             hdrr_cbSsOffset, hdrr_issExtMax, hdrr_cbSsExtOffset, hdrr_ifdMax, \
             hdrr_cbFdOffset, hdrr_crfd, hdrr_cbRfdOffset, hdrr_iextMax, \
-            hdrr_cbExtOffset = struct.unpack(">HHIIIIIIIIIIIIIIIIIIIIIII", self.data[0:0x60])
+            hdrr_cbExtOffset = self.fmt.unpack("HHIIIIIIIIIIIIIIIIIIIIIII", self.data[0:0x60])
 
-        assert hdrr_magic == 0x7009 , "Invalid magic value for .mdebug symbolic header"
+        assert hdrr_magic == 0x7009, "Invalid magic value for .mdebug symbolic header"
 
         hdrr_cbLineOffset += shift_by
         hdrr_cbDnOffset += shift_by
@@ -297,7 +312,7 @@ class Section:
         hdrr_cbRfdOffset += shift_by
         hdrr_cbExtOffset += shift_by
 
-        new_data[0:0x60] = struct.pack(">HHIIIIIIIIIIIIIIIIIIIIIII", hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \
+        new_data[0:0x60] = self.fmt.pack("HHIIIIIIIIIIIIIIIIIIIIIII", hdrr_magic, hdrr_vstamp, hdrr_ilineMax, hdrr_cbLine, \
             hdrr_cbLineOffset, hdrr_idnMax, hdrr_cbDnOffset, hdrr_ipdMax, \
             hdrr_cbPdOffset, hdrr_isymMax, hdrr_cbSymOffset, hdrr_ioptMax, \
             hdrr_cbOptOffset, hdrr_iauxMax, hdrr_cbAuxOffset, hdrr_issMax, \
@@ -313,15 +328,16 @@ class ElfFile:
         assert data[:4] == b'\x7fELF', "not an ELF file"
 
         self.elf_header = ElfHeader(data[0:52])
+        self.fmt = self.elf_header.fmt
 
         offset, size = self.elf_header.e_shoff, self.elf_header.e_shentsize
-        null_section = Section(data[offset:offset + size], data, 0)
+        null_section = Section(self.fmt, data[offset:offset + size], data, 0)
         num_sections = self.elf_header.e_shnum or null_section.sh_size
 
         self.sections = [null_section]
         for i in range(1, num_sections):
             ind = offset + i * size
-            self.sections.append(Section(data[ind:ind + size], data, i))
+            self.sections.append(Section(self.fmt, data[ind:ind + size], data, i))
 
         symtab = None
         for s in self.sections:
@@ -345,7 +361,7 @@ class ElfFile:
     def add_section(self, name, sh_type, sh_flags, sh_link, sh_info, sh_addralign, sh_entsize, data):
         shstr = self.sections[self.elf_header.e_shstrndx]
         sh_name = shstr.add_str(name)
-        s = Section.from_parts(sh_name=sh_name, sh_type=sh_type,
+        s = Section.from_parts(self.fmt, sh_name=sh_name, sh_type=sh_type,
                 sh_flags=sh_flags, sh_link=sh_link, sh_info=sh_info,
                 sh_addralign=sh_addralign, sh_entsize=sh_entsize, data=data,
                 index=len(self.sections))
@@ -476,6 +492,7 @@ class GlobalAsmBlock:
     def count_quoted_size(self, line, z, real_line, output_enc):
         line = line.encode(output_enc).decode('latin1')
         in_quote = False
+        has_comma = True
         num_parts = 0
         ret = 0
         i = 0
@@ -486,10 +503,15 @@ class GlobalAsmBlock:
             if not in_quote:
                 if c == '"':
                     in_quote = True
+                    if z and not has_comma:
+                        self.fail(".asciiz with glued strings is not supported due to GNU as version diffs")
                     num_parts += 1
+                elif c == ',':
+                    has_comma = True
             else:
                 if c == '"':
                     in_quote = False
+                    has_comma = False
                     continue
                 ret += 1
                 if c != '\\':
@@ -554,7 +576,7 @@ class GlobalAsmBlock:
             self.text_glabels.append(line.split()[1])
         if not line:
             pass # empty line
-        elif line.startswith('glabel ') or (' ' not in line and line.endswith(':')):
+        elif line.startswith('glabel ') or line.startswith('dlabel ') or line.startswith('endlabel ') or (' ' not in line and line.endswith(':')):
             pass # label
         elif line.startswith('.section') or line in ['.text', '.data', '.rdata', '.rodata', '.bss', '.late_rodata']:
             # section change
@@ -783,6 +805,13 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
         else:
             min_instr_count = 2
             skip_instr_count = 1
+    elif opt == 'O0':
+        if framepointer:
+            min_instr_count = 8
+            skip_instr_count = 8
+        else:
+            min_instr_count = 4
+            skip_instr_count = 4
     elif opt == 'g':
         if framepointer:
             min_instr_count = 7
@@ -792,7 +821,7 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
             skip_instr_count = 4
     else:
         if opt != 'g3':
-            raise Failure("must pass one of -g, -O1, -O2, -O2 -g3")
+            raise Failure("must pass one of -g, -O0, -O1, -O2, -O2 -g3")
         if framepointer:
             min_instr_count = 4
             skip_instr_count = 4
@@ -813,6 +842,7 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
     ]
 
     is_cutscene_data = False
+    is_early_include = False
 
     for line_no, raw_line in enumerate(f, 1):
         raw_line = raw_line.rstrip()
@@ -832,55 +862,69 @@ def parse_source(f, opt, framepointer, mips1, input_enc, output_enc, out_depende
                 global_asm = None
             else:
                 global_asm.process_line(raw_line, output_enc)
+        elif line in ['GLOBAL_ASM(', '#pragma GLOBAL_ASM(']:
+            global_asm = GlobalAsmBlock("GLOBAL_ASM block at line " + str(line_no))
+            start_index = len(output_lines)
+        elif ((line.startswith('GLOBAL_ASM("') or line.startswith('#pragma GLOBAL_ASM("'))
+                and line.endswith('")')):
+            fname = line[line.index('(') + 2 : -2]
+            out_dependencies.append(fname)
+            global_asm = GlobalAsmBlock(fname)
+            with open(fname, encoding=input_enc) as f:
+                for line2 in f:
+                    global_asm.process_line(line2.rstrip(), output_enc)
+            src, fn = global_asm.finish(state)
+            output_lines[-1] = ''.join(src)
+            asm_functions.append(fn)
+            global_asm = None
+        elif line == '#pragma asmproc recurse':
+            # C includes qualified as
+            # #pragma asmproc recurse
+            # #include "file.c"
+            # will be processed recursively when encountered
+            is_early_include = True
+        elif is_early_include:
+            # Previous line was a #pragma asmproc recurse
+            is_early_include = False
+            if not line.startswith("#include "):
+                raise Failure("#pragma asmproc recurse must be followed by an #include ")
+            fpath = os.path.dirname(f.name)
+            fname = os.path.join(fpath, line[line.index(' ') + 2 : -1])
+            out_dependencies.append(fname)
+            include_src = StringIO()
+            with open(fname, encoding=input_enc) as include_file:
+                parse_source(include_file, opt, framepointer, mips1, input_enc, output_enc, out_dependencies, include_src)
+            include_src.write('#line ' + str(line_no + 1) + ' "' + f.name + '"')
+            output_lines[-1] = include_src.getvalue()
+            include_src.close()
         else:
-            if line in ['GLOBAL_ASM(', '#pragma GLOBAL_ASM(']:
+            # This is a hack to replace all floating-point numbers in an array of a particular type
-                global_asm = GlobalAsmBlock("GLOBAL_ASM block at line " + str(line_no))
+            # (in this case CutsceneData) with their corresponding IEEE-754 hexadecimal representation
-                start_index = len(output_lines)
+            if cutscene_data_regexpr.search(line) is not None:
-            elif ((line.startswith('GLOBAL_ASM("') or line.startswith('#pragma GLOBAL_ASM("'))
+                is_cutscene_data = True
-                    and line.endswith('")')):
+            elif line.endswith("};"):
-                fname = line[line.index('(') + 2 : -2]
+                is_cutscene_data = False
-                out_dependencies.append(fname)
+            if is_cutscene_data:
-                global_asm = GlobalAsmBlock(fname)
+                raw_line = re.sub(float_regexpr, repl_float_hex, raw_line)
-                with open(fname, encoding=input_enc) as f:
+            output_lines[-1] = raw_line
-                    for line2 in f:
-                        global_asm.process_line(line2.rstrip(), output_enc)
-                src, fn = global_asm.finish(state)
-                output_lines[-1] = ''.join(src)
-                asm_functions.append(fn)
-                global_asm = None
-            elif line.startswith('#include "') and line.endswith('" EARLY'):
-                # C includes qualified with EARLY (i.e. #include "file.c" EARLY) will be
-                # processed recursively when encountered
-                fpath = os.path.dirname(f.name)
-                fname = os.path.join(fpath, line[line.index(' ') + 2 : -7])
-                out_dependencies.append(fname)
-                include_src = StringIO()
-                with open(fname, encoding=input_enc) as include_file:
-                    parse_source(include_file, opt, framepointer, mips1, input_enc, output_enc, out_dependencies, include_src)
-                include_src.write('#line ' + str(line_no + 1) + ' "' + f.name + '"')
-                output_lines[-1] = include_src.getvalue()
-                include_src.close()
-            else:
-                # This is a hack to replace all floating-point numbers in an array of a particular type
-                # (in this case CutsceneData) with their corresponding IEEE-754 hexadecimal representation
-                if cutscene_data_regexpr.search(line) is not None:
-                    is_cutscene_data = True
-                elif line.endswith("};"):
-                    is_cutscene_data = False
-                if is_cutscene_data:
-                    raw_line = re.sub(float_regexpr, repl_float_hex, raw_line)
-                output_lines[-1] = raw_line
 
     if print_source:
         if isinstance(print_source, StringIO):
             for line in output_lines:
                 print_source.write(line + '\n')
         else:
+            newline_encoded = "\n".encode(output_enc)
             for line in output_lines:
-                print_source.write(line.encode(output_enc) + b'\n')
+                try:
+                    line_encoded = line.encode(output_enc)
+                except UnicodeEncodeError:
+                    print("Failed to encode a line to", output_enc)
+                    print("The line:", line)
+                    print("The line, utf-8-encoded:", line.encode("utf-8"))
+                    raise
+                print_source.write(line_encoded)
+                print_source.write(newline_encoded)
             print_source.flush()
-            if print_source != sys.stdout.buffer:
-                print_source.close()
 
     return asm_functions
 
@@ -889,6 +933,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
 
     with open(objfile_name, 'rb') as f:
         objfile = ElfFile(f.read())
+    fmt = objfile.fmt
 
     prev_locs = {
         '.text': 0,
@@ -1039,6 +1084,8 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
             new_data = list(target.data)
             for dummy_bytes_list, jtbl_rodata_size in zip(all_late_rodata_dummy_bytes, all_jtbl_rodata_size):
                 for index, dummy_bytes in enumerate(dummy_bytes_list):
+                    if not fmt.is_big_endian:
+                        dummy_bytes = dummy_bytes[::-1]
                     pos = target.data.index(dummy_bytes, last_rodata_pos)
                     # This check is nice, but makes time complexity worse for large files:
                     if SLOW_CHECKS and target.data.find(dummy_bytes, pos + 4) != -1:
@@ -1119,15 +1166,15 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
         if mdebug_section:
             strtab_index = len(objfile.symtab.strtab.data)
             new_strtab_data = []
-            ifd_max, cb_fd_offset = struct.unpack('>II', mdebug_section.data[18*4 : 20*4])
+            ifd_max, cb_fd_offset = fmt.unpack('II', mdebug_section.data[18*4 : 20*4])
-            cb_sym_offset, = struct.unpack('>I', mdebug_section.data[9*4 : 10*4])
+            cb_sym_offset, = fmt.unpack('I', mdebug_section.data[9*4 : 10*4])
-            cb_ss_offset, = struct.unpack('>I', mdebug_section.data[15*4 : 16*4])
+            cb_ss_offset, = fmt.unpack('I', mdebug_section.data[15*4 : 16*4])
             for i in range(ifd_max):
                 offset = cb_fd_offset + 18*4*i
-                iss_base, _, isym_base, csym = struct.unpack('>IIII', objfile.data[offset + 2*4 : offset + 6*4])
+                iss_base, _, isym_base, csym = fmt.unpack('IIII', objfile.data[offset + 2*4 : offset + 6*4])
                 for j in range(csym):
                     offset2 = cb_sym_offset + 12 * (isym_base + j)
-                    iss, value, st_sc_index = struct.unpack('>III', objfile.data[offset2 : offset2 + 12])
+                    iss, value, st_sc_index = fmt.unpack('III', objfile.data[offset2 : offset2 + 12])
                     st = (st_sc_index >> 26)
                     sc = (st_sc_index >> 21) & 0x1f
                     if st in [MIPS_DEBUG_ST_STATIC, MIPS_DEBUG_ST_STATIC_PROC]:
@@ -1140,6 +1187,7 @@ def fixup_objfile(objfile_name, functions, asm_prelude, assembler, output_enc, d
                         section = objfile.find_section(section_name)
                         symtype = STT_FUNC if sc == 1 else STT_OBJECT
                         sym = Symbol.from_parts(
+                            fmt,
                             st_name=strtab_index,
                             st_value=value,
                             st_size=0,
@@ -1239,6 +1287,7 @@ def run_wrapped(argv, outfile, functions):
     parser.add_argument('-mips1', dest='mips1', action='store_true')
     parser.add_argument('-g3', dest='g3', action='store_true')
     group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument('-O0', dest='opt', action='store_const', const='O0')
     group.add_argument('-O1', dest='opt', action='store_const', const='O1')
     group.add_argument('-O2', dest='opt', action='store_const', const='O2')
     group.add_argument('-g', dest='opt', action='store_const', const='g')

tools/asm_processor/build.py

@@ -1,64 +1,114 @@
 #!/usr/bin/env python3
 import sys
-import os
+from pathlib import Path
 import shlex
 import subprocess
 import tempfile
+import uuid
 import asm_processor
 
-dir_path = os.path.dirname(os.path.realpath(__file__))
+# Boolean for debugging purposes
-prelude = os.path.join(dir_path, "prelude.inc")
+# Preprocessed files are temporary, set to True to keep a copy
+keep_preprocessed_files = False
+
+dir_path = Path(__file__).resolve().parent
+asm_prelude_path = dir_path / "prelude.inc"
 
 all_args = sys.argv[1:]
-sep1 = all_args.index('--')
+sep0 = next(index for index, arg in enumerate(all_args) if not arg.startswith("-"))
-sep2 = all_args.index('--', sep1+1)
+sep1 = all_args.index("--")
+sep2 = all_args.index("--", sep1 + 1)
 
-compiler = all_args[:sep1]
+asmproc_flags = all_args[:sep0]
+compiler = all_args[sep0:sep1]
 
-assembler = all_args[sep1+1:sep2]
+assembler_args = all_args[sep1 + 1 : sep2]
-assembler_sh = ' '.join(shlex.quote(x) for x in assembler)
+assembler_sh = " ".join(shlex.quote(x) for x in assembler_args)
 
-compile_args = all_args[sep2+1:]
+
-in_file = compile_args[-1]
+compile_args = all_args[sep2 + 1 :]
-out_ind = compile_args.index('-o')
+
-out_file = compile_args[out_ind + 1]
+in_file = Path(compile_args[-1])
 del compile_args[-1]
+
+out_ind = compile_args.index("-o")
+out_file = Path(compile_args[out_ind + 1])
 del compile_args[out_ind + 1]
 del compile_args[out_ind]
 
-in_dir = os.path.split(os.path.realpath(in_file))[0]
-opt_flags = [x for x in compile_args if x in ['-g3', '-g', '-O1', '-O2', '-framepointer']]
 
-preprocessed_file = tempfile.NamedTemporaryFile(prefix='preprocessed', suffix='.c', delete=False)
+in_dir = in_file.resolve().parent
+opt_flags = [
+    x for x in compile_args if x in {"-g3", "-g", "-O0", "-O1", "-O2", "-framepointer"}
+]
+if "-mips2" not in compile_args:
+    opt_flags.append("-mips1")
 
-try:
+asmproc_flags += opt_flags + [str(in_file)]
-    asmproc_flags = opt_flags + [in_file, '--input-enc', 'utf-8', '--output-enc', 'euc-jp']
+
-    compile_cmdline = compiler + compile_args + ['-I', in_dir, '-o', out_file, preprocessed_file.name]
+# Drop .mdebug and .gptab sections from resulting binaries. This makes
+# resulting .o files much smaller and speeds up builds, but loses line
+# number debug data.
+# asmproc_flags += ["--drop-mdebug-gptab"]
+
+# Convert encoding before compiling.
+asmproc_flags += ["--input-enc", "utf-8", "--output-enc", "euc-jp"]
+
+with tempfile.TemporaryDirectory(prefix="asm_processor") as tmpdirname:
+    tmpdir_path = Path(tmpdirname)
+    preprocessed_filename = "preprocessed_" + uuid.uuid4().hex + ".c"
+    preprocessed_path = tmpdir_path / preprocessed_filename
+
+    with preprocessed_path.open("wb") as f:
+        functions, deps = asm_processor.run(asmproc_flags, outfile=f)
+
+    if keep_preprocessed_files:
+        import shutil
+
+        keep_output_dir = Path("./asm_processor_preprocessed")
+        keep_output_dir.mkdir(parents=True, exist_ok=True)
+
+        shutil.copy(
+            preprocessed_path,
+            keep_output_dir / (in_file.stem + "_" + preprocessed_filename),
+        )
+
+    compile_cmdline = (
+        compiler
+        + compile_args
+        + ["-I", str(in_dir), "-o", str(out_file), str(preprocessed_path)]
+    )
 
-    functions, deps = asm_processor.run(asmproc_flags, outfile=preprocessed_file)
     try:
         subprocess.check_call(compile_cmdline)
     except subprocess.CalledProcessError as e:
-        print("Failed to compile file " + in_file + ". Command line:")
+        print("Failed to compile file " + str(in_file) + ". Command line:")
         print()
-        print(' '.join(shlex.quote(x) for x in compile_cmdline))
+        print(" ".join(shlex.quote(x) for x in compile_cmdline))
         print()
         sys.exit(55)
-        # To keep the preprocessed file:
-        # os._exit(1)
 
-    asm_processor.run(asmproc_flags + ['--post-process', out_file, '--assembler', assembler_sh, '--asm-prelude', prelude], functions=functions)
+    asm_processor.run(
+        asmproc_flags
+        + [
+            "--post-process",
+            str(out_file),
+            "--assembler",
+            assembler_sh,
+            "--asm-prelude",
+            str(asm_prelude_path),
+        ],
+        functions=functions,
+    )
 
-    deps_file = out_file[:-2] + ".asmproc.d"
+    deps_file = out_file.with_suffix(".asmproc.d")
     if deps:
-        with open(deps_file, "w") as f:
+        with deps_file.open("w") as f:
-            f.write(out_file + ": " + " \\\n    ".join(deps) + "\n")
+            f.write(str(out_file) + ": " + " \\\n    ".join(deps) + "\n")
             for dep in deps:
                 f.write("\n" + dep + ":\n")
     else:
         try:
-            os.remove(deps_file)
+            deps_file.unlink()
         except OSError:
             pass
-finally:
-    os.remove(preprocessed_file.name)