From 70716be96d1ede6b11eef078d205678ee5a422bf Mon Sep 17 00:00:00 2001 From: cadmic Date: Wed, 14 Aug 2024 20:39:34 -0700 Subject: [PATCH] Build all GC retail ROMs in Jenkins (#2027) * Build all GC retail ROMs in Jenkins * Delete progress tracking Doesn't play nicely with deleting extracted/ and build/ after each version, and it currently serves no purpose * Update Jenkinsfile Co-authored-by: Dragorn421 * Reorder ROMs * Symlink ROMs instead of copying --------- Co-authored-by: Dragorn421 --- Jenkinsfile | 89 ++++++++++++++++++------------ progress.py | 153 ---------------------------------------------------- 2 files changed, 54 insertions(+), 188 deletions(-) delete mode 100755 progress.py diff --git a/Jenkinsfile b/Jenkinsfile index 2566f13e3d..e8934a557f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -24,52 +24,71 @@ pipeline { sh 'python3 tools/check_format.py --verbose --compare-to origin/main' } } - stage('Setup gc-eu-mq-dbg') { + // The ROMs are built in an order that maximizes compiler flags coverage in a "fail fast" approach. + // Specifically we start with a retail ROM for BSS ordering, and make sure we cover all of + // NTSC/PAL/MQ/DEBUG as quickly as possible. + stage('Build gc-jp') { steps { - sh 'cp /usr/local/etc/roms/oot-gc-eu-mq-dbg.z64 baseroms/gc-eu-mq-dbg/baserom.z64' - sh 'make -j setup' - } - } - stage('Build gc-eu-mq-dbg') { - steps { - sh 'make -j RUN_CC_CHECK=0' - } - } - stage('Setup gc-eu-mq') { - steps { - sh 'cp /usr/local/etc/roms/oot-gc-eu-mq.z64 baseroms/gc-eu-mq/baserom.z64' - sh 'make -j setup VERSION=gc-eu-mq' + sh 'ln -s /usr/local/etc/roms/oot-gc-jp.z64 baseroms/gc-jp/baserom.z64' + sh 'make -j setup VERSION=gc-jp' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-jp' + sh 'make clean assetclean VERSION=gc-jp' } } stage('Build gc-eu-mq') { steps { - sh 'make -j VERSION=gc-eu-mq RUN_CC_CHECK=0' + sh 'ln -s /usr/local/etc/roms/oot-gc-eu-mq.z64 baseroms/gc-eu-mq/baserom.z64' + sh 'make -j setup VERSION=gc-eu-mq' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-eu-mq' + sh 'make clean assetclean VERSION=gc-eu-mq' } } - stage('Report Progress') { - when { - branch 'main' - } + stage('Build gc-eu-mq-dbg') { steps { - sh 'mkdir reports' - sh 'python3 progress.py csv >> reports/progress-oot-nonmatching.csv' - sh 'python3 progress.py csv -m >> reports/progress-oot-matching.csv' - sh 'python3 progress.py shield-json > reports/progress-oot-shield.json' - stash includes: 'reports/*', name: 'reports' + sh 'ln -s /usr/local/etc/roms/oot-gc-eu-mq-dbg.z64 baseroms/gc-eu-mq-dbg/baserom.z64' + sh 'make -j setup VERSION=gc-eu-mq-dbg' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-eu-mq-dbg' + sh 'make clean assetclean VERSION=gc-eu-mq-dbg' } } - stage('Update Progress') { - when { - branch 'main' - } - agent { - label 'zeldaret_website' - } + stage('Build gc-us') { steps { - unstash 'reports' - sh 'cat reports/progress-oot-nonmatching.csv >> /var/www/zelda64.dev/assets/csv/progress-oot-nonmatching.csv' - sh 'cat reports/progress-oot-matching.csv >> /var/www/zelda64.dev/assets/csv/progress-oot-matching.csv' - sh 'cat reports/progress-oot-shield.json > /var/www/zelda64.dev/assets/csv/progress-oot-shield.json' + sh 'ln -s /usr/local/etc/roms/oot-gc-us.z64 baseroms/gc-us/baserom.z64' + sh 'make -j setup VERSION=gc-us' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-us' + sh 'make clean assetclean VERSION=gc-us' + } + } + stage('Build gc-jp-ce') { + steps { + sh 'ln -s /usr/local/etc/roms/oot-gc-jp-ce.z64 baseroms/gc-jp-ce/baserom.z64' + sh 'make -j setup VERSION=gc-jp-ce' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-jp-ce' + sh 'make clean assetclean VERSION=gc-jp-ce' + } + } + stage('Build gc-eu') { + steps { + sh 'ln -s /usr/local/etc/roms/oot-gc-eu.z64 baseroms/gc-eu/baserom.z64' + sh 'make -j setup VERSION=gc-eu' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-eu' + sh 'make clean assetclean VERSION=gc-eu' + } + } + stage('Build gc-jp-mq') { + steps { + sh 'ln -s /usr/local/etc/roms/oot-gc-jp-mq.z64 baseroms/gc-jp-mq/baserom.z64' + sh 'make -j setup VERSION=gc-jp-mq' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-jp-mq' + sh 'make clean assetclean VERSION=gc-jp-mq' + } + } + stage('Build gc-us-mq') { + steps { + sh 'ln -s /usr/local/etc/roms/oot-gc-us-mq.z64 baseroms/gc-us-mq/baserom.z64' + sh 'make -j setup VERSION=gc-us-mq' + sh 'make -j RUN_CC_CHECK=0 VERSION=gc-us-mq' + sh 'make clean assetclean VERSION=gc-us-mq' } } } diff --git a/progress.py b/progress.py deleted file mode 100755 index 0556ae6f77..0000000000 --- a/progress.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import json -import git -import os -import re - -parser = argparse.ArgumentParser(description="Computes current progress throughout the whole project.") -parser.add_argument("format", nargs="?", default="text", choices=["text", "csv", "shield-json"]) -parser.add_argument("-m", "--matching", dest='matching', action='store_true', - help="Output matching progress instead of decompilation progress") -args = parser.parse_args() - -NON_MATCHING_PATTERN = r"#ifdef\s+NON_MATCHING.*?#pragma\s+GLOBAL_ASM\s*\(\s*\"(.*?)\"\s*\).*?#endif" - -def GetNonMatchingFunctions(files): - functions = [] - - for file in files: - with open(file) as f: - functions += re.findall(NON_MATCHING_PATTERN, f.read(), re.DOTALL) - - return functions - -def ReadAllLines(fileName): - lineList = list() - with open(fileName) as f: - lineList = f.readlines() - - return lineList - -def GetFiles(path, ext): - files = [] - - for r, d, f in os.walk(path): - for file in f: - if file.endswith(ext): - files.append(os.path.join(r, file)) - - return files - -nonMatchingFunctions = GetNonMatchingFunctions(GetFiles("src", ".c")) if not args.matching else [] - -def GetNonMatchingSize(path): - size = 0 - - asmFiles = GetFiles(path, ".s") - - for asmFilePath in asmFiles: - if asmFilePath not in nonMatchingFunctions: - asmLines = ReadAllLines(asmFilePath) - - for asmLine in asmLines: - if (asmLine.startswith("/*")): - size += 4 - - return size - -def IsCFile(objfile): - srcfile = objfile.strip().replace("build/gc-eu-mq-dbg/", "").replace(".o", ".c") - return os.path.isfile(srcfile) - -mapFile = ReadAllLines("build/gc-eu-mq-dbg/oot-gc-eu-mq-dbg.map") -curSegment = None -src = 0 -code = 0 -boot = 0 -ovl = 0 - -for line in mapFile: - - if "_codeSegmentStart" in line: - curSegment = "code" - elif "_bootSegmentStart" in line: - curSegment = "boot" - elif "_codeSegmentEnd" in line or "_bootSegmentEnd" in line: - curSegment = None - - lineSplit = list(filter(None, line.split(" "))) - - if (len(lineSplit) == 4 and lineSplit[0].startswith(".")): - section = lineSplit[0] - size = int(lineSplit[2], 16) - objFile = lineSplit[3] - - if (section == ".text" and IsCFile(objFile)): - if objFile.startswith("build/gc-eu-mq-dbg/src"): - src += size - - if curSegment == "code": - code += size - elif curSegment == "boot": - boot += size - else: - ovl += size - -nonMatchingASM = GetNonMatchingSize("asm/non_matchings") -nonMatchingASMBoot = GetNonMatchingSize("asm/non_matchings/boot") -nonMatchingASMCode = GetNonMatchingSize("asm/non_matchings/code") -nonMatchingASMOvl = GetNonMatchingSize("asm/non_matchings/overlays") - -src -= nonMatchingASM -code -= nonMatchingASMCode -boot -= nonMatchingASMBoot -ovl -= nonMatchingASMOvl - -bootSize = 31408 # decompilable code only -codeSize = 999984 # decompilable code only -ovlSize = 2812000 # .text sections - -total = src + nonMatchingASM -srcPct = 100 * src / total -codePct = 100 * code / codeSize -bootPct = 100 * boot / bootSize -ovlPct = 100 * ovl / ovlSize - -bytesPerHeartPiece = total // 80 - -if args.format == 'csv': - csv_version = 2 - git_object = git.Repo().head.object - timestamp = str(git_object.committed_date) - git_hash = git_object.hexsha - csv_list = [str(csv_version), timestamp, git_hash, str(src), str(total), str(code), str(codeSize), str(boot), str(bootSize), str(ovl), str(ovlSize), str(nonMatchingASM), str(len(nonMatchingFunctions))] - print(",".join(csv_list)) -elif args.format == 'shield-json': - # https://shields.io/endpoint - print(json.dumps({ - "schemaVersion": 1, - "label": "progress", - "message": f"{srcPct:.3g}%", - "color": 'yellow' if srcPct < 100 else 'brightgreen', - })) -elif args.format == 'text': - adjective = "decompiled" if not args.matching else "matched" - - print(str(total) + " total bytes of decompilable code\n") - print(str(src) + " bytes " + adjective + " in src " + str(srcPct) + "%\n") - print(str(boot) + "/" + str(bootSize) + " bytes " + adjective + " in boot " + str(bootPct) + "%\n") - print(str(code) + "/" + str(codeSize) + " bytes " + adjective + " in code " + str(codePct) + "%\n") - print(str(ovl) + "/" + str(ovlSize) + " bytes " + adjective + " in overlays " + str(ovlPct) + "%\n") - print("------------------------------------\n") - - heartPieces = int(src / bytesPerHeartPiece) - rupees = int(((src % bytesPerHeartPiece) * 100) / bytesPerHeartPiece) - - if (rupees > 0): - print("You have " + str(heartPieces) + "/80 heart pieces and " + str(rupees) + " rupee(s).\n") - else: - print("You have " + str(heartPieces) + "/80 heart pieces.\n") -else: - print("Unknown format argument: " + args.format)