From 5dd19e4862efdcca3b6d0c9f48c5ea829e583fd2 Mon Sep 17 00:00:00 2001 From: cadmic Date: Wed, 24 Jan 2024 16:39:13 -0800 Subject: [PATCH] Use virtualenv in Makefile for python dependencies (#1619) * Use virtualenv in Makefile for python dependencies * Create separate targets for venv creation and installation * Fix install-python-deps prerequisite * Try simplifying Jenkinsfile * Simplify venv targets * Fix merge * Update README.md Co-authored-by: Dragorn421 --------- Co-authored-by: Dragorn421 --- Jenkinsfile | 21 +++------------------ Makefile | 15 +++++++++++---- README.md | 33 +++++---------------------------- docs/tutorial/helper_scripts.md | 21 +++++++++++++++++++++ 4 files changed, 40 insertions(+), 50 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5d7333f65f..958b46590f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,21 +4,10 @@ pipeline { } stages { - stage('Install Python dependencies') { - steps { - echo 'Installing Python dependencies' - sh 'python3 -m venv .venv' - sh '''. .venv/bin/activate - python3 -m pip install -U -r requirements.txt - ''' - } - } stage('Setup') { steps { sh 'cp /usr/local/etc/roms/baserom_oot.z64 baseroms/gc-eu-mq-dbg/baserom.z64' - sh '''. .venv/bin/activate - make -j setup - ''' + sh 'make -j setup' } } stage('Build (qemu-irix)') { @@ -26,9 +15,7 @@ pipeline { branch 'main' } steps { - sh '''. .venv/bin/activate - make -j ORIG_COMPILER=1 - ''' + sh 'make -j ORIG_COMPILER=1' } } stage('Build') { @@ -38,9 +25,7 @@ pipeline { } } steps { - sh '''. .venv/bin/activate - make -j - ''' + sh 'make -j' } } stage('Report Progress') { diff --git a/Makefile b/Makefile index 6fbac669a6..a8869f3fea 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ endif PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST)))) BUILD_DIR := build/$(VERSION) +VENV := .venv MAKE = make CFLAGS += -DOOT_DEBUG @@ -120,7 +121,7 @@ MKDMADATA := tools/mkdmadata ELF2ROM := tools/elf2rom ZAPD := tools/ZAPD/ZAPD.out FADO := tools/fado/fado.elf -PYTHON ?= python3 +PYTHON ?= $(VENV)/bin/python3 # Command to replace path variables in the spec file. We can't use the C # preprocessor for this because it won't substitute inside string literals. @@ -262,6 +263,8 @@ endif #### Main Targets ### +all: rom compress + rom: $(ROM) ifneq ($(COMPARE),0) @md5sum $(ROM) @@ -287,7 +290,12 @@ distclean: clean assetclean $(RM) -r baseroms/$(VERSION)/segments $(MAKE) -C tools distclean -setup: +venv: + test -d $(VENV) || python3 -m venv $(VENV) + $(PYTHON) -m pip install -U pip + $(PYTHON) -m pip install -U -r requirements.txt + +setup: venv $(MAKE) -C tools $(PYTHON) tools/decompress_baserom.py $(VERSION) $(PYTHON) extract_baserom.py @@ -300,9 +308,8 @@ endif $(N64_EMULATOR) $< -.PHONY: all rom compress clean setup run distclean assetclean +.PHONY: all rom compress clean assetclean distclean venv setup run .DEFAULT_GOAL := rom -all: rom compress #### Various Recipes #### diff --git a/README.md b/README.md index 30adba5afa..f58d22d95f 100644 --- a/README.md +++ b/README.md @@ -100,37 +100,13 @@ This will copy the GitHub repository contents into a new folder in the current d cd oot ``` -#### 3. Install python dependencies - -The build process has a few python packages required that are located in `requirements.txt`. - -It is recommended to set up a virtual environment for python to contain all dependencies. To create a virtual environment: - -```bash -python3 -m venv .venv -``` - -To start using the virtual environment in your current terminal run: - -```bash -. .venv/bin/activate -``` - -Keep in mind that for each new terminal session, you will need to **activate** the Python virtual environment again. That is, run the above `. .venv/bin/activate` command. - -Now you can install the Python dependencies, to do so run: - -```bash -pip install -U -r requirements.txt -``` - -#### 4. Prepare a base ROM +#### 3. Prepare a base ROM Place a copy of the Master Quest (Debug) ROM inside the `baseroms/gc-eu-mq-dbg/` folder. Rename the file to `baserom.z64`, `baserom.n64` or `baserom.v64`, depending on the original extension. -#### 5. Setup the ROM and build process +#### 4. Setup the ROM and build process Setup and extract everything from your ROM with the following command: @@ -138,10 +114,11 @@ Setup and extract everything from your ROM with the following command: make setup ``` -This will generate a new ROM "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64" that will have the overdump removed and the header patched. +This downloads some dependencies (from pip), and compiles tools for the build process. +Then it generates a new ROM "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64" that will have the overdump removed and the header patched. It will also extract the individual assets from the ROM. -#### 6. Build the ROM +#### 5. Build the ROM Run make to build the ROM. Make sure your path to the project is not too long, otherwise this process may error. diff --git a/docs/tutorial/helper_scripts.md b/docs/tutorial/helper_scripts.md index cef356392d..74c580904a 100644 --- a/docs/tutorial/helper_scripts.md +++ b/docs/tutorial/helper_scripts.md @@ -2,6 +2,27 @@ This list gives brief information on the most common usage cases. For more information, first try using `-h` or `--help` as an argument, and failing that, ask in #oot-decomp-help or #tools-other in the Discord. +Many tools require activating a Python virtual environment that contains Python +dependencies. This virtual environment is automatically installed into the +`.venv` directory by `make setup`, but you need to **activate** it in your +current terminal session in order to run Python tools. To start using the +virtual environment in your current terminal run: + +```bash +source .venv/bin/activate +``` + +Keep in mind that for each new terminal session, you will need to activate the +Python virtual environment again. That is, run the above `source .venv/bin/activate` command. + +To deactivate the virtual environment, run + +```bash +deactivate +``` + +and your terminal session state will be restored to what it was before. + ## m2ctx This generates the context for mips2c to use to type objects in its output. It lives in the tools directory. Running