Use virtualenv in Makefile for python dependencies (#1619)

* Use virtualenv in Makefile for python dependencies

* Create separate targets for venv creation and installation

* Fix install-python-deps prerequisite

* Try simplifying Jenkinsfile

* Simplify venv targets

* Fix merge

* Update README.md

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

---------

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

Jenkinsfile

@@ -4,21 +4,10 @@ pipeline {
     }
 
     stages {
-        stage('Install Python dependencies') {
-            steps {
-                echo 'Installing Python dependencies'
-                sh 'python3 -m venv .venv'
-                sh '''. .venv/bin/activate
-                python3 -m pip install -U -r requirements.txt
-                '''
-            }
-        }
         stage('Setup') {
             steps {
                 sh 'cp /usr/local/etc/roms/baserom_oot.z64 baseroms/gc-eu-mq-dbg/baserom.z64'
-                sh '''. .venv/bin/activate
+                sh 'make -j setup'
-                make -j setup
-                '''
             }
         }
         stage('Build (qemu-irix)') {
@@ -26,9 +15,7 @@ pipeline {
                 branch 'main'
             }
             steps {
-                sh '''. .venv/bin/activate
+                sh 'make -j ORIG_COMPILER=1'
-                make -j ORIG_COMPILER=1
-                '''
             }
         }
         stage('Build') {
@@ -38,9 +25,7 @@ pipeline {
                 }
             }
             steps {
-                sh '''. .venv/bin/activate
+                sh 'make -j'
-                make -j
-                '''
             }
         }
         stage('Report Progress') {

Makefile

@@ -55,6 +55,7 @@ endif
 
 PROJECT_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
 BUILD_DIR := build/$(VERSION)
+VENV := .venv
 
 MAKE = make
 CFLAGS += -DOOT_DEBUG
@@ -120,7 +121,7 @@ MKDMADATA  := tools/mkdmadata
 ELF2ROM    := tools/elf2rom
 ZAPD       := tools/ZAPD/ZAPD.out
 FADO       := tools/fado/fado.elf
-PYTHON     ?= python3
+PYTHON     ?= $(VENV)/bin/python3
 
 # Command to replace path variables in the spec file. We can't use the C
 # preprocessor for this because it won't substitute inside string literals.
@@ -262,6 +263,8 @@ endif
 
 #### Main Targets ###
 
+all: rom compress
+
 rom: $(ROM)
 ifneq ($(COMPARE),0)
 	@md5sum $(ROM)
@@ -287,7 +290,12 @@ distclean: clean assetclean
 	$(RM) -r baseroms/$(VERSION)/segments
 	$(MAKE) -C tools distclean
 
-setup:
+venv:
+	test -d $(VENV) || python3 -m venv $(VENV)
+	$(PYTHON) -m pip install -U pip
+	$(PYTHON) -m pip install -U -r requirements.txt
+
+setup: venv
 	$(MAKE) -C tools
 	$(PYTHON) tools/decompress_baserom.py $(VERSION)
 	$(PYTHON) extract_baserom.py
@@ -300,9 +308,8 @@ endif
 	$(N64_EMULATOR) $<
 
 
-.PHONY: all rom compress clean setup run distclean assetclean
+.PHONY: all rom compress clean assetclean distclean venv setup run
 .DEFAULT_GOAL := rom
-all: rom compress
 
 #### Various Recipes ####
 

README.md

@@ -100,37 +100,13 @@ This will copy the GitHub repository contents into a new folder in the current d
 cd oot
 ```
 
-#### 3. Install python dependencies
+#### 3. Prepare a base ROM
-
-The build process has a few python packages required that are located in `requirements.txt`.
-
-It is recommended to set up a virtual environment for python to contain all dependencies. To create a virtual environment:
-
-```bash
-python3 -m venv .venv
-```
-
-To start using the virtual environment in your current terminal run:
-
-```bash
-. .venv/bin/activate
-```
-
-Keep in mind that for each new terminal session, you will need to **activate** the Python virtual environment again. That is, run the above `. .venv/bin/activate` command.
-
-Now you can install the Python dependencies, to do so run:
-
-```bash
-pip install -U -r requirements.txt
-```
-
-#### 4. Prepare a base ROM
 
 Place a copy of the Master Quest (Debug) ROM inside the `baseroms/gc-eu-mq-dbg/` folder.
 
 Rename the file to `baserom.z64`, `baserom.n64` or `baserom.v64`, depending on the original extension.
 
-#### 5. Setup the ROM and build process
+#### 4. Setup the ROM and build process
 
 Setup and extract everything from your ROM with the following command:
 
@@ -138,10 +114,11 @@ Setup and extract everything from your ROM with the following command:
 make setup
 ```
 
-This will generate a new ROM "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64" that will have the overdump removed and the header patched.
+This downloads some dependencies (from pip), and compiles tools for the build process.
+Then it generates a new ROM "baseroms/gc-eu-mq-dbg/baserom-decompressed.z64" that will have the overdump removed and the header patched.
 It will also extract the individual assets from the ROM.
 
-#### 6. Build the ROM
+#### 5. Build the ROM
 
 Run make to build the ROM.
 Make sure your path to the project is not too long, otherwise this process may error.

docs/tutorial/helper_scripts.md

@@ -2,6 +2,27 @@
 
 This list gives brief information on the most common usage cases. For more information, first try using `-h` or `--help` as an argument, and failing that, ask in #oot-decomp-help or #tools-other in the Discord.
 
+Many tools require activating a Python virtual environment that contains Python
+dependencies. This virtual environment is automatically installed into the
+`.venv` directory by `make setup`, but you need to **activate** it in your
+current terminal session in order to run Python tools. To start using the
+virtual environment in your current terminal run:
+
+```bash
+source .venv/bin/activate
+```
+
+Keep in mind that for each new terminal session, you will need to activate the
+Python virtual environment again. That is, run the above `source .venv/bin/activate` command.
+
+To deactivate the virtual environment, run
+
+```bash
+deactivate
+```
+
+and your terminal session state will be restored to what it was before.
+
 ## m2ctx
 
 This generates the context for mips2c to use to type objects in its output. It lives in the tools directory. Running