From 2d454933f3de1cef67b23f8896a16f22365924c3 Mon Sep 17 00:00:00 2001
From: Tharo <17233964+Thar0@users.noreply.github.com>
Date: Tue, 12 Nov 2024 13:47:34 +0000
Subject: [PATCH] [Audio 9/?] Multiversion samplebank and soundfont extraction
xmls, 1.0 and 1.1 audio extraction (#2291)
* [Audio 9/?] Multiversion samplebank and soundfont extraction xmls, 1.0 and 1.1 audio extraction
* Rework multiversion samplebanks to reduce duplicates
---
assets/xml/audio/samplebanks/SampleBank_0.xml | 863 +++++++++---------
assets/xml/audio/samplebanks/SampleBank_2.xml | 2 +-
assets/xml/audio/samplebanks/SampleBank_3.xml | 10 +-
assets/xml/audio/samplebanks/SampleBank_4.xml | 10 +-
assets/xml/audio/samplebanks/SampleBank_5.xml | 12 +-
assets/xml/audio/samplebanks/SampleBank_6.xml | 14 +-
assets/xml/audio/soundfonts/Soundfont_10.xml | 13 +-
tools/audio/extraction/audio_extract.py | 54 +-
tools/audio/extraction/audiobank_file.py | 114 ++-
tools/audio/extraction/audiotable.py | 96 +-
tools/audio/extraction/extraction_xml.py | 135 +++
tools/audio_extraction.py | 3 +-
12 files changed, 741 insertions(+), 585 deletions(-)
create mode 100644 tools/audio/extraction/extraction_xml.py
diff --git a/assets/xml/audio/samplebanks/SampleBank_0.xml b/assets/xml/audio/samplebanks/SampleBank_0.xml
index e90a084930..5de4e44b12 100644
--- a/assets/xml/audio/samplebanks/SampleBank_0.xml
+++ b/assets/xml/audio/samplebanks/SampleBank_0.xml
@@ -1,433 +1,436 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/assets/xml/audio/samplebanks/SampleBank_2.xml b/assets/xml/audio/samplebanks/SampleBank_2.xml
index 21e76424e2..05d2563d1a 100644
--- a/assets/xml/audio/samplebanks/SampleBank_2.xml
+++ b/assets/xml/audio/samplebanks/SampleBank_2.xml
@@ -1,4 +1,4 @@
-
+
diff --git a/assets/xml/audio/samplebanks/SampleBank_3.xml b/assets/xml/audio/samplebanks/SampleBank_3.xml
index e6738f8b39..268db38821 100644
--- a/assets/xml/audio/samplebanks/SampleBank_3.xml
+++ b/assets/xml/audio/samplebanks/SampleBank_3.xml
@@ -1,8 +1,8 @@
-
-
-
-
-
+
+
+
+
+
diff --git a/assets/xml/audio/samplebanks/SampleBank_4.xml b/assets/xml/audio/samplebanks/SampleBank_4.xml
index 8d68e285ff..11688e4bc7 100644
--- a/assets/xml/audio/samplebanks/SampleBank_4.xml
+++ b/assets/xml/audio/samplebanks/SampleBank_4.xml
@@ -1,8 +1,8 @@
-
-
-
-
-
+
+
+
+
+
diff --git a/assets/xml/audio/samplebanks/SampleBank_5.xml b/assets/xml/audio/samplebanks/SampleBank_5.xml
index 6eb7356935..899efe7f9c 100644
--- a/assets/xml/audio/samplebanks/SampleBank_5.xml
+++ b/assets/xml/audio/samplebanks/SampleBank_5.xml
@@ -1,9 +1,9 @@
-
-
-
-
-
-
+
+
+
+
+
+
diff --git a/assets/xml/audio/samplebanks/SampleBank_6.xml b/assets/xml/audio/samplebanks/SampleBank_6.xml
index e6971659b0..6b565dd112 100644
--- a/assets/xml/audio/samplebanks/SampleBank_6.xml
+++ b/assets/xml/audio/samplebanks/SampleBank_6.xml
@@ -1,10 +1,10 @@
-
-
-
-
-
-
-
+
+
+
+
+
+
+
diff --git a/assets/xml/audio/soundfonts/Soundfont_10.xml b/assets/xml/audio/soundfonts/Soundfont_10.xml
index 578101fd4d..126488b548 100644
--- a/assets/xml/audio/soundfonts/Soundfont_10.xml
+++ b/assets/xml/audio/soundfonts/Soundfont_10.xml
@@ -5,16 +5,19 @@
-
+
+
+
+
-
-
-
-
+
+
+
+
diff --git a/tools/audio/extraction/audio_extract.py b/tools/audio/extraction/audio_extract.py
index 9107288068..29bf632e3a 100644
--- a/tools/audio/extraction/audio_extract.py
+++ b/tools/audio/extraction/audio_extract.py
@@ -8,17 +8,18 @@ import os, shutil, time
from dataclasses import dataclass
from multiprocessing.pool import ThreadPool
from typing import Dict, List, Tuple, Union
-from xml.etree import ElementTree
-from xml.etree.ElementTree import Element
from .audio_tables import AudioCodeTable, AudioCodeTableEntry, AudioStorageMedium
from .audiotable import AudioTableData, AudioTableFile, AudioTableSample
from .audiobank_file import AudiobankFile
from .disassemble_sequence import CMD_SPEC, SequenceDisassembler, SequenceTableSpec, MMLVersion
-from .util import align, debugm, error, incbin, program_get, XMLWriter
+from .extraction_xml import ExtractionDescription, SampleBankExtractionDescription, SoundFontExtractionDescription, SequenceExtractionDescription
+from .util import align, incbin, program_get, XMLWriter
@dataclass
class GameVersionInfo:
+ # Version Name
+ version_name : str
# Music Macro Language Version
mml_version : MMLVersion
# Soundfont table code offset
@@ -49,7 +50,7 @@ BASEROM_DEBUG = False
# ======================================================================================================================
def collect_sample_banks(audiotable_seg : memoryview, extracted_dir : str, version_info : GameVersionInfo,
- table : AudioCodeTable, samplebank_xmls : Dict[int, Tuple[str, Element]]):
+ table : AudioCodeTable, samplebank_descs : Dict[int, SampleBankExtractionDescription]):
sample_banks : List[Union[AudioTableFile, int]] = []
for i,entry in enumerate(table):
@@ -72,7 +73,7 @@ def collect_sample_banks(audiotable_seg : memoryview, extracted_dir : str, versi
bug = i in version_info.audiotable_buffer_bugs
bank = AudioTableFile(i, audiotable_seg, entry, table.rom_addr, buffer_bug=bug,
- extraction_xml=samplebank_xmls.get(i, None))
+ extraction_desc=samplebank_descs.get(i, None))
if BASEROM_DEBUG:
bank.dump_bin(f"{extracted_dir}/baserom_audiotest/audiotable_files/{bank.file_name}.bin")
@@ -90,7 +91,7 @@ def bank_data_lookup(sample_banks : List[Union[AudioTableFile, int]], e : Union[
return e
def collect_soundfonts(audiobank_seg : memoryview, extracted_dir : str, version_info : GameVersionInfo,
- sound_font_table : AudioCodeTable, soundfont_xmls : Dict[int, Tuple[str, Element]],
+ sound_font_table : AudioCodeTable, soundfont_descs : Dict[int, SoundFontExtractionDescription],
sample_banks : List[Union[AudioTableFile, int]]):
soundfonts = []
@@ -104,7 +105,7 @@ def collect_soundfonts(audiobank_seg : memoryview, extracted_dir : str, version_
# Read the data
soundfont = AudiobankFile(audiobank_seg, i, entry, sound_font_table.rom_addr, bank1, bank2,
entry.sample_bank_id_1, entry.sample_bank_id_2,
- extraction_xml=soundfont_xmls.get(i, None))
+ extraction_desc=soundfont_descs.get(i, None))
soundfonts.append(soundfont)
if BASEROM_DEBUG:
@@ -186,7 +187,7 @@ def disassemble_one_sequence(extracted_dir : str, version_info : GameVersionInfo
def extract_sequences(audioseq_seg : memoryview, extracted_dir : str, version_info : GameVersionInfo, write_xml : bool,
sequence_table : AudioCodeTable, sequence_font_table : memoryview,
- sequence_xmls : Dict[int, Element], soundfonts : List[AudiobankFile]):
+ sequence_descs : Dict[int, SequenceExtractionDescription], soundfonts : List[AudiobankFile]):
sequence_font_table_cvg = [0] * len(sequence_font_table)
@@ -237,13 +238,13 @@ def extract_sequences(audioseq_seg : memoryview, extracted_dir : str, version_in
with open(f"{extracted_dir}/baserom_audiotest/audioseq_files/seq_{i}{ext}.aseq", "wb") as outfile:
outfile.write(seq_data)
- extraction_xml = sequence_xmls.get(i, None)
- if extraction_xml is None:
+ extraction_desc = sequence_descs.get(i, None)
+ if extraction_desc is None:
sequence_filename = f"seq_{i}"
sequence_name = f"Sequence_{i}"
else:
- sequence_filename = extraction_xml[0]
- sequence_name = extraction_xml[1].attrib["Name"]
+ sequence_filename = extraction_desc.file_name
+ sequence_name = extraction_desc.name
# Write extraction xml entry
if write_xml:
@@ -359,27 +360,22 @@ def extract_audio_for_version(version_info : GameVersionInfo, extracted_dir : st
# Collect extraction xmls
# ==================================================================================================================
- samplebank_xmls : Dict[int, Tuple[str, Element]] = {}
- soundfont_xmls : Dict[int, Tuple[str, Element]] = {}
- sequence_xmls : Dict[int, Tuple[str, Element]] = {}
+ samplebank_descs : Dict[int, SampleBankExtractionDescription] = {}
+ soundfont_descs : Dict[int, SoundFontExtractionDescription] = {}
+ sequence_descs : Dict[int, SequenceExtractionDescription] = {}
if read_xml:
# Read all present xmls
- def walk_xmls(out_dict : Dict[int, Tuple[str, Element]], path : str, typename : str):
+ def walk_xmls(T : type, out_dict : Dict[int, ExtractionDescription], path : str):
for root,_,files in os.walk(path):
for f in files:
- fullpath = os.path.join(root, f)
- xml = ElementTree.parse(fullpath)
- xml_root = xml.getroot()
+ desc : ExtractionDescription = T(os.path.join(root, f), f, version_info.version_name)
+ out_dict[desc.index] = desc
- if xml_root.tag != typename or "Name" not in xml_root.attrib or "Index" not in xml_root.attrib:
- error(f"Malformed {typename} extraction xml: \"{fullpath}\"")
- out_dict[int(xml_root.attrib["Index"])] = (f.replace(".xml", ""), xml_root)
-
- walk_xmls(samplebank_xmls, f"assets/xml/audio/samplebanks", "SampleBank")
- walk_xmls(soundfont_xmls, f"assets/xml/audio/soundfonts", "SoundFont")
- walk_xmls(sequence_xmls, f"assets/xml/audio/sequences", "Sequence")
+ walk_xmls(SampleBankExtractionDescription, samplebank_descs, f"assets/xml/audio/samplebanks")
+ walk_xmls(SoundFontExtractionDescription, soundfont_descs, f"assets/xml/audio/soundfonts")
+ walk_xmls(SequenceExtractionDescription, sequence_descs, f"assets/xml/audio/sequences")
# TODO warn about any missing xmls or xmls with a bad index
@@ -389,7 +385,7 @@ def extract_audio_for_version(version_info : GameVersionInfo, extracted_dir : st
if BASEROM_DEBUG:
os.makedirs(f"{extracted_dir}/baserom_audiotest/audiotable_files", exist_ok=True)
- sample_banks = collect_sample_banks(audiotable_seg, extracted_dir, version_info, sample_bank_table, samplebank_xmls)
+ sample_banks = collect_sample_banks(audiotable_seg, extracted_dir, version_info, sample_bank_table, samplebank_descs)
# ==================================================================================================================
# Collect soundfonts
@@ -397,7 +393,7 @@ def extract_audio_for_version(version_info : GameVersionInfo, extracted_dir : st
if BASEROM_DEBUG:
os.makedirs(f"{extracted_dir}/baserom_audiotest/audiobank_files", exist_ok=True)
- soundfonts = collect_soundfonts(audiobank_seg, extracted_dir, version_info, sound_font_table, soundfont_xmls,
+ soundfonts = collect_soundfonts(audiobank_seg, extracted_dir, version_info, sound_font_table, soundfont_descs,
sample_banks)
# ==================================================================================================================
@@ -459,4 +455,4 @@ def extract_audio_for_version(version_info : GameVersionInfo, extracted_dir : st
print("Extracting sequences...")
extract_sequences(audioseq_seg, extracted_dir, version_info, write_xml, sequence_table, sequence_font_table,
- sequence_xmls, soundfonts)
+ sequence_descs, soundfonts)
diff --git a/tools/audio/extraction/audiobank_file.py b/tools/audio/extraction/audiobank_file.py
index 83efbc71c7..cd4e48740b 100644
--- a/tools/audio/extraction/audiobank_file.py
+++ b/tools/audio/extraction/audiobank_file.py
@@ -5,13 +5,13 @@
#
import struct
-from typing import Optional, Tuple
-from xml.etree.ElementTree import Element
+from typing import Optional
from .audio_tables import AudioCodeTableEntry
from .audiobank_structs import AdpcmBook, AdpcmLoop, Drum, Instrument, SoundFontSample, SoundFontSound
-from .envelope import Envelope
from .audiotable import AudioTableFile, AudioTableSample
+from .envelope import Envelope
+from .extraction_xml import SoundFontExtractionDescription
from .tuning import pitch_names
from .util import XMLWriter, align, debugm, merge_like_ranges, merge_ranges
@@ -183,7 +183,7 @@ class AudiobankFile:
def __init__(self, audiobank_seg : memoryview, index : int, table_entry : AudioCodeTableEntry,
seg_offset : int, bank1 : AudioTableFile, bank2 : AudioTableFile, bank1_num : int, bank2_num : int,
- extraction_xml : Tuple[str, Element] = None):
+ extraction_desc : Optional[SoundFontExtractionDescription] = None):
self.bank_num = index
self.table_entry : AudioCodeTableEntry = table_entry
self.num_instruments = self.table_entry.num_instruments
@@ -193,7 +193,7 @@ class AudiobankFile:
self.bank1_num = bank1_num
self.bank2_num = bank2_num
- if extraction_xml is None:
+ if extraction_desc is None:
self.file_name = f"Soundfont_{self.bank_num}"
self.name = f"Soundfont_{self.bank_num}"
@@ -201,32 +201,22 @@ class AudiobankFile:
self.extraction_instruments_info = None
self.extraction_drums_info = None
self.extraction_effects_info = None
+ self.extraction_envelopes_info_versions = []
+ self.extraction_instruments_info_versions = {}
+ self.extraction_drums_info_versions = []
+ self.extraction_effects_info_versions = []
else:
- self.file_name = extraction_xml[0]
- self.name = extraction_xml[1].attrib["Name"]
+ self.file_name = extraction_desc.file_name
+ self.name = extraction_desc.name
- self.extraction_envelopes_info = []
- self.extraction_instruments_info = {}
- self.extraction_drums_info = []
- self.extraction_effects_info = []
-
- for item in extraction_xml[1]:
- if item.tag == "Envelopes":
- for env in item:
- assert env.tag == "Envelope"
- self.extraction_envelopes_info.append(env.attrib["Name"])
- elif item.tag == "Instruments":
- for instr in item:
- assert instr.tag == "Instrument"
- self.extraction_instruments_info[int(instr.attrib["ProgramNumber"])] = instr.attrib["Name"]
- elif item.tag == "Drums":
- for drum in item:
- self.extraction_drums_info.append(drum.attrib["Name"])
- elif item.tag == "Effects":
- for effect in item:
- self.extraction_effects_info.append(effect.attrib["Name"])
- else:
- assert False, item.tag
+ self.extraction_envelopes_info = extraction_desc.envelopes_info
+ self.extraction_instruments_info = extraction_desc.instruments_info
+ self.extraction_drums_info = extraction_desc.drums_info
+ self.extraction_effects_info = extraction_desc.effects_info
+ self.extraction_envelopes_info_versions = extraction_desc.envelopes_info_versions
+ self.extraction_instruments_info_versions = extraction_desc.instruments_info_versions
+ self.extraction_drums_info_versions = extraction_desc.drums_info_versions
+ self.extraction_effects_info_versions = extraction_desc.effects_info_versions
# Coverage consists of a list of itervals of the form [[start,type],[end,type]]
self.coverage = []
@@ -755,25 +745,25 @@ class AudiobankFile:
# TODO resolve decay/release index overrides?
def envelope_name(self, index):
- if self.extraction_envelopes_info is not None:
+ if self.extraction_envelopes_info is not None and index < len(self.extraction_envelopes_info):
return self.extraction_envelopes_info[index]
else:
return f"Env{index}"
def instrument_name(self, program_number):
- if self.extraction_instruments_info is not None:
+ if self.extraction_instruments_info is not None and program_number in self.extraction_instruments_info:
return self.extraction_instruments_info[program_number]
else:
return f"INST_{program_number}"
def drum_grp_name(self, index):
- if self.extraction_drums_info is not None:
+ if self.extraction_drums_info is not None and index < len(self.extraction_drums_info):
return self.extraction_drums_info[index]
else:
return f"DRUM_{index}"
def effect_name(self, index):
- if self.extraction_effects_info is not None:
+ if self.extraction_effects_info is not None and index < len(self.extraction_effects_info):
return self.extraction_effects_info[index]
else:
return f"EFFECT_{index}"
@@ -905,21 +895,41 @@ class AudiobankFile:
# add contents for names
- if len(self.envelopes) != 0:
+ if len(self.envelopes) != 0 or len(self.extraction_envelopes_info_versions) != 0:
xml.write_start_tag("Envelopes")
- for i in range(len(self.envelopes)):
+ # First write envelopes that were defined in the extraction xml, possibly interleaved with envelopes
+ # we ignored for this version
+ i = 0
+ for envelope_entry,in_version in self.extraction_envelopes_info_versions:
+ xml.write_element("Envelope", envelope_entry)
+ # Count how many envelopes we saw that were defined for this version
+ i += in_version
+
+ # Write any remaining envelopes that weren't defined in the xml
+ for j in range(i, len(self.envelopes)):
xml.write_element("Envelope", {
- "Name" : self.envelope_name(i)
+ "Name" : self.envelope_name(j)
})
xml.write_end_tag()
- if len(self.instruments) != 0:
+ if len(self.instruments) != 0 or len(self.extraction_instruments_info_versions) != 0:
xml.write_start_tag("Instruments")
# Write in struct order
- for instr in sorted(self.instruments, key=lambda instr : instr.struct_index):
+ sorted_instruments = tuple(sorted(self.instruments, key=lambda instr : instr.struct_index))
+
+ # First write instruments that were defined in the extraction xml, possibly interleaved with instruments
+ # we ignored for this version
+ i = 0
+ for instr_entry,in_version in self.extraction_instruments_info_versions:
+ xml.write_element("Instrument", instr_entry)
+ # Count how many instruments we saw that were defined for this version
+ i += in_version
+
+ # Write any remaining instruments that weren't defined in the xml
+ for instr in sorted_instruments[i:]:
instr : Instrument
if not instr.unused:
xml.write_element("Instrument", {
@@ -929,23 +939,39 @@ class AudiobankFile:
xml.write_end_tag()
- if any(isinstance(dg, DrumGroup) for dg in self.drum_groups):
+ if any(isinstance(dg, DrumGroup) for dg in self.drum_groups) or len(self.extraction_drums_info_versions):
xml.write_start_tag("Drums")
- for i,drum_grp in enumerate(self.drum_groups):
+ # First write drums that were defined in the extraction xml, possibly interleaved with drums
+ # we ignored for this version
+ i = 0
+ for drum_entry,in_version in self.extraction_drums_info_versions:
+ xml.write_element("Drum", drum_entry)
+ # Count how many drum groups we saw that were defined for this version
+ i += in_version
+
+ for j,drum_grp in enumerate(self.drum_groups[i:], i):
if isinstance(drum_grp, DrumGroup):
xml.write_element("Drum", {
- "Name" : self.drum_grp_name(i)
+ "Name" : self.drum_grp_name(j)
})
xml.write_end_tag()
- if len(self.sfx) != 0:
+ if len(self.sfx) != 0 or len(self.extraction_effects_info_versions):
xml.write_start_tag("Effects")
- for i,sfx in enumerate(self.sfx):
+ # First write effects that were defined in the extraction xml, possibly interleaved with effects
+ # we ignored for this version
+ i = 0
+ for sfx_entry,in_version in self.extraction_effects_info_versions:
+ xml.write_element("Effect", sfx_entry)
+ # Count how many effects we saw that were defined for this version
+ i += in_version
+
+ for j,sfx in enumerate(self.sfx[i:], i):
xml.write_element("Effect", {
- "Name" : self.effect_name(i)
+ "Name" : self.effect_name(j)
})
xml.write_end_tag()
diff --git a/tools/audio/extraction/audiotable.py b/tools/audio/extraction/audiotable.py
index 9511acecda..29a27695ae 100644
--- a/tools/audio/extraction/audiotable.py
+++ b/tools/audio/extraction/audiotable.py
@@ -5,11 +5,11 @@
#
import math, struct
-from typing import Dict, Tuple
-from xml.etree.ElementTree import Element
+from typing import Dict, Optional
from .audio_tables import AudioCodeTableEntry
from .audiobank_structs import AudioSampleCodec, SoundFontSample, AdpcmBook, AdpcmLoop
+from .extraction_xml import SampleBankExtractionDescription
from .tuning import pitch_names, note_z64_to_midi, recalc_tuning, rate_from_tuning, rank_rates_notes, BAD_FLOATS
from .util import align, error, XMLWriter, f32_to_u32
@@ -207,7 +207,7 @@ class AudioTableSample(AudioTableData):
def base_note_number(self):
return note_z64_to_midi(pitch_names.index(self.base_note))
- def resolve_basenote_rate(self, extraction_sample_info : Dict[int, Dict[str,str]]):
+ def resolve_basenote_rate(self, extraction_sample_info : Optional[Dict[str,str]]):
assert len(self.notes_rates) != 0
# rate_3ds = None
@@ -285,13 +285,9 @@ class AudioTableSample(AudioTableData):
final_rate,(final_note,) = rank_rates_notes(finalists)
if extraction_sample_info is not None:
- if self.start in extraction_sample_info:
- entry = extraction_sample_info[self.start]
- if "SampleRate" in entry and "BaseNote" in entry:
- final_rate = int(entry["SampleRate"])
- final_note = entry["BaseNote"]
- else:
- print(f"WARNING: Missing extraction xml entry for sample at offset=0x{self.start:X}")
+ assert "SampleRate" in extraction_sample_info and "BaseNote" in extraction_sample_info
+ final_rate = int(extraction_sample_info["SampleRate"])
+ final_note = extraction_sample_info["BaseNote"]
# print(" ",len(FINAL_NOTES_RATES), FINAL_NOTES_RATES)
# if rate_3ds is not None and len(FINAL_NOTES_RATES) == 1:
@@ -385,7 +381,8 @@ class AudioTableFile:
"""
def __init__(self, bank_num : int, audiotable_seg : memoryview, table_entry : AudioCodeTableEntry,
- seg_offset : int, buffer_bug : bool = False, extraction_xml : Tuple[str, Element] = None):
+ seg_offset : int, buffer_bug : bool = False,
+ extraction_desc : Optional[SampleBankExtractionDescription] = None):
self.bank_num = bank_num
self.table_entry : AudioCodeTableEntry = table_entry
self.data = self.table_entry.data(audiotable_seg, seg_offset)
@@ -393,24 +390,18 @@ class AudioTableFile:
self.samples_final = None
- if extraction_xml is None:
+ if extraction_desc is None:
self.file_name = f"SampleBank_{self.bank_num}"
self.name = f"SampleBank_{self.bank_num}"
+ self.extraction_sample_info_versions = []
self.extraction_sample_info = None
self.extraction_blob_info = None
else:
- self.file_name = extraction_xml[0]
- self.name = extraction_xml[1].attrib["Name"]
-
- self.extraction_sample_info = {}
- self.extraction_blob_info = {}
- for item in extraction_xml[1]:
- if item.tag == "Sample":
- self.extraction_sample_info[int(item.attrib["Offset"], 16)] = item.attrib
- elif item.tag == "Blob":
- self.extraction_blob_info[int(item.attrib["Offset"], 16)] = item.attrib
- else:
- assert False
+ self.file_name = extraction_desc.file_name
+ self.name = extraction_desc.name
+ self.extraction_sample_info_versions = extraction_desc.sample_info_versions
+ self.extraction_sample_info = extraction_desc.sample_info
+ self.extraction_blob_info = extraction_desc.blob_info
self.pointer_indices = []
@@ -461,28 +452,24 @@ class AudioTableFile:
return self.samples[offset]
def sample_name(self, sample : AudioTableSample, index : int):
- if self.extraction_sample_info is not None:
- if sample.start in self.extraction_sample_info:
- return self.extraction_sample_info[sample.start]["Name"]
- print(f"WARNING: Missing extraction xml entry for sample at offset=0x{sample.start:X}")
+ if self.extraction_sample_info is not None and index < len(self.extraction_sample_info):
+ return self.extraction_sample_info[index]["Name"]
+
return f"SAMPLE_{self.bank_num}_{index}"
def sample_filename(self, sample : AudioTableSample, index : int):
ext = sample.codec_file_extension_compressed()
- if self.extraction_sample_info is not None:
- if sample.start in self.extraction_sample_info:
- return self.extraction_sample_info[sample.start]["FileName"] + ext
- print(f"WARNING: Missing extraction xml entry for sample at offset=0x{sample.start:X}")
+ if self.extraction_sample_info is not None and index < len(self.extraction_sample_info):
+ return self.extraction_sample_info[index]["FileName"] + ext
npad = int(math.floor(1 + math.log10(len(self.samples)))) if len(self.samples) != 0 else 0
return f"Sample{index:0{npad}}{ext}"
- def blob_filename(self, start, end):
- if self.extraction_blob_info is not None:
- if start in self.extraction_blob_info:
- return self.extraction_blob_info[start]["Name"]
- print(f"WARNING: Missing extraction xml entry for blob at offset=0x{start:X}")
+ def blob_filename(self, start, end, index):
+ if self.extraction_blob_info is not None and index < len(self.extraction_blob_info):
+ return self.extraction_blob_info[index]["Name"]
+
return f"UNACCOUNTED_{start:X}_{end:X}"
def finalize_samples(self):
@@ -490,7 +477,7 @@ class AudioTableFile:
for i,sample in enumerate(self.samples_final):
sample : AudioTableSample
- sample.resolve_basenote_rate(self.extraction_sample_info)
+ sample.resolve_basenote_rate(self.extraction_sample_info[i] if self.extraction_sample_info is not None else None)
def finalize_coverage(self, all_sample_banks):
if len(self.coverage) != 0:
@@ -577,6 +564,7 @@ class AudioTableFile:
def assign_names(self):
i = 0
+ j = 0
for sample in self.samples_final:
if isinstance(sample, AudioTableSample):
sample : AudioTableSample
@@ -587,9 +575,10 @@ class AudioTableFile:
else:
sample : AudioTableData
- name = self.blob_filename(sample.start, sample.end)
+ name = self.blob_filename(sample.start, sample.end, j)
sample.name = name
sample.filename = f"{name}.bin"
+ j += 1
def to_xml(self, base_path):
xml = XMLWriter()
@@ -635,33 +624,36 @@ class AudioTableFile:
xml.write_comment("This file is only for extraction of vanilla data. For other purposes see assets/audio/samplebanks/")
- start = {
+ xml.write_start_tag("SampleBank", {
"Name" : self.name,
"Index" : self.bank_num,
- }
- xml.write_start_tag("SampleBank", start)
+ })
+ # Write elements from the old xml version verbatim
i = 0
- for sample in self.samples_final:
+ for entry_name,entry_attrs,in_version in self.extraction_sample_info_versions:
+ xml.write_element(entry_name, entry_attrs)
+ i += in_version
+
+ # Write any new elements
+ for sample in self.samples_final[i:]:
if isinstance(sample, AudioTableSample):
sample : AudioTableSample
- xml.write_element("Sample", {
+ attrs = {
"Name" : sample.name,
"FileName" : sample.filename.replace(sample.codec_file_extension_compressed(), ""),
- "Offset" : f"0x{sample.start:06X}",
"SampleRate" : sample.sample_rate,
"BaseNote" : sample.base_note,
- })
- i += 1
+ }
+ xml.write_element("Sample", attrs)
else:
sample : AudioTableData
- xml.write_element("Blob", {
- "Name" : sample.name,
- "Offset" : f"0x{sample.start:06X}",
- "Size" : f"0x{sample.end - sample.start:X}",
- })
+ attrs = {
+ "Name" : sample.name,
+ }
+ xml.write_element("Blob", attrs)
xml.write_end_tag()
diff --git a/tools/audio/extraction/extraction_xml.py b/tools/audio/extraction/extraction_xml.py
new file mode 100644
index 0000000000..f54b7749f0
--- /dev/null
+++ b/tools/audio/extraction/extraction_xml.py
@@ -0,0 +1,135 @@
+# SPDX-FileCopyrightText: © 2024 ZeldaRET
+# SPDX-License-Identifier: CC0-1.0
+#
+#
+#
+
+from xml.etree import ElementTree
+from xml.etree.ElementTree import Element
+
+from .util import error
+
+class ExtractionDescription:
+
+ def __init__(self, file_path : str, file_name : str, version_name : str) -> None:
+ self.type_name = type(self).__name__.replace("ExtractionDescription", "")
+ self.file_name = file_name.replace(".xml", "")
+ self.file_path = file_path
+
+ xml_root = ElementTree.parse(file_path).getroot()
+ if xml_root.tag != self.type_name or "Name" not in xml_root.attrib or "Index" not in xml_root.attrib:
+ error(f"Malformed {self.type_name} extraction xml: \"{file_path}\"")
+
+ self.name = xml_root.attrib["Name"]
+ self.index = int(xml_root.attrib["Index"])
+
+ self.post_init(xml_root, version_name)
+
+ def post_init(self, xml_root : Element, version_name : str):
+ raise NotImplementedError() # Implement in subclass
+
+ def in_version(self, version_include, version_exclude, version_name : str):
+ if version_include == "":
+ version_include = "All"
+ if version_exclude == "":
+ version_exclude = "None"
+
+ # Determine if this layout is the one we need
+ if version_include != "All":
+ version_include = version_include.split(",")
+ if version_exclude != "None":
+ version_exclude = version_exclude.split(",")
+
+ included = version_include == "All" or version_name in version_include
+ excluded = version_exclude != "None" and version_name in version_exclude
+
+ return included and not excluded
+
+class SampleBankExtractionDescription(ExtractionDescription):
+
+ def post_init(self, xml_root : Element, version_name : str):
+ self.included_version = None
+ self.sample_info = []
+ self.sample_info_versions = []
+ self.blob_info = []
+
+ for item in xml_root:
+ if item.tag == "Sample":
+ version_include = item.attrib.get("VersionInclude", "")
+ version_exclude = item.attrib.get("VersionExclude", "")
+ in_version = self.in_version(version_include, version_exclude, version_name)
+ if in_version:
+ self.sample_info.append(item.attrib)
+ self.sample_info_versions.append((item.tag, item.attrib, in_version))
+ elif item.tag == "Blob":
+ version_include = item.attrib.get("VersionInclude", "")
+ version_exclude = item.attrib.get("VersionExclude", "")
+ in_version = self.in_version(version_include, version_exclude, version_name)
+ if in_version:
+ self.blob_info.append(item.attrib)
+ self.sample_info_versions.append((item.attrib, in_version))
+ else:
+ print(xml_root.attrib)
+ assert False, item.tag
+
+class SoundFontExtractionDescription(ExtractionDescription):
+
+ def post_init(self, xml_root : Element, version_name : str):
+ self.envelopes_info = []
+ self.instruments_info = {}
+ self.drums_info = []
+ self.effects_info = []
+ self.envelopes_info_versions = []
+ self.instruments_info_versions = []
+ self.drums_info_versions = []
+ self.effects_info_versions = []
+
+ for item in xml_root:
+ if item.tag == "Envelopes":
+ for env in item:
+ assert env.tag == "Envelope"
+
+ version_include = env.attrib.get("VersionInclude", "")
+ version_exclude = env.attrib.get("VersionExclude", "")
+ in_version = self.in_version(version_include, version_exclude, version_name)
+ if in_version:
+ self.envelopes_info.append(env.attrib["Name"])
+ self.envelopes_info_versions.append((env.attrib, in_version))
+ elif item.tag == "Instruments":
+ for instr in item:
+ assert instr.tag == "Instrument"
+ prg_num = int(instr.attrib["ProgramNumber"])
+
+ version_include = instr.attrib.get("VersionInclude", "")
+ version_exclude = instr.attrib.get("VersionExclude", "")
+ in_version = self.in_version(version_include, version_exclude, version_name)
+ if in_version:
+ self.instruments_info[prg_num] = instr.attrib["Name"]
+ self.instruments_info_versions.append((instr.attrib, in_version))
+ elif item.tag == "Drums":
+ for drum in item:
+ assert drum.tag == "Drum"
+
+ version_include = drum.attrib.get("VersionInclude", "")
+ version_exclude = drum.attrib.get("VersionExclude", "")
+ in_version = self.in_version(version_include, version_exclude, version_name)
+ if in_version:
+ self.drums_info.append(drum.attrib["Name"])
+ self.drums_info_versions.append((drum.attrib, in_version))
+ elif item.tag == "Effects":
+ for effect in item:
+ assert effect.tag == "Effect"
+
+ version_include = effect.attrib.get("VersionInclude", "")
+ version_exclude = effect.attrib.get("VersionExclude", "")
+ in_version = self.in_version(version_include, version_exclude, version_name)
+ if in_version:
+ self.effects_info.append(effect.attrib["Name"])
+ self.effects_info_versions.append((effect.attrib, in_version))
+ else:
+ assert False, item.tag
+
+class SequenceExtractionDescription(ExtractionDescription):
+
+ def post_init(self, xml_root : Element, version_name : str):
+ pass
diff --git a/tools/audio_extraction.py b/tools/audio_extraction.py
index 9c3a202648..ab5d5b3fb4 100644
--- a/tools/audio_extraction.py
+++ b/tools/audio_extraction.py
@@ -184,7 +184,8 @@ if __name__ == '__main__':
),
}
- version_info = GameVersionInfo(MMLVersion.OOT,
+ version_info = GameVersionInfo(version,
+ MMLVersion.OOT,
soundfont_table_code_offset,
seq_font_table_code_offset,
seq_table_code_offset,