Add n64texconv and bin2c tools to convert extracted .png and .bin to C arrays during build (#2477)

* n64texconv and bin2c

* mv tools/n64texconv tools/assets/

* fix

* more light fixes

* Silence -Wshadow for libimagequant

* Add reference counting gc for palette objects in python bindings

* Fix missing alignment in n64texconv_*_to_c functions

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

* Check palette size in n64texconv_image_from_png

* accept memoryview as well as bytes for binary data

* minimal doc on n64texconv_quantize_shared

* fix a buffer size passed to libimagequant

* assert pal count <= 256 on png write

* Disable palette size check for input pngs, ZAPD fails the check

* No OpenMP for clang

* When reading an indexed png into a CI format, requantize if there are too many colors for the target texel size

---------

Co-authored-by: Dragorn421 <Dragorn421@users.noreply.github.com>

tools/assets/n64texconv/src/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 ZeldaRET
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tools/assets/n64texconv/src/app/main.c

@@ -0,0 +1,137 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#include <assert.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#define NORETURN       __attribute__((noreturn))
+#define ARRLEN(a)      (sizeof(a) / sizeof((a)[0]))
+#define strequ(s1, s2) (strcmp((s1), (s2)) == 0)
+
+#include "../libn64texconv/n64texconv.h"
+#include "../libn64texconv/jfif.h"
+
+static bool
+is_regular_file(const char *path)
+{
+    struct stat sb;
+    stat(path, &sb);
+    return S_ISREG(sb.st_mode);
+}
+
+static NORETURN void
+usage(const char *progname)
+{
+    fprintf(stderr,
+            "n64texconv: Convert an input png to N64 data in the desired format.\n"
+            "Usage: %s <type> <in.png> <out.c> [pal_out.c]\n"
+            "    Valid types: i4 / i8 / ci4 / ci8 / ia4 / ia8 / ia16 / rgba16 / rgba32 / JFIF\n",
+            progname);
+    exit(EXIT_FAILURE);
+}
+
+static const struct fmt_info {
+    const char *name;
+    int fmt;
+    int siz;
+} fmt_map[] = {
+  // clang-format off
+    { "i4",     G_IM_FMT_I,    G_IM_SIZ_4b,  },
+    { "i8",     G_IM_FMT_I,    G_IM_SIZ_8b,  },
+    { "ci4",    G_IM_FMT_CI,   G_IM_SIZ_4b,  },
+    { "ci8",    G_IM_FMT_CI,   G_IM_SIZ_8b,  },
+    { "ia4",    G_IM_FMT_IA,   G_IM_SIZ_4b,  },
+    { "ia8",    G_IM_FMT_IA,   G_IM_SIZ_8b,  },
+    { "ia16",   G_IM_FMT_IA,   G_IM_SIZ_16b, },
+    { "rgba16", G_IM_FMT_RGBA, G_IM_SIZ_16b, },
+    { "rgba32", G_IM_FMT_RGBA, G_IM_SIZ_32b, },
+  // clang-format on
+};
+
+int
+main(int argc, char **argv)
+{
+    const char *progname = argv[0];
+    const char *fmt;
+    const char *array_fmt;
+    const char *in;
+    const char *out;
+    const char *pal_out;
+
+    if (argc < 5)
+        usage(progname);
+
+    fmt = argv[1];
+    array_fmt = argv[2];
+    in = argv[3];
+    out = argv[4];
+    pal_out = (argc > 5) ? argv[5] : NULL;
+
+    unsigned int byte_width = (strequ(array_fmt, "u32") ? 4 : 8);
+
+    if (!is_regular_file(in)) {
+        fprintf(stderr, "Could not open input file %s\n", in);
+        return EXIT_FAILURE;
+    }
+
+    if (strequ(fmt, "JFIF")) {
+        struct JFIF *jfif = jfif_fromfile(in, JFIF_BUFFER_SIZE);
+        if (jfif == NULL) {
+            fprintf(stderr, "Could not open input file %s\n", in);
+            return EXIT_FAILURE;
+        }
+        if (jfif_to_c_file(out, jfif, JFIF_BUFFER_SIZE)) {
+            fprintf(stderr, "Could not save output C file %s\n", out);
+            return EXIT_FAILURE;
+        }
+        jfif_free(jfif);
+    } else {
+        int rv;
+        const struct fmt_info *fmt_info = NULL;
+        for (size_t i = 0; i < ARRLEN(fmt_map); i++) {
+            if (strequ(fmt, fmt_map[i].name)) {
+                fmt_info = &fmt_map[i];
+                break;
+            }
+        }
+        if (fmt_info == NULL) {
+            fprintf(stderr, "Error: Invalid fmt %s\n", fmt);
+            return EXIT_FAILURE;
+        }
+
+        struct n64_image *img = n64texconv_image_from_png(in, fmt_info->fmt, fmt_info->siz, G_IM_FMT_RGBA);
+        if (img == NULL) {
+            fprintf(stderr, "Could not open input file %s\n", in);
+            return EXIT_FAILURE;
+        }
+
+        if (img->pal != NULL) {
+            if (pal_out == NULL) {
+                fprintf(stderr, "Input file %s is color indexed, a palette output C file must be provided.\n", in);
+                usage(progname);
+            }
+
+            if (rv = n64texconv_palette_to_c_file(pal_out, img->pal, false, byte_width), rv != 0) {
+                fprintf(stderr, "Could not save output C file %s (error %d)\n", pal_out, rv);
+                return EXIT_FAILURE;
+            }
+        }
+
+        if (rv = n64texconv_image_to_c_file(out, img, false, false, byte_width), rv != 0) {
+            fprintf(stderr, "Could not save output C file %s (error %d)\n", out, rv);
+            return EXIT_FAILURE;
+        }
+
+        if (img->pal != NULL)
+            n64texconv_palette_free(img->pal);
+        n64texconv_image_free(img);
+    }
+
+    return EXIT_SUCCESS;
+}

tools/assets/n64texconv/src/libn64texconv/bin2c.c

@@ -0,0 +1,130 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#include <assert.h>
+#include <errno.h>
+#include <inttypes.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "bin2c.h"
+#include "endian.h"
+
+#define BYTES_PER_ROW 32
+#define LINE_MASK     (BYTES_PER_ROW - 1)
+
+int
+bin2c(char **out, size_t *size_out, void *bin, size_t size, size_t pad_to_size, unsigned int byte_width)
+{
+    assert(out != NULL);
+    assert(size_out != NULL);
+    assert(bin != NULL);
+
+    if (byte_width != 1 && byte_width != 2 && byte_width != 4 && byte_width != 8)
+        return -2;
+
+    size_t end_size = (pad_to_size > size) ? pad_to_size : size;
+
+    if ((end_size & (byte_width - 1)) != 0)
+        return -3;
+
+    size_t size_out_ = (1 + 1 + 2 * byte_width + 1 + 1) * ((end_size + byte_width - 1) / byte_width) + 2;
+    char *out_ = malloc(size_out_);
+    if (out_ == NULL)
+        return -1;
+
+    char *pos = out_;
+    bool was_newline = false;
+    for (size_t p = 0; p < size; p += byte_width) {
+        size_t rem = byte_width;
+        if (rem > size - p) // For any remaining unaligned data, rest will be padded with 0
+            rem = size - p;
+
+        // Read input
+        uint64_t d = 0;
+        memcpy(&d, &((uint8_t *)bin)[p], rem);
+
+        // Byteswap + shift
+        d = be64toh(d) >> (64 - 8 * byte_width);
+
+        // Write output
+        was_newline = (((p + byte_width) & LINE_MASK) == 0);
+        char end = was_newline ? '\n' : ' ';
+        pos += sprintf(pos, "0x%0*" PRIX64 ",%c", 2 * byte_width, d, end);
+    }
+
+    for (size_t p = (size + byte_width - 1) & ~(byte_width - 1); p < pad_to_size; p += byte_width) {
+        was_newline = (((p + byte_width) & LINE_MASK) == 0);
+        char end = was_newline ? '\n' : ' ';
+        pos += sprintf(pos, "0x%0*" PRIX64 ",%c", 2 * byte_width, (uint64_t)0, end);
+    }
+
+    if (!was_newline)
+        *pos++ = '\n';
+
+    *pos++ = '\0';
+
+    *out = out_;
+    *size_out = size_out_;
+    return 0;
+}
+
+int
+bin2c_file(const char *out_path, void *bin, size_t size, size_t pad_to_size, unsigned int byte_width)
+{
+    assert(out_path != NULL);
+    assert(bin != NULL);
+
+    if (byte_width != 1 && byte_width != 2 && byte_width != 4 && byte_width != 8)
+        return -2;
+
+    size_t end_size = (pad_to_size > size) ? pad_to_size : size;
+
+    if ((end_size & (byte_width - 1)) != 0)
+        return -3;
+
+    FILE *of = fopen(out_path, "w");
+    if (of == NULL)
+        return -1;
+
+    bool was_newline = false;
+    for (size_t p = 0; p < size; p += byte_width) {
+        size_t rem = byte_width;
+        if (rem > size - p) // For any remaining unaligned data, rest will be padded with 0
+            rem = size - p;
+
+        // Read input
+        uint64_t d = 0;
+        memcpy(&d, &((uint8_t *)bin)[p], rem);
+
+        // Byteswap + shift
+        d = be64toh(d) >> (64 - 8 * byte_width);
+
+        // Write output
+        was_newline = (((p + byte_width) & LINE_MASK) == 0);
+        char end = was_newline ? '\n' : ' ';
+
+        if (fprintf(of, "0x%0*" PRIX64 ",%c", 2 * byte_width, d, end) < 0)
+            goto error_post_open;
+    }
+
+    for (size_t p = (size + byte_width - 1) & ~(byte_width - 1); p < pad_to_size; p += byte_width) {
+        was_newline = (((p + byte_width) & LINE_MASK) == 0);
+        char end = was_newline ? '\n' : ' ';
+        if (fprintf(of, "0x%0*" PRIX64 ",%c", 2 * byte_width, (uint64_t)0, end) < 0)
+            goto error_post_open;
+    }
+
+    if (!was_newline)
+        fputs("\n", of);
+
+    fclose(of);
+    return 0;
+error_post_open:
+    fclose(of);
+    if (remove(out_path) != 0)
+        fprintf(stderr, "error calling remove(\"%s\"): %s", out_path, strerror(errno));
+    return -4;
+}

tools/assets/n64texconv/src/libn64texconv/bin2c.h

@@ -0,0 +1,14 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#ifndef BIN2C_H
+#define BIN2C_H
+
+#include <stddef.h>
+
+int
+bin2c(char **out, size_t *size_out, void *bin, size_t size, size_t pad_to_size, unsigned int byte_width);
+
+int
+bin2c_file(const char *out_path, void *bin, size_t size, size_t pad_to_size, unsigned int byte_width);
+
+#endif

tools/assets/n64texconv/src/libn64texconv/endian.h

@@ -0,0 +1,69 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#ifndef ENDIAN_H
+#define ENDIAN_H
+
+#if defined(__linux__) || defined(__CYGWIN__)
+#include <endian.h>
+#elif defined(__APPLE__)
+#include <libkern/OSByteOrder.h>
+
+#define htobe16(x) OSSwapHostToBigInt16(x)
+#define htole16(x) OSSwapHostToLittleInt16(x)
+#define be16toh(x) OSSwapBigToHostInt16(x)
+#define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#define htobe32(x) OSSwapHostToBigInt32(x)
+#define htole32(x) OSSwapHostToLittleInt32(x)
+#define be32toh(x) OSSwapBigToHostInt32(x)
+#define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#define htobe64(x) OSSwapHostToBigInt64(x)
+#define htole64(x) OSSwapHostToLittleInt64(x)
+#define be64toh(x) OSSwapBigToHostInt64(x)
+#define le64toh(x) OSSwapLittleToHostInt64(x)
+#else
+
+#if !defined(__BYTE_ORDER__)
+#error "No endian define provided by compiler"
+#endif
+
+#if (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+
+#define htobe16(x) (x)
+#define htole16(x) __builtin_bswap16(x)
+#define be16toh(x) (x)
+#define le16toh(x) __builtin_bswap16(x)
+
+#define htobe32(x) (x)
+#define htole32(x) __builtin_bswap32(x)
+#define be32toh(x) (x)
+#define le32toh(x) __builtin_bswap32(x)
+
+#define htobe64(x) (x)
+#define htole64(x) __builtin_bswap64(x)
+#define be64toh(x) (x)
+#define le64toh(x) __builtin_bswap64(x)
+
+#else
+
+#define htobe16(x) __builtin_bswap16(x)
+#define htole16(x) (x)
+#define be16toh(x) __builtin_bswap16(x)
+#define le16toh(x) (x)
+
+#define htobe32(x) __builtin_bswap32(x)
+#define htole32(x) (x)
+#define be32toh(x) __builtin_bswap32(x)
+#define le32toh(x) (x)
+
+#define htobe64(x) __builtin_bswap64(x)
+#define htole64(x) (x)
+#define be64toh(x) __builtin_bswap64(x)
+#define le64toh(x) (x)
+
+#endif
+
+#endif
+
+#endif

tools/assets/n64texconv/src/libn64texconv/jfif.c

@@ -0,0 +1,76 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "endian.h"
+#include "bin2c.h"
+#include "jfif.h"
+
+struct JFIF *
+jfif_fromfile(const char *path, size_t max_size)
+{
+    assert(path != NULL);
+    FILE *f = fopen(path, "rb");
+    if (f == NULL)
+        return NULL;
+
+    fseek(f, 0, SEEK_END);
+    size_t data_size = ftell(f);
+    fseek(f, 0, SEEK_SET);
+
+    struct JFIF *jfif = malloc(((sizeof(struct JFIF) + 3) & ~3) + data_size);
+    if (jfif != NULL) {
+        jfif->data = (void *)(jfif + 1);
+        jfif->data_size = data_size;
+
+        if (fread(jfif->data, 1, data_size, f) != data_size) {
+            free(jfif);
+            jfif = NULL;
+        } else {
+            uint8_t *data8 = jfif->data;
+            uint16_t *data16 = jfif->data;
+            uint32_t *data32 = jfif->data;
+
+            if (be32toh(data32[0]) != 0xFFD8FFE0)
+                printf("[Warning] Missing JPEG marker\n");
+            if (data8[6] != 'J' || data8[7] != 'F' || data8[8] != 'I' || data8[9] != 'F')
+                printf("[Warning] Not JFIF\n");
+            if (data8[11] != 0x01 || data8[12] != 0x01)
+                printf("[Warning] Not JFIF version 1.01\n");
+            if (be16toh(data16[10]) != 0xFFDB)
+                printf("[Warning] Data before DQT\n");
+            if (jfif->data_size > max_size)
+                printf("[Warning] JFIF image too large\n");
+        }
+    }
+
+    fclose(f);
+    return jfif;
+}
+
+void
+jfif_free(struct JFIF *jfif)
+{
+    assert(jfif != NULL);
+    free(jfif);
+}
+
+int
+jfif_to_c(char **out, size_t *size_out, struct JFIF *jfif, size_t pad_to_size)
+{
+    assert(out != NULL);
+    assert(size_out != NULL);
+    return bin2c(out, size_out, jfif->data, jfif->data_size, pad_to_size, 8);
+}
+
+int
+jfif_to_c_file(const char *out_path, struct JFIF *jfif, size_t pad_to_size)
+{
+    assert(out_path != NULL);
+    assert(jfif != NULL);
+    return bin2c_file(out_path, jfif->data, jfif->data_size, pad_to_size, 8);
+}

tools/assets/n64texconv/src/libn64texconv/jfif.h

@@ -0,0 +1,28 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#ifndef JFIF_H
+#define JFIF_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+struct JFIF {
+    void *data;
+    size_t data_size;
+};
+
+#define JFIF_BUFFER_SIZE (320 * 240 * sizeof(uint16_t))
+
+struct JFIF *
+jfif_fromfile(const char *path, size_t max_size);
+
+void
+jfif_free(struct JFIF *jfif);
+
+int
+jfif_to_c(char **out, size_t *size_out, struct JFIF *jfif, size_t pad_to_size);
+
+int
+jfif_to_c_file(const char *out_path, struct JFIF *jfif, size_t pad_to_size);
+
+#endif

tools/assets/n64texconv/src/libn64texconv/n64texconv.h

@@ -0,0 +1,122 @@
+/* SPDX-FileCopyrightText: Copyright (C) 2025 ZeldaRET */
+/* SPDX-License-Identifier: MIT */
+#ifndef N64TEXCONV_H
+#define N64TEXCONV_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define FMT_NONE      -1
+#define FMT_MAX       5
+#define G_IM_FMT_RGBA 0
+#define G_IM_FMT_YUV  1
+#define G_IM_FMT_CI   2
+#define G_IM_FMT_IA   3
+#define G_IM_FMT_I    4
+
+#define SIZ_NONE     -1
+#define SIZ_MAX      4
+#define G_IM_SIZ_4b  0
+#define G_IM_SIZ_8b  1
+#define G_IM_SIZ_16b 2
+#define G_IM_SIZ_32b 3
+
+struct color {
+    union {
+        struct {
+            uint8_t r, g, b, a;
+        };
+        uint32_t w;
+    };
+};
+
+static inline __attribute__((always_inline)) size_t
+texel_size_bytes(size_t ntexels, int siz)
+{
+    return (siz == G_IM_SIZ_4b) ? (ntexels / 2) : (ntexels * ((1 << (unsigned)siz) >> 1));
+}
+
+struct n64_palette {
+    struct color *texels;
+    int fmt;
+    size_t count;
+};
+
+struct n64_palette *
+n64texconv_palette_new(size_t count, int fmt);
+
+void
+n64texconv_palette_free(struct n64_palette *pal);
+
+struct n64_palette *
+n64texconv_palette_copy(struct n64_palette *pal);
+
+struct n64_palette *
+n64texconv_palette_reformat(struct n64_palette *pal, int fmt);
+
+struct n64_palette *
+n64texconv_palette_from_png(const char *path, int fmt);
+
+struct n64_palette *
+n64texconv_palette_from_bin(void *data, size_t count, int fmt);
+
+int
+n64texconv_palette_to_png(const char *outpath, struct n64_palette *pal);
+
+void *
+n64texconv_palette_to_bin(struct n64_palette *pal, bool pad_to_8b);
+
+int
+n64texconv_palette_to_c(char **out, size_t *size_out, struct n64_palette *pal, bool pad_to_8b, unsigned int byte_width);
+
+int
+n64texconv_palette_to_c_file(const char *out_path, struct n64_palette *pal, bool pad_to_8b, unsigned int byte_width);
+
+struct n64_image {
+    size_t width;
+    size_t height;
+    int fmt;
+    int siz;
+    struct n64_palette *pal;
+    struct color *texels;
+    uint8_t *color_indices;
+};
+
+struct n64_image *
+n64texconv_image_new(size_t width, size_t height, int fmt, int siz, struct n64_palette *pal);
+
+void
+n64texconv_image_free(struct n64_image *img);
+
+struct n64_image *
+n64texconv_image_copy(struct n64_image *img);
+
+struct n64_image *
+n64texconv_image_from_png(const char *path, int fmt, int siz, int pal_fmt);
+
+struct n64_image *
+n64texconv_image_from_bin(void *data, size_t width, size_t height, int fmt, int siz, struct n64_palette *pal,
+                          bool preswapped);
+
+struct n64_image *
+n64texconv_image_reformat(struct n64_image *img, int fmt, int siz, struct n64_palette *pal);
+
+int
+n64texconv_image_to_png(const char *outpath, struct n64_image *img, bool intensity_alpha);
+
+void *
+n64texconv_image_to_bin(struct n64_image *img, bool pad_to_8b, bool preswap);
+
+int
+n64texconv_image_to_c(char **out, size_t *size_out, struct n64_image *img, bool pad_to_8b, bool preswap,
+                      unsigned int byte_width);
+
+int
+n64texconv_image_to_c_file(const char *out_path, struct n64_image *img, bool pad_to_8b, bool preswap,
+                           unsigned int byte_width);
+
+const char *
+n64texconv_png_extension(struct n64_image *img);
+
+#endif