Merge pull request #19 from Progmasoft/feature/yyjson-pugixml

Read and write JSON with yyjson and JUnit XML with pugixml

2 files changed, 185 insertions, 0 deletions

diff --git a/tools/scripts/generateAmalgamatedFiles.py b/tools/scripts/generateAmalgamatedFiles.py
index ffd069ef..7567a1c8 100755
--- a/tools/scripts/generateAmalgamatedFiles.py
+++ b/tools/scripts/generateAmalgamatedFiles.py
@@ -16,6 +16,8 @@ root_path = os.path.join(catchPath, 'src')
 starting_header = os.path.join(root_path, 'Progmasoft', 'Catch3', 'All.hpp')
 output_header = os.path.join(catchPath, 'extras', 'catch_amalgamated.hpp')
 output_cpp = os.path.join(catchPath, 'extras', 'catch_amalgamated.cpp')
+output_third_party = os.path.join(catchPath, 'extras', 'catch_amalgamated_third_party.hpp')
+third_party_path = os.path.join(catchPath, 'third_party')
 
 # REUSE-IgnoreStart
 
@@ -51,6 +53,21 @@ file_header = '''\
 //  ----------------------------------------------------------
 '''
 
+# The header of the file with yyjson and pugixml.
+third_party_header = '''\
+// Catch3 v{catch3_version}: yyjson and pugixml for catch_amalgamated.cpp
+// Generated: {generation_time}
+// ----------------------------------------------------------
+// This file is an amalgamation of src/Progmasoft/Runner/Yyjson.inl,
+// src/Progmasoft/PugiXml.inl and the libraries they include from
+// third_party/. You probably shouldn't edit it directly.
+//
+// yyjson and pugixml keep their MIT licenses, stated in their text below;
+// the rest is MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1. Only
+// catch_amalgamated.cpp includes this file.
+// ----------------------------------------------------------
+'''
+
 # REUSE-IgnoreEnd
 
 # Returns file header with proper version string and generation time
@@ -145,6 +162,21 @@ def generate_header():
 progmasoft_include_parser = re.compile(r'\s*#include (?:<(Progmasoft/[^>]+)>|"([A-Za-z/]+\.hpp)").*')
 concatenated_runner_headers = set()
 
+# The runner compiles yyjson and pugixml into its own code through these
+# files; in the amalgamation they are one header, included where the first
+# of them was.
+third_party_wrappers = ['Runner/Yyjson.inl', 'PugiXml.inl']
+wrapper_include_parser = re.compile(r'\s*#include "([A-Za-z/]+\.inl)".*')
+third_party_files = {
+    'yyjson.c': 'yyjson/yyjson.c',
+    'yyjson.h': 'yyjson/yyjson.h',
+    'pugixml.cpp': 'pugixml/pugixml.cpp',
+    'pugixml.hpp': 'pugixml/pugixml.hpp',
+    'pugiconfig.hpp': 'pugixml/pugiconfig.hpp',
+}
+third_party_include_parser = re.compile(r'\s*#\s*include\s+[<"]([A-Za-z.]+)[>"].*')
+third_party_included = False
+
 
 def concatenate_runner_file(out, filename: str, expand_headers: bool):
     """Copies a file of the runner, expanding the runner's headers it
@@ -157,6 +189,12 @@ def concatenate_runner_file(out, filename: str, expand_headers: bool):
             # The engine's headers are all in the amalgamated header.
             if engine_include(line):
                 continue
+            if wrapper_include_parser.match(line):
+                global third_party_included
+                if not third_party_included:
+                    out.write('#include "catch_amalgamated_third_party.hpp"\n')
+                    third_party_included = True
+                continue
             m = progmasoft_include_parser.match(line)
             if not m:
                 out.write(line)
@@ -171,6 +209,49 @@ def concatenate_runner_file(out, filename: str, expand_headers: bool):
             concatenate_runner_file(out, header, True)
 
 
+def concatenate_third_party_file(out, filename: str, done: set):
+    """Copies a file of yyjson or pugixml, with the files of the two
+    libraries it includes expanded once."""
+    with open(filename, mode='r', encoding='utf-8') as input:
+        for line in input:
+            m = third_party_include_parser.match(line)
+            name = m.group(1) if m else None
+            # pugixml includes its source through a macro in header-only mode.
+            if line.strip() == '#\tinclude PUGIXML_SOURCE':
+                name = 'pugixml.cpp'
+            if name not in third_party_files:
+                out.write(line)
+                continue
+            if name not in done:
+                done.add(name)
+                concatenate_third_party_file(
+                    out, os.path.join(third_party_path, third_party_files[name]), done)
+
+
+def generate_third_party():
+    with open(output_third_party, mode='w', encoding='utf-8', newline='\n') as header:
+        header.write(third_party_header.format(catch3_version=catch3_version(),
+                                               generation_time=datetime.datetime.now()))
+        header.write('#pragma once\n')
+        done = set()
+        for wrapper in third_party_wrappers:
+            path = os.path.join(root_path, 'Progmasoft', wrapper)
+            with open(path, mode='r', encoding='utf-8') as input:
+                for line in input:
+                    if line.startswith(('// Copyright', '// SPDX-License-Identifier')):
+                        continue
+                    m = third_party_include_parser.match(line)
+                    if m and m.group(1) in third_party_files:
+                        name = m.group(1)
+                        if name not in done:
+                            done.add(name)
+                            concatenate_third_party_file(
+                                header, os.path.join(third_party_path, third_party_files[name]), done)
+                        continue
+                    header.write(line)
+    print('Wrote {}'.format(os.path.relpath(output_third_party, catchPath)))
+
+
 def generate_cpp():
     from glob import glob
     cpp_files = sorted(os.path.join(catchPath, path) for path in upstream_files
@@ -200,6 +281,7 @@ def generate_cpp():
 if __name__ == "__main__":
     generate_header()
     generate_cpp()
+    generate_third_party()
 
 
 # Notes:
diff --git a/tools/scripts/updateThirdParty.py b/tools/scripts/updateThirdParty.py
new file mode 100755
index 00000000..a28ead3c
--- /dev/null
+++ b/tools/scripts/updateThirdParty.py
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026 Progmasoft.
+# SPDX-License-Identifier: MPL-2.0 WITH AdditionRef-Progmasoft-Exception-1.1
+
+"""Keeps third_party/ equal to the pinned release archives of its libraries.
+
+    updateThirdParty.py --check    fails when a vendored file differs
+    updateThirdParty.py --update   writes the files of the pinned archives
+
+The table below is the only record of the versions. Renovate proposes a new
+version by changing "version"; whoever takes it sets "sha256" to the digest
+of the new archive, after reading the release, and runs --update.
+"""
+
+import argparse
+import hashlib
+import io
+import sys
+import tarfile
+import urllib.request
+from pathlib import Path
+
+from scriptCommon import catchPath
+
+LIBRARIES = {
+    "yyjson": {
+        # renovate: datasource=github-tags depName=ibireme/yyjson
+        "version": "0.13.0",
+        "url": "https://github.com/ibireme/yyjson/archive/refs/tags/{version}.tar.gz",
+        "sha256": "34e0f62a2bc11ab20d601e8ca1cc2b2079503aa45119a19133d89d19b94a0fae",
+        "files": {"src/yyjson.c": "yyjson.c", "src/yyjson.h": "yyjson.h"},
+    },
+    "pugixml": {
+        # renovate: datasource=github-tags depName=zeux/pugixml extractVersion=^v(?<version>.+)$
+        "version": "1.16",
+        "url": "https://github.com/zeux/pugixml/archive/refs/tags/v{version}.tar.gz",
+        "sha256": "357bcab8877dc9943f355d3a72daba1b053238ba955f50fa81586afb65090219",
+        "files": {
+            "src/pugiconfig.hpp": "pugiconfig.hpp",
+            "src/pugixml.cpp": "pugixml.cpp",
+            "src/pugixml.hpp": "pugixml.hpp",
+        },
+    },
+}
+
+
+def archive_files(name: str, library: dict) -> dict:
+    """The files of the pinned archive, after its digest is checked."""
+    url = library["url"].format(version=library["version"])
+    with urllib.request.urlopen(url, timeout=60) as response:
+        data = response.read()
+    digest = hashlib.sha256(data).hexdigest()
+    if digest != library["sha256"]:
+        sys.exit(f"{name}: {url} has sha256 {digest}, "
+                 f"the table pins {library['sha256']}")
+    files = {}
+    with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as archive:
+        for member in archive.getmembers():
+            # The archive has one top directory, named after the release.
+            inner = member.name.split("/", 1)[-1]
+            if member.isfile() and inner in library["files"]:
+                extracted = archive.extractfile(member)
+                assert extracted is not None
+                files[library["files"][inner]] = extracted.read()
+    missing = set(library["files"].values()) - set(files)
+    if missing:
+        sys.exit(f"{name}: the archive has no {', '.join(sorted(missing))}")
+    return files
+
+
+def normalized(data: bytes) -> bytes:
+    # Git may check the files out with CRLF on Windows.
+    return data.replace(b"\r\n", b"\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--check", action="store_true",
+                      help="fail when a vendored file differs from its archive")
+    mode.add_argument("--update", action="store_true",
+                      help="write the files of the pinned archives")
+    arguments = parser.parse_args()
+
+    differences = 0
+    for name, library in LIBRARIES.items():
+        directory = Path(catchPath) / "third_party" / name
+        for file_name, data in archive_files(name, library).items():
+            target = directory / file_name
+            if arguments.update:
+                target.write_bytes(data)
+                print(f"{name} {library['version']}: wrote {target.name}")
+            elif not target.is_file() or normalized(target.read_bytes()) != normalized(data):
+                print(f"{name} {library['version']}: {target} differs from the archive")
+                differences += 1
+    if differences:
+        print("Run tools/scripts/updateThirdParty.py --update.")
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())