Merge branch 'mlimonci/upstream-packaging' into 'main'

Add support to generate deb/rpm packages

See merge request kernel-firmware/linux-firmware!20

.gitlab-ci.yml

@@ -1,3 +1,12 @@
+.debcommon: &debcommon
+  - apt update
+  - apt install build-essential debhelper git python3-jinja2 -y
+  - make deb
+
+.rpmcommon: &rpmcommon
+  - dnf install make git python3-jinja2 rpmdevtools -y
+  - make rpm
+
 check-commits:
   stage: test
   image: registry.gitlab.com/kernel-firmware/linux-firmware
@@ -16,6 +25,52 @@ pre-commit:
   script:
     - pre-commit run --all-files
 
+rpm:
+  stage: test
+  image: fedora
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  script:
+    - *rpmcommon
+
+deb:
+  stage: test
+  image: debian
+  rules:
+    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  script:
+    - *debcommon
+
+deb-release:
+  stage: deploy
+  image: debian
+  rules:
+    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      when: never
+    - if: $RELEASE_TOKEN
+  artifacts:
+    paths:
+      - dist/*
+  script:
+    - *debcommon
+
+rpm-release:
+  stage: deploy
+  image: fedora
+  rules:
+    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
+      when: never
+    - if: $RELEASE_TOKEN
+  artifacts:
+    paths:
+      - dist/*
+  script:
+    - *rpmcommon
+
 release:
   stage: deploy
   rules:

Makefile

@@ -20,6 +20,12 @@ dist:
 	echo "Created dist/$${TARGET}"
 	@rm -rf release
 
+deb:
+	./build_packages.py --deb
+
+rpm:
+	./build_packages.py --rpm
+
 install:
 	install -d $(DESTDIR)$(FIRMWAREDIR)
 	./copy-firmware.sh $(DESTDIR)$(FIRMWAREDIR)

build_packages.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+
+import argparse
+import datetime
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from jinja2 import Environment, FileSystemLoader
+
+
+def version_str() -> str:
+    try:
+        return subprocess.check_output(["git", "describe"]).strip().decode("utf-8")
+    except subprocess.CalledProcessError:
+        return "0"
+
+
+def prep_tree(package) -> tuple:
+    tmpdir = tempfile.mkdtemp()
+    builddir = os.path.join(tmpdir, package)
+    fwdir = os.path.join(builddir, "updates")
+    targetdir = "dist"
+
+    os.makedirs(targetdir, exist_ok=True)
+    os.makedirs(builddir, exist_ok=False)
+    os.makedirs(fwdir, exist_ok=False)
+
+    subprocess.check_output(["./copy-firmware.sh", fwdir])
+    shutil.copy("WHENCE", os.path.join(builddir, "WHENCE"))
+
+    return (tmpdir, builddir, fwdir, targetdir)
+
+
+def build_deb_package(package, builddir) -> None:
+    env = Environment(loader=FileSystemLoader(os.path.join("contrib", "templates")))
+
+    d = {
+        "package": package,
+        "date": datetime.datetime.now()
+        .astimezone()
+        .strftime("%a, %d %b %Y %H:%M:%S %z"),
+        "version": version_str(),
+    }
+
+    templates = {
+        "debian.control": "control",
+        "debian.changelog": "changelog",
+        "debian.copyright": "copyright",
+    }
+
+    os.makedirs(os.path.join(builddir, "debian"))
+    for f in templates:
+        template = env.get_template(f)
+        with open(os.path.join(builddir, "debian", templates[f]), "w") as w:
+            w.write(template.render(d))
+
+    with open(os.path.join(builddir, "debian", "install"), "w") as w:
+        w.write("updates lib/firmware\n")
+
+    with open(os.path.join(builddir, "debian", "docs"), "w") as w:
+        w.write("WHENCE\n")
+
+    with open(os.path.join(builddir, "debian", "rules"), "w") as w:
+        w.write("#!/usr/bin/make -f\n")
+        w.write("%:\n")
+        w.write("\tdh $@\n")
+    os.chmod(os.path.join(builddir, "debian", "rules"), 0o755)
+
+    os.mkdir(os.path.join(builddir, "debian", "source"))
+    with open(os.path.join(builddir, "debian", "source", "format"), "w") as w:
+        w.write("3.0 (native)\n")
+
+    # build the package
+    os.environ["DEB_BUILD_OPTIONS"] = "nostrip"
+    subprocess.check_output(["dpkg-buildpackage", "-us", "-uc", "-b"], cwd=builddir)
+
+    # result is in tmpdir (not builddir!)
+    return os.path.join(
+        "..",
+        "{package}_{version}_all.deb".format(package=package, version=version_str()),
+    )
+
+
+def build_rpm_package(package, builddir) -> None:
+
+    v = version_str().replace("-", "_")
+    env = Environment(loader=FileSystemLoader(os.path.join("contrib", "templates")))
+
+    d = {
+        "package": package,
+        "version": v,
+        "cwd": builddir,
+    }
+
+    template = env.get_template("rpm.spec")
+    with open(os.path.join(builddir, "package.spec"), "wt") as w:
+        w.write(template.render(d))
+    cmd = ["rpmbuild", "-bb", "--build-in-place", "package.spec"]
+    subprocess.check_call(cmd, cwd=builddir, stderr=subprocess.STDOUT)
+
+    # result is in ~/rpmbuild/RPMS/noarch/
+    for root, dirs, files in os.walk(
+        os.path.join(os.getenv("HOME"), "rpmbuild", "RPMS", "noarch")
+    ):
+        for f in files:
+            if f.startswith(package) and f.endswith(".rpm") and v in f:
+                return os.path.join(root, f)
+    raise FileNotFoundError("RPM package not found")
+
+
+def parse_args():
+    parser = argparse.ArgumentParser("Build upstream packages using Jinja2 templates")
+    parser.add_argument("--deb", help="Build DEB package", action="store_true")
+    parser.add_argument("--rpm", help="Build RPM package", action="store_true")
+    parser.add_argument("--debug", help="Enable debug output", action="store_true")
+    args = parser.parse_args()
+
+    if not (args.rpm or args.deb) or (args.rpm and args.deb):
+        parser.print_help()
+        sys.exit(1)
+
+    return args
+
+
+if __name__ == "__main__":
+    args = parse_args()
+
+    package = "linux-firmware-upstream"
+    tmpdir, builddir, fwdir, targetdir = prep_tree(package)
+
+    try:
+        if args.deb:
+            result = build_deb_package(package, builddir)
+        elif args.rpm:
+            result = build_rpm_package(package, builddir)
+        shutil.copy(os.path.join(builddir, result), targetdir)
+        print(
+            "Built package: {}".format(
+                os.path.join(targetdir, os.path.basename(result))
+            )
+        )
+    finally:
+        if not args.debug:
+            shutil.rmtree(tmpdir)

check_whence.py

@@ -76,6 +76,7 @@ def main():
             ".codespell.cfg",
             ".gitlab-ci.yml",
             ".pre-commit-config.yaml",
+            "build_packages.py",
             "check_whence.py",
             "configure",
             "Makefile",
@@ -83,6 +84,10 @@ def main():
             "copy-firmware.sh",
             "WHENCE",
             "Dockerfile",
+            "contrib/templates/debian.changelog",
+            "contrib/templates/debian.control",
+            "contrib/templates/debian.copyright",
+            "contrib/templates/rpm.spec",
         ]
     )
     known_prefixes = set(name for name in whence_list if name.endswith("/"))

contrib/templates/debian.changelog

@@ -0,0 +1,5 @@
+{{package}} ({{version}}) unstable; urgency=medium
+
+  * Automated release
+
+ -- Linux Firmware <linux-firmware@kernel.org>  {{ date }}

contrib/templates/debian.control

@@ -0,0 +1,18 @@
+Source: {{package}}
+Section: unknown
+Priority: optional
+Maintainer: Linux Firmware <linux-firmware@kernel.org>
+Build-Depends: debhelper-compat (= 13)
+Standards-Version: 4.6.0
+Homepage: https://gitlab.com/kernel-firmware/linux-firmware
+Vcs-Browser: https://gitlab.com/kernel-firmware/linux-firmware
+Vcs-Git: https://gitlab.com/kernel-firmware/linux-firmware
+Rules-Requires-Root: no
+
+Package: {{package}}
+Architecture: all
+Depends: ${misc:Depends}
+Description: Upstream snapshot of linux-firmware package
+ Monolithic snapshot of upstream linux-firmware package, intended to
+ to validate upstream firmware without conflicts to the distribution
+ package.

contrib/templates/debian.copyright

@@ -0,0 +1,28 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: {{ package }}
+Upstream-Contact: linux-firmware@kernel.org
+Source: https://gitlab.com/kernel-firmware/linux-firmware
+
+Files: *
+Copyright: Various
+License: Various
+ License for all firmware is described in /usr/share/doc/{{ package }}/WHENCE.gz
+
+Files: debian/*
+Copyright: 2023 Linux Firmware <firmware@kernel.org>
+License: GPL-2+
+ This package is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ .
+ This package is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>
+ .
+ On Debian systems, the complete text of the GNU General
+ Public License version 2 can be found in "/usr/share/common-licenses/GPL-2".

contrib/templates/rpm.spec

@@ -0,0 +1,31 @@
+Name:           {{ package }}
+Version:        {{ version }}
+Release:        1%{?dist}
+Summary:        Linux Firmware (upstream)
+License:        Redistributable, no modification permitted
+URL:            http://gitlab.com/kernel-firmware/linux-firmware
+Source:         /dev/null
+BuildArch:      noarch
+
+%description
+Monolithic snapshot of upstream linux-firmware package, intended to
+to validate upstream firmware without conflicts to the distribution
+package.
+
+%prep
+%setup -q
+
+%build
+
+%install
+%define __strip /bin/true
+rm -rf %{buildroot}
+mkdir -p %{buildroot}/lib/firmware
+cp -aR {{ cwd }}/updates %{buildroot}/lib/firmware
+
+%files
+%defattr(-,root,root,-)
+/lib/firmware/updates/*
+
+%post
+dracut -fp --regenerate-all