From 092c1bdb783e960ca5c4fba37428b9eacfd5d72e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sat, 17 Feb 2018 11:46:23 -0500 Subject: [PATCH] Add comps-sync script, use it to trim package set This script looks for packages not in comps at all, which helps us lose things that were there only as dependencies. Also, it now supports showing packages in the workstation environment but not in the manifest. (I'm not yet taking action on those though) Some leaf packages do drop out. All of these I believe are right to stop shipping by default with the possible exception of `media-player-info`...I'm not sure if that was intentional or not. Anyways let's start this sync process. ``` -Installing 1344 packages: +Installing 1335 packages: - bcache-tools-1.0.8-10.fc27.x86_64 (fedora-rawhide) - isomd5sum-1:1.2.2-1.fc28.x86_64 (fedora-rawhide) - kexec-tools-2.0.16-3.fc28.x86_64 (fedora-rawhide) - keybinder3-0.3.2-3.fc27.x86_64 (fedora-rawhide) - langtable-0.0.38-2.fc28.noarch (fedora-rawhide) - langtable-data-0.0.38-2.fc28.noarch (fedora-rawhide) - langtable-python3-0.0.38-2.fc28.noarch (fedora-rawhide) - lpsolve-5.5.2.0-16.fc27.x86_64 (fedora-rawhide) - media-player-info-23-1.fc28.noarch (fedora-rawhide) ``` --- comps-sync.py | 98 +++++++++++++++++++++++++++++++ fedora-workstation-base-pkgs.json | 93 +---------------------------- 2 files changed, 99 insertions(+), 92 deletions(-) create mode 100755 comps-sync.py diff --git a/comps-sync.py b/comps-sync.py new file mode 100755 index 0000000..7c5e889 --- /dev/null +++ b/comps-sync.py @@ -0,0 +1,98 @@ +#!/usr/bin/python3 +# Usage: ./comps-sync.py /path/to/comps-f28.xml.in +# Currently just *removes* packages from the manifest +# which are not mentioned in comps. + +import os, sys, subprocess, argparse, shlex, json +import libcomps + +def fatal(msg): + print >>sys.stderr, msg + sys.exit(1) + +parser = argparse.ArgumentParser() +parser.add_argument("--save", help="Write changes", action='store_true') +parser.add_argument("src", help="Source path") + +args = parser.parse_args() + +base_pkgs_path = 'fedora-workstation-base-pkgs.json' +with open(base_pkgs_path) as f: + manifest = json.load(f) + +manifest_packages = set(manifest['packages']) + +comps_unknown = set() + +comps_packages = set() +workstation_product_packages = set() +# Parse comps, and build up a set of all packages so we +# can find packages not listed in comps *at all*, beyond +# just the workstation environment. +comps = libcomps.Comps() +comps.fromxml_f(args.src) +for group in comps.groups: + for pkg in group.packages: + comps_packages.add(pkg.name) +for pkg in manifest_packages: + if pkg not in comps_packages: + comps_unknown.add(pkg) + +# Parse the workstation-product environment, gathering +# default or mandatory packages. +ws_environ = comps.environments['workstation-product-environment'] +ws_pkgs = {} +for gid in ws_environ.group_ids: + group = comps.groups_match(id=gid.name)[0] + for pkg in group.packages: + pkgname = pkg.name + if pkg.type not in (libcomps.PACKAGE_TYPE_DEFAULT, + libcomps.PACKAGE_TYPE_MANDATORY): + continue + pkgdata = ws_pkgs.get(pkgname) + if pkgdata is None: + ws_pkgs[pkgname] = pkgdata = (pkg.type, set([gid.name])) + if (pkgdata[0] == libcomps.PACKAGE_TYPE_DEFAULT and + pkg.type == libcomps.PACKAGE_TYPE_MANDATORY): + ws_pkgs[pkgname] = pkgdata = (pkg.type, pkgdata[1]) + pkgdata[1].add(gid.name) + +# Look for packages in the manifest but not in comps at all +n_manifest_new = len(comps_unknown) +if n_manifest_new == 0: + print("All manifest packages are already listed in comps.") +else: + print("{} packages not in comps:".format(n_manifest_new)) + for pkg in sorted(comps_unknown): + print(' ' + pkg) + manifest_packages.remove(pkg) + +# Look for packages in workstation but not in the manifest +ws_added = {} +for (pkg,data) in ws_pkgs.items(): + if pkg not in manifest_packages: + ws_added[pkg] = data + +def format_pkgtype(n): + if n == libcomps.PACKAGE_TYPE_DEFAULT: + return 'default' + elif n == libcomps.PACKAGE_TYPE_MANDATORY: + return 'mandatory' + else: + assert False + +n_comps_new = len(ws_added) +if n_comps_new == 0: + print("All comps packages are already listed in manifest.") +else: + print("{} packages not in manifest:".format(n_comps_new)) + for pkg in sorted(ws_added): + (req, groups) = ws_added[pkg] + print(' {} ({}, groups: {})'.format(pkg, format_pkgtype(req), ', '.join(groups))) + +if (n_manifest_new > 0 or n_comps_new > 0) and args.save: + manifest['packages'] = sorted(manifest_packages) + with open(base_pkgs_path, 'w') as f: + json.dump(manifest, f, indent=4, sort_keys=True) + f.write('\n') + print("Wrote {}".format(base_pkgs_path)) diff --git a/fedora-workstation-base-pkgs.json b/fedora-workstation-base-pkgs.json index 2504d02..34daa49 100644 --- a/fedora-workstation-base-pkgs.json +++ b/fedora-workstation-base-pkgs.json @@ -1,5 +1,5 @@ { - "00packages-comment": "This content may be auto-generated in the future", + "00packages-comment": "DO NOT EDIT! This content is generated from comps-sync.py", "packages": [ "ModemManager", "NetworkManager", @@ -35,8 +35,6 @@ "bash", "bash-completion", "bc", - "bcache-tools", - "bind-license", "bind-utils", "bluez-cups", "bridge-utils", @@ -50,20 +48,15 @@ "cifs-utils", "control-center", "coreutils", - "crda", "cryptsetup", "cups", - "cups-client", - "cups-filesystem", "cups-filters", "curl", "dejavu-sans-fonts", "dejavu-sans-mono-fonts", "dejavu-serif-fonts", "device-mapper-multipath", - "dmidecode", "dmraid", - "dmraid-events", "dos2unix", "dracut-config-generic", "dracut-network", @@ -79,15 +72,10 @@ "file-roller", "firefox", "firewalld", - "firewalld-filesystem", - "flatpak", "foomatic", - "foomatic-db", - "foomatic-db-filesystem", "foomatic-db-ppds", "fpaste", "fprintd-pam", - "fros", "fros-gnome", "gdm", "gedit", @@ -122,11 +110,9 @@ "gnome-user-docs", "gnome-user-share", "gnome-weather", - "gnu-free-fonts-common", "gnu-free-mono-fonts", "gnu-free-sans-fonts", "gnu-free-serif-fonts", - "gnupg", "gnupg2", "google-noto-emoji-fonts", "google-noto-sans-lisu-fonts", @@ -151,11 +137,6 @@ "hostname", "hplip", "hyperv-daemons", - "hyperv-daemons-license", - "hypervfcopyd", - "hypervkvpd", - "hypervvssd", - "ibus", "ibus-gtk2", "ibus-gtk3", "ibus-hangul", @@ -165,17 +146,13 @@ "ibus-m17n", "ibus-qt", "ibus-rawcode", - "ibus-setup", "ibus-typing-booster", "initscripts", "iproute", - "ipset", "iptstate", "iputils", "ipw2100-firmware", "ipw2200-firmware", - "isomd5sum", - "iw", "iwl100-firmware", "iwl1000-firmware", "iwl105-firmware", @@ -195,39 +172,16 @@ "jomolhari-fonts", "jwhois", "kbd", - "kbd-legacy", - "kbd-misc", "kernel", "kernel-modules-extra", - "kexec-tools", - "keybinder3", "khmeros-base-fonts", - "khmeros-fonts-common", - "langtable", - "langtable-data", - "langtable-python3", "less", - "libXxf86dga", "libcanberra-gtk2", "libcanberra-gtk3", "libertas-usb8388-firmware", "libproxy-mozjs", - "libsss_idmap", - "libsss_nss_idmap", "libvirt-client", - "libvirt-daemon", - "libvirt-daemon-driver-interface", - "libvirt-daemon-driver-network", - "libvirt-daemon-driver-nodedev", - "libvirt-daemon-driver-nwfilter", - "libvirt-daemon-driver-qemu", - "libvirt-daemon-driver-secret", - "libvirt-daemon-driver-storage", "libvirt-daemon-kvm", - "libvirt-gconfig", - "libvirt-glib", - "libvirt-gobject", - "lldpad", "logrotate", "lohit-assamese-fonts", "lohit-bengali-fonts", @@ -238,37 +192,27 @@ "lohit-odia-fonts", "lohit-tamil-fonts", "lohit-telugu-fonts", - "lpsolve", "lrzsz", "lsof", - "lttng-ust", "lvm2", "lzop", "m17n-db", - "m17n-lib", "mactel-boot", "man-db", "man-pages", "mcelog", - "media-player-info", "mesa-dri-drivers", - "mesa-libxatracker", "microcode_ctl", "mlocate", "mousetweaks", "mpage", - "mtools", "mtr", "nautilus", "nautilus-sendto", - "naver-nanum-fonts-common", "naver-nanum-gothic-fonts", "nfs-utils", "nss-altfiles", "nss-mdns", - "nss-softokn-freebl", - "numad", - "opencc", "openssh-clients", "openssh-server", "orca", @@ -278,24 +222,17 @@ "paratype-pt-sans-fonts", "passwd", "passwdqc", - "passwdqc-lib", "pciutils", "pinfo", "plymouth", - "plymouth-plugin-label", - "plymouth-plugin-two-step", "plymouth-system-theme", - "plymouth-theme-charge", "policycoreutils", "polkit", "poppler-utils", "procps-ng", "psacct", - "qemu-common", "qemu-guest-agent", - "qemu-img", "qemu-kvm", - "qemu-system-x86", "qgnomeplatform", "radvd", "rdist", @@ -307,35 +244,20 @@ "samba-client", "sane-backends-drivers-scanners", "scl-utils", - "seabios-bin", - "seavgabios-bin", "selinux-policy-targeted", "setup", "setuptool", - "sgabios-bin", - "sgpio", "shadow-utils", "sheepdog", - "shim", "sil-abyssinica-fonts", "sil-mingzat-fonts", "sil-nuosu-fonts", "sil-padauk-fonts", - "skkdic", - "smc-fonts-common", "smc-meera-fonts", "sos", - "spice-server", "spice-vdagent", "sssd", - "sssd-ad", - "sssd-client", "sssd-common", - "sssd-common-pac", - "sssd-ipa", - "sssd-krb5", - "sssd-krb5-common", - "sssd-ldap", "stix-fonts", "sudo", "sushi", @@ -344,11 +266,9 @@ "systemd", "tabish-eeyek-fonts", "tar", - "tcl", "tcpdump", "teamd", "telnet", - "thai-scalable-fonts-common", "thai-scalable-waree-fonts", "time", "totem", @@ -356,28 +276,17 @@ "traceroute", "tree", "usb_modeswitch", - "usb_modeswitch-data", "usbutils", - "usermode", - "userspace-rcu", "util-linux", "vconfig", "vim-minimal", - "virglrenderer", "vlgothic-fonts", "wget", "which", "wireless-tools", "words", - "wpa_supplicant", "wvdial", - "xcb-util-image", - "xcb-util-keysyms", - "xcb-util-renderutil", - "xcb-util-wm", "xdg-user-dirs-gtk", - "xdg-utils", - "xen-licenses", "xfsprogs", "xorg-x11-drv-ati", "xorg-x11-drv-evdev",