Add Unattended-Upgrade::Remove-New-Unused-Dependencies

The new Unattended-Upgrade::Remove-New-Unused-Dependencies will
remove anything that is no longer required after an
unattended-upgrade run. It defaults to "true" to ensure that
e.g. old kernels are removed and /boot will not fill up.

Closes: LP: #1357093

Author: mvo5

README.md

@@ -200,6 +200,10 @@ Supported Options Reference
 
  Remove all unused dependencies after the upgrade finished.
 
+* `Unattended-Upgrade::Remove-New-Unused-Dependencies` - boolean (default:True)
+
+ Remove any new unused dependencies after the upgrade finished.
+
 * `Unattended-Upgrade::Automatic-Reboot` - boolean (default:False)
 
  Automatically reboot *WITHOUT CONFIRMATION* if the file

test/root.unused-deps/etc/apt/apt.conf

@@ -1,4 +1,5 @@
+
 Unattended-Upgrade::Allowed-Origins {
-	"Ubuntu:lucid-security";
+    "Ubuntu:lucid-security";
 };
-Unattended-Upgrade::Remove-Unused-Dependencies "true";
+Unattended-Upgrade::Remove-New-Unused-Dependencies "true";

test/root.unused-deps/usr/bin/dpkg

@@ -0,0 +1,16 @@
+#!/usr/bin/python
+
+import os
+import sys
+import subprocess
+
+if __name__ == "__main__":
+   if "--unpack" in sys.argv:
+      dpkg_status = os.path.join(
+          os.path.dirname(sys.argv[0]), "..", "..",
+          "var", "lib", "dpkg", "status")
+      # pretend a new version without the previous dependency got installed
+      subprocess.check_call(
+          ["sed", "-i", "/Depends:\ test-package-dependency/d", dpkg_status])
+      subprocess.check_call(
+          ["sed", "-i", "s/1.0.test.pkg/2.0.test.pkg/", dpkg_status])

test/root.unused-deps/var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_lucid-security_main_binary-amd64_Packages

@@ -1,6 +1,6 @@
 Package: test-package
 Architecture: all
-Version: 2.0
+Version: 2.0.test.pkg
 Description-en: test package
 Filename: test-package_2.0_all.deb
 SHA256: 01704da7de63ca12109abff5c2c480ad9c757e669c4428424de23ec97bf24248

test/root.unused-deps/var/lib/dpkg/status

@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import os
+import subprocess
 import unittest
 
 import apt
@@ -13,7 +14,7 @@
 class MockOptions(object):
     debug = False
     verbose = False
-    dry_run = True
+    dry_run = False
     apt_debug = False
     minimal_upgrade_steps = False
 
@@ -24,23 +25,96 @@ def setUp(self):
         self.rootdir = os.path.abspath("./root.unused-deps")
         dpkg_status = os.path.abspath(
             os.path.join(self.rootdir, "var", "lib", "dpkg", "status"))
+        # fake dpkg status
+        with open(dpkg_status, "w") as fp:
+            fp.write("""Package: test-package
+Status: install ok installed
+Architecture: all
+Version: 1.0.test.pkg
+Depends: test-package-dependency
+
+Package: test-package-dependency
+Status: install ok installed
+Architecture: all
+Version: 1.0
+
+Package: z-package
+Status: install ok installed
+Architecture: all
+Version: 1.0
+
+Package: old-unused-dependency
+Status: install ok installed
+Architecture: all
+Version: 1.0
+""")
         apt.apt_pkg.config.set("Dir::State::status", dpkg_status)
         apt.apt_pkg.config.clear("DPkg::Pre-Invoke")
         apt.apt_pkg.config.clear("DPkg::Post-Invoke")
+        apt.apt_pkg.config.set("Debug::NoLocking", "true")
+        # we don't really run dpkg
+        apt.apt_pkg.config.set(
+            "Dir::Bin::Dpkg", os.path.join(self.rootdir, "bin", "dpkg"))
         # pretend test-package-dependency is auto-installed
         extended_states = os.path.join(
             self.rootdir, "var", "lib", "apt", "extended_states")
         with open(extended_states, "w") as f:
-            f.write("Package: test-package-dependency\nAuto-Installed: 1\n")
+            f.write("""
+Package: old-unused-dependency
+Architecture: all
+Auto-Installed: 1
+
+Package: test-package-dependency
+Architecture: all
+Auto-Installed: 1
+""")
+        # clean log
+        self.log = os.path.join(
+            self.rootdir, "var", "log", "unattended-upgrades",
+            "unattended-upgrades.log")
+        if not os.path.exists(os.path.dirname(self.log)):
+            os.makedirs(os.path.dirname(self.log))
+        with open(self.log, "w"):
+            pass
+        # clean cache
+        subprocess.check_call(
+            ["rm", "-f",
+             os.path.join(self.rootdir, "var", "cache", "apt", "*.bin")])
 
     def test_remove_unused_dependencies(self):
+        apt_conf = os.path.join(self.rootdir, "etc", "apt", "apt.conf")
+        with open(apt_conf, "w") as fp:
+            fp.write("""
+Unattended-Upgrade::Allowed-Origins {
+    "Ubuntu:lucid-security";
+};
+Unattended-Upgrade::Remove-Unused-Dependencies "true";
+""")
         options = MockOptions()
         unattended_upgrade.main(
             options, rootdir="./root.unused-deps")
-        log = os.path.join(
-            self.rootdir, "var", "log", "unattended-upgrades",
-            "unattended-upgrades.log")
-        with open(log) as f:
+        with open(self.log) as f:
+            # both the new and the old unused dependency are removed
+            needle = "Packages that are auto removed: "\
+                     "'test-package-dependency old-unused-dependency'"
+            haystack = f.read()
+            self.assertTrue(needle in haystack,
+                            "Can not find '%s' in '%s'" % (needle, haystack))
+
+    def test_remove_unused_dependencies_new_unused_only(self):
+        apt_conf = os.path.join(self.rootdir, "etc", "apt", "apt.conf")
+        with open(apt_conf, "w") as fp:
+            fp.write("""
+Unattended-Upgrade::Allowed-Origins {
+    "Ubuntu:lucid-security";
+};
+Unattended-Upgrade::Remove-New-Unused-Dependencies "true";
+""")
+        options = MockOptions()
+        unattended_upgrade.main(
+            options, rootdir="./root.unused-deps")
+        with open(self.log) as f:
+            # ensure its only exactly one package that is removed
             needle = "Packages that are auto removed: "\
                      "'test-package-dependency'"
             haystack = f.read()

test/root.unused-deps/var/run/.silly-git

@@ -1088,8 +1088,7 @@ def get_auto_removable(cache):
                 if pkg.is_auto_removable])
 
 
-def do_auto_remove(cache, options, logfile_dpkg, verbose=False):
-    auto_removable = get_auto_removable(cache)
+def do_auto_remove(cache, auto_removable, logfile_dpkg, verbose=False):
     if not auto_removable:
         return True
 
@@ -1300,9 +1299,10 @@ def main(options, rootdir=""):
         logging.debug("dpkg is configured not to cause conffile prompts")
 
     # auto-removal
+    previous_autoremovals = get_auto_removable(cache)
     if apt_pkg.config.find_b(
             "Unattended-Upgrade::Remove-Unused-Dependencies", False):
-        pending_autoremovals = get_auto_removable(cache)
+        pending_autoremovals = previous_autoremovals
     else:
         pending_autoremovals = []
 
@@ -1342,7 +1342,7 @@ def main(options, rootdir=""):
     logfile_dpkg = os.path.join(_get_logdir(), 'unattended-upgrades-dpkg.log')
 
     # only perform install step if we actually have packages to install
-    pkg_install_success = None
+    pkg_install_success = True
     shutdown_lock = -1
     if len(pkgs_to_upgrade) > 0:
         # lock for the shutdown check - its fine if the system
@@ -1356,21 +1356,30 @@ def main(options, rootdir=""):
                                          options,
                                          logfile_dpkg)
 
-    # remove unneeded packages if there are any
+    # now check if any auto-removing needs to be done
+    cache = UnattendedUpgradesCache(
+        rootdir=rootdir, allowed_origins=allowed_origins)
+    if cache._depcache.broken_count > 0:
+        print(_("Cache has broken packages, exiting"))
+        logging.error(_("Cache has broken packages, exiting"))
+        sys.exit(1)
+
+    # the user wants *all* auto-removals to be removed
     if apt_pkg.config.find_b(
             "Unattended-Upgrade::Remove-Unused-Dependencies", False):
-        cache = UnattendedUpgradesCache(rootdir=rootdir,
-                                        allowed_origins=allowed_origins)
-        if cache._depcache.broken_count > 0:
-            print(_("Cache has broken packages, exiting"))
-            logging.error(_("Cache has broken packages, exiting"))
-            sys.exit(1)
-
-        remove_success = do_auto_remove(
-            cache, options, logfile_dpkg,
+        auto_removals = get_auto_removable(cache)
+        pkg_install_success &= do_auto_remove(
+            cache, auto_removals, logfile_dpkg,
+            options.verbose or options.debug)
+    # the user wants *only new* auto-removals to be removed
+    elif apt_pkg.config.find_b(
+            "Unattended-Upgrade::Remove-New-Unused-Dependencies", True):
+        # calculate the new auto-removals
+        new_pending_autoremovals = get_auto_removable(cache)
+        auto_removals = new_pending_autoremovals - previous_autoremovals
+        pkg_install_success &= do_auto_remove(
+            cache, auto_removals, logfile_dpkg,
             options.verbose or options.debug)
-        if pkg_install_success is not False:
-            pkg_install_success = remove_success
 
     logging.debug("InstCount=%i DelCount=%i BrokenCount=%i"
                   % (cache._depcache.inst_count,