From 5189dc097b5bfe246c683102b6b4caabaefe00f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Hertzog?= Date: Thu, 31 May 2018 13:46:14 +0200 Subject: Generalize the code to fetch Packages/Sources updates It's now easy to extend by adding a supplementary lib/foobar-releases.mk file. I verified that the set of downloaded files are identical with the exception of a bug that got fixed in the process: the update-backports-% rule used to download files for all sections (main, contrib, non-free) but saved them all under the name corresponding to the main section. --- Makefile | 208 ++++++++++++++++----------------------------------------------- 1 file changed, 51 insertions(+), 157 deletions(-) (limited to 'Makefile') diff --git a/Makefile b/Makefile index 0200dd8ca6..b437701676 100644 --- a/Makefile +++ b/Makefile @@ -1,21 +1,39 @@ PYTHON = python PYTHON_MODULES = $(wildcard lib/python/*.py) -BUG_LISTS = $(wildcard data/*/list) -# Adjust these if necessary. The architecture selection is rather -# arbitrary at the moment. More architectures can be added later. +# The following variables need to be kept up-to-date and can be adjusted +OLDOLDSTABLE = wheezy +OLDSTABLE = jessie +STABLE = stretch +TESTING = buster + +MIRROR = http://debian.csail.mit.edu/debian +SECURITY_MIRROR = http://security.debian.org/debian-security -MIRROR = http://debian.csail.mit.edu/debian/ wheezy_ARCHS = amd64 armel armhf i386 jessie_ARCHS = amd64 arm64 armel armhf i386 mips mipsel powerpc ppc64el s390x stretch_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x buster_ARCHS = amd64 arm64 armel armhf i386 mips mips64el mipsel ppc64el s390x sid_ARCHS = amd64 arm64 armel armhf hurd-i386 i386 kfreebsd-i386 kfreebsd-amd64 mips mips64el mipsel powerpc ppc64el s390x -OLDOLDSTABLE = wheezy -OLDSTABLE = jessie -STABLE = stretch -TESTING = buster +# The rest of the file should not need to be edited + +# Include the definitions of the releases to be fetched +include lib/*-releases.mk + +# There's a "RELEASES" variable defining the name of all releases to +# process. Then for each release, there are 5 associated variables: +# _MIRROR: the base URL of the mirror hosting the repository +# _DIST: the sub-directory in $MIRROR/dists so that +# $MIRROR/dists/$DIST/Release is a valid URL +# _ARCHS: the list of architectures supported in this release +# _RELEASE: the release name for the security tracker +# _SUBRELEASE: the sub-release identifier for the security tracker + +Q=@ +ifneq ($(V),) +Q= +endif all: $(PYTHON) bin/update-db data/security.db @@ -51,157 +69,33 @@ stamps/DLA-syntax: data/DLA/list bin/check-syntax $(PYTHON_MODULES) serve: @bash bin/test-web-server -.PHONY: update-packages update-testing update-stable update-oldstable update-oldoldstable -update-packages: update-testing - set -e ; for rel in sid ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/source/Sources \ - data/packages/$${rel}__$${archive}_Sources ; \ - done ; \ - for arch in $(sid_ARCHS) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \ - data/packages/$${rel}__$${archive}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -update-testing: - set -e ; for rel in $(TESTING) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/source/Sources \ - data/packages/$${rel}__$${archive}_Sources ; \ - done ; \ - for arch in $($(TESTING)_ARCHS) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \ - data/packages/$${rel}__$${archive}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -update-stable: - set -e ; for rel in $(STABLE) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/source/Sources \ - data/packages/$${rel}__$${archive}_Sources ; \ - done ; \ - for arch in $($(STABLE)_ARCHS) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \ - data/packages/$${rel}__$${archive}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -update-oldstable: - set -e ; for rel in $(OLDSTABLE) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/source/Sources \ - data/packages/$${rel}__$${archive}_Sources ; \ - done ; \ - for arch in $($(OLDSTABLE)_ARCHS) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \ - data/packages/$${rel}__$${archive}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -update-oldoldstable: - set -e ; for rel in $(OLDOLDSTABLE) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/source/Sources \ - data/packages/$${rel}__$${archive}_Sources ; \ - done ; \ - for arch in $($(OLDOLDSTABLE)_ARCHS) ; do \ - for archive in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$$rel/$$archive/binary-$$arch/Packages \ - data/packages/$${rel}__$${archive}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -.PHONY: update-security update-old-security update-oldold-security update-testing-security -ST_MIRROR = http://security.debian.org/dists/$(TESTING)/updates -ST_FILE = data/packages/$(TESTING)_security_ -update-testing-security: +.PHONY: update-packages +update-packages: $(foreach release,$(RELEASES),update-$(release)) + +# This rule is a bit complicated as we need to escape $ for the shell twice, +# once for the eval and once for the usual make processing +define add_update_rule = +.PHONY: update-$(1) +update-$(1): + $$(Q)set -e; \ + prefix="$$($(1)_RELEASE)_$$($(1)_SUBRELEASE)"; \ + dist="$$($(1)_DIST)"; \ + mirror="$$($(1)_MIRROR)"; \ for section in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(ST_MIRROR)/$$section/source/Sources $(ST_FILE)$${section}_Sources ; \ - set -e ; for arch in $($(TESTING)_ARCHS) ; do \ - $(PYTHON) bin/apt-update-file \ - $(ST_MIRROR)/$$section/binary-$${arch}/Packages $(ST_FILE)$${section}_$${arch}_Packages ; \ - done ; \ - done - -SEC_MIRROR = http://security.debian.org/dists -update-security: update-old-security update-oldold-security - for archive in $(STABLE); do \ - for section in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \ - data/packages/$${archive}_security_$${section}_Sources ; \ - for arch in $($(STABLE)_ARCHS) ; do \ - $(PYTHON) bin/apt-update-file \ - $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \ - data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -update-old-security: - for archive in $(OLDSTABLE); do \ - for section in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \ - data/packages/$${archive}_security_$${section}_Sources ; \ - for arch in $($(OLDSTABLE)_ARCHS) ; do \ - $(PYTHON) bin/apt-update-file \ - $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \ - data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \ - done ; \ - done ; \ - done - -update-oldold-security: - for archive in $(OLDOLDSTABLE); do \ - for section in main contrib non-free ; do \ - $(PYTHON) bin/apt-update-file \ - $(SEC_MIRROR)/$$archive/updates/$$section/source/Sources \ - data/packages/$${archive}_security_$${section}_Sources ; \ - for arch in $($(OLDOLDSTABLE)_ARCHS) ; do \ - $(PYTHON) bin/apt-update-file \ - $(SEC_MIRROR)/$$archive/updates/$$section/binary-$$arch/Packages \ - data/packages/$${archive}_security_$${section}_$${arch}_Packages ; \ - done ; \ - done ; \ + echo "Fetching $$$$mirror/dists/$$$$dist/$$$$section/source/Sources"; \ + $$(PYTHON) bin/apt-update-file \ + $$$$mirror/dists/$$$$dist/$$$$section/source/Sources \ + data/packages/$$$${prefix}_$$$${section}_Sources ; \ + for arch in $$($(1)_ARCHS) ; do \ + echo "Fetching $$$$mirror/dists/$$$$dist/$$$$section/binary-$$$$arch/Packages"; \ + $$(PYTHON) bin/apt-update-file \ + $$$$mirror/dists/$$$$dist/$$$$section/binary-$$$$arch/Packages \ + data/packages/$$$${prefix}_$$$${section}_$$$${arch}_Packages ; \ + done; \ done -update-backports: update-backports-$(STABLE) update-backports-$(OLDSTABLE) update-backports-$(OLDOLDSTABLE) - -update-backports-%: - set -e && archive=$(shell echo $@ | cut -d- -f3) ; \ - for section in main contrib non-free ; do \ - for arch in $($(shell echo $@ | cut -d- -f3)_ARCHS) ; do \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$${archive}-backports/$$section/binary-$$arch/Packages \ - data/packages/$${archive}-backports__main_$${arch}_Packages ; \ - done ; \ - $(PYTHON) bin/apt-update-file \ - $(MIRROR)/dists/$${archive}-backports/$$section/source/Sources \ - data/packages/$${archive}-backports__main_Sources ; \ - done ; \ +endef +$(foreach release,$(RELEASES),$(eval $(call add_update_rule,$(release)))) update-lists: git fetch -q origin && git checkout -f origin/master -- data @@ -228,4 +122,4 @@ update-compare-nvd: done bin/compare-nvd-cve 2> compare-nvd-cve.log -update-all: update-nvd update-lists update-packages update-oldoldstable update-oldstable update-stable update-security update-testing-security update-packages update-backports all +update-all: update-nvd update-lists update-packages all -- cgit v1.2.3