From a1d0fa50ffd0576967ee7872dc38b0e49ea544e8 Mon Sep 17 00:00:00 2001 From: Michele Locati Date: Thu, 15 Mar 2018 16:24:16 +0100 Subject: Allow filtering tags --- README.md | 1 + bin/incremental-git-filterbranch.sh | 146 ++++++++++++++------- .../filter-directory-filtered-tags-bl.success | 16 +++ .../filter-directory-filtered-tags-blrx.success | 16 +++ .../filter-directory-filtered-tags-wl.success | 16 +++ .../filter-directory-filtered-tags-wlrx.success | 16 +++ 6 files changed, 167 insertions(+), 44 deletions(-) create mode 100755 test/tests/filter-directory-filtered-tags-bl.success create mode 100755 test/tests/filter-directory-filtered-tags-blrx.success create mode 100755 test/tests/filter-directory-filtered-tags-wl.success create mode 100755 test/tests/filter-directory-filtered-tags-wlrx.success diff --git a/README.md b/README.md index de5cd10..04c6b77 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Get the script and read the syntax using the `--help` option. ```sh ./incremental-git-filterbranch.sh \ --branch-whitelist 'develop master rx:release\/.*' \ + --tag-blacklist 'rx:5\..*' \ --tags-plan all --tags-max-history-lookup 10 \ https://github.com/concrete5/concrete5.git \ '--prune-empty --subdirectory-filter concrete' \ diff --git a/bin/incremental-git-filterbranch.sh b/bin/incremental-git-filterbranch.sh index 832a4f2..fc6c687 100755 --- a/bin/incremental-git-filterbranch.sh +++ b/bin/incremental-git-filterbranch.sh @@ -31,6 +31,7 @@ usage () { printf '%s' "Usage: ${0} [-h | --help] [--workdir ] [--branch-whitelist ] [--branch-blacklist ] + [--tag-whitelist ] [--tag-blacklist ] [--tags-plan (visited|all|none)] [--tags-max-history-lookup ] [--no-hardlinks] [--no-atomic] [--no-lock] [--] @@ -43,7 +44,7 @@ Where: set the path to the directory where the temporary local repositories are created. By default, we'll use a directory named temp in the current directory. --branch-whitelist - a whitespace-separated list of branches be included in the process. + a whitespace-separated list of branches to be included in the process. Multiple options can be specified. By default, all branches will be processed. --branch-blacklist @@ -51,6 +52,13 @@ Where: Multiple options can be specified. By default, all branches will be processed. Blacklisted branches take the precedence over whitelisted ones. +--tag-whitelist + a whitespace-separated list of tags to be included in the process. + Multiple options can be specified. +--tag-blacklist + a whitespace-separated list of tags to be excluded from the process. + Multiple options can be specified. + Blacklisted tags take the precedence over whitelisted ones. --tags-plan how tags should be processed. This can be one of these values: visited: process only the tags visited (default) @@ -72,7 +80,7 @@ filter destinationrepository The URL or path to the destination repository. -You can prefix branch names in both whitelist and blacklist with 'rx:': in this case a regular expression check will be performed. +You can prefix branch/tag names in both whitelists and blacklists with 'rx:': in this case a regular expression check will be performed. For instance: --branch-whitelist 'master rx:release\\/\\d+(\\.\\d+)*' will match 'master' and 'release/1.1' " exit 0 @@ -83,11 +91,13 @@ readParameters () { WORK_DIRECTORY="$(pwd)/temp" BRANCH_WHITELIST='' BRANCH_BLACKLIST='' + TAG_WHITELIST='' + TAG_BLACKLIST='' + TAGS_PLAN='visited' + PROCESS_TAGS_MAXHISTORYLOOKUP=20 NO_HARDLINKS='' ATOMIC='--atomic' NO_LOCK='' - PROCESS_TAGS='visited' - PROCESS_TAGS_MAXHISTORYLOOKUP=20 while : do if test $# -lt 1 @@ -131,6 +141,22 @@ readParameters () { BRANCH_BLACKLIST="${BRANCH_BLACKLIST} ${2}" shift 2 ;; + --tag-whitelist) + if test $# -lt 2 + then + usage 'Not enough arguments' + fi + TAG_WHITELIST="${TAG_WHITELIST} ${2}" + shift 2 + ;; + --tag-blacklist) + if test $# -lt 2 + then + usage 'Not enough arguments' + fi + TAG_BLACKLIST="${TAG_BLACKLIST} ${2}" + shift 2 + ;; --tags-plan) if test $# -lt 2 then @@ -138,13 +164,13 @@ readParameters () { fi case "${2}" in 'all') - PROCESS_TAGS='all' + TAGS_PLAN='all' ;; 'visited') - PROCESS_TAGS='visited' + TAGS_PLAN='visited' ;; 'none') - PROCESS_TAGS='' + TAGS_PLAN='' ;; *) usage "Invalid value of the ${readParameters_currentArgument} option" @@ -188,6 +214,13 @@ readParameters () { ;; esac done + if test -z "${TAGS_PLAN}" + then + if test -n "${TAG_WHITELIST}" -o -n "${TAG_BLACKLIST}" + then + die "You can't use --tag-whitelist or --tag-blacklist when you specify '--tags-plan none'" + fi + fi if test $# -lt 3 then usage 'Not enough arguments' @@ -213,6 +246,7 @@ readParameters () { fi } + absolutizeUrl () { absolutizeUrl_url="${1}" if test -d "${1}" @@ -222,6 +256,7 @@ absolutizeUrl () { printf '%s' "${absolutizeUrl_url}" } + checkFilter () { checkFilter_some=0 while : @@ -264,6 +299,7 @@ checkFilter () { fi } + normalizeParameters () { echo '# Normalizing source repository URL' SOURCE_REPOSITORY_URL=$(absolutizeUrl "${SOURCE_REPOSITORY_URL}") @@ -274,6 +310,7 @@ normalizeParameters () { checkFilter ${FILTER} } + checkEnvironment () { if test -z "${NO_LOCK}" then @@ -362,23 +399,23 @@ getTagList () { } -branchInList () { - branchInList_branch="${1}" - branchInList_list="${2}" - for branchInList_listItem in ${branchInList_list} +stringInList () { + stringInList_string="${1}" + stringInList_list="${2}" + for stringInList_listItem in ${stringInList_list} do - if test -n "${branchInList_listItem}" + if test -n "${stringInList_listItem}" then - case "${branchInList_listItem}" in + case "${stringInList_listItem}" in rx:*) - branchInList_substring=$(printf '%s' "${branchInList_listItem}" | cut -c4-) - if printf '%s' "${branchInList_branch}" | grep -Eq "^${branchInList_substring}$" + stringInList_substring=$(printf '%s' "${stringInList_listItem}" | cut -c4-) + if printf '%s' "${stringInList_string}" | grep -Eq "^${stringInList_substring}$" then return 0 fi ;; *) - if test "${branchInList_branch}" = "${branchInList_listItem}" + if test "${stringInList_string}" = "${stringInList_listItem}" then return 0 fi @@ -390,25 +427,34 @@ branchInList () { } +# $1: the string +# $2: the whitelist +# $3: the blacklist +stringPassesLists () { + if stringInList "${1}" "${3}" + then + return 1 + fi + if test -z "${2}" + then + return 0 + fi + if stringInList "${1}" "${2}" + then + return 0 + fi + return 1 +} + + getBranchesToProcess () { echo '# Determining branches to be processed' WORK_BRANCHES='' for getBranchesToProcess_sourceBranch in ${SOURCE_BRANCHES} do - if ! branchInList "${getBranchesToProcess_sourceBranch}" "${BRANCH_BLACKLIST}" + if stringPassesLists "${getBranchesToProcess_sourceBranch}" "${BRANCH_WHITELIST}" "${BRANCH_BLACKLIST}" then - getBranchesToProcess_branchPassed='' - if test -z "${BRANCH_WHITELIST}" - then - getBranchesToProcess_branchPassed='yes' - elif branchInList "${getBranchesToProcess_sourceBranch}" "${BRANCH_WHITELIST}" - then - getBranchesToProcess_branchPassed='yes' - fi - if test -n "${getBranchesToProcess_branchPassed}" - then - WORK_BRANCHES="${WORK_BRANCHES} ${getBranchesToProcess_sourceBranch}" - fi + WORK_BRANCHES="${WORK_BRANCHES} ${getBranchesToProcess_sourceBranch}" fi done if test -z "${WORK_BRANCHES}" @@ -474,7 +520,7 @@ processBranch () { rm -rf "${WORKER_REPOSITORY_DIR}.filter-branch" echo " - filtering commits" processBranch_tags='' - if test -z "${PROCESS_TAGS}" + if test -z "${TAGS_PLAN}" then processBranch_tags='' else @@ -514,14 +560,20 @@ processBranch () { die 'git failed' fi else - if test "${PROCESS_TAGS}" = 'all' -a -n "${processBranch_tags}" + if test "${TAGS_PLAN}" = 'all' -a -n "${processBranch_tags}" then - processBranchTag_availableTags="$(git -C "${WORKER_REPOSITORY_DIR}" tag --list | grep -E '^filter-branch/converted-tags/' | sed -E 's:^filter-branch/converted-tags/::')" + if ! processBranchTag_availableTags="$(git -C "${WORKER_REPOSITORY_DIR}" tag --list | grep -E '^filter-branch/converted-tags/' | sed -E 's:^filter-branch/converted-tags/::')" + then + processBranchTag_availableTags='' + fi for processBranch_tag in ${processBranch_tags} do if ! itemInList "${processBranch_tag}" "${processBranchTag_availableTags}" then - processNotConvertedTag "${processBranch_tag}" + if stringPassesLists "${processBranch_tag}" "${TAG_WHITELIST}" "${TAG_BLACKLIST}" + then + processNotConvertedTag "${processBranch_tag}" + fi fi done fi @@ -597,17 +649,23 @@ processBranches () { processBranch "${processBranches_branch}" processBranches_pushRefSpec="${processBranches_pushRefSpec} filter-branch/result/${processBranches_branch}:${processBranches_branch}" done - echo '# Listing source tags' - processBranches_sourceTags=$(getTagList "${SOURCE_REPOSITORY_DIR}") - echo '# Determining destination tags' - for processBranches_sourceTag in ${processBranches_sourceTags} - do - processBranches_rewrittenTag="filter-branch/converted-tags/${processBranches_sourceTag}" - if git -C "${WORKER_REPOSITORY_DIR}" rev-list --max-count=0 "${processBranches_rewrittenTag}" 2>/dev/null - then - processBranches_pushRefSpec="${processBranches_pushRefSpec} ${processBranches_rewrittenTag}:${processBranches_sourceTag}" - fi - done + if test -n "${TAGS_PLAN}" + then + echo '# Listing source tags' + processBranches_sourceTags=$(getTagList "${SOURCE_REPOSITORY_DIR}") + echo '# Determining destination tags' + for processBranches_sourceTag in ${processBranches_sourceTags} + do + if stringPassesLists "${processBranches_sourceTag}" "${TAG_WHITELIST}" "${TAG_BLACKLIST}" + then + processBranches_rewrittenTag="filter-branch/converted-tags/${processBranches_sourceTag}" + if git -C "${WORKER_REPOSITORY_DIR}" rev-list --max-count=0 "${processBranches_rewrittenTag}" 2>/dev/null + then + processBranches_pushRefSpec="${processBranches_pushRefSpec} ${processBranches_rewrittenTag}:${processBranches_sourceTag}" + fi + fi + done + fi echo "# Pushing to destination repository" # shellcheck disable=SC2086 git -C "${WORKER_REPOSITORY_DIR}" push --quiet --force ${ATOMIC} destination ${processBranches_pushRefSpec} diff --git a/test/tests/filter-directory-filtered-tags-bl.success b/test/tests/filter-directory-filtered-tags-bl.success new file mode 100755 index 0000000..8bc0c46 --- /dev/null +++ b/test/tests/filter-directory-filtered-tags-bl.success @@ -0,0 +1,16 @@ +#!/bin/sh + +. "$(cd -- "$(dirname -- "${0}")" && pwd -P)/../bootstrap.sh" + +initializeRepositories + +"${BIN_MAIN}" --workdir "${DIR_TEMP}" --branch-whitelist 'master' --tag-blacklist 'tag-02' -- "${DIR_SOURCE}" '--prune-empty --subdirectory-filter subdir' "${DIR_DESTINATION}" + +echo 'Fetching tags' +actualTags="$(getTagList "${DIR_DESTINATION}")" +expectedTags='' +if test "${actualTags}" != "${expectedTags}" +then + printf 'Expected tags: %s\nResulting tags with blacklist: %s\n' "${expectedTags}" "${actualTags}">&2 + exit 1 +fi diff --git a/test/tests/filter-directory-filtered-tags-blrx.success b/test/tests/filter-directory-filtered-tags-blrx.success new file mode 100755 index 0000000..ff6e155 --- /dev/null +++ b/test/tests/filter-directory-filtered-tags-blrx.success @@ -0,0 +1,16 @@ +#!/bin/sh + +. "$(cd -- "$(dirname -- "${0}")" && pwd -P)/../bootstrap.sh" + +initializeRepositories + +"${BIN_MAIN}" --workdir "${DIR_TEMP}" --branch-whitelist 'master' --tag-blacklist 'rx:tag-[0-9]+' -- "${DIR_SOURCE}" '--prune-empty --subdirectory-filter subdir' "${DIR_DESTINATION}" + +echo 'Fetching tags' +actualTags="$(getTagList "${DIR_DESTINATION}")" +expectedTags='' +if test "${actualTags}" != "${expectedTags}" +then + printf 'Expected tags: %s\nResulting tags with rx blacklist: %s\n' "${expectedTags}" "${actualTags}">&2 + exit 1 +fi diff --git a/test/tests/filter-directory-filtered-tags-wl.success b/test/tests/filter-directory-filtered-tags-wl.success new file mode 100755 index 0000000..3e08be0 --- /dev/null +++ b/test/tests/filter-directory-filtered-tags-wl.success @@ -0,0 +1,16 @@ +#!/bin/sh + +. "$(cd -- "$(dirname -- "${0}")" && pwd -P)/../bootstrap.sh" + +initializeRepositories + +"${BIN_MAIN}" --workdir "${DIR_TEMP}" --branch-whitelist 'master' --tag-whitelist 'tag-02' -- "${DIR_SOURCE}" '--prune-empty --subdirectory-filter subdir' "${DIR_DESTINATION}" + +echo 'Fetching tags' +actualTags="$(getTagList "${DIR_DESTINATION}")" +expectedTags='tag-02' +if test "${actualTags}" != "${expectedTags}" +then + printf 'Expected tags: %s\nResulting tags with whitelist: %s\n' "${expectedTags}" "${actualTags}">&2 + exit 1 +fi diff --git a/test/tests/filter-directory-filtered-tags-wlrx.success b/test/tests/filter-directory-filtered-tags-wlrx.success new file mode 100755 index 0000000..7e101a0 --- /dev/null +++ b/test/tests/filter-directory-filtered-tags-wlrx.success @@ -0,0 +1,16 @@ +#!/bin/sh + +. "$(cd -- "$(dirname -- "${0}")" && pwd -P)/../bootstrap.sh" + +initializeRepositories + +"${BIN_MAIN}" --workdir "${DIR_TEMP}" --branch-whitelist 'master' --tag-whitelist 'rx:tag-[0-9]+' -- "${DIR_SOURCE}" '--prune-empty --subdirectory-filter subdir' "${DIR_DESTINATION}" + +echo 'Fetching tags' +actualTags="$(getTagList "${DIR_DESTINATION}")" +expectedTags='tag-02' +if test "${actualTags}" != "${expectedTags}" +then + printf 'Expected tags: %s\nResulting tags with rx whitelist: %s\n' "${expectedTags}" "${actualTags}">&2 + exit 1 +fi -- cgit v1.2.3