From afdb29732ca24242f1a2e94f98e90034f9925f46 Mon Sep 17 00:00:00 2001 From: Steve McIntyre <93sam> Date: Wed, 30 May 2018 02:26:19 +0000 Subject: Major updates to perl scripts Add new git backend in VCS_git.pm Switch from an old-style set of function calls to an OO API. This allows for initialisation and some state to be kept in the VCS_git.pm module - namely a per-file cache of commit hashes for a massive performance boost when doing lots of lookups. Extend the API with 2 new utility functions: * get_oldest_revision() * next_revision() Extended the vcs_cmp_rev() function to take a filename too. Add a test harness to validate the git and cvs backends. Add switch_to_git_translations.pl to walk the tree and switch from cvs revisions to git commit hashes in translation-check headers. Change all of our local scripts to use the new Local::VCS frontend *where it makes sense*. Some scripts will behave slightly differently, as the new world can't exactly match the old behaviour. CVS version numbers check_desc_trans.pl: 1.9 -> 1.10 check_trans.pl: 1.93 -> 1.94 copypage.pl: 1.42 -> 1.43 karma.pl: 1.6 -> 1.7 remove_stale.pl: 1.22 -> 1.23 smart_change.pl: 1.8 -> 1.9 stattrans.pl: 1.127 -> 1.128 switch_to_git_translations.pl: INITIAL -> 1.1 touch_translations.pl: 1.9 -> 1.10 vcs-test.pl: INITIAL -> 1.1 Perl/Local/Util.pm: 1.4 -> 1.5 Perl/Local/VCS.pm: 1.3 -> 1.4 Perl/Local/VCS_CVS.pm: 1.13 -> 1.14 Perl/Local/VCS_git.pm: 1.12 -> 1.13 Perl/Webwml/Langs.pm: 1.5 -> 1.6 Perl/Webwml/TransIgnore.pm: 1.3 -> 1.4 --- switch_to_git_translations.pl | 226 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 226 insertions(+) create mode 100755 switch_to_git_translations.pl (limited to 'switch_to_git_translations.pl') diff --git a/switch_to_git_translations.pl b/switch_to_git_translations.pl new file mode 100755 index 00000000000..76e32b92c2e --- /dev/null +++ b/switch_to_git_translations.pl @@ -0,0 +1,226 @@ +#!/usr/bin/perl + +# This script walks the webwml tree to look for translated files. It +# looks for the wml::debian::translation-check header to see if a file +# is a stranslation of an original, then checks for the revision +# status of the master document. +# +# Part of the effort to switch from CVS to Git +# +# Originally written 2018 by Steve McIntyre <93sam@debian.org> +# © Copyright 2018 Software in the public interest, Inc. +# This program is released under the GNU General Public License, v2. + +use strict; +use warnings; + +use Getopt::Long; +use Data::Dumper; +use File::Spec::Functions; +use File::Find; +use lib ($0 =~ m|(.*)/|, $1 or ".") ."/Perl"; +use Webwml::TransCheck; + +my $help = 0; +my $verbose = 0; +my $dry_run = 0; +my $revs_file = ""; +my %rev_map; + +sub usage { + print <<'EOT'; +Usage: switch_to_git_translations.pl [options] +Options: + --help display this message + --verbose run verbosely + --dry-run do not modify translation-check headers + --revisions=REVISIONS location of the cvs2git revisions map file + +Find all wml/src/etc. files under the current directory, updating revisions for +translations. +EOT + exit(0); +} + +# log very verbose messages +sub vvlog { + if ($verbose >= 2) { + print STDOUT $_[0] . "\n"; + } +} + +# log verbose messages +sub vlog { + if ($verbose >= 1) { + print STDOUT $_[0] . "\n"; + } +} + +# Parse the revisions file for use, building a hash of the git and cvs versions for each file +sub parse_revisions +{ + my $revs_file = shift; + open(IN, "<", "$revs_file") or die "Can't open revisions file \$revs_file\" for reading: $!\n"; + while (my $line = ) { + chomp $line; + my ($file, $cvs_ver, $commit_hash); + if ($line =~ m,^(\S+) ([.\d]+) ([[:xdigit:]]+)$,) + { + $file = $1; + $cvs_ver = $2; + $commit_hash = $3; + $rev_map{"$file"}{"$cvs_ver"}{"commit_hash"} = $commit_hash; + } else { + die "Failed to parse revisions file at line $.\n"; + } + vvlog("Found file $file with CVS version $cvs_ver in commit hash $commit_hash"); + } + close IN; + vlog("Parsed revisions file \"$revs_file\", found revisions for " . scalar(keys %rev_map) . " files"); +} + +# return a list of filenames with the given extension +sub find_files_ext +{ + my $dir = shift or die('Internal error: No dir specified'); + my $ext = shift or die('Internal error: No ext specified'); + + my @files; + find( sub { if (-f and m/\.$ext$/) { my $filename = $File::Find::name; $filename =~ s,\.\/,,; push @files, $filename }}, $dir ); + return @files; +} + +# Update the translation-check metadata header in a wml file +sub update_file_metadata +{ + my $file = shift; + my $revision = shift; + my $hash = shift; + my $text = ""; + + open (IN, "< $file") or die "Can't open $file for reading: $!\n"; + while () { + if (m/^#use wml::debian::translation-check/) { + s/(translation="?)($revision)("?)/$1$hash$3/; + } + $text .= $_; + } + close(IN); + open(OUT, "> $file") or die "Can't open $file for writing: $!\n"; + print OUT $text; + close OUT; +} + +# Parse a file, and see if there's a translation-check header. If so, +# use the rev_map data to switch the translation information from the +# cvs version to the git hash *if available*. If it's not available, +# report an error. +sub parse_file +{ + my $file = shift; + my $info = 0; # Do we have any translation header info at all? + my $tc = Webwml::TransCheck->new("$file") or die "Failed transcheck: $!\n"; + vlog("Looking at wml file $file"); + my $target_lang = "english"; + my $maint = $tc->maintainer(); + if (defined($maint)) { + vvlog(" Maintainer: $maint"); + $info += 1; + } + my $revision = $tc->revision(); + if (defined($revision)) { + vvlog(" Revision: $revision"); + $info += 1; + } + my $original = $tc->original(); + if (defined($original)) { + vvlog(" Original: $original"); + $info += 1; + $target_lang = $original; + } + my $mindelta = $tc->mindelta(); + if (defined($mindelta)) { + vvlog(" Mindelta: $mindelta"); + $info += 1; + } + my $maxdelta = $tc->maxdelta(); + if (defined($maxdelta)) { + vvlog(" Maxdelta: $maxdelta"); + $info += 1; + } + if ($info > 0) { + my $targetfile = $file; + $targetfile =~ s,^[^/]+,$target_lang,; + vvlog(" Depends on $targetfile"); + if (defined($revision)) { + # Do we have a cvs->git map for that file and revision? + my $hash = $rev_map{"$targetfile"}{"$revision"}{"commit_hash"}; +# my $file_hash = $rev_map{"$targetfile"}{"$revision"}{"file_hash"}; + if (defined $hash) { +# if (!defined $file_hash) { +# $file_hash = `git ls-tree -r $hash $targetfile`; +# if ($file_hash =~ m/^\s*\d+\s*blob\s+([[:xdigit:]]+)\s+\S+$/) { +# $file_hash = $1; +# } +# # Cache the result +# $rev_map{"$targetfile"}{"$revision"}{"file_hash"} = $file_hash; +# } + vlog(" Depends on $targetfile with cvs rev $revision, commit hash $hash"); + } else { + vlog(" Looking up $targetfile with cvs rev $revision, no mapping found"); + return 1; + } + if (!$dry_run) { + vlog (" Updating the file data"); + + update_file_metadata($file, $revision, $hash); + } + } else { + vlog(" But no revision data!"); + return 1; + } + } +} + +# open(IN, "<", "$file") or die "Can't open file \$wml_file\" for reading: $!#\n"; +# while (my $line = ) { +# chomp $line; +# if ($line =~ m/^#use wml::debian::translation-check/) { +# my $original="english"; # default +# } +# } +#} + +# "main" + +if (not GetOptions ("help" => \$help, + "verbose=i" => \$verbose, + "dry-run" => \$dry_run, + "revisions=s" => \$revs_file)) +{ + warn "Try `$0 --help' for more information.\n"; + exit(1); +} + +if ($help) { + usage(); +} + +if (! -f $revs_file) { + die "Can't open revisions file, abort!\n"; +} +parse_revisions($revs_file); + +my @wmlfiles = sort(find_files_ext(".", 'wml')); +my @incfiles = sort(find_files_ext(".", 'inc')); +my @pofiles = sort(find_files_ext(".", 'po')); +my @srcfiles = sort(find_files_ext(".", 'src')); +my @files; +push @files, @wmlfiles; +push @files, @incfiles; +push @files, @pofiles; +push @files, @srcfiles; +vlog("Found " . scalar(@files) . " files to work on\n"); +for my $file (@files) { + parse_file($file); +} -- cgit v1.2.3