From a9e38359e356de7d6397395bdde8af61440262d0 Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Thu, 3 Sep 2015 11:14:07 +0200 Subject: [PATCH 1/3] git-p4: add config git-p4.pathEncoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Perforce keeps the encoding of a path as given by the originating OS. Git expects paths encoded as UTF-8. Add a config to tell git-p4 what encoding Perforce had used for the paths. This encoding is used to transcode the paths to UTF-8. As an example, Perforce on Windows often uses “cp1252” to encode path names. Signed-off-by: Lars Schneider Acked-by: Luke Diamand Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- Documentation/git-p4.txt | 7 ++++ git-p4.py | 11 +++++++ t/t9822-git-p4-path-encoding.sh | 58 +++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+) create mode 100755 t/t9822-git-p4-path-encoding.sh diff --git a/Documentation/git-p4.txt b/Documentation/git-p4.txt index 82aa5d6073..12a57d49f4 100644 --- a/Documentation/git-p4.txt +++ b/Documentation/git-p4.txt @@ -510,6 +510,13 @@ git-p4.useClientSpec:: option '--use-client-spec'. See the "CLIENT SPEC" section above. This variable is a boolean, not the name of a p4 client. +git-p4.pathEncoding:: + Perforce keeps the encoding of a path as given by the originating OS. + Git expects paths encoded as UTF-8. Use this config to tell git-p4 + what encoding Perforce had used for the paths. This encoding is used + to transcode the paths to UTF-8. As an example, Perforce on Windows + often uses “cp1252” to encode path names. + Submit variables ~~~~~~~~~~~~~~~~ git-p4.detectRenames:: diff --git a/git-p4.py b/git-p4.py index 073f87bbfd..b1ad86de7f 100755 --- a/git-p4.py +++ b/git-p4.py @@ -2213,6 +2213,17 @@ class P4Sync(Command, P4UserMap): text = regexp.sub(r'$\1$', text) contents = [ text ] + if gitConfig("git-p4.pathEncoding"): + relPath = relPath.decode(gitConfig("git-p4.pathEncoding")).encode('utf8', 'replace') + elif self.verbose: + try: + relPath.decode('ascii') + except: + print ( + "Path with Non-ASCII characters detected and no path encoding defined. " + "Please check the encoding: %s" % relPath + ) + self.gitStream.write("M %s inline %s\n" % (git_mode, relPath)) # total length... diff --git a/t/t9822-git-p4-path-encoding.sh b/t/t9822-git-p4-path-encoding.sh new file mode 100755 index 0000000000..7b83e696a9 --- /dev/null +++ b/t/t9822-git-p4-path-encoding.sh @@ -0,0 +1,58 @@ +#!/bin/sh + +test_description='Clone repositories with non ASCII paths' + +. ./lib-git-p4.sh + +UTF8_ESCAPED="a-\303\244_o-\303\266_u-\303\274.txt" +ISO8859_ESCAPED="a-\344_o-\366_u-\374.txt" + +test_expect_success 'start p4d' ' + start_p4d +' + +test_expect_success 'Create a repo containing iso8859-1 encoded paths' ' + ( + cd "$cli" && + ISO8859="$(printf "$ISO8859_ESCAPED")" && + echo content123 >"$ISO8859" && + p4 add "$ISO8859" && + p4 submit -d "test commit" + ) +' + +test_expect_failure 'Clone auto-detects depot with iso8859-1 paths' ' + git p4 clone --destination="$git" //depot && + test_when_finished cleanup_git && + ( + cd "$git" && + UTF8="$(printf "$UTF8_ESCAPED")" && + echo "$UTF8" >expect && + git -c core.quotepath=false ls-files >actual && + test_cmp expect actual + ) +' + +test_expect_success 'Clone repo containing iso8859-1 encoded paths with git-p4.pathEncoding' ' + test_when_finished cleanup_git && + ( + cd "$git" && + git init . && + git config git-p4.pathEncoding iso8859-1 && + git p4 clone --use-client-spec --destination="$git" //depot && + UTF8="$(printf "$UTF8_ESCAPED")" && + echo "$UTF8" >expect && + git -c core.quotepath=false ls-files >actual && + test_cmp expect actual && + + echo content123 >expect && + cat "$UTF8" >actual && + test_cmp expect actual + ) +' + +test_expect_success 'kill p4d' ' + kill_p4d +' + +test_done From 00a9403a1069b19f27d690853db34459b32b3d3d Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Wed, 16 Sep 2015 14:37:04 +0200 Subject: [PATCH 2/3] git-p4: improve path encoding verbose output If a path with non-ASCII characters is detected then print the encoding and the encoded string in verbose mode. Signed-off-by: Lars Schneider Signed-off-by: Junio C Hamano --- git-p4.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/git-p4.py b/git-p4.py index b1ad86de7f..65feb22f58 100755 --- a/git-p4.py +++ b/git-p4.py @@ -2213,16 +2213,15 @@ class P4Sync(Command, P4UserMap): text = regexp.sub(r'$\1$', text) contents = [ text ] - if gitConfig("git-p4.pathEncoding"): - relPath = relPath.decode(gitConfig("git-p4.pathEncoding")).encode('utf8', 'replace') - elif self.verbose: - try: - relPath.decode('ascii') - except: - print ( - "Path with Non-ASCII characters detected and no path encoding defined. " - "Please check the encoding: %s" % relPath - ) + try: + relPath.decode('ascii') + except: + encoding = 'utf8' + if gitConfig('git-p4.pathEncoding'): + encoding = gitConfig('git-p4.pathEncoding') + relPath = relPath.decode(encoding).encode('utf8', 'replace') + if self.verbose: + print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath) self.gitStream.write("M %s inline %s\n" % (git_mode, relPath)) From 4cb870d8042bef3fdd953c633463eda24ce78f3d Mon Sep 17 00:00:00 2001 From: Lars Schneider Date: Mon, 21 Sep 2015 10:49:18 +0200 Subject: [PATCH 3/3] git-p4: use replacement character for non UTF-8 characters in paths If non UTF-8 characters are detected in paths then replace them with a placeholder instead of throwing a UnicodeDecodeError exception. This restores the original (implicit) implementation that was broken in 00a9403. Signed-off-by: Lars Schneider Reviewed-by: Luke Diamand Signed-off-by: Junio C Hamano --- git-p4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/git-p4.py b/git-p4.py index 65feb22f58..603045050c 100755 --- a/git-p4.py +++ b/git-p4.py @@ -2219,7 +2219,7 @@ class P4Sync(Command, P4UserMap): encoding = 'utf8' if gitConfig('git-p4.pathEncoding'): encoding = gitConfig('git-p4.pathEncoding') - relPath = relPath.decode(encoding).encode('utf8', 'replace') + relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace') if self.verbose: print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)