|
|
From e87ab5b991b08092a7e07af82b3ec822a8604151 Mon Sep 17 00:00:00 2001 |
|
|
From: Ondrej Oprala <ooprala@redhat.com> |
|
|
Date: Wed, 5 Aug 2015 09:15:09 +0200 |
|
|
Subject: [PATCH] expand,unexpand: add multibyte support |
|
|
MIME-Version: 1.0 |
|
|
Content-Type: text/plain; charset=UTF-8 |
|
|
Content-Transfer-Encoding: 8bit |
|
|
|
|
|
* NEWS: Mention the changes. |
|
|
* bootstrap.conf: Add mbfile to the list of modules. |
|
|
* configure.ac: Properly initialize mbfile. |
|
|
* src/expand.c (expand): Iterate over multibyte characters properly. |
|
|
* src/unexpand.c (unexpand): Iterate over multibyte characters |
|
|
properly. |
|
|
* tests/local.mk: Add new tests. |
|
|
* tests/{expand,unexpand}/mb.sh: New tests. |
|
|
|
|
|
Co-authored-by: Pádraig Brady <pbrady@redhat.com> |
|
|
--- |
|
|
bootstrap.conf | 1 + |
|
|
configure.ac | 2 + |
|
|
lib/mbfile.c | 3 + |
|
|
lib/mbfile.h | 255 +++++++++++++++++++++++++++++++++++++++++++++++++++ |
|
|
m4/mbfile.m4 | 14 +++ |
|
|
src/expand.c | 43 +++++---- |
|
|
src/local.mk | 4 +- |
|
|
src/unexpand.c | 54 +++++++---- |
|
|
tests/expand/mb.sh | 98 ++++++++++++++++++++ |
|
|
tests/local.mk | 2 + |
|
|
tests/unexpand/mb.sh | 97 ++++++++++++++++++++ |
|
|
10 files changed, 535 insertions(+), 34 deletions(-) |
|
|
create mode 100644 lib/mbfile.c |
|
|
create mode 100644 lib/mbfile.h |
|
|
create mode 100644 m4/mbfile.m4 |
|
|
create mode 100755 tests/expand/mb.sh |
|
|
create mode 100755 tests/unexpand/mb.sh |
|
|
|
|
|
diff --git a/bootstrap.conf b/bootstrap.conf |
|
|
index 8a0ff31..a1c78b2 100644 |
|
|
--- a/bootstrap.conf |
|
|
+++ b/bootstrap.conf |
|
|
@@ -152,6 +152,7 @@ gnulib_modules=" |
|
|
maintainer-makefile |
|
|
malloc-gnu |
|
|
manywarnings |
|
|
+ mbfile |
|
|
mbrlen |
|
|
mbrtowc |
|
|
mbsalign |
|
|
diff --git a/configure.ac b/configure.ac |
|
|
index 1e74b36..24c9725 100644 |
|
|
--- a/configure.ac |
|
|
+++ b/configure.ac |
|
|
@@ -427,6 +427,8 @@ fi |
|
|
# I'm leaving it here for now. This whole thing needs to be modernized... |
|
|
gl_WINSIZE_IN_PTEM |
|
|
|
|
|
+gl_MBFILE |
|
|
+ |
|
|
gl_HEADER_TIOCGWINSZ_IN_TERMIOS_H |
|
|
|
|
|
if test $gl_cv_sys_tiocgwinsz_needs_termios_h = no && \ |
|
|
diff --git a/lib/mbfile.c b/lib/mbfile.c |
|
|
new file mode 100644 |
|
|
index 0000000..b0a468e |
|
|
--- /dev/null |
|
|
+++ b/lib/mbfile.c |
|
|
@@ -0,0 +1,3 @@ |
|
|
+#include <config.h> |
|
|
+#define MBFILE_INLINE _GL_EXTERN_INLINE |
|
|
+#include "mbfile.h" |
|
|
diff --git a/lib/mbfile.h b/lib/mbfile.h |
|
|
new file mode 100644 |
|
|
index 0000000..11f1b12 |
|
|
--- /dev/null |
|
|
+++ b/lib/mbfile.h |
|
|
@@ -0,0 +1,255 @@ |
|
|
+/* Multibyte character I/O: macros for multi-byte encodings. |
|
|
+ Copyright (C) 2001, 2005, 2009-2015 Free Software Foundation, Inc. |
|
|
+ |
|
|
+ This program is free software: you can redistribute it and/or modify |
|
|
+ it under the terms of the GNU General Public License as published by |
|
|
+ the Free Software Foundation; either version 3 of the License, or |
|
|
+ (at your option) any later version. |
|
|
+ |
|
|
+ This program is distributed in the hope that it will be useful, |
|
|
+ but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
+ GNU General Public License for more details. |
|
|
+ |
|
|
+ You should have received a copy of the GNU General Public License |
|
|
+ along with this program. If not, see <http://www.gnu.org/licenses/>. */ |
|
|
+ |
|
|
+/* Written by Mitsuru Chinen <mchinen@yamato.ibm.com> |
|
|
+ and Bruno Haible <bruno@clisp.org>. */ |
|
|
+ |
|
|
+/* The macros in this file implement multi-byte character input from a |
|
|
+ stream. |
|
|
+ |
|
|
+ mb_file_t |
|
|
+ is the type for multibyte character input stream, usable for variable |
|
|
+ declarations. |
|
|
+ |
|
|
+ mbf_char_t |
|
|
+ is the type for multibyte character or EOF, usable for variable |
|
|
+ declarations. |
|
|
+ |
|
|
+ mbf_init (mbf, stream) |
|
|
+ initializes the MB_FILE for reading from stream. |
|
|
+ |
|
|
+ mbf_getc (mbc, mbf) |
|
|
+ reads the next multibyte character from mbf and stores it in mbc. |
|
|
+ |
|
|
+ mb_iseof (mbc) |
|
|
+ returns true if mbc represents the EOF value. |
|
|
+ |
|
|
+ Here are the function prototypes of the macros. |
|
|
+ |
|
|
+ extern void mbf_init (mb_file_t mbf, FILE *stream); |
|
|
+ extern void mbf_getc (mbf_char_t mbc, mb_file_t mbf); |
|
|
+ extern bool mb_iseof (const mbf_char_t mbc); |
|
|
+ */ |
|
|
+ |
|
|
+#ifndef _MBFILE_H |
|
|
+#define _MBFILE_H 1 |
|
|
+ |
|
|
+#include <assert.h> |
|
|
+#include <stdbool.h> |
|
|
+#include <stdio.h> |
|
|
+#include <string.h> |
|
|
+ |
|
|
+/* Tru64 with Desktop Toolkit C has a bug: <stdio.h> must be included before |
|
|
+ <wchar.h>. |
|
|
+ BSD/OS 4.1 has a bug: <stdio.h> and <time.h> must be included before |
|
|
+ <wchar.h>. */ |
|
|
+#include <stdio.h> |
|
|
+#include <time.h> |
|
|
+#include <wchar.h> |
|
|
+ |
|
|
+#include "mbchar.h" |
|
|
+ |
|
|
+#ifndef _GL_INLINE_HEADER_BEGIN |
|
|
+ #error "Please include config.h first." |
|
|
+#endif |
|
|
+_GL_INLINE_HEADER_BEGIN |
|
|
+#ifndef MBFILE_INLINE |
|
|
+# define MBFILE_INLINE _GL_INLINE |
|
|
+#endif |
|
|
+ |
|
|
+struct mbfile_multi { |
|
|
+ FILE *fp; |
|
|
+ bool eof_seen; |
|
|
+ bool have_pushback; |
|
|
+ mbstate_t state; |
|
|
+ unsigned int bufcount; |
|
|
+ char buf[MBCHAR_BUF_SIZE]; |
|
|
+ struct mbchar pushback; |
|
|
+}; |
|
|
+ |
|
|
+MBFILE_INLINE void |
|
|
+mbfile_multi_getc (struct mbchar *mbc, struct mbfile_multi *mbf) |
|
|
+{ |
|
|
+ size_t bytes; |
|
|
+ |
|
|
+ /* If EOF has already been seen, don't use getc. This matters if |
|
|
+ mbf->fp is connected to an interactive tty. */ |
|
|
+ if (mbf->eof_seen) |
|
|
+ goto eof; |
|
|
+ |
|
|
+ /* Return character pushed back, if there is one. */ |
|
|
+ if (mbf->have_pushback) |
|
|
+ { |
|
|
+ mb_copy (mbc, &mbf->pushback); |
|
|
+ mbf->have_pushback = false; |
|
|
+ return; |
|
|
+ } |
|
|
+ |
|
|
+ /* Before using mbrtowc, we need at least one byte. */ |
|
|
+ if (mbf->bufcount == 0) |
|
|
+ { |
|
|
+ int c = getc (mbf->fp); |
|
|
+ if (c == EOF) |
|
|
+ { |
|
|
+ mbf->eof_seen = true; |
|
|
+ goto eof; |
|
|
+ } |
|
|
+ mbf->buf[0] = (unsigned char) c; |
|
|
+ mbf->bufcount++; |
|
|
+ } |
|
|
+ |
|
|
+ /* Handle most ASCII characters quickly, without calling mbrtowc(). */ |
|
|
+ if (mbf->bufcount == 1 && mbsinit (&mbf->state) && is_basic (mbf->buf[0])) |
|
|
+ { |
|
|
+ /* These characters are part of the basic character set. ISO C 99 |
|
|
+ guarantees that their wide character code is identical to their |
|
|
+ char code. */ |
|
|
+ mbc->wc = mbc->buf[0] = mbf->buf[0]; |
|
|
+ mbc->wc_valid = true; |
|
|
+ mbc->ptr = &mbc->buf[0]; |
|
|
+ mbc->bytes = 1; |
|
|
+ mbf->bufcount = 0; |
|
|
+ return; |
|
|
+ } |
|
|
+ |
|
|
+ /* Use mbrtowc on an increasing number of bytes. Read only as many bytes |
|
|
+ from mbf->fp as needed. This is needed to give reasonable interactive |
|
|
+ behaviour when mbf->fp is connected to an interactive tty. */ |
|
|
+ for (;;) |
|
|
+ { |
|
|
+ /* We don't know whether the 'mbrtowc' function updates the state when |
|
|
+ it returns -2, - this is the ISO C 99 and glibc-2.2 behaviour - or |
|
|
+ not - amended ANSI C, glibc-2.1 and Solaris 2.7 behaviour. We |
|
|
+ don't have an autoconf test for this, yet. |
|
|
+ The new behaviour would allow us to feed the bytes one by one into |
|
|
+ mbrtowc. But the old behaviour forces us to feed all bytes since |
|
|
+ the end of the last character into mbrtowc. Since we want to retry |
|
|
+ with more bytes when mbrtowc returns -2, we must backup the state |
|
|
+ before calling mbrtowc, because implementations with the new |
|
|
+ behaviour will clobber it. */ |
|
|
+ mbstate_t backup_state = mbf->state; |
|
|
+ |
|
|
+ bytes = mbrtowc (&mbc->wc, &mbf->buf[0], mbf->bufcount, &mbf->state); |
|
|
+ |
|
|
+ if (bytes == (size_t) -1) |
|
|
+ { |
|
|
+ /* An invalid multibyte sequence was encountered. */ |
|
|
+ /* Return a single byte. */ |
|
|
+ bytes = 1; |
|
|
+ mbc->wc_valid = false; |
|
|
+ break; |
|
|
+ } |
|
|
+ else if (bytes == (size_t) -2) |
|
|
+ { |
|
|
+ /* An incomplete multibyte character. */ |
|
|
+ mbf->state = backup_state; |
|
|
+ if (mbf->bufcount == MBCHAR_BUF_SIZE) |
|
|
+ { |
|
|
+ /* An overlong incomplete multibyte sequence was encountered. */ |
|
|
+ /* Return a single byte. */ |
|
|
+ bytes = 1; |
|
|
+ mbc->wc_valid = false; |
|
|
+ break; |
|
|
+ } |
|
|
+ else |
|
|
+ { |
|
|
+ /* Read one more byte and retry mbrtowc. */ |
|
|
+ int c = getc (mbf->fp); |
|
|
+ if (c == EOF) |
|
|
+ { |
|
|
+ /* An incomplete multibyte character at the end. */ |
|
|
+ mbf->eof_seen = true; |
|
|
+ bytes = mbf->bufcount; |
|
|
+ mbc->wc_valid = false; |
|
|
+ break; |
|
|
+ } |
|
|
+ mbf->buf[mbf->bufcount] = (unsigned char) c; |
|
|
+ mbf->bufcount++; |
|
|
+ } |
|
|
+ } |
|
|
+ else |
|
|
+ { |
|
|
+ if (bytes == 0) |
|
|
+ { |
|
|
+ /* A null wide character was encountered. */ |
|
|
+ bytes = 1; |
|
|
+ assert (mbf->buf[0] == '\0'); |
|
|
+ assert (mbc->wc == 0); |
|
|
+ } |
|
|
+ mbc->wc_valid = true; |
|
|
+ break; |
|
|
+ } |
|
|
+ } |
|
|
+ |
|
|
+ /* Return the multibyte sequence mbf->buf[0..bytes-1]. */ |
|
|
+ mbc->ptr = &mbc->buf[0]; |
|
|
+ memcpy (&mbc->buf[0], &mbf->buf[0], bytes); |
|
|
+ mbc->bytes = bytes; |
|
|
+ |
|
|
+ mbf->bufcount -= bytes; |
|
|
+ if (mbf->bufcount > 0) |
|
|
+ { |
|
|
+ /* It's not worth calling memmove() for so few bytes. */ |
|
|
+ unsigned int count = mbf->bufcount; |
|
|
+ char *p = &mbf->buf[0]; |
|
|
+ |
|
|
+ do |
|
|
+ { |
|
|
+ *p = *(p + bytes); |
|
|
+ p++; |
|
|
+ } |
|
|
+ while (--count > 0); |
|
|
+ } |
|
|
+ return; |
|
|
+ |
|
|
+eof: |
|
|
+ /* An mbchar_t with bytes == 0 is used to indicate EOF. */ |
|
|
+ mbc->ptr = NULL; |
|
|
+ mbc->bytes = 0; |
|
|
+ mbc->wc_valid = false; |
|
|
+ return; |
|
|
+} |
|
|
+ |
|
|
+MBFILE_INLINE void |
|
|
+mbfile_multi_ungetc (const struct mbchar *mbc, struct mbfile_multi *mbf) |
|
|
+{ |
|
|
+ mb_copy (&mbf->pushback, mbc); |
|
|
+ mbf->have_pushback = true; |
|
|
+} |
|
|
+ |
|
|
+typedef struct mbfile_multi mb_file_t; |
|
|
+ |
|
|
+typedef mbchar_t mbf_char_t; |
|
|
+ |
|
|
+#define mbf_init(mbf, stream) \ |
|
|
+ ((mbf).fp = (stream), \ |
|
|
+ (mbf).eof_seen = false, \ |
|
|
+ (mbf).have_pushback = false, \ |
|
|
+ memset (&(mbf).state, '\0', sizeof (mbstate_t)), \ |
|
|
+ (mbf).bufcount = 0) |
|
|
+ |
|
|
+#define mbf_getc(mbc, mbf) mbfile_multi_getc (&(mbc), &(mbf)) |
|
|
+ |
|
|
+#define mbf_ungetc(mbc, mbf) mbfile_multi_ungetc (&(mbc), &(mbf)) |
|
|
+ |
|
|
+#define mb_iseof(mbc) ((mbc).bytes == 0) |
|
|
+ |
|
|
+#ifndef _GL_INLINE_HEADER_BEGIN |
|
|
+ #error "Please include config.h first." |
|
|
+#endif |
|
|
+_GL_INLINE_HEADER_BEGIN |
|
|
+ |
|
|
+#endif /* _MBFILE_H */ |
|
|
diff --git a/m4/mbfile.m4 b/m4/mbfile.m4 |
|
|
new file mode 100644 |
|
|
index 0000000..8589902 |
|
|
--- /dev/null |
|
|
+++ b/m4/mbfile.m4 |
|
|
@@ -0,0 +1,14 @@ |
|
|
+# mbfile.m4 serial 7 |
|
|
+dnl Copyright (C) 2005, 2008-2015 Free Software Foundation, Inc. |
|
|
+dnl This file is free software; the Free Software Foundation |
|
|
+dnl gives unlimited permission to copy and/or distribute it, |
|
|
+dnl with or without modifications, as long as this notice is preserved. |
|
|
+ |
|
|
+dnl autoconf tests required for use of mbfile.h |
|
|
+dnl From Bruno Haible. |
|
|
+ |
|
|
+AC_DEFUN([gl_MBFILE], |
|
|
+[ |
|
|
+ AC_REQUIRE([AC_TYPE_MBSTATE_T]) |
|
|
+ : |
|
|
+]) |
|
|
diff --git a/src/expand.c b/src/expand.c |
|
|
index 9fa2e10..380e020 100644 |
|
|
--- a/src/expand.c |
|
|
+++ b/src/expand.c |
|
|
@@ -37,6 +37,9 @@ |
|
|
#include <stdio.h> |
|
|
#include <getopt.h> |
|
|
#include <sys/types.h> |
|
|
+ |
|
|
+#include <mbfile.h> |
|
|
+ |
|
|
#include "system.h" |
|
|
#include "die.h" |
|
|
#include "xstrndup.h" |
|
|
@@ -100,19 +103,19 @@ expand (void) |
|
|
{ |
|
|
/* Input stream. */ |
|
|
FILE *fp = next_file (NULL); |
|
|
+ mb_file_t mbf; |
|
|
+ mbf_char_t c; |
|
|
|
|
|
if (!fp) |
|
|
return; |
|
|
|
|
|
+ mbf_init (mbf, fp); |
|
|
+ |
|
|
while (true) |
|
|
{ |
|
|
- /* Input character, or EOF. */ |
|
|
- int c; |
|
|
- |
|
|
/* If true, perform translations. */ |
|
|
bool convert = true; |
|
|
|
|
|
- |
|
|
/* The following variables have valid values only when CONVERT |
|
|
is true: */ |
|
|
|
|
|
@@ -122,17 +125,23 @@ expand (void) |
|
|
/* Index in TAB_LIST of next tab stop to examine. */ |
|
|
size_t tab_index = 0; |
|
|
|
|
|
- |
|
|
/* Convert a line of text. */ |
|
|
|
|
|
do |
|
|
{ |
|
|
- while ((c = getc (fp)) < 0 && (fp = next_file (fp))) |
|
|
- continue; |
|
|
+ do { |
|
|
+ mbf_getc (c, mbf); |
|
|
+ if (mb_iseof (c)) |
|
|
+ { |
|
|
+ mbf_init (mbf, fp = next_file (fp)); |
|
|
+ continue; |
|
|
+ } |
|
|
+ } |
|
|
+ while (false); |
|
|
|
|
|
if (convert) |
|
|
{ |
|
|
- if (c == '\t') |
|
|
+ if (mb_iseq (c, '\t')) |
|
|
{ |
|
|
/* Column the next input tab stop is on. */ |
|
|
uintmax_t next_tab_column; |
|
|
@@ -151,32 +160,34 @@ expand (void) |
|
|
if (putchar (' ') < 0) |
|
|
die (EXIT_FAILURE, errno, _("write error")); |
|
|
|
|
|
- c = ' '; |
|
|
+ mb_setascii (&c, ' '); |
|
|
} |
|
|
- else if (c == '\b') |
|
|
+ else if (mb_iseq (c, '\b')) |
|
|
{ |
|
|
/* Go back one column, and force recalculation of the |
|
|
next tab stop. */ |
|
|
column -= !!column; |
|
|
tab_index -= !!tab_index; |
|
|
} |
|
|
- else |
|
|
+ /* A leading control character could make us trip over. */ |
|
|
+ else if (!mb_iscntrl (c)) |
|
|
{ |
|
|
- column++; |
|
|
+ column += mb_width (c); |
|
|
if (!column) |
|
|
die (EXIT_FAILURE, 0, _("input line is too long")); |
|
|
} |
|
|
|
|
|
- convert &= convert_entire_line || !! isblank (c); |
|
|
+ convert &= convert_entire_line || mb_isblank (c); |
|
|
} |
|
|
|
|
|
- if (c < 0) |
|
|
+ if (mb_iseof (c)) |
|
|
return; |
|
|
|
|
|
- if (putchar (c) < 0) |
|
|
+ mb_putc (c, stdout); |
|
|
+ if (ferror (stdout)) |
|
|
die (EXIT_FAILURE, errno, _("write error")); |
|
|
} |
|
|
- while (c != '\n'); |
|
|
+ while (!mb_iseq (c, '\n')); |
|
|
} |
|
|
} |
|
|
|
|
|
diff --git a/src/local.mk b/src/local.mk |
|
|
index 72db9c704..ef3bfa469 100644 |
|
|
--- a/src/local.mk |
|
|
+++ b/src/local.mk |
|
|
@@ -415,8 +415,8 @@ src_basenc_CPPFLAGS = -DBASE_TYPE=42 $(AM_CPPFLAGS) |
|
|
|
|
|
src_ginstall_CPPFLAGS = -DENABLE_MATCHPATHCON=1 $(AM_CPPFLAGS) |
|
|
|
|
|
-src_expand_SOURCES = src/expand.c src/expand-common.c |
|
|
-src_unexpand_SOURCES = src/unexpand.c src/expand-common.c |
|
|
+src_expand_SOURCES = src/expand.c src/expand-common.c lib/mbfile.c |
|
|
+src_unexpand_SOURCES = src/unexpand.c src/expand-common.c lib/mbfile.c |
|
|
|
|
|
# Ensure we don't link against libcoreutils.a as that lib is |
|
|
# not compiled with -fPIC which causes issues on 64 bit at least |
|
|
diff --git a/src/unexpand.c b/src/unexpand.c |
|
|
index 7801274..569a7ee 100644 |
|
|
--- a/src/unexpand.c |
|
|
+++ b/src/unexpand.c |
|
|
@@ -38,6 +38,9 @@ |
|
|
#include <stdio.h> |
|
|
#include <getopt.h> |
|
|
#include <sys/types.h> |
|
|
+ |
|
|
+#include <mbfile.h> |
|
|
+ |
|
|
#include "system.h" |
|
|
#include "die.h" |
|
|
#include "xstrndup.h" |
|
|
@@ -107,11 +110,12 @@ unexpand (void) |
|
|
{ |
|
|
/* Input stream. */ |
|
|
FILE *fp = next_file (NULL); |
|
|
+ mb_file_t mbf; |
|
|
|
|
|
/* The array of pending blanks. In non-POSIX locales, blanks can |
|
|
include characters other than spaces, so the blanks must be |
|
|
stored, not merely counted. */ |
|
|
- char *pending_blank; |
|
|
+ mbf_char_t *pending_blank; |
|
|
|
|
|
if (!fp) |
|
|
return; |
|
|
@@ -119,12 +123,14 @@ unexpand (void) |
|
|
/* The worst case is a non-blank character, then one blank, then a |
|
|
tab stop, then MAX_COLUMN_WIDTH - 1 blanks, then a non-blank; so |
|
|
allocate MAX_COLUMN_WIDTH bytes to store the blanks. */ |
|
|
- pending_blank = xmalloc (max_column_width); |
|
|
+ pending_blank = xmalloc (max_column_width * sizeof (mbf_char_t)); |
|
|
+ |
|
|
+ mbf_init (mbf, fp); |
|
|
|
|
|
while (true) |
|
|
{ |
|
|
/* Input character, or EOF. */ |
|
|
- int c; |
|
|
+ mbf_char_t c; |
|
|
|
|
|
/* If true, perform translations. */ |
|
|
bool convert = true; |
|
|
@@ -158,12 +164,19 @@ unexpand (void) |
|
|
|
|
|
do |
|
|
{ |
|
|
- while ((c = getc (fp)) < 0 && (fp = next_file (fp))) |
|
|
- continue; |
|
|
+ do { |
|
|
+ mbf_getc (c, mbf); |
|
|
+ if (mb_iseof (c)) |
|
|
+ { |
|
|
+ mbf_init (mbf, fp = next_file (fp)); |
|
|
+ continue; |
|
|
+ } |
|
|
+ } |
|
|
+ while (false); |
|
|
|
|
|
if (convert) |
|
|
{ |
|
|
- bool blank = !! isblank (c); |
|
|
+ bool blank = mb_isblank (c); |
|
|
|
|
|
if (blank) |
|
|
{ |
|
|
@@ -180,16 +193,16 @@ unexpand (void) |
|
|
if (next_tab_column < column) |
|
|
die (EXIT_FAILURE, 0, _("input line is too long")); |
|
|
|
|
|
- if (c == '\t') |
|
|
+ if (mb_iseq (c, '\t')) |
|
|
{ |
|
|
column = next_tab_column; |
|
|
|
|
|
if (pending) |
|
|
- pending_blank[0] = '\t'; |
|
|
+ mb_setascii (&pending_blank[0], '\t'); |
|
|
} |
|
|
else |
|
|
{ |
|
|
- column++; |
|
|
+ column += mb_width (c); |
|
|
|
|
|
if (! (prev_blank && column == next_tab_column)) |
|
|
{ |
|
|
@@ -197,13 +210,14 @@ unexpand (void) |
|
|
will be replaced by tabs. */ |
|
|
if (column == next_tab_column) |
|
|
one_blank_before_tab_stop = true; |
|
|
- pending_blank[pending++] = c; |
|
|
+ mb_copy (&pending_blank[pending++], &c); |
|
|
prev_blank = true; |
|
|
continue; |
|
|
} |
|
|
|
|
|
/* Replace the pending blanks by a tab or two. */ |
|
|
- pending_blank[0] = c = '\t'; |
|
|
+ mb_setascii (&c, '\t'); |
|
|
+ mb_setascii (&pending_blank[0], '\t'); |
|
|
} |
|
|
|
|
|
/* Discard pending blanks, unless it was a single |
|
|
@@ -211,7 +225,7 @@ unexpand (void) |
|
|
pending = one_blank_before_tab_stop; |
|
|
} |
|
|
} |
|
|
- else if (c == '\b') |
|
|
+ else if (mb_iseq (c, '\b')) |
|
|
{ |
|
|
/* Go back one column, and force recalculation of the |
|
|
next tab stop. */ |
|
|
@@ -221,7 +235,7 @@ unexpand (void) |
|
|
} |
|
|
else |
|
|
{ |
|
|
- column++; |
|
|
+ column += mb_width (c); |
|
|
if (!column) |
|
|
die (EXIT_FAILURE, 0, _("input line is too long")); |
|
|
} |
|
|
@@ -229,8 +243,11 @@ unexpand (void) |
|
|
if (pending) |
|
|
{ |
|
|
if (pending > 1 && one_blank_before_tab_stop) |
|
|
- pending_blank[0] = '\t'; |
|
|
- if (fwrite (pending_blank, 1, pending, stdout) != pending) |
|
|
+ mb_setascii (&pending_blank[0], '\t'); |
|
|
+ |
|
|
+ for (int n = 0; n < pending; ++n) |
|
|
+ mb_putc (pending_blank[n], stdout); |
|
|
+ if (ferror (stdout)) |
|
|
die (EXIT_FAILURE, errno, _("write error")); |
|
|
pending = 0; |
|
|
one_blank_before_tab_stop = false; |
|
|
@@ -240,16 +257,17 @@ unexpand (void) |
|
|
convert &= convert_entire_line || blank; |
|
|
} |
|
|
|
|
|
- if (c < 0) |
|
|
+ if (mb_iseof (c)) |
|
|
{ |
|
|
free (pending_blank); |
|
|
return; |
|
|
} |
|
|
|
|
|
- if (putchar (c) < 0) |
|
|
+ mb_putc (c, stdout); |
|
|
+ if (ferror (stdout)) |
|
|
die (EXIT_FAILURE, errno, _("write error")); |
|
|
} |
|
|
- while (c != '\n'); |
|
|
+ while (!mb_iseq (c, '\n')); |
|
|
} |
|
|
} |
|
|
|
|
|
diff --git a/tests/expand/mb.sh b/tests/expand/mb.sh |
|
|
new file mode 100755 |
|
|
index 0000000..7971e18 |
|
|
--- /dev/null |
|
|
+++ b/tests/expand/mb.sh |
|
|
@@ -0,0 +1,98 @@ |
|
|
+#!/bin/sh |
|
|
+ |
|
|
+# Copyright (C) 2012-2015 Free Software Foundation, Inc. |
|
|
+ |
|
|
+# This program is free software: you can redistribute it and/or modify |
|
|
+# it under the terms of the GNU General Public License as published by |
|
|
+# the Free Software Foundation, either version 3 of the License, or |
|
|
+# (at your option) any later version. |
|
|
+ |
|
|
+# This program is distributed in the hope that it will be useful, |
|
|
+# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
+# GNU General Public License for more details. |
|
|
+ |
|
|
+# You should have received a copy of the GNU General Public License |
|
|
+# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
|
+ |
|
|
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src |
|
|
+print_ver_ expand |
|
|
+ |
|
|
+export LC_ALL=en_US.UTF-8 |
|
|
+ |
|
|
+#input containing multibyte characters |
|
|
+cat <<\EOF > in || framework_failure_ |
|
|
+1234567812345678123456781 |
|
|
+. . . . |
|
|
+a b c d |
|
|
+. . . . |
|
|
+ä ö ü ß |
|
|
+. . . . |
|
|
+EOF |
|
|
+env printf ' äöü\t. öüä. \tä xx\n' >> in || framework_failure_ |
|
|
+ |
|
|
+cat <<\EOF > exp || framework_failure_ |
|
|
+1234567812345678123456781 |
|
|
+. . . . |
|
|
+a b c d |
|
|
+. . . . |
|
|
+ä ö ü ß |
|
|
+. . . . |
|
|
+ äöü . öüä. ä xx |
|
|
+EOF |
|
|
+ |
|
|
+expand < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+#test characters with display widths != 1 |
|
|
+env printf '12345678 |
|
|
+e\t|ascii(1) |
|
|
+\u00E9\t|composed(1) |
|
|
+e\u0301\t|decomposed(1) |
|
|
+\u3000\t|ideo-space(2) |
|
|
+\uFF0D\t|full-hypen(2) |
|
|
+' > in || framework_failure_ |
|
|
+ |
|
|
+env printf '12345678 |
|
|
+e |ascii(1) |
|
|
+\u00E9 |composed(1) |
|
|
+e\u0301 |decomposed(1) |
|
|
+\u3000 |ideo-space(2) |
|
|
+\uFF0D |full-hypen(2) |
|
|
+' > exp || framework_failure_ |
|
|
+ |
|
|
+expand < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+#shouldn't fail with "input line too long" |
|
|
+#when a line starts with a control character |
|
|
+env printf '\n' > in || framework_failure_ |
|
|
+ |
|
|
+expand < in > out || fail=1 |
|
|
+compare in out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+#non-Unicode characters interspersed between Unicode ones |
|
|
+env printf '12345678 |
|
|
+\t\xFF| |
|
|
+\xFF\t| |
|
|
+\t\xFFä| |
|
|
+ä\xFF\t| |
|
|
+\tä\xFF| |
|
|
+\xFF\tä| |
|
|
+äbcdef\xFF\t| |
|
|
+' > in || framework_failure_ |
|
|
+ |
|
|
+env printf '12345678 |
|
|
+ \xFF| |
|
|
+\xFF | |
|
|
+ \xFFä| |
|
|
+ä\xFF | |
|
|
+ ä\xFF| |
|
|
+\xFF ä| |
|
|
+äbcdef\xFF | |
|
|
+' > exp || framework_failure_ |
|
|
+ |
|
|
+expand < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+exit $fail |
|
|
diff --git a/tests/local.mk b/tests/local.mk |
|
|
index 192f776..8053397 100644 |
|
|
--- a/tests/local.mk |
|
|
+++ b/tests/local.mk |
|
|
@@ -544,6 +544,7 @@ all_tests = \ |
|
|
tests/du/threshold.sh \ |
|
|
tests/du/trailing-slash.sh \ |
|
|
tests/du/two-args.sh \ |
|
|
+ tests/expand/mb.sh \ |
|
|
tests/id/gnu-zero-uids.sh \ |
|
|
tests/id/no-context.sh \ |
|
|
tests/id/context.sh \ |
|
|
@@ -684,6 +685,7 @@ all_tests = \ |
|
|
tests/touch/read-only.sh \ |
|
|
tests/touch/relative.sh \ |
|
|
tests/touch/trailing-slash.sh \ |
|
|
+ tests/unexpand/mb.sh \ |
|
|
$(all_root_tests) |
|
|
|
|
|
# See tests/factor/create-test.sh. |
|
|
diff --git a/tests/unexpand/mb.sh b/tests/unexpand/mb.sh |
|
|
new file mode 100755 |
|
|
index 0000000..60d4c1a |
|
|
--- /dev/null |
|
|
+++ b/tests/unexpand/mb.sh |
|
|
@@ -0,0 +1,97 @@ |
|
|
+#!/bin/sh |
|
|
+ |
|
|
+# Copyright (C) 2012-2015 Free Software Foundation, Inc. |
|
|
+ |
|
|
+# This program is free software: you can redistribute it and/or modify |
|
|
+# it under the terms of the GNU General Public License as published by |
|
|
+# the Free Software Foundation, either version 3 of the License, or |
|
|
+# (at your option) any later version. |
|
|
+ |
|
|
+# This program is distributed in the hope that it will be useful, |
|
|
+# but WITHOUT ANY WARRANTY; without even the implied warranty of |
|
|
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|
|
+# GNU General Public License for more details. |
|
|
+ |
|
|
+# You should have received a copy of the GNU General Public License |
|
|
+# along with this program. If not, see <http://www.gnu.org/licenses/>. |
|
|
+ |
|
|
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src |
|
|
+print_ver_ unexpand |
|
|
+ |
|
|
+export LC_ALL=en_US.UTF-8 |
|
|
+ |
|
|
+#input containing multibyte characters |
|
|
+cat > in <<\EOF |
|
|
+1234567812345678123456781 |
|
|
+. . . . |
|
|
+a b c d |
|
|
+. . . . |
|
|
+ä ö ü ß |
|
|
+. . . . |
|
|
+ äöü . öüä. ä xx |
|
|
+EOF |
|
|
+ |
|
|
+cat > exp <<\EOF |
|
|
+1234567812345678123456781 |
|
|
+. . . . |
|
|
+a b c d |
|
|
+. . . . |
|
|
+ä ö ü ß |
|
|
+. . . . |
|
|
+ äöü . öüä. ä xx |
|
|
+EOF |
|
|
+ |
|
|
+unexpand -a < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+#test characters with a display width larger than 1 |
|
|
+ |
|
|
+env printf '12345678 |
|
|
+e |ascii(1) |
|
|
+\u00E9 |composed(1) |
|
|
+e\u0301 |decomposed(1) |
|
|
+\u3000 |ideo-space(2) |
|
|
+\uFF0D |full-hypen(2) |
|
|
+' > in || framework_failure_ |
|
|
+ |
|
|
+env printf '12345678 |
|
|
+e\t|ascii(1) |
|
|
+\u00E9\t|composed(1) |
|
|
+e\u0301\t|decomposed(1) |
|
|
+\u3000\t|ideo-space(2) |
|
|
+\uFF0D\t|full-hypen(2) |
|
|
+' > exp || framework_failure_ |
|
|
+ |
|
|
+unexpand -a < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+#test input where a blank of width > 1 is not being substituted |
|
|
+in="$(LC_ALL=en_US.UTF-8 printf ' \u3000 ö ü ß')" |
|
|
+exp=' ö ü ß' |
|
|
+ |
|
|
+unexpand -a < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
+ |
|
|
+#non-Unicode characters interspersed between Unicode ones |
|
|
+env printf '12345678 |
|
|
+ \xFF| |
|
|
+\xFF | |
|
|
+ \xFFä| |
|
|
+ä\xFF | |
|
|
+ ä\xFF| |
|
|
+\xFF ä| |
|
|
+äbcdef\xFF | |
|
|
+' > in || framework_failure_ |
|
|
+ |
|
|
+env printf '12345678 |
|
|
+\t\xFF| |
|
|
+\xFF\t| |
|
|
+\t\xFFä| |
|
|
+ä\xFF\t| |
|
|
+\tä\xFF| |
|
|
+\xFF\tä| |
|
|
+äbcdef\xFF\t| |
|
|
+' > exp || framework_failure_ |
|
|
+ |
|
|
+unexpand -a < in > out || fail=1 |
|
|
+compare exp out > /dev/null 2>&1 || fail=1 |
|
|
-- |
|
|
2.7.4 |
|
|
|
|
|
|