subversion-1.9.7HEADsubversion-1.9.7master

1 files changed, 530 insertions, 0 deletions

diff --git a/subversion/libsvn_subr/utf8proc.c b/subversion/libsvn_subr/utf8proc.c
new file mode 100644
index 0000000..1e705f5
--- /dev/null
+++ b/subversion/libsvn_subr/utf8proc.c
@@ -0,0 +1,530 @@
+/*
+ * utf8proc.c:  Wrappers for the utf8proc library
+ *
+ * ====================================================================
+ *    Licensed to the Apache Software Foundation (ASF) under one
+ *    or more contributor license agreements.  See the NOTICE file
+ *    distributed with this work for additional information
+ *    regarding copyright ownership.  The ASF licenses this file
+ *    to you under the Apache License, Version 2.0 (the
+ *    "License"); you may not use this file except in compliance
+ *    with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing,
+ *    software distributed under the License is distributed on an
+ *    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *    KIND, either express or implied.  See the License for the
+ *    specific language governing permissions and limitations
+ *    under the License.
+ * ====================================================================
+ */
+
+
+
+#include <apr_fnmatch.h>
+
+#include "private/svn_string_private.h"
+#include "private/svn_utf_private.h"
+#include "svn_private_config.h"
+
+#define UTF8PROC_INLINE
+/* Somehow utf8proc thinks it is nice to use strlen as an argument name,
+   while this function is already defined via apr.h */
+#define strlen svn__strlen_var
+#include "utf8proc/utf8proc.c"
+#undef strlen
+
+
+
+const char *
+svn_utf__utf8proc_compiled_version(void)
+{
+  static const char utf8proc_version[] =
+                                  APR_STRINGIFY(UTF8PROC_VERSION_MAJOR) "."
+                                  APR_STRINGIFY(UTF8PROC_VERSION_MINOR) "."
+                                  APR_STRINGIFY(UTF8PROC_VERSION_PATCH);
+  return utf8proc_version;
+}
+
+const char *
+svn_utf__utf8proc_runtime_version(void)
+{
+  /* Unused static function warning removal hack. */
+  SVN_UNUSED(utf8proc_NFD);
+  SVN_UNUSED(utf8proc_NFC);
+  SVN_UNUSED(utf8proc_NFKD);
+  SVN_UNUSED(utf8proc_NFKC);
+
+  return utf8proc_version();
+}
+
+
+
+/* Fill the given BUFFER with decomposed UCS-4 representation of the
+ * UTF-8 STRING. If LENGTH is SVN_UTF__UNKNOWN_LENGTH, assume STRING
+ * is NUL-terminated; otherwise look only at the first LENGTH bytes in
+ * STRING. Upon return, BUFFER->data points at an array of UCS-4
+ * characters, and return the length of the array. TRANSFORM_FLAGS
+ * define exactly how the decomposition is performed.
+ *
+ * A negative return value is an utf8proc error code and may indicate
+ * that STRING contains invalid UTF-8 or was so long that an overflow
+ * occurred.
+ */
+static ssize_t
+unicode_decomposition(int transform_flags,
+                      const char *string, apr_size_t length,
+                      svn_membuf_t *buffer)
+{
+  const int nullterm = (length == SVN_UTF__UNKNOWN_LENGTH
+                        ? UTF8PROC_NULLTERM : 0);
+
+  for (;;)
+    {
+      apr_int32_t *const ucs4buf = buffer->data;
+      const ssize_t ucs4len = buffer->size / sizeof(*ucs4buf);
+      const ssize_t result =
+        utf8proc_decompose((const void*) string, length, ucs4buf, ucs4len,
+                           UTF8PROC_DECOMPOSE | UTF8PROC_STABLE
+                           | transform_flags | nullterm);
+
+      if (result < 0 || result <= ucs4len)
+        return result;
+
+      /* Increase the decomposition buffer size and retry */
+      svn_membuf__ensure(buffer, result * sizeof(*ucs4buf));
+    }
+}
+
+/* Fill the given BUFFER with an NFD UCS-4 representation of the UTF-8
+ * STRING. If LENGTH is SVN_UTF__UNKNOWN_LENGTH, assume STRING is
+ * NUL-terminated; otherwise look only at the first LENGTH bytes in
+ * STRING. Upon return, BUFFER->data points at an array of UCS-4
+ * characters and *RESULT_LENGTH contains the length of the array.
+ *
+ * A returned error may indicate that STRING contains invalid UTF-8 or
+ * invalid Unicode codepoints. Any error message comes from utf8proc.
+ */
+static svn_error_t *
+decompose_normalized(apr_size_t *result_length,
+                     const char *string, apr_size_t length,
+                     svn_membuf_t *buffer)
+{
+  ssize_t result = unicode_decomposition(0, string, length, buffer);
+  if (result < 0)
+    return svn_error_create(SVN_ERR_UTF8PROC_ERROR, NULL,
+                            gettext(utf8proc_errmsg(result)));
+  *result_length = result;
+  return SVN_NO_ERROR;
+}
+
+/* Fill the given BUFFER with an NFC UTF-8 representation of the UTF-8
+ * STRING. If LENGTH is SVN_UTF__UNKNOWN_LENGTH, assume STRING is
+ * NUL-terminated; otherwise look only at the first LENGTH bytes in
+ * STRING. Upon return, BUFFER->data points at a NUL-terminated string
+ * of UTF-8 characters.
+ *
+ * A returned error may indicate that STRING contains invalid UTF-8 or
+ * invalid Unicode codepoints. Any error message comes from utf8proc.
+ */
+static svn_error_t *
+normalize_cstring(apr_size_t *result_length,
+                  const char *string, apr_size_t length,
+                  svn_membuf_t *buffer)
+{
+  ssize_t result = unicode_decomposition(0, string, length, buffer);
+  if (result >= 0)
+    {
+      svn_membuf__resize(buffer, result * sizeof(apr_int32_t) + 1);
+      result = utf8proc_reencode(buffer->data, result,
+                                 UTF8PROC_COMPOSE | UTF8PROC_STABLE);
+    }
+  if (result < 0)
+    return svn_error_create(SVN_ERR_UTF8PROC_ERROR, NULL,
+                            gettext(utf8proc_errmsg(result)));
+  *result_length = result;
+  return SVN_NO_ERROR;
+}
+
+/* Compare two arrays of UCS-4 codes, BUFA of length LENA and BUFB of
+ * length LENB. Return 0 if they're equal, a negative value if BUFA is
+ * less than BUFB, otherwise a positive value.
+ *
+ * Yes, this is strcmp for known-length UCS-4 strings.
+ */
+static int
+ucs4cmp(const apr_int32_t *bufa, apr_size_t lena,
+        const apr_int32_t *bufb, apr_size_t lenb)
+{
+  const apr_size_t len = (lena < lenb ? lena : lenb);
+  apr_size_t i;
+
+  for (i = 0; i < len; ++i)
+    {
+      const int diff = bufa[i] - bufb[i];
+      if (diff)
+        return diff;
+    }
+  return (lena == lenb ? 0 : (lena < lenb ? -1 : 1));
+}
+
+svn_error_t *
+svn_utf__normcmp(int *result,
+                 const char *str1, apr_size_t len1,
+                 const char *str2, apr_size_t len2,
+                 svn_membuf_t *buf1, svn_membuf_t *buf2)
+{
+  apr_size_t buflen1;
+  apr_size_t buflen2;
+
+  /* Shortcut-circuit the decision if at least one of the strings is empty. */
+  const svn_boolean_t empty1 =
+    (0 == len1 || (len1 == SVN_UTF__UNKNOWN_LENGTH && !*str1));
+  const svn_boolean_t empty2 =
+    (0 == len2 || (len2 == SVN_UTF__UNKNOWN_LENGTH && !*str2));
+  if (empty1 || empty2)
+    {
+      *result = (empty1 == empty2 ? 0 : (empty1 ? -1 : 1));
+      return SVN_NO_ERROR;
+    }
+
+  SVN_ERR(decompose_normalized(&buflen1, str1, len1, buf1));
+  SVN_ERR(decompose_normalized(&buflen2, str2, len2, buf2));
+  *result = ucs4cmp(buf1->data, buflen1, buf2->data, buflen2);
+  return SVN_NO_ERROR;
+}
+
+svn_error_t*
+svn_utf__normalize(const char **result,
+                   const char *str, apr_size_t len,
+                   svn_membuf_t *buf)
+{
+  apr_size_t result_length;
+  SVN_ERR(normalize_cstring(&result_length, str, len, buf));
+  *result = (const char*)(buf->data);
+  return SVN_NO_ERROR;
+}
+
+/* Decode a single UCS-4 code point to UTF-8, appending the result to BUFFER.
+ * Assume BUFFER is already filled to *LENGTH and return the new size there.
+ * This function does *not* nul-terminate the stringbuf!
+ *
+ * A returned error indicates that the codepoint is invalid.
+ */
+static svn_error_t *
+encode_ucs4(svn_membuf_t *buffer, apr_int32_t ucs4chr, apr_size_t *length)
+{
+  apr_size_t utf8len;
+
+  if (buffer->size - *length < 4)
+    svn_membuf__resize(buffer, buffer->size + 4);
+
+  utf8len = utf8proc_encode_char(ucs4chr, ((uint8_t*)buffer->data + *length));
+  if (!utf8len)
+    return svn_error_createf(SVN_ERR_UTF8PROC_ERROR, NULL,
+                             _("Invalid Unicode character U+%04lX"),
+                             (long)ucs4chr);
+  *length += utf8len;
+  return SVN_NO_ERROR;
+}
+
+svn_error_t *
+svn_utf__encode_ucs4_string(svn_membuf_t *buffer,
+                            const apr_int32_t *ucs4str,
+                            apr_size_t length,
+                            apr_size_t *result_length)
+{
+  *result_length = 0;
+  while (length-- > 0)
+    SVN_ERR(encode_ucs4(buffer, *ucs4str++, result_length));
+  svn_membuf__resize(buffer, *result_length + 1);
+  ((char*)buffer->data)[*result_length] = '\0';
+  return SVN_NO_ERROR;
+}
+
+
+svn_error_t *
+svn_utf__glob(svn_boolean_t *match,
+              const char *pattern, apr_size_t pattern_len,
+              const char *string, apr_size_t string_len,
+              const char *escape, apr_size_t escape_len,
+              svn_boolean_t sql_like,
+              svn_membuf_t *pattern_buf,
+              svn_membuf_t *string_buf,
+              svn_membuf_t *temp_buf)
+{
+  apr_size_t patternbuf_len;
+  apr_size_t tempbuf_len;
+
+  /* If we're in GLOB mode, we don't do custom escape chars. */
+  if (escape && !sql_like)
+    return svn_error_create(SVN_ERR_UTF8_GLOB, NULL,
+                            _("Cannot use a custom escape token"
+                              " in glob matching mode"));
+
+  /* Convert the patern to NFD UTF-8. We can't use the UCS-4 result
+     because apr_fnmatch can't handle it.*/
+  SVN_ERR(decompose_normalized(&tempbuf_len, pattern, pattern_len, temp_buf));
+  if (!sql_like)
+    SVN_ERR(svn_utf__encode_ucs4_string(pattern_buf, temp_buf->data,
+                                        tempbuf_len, &patternbuf_len));
+  else
+    {
+      /* Convert a LIKE pattern to a GLOB pattern that apr_fnmatch can use. */
+      const apr_int32_t *like = temp_buf->data;
+      apr_int32_t ucs4esc;
+      svn_boolean_t escaped;
+      apr_size_t i;
+
+      if (!escape)
+        ucs4esc = -1;           /* Definitely an invalid UCS-4 character. */
+      else
+        {
+          const int nullterm = (escape_len == SVN_UTF__UNKNOWN_LENGTH
+                                ? UTF8PROC_NULLTERM : 0);
+          ssize_t result =
+            utf8proc_decompose((const void*) escape, escape_len, &ucs4esc, 1,
+                               UTF8PROC_DECOMPOSE | UTF8PROC_STABLE | nullterm);
+          if (result < 0)
+            return svn_error_create(SVN_ERR_UTF8PROC_ERROR, NULL,
+                                    gettext(utf8proc_errmsg(result)));
+          if (result == 0 || result > 1)
+            return svn_error_create(SVN_ERR_UTF8_GLOB, NULL,
+                                    _("Escape token must be one character"));
+          if ((ucs4esc & 0xFF) != ucs4esc)
+            return svn_error_createf(SVN_ERR_UTF8_GLOB, NULL,
+                                     _("Invalid escape character U+%04lX"),
+                                     (long)ucs4esc);
+        }
+
+      patternbuf_len = 0;
+      svn_membuf__ensure(pattern_buf, tempbuf_len + 1);
+      for (i = 0, escaped = FALSE; i < tempbuf_len; ++i, ++like)
+        {
+          if (*like == ucs4esc && !escaped)
+            {
+              svn_membuf__resize(pattern_buf, patternbuf_len + 1);
+              ((char*)pattern_buf->data)[patternbuf_len++] = '\\';
+              escaped = TRUE;
+            }
+          else if (escaped)
+            {
+              SVN_ERR(encode_ucs4(pattern_buf, *like, &patternbuf_len));
+              escaped = FALSE;
+            }
+          else
+            {
+              if ((*like == '[' || *like == '\\') && !escaped)
+                {
+                  /* Escape brackets and backslashes which are always
+                     literals in LIKE patterns. */
+                  svn_membuf__resize(pattern_buf, patternbuf_len + 1);
+                  ((char*)pattern_buf->data)[patternbuf_len++] = '\\';
+                  escaped = TRUE;
+                  --i; --like;
+                  continue;
+                }
+
+              /* Replace LIKE wildcards with their GLOB equivalents. */
+              if (*like == '%' || *like == '_')
+                {
+                  const char wildcard = (*like == '%' ? '*' : '?');
+                  svn_membuf__resize(pattern_buf, patternbuf_len + 1);
+                  ((char*)pattern_buf->data)[patternbuf_len++] = wildcard;
+                }
+              else
+                SVN_ERR(encode_ucs4(pattern_buf, *like, &patternbuf_len));
+            }
+        }
+      svn_membuf__resize(pattern_buf, patternbuf_len + 1);
+      ((char*)pattern_buf->data)[patternbuf_len] = '\0';
+    }
+
+  /* Now normalize the string */
+  SVN_ERR(decompose_normalized(&tempbuf_len, string, string_len, temp_buf));
+  SVN_ERR(svn_utf__encode_ucs4_string(string_buf, temp_buf->data,
+                                      tempbuf_len, &tempbuf_len));
+
+  *match = !apr_fnmatch(pattern_buf->data, string_buf->data, 0);
+  return SVN_NO_ERROR;
+}
+
+svn_boolean_t
+svn_utf__is_normalized(const char *string, apr_pool_t *scratch_pool)
+{
+  svn_error_t *err;
+  svn_membuf_t buffer;
+  apr_size_t result_length;
+  const apr_size_t length = strlen(string);
+  svn_membuf__create(&buffer, length * sizeof(apr_int32_t), scratch_pool);
+  err = normalize_cstring(&result_length, string, length, &buffer);
+  if (err)
+    {
+      svn_error_clear(err);
+      return FALSE;
+    }
+  return (length == result_length && 0 == strcmp(string, buffer.data));
+}
+
+const char *
+svn_utf__fuzzy_escape(const char *src, apr_size_t length, apr_pool_t *pool)
+{
+  /* Hexadecimal digits for code conversion. */
+  static const char digits[] = "0123456789ABCDEF";
+
+  /* Flags used for Unicode decomposition. */
+  static const int decomp_flags = (
+      UTF8PROC_COMPAT | UTF8PROC_STABLE | UTF8PROC_LUMP
+      | UTF8PROC_NLF2LF | UTF8PROC_STRIPCC | UTF8PROC_STRIPMARK);
+
+  svn_stringbuf_t *result;
+  svn_membuf_t buffer;
+  ssize_t decomp_length;
+  ssize_t len;
+
+  /* Decompose to a non-reversible compatibility format. */
+  svn_membuf__create(&buffer, length * sizeof(apr_int32_t), pool);
+  decomp_length = unicode_decomposition(decomp_flags, src, length, &buffer);
+  if (decomp_length < 0)
+    {
+      svn_membuf_t part;
+      apr_size_t done, prev;
+
+      /* The only other error we can receive here indicates an integer
+         overflow due to the length of the input string. Not very
+         likely, but we certainly shouldn't continue in that case. */
+      SVN_ERR_ASSERT_NO_RETURN(decomp_length == UTF8PROC_ERROR_INVALIDUTF8);
+
+      /* Break the decomposition into parts that are valid UTF-8, and
+         bytes that are not. Represent the invalid bytes in the target
+         erray by their negative value. This works because utf8proc
+         will not generate Unicode code points with values larger than
+         U+10FFFF. */
+      svn_membuf__create(&part, sizeof(apr_int32_t), pool);
+      decomp_length = 0;
+      done = prev = 0;
+      while (done < length)
+        {
+          apr_int32_t uc;
+
+          while (done < length)
+            {
+              len = utf8proc_iterate((uint8_t*)src + done, length - done, &uc);
+              if (len < 0)
+                break;
+              done += len;
+            }
+
+          /* Decompose the valid part */
+          if (done > prev)
+            {
+              len = unicode_decomposition(
+                  decomp_flags, src + prev, done - prev, &part);
+              SVN_ERR_ASSERT_NO_RETURN(len > 0);
+              svn_membuf__resize(
+                  &buffer, (decomp_length + len) * sizeof(apr_int32_t));
+              memcpy((apr_int32_t*)buffer.data + decomp_length,
+                     part.data, len * sizeof(apr_int32_t));
+              decomp_length += len;
+              prev = done;
+            }
+
+          /* What follows could be a valid UTF-8 sequence, but not
+             a valid Unicode character. */
+          if (done < length)
+            {
+              const char *last;
+
+              /* Determine the length of the UTF-8 sequence */
+              const char *const p = src + done;
+              len = utf8proc_utf8class[(uint8_t)*p];
+
+              /* Check if the multi-byte sequence is valid UTF-8. */
+              if (len > 1 && len <= (apr_ssize_t)(length - done))
+                last = svn_utf__last_valid(p, len);
+              else
+                last = NULL;
+
+              /* Might not be a valid UTF-8 sequence at all */
+              if (!last || (last && last - p < len))
+                {
+                  uc = -((apr_int32_t)(*p & 0xff));
+                  len = 1;
+                }
+              else
+                {
+                  switch (len)
+                    {
+                      /* Decode the UTF-8 sequence without validation. */
+                    case 2:
+                      uc = ((p[0] & 0x1f) <<  6) + (p[1] & 0x3f);
+                      break;
+                    case 3:
+                      uc = (((p[0] & 0x0f) << 12) + ((p[1] & 0x3f) <<  6)
+                            + (p[2] & 0x3f));
+                      break;
+                    case 4:
+                      uc = (((p[0] & 0x07) << 18) + ((p[1] & 0x3f) << 12)
+                            + ((p[2] & 0x3f) <<  6) + (p[3] & 0x3f));
+                      break;
+                    default:
+                      SVN_ERR_ASSERT_NO_RETURN(
+                          !"Unexpected invalid UTF-8 byte");
+                    }
+
+                }
+
+              svn_membuf__resize(
+                  &buffer, (decomp_length + 1) * sizeof(apr_int32_t));
+              ((apr_int32_t*)buffer.data)[decomp_length++] = uc;
+              done += len;
+              prev = done;
+            }
+        }
+    }
+
+  /* Scan the result and deleting any combining diacriticals and
+     inserting placeholders where any non-ascii characters remain.  */
+  result = svn_stringbuf_create_ensure(decomp_length, pool);
+  for (len = 0; len < decomp_length; ++len)
+    {
+      const apr_int32_t cp = ((apr_int32_t*)buffer.data)[len];
+      if (cp > 0 && cp < 127)
+        svn_stringbuf_appendbyte(result, (char)cp);
+      else if (cp == 0)
+        svn_stringbuf_appendcstr(result, "\\0");
+      else if (cp < 0)
+        {
+          const apr_int32_t rcp = ((-cp) & 0xff);
+          svn_stringbuf_appendcstr(result, "?\\");
+          svn_stringbuf_appendbyte(result, digits[(rcp & 0x00f0) >> 4]);
+          svn_stringbuf_appendbyte(result, digits[(rcp & 0x000f)]);
+        }
+      else
+        {
+          if (utf8proc_codepoint_valid(cp))
+            {
+              const utf8proc_property_t *prop = utf8proc_get_property(cp);
+              if (prop->combining_class != 0)
+                continue;           /* Combining mark; ignore */
+              svn_stringbuf_appendcstr(result, "{U+");
+            }
+          else
+            svn_stringbuf_appendcstr(result, "{U?");
+          if (cp > 0xffff)
+            {
+              svn_stringbuf_appendbyte(result, digits[(cp & 0xf00000) >> 20]);
+              svn_stringbuf_appendbyte(result, digits[(cp & 0x0f0000) >> 16]);
+            }
+          svn_stringbuf_appendbyte(result, digits[(cp & 0xf000) >> 12]);
+          svn_stringbuf_appendbyte(result, digits[(cp & 0x0f00) >> 8]);
+          svn_stringbuf_appendbyte(result, digits[(cp & 0x00f0) >> 4]);
+          svn_stringbuf_appendbyte(result, digits[(cp & 0x000f)]);
+          svn_stringbuf_appendbyte(result, '}');
+        }
+    }
+
+  return result->data;
+}