[ovs-dev] [PATCH v3] python: Fix decoding error when the received data is larger than 4096.

Guoshuai Li ligs at dtdream.com
Mon Feb 26 05:45:54 UTC 2018


It can only receive 4096 bytes of data each time in jsonrpc,
when there are similar and Chinese characters occupy multiple bytes,
it may receive half a character, this time the decoding will be abnormal.
We need to receive the completed character to decode.

Change-Id: Ie20c2b4632825fa8dfc60586e9b6d51a03355f4c
Signed-off-by: Guoshuai Li <ligs at dtdream.com>
---
 python/ovs/jsonrpc.py | 16 +++++++++++-----
 tests/ovsdb-idl.at    | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+), 5 deletions(-)

diff --git a/python/ovs/jsonrpc.py b/python/ovs/jsonrpc.py
index d284190e0..95e7837af 100644
--- a/python/ovs/jsonrpc.py
+++ b/python/ovs/jsonrpc.py
@@ -262,16 +262,22 @@ class Connection(object):
         if self.status:
             return self.status, None
 
+        recv_buffer = bytearray()
         while True:
             if not self.input:
+                recv_data = None
                 error, data = self.stream.recv(4096)
+                recv_buffer.extend(bytes(data))
                 # Python 3 has separate types for strings and bytes.  We
                 # received bytes from a socket.  We expect it to be string
                 # data, so we convert it here as soon as possible.
-                if data and not error:
+                if recv_buffer and not error:
                     try:
-                        data = data.decode('utf-8')
+                        recv_data = recv_buffer.decode('utf-8')
+                        recv_buffer = bytearray()
                     except UnicodeError:
+                        if len(recv_buffer) % 4096 == 0:
+                            continue
                         error = errno.EILSEQ
                 if error:
                     if (sys.platform == "win32" and
@@ -287,12 +293,12 @@ class Connection(object):
                                   % (self.name, os.strerror(error)))
                         self.error(error)
                         return self.status, None
-                elif not data:
+                elif not recv_data:
                     self.error(EOF)
                     return EOF, None
                 else:
-                    self.input += data
-                    self.received_bytes += len(data)
+                    self.input += recv_data
+                    self.received_bytes += len(recv_data)
             else:
                 if self.parser is None:
                     self.parser = ovs.json.Parser()
diff --git a/tests/ovsdb-idl.at b/tests/ovsdb-idl.at
index 59b2c1991..d63b7758f 100644
--- a/tests/ovsdb-idl.at
+++ b/tests/ovsdb-idl.at
@@ -328,6 +328,40 @@ OVSDB_CHECK_IDL([simple idl, writing via IDL with unicode],
 003: done
 ]])
 
+# same as OVSDB_CHECK_IDL but uses the Python IDL implementation.
+m4_define([OVSDB_CHECK_IDL_PYN_WITH_EXPOUT],
+  [AT_SETUP([$1])
+   AT_SKIP_IF([test $7 = no])
+   AT_KEYWORDS([ovsdb server idl positive Python $5])
+   AT_CHECK([ovsdb_start_idltest])
+   m4_if([$2], [], [],
+     [AT_CHECK([ovsdb-client transact unix:socket $2], [0], [ignore], [ignore])])
+   AT_CHECK([$8 $srcdir/test-ovsdb.py  -t10 idl $srcdir/idltest.ovsschema unix:socket $3],
+            [0], [stdout], [ignore])
+   echo "$4" > expout
+   AT_CHECK([sort stdout | uuidfilt]m4_if([$6],,, [[| $6]]),
+            [0], [expout])
+   OVSDB_SERVER_SHUTDOWN
+   AT_CLEANUP])
+
+m4_define([OVSDB_CHECK_IDL_PY_WITH_EXPOUT],
+   [OVSDB_CHECK_IDL_PYN_WITH_EXPOUT([$1 - Python2], [$2], [$3], [$4], [$5], [$6],
+                                    [$HAVE_PYTHON], [$PYTHON])
+    OVSDB_CHECK_IDL_PYN_WITH_EXPOUT([$1 - Python3], [$2], [$3], [$4], [$5], [$6],
+                                    [$HAVE_PYTHON3], [$PYTHON3])])
+
+OVSDB_CHECK_IDL_PY_WITH_EXPOUT([simple idl, writing large data via IDL with unicode],
+  [['["idltest",
+      {"op": "insert",
+       "table": "simple",
+       "row": {"s": "'$(printf "测试超过四千零九十六个字节的中文字符串以使解码出现问题。%.0s" {1..50})'"}}]']],
+  [['set 0 b 1, insert 1, set 1 s '$(printf "测试超过四千零九十六个字节的中文字符串以使解码出现问题。%.0s" {1..100})'']],
+  [[000: i=0 r=0 b=false s=$(printf "测试超过四千零九十六个字节的中文字符串以使解码出现问题。%.0s" {1..50}) u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1>
+001: commit, status=success
+002: i=0 r=0 b=true s=$(printf "测试超过四千零九十六个字节的中文字符串以使解码出现问题。%.0s" {1..50}) u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<1>
+002: i=1 r=0 b=false s=$(printf "测试超过四千零九十六个字节的中文字符串以使解码出现问题。%.0s" {1..100}) u=<0> ia=[] ra=[] ba=[] sa=[] ua=[] uuid=<2>
+003: done]])
+
 OVSDB_CHECK_IDL([simple idl, handling verification failure],
   [['["idltest",
       {"op": "insert",
-- 
2.13.2.windows.1



More information about the dev mailing list