From 89cecbe8720011e87cd80d32bd8e3a5ef662b6c3 Mon Sep 17 00:00:00 2001 From: Ivan Chernetsky Date: Thu, 25 Jun 2015 20:03:27 +0000 Subject: [PATCH 1/4] Make mailbox_gen() generator compatible with Python 2.x --- erlastic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erlastic/__init__.py b/erlastic/__init__.py index 928a7d8..e584d23 100644 --- a/erlastic/__init__.py +++ b/erlastic/__init__.py @@ -15,7 +15,7 @@ def mailbox_gen(): while True: len_bin = sys.stdin.buffer.read(4) - if len(len_bin) != 4: return None + if len(len_bin) != 4: return (length,) = struct.unpack('!I',len_bin) yield decode(sys.stdin.buffer.read(length)) def port_gen(): From 0693ca83733985735f08024c63457d2ebae8cb40 Mon Sep 17 00:00:00 2001 From: Ivan Chernetsky Date: Thu, 25 Jun 2015 23:05:37 +0000 Subject: [PATCH 2/4] Add six as a dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index febc36c..ef8633e 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ author_email = 'samuel@descolada.com', url = 'http://github.com/samuel/python-erlastic', packages = ['erlastic'], + install_requires=['six'], classifiers = [ 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', From 7f64d5f47dca13452d64ecd321f17cdff700725d Mon Sep 17 00:00:00 2001 From: Ivan Chernetsky Date: Thu, 25 Jun 2015 23:08:56 +0000 Subject: [PATCH 3/4] Make erlastic compatible with Python 2.x with help of six --- erlastic/codec.py | 99 ++++++++++++++++++++++++---------------------- erlastic/compat.py | 10 +++++ tests.py | 44 +++++++++++---------- 3 files changed, 85 insertions(+), 68 deletions(-) create mode 100644 erlastic/compat.py diff --git a/erlastic/codec.py b/erlastic/codec.py index 3a284fe..60f67ae 100644 --- a/erlastic/codec.py +++ b/erlastic/codec.py @@ -1,9 +1,11 @@ from __future__ import division +import six import struct import zlib +from erlastic.compat import * from erlastic.constants import * from erlastic.types import * @@ -23,17 +25,17 @@ def __init__(self): except: pass def decode(self, buf, offset=0): - version = buf[offset] + version = six.indexbytes(buf, offset) if version != FORMAT_VERSION: raise EncodingError("Bad version number. Expected %d found %d" % (FORMAT_VERSION, version)) return self.decode_part(buf, offset+1)[0] def decode_part(self, buf, offset=0): - return self.decoders[buf[offset]](buf, offset+1) + return self.decoders[six.indexbytes(buf, offset)](buf, offset+1) def decode_97(self, buf, offset): """SMALL_INTEGER_EXT""" - return buf[offset], offset+1 + return six.indexbytes(buf, offset), offset+1 def decode_98(self, buf, offset): """INTEGER_EXT""" @@ -41,7 +43,7 @@ def decode_98(self, buf, offset): def decode_99(self, buf, offset): """FLOAT_EXT""" - return float(buf[offset:offset+31].split(b'\x00', 1)[0]), offset+31 + return float(buf[offset:offset+31].split(six.b('\x00'), 1)[0]), offset+31 def decode_70(self, buf, offset): """NEW_FLOAT_EXT""" @@ -55,13 +57,13 @@ def decode_100(self, buf, offset): def decode_115(self, buf, offset): """SMALL_ATOM_EXT""" - atom_len = buf[offset] + atom_len = six.intexbytes(buf, offset) atom = buf[offset+1:offset+1+atom_len] return self.convert_atom(atom), offset+atom_len+1 def decode_104(self, buf, offset): """SMALL_TUPLE_EXT""" - arity = buf[offset] + arity = six.indexbytes(buf, offset) offset += 1 items = [] @@ -112,7 +114,7 @@ def decode_109(self, buf, offset): def decode_110(self, buf, offset): """SMALL_BIG_EXT""" - n = buf[offset] + n = six.indexbytes(buf, offset) offset += 1 return self.decode_bigint(n, buf, offset) @@ -123,12 +125,12 @@ def decode_111(self, buf, offset): return self.decode_bigint(n, buf, offset) def decode_bigint(self, n, buf, offset): - sign = buf[offset] + sign = six.indexbytes(buf, offset) offset += 1 b = 1 val = 0 for i in range(n): - val += buf[offset] * b + val += six.indexbytes(buf, offset) * b b <<= 8 offset += 1 if sign != 0: @@ -149,7 +151,7 @@ def decode_114(self, buf, offset): node, offset = self.decode_part(buf, offset+2) if not isinstance(node, Atom): raise EncodingError("Expected atom while parsing NEW_REFERENCE_EXT, found %r instead" % node) - creation = buf[offset] + creation = six.indexbytes(buf, offset) reference_id = struct.unpack(">%dL" % id_len, buf[offset+1:offset+1+4*id_len]) return Reference(node, reference_id, creation), offset+1+4*id_len @@ -178,7 +180,7 @@ def decode_113(self, buf, offset): if not isinstance(function, Atom): raise EncodingError("Expected atom while parsing EXPORT_EXT, found %r instead" % function) arity, offset = self.decode_part(buf, offset) - if not isinstance(arity, int): + if not isinstance(arity, six.integer_types): raise EncodingError("Expected integer while parsing EXPORT_EXT, found %r instead" % arity) return Export(module, function, arity), offset+1 @@ -206,11 +208,11 @@ def encode(self, obj, compressed=False): import sys import pprint #pprint.pprint(self.encode_part(obj),stream=sys.stderr) - ubuf = b"".join(self.encode_part(obj)) + ubuf = six.b('').join(self.encode_part(obj)) if compressed is True: compressed = 6 if not (compressed is False \ - or (isinstance(compressed, int) \ + or (isinstance(compressed, six.integer_types) \ and compressed >= 0 and compressed <= 9)): raise TypeError("compressed must be True, False or " "an integer between 0 and 9") @@ -219,20 +221,20 @@ def encode(self, obj, compressed=False): if len(cbuf) < len(ubuf): usize = struct.pack(">L", len(ubuf)) ubuf = "".join([COMPRESSED, usize, cbuf]) - return bytes([FORMAT_VERSION]) + ubuf + return pack_bytes([FORMAT_VERSION]) + ubuf def encode_part(self, obj): if obj is False: - return [bytes([ATOM_EXT]), struct.pack(">H", 5), b"false"] + return [pack_bytes([ATOM_EXT]), struct.pack(">H", 5), b"false"] elif obj is True: - return [bytes([ATOM_EXT]), struct.pack(">H", 4), b"true"] + return [pack_bytes([ATOM_EXT]), struct.pack(">H", 4), b"true"] elif obj is None: - return [bytes([ATOM_EXT]), struct.pack(">H", 4), b"none"] - elif isinstance(obj, int): + return [pack_bytes([ATOM_EXT]), struct.pack(">H", 4), b"none"] + elif isinstance(obj, six.integer_types): if 0 <= obj <= 255: - return [bytes([SMALL_INTEGER_EXT,obj])] + return [pack_bytes([SMALL_INTEGER_EXT,obj])] elif -2147483648 <= obj <= 2147483647: - return [bytes([INTEGER_EXT]), struct.pack(">l", obj)] + return [pack_bytes([INTEGER_EXT]), struct.pack(">l", obj)] else: sign = obj < 0 obj = abs(obj) @@ -243,54 +245,57 @@ def encode_part(self, obj): obj >>= 8 if len(big_buf) < 256: - return [bytes([SMALL_BIG_EXT,len(big_buf),sign]),bytes(big_buf)] + return [pack_bytes([SMALL_BIG_EXT,len(big_buf),sign]), + pack_bytes(big_buf)] else: - return [bytes([LARGE_BIG_EXT]), struct.pack(">L", len(big_buf)), bytes([sign]), bytes(big_buf)] + return [pack_bytes([LARGE_BIG_EXT]), + struct.pack(">L", len(big_buf)), + pack_bytes([sign]), pack_bytes(big_buf)] elif isinstance(obj, float): floatstr = ("%.20e" % obj).encode('ascii') - return [bytes([FLOAT_EXT]), floatstr + b"\x00"*(31-len(floatstr))] + return [pack_bytes([FLOAT_EXT]), floatstr + b"\x00"*(31-len(floatstr))] elif isinstance(obj, Atom): st = obj.encode('latin-1') - return [bytes([ATOM_EXT]), struct.pack(">H", len(st)), st] - elif isinstance(obj, str): + return [pack_bytes([ATOM_EXT]), struct.pack(">H", len(st)), st] + elif isinstance(obj, six.string_types): st = obj.encode('utf-8') - return [bytes([BINARY_EXT]), struct.pack(">L", len(st)), st] - elif isinstance(obj, bytes): - return [bytes([BINARY_EXT]), struct.pack(">L", len(obj)), obj] + return [pack_bytes([BINARY_EXT]), struct.pack(">L", len(st)), st] + elif isinstance(obj, six.binary_type): + return [pack_bytes([BINARY_EXT]), struct.pack(">L", len(obj)), obj] elif isinstance(obj, tuple): n = len(obj) if n < 256: - buf = [bytes([SMALL_TUPLE_EXT,n])] + buf = [pack_bytes([SMALL_TUPLE_EXT,n])] else: - buf = [bytes([LARGE_TUPLE_EXT]), struct.pack(">L", n)] + buf = [pack_bytes([LARGE_TUPLE_EXT]), struct.pack(">L", n)] for item in obj: buf += self.encode_part(item) return buf elif obj == []: - return [bytes([NIL_EXT])] + return [pack_bytes([NIL_EXT])] elif isinstance(obj, list): - buf = [bytes([LIST_EXT]), struct.pack(">L", len(obj))] + buf = [pack_bytes([LIST_EXT]), struct.pack(">L", len(obj))] for item in obj: buf += self.encode_part(item) - buf.append(bytes([NIL_EXT])) # list tail - no such thing in Python + buf.append(pack_bytes([NIL_EXT])) # list tail - no such thing in Python return buf elif isinstance(obj, Reference): - return [bytes([NEW_REFERENCE_EXT]), - struct.pack(">H", len(obj.ref_id)), - bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'), - bytes([obj.creation]), struct.pack(">%dL" % len(obj.ref_id), *obj.ref_id)] + return [pack_bytes([NEW_REFERENCE_EXT]), struct.pack(">H", len(obj.ref_id)), + pack_bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), + obj.node.encode('latin-1'), pack_bytes([obj.creation]), + struct.pack(">%dL" % len(obj.ref_id), *obj.ref_id)] elif isinstance(obj, Port): - return [bytes([PORT_EXT]), - bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'), - struct.pack(">LB", obj.port_id, obj.creation)] + return [pack_bytes([PORT_EXT]), pack_bytes([ATOM_EXT]), + struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'), + struct.pack(">LB", obj.port_id, obj.creation)] elif isinstance(obj, PID): - return [bytes([PID_EXT]), - bytes([ATOM_EXT]), struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'), - struct.pack(">LLB", obj.pid_id, obj.serial, obj.creation)] + return [pack_bytes([PID_EXT]), pack_bytes([ATOM_EXT]), + struct.pack(">H", len(obj.node)), obj.node.encode('latin-1'), + struct.pack(">LLB", obj.pid_id, obj.serial, obj.creation)] elif isinstance(obj, Export): - return [bytes([EXPORT_EXT]), - bytes([ATOM_EXT]), struct.pack(">H", len(obj.module)), obj.module.encode('latin-1'), - bytes([ATOM_EXT]), struct.pack(">H", len(obj.function)), obj.function.encode('latin-1'), - bytes([SMALL_INTEGER_EXT,obj.arity])] + return [pack_bytes([EXPORT_EXT]), pack_bytes([ATOM_EXT]), + struct.pack(">H", len(obj.module)), obj.module.encode('latin-1'), + pack_bytes([ATOM_EXT]), struct.pack(">H", len(obj.function)), + obj.function.encode('latin-1'), pack_bytes([SMALL_INTEGER_EXT,obj.arity])] else: raise NotImplementedError("Unable to serialize %r" % obj) diff --git a/erlastic/compat.py b/erlastic/compat.py new file mode 100644 index 0000000..c6df9bd --- /dev/null +++ b/erlastic/compat.py @@ -0,0 +1,10 @@ +import sys + +from array import array + +__all__ = ['pack_bytes'] + +if sys.version_info < (3,): + pack_bytes = lambda a: array('B', a).tostring() +else: + pack_bytes = bytes diff --git a/tests.py b/tests.py index 3ea17cf..6af7de9 100755 --- a/tests.py +++ b/tests.py @@ -1,58 +1,60 @@ #!/usr/bin/env python +import six import unittest from erlastic import decode, encode +from erlastic.compat import * from erlastic.types import * erlang_term_binaries = [ # nil - ([], list, b"\x83j"), + ([], list, six.b('\x83j')), # binary - (b"foo", bytes, b'\x83m\x00\x00\x00\x03foo'), + (six.b('foo'), six.binary_type, six.b('\x83m\x00\x00\x00\x03foo')), # atom - (Atom("foo"), Atom, b'\x83d\x00\x03foo'), + (Atom("foo"), Atom, six.b('\x83d\x00\x03foo')), # atom true - (True, bool, b'\x83d\x00\x04true'), + (True, bool, six.b('\x83d\x00\x04true')), # atom false - (False, bool, b'\x83d\x00\x05false'), + (False, bool, six.b('\x83d\x00\x05false')), # atom none - (None, type(None), b'\x83d\x00\x04none'), + (None, type(None), six.b('\x83d\x00\x04none')), # small integer - (123, int, b'\x83a{'), + (123, six.integer_types, six.b('\x83a{')), # integer - (12345, int, b'\x83b\x00\x0009'), + (12345, six.integer_types, six.b('\x83b\x00\x0009')), # float - (1.2345, float, b'\x83c1.23449999999999993072e+00\x00\x00\x00\x00\x00'), + (1.2345, float, six.b('\x83c1.23449999999999993072e+00\x00\x00\x00\x00\x00')), # tuple - ((Atom("foo"), b"test", 123), tuple, b'\x83h\x03d\x00\x03foom\x00\x00\x00\x04testa{'), + ((Atom("foo"), b"test", 123), tuple, six.b('\x83h\x03d\x00\x03foom\x00\x00\x00\x04testa{')), # list - ([1024, b"test", 4.096], list, b'\x83l\x00\x00\x00\x03b\x00\x00\x04\x00m\x00\x00\x00\x04testc4.09600000000000008527e+00\x00\x00\x00\x00\x00j'), - ([102, 111, 111], list, b'\x83l\x00\x00\x00\x03afaoaoj'), + ([1024, b"test", 4.096], list, six.b('\x83l\x00\x00\x00\x03b\x00\x00\x04\x00m\x00\x00\x00\x04testc4.09600000000000008527e+00\x00\x00\x00\x00\x00j')), + ([102, 111, 111], list, six.b('\x83l\x00\x00\x00\x03afaoaoj')), # small big - (12345678901234567890, int, b'\x83n\x08\x00\xd2\n\x1f\xeb\x8c\xa9T\xab'), + (12345678901234567890, six.integer_types, six.b('\x83n\x08\x00\xd2\n\x1f\xeb\x8c\xa9T\xab')), # large big (123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890, - int, b'\x83o\x00\x00\x01D\x00\xd2\n?\xce\x96\xf1\xcf\xacK\xf1{\xefa\x11=$^\x93\xa9\x88\x17\xa0\xc2\x01\xa5%\xb7\xe3Q\x1b\x00\xeb\xe7\xe5\xd5Po\x98\xbd\x90\xf1\xc3\xddR\x83\xd1)\xfc&\xeaH\xc31w\xf1\x07\xf3\xf33\x8f\xb7\x96\x83\x05t\xeci\x9cY"\x98\x98i\xca\x11bY=\xcc\xa1\xb4R\x1bl\x01\x86\x18\xe9\xa23\xaa\x14\xef\x11[}O\x14RU\x18$\xfe\x7f\x96\x94\xcer?\xd7\x8b\x9a\xa7v\xbd\xbb+\x07X\x94x\x7fI\x024.\xa0\xcc\xde\xef:\xa7\x89~\xa4\xafb\xe4\xc1\x07\x1d\xf3cl|0\xc9P`\xbf\xab\x95z\xa2DQf\xf7\xca\xef\xb0\xc4=\x11\x06*:Y\xf58\xaf\x18\xa7\x81\x13\xdf\xbdTl4\xe0\x00\xee\x93\xd6\x83V\xc9<\xe7I\xdf\xa8.\xf5\xfc\xa4$R\x95\xef\xd1\xa7\xd2\x89\xceu!\xf8\x08\xb1Zv\xa6\xd9z\xdb0\x88\x10\xf3\x7f\xd3sc\x98[\x1a\xac6V\x1f\xad0)\xd0\x978\xd1\x02\xe6\xfbH\x149\xdc).\xb5\x92\xf6\x91A\x1b\xcd\xb8`B\xc6\x04\x83L\xc0\xb8\xafN+\x81\xed\xec?;\x1f\xab1\xc1^J\xffO\x1e\x01\x87H\x0f.ZD\x06\xf0\xbak\xaagVH]\x17\xe6I.B\x14a2\xc1;\xd1+\xea.\xe4\x92\x15\x93\xe9\'E\xd0(\xcd\x90\xfb\x10'), + six.integer_types, six.b('\x83o\x00\x00\x01D\x00\xd2\n?\xce\x96\xf1\xcf\xacK\xf1{\xefa\x11=$^\x93\xa9\x88\x17\xa0\xc2\x01\xa5%\xb7\xe3Q\x1b\x00\xeb\xe7\xe5\xd5Po\x98\xbd\x90\xf1\xc3\xddR\x83\xd1)\xfc&\xeaH\xc31w\xf1\x07\xf3\xf33\x8f\xb7\x96\x83\x05t\xeci\x9cY"\x98\x98i\xca\x11bY=\xcc\xa1\xb4R\x1bl\x01\x86\x18\xe9\xa23\xaa\x14\xef\x11[}O\x14RU\x18$\xfe\x7f\x96\x94\xcer?\xd7\x8b\x9a\xa7v\xbd\xbb+\x07X\x94x\x7fI\x024.\xa0\xcc\xde\xef:\xa7\x89~\xa4\xafb\xe4\xc1\x07\x1d\xf3cl|0\xc9P`\xbf\xab\x95z\xa2DQf\xf7\xca\xef\xb0\xc4=\x11\x06*:Y\xf58\xaf\x18\xa7\x81\x13\xdf\xbdTl4\xe0\x00\xee\x93\xd6\x83V\xc9<\xe7I\xdf\xa8.\xf5\xfc\xa4$R\x95\xef\xd1\xa7\xd2\x89\xceu!\xf8\x08\xb1Zv\xa6\xd9z\xdb0\x88\x10\xf3\x7f\xd3sc\x98[\x1a\xac6V\x1f\xad0)\xd0\x978\xd1\x02\xe6\xfbH\x149\xdc).\xb5\x92\xf6\x91A\x1b\xcd\xb8`B\xc6\x04\x83L\xc0\xb8\xafN+\x81\xed\xec?;\x1f\xab1\xc1^J\xffO\x1e\x01\x87H\x0f.ZD\x06\xf0\xbak\xaagVH]\x17\xe6I.B\x14a2\xc1;\xd1+\xea.\xe4\x92\x15\x93\xe9\'E\xd0(\xcd\x90\xfb\x10')), # reference - (Reference('nonode@nohost', [33, 0, 0], 0), Reference, b'\x83r\x00\x03d\x00\rnonode@nohost\x00\x00\x00\x00!\x00\x00\x00\x00\x00\x00\x00\x00'), + (Reference('nonode@nohost', [33, 0, 0], 0), Reference, six.b('\x83r\x00\x03d\x00\rnonode@nohost\x00\x00\x00\x00!\x00\x00\x00\x00\x00\x00\x00\x00')), # port - (Port('nonode@nohost', 455, 0), Port, b'\x83fd\x00\rnonode@nohost\x00\x00\x01\xc7\x00'), + (Port('nonode@nohost', 455, 0), Port, six.b('\x83fd\x00\rnonode@nohost\x00\x00\x01\xc7\x00')), # pid - (PID('nonode@nohost', 31, 0, 0), PID, b'\x83gd\x00\rnonode@nohost\x00\x00\x00\x1f\x00\x00\x00\x00\x00'), + (PID('nonode@nohost', 31, 0, 0), PID, six.b('\x83gd\x00\rnonode@nohost\x00\x00\x00\x1f\x00\x00\x00\x00\x00')), # function export - (Export('jobqueue', 'stats', 0), Export, b'\x83qd\x00\x08jobqueued\x00\x05statsa\x00'), + (Export('jobqueue', 'stats', 0), Export, six.b('\x83qd\x00\x08jobqueued\x00\x05statsa\x00')), ] erlang_term_decode = [ ## ext_string is an optimized way to send list of bytes, so not bijective (no erlang representation), only decode - (bytes([102, 111, 111]), bytes, b'\x83k\x00\x03foo') + (pack_bytes([102, 111, 111]), six.binary_type, six.b('\x83k\x00\x03foo')) ] erlang_term_encode = [ - ## python3 choice : encode string as binary utf-8, so not bijective (decoded binary utf-8 is of type "bytes()"), only encode - ("foo", str, b'\x83m\x00\x00\x00\x03foo') + ## python3 choice : encode string as binary utf-8, so not bijective (decoded binary utf-8 is of type "six.binary_type"), only encode + ("foo", six.string_types, six.b('\x83m\x00\x00\x00\x03foo')) ] class ErlangTestCase(unittest.TestCase): From 434dc15da4967c5985b29af3d239453d08dbe63b Mon Sep 17 00:00:00 2001 From: Ivan Chernetsky Date: Tue, 29 Mar 2016 11:33:18 -0700 Subject: [PATCH 4/4] Fix a typo pointed out by lukebakken --- erlastic/codec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erlastic/codec.py b/erlastic/codec.py index 60f67ae..76f89a7 100644 --- a/erlastic/codec.py +++ b/erlastic/codec.py @@ -57,7 +57,7 @@ def decode_100(self, buf, offset): def decode_115(self, buf, offset): """SMALL_ATOM_EXT""" - atom_len = six.intexbytes(buf, offset) + atom_len = six.indexbytes(buf, offset) atom = buf[offset+1:offset+1+atom_len] return self.convert_atom(atom), offset+atom_len+1