# else tmp is empty, and we're done
self.save_reduce(set, (list(obj),), obj=obj)
batch = list(islice(it, self._BATCHSIZE))
def save_frozenset(self, obj):
self.save_reduce(frozenset, (list(obj),), obj=obj)
# If the object is already in the memo, this means it is
# recursive. In this case, throw away everything we put on the
# stack, and fetch the object back from the memo.
write(POP_MARK + self.get(self.memo[id(obj)][0]))
dispatch[frozenset] = save_frozenset
def save_global(self, obj, name=None):
name = getattr(obj, '__qualname__', None)
module_name = whichmodule(obj, name)
__import__(module_name, level=0)
module = sys.modules[module_name]
obj2, parent = _getattribute(module, name)
except (ImportError, KeyError, AttributeError):
"Can't pickle %r: it's not found as %s.%s" %
(obj, module_name, name)) from None
"Can't pickle %r: it's not the same object as %s.%s" %
(obj, module_name, name))
code = _extension_registry.get((module_name, name))
write(EXT1 + pack("<B", code))
write(EXT2 + pack("<H", code))
write(EXT4 + pack("<i", code))
lastname = name.rpartition('.')[2]
# Non-ASCII identifiers are supported only with protocols >= 3.
elif parent is not module:
self.save_reduce(getattr, (parent, lastname))
write(GLOBAL + bytes(module_name, "utf-8") + b'\n' +
bytes(name, "utf-8") + b'\n')
r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING
r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING
if (module_name, name) in r_name_mapping:
module_name, name = r_name_mapping[(module_name, name)]
elif module_name in r_import_mapping:
module_name = r_import_mapping[module_name]
write(GLOBAL + bytes(module_name, "ascii") + b'\n' +
bytes(name, "ascii") + b'\n')
except UnicodeEncodeError:
"can't pickle global identifier '%s.%s' using "
"pickle protocol %i" % (module, name, self.proto)) from None
def save_type(self, obj):
return self.save_reduce(type, (None,), obj=obj)
elif obj is type(NotImplemented):
return self.save_reduce(type, (NotImplemented,), obj=obj)
return self.save_reduce(type, (...,), obj=obj)
return self.save_global(obj)
dispatch[FunctionType] = save_global
dispatch[type] = save_type
def __init__(self, file, *, fix_imports=True,
encoding="ASCII", errors="strict", buffers=None):
"""This takes a binary file for reading a pickle data stream.
The protocol version of the pickle is detected automatically, so
no proto argument is needed.
The argument *file* must have two methods, a read() method that
takes an integer argument, and a readline() method that requires
no arguments. Both methods should return bytes. Thus *file*
can be a binary file object opened for reading, an io.BytesIO
object, or any other custom object that meets this interface.
The file-like object must have two methods, a read() method
that takes an integer argument, and a readline() method that
requires no arguments. Both methods should return bytes.
Thus file-like object can be a binary file object opened for
reading, a BytesIO object, or any other custom object that
If *buffers* is not None, it should be an iterable of buffer-enabled
objects that is consumed each time the pickle stream references
an out-of-band buffer view. Such buffers have been given in order
to the *buffer_callback* of a Pickler object.
If *buffers* is None (the default), then the buffers are taken
from the pickle stream, assuming they are serialized there.
It is an error for *buffers* to be None if the pickle stream
was produced with a non-None *buffer_callback*.
Other optional arguments are *fix_imports*, *encoding* and
*errors*, which are used to control compatibility support for
pickle stream generated by Python 2. If *fix_imports* is True,
pickle will try to map the old Python 2 names to the new names
used in Python 3. The *encoding* and *errors* tell pickle how
to decode 8-bit string instances pickled by Python 2; these
default to 'ASCII' and 'strict', respectively. *encoding* can be
'bytes' to read theses 8-bit string instances as bytes objects.
self._buffers = iter(buffers) if buffers is not None else None
self._file_readline = file.readline
self._file_read = file.read
self.fix_imports = fix_imports
"""Read a pickled object representation from the open file.
Return the reconstituted object hierarchy specified in the file.
# Check whether Unpickler was initialized correctly. This is
# only needed to mimic the behavior of _pickle.Unpickler.dump().
if not hasattr(self, "_file_read"):
raise UnpicklingError("Unpickler.__init__() was not called by "
"%s.__init__()" % (self.__class__.__name__,))
self._unframer = _Unframer(self._file_read, self._file_readline)
self.read = self._unframer.read
self.readinto = self._unframer.readinto
self.readline = self._unframer.readline
self.append = self.stack.append
assert isinstance(key, bytes_types)
except _Stop as stopinst:
# Return a list of items pushed in the stack after last MARK instruction.
self.stack = self.metastack.pop()
self.append = self.stack.append
def persistent_load(self, pid):
raise UnpicklingError("unsupported persistent id encountered")
if not 0 <= proto <= HIGHEST_PROTOCOL:
raise ValueError("unsupported pickle protocol: %d" % proto)
dispatch[PROTO[0]] = load_proto
frame_size, = unpack('<Q', self.read(8))
if frame_size > sys.maxsize:
raise ValueError("frame size > sys.maxsize: %d" % frame_size)
self._unframer.load_frame(frame_size)
dispatch[FRAME[0]] = load_frame
pid = self.readline()[:-1].decode("ascii")
except UnicodeDecodeError:
"persistent IDs in protocol 0 must be ASCII strings")
self.append(self.persistent_load(pid))
dispatch[PERSID[0]] = load_persid
def load_binpersid(self):
self.append(self.persistent_load(pid))
dispatch[BINPERSID[0]] = load_binpersid
dispatch[NONE[0]] = load_none
dispatch[NEWFALSE[0]] = load_false
dispatch[NEWTRUE[0]] = load_true
dispatch[INT[0]] = load_int
self.append(unpack('<i', self.read(4))[0])
dispatch[BININT[0]] = load_binint
self.append(self.read(1)[0])
dispatch[BININT1[0]] = load_binint1
self.append(unpack('<H', self.read(2))[0])
dispatch[BININT2[0]] = load_binint2
val = self.readline()[:-1]
if val and val[-1] == b'L'[0]:
dispatch[LONG[0]] = load_long
self.append(decode_long(data))
dispatch[LONG1[0]] = load_long1
n, = unpack('<i', self.read(4))
# Corrupt or hostile pickle -- we never write one like this
raise UnpicklingError("LONG pickle has negative byte count")
self.append(decode_long(data))
dispatch[LONG4[0]] = load_long4
self.append(float(self.readline()[:-1]))
dispatch[FLOAT[0]] = load_float
self.append(unpack('>d', self.read(8))[0])
dispatch[BINFLOAT[0]] = load_binfloat
def _decode_string(self, value):
# Used to allow strings from Python 2 to be decoded either as
# bytes or Unicode strings. This should be used only with the
# STRING, BINSTRING and SHORT_BINSTRING opcodes.
if self.encoding == "bytes":
return value.decode(self.encoding, self.errors)
data = self.readline()[:-1]
if len(data) >= 2 and data[0] == data[-1] and data[0] in b'"\'':
raise UnpicklingError("the STRING opcode argument must be quoted")
self.append(self._decode_string(codecs.escape_decode(data)[0]))
dispatch[STRING[0]] = load_string
def load_binstring(self):
# Deprecated BINSTRING uses signed 32-bit length
len, = unpack('<i', self.read(4))
raise UnpicklingError("BINSTRING pickle has negative byte count")
self.append(self._decode_string(data))
dispatch[BINSTRING[0]] = load_binstring
len, = unpack('<I', self.read(4))
raise UnpicklingError("BINBYTES exceeds system's maximum size "
self.append(self.read(len))
dispatch[BINBYTES[0]] = load_binbytes
self.append(str(self.readline()[:-1], 'raw-unicode-escape'))
dispatch[UNICODE[0]] = load_unicode
def load_binunicode(self):
len, = unpack('<I', self.read(4))
raise UnpicklingError("BINUNICODE exceeds system's maximum size "
self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
dispatch[BINUNICODE[0]] = load_binunicode
def load_binunicode8(self):
len, = unpack('<Q', self.read(8))
raise UnpicklingError("BINUNICODE8 exceeds system's maximum size "
self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
dispatch[BINUNICODE8[0]] = load_binunicode8
def load_binbytes8(self):
len, = unpack('<Q', self.read(8))
raise UnpicklingError("BINBYTES8 exceeds system's maximum size "
self.append(self.read(len))
dispatch[BINBYTES8[0]] = load_binbytes8
def load_bytearray8(self):
len, = unpack('<Q', self.read(8))
raise UnpicklingError("BYTEARRAY8 exceeds system's maximum size "
dispatch[BYTEARRAY8[0]] = load_bytearray8
def load_next_buffer(self):
if self._buffers is None:
raise UnpicklingError("pickle stream refers to out-of-band data "
"but no *buffers* argument was given")
buf = next(self._buffers)
raise UnpicklingError("not enough out-of-band buffers")
dispatch[NEXT_BUFFER[0]] = load_next_buffer
def load_readonly_buffer(self):
with memoryview(buf) as m:
self.stack[-1] = m.toreadonly()
dispatch[READONLY_BUFFER[0]] = load_readonly_buffer
def load_short_binstring(self):
self.append(self._decode_string(data))
dispatch[SHORT_BINSTRING[0]] = load_short_binstring
def load_short_binbytes(self):
self.append(self.read(len))
dispatch[SHORT_BINBYTES[0]] = load_short_binbytes
def load_short_binunicode(self):
self.append(str(self.read(len), 'utf-8', 'surrogatepass'))
dispatch[SHORT_BINUNICODE[0]] = load_short_binunicode
self.append(tuple(items))
dispatch[TUPLE[0]] = load_tuple
def load_empty_tuple(self):
dispatch[EMPTY_TUPLE[0]] = load_empty_tuple
self.stack[-1] = (self.stack[-1],)
dispatch[TUPLE1[0]] = load_tuple1
self.stack[-2:] = [(self.stack[-2], self.stack[-1])]
dispatch[TUPLE2[0]] = load_tuple2
self.stack[-3:] = [(self.stack[-3], self.stack[-2], self.stack[-1])]
dispatch[TUPLE3[0]] = load_tuple3
def load_empty_list(self):
dispatch[EMPTY_LIST[0]] = load_empty_list
def load_empty_dictionary(self):
dispatch[EMPTY_DICT[0]] = load_empty_dictionary
def load_empty_set(self):
dispatch[EMPTY_SET[0]] = load_empty_set
def load_frozenset(self):
self.append(frozenset(items))
dispatch[FROZENSET[0]] = load_frozenset
dispatch[LIST[0]] = load_list
d = {items[i]: items[i+1]
for i in range(0, len(items), 2)}
dispatch[DICT[0]] = load_dict
# INST and OBJ differ only in how they get a class object. It's not
# only sensible to do the rest in a common routine, the two routines
# previously diverged and grew different bugs.
# klass is the class to instantiate, and k points to the topmost mark
# object, following which are the arguments for klass.__init__.
def _instantiate(self, klass, args):
if (args or not isinstance(klass, type) or
hasattr(klass, "__getinitargs__")):
raise TypeError("in constructor for %s: %s" %
(klass.__name__, str(err)), sys.exc_info()[2])
value = klass.__new__(klass)
module = self.readline()[:-1].decode("ascii")
name = self.readline()[:-1].decode("ascii")
klass = self.find_class(module, name)
self._instantiate(klass, self.pop_mark())
It is recommended that you Edit text format, this type of Fix handles quite a lot in one request
