spark-user mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From Selvam Raman <sel...@gmail.com>
Subject Re: Pyspark UDF/map fucntion throws pickling exception
Date Thu, 15 Feb 2018 12:01:01 GMT
pyspark - 2.2.1
spacy - 2.0.7
python - 3.6


Placing full logs here

Traceback (most recent call last):
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 148, in dump
    return Pickler.dump(self, obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 409, in dump
    self.save(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 751, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 808, in _batch_appends
    save(tmp[0])
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 249, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 297, in save_function_tuple
    save(f_globals)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 821, in save_dict
    self._batch_setitems(obj.items())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 852, in _batch_setitems
    save(v)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 249, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 297, in save_function_tuple
    save(f_globals)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 821, in save_dict
    self._batch_setitems(obj.items())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 852, in _batch_setitems
    save(v)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 521, in save
    self.save_reduce(obj=obj, *rv)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 600, in save_reduce
    save(state)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 821, in save_dict
    self._batch_setitems(obj.items())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 847, in _batch_setitems
    save(v)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 521, in save
    self.save_reduce(obj=obj, *rv)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 582, in save_reduce
    save(args)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 751, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 368, in save_builtin_function
    return self.save_function(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 247, in save_function
    if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule
is None:
AttributeError: 'builtin_function_or_method' object has no attribute
'__code__'
Traceback (most recent call last):
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 148, in dump
    return Pickler.dump(self, obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 409, in dump
    self.save(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 751, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 805, in _batch_appends
    save(x)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 255, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 292, in save_function_tuple
    save((code, closure, base_globals))
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 736, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 781, in save_list
    self._batch_appends(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 808, in _batch_appends
    save(tmp[0])
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 249, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 297, in save_function_tuple
    save(f_globals)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 821, in save_dict
    self._batch_setitems(obj.items())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 852, in _batch_setitems
    save(v)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 249, in save_function
    self.save_function_tuple(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 297, in save_function_tuple
    save(f_globals)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 821, in save_dict
    self._batch_setitems(obj.items())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 852, in _batch_setitems
    save(v)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 521, in save
    self.save_reduce(obj=obj, *rv)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 600, in save_reduce
    save(state)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 821, in save_dict
    self._batch_setitems(obj.items())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 847, in _batch_setitems
    save(v)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 521, in save
    self.save_reduce(obj=obj, *rv)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 582, in save_reduce
    save(args)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 751, in save_tuple
    save(element)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
line 476, in save
    f(self, obj) # Call unbound method with explicit self
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 368, in save_builtin_function
    return self.save_function(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 247, in save_function
    if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule
is None:
AttributeError: 'builtin_function_or_method' object has no attribute
'__code__'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File
"/Users/rs/PycharmProjects/SparkDemo/com/elsevier/vtw/ExtractDescription.py",
line 30, in <module>
    #description.rdd.flatMap(lambda row: getPhrases(row.desc)).foreach(f)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 782, in foreach
    self.mapPartitions(processPartition).count()  # Force evaluation
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 1041, in count
    return self.mapPartitions(lambda i: [sum(1 for _ in i)]).sum()
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 1032, in sum
    return self.mapPartitions(lambda x: [sum(x)]).fold(0, operator.add)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 906, in fold
    vals = self.mapPartitions(func).collect()
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 809, in collect
    port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 2455, in _jrdd
    self._jrdd_deserializer, profiler)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 2388, in _wrap_function
    pickled_command, broadcast_vars, env, includes =
_prepare_for_python_RDD(sc, command)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/rdd.py",
line 2374, in _prepare_for_python_RDD
    pickled_command = ser.dumps(command)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/serializers.py",
line 464, in dumps
    return cloudpickle.dumps(obj, 2)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 704, in dumps
    cp.dump(obj)
  File
"/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pyspark/cloudpickle.py",
line 162, in dump
    raise pickle.PicklingError(msg)
_pickle.PicklingError: Could not serialize object: AttributeError:
'builtin_function_or_method' object has no attribute '__code__'

On Thu, Feb 15, 2018 at 11:44 AM, Selvam Raman <selmna@gmail.com> wrote:

> import spacy
>
> nlp = spacy.load('en')
>
>
>
> def getPhrases(content):
>     phrases = []
>     doc = nlp(str(content))
>     for chunks in doc.noun_chunks:
>         phrases.append(chunks.text)
>     return phrases
>
> the above function will retrieve the noun phrases from the content and
> return list of phrases.
>
>
> def f(x) : print(x)
>
>
> description = xmlData.filter(col("dcterms:description").isNotNull()).select(col("dcterms:description").alias("desc"))
>
> description.rdd.flatMap(lambda row: getPhrases(row.desc)).foreach(f)
>
> when i am trying to access getphrases i am getting below exception
>
> """if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule
> is None:
> AttributeError: 'builtin_function_or_method' object has no attribute
> '__code__' """
>
> Full stack trace is below
>
> Traceback (most recent call last):
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 148, in dump
>     return Pickler.dump(self, obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 409, in dump
>     self.save(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 751, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 255, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 292, in
> save_function_tuple
>     save((code, closure, base_globals))
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 736, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 781, in save_list
>     self._batch_appends(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 805, in _batch_appends
>     save(x)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 255, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 292, in
> save_function_tuple
>     save((code, closure, base_globals))
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 736, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 781, in save_list
>     self._batch_appends(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 805, in _batch_appends
>     save(x)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 255, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 292, in
> save_function_tuple
>     save((code, closure, base_globals))
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 736, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 781, in save_list
>     self._batch_appends(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 805, in _batch_appends
>     save(x)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 255, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 292, in
> save_function_tuple
>     save((code, closure, base_globals))
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 736, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 781, in save_list
>     self._batch_appends(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 805, in _batch_appends
>     save(x)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 255, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 292, in
> save_function_tuple
>     save((code, closure, base_globals))
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 736, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 781, in save_list
>     self._batch_appends(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 808, in _batch_appends
>     save(tmp[0])
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 249, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 297, in
> save_function_tuple
>     save(f_globals)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 821, in save_dict
>     self._batch_setitems(obj.items())
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 852, in _batch_setitems
>     save(v)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 249, in
> save_function
>     self.save_function_tuple(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 297, in
> save_function_tuple
>     save(f_globals)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 821, in save_dict
>     self._batch_setitems(obj.items())
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 852, in _batch_setitems
>     save(v)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 521, in save
>     self.save_reduce(obj=obj, *rv)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 600, in save_reduce
>     save(state)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 821, in save_dict
>     self._batch_setitems(obj.items())
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 847, in _batch_setitems
>     save(v)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 521, in save
>     self.save_reduce(obj=obj, *rv)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 582, in save_reduce
>     save(args)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 751, in save_tuple
>     save(element)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/pickle.py",
> line 476, in save
>     f(self, obj) # Call unbound method with explicit self
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 368, in
> save_builtin_function
>     return self.save_function(obj)
>   File "/Library/Frameworks/Python.framework/Versions/3.6/lib/
> python3.6/site-packages/pyspark/cloudpickle.py", line 247, in
> save_function
>     if islambda(obj) or obj.__code__.co_filename == '<stdin>' or themodule
> is None:
> AttributeError: 'builtin_function_or_method' object has no attribute
> '__code__'
>
>
>
> please help me.
>
>
>
> --
> Selvam Raman
> "லஞ்சம் தவிர்த்து நெஞ்சம் நிமிர்த்து"
>



-- 
Selvam Raman
"லஞ்சம் தவிர்த்து நெஞ்சம் நிமிர்த்து"

Mime
View raw message