spark-issues mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Prabir Ghosh (JIRA)" <j...@apache.org>
Subject [jira] [Created] (SPARK-22186) Getting this issue while initializing spark on windows
Date Mon, 02 Oct 2017 18:50:03 GMT
Prabir Ghosh created SPARK-22186:
------------------------------------

             Summary: Getting this issue while initializing spark on windows
                 Key: SPARK-22186
                 URL: https://issues.apache.org/jira/browse/SPARK-22186
             Project: Spark
          Issue Type: Bug
          Components: Build
    Affects Versions: 2.1.1
         Environment: Windows 10, Python 3.6
            Reporter: Prabir Ghosh


Py4JJavaError                             Traceback (most recent call last)
<ipython-input-2-5e2ed3915947> in <module>()
----> 1 student = learner.addmission('problem.json')

C:\Users\prabi\Modeling\app/..\learners.py in __init__(self, problempath)
     67                 self.__identifier__ = identifier
     68 
---> 69                 spark_st = spark_setup(data_config["spark_path"],data_config["py4j_folder"])
     70                 self.__sc__ = spark_st.sc()
     71                 self.__GridSearchCV__ = spark_st.GridSearchCV()

C:\Users\prabi\Modeling\app/..\learners.py in __init__(self, spark_path, py4j_path)
     17         __GridSearchCV__ = None
     18         def __init__(self, spark_path, py4j_path):
---> 19                 __spark__ = spark.Setup(spark_path, py4j_path)
     20                 self.__sc__ = __spark__.SparkContext()
     21                 self.__GridSearchCV__ = __spark__.GridSearchCV()

C:\Users\prabi\Modeling\app/..\School\spark_setup.py in __init__(self, spark_path, py4j_path,
app_name)
     33                 from spark_sklearn import GridSearchCV
     34 
---> 35                 self.__sc__ = SparkContext(conf=conf)
     36                 self.__grid_serach__ = GridSearchCV
     37 

/ayata/spark/spark-2.1.1-bin-hadoop2.7/python\pyspark\context.py in __init__(self, master,
appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
    116         try:
    117             self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize,
serializer,
--> 118                           conf, jsc, profiler_cls)
    119         except:
    120             # If an error occurs, clean up in order to allow future SparkContext creation:

/ayata/spark/spark-2.1.1-bin-hadoop2.7/python\pyspark\context.py in _do_init(self, master,
appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)
    180 
    181         # Create the Java SparkContext through Py4J
--> 182         self._jsc = jsc or self._initialize_context(self._conf._jconf)
    183         # Reset the SparkConf to the one actually used by the SparkContext in JVM.
    184         self._conf = SparkConf(_jconf=self._jsc.sc().conf())

/ayata/spark/spark-2.1.1-bin-hadoop2.7/python\pyspark\context.py in _initialize_context(self,
jconf)
    247         Initialize SparkContext in function to allow subclass specific initialization
    248         """
--> 249         return self._jvm.JavaSparkContext(jconf)
    250 
    251     @classmethod

\ayata\spark\spark-2.1.1-bin-hadoop2.7\python\lib\py4j-0.10.4-src.zip\py4j\java_gateway.py
in __call__(self, *args)
   1399         answer = self._gateway_client.send_command(command)
   1400         return_value = get_return_value(
-> 1401             answer, self._gateway_client, None, self._fqn)
   1402 
   1403         for temp_arg in temp_args:

\ayata\spark\spark-2.1.1-bin-hadoop2.7\python\lib\py4j-0.10.4-src.zip\py4j\protocol.py in
get_return_value(answer, gateway_client, target_id, name)
    317                 raise Py4JJavaError(
    318                     "An error occurred while calling {0}{1}{2}.\n".
--> 319                     format(target_id, ".", name), value)
    320             else:
    321                 raise Py4JError(

Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
: java.lang.ExceptionInInitializerError
	at org.apache.hadoop.util.StringUtils.<clinit>(StringUtils.java:80)
	at org.apache.hadoop.security.SecurityUtil.getAuthenticationMethod(SecurityUtil.java:611)
	at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:273)
	at org.apache.hadoop.security.UserGroupInformation.ensureInitialized(UserGroupInformation.java:261)
	at org.apache.hadoop.security.UserGroupInformation.loginUserFromSubject(UserGroupInformation.java:791)
	at org.apache.hadoop.security.UserGroupInformation.getLoginUser(UserGroupInformation.java:761)
	at org.apache.hadoop.security.UserGroupInformation.getCurrentUser(UserGroupInformation.java:634)
	at org.apache.spark.util.Utils$$anonfun$getCurrentUserName$1.apply(Utils.scala:2391)
	at org.apache.spark.util.Utils$$anonfun$getCurrentUserName$1.apply(Utils.scala:2391)
	at scala.Option.getOrElse(Option.scala:121)
	at org.apache.spark.util.Utils$.getCurrentUserName(Utils.scala:2391)
	at org.apache.spark.SparkContext.<init>(SparkContext.scala:295)
	at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
	at java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(Unknown Source)
	at java.base/java.lang.reflect.Constructor.newInstance(Unknown Source)
	at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
	at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
	at py4j.Gateway.invoke(Gateway.java:236)
	at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
	at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
	at py4j.GatewayConnection.run(GatewayConnection.java:214)
	at java.base/java.lang.Thread.run(Unknown Source)
Caused by: java.lang.StringIndexOutOfBoundsException: begin 0, end 3, length 1
	at java.base/java.lang.String.checkBoundsBeginEnd(Unknown Source)
	at java.base/java.lang.String.substring(Unknown Source)
	at org.apache.hadoop.util.Shell.<clinit>(Shell.java:52)
	... 24 more





--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org


Mime
View raw message