Unable to launch spark

apache-spark

#1

Hi,
I am unable to launch spark. When I use the command -
pyspark --master yarn --conf spark.ui.port=65534
I get the below stated exception -
[nandanasgn@gw02 ~]$ pyspark --master yarn --conf spark.ui.port=65534
Multiple versions of Spark are installed but SPARK_MAJOR_VERSION is not set
Spark1 will be picked by default
… … …
The number of live datanodes 5 has reached the minimum number 0. Safe mode will be turn ed off automatically once the thresholds have been reached.
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkNameNodeSafeMode(FS Namesystem.java:1359)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.mkdirs(FSNamesystem.java :4010)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.mkdirs(NameNodeRpcS erver.java:1102)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslator …
at org.apache.hadoop.ipc.Client.call(Client.java:1496)
at org.apache.hadoop.ipc.Client.call(Client.java:1396)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.jav a:233)
at com.sun.proxy.$Proxy24.mkdirs(Unknown Source)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.mkdirs( ClientNamenodeProtocolTranslatorPB.java:603)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62 )
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl .java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocati onHandler.java:278)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHand ler.java:194)
at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHand ler.java:176)
at com.sun.proxy.$Proxy25.mkdirs(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.primitiveMkdir(DFSClient.java:3061)
at org.apache.hadoop.hdfs.DFSClient.mkdirs(DFSClient.java:3031)
at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem .java:1162)
at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem .java:1158)
at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.j ava:81)
at org.apache.hadoop.hdfs.DistributedFileSystem.mkdirsInternal(DistributedFileS ystem.java:1158)
at org.apache.hadoop.hdfs.DistributedFileSystem.mkdirs(DistributedFileSystem.ja va:1150)
at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:1898)
at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:613)
at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:394)
at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scal a:763)
at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:143)
at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClie ntSchedulerBackend.scala:56)
at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:1 44)
at org.apache.spark.SparkContext.(SparkContext.scala:530)
at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:59)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcces sorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstruc torAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:234)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
at py4j.Gateway.invoke(Gateway.java:214)
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:7 9)
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:68)
at py4j.GatewayConnection.run(GatewayConnection.java:209)
at java.lang.Thread.run(Thread.java:745)
18/07/20 04:25:27 ERROR SparkContext: Error initializing SparkContext.
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.SafeModeEx ception): Cannot create directory /user/nandanasgn/.sparkStaging/application_1528589352 821_28823. Name node is in safe mode.
The reported blocks 821815 needs additional 67 blocks to reach the threshold 1.0000 of total blocks 821881.
The number of live datanodes 5 has reached the minimum number 0. Safe mode will be turn ed off automatically once the thresholds have been reached.
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkNameNodeSafeMode(FS Namesystem.java:1359)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.mkdirs(FSNamesystem.java :4010)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.mkdirs(NameNodeRpcS erver.java:1102)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslator PB.mkdirs(ClientNamenodeProtocolServerSideTranslatorPB.java:630)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNam enodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(Proto bufRpcEngine.java:640)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2313)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2309)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.ja va:1724)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2307)

    at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1552)
    at org.apache.hadoop.ipc.Client.call(Client.java:1496)
    at org.apache.hadoop.ipc.Client.call(Client.java:1396)
    at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.jav                                                                                                     a:233)
    at com.sun.proxy.$Proxy24.mkdirs(Unknown Source)
    at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.mkdirs(                                                                                                     ClientNamenodeProtocolTranslatorPB.java:603)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62                                                                                                     )
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl                                                                                                     .java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocati                                                                                                     onHandler.java:278)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHand                                                                                                     ler.java:194)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHand                                                                                                     ler.java:176)
    at com.sun.proxy.$Proxy25.mkdirs(Unknown Source)
    at org.apache.hadoop.hdfs.DFSClient.primitiveMkdir(DFSClient.java:3061)
    at org.apache.hadoop.hdfs.DFSClient.mkdirs(DFSClient.java:3031)
    at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem                                                                                                     .java:1162)
    at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem                                                                                                     .java:1158)
    at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.j                                                                                                     ava:81)
    at org.apache.hadoop.hdfs.DistributedFileSystem.mkdirsInternal(DistributedFileS                                                                                                     ystem.java:1158)
    at org.apache.hadoop.hdfs.DistributedFileSystem.mkdirs(DistributedFileSystem.ja                                                                                                     va:1150)
    at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:1898)
    at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:613)
    at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:394)
    at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scal                                                                                                     a:763)
    at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:143)
    at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClie                                                                                                     ntSchedulerBackend.scala:56)
    at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:1                                                                                                     44)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:530)
    at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:59)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcces                                                                                                     sorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstruc                                                                                                     torAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:234)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
    at py4j.Gateway.invoke(Gateway.java:214)
    at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:7                                                                                                     9)
    at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:68)
    at py4j.GatewayConnection.run(GatewayConnection.java:209)
    at java.lang.Thread.run(Thread.java:745)

18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/st age/kill,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,nu ll}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors /threadDump/json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors /threadDump,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors /json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors ,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/environme nt/json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/environme nt,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/r dd/json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/r dd,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/j son,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,n ull}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/po ol/json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/po ol,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/st age/json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/st age,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/js on,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,nu ll}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/ json,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job, null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json ,null}
18/07/20 04:25:27 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null }
18/07/20 04:25:27 INFO SparkUI: Stopped Spark web UI at http://172.16.1.109:65534
18/07/20 04:25:27 WARN YarnSchedulerBackend$YarnSchedulerEndpoint: Attempted to request executors before the AM has registered!
18/07/20 04:25:27 INFO YarnClientSchedulerBackend: Stopped
18/07/20 04:25:27 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint s topped!
18/07/20 04:25:27 ERROR Utils: Uncaught exception in thread Thread-2
java.lang.NullPointerException
at org.apache.spark.network.shuffle.ExternalShuffleClient.close(ExternalShuffle Client.java:152)
at org.apache.spark.storage.BlockManager.stop(BlockManager.scala:1286)
at org.apache.spark.SparkEnv.stop(SparkEnv.scala:96)
at org.apache.spark.SparkContext$$anonfun$stop$12.apply$mcV$sp(SparkContext.sca la:1780)
at org.apache.spark.util.Utils$.tryLogNonFatalError(Utils.scala:1219)
at org.apache.spark.SparkContext.stop(SparkContext.scala:1779)
at org.apache.spark.SparkContext.(SparkContext.scala:602)
at org.apache.spark.api.java.JavaSparkContext.(JavaSparkContext.scala:59)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcces sorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstruc torAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:234)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
at py4j.Gateway.invoke(Gateway.java:214)
at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:7 9)
at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:68)
at py4j.GatewayConnection.run(GatewayConnection.java:209)
at java.lang.Thread.run(Thread.java:745)
18/07/20 04:25:27 INFO SparkContext: Successfully stopped SparkContext
Traceback (most recent call last):
File “/usr/hdp/2.5.0.0-1245/spark/python/pyspark/shell.py”, line 43, in
sc = SparkContext(pyFiles=add_files)
File “/usr/hdp/2.5.0.0-1245/spark/python/pyspark/context.py”, line 115, in init
conf, jsc, profiler_cls)
File “/usr/hdp/2.5.0.0-1245/spark/python/pyspark/context.py”, line 172, in _do_init
self._jsc = jsc or self._initialize_context(self._conf._jconf)
File “/usr/hdp/2.5.0.0-1245/spark/python/pyspark/context.py”, line 235, in _initializ e_context
return self._jvm.JavaSparkContext(jconf)
File “/usr/hdp/2.5.0.0-1245/spark/python/lib/py4j-0.9-src.zip/py4j/java_gateway.py”, line 1064, in call
File “/usr/hdp/2.5.0.0-1245/spark/python/lib/py4j-0.9-src.zip/py4j/protocol.py”, line 308, in get_return_value
py4j.protocol.Py4JJavaError: An error occurred while calling None.org.apache.spark.api. java.JavaSparkContext.
: org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.hdfs.server.namenode.SafeMode Exception): Cannot create directory /user/nandanasgn/.sparkStaging/application_15285893 52821_28823. Name node is in safe mode.
The reported blocks 821815 needs additional 67 blocks to reach the threshold 1.0000 of total blocks 821881.
The number of live datanodes 5 has reached the minimum number 0. Safe mode will be turn ed off automatically once the thresholds have been reached.
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkNameNodeSafeMode(FS Namesystem.java:1359)
at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.mkdirs(FSNamesystem.java :4010)
at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.mkdirs(NameNodeRpcS erver.java:1102)
at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslator PB.mkdirs(ClientNamenodeProtocolServerSideTranslatorPB.java:630)
at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNam enodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(Proto bufRpcEngine.java:640)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2313)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2309)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.ja va:1724)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2307)

    at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1552)
    at org.apache.hadoop.ipc.Client.call(Client.java:1496)
    at org.apache.hadoop.ipc.Client.call(Client.java:1396)
    at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.jav                                                                                                     a:233)
    at com.sun.proxy.$Proxy24.mkdirs(Unknown Source)
    at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.mkdirs(                                                                                                     ClientNamenodeProtocolTranslatorPB.java:603)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62                                                                                                     )
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl                                                                                                     .java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocati                                                                                                     onHandler.java:278)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHand                                                                                                     ler.java:194)
    at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHand                                                                                                     ler.java:176)
    at com.sun.proxy.$Proxy25.mkdirs(Unknown Source)
    at org.apache.hadoop.hdfs.DFSClient.primitiveMkdir(DFSClient.java:3061)
    at org.apache.hadoop.hdfs.DFSClient.mkdirs(DFSClient.java:3031)
    at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem                                                                                                     .java:1162)
    at org.apache.hadoop.hdfs.DistributedFileSystem$24.doCall(DistributedFileSystem                                                                                                     .java:1158)
    at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.j                                                                                                     ava:81)
    at org.apache.hadoop.hdfs.DistributedFileSystem.mkdirsInternal(DistributedFileS                                                                                                     ystem.java:1158)
    at org.apache.hadoop.hdfs.DistributedFileSystem.mkdirs(DistributedFileSystem.ja                                                                                                     va:1150)
    at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:1898)
    at org.apache.hadoop.fs.FileSystem.mkdirs(FileSystem.java:613)
    at org.apache.spark.deploy.yarn.Client.prepareLocalResources(Client.scala:394)
    at org.apache.spark.deploy.yarn.Client.createContainerLaunchContext(Client.scal                                                                                                     a:763)
    at org.apache.spark.deploy.yarn.Client.submitApplication(Client.scala:143)
    at org.apache.spark.scheduler.cluster.YarnClientSchedulerBackend.start(YarnClie                                                                                                     ntSchedulerBackend.scala:56)
    at org.apache.spark.scheduler.TaskSchedulerImpl.start(TaskSchedulerImpl.scala:1                                                                                                     44)
    at org.apache.spark.SparkContext.<init>(SparkContext.scala:530)
    at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:59)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
    at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAcces                                                                                                     sorImpl.java:62)
    at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstruc                                                                                                     torAccessorImpl.java:45)
    at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
    at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:234)
    at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:381)
    at py4j.Gateway.invoke(Gateway.java:214)
    at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:7                                                                                                     9)
    at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:68)
    at py4j.GatewayConnection.run(GatewayConnection.java:209)
    at java.lang.Thread.run(Thread.java:745)

#2

@nandanasgn Issue resolved.Can you please try again?


#3

Thank you @annapurna, I could launch spark successfully.


#4