Simple Dataframe insert into Hive table failing on cluster seems like @ Admin issue

apache-spark

#1

My Code - If I run individually in spark -shell it runs. But when I create jar file and try executing on cluster it throws error-

Code
package MavenDemo.project_scala
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.functions._
import org.apache.spark.{SparkConf, SparkContext}
import scala.collection.mutable.ListBuffer
import org.apache.spark.sql.SparkSession
//import org.apache.spark.

object OrdersOrdItemsJoin {
def main(args: Array[String]): Unit = {

val spark = SparkSession
.builder()
.appName(“Spark SQL basic example”)
.config(“spark.some.config.option”, “some-value”)
.getOrCreate()

import spark.implicits._

val orders_ord_items_join_df= spark.sql(""“select op.order_id,op.order_date,
op.order_status,oi.order_item_subtotal from ishti_sqoop_import.orders op
left join ishti_sqoop_import.order_items oi on op.order_id=oi.order_item_order_id
where op.order_status=‘CLOSED’ or op.order_status=‘COMPLETED’ “””)

orders_ord_items_join_df.write.mode(“overwrite”).insertInto(“ishti_sqoop_import.orders_ord_items_join”)
}
}

====================
Error-

18/11/01 03:42:06 INFO YarnClientSchedulerBackend: Application application_1540458187951_2898 has started running.
18/11/01 03:42:06 INFO Utils: Successfully started service ‘org.apache.spark.network.netty.NettyBlockTransferService’ on port 41164.
18/11/01 03:42:06 INFO NettyBlockTransferService: Server created on gw03.itversity.com:41164
18/11/01 03:42:06 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
18/11/01 03:42:06 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, gw03.itversity.com, 41164, None)
18/11/01 03:42:06 INFO BlockManagerMasterEndpoint: Registering block manager gw03.itversity.com:41164 with 366.3 MB RAM, BlockManagerId(driver, gw03.itversity.com, 41164, None)
18/11/01 03:42:06 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, gw03.itversity.com, 41164, None)
18/11/01 03:42:06 INFO BlockManager: external shuffle service port = 7447
18/11/01 03:42:06 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, gw03.itversity.com, 41164, None)
18/11/01 03:42:07 INFO EventLoggingListener: Logging events to hdfs:/spark2-history/application_1540458187951_2898
18/11/01 03:42:07 INFO Utils: Using initial executors = 0, max of spark.dynamicAllocation.initialExecutors, spark.dynamicAllocation.minExecutors and spark.executor.instances
18/11/01 03:42:07 INFO YarnClientSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.8
18/11/01 03:42:07 INFO SharedState: loading hive config file: file:/etc/spark2/2.6.5.0-292/0/hive-site.xml
18/11/01 03:42:07 INFO SharedState: Setting hive.metastore.warehouse.dir (‘null’) to the value of spark.sql.warehouse.dir (‘file:/etc/spark2/2.6.5.0-292/0/spark-warehouse’).
18/11/01 03:42:07 INFO SharedState: Warehouse path is ‘file:/etc/spark2/2.6.5.0-292/0/spark-warehouse’.
18/11/01 03:42:08 INFO StateStoreCoordinatorRef: Registered StateStoreCoordinator endpoint
Exception in thread “main” org.apache.spark.sql.AnalysisException: Table or view not found: ishti_sqoop_import.orders; line 2 pos 47;
'Project ['op.order_id, 'op.order_date, 'op.order_status, 'oi.order_item_subtotal]
± 'Filter (('op.order_status = CLOSED) || ('op.order_status = COMPLETED))
± 'Join LeftOuter, ('op.order_id = 'oi.order_item_order_id)
:- 'SubqueryAlias op
: ± 'UnresolvedRelation ishti_sqoop_import.orders
± 'SubqueryAlias oi
± 'UnresolvedRelation ishti_sqoop_import.order_items

    at org.apache.spark.sql.catalyst.analysis.package$AnalysisErrorAt.failAnalysis(package.scala:42)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:82)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$$anonfun$checkAnalysis$1.apply(CheckAnalysis.scala:80)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:127)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$foreachUp$1.apply(TreeNode.scala:126)
    at scala.collection.immutable.List.foreach(List.scala:381)
    at org.apache.spark.sql.catalyst.trees.TreeNode.foreachUp(TreeNode.scala:126)
    at org.apache.spark.sql.catalyst.analysis.CheckAnalysis$class.checkAnalysis(CheckAnalysis.scala:80)
    at org.apache.spark.sql.catalyst.analysis.Analyzer.checkAnalysis(Analyzer.scala:92)
    at org.apache.spark.sql.catalyst.analysis.Analyzer.executeAndCheck(Analyzer.scala:105)
    at org.apache.spark.sql.execution.QueryExecution.analyzed$lzycompute(QueryExecution.scala:57)
    at org.apache.spark.sql.execution.QueryExecution.analyzed(QueryExecution.scala:55)
    at org.apache.spark.sql.execution.QueryExecution.assertAnalyzed(QueryExecution.scala:47)
    at org.apache.spark.sql.Dataset$.ofRows(Dataset.scala:74)
    at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:638)
    at MavenDemo.project_scala.OrdersOrdItemsJoin$.main(OrdersOrdItemsJoin.scala:24)
    at MavenDemo.project_scala.OrdersOrdItemsJoin.main(OrdersOrdItemsJoin.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
    at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:906)
    at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:197)
    at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:227)
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:136)
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

18/11/01 03:42:09 INFO SparkContext: Invoking stop() from shutdown hook
18/11/01 03:42:09 INFO SparkUI: Stopped Spark web UI at http://gw03.itversity.com:4048
18/11/01 03:42:09 INFO YarnClientSchedulerBackend: Interrupting monitor thread
18/11/01 03:42:09 INFO YarnClientSchedulerBackend: Shutting down all executors
18/11/01 03:42:09 INFO YarnSchedulerBackend$YarnDriverEndpoint: Asking each executor to shut down
18/11/01 03:42:09 INFO SchedulerExtensionServices: Stopping SchedulerExtensionServices