SparkSQL03

在object LogApp {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .master("local[4]")
      .appName("LogApp")
      .getOrCreate()

    // ETL: 一定保留原有的数据   最完整
    var inputDF = spark.read.json("data/data-test.json")
   inputDF = inputDF.withColumn("province", MydataUDF.getProvince(inputDF.col("ip")))
   inputDF = inputDF.withColumn("city", MydataUDF.getCity(inputDF.col("ip")))

    // ETL==>ODS
  //  inputDF.coalesce(1).write.format("parquet")     //orc /parquet
 //     .option("compression","snappy").save(" path")    //别使用snappy 用lzo

    inputDF.createOrReplaceTempView("log")

    spark.conf.set("spark.sql.shuffle.partitions","400") // --conf  这个东西不建议写在代码里 建议写在 spark-submit --conf 那块 明白吗？
    
    val areaSQL01 = "select province,city, " +
      "sum(case when requestmode=1 and processnode >=1 then 1 else 0 end) origin_request," +
      "sum(case when requestmode=1 and processnode >=2 then 1 else 0 end) valid_request," +
      "sum(case when requestmode=1 and processnode =3 then 1 else 0 end) ad_request," +
      "sum(case when adplatformproviderid>=100000 and iseffective=1 and isbilling=1 and isbid=1 and adorderid!=0 then 1 else 0 end) bid_cnt," +
      "sum(case when adplatformproviderid>=100000 and iseffective=1 and isbilling=1 and iswin=1 then 1 else 0 end) bid_success_cnt," +
      "sum(case when requestmode=2 and iseffective=1 then 1 else 0 end) ad_display_cnt," +
      "sum(case when requestmode=3 and processnode=1 then 1 else 0 end) ad_click_cnt," +
      "sum(case when requestmode=2 and iseffective=1 and isbilling=1 then 1 else 0 end) medium_display_cnt," +
      "sum(case when requestmode=3 and iseffective=1 and isbilling=1 then 1 else 0 end) medium_click_cnt," +
      "sum(case when adplatformproviderid>=100000 and iseffective=1 and isbilling=1 and iswin=1 and adorderid>20000  then 1*winprice/1000 else 0 end) ad_consumption," +
      "sum(case when adplatformproviderid>=100000 and iseffective=1 and isbilling=1 and iswin=1 and adorderid>20000  then 1*adpayment/1000 else 0 end) ad_cost " +
      "from log group by province,city"
    spark.sql(areaSQL01).show(false)//.createOrReplaceTempView("area_tmp")



    val areaSQL02 = "select province,city, " +
      "origin_request," +
      "valid_request," +
      "ad_request," +
      "bid_cnt," +
      "bid_success_cnt," +
      "bid_success_cnt/bid_cnt bid_success_rate," +
      "ad_display_cnt," +
      "ad_click_cnt," +
      "ad_click_cnt/ad_display_cnt ad_click_rate," +
      "ad_consumption," +
      "ad_cost from area_tmp " +
      "where bid_cnt!=0 and ad_display_cnt!=0"

    Thread.sleep(10000)

    spark.sql(areaSQL02).show(false)

    spark.stop()
  }
}


object MydataUDF {

  import org.apache.spark.sql.functions._

  def getProvince = udf((ip:String)=>{
    IPUtil.getInstance().getInfos(ip)(1)
  })

  def getCity = udf((ip:String)=>{
    IPUtil.getInstance().getInfos(ip)(2)
  })
}


问题：
有什么问题？
1.spark.conf.set("spark.sql.shuffle.partitions","400") // --conf  
这个东西不建议写在代码里 建议写在 spark-submit --conf 那块 明白吗？ 或者通过代码判断输入值 eg:400

2. inputDF.coalesce(1).write.format("parquet")     //orc /parquet
.option("compression","snappy").save(" path")    //别使用snappy 用lzo
spark默认是snappy 别用哈 看压缩篇

coalesce(1) 这个值 看下面给的建议  处理小文件的

演示上面代码可能的问题
在这里插入图片描述

这200 哪里来的 ？

    官网 sparksql 调优章节 
1.spark.sql.shuffle.partitions 参数 默认 200   这是sparksql里面 设置的shuffle参数 

2.RDD里的 reduceByKey(，numPartitions）还有印象吗？rdd是在这里设置的 

sparksql 默认200  生产上绝对是不够的 只要你数据量稍微大一点 200个 一定是扛不住的 

这个参数 你可以理解为mapreduce里的reduce的数量 ，reduce数量如果大了 
会导致上面问题？程序跑起来是快了 但是 小文件过多 

那么这个值 该怎么设置呢？
给你个思路 估计你读进来的数据量大小 + 你预估你每个task处理的数据量是多少 
来设计 这个值

还有一点就是：
加入这个值是400  
400
    大：小文件多点、
    10exe * 2core = 20task   同一时间点 20个task
               400/20=20轮
               600/20=30轮

ETL

ETL
    input:json
    清洗 ==> ODS  大宽表  HDFS/Hive/SparkSQL
    output: 列式存储  ORC/Parquet   这块一定是要落地的 

    Stat
        ==>  一个非常简单的SQL搞定
        ==>  复杂：多个SQL 或者 一个复杂SQL搞定

Choose Parquet for Spark SQL

行式存储：MySQL
    一条记录有多个列  一行数据是存储在一起的
    优点：
        你每次查询都使用到所有的列
    缺点：
        大宽表有N多列，但是我们仅仅使用其中几列
 	
 	因为我使用大宽表(有100列)的时候 假如只用到其中的3个列，
 	如果我使用 行式存储   加载数据的时候会把 你一行的所有列都加载出来 意味着浪费了97%资源
 
 列式存储很好的解决这个问题

列式：Orc Parquet
    特点：把每一列的数据存放在一起
    优点：减少IO 需要哪几列就直接获取哪几列
    缺点：如果你还是要获取每一行中的所有列，那么性能比行式的差

在这里插入图片描述

使用行式存储 spark跑程序的时候官网也列举了很多问题 
eg：

在这里插入图片描述

那么Most of these failures force Spark to re-try by re-queuing tasks：
spark会重试跑失败的task 
注意：
	重试 一般是跑不出来的  如果没有倾斜 和资源够 可能会跑出来
	假设10个task 3个task挂掉了 那么重新起的task 你能确定 
	重启来的task 会在 3个task之前挂掉的executor上面么？
	不能确定 很可能起到别的executor上面 
	（别的executor 可能现在也在跑 其余7个task中的某些task）
	对于这个 executor压力更大 可能会导致你的应用程序被干掉

存储是结合 压缩来用的   eg：orc + lzo
减少disk io

beeline/jdbc

生产上是用的

hiveserver2  beeline/jdbc     Hive里的 
thriftserver beeline/jdbc     spark里的 

怎么用呢？

[double_happy@hadoop101 sbin]$ ./start-thriftserver.sh --jars ~/software/mysql-connector-java-5.1.47.jar 
starting org.apache.spark.sql.hive.thriftserver.HiveThriftServer2, logging to /home/double_happy/app/spark/logs/spark-double_happy-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1-hadoop101.out

[double_happy@hadoop101 sbin]$ tail -200f /home/double_happy/app/spark/logs/spark-double_happy-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1-hadoop101.out


Spark Command: /usr/java/java/bin/java -cp /home/double_happy/app/spark/conf/:/home/double_happy/app/spark/jars/*:/home/double_happy/app/hadoop/etc/hadoop/ -Xmx1g org.apache.spark.deploy.SparkSubmit --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 --name Thrift JDBC/ODBC Server --jars /home/double_happy/software/mysql-connector-java-5.1.47.jar spark-internal
========================================
19/10/28 22:26:55 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
19/10/28 22:26:56 INFO HiveThriftServer2: Started daemon with process name: 2297@hadoop101
19/10/28 22:26:56 INFO SignalUtils: Registered signal handler for TERM
19/10/28 22:26:56 INFO SignalUtils: Registered signal handler for HUP
19/10/28 22:26:56 INFO SignalUtils: Registered signal handler for INT
19/10/28 22:26:56 INFO HiveThriftServer2: Starting SparkContext
19/10/28 22:26:56 INFO SparkContext: Running Spark version 2.4.4
19/10/28 22:26:56 INFO SparkContext: Submitted application: Thrift JDBC/ODBC Server
19/10/28 22:26:57 INFO SecurityManager: Changing view acls to: double_happy
19/10/28 22:26:57 INFO SecurityManager: Changing modify acls to: double_happy
19/10/28 22:26:57 INFO SecurityManager: Changing view acls groups to: 
19/10/28 22:26:57 INFO SecurityManager: Changing modify acls groups to: 
19/10/28 22:26:57 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(double_happy); groups with view permissions: Set(); users  with modify permissions: Set(double_happy); groups with modify permissions: Set()
19/10/28 22:26:57 INFO Utils: Successfully started service 'sparkDriver' on port 35237.
19/10/28 22:26:57 INFO SparkEnv: Registering MapOutputTracker
19/10/28 22:26:57 INFO SparkEnv: Registering BlockManagerMaster
19/10/28 22:26:57 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
19/10/28 22:26:57 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
19/10/28 22:26:57 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-cb4561a2-a2c1-42a6-a313-96e3ff47a7fb
19/10/28 22:26:57 INFO MemoryStore: MemoryStore started with capacity 366.3 MB
19/10/28 22:26:58 INFO SparkEnv: Registering OutputCommitCoordinator
19/10/28 22:26:58 INFO Utils: Successfully started service 'SparkUI' on port 4040.
19/10/28 22:26:58 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://hadoop101:4040
19/10/28 22:26:58 INFO SparkContext: Added JAR file:///home/double_happy/software/mysql-connector-java-5.1.47.jar at spark://hadoop101:35237/jars/mysql-connector-java-5.1.47.jar with timestamp 1572272818597
19/10/28 22:26:58 INFO Executor: Starting executor ID driver on host localhost
19/10/28 22:26:59 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 33661.
19/10/28 22:26:59 INFO NettyBlockTransferService: Server created on hadoop101:33661
19/10/28 22:26:59 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
19/10/28 22:26:59 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, hadoop101, 33661, None)
19/10/28 22:26:59 INFO BlockManagerMasterEndpoint: Registering block manager hadoop101:33661 with 366.3 MB RAM, BlockManagerId(driver, hadoop101, 33661, None)
19/10/28 22:26:59 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, hadoop101, 33661, None)
19/10/28 22:26:59 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, hadoop101, 33661, None)
19/10/28 22:27:01 INFO EventLoggingListener: Logging events to hdfs://hadoop101:8020/spark_directory/local-1572272818750
19/10/28 22:27:01 INFO SharedState: loading hive config file: file:/home/double_happy/app/spark-2.4.4-bin-2.6.0-cdh5.15.1/conf/hive-site.xml
19/10/28 22:27:01 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/double_happy/app/spark-2.4.4-bin-2.6.0-cdh5.15.1/sbin/spark-warehouse').
19/10/28 22:27:01 INFO SharedState: Warehouse path is 'file:/home/double_happy/app/spark-2.4.4-bin-2.6.0-cdh5.15.1/sbin/spark-warehouse'.
19/10/28 22:27:01 INFO HiveUtils: Initializing HiveMetastoreConnection version 1.2.1 using Spark classes.
19/10/28 22:27:03 INFO HiveMetaStore: 0: Opening raw store with implemenation class:org.apache.hadoop.hive.metastore.ObjectStore
19/10/28 22:27:03 INFO ObjectStore: ObjectStore, initialize called
19/10/28 22:27:03 INFO Persistence: Property hive.metastore.integral.jdo.pushdown unknown - will be ignored
19/10/28 22:27:03 INFO Persistence: Property datanucleus.cache.level2 unknown - will be ignored
19/10/28 22:27:05 INFO ObjectStore: Setting MetaStore object pin classes with hive.metastore.cache.pinobjtypes="Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"
19/10/28 22:27:07 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:07 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:08 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:08 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:08 INFO Query: Reading in results for query "org.datanucleus.store.rdbms.query.SQLQuery@0" since the connection used is closing
19/10/28 22:27:08 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is MYSQL
19/10/28 22:27:08 INFO ObjectStore: Initialized ObjectStore
19/10/28 22:27:08 INFO HiveMetaStore: Added admin role in metastore
19/10/28 22:27:08 INFO HiveMetaStore: Added public role in metastore
19/10/28 22:27:09 INFO HiveMetaStore: No user is added in admin role, since config is empty
19/10/28 22:27:09 INFO HiveMetaStore: 0: get_all_databases
19/10/28 22:27:09 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_all_databases
19/10/28 22:27:09 INFO HiveMetaStore: 0: get_functions: db=default pat=*
19/10/28 22:27:09 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_functions: db=default pat=*
19/10/28 22:27:09 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MResourceUri" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:09 INFO HiveMetaStore: 0: get_functions: db=homework pat=*
19/10/28 22:27:09 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_functions: db=homework pat=*
19/10/28 22:27:09 INFO HiveMetaStore: 0: get_function: homework.add_prefix_new
19/10/28 22:27:09 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_function: homework.add_prefix_new
19/10/28 22:27:10 INFO HiveMetaStore: 0: get_function: homework.remove_prefix_new
19/10/28 22:27:10 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_function: homework.remove_prefix_new
19/10/28 22:27:10 INFO SessionState: Created local directory: /tmp/41aaf1c8-5deb-45c7-9c03-ef172a6058a3_resources
19/10/28 22:27:10 INFO SessionState: Created HDFS directory: /tmp/hive/double_happy/41aaf1c8-5deb-45c7-9c03-ef172a6058a3
19/10/28 22:27:10 INFO SessionState: Created local directory: /tmp/double_happy/41aaf1c8-5deb-45c7-9c03-ef172a6058a3
19/10/28 22:27:10 INFO SessionState: Created HDFS directory: /tmp/hive/double_happy/41aaf1c8-5deb-45c7-9c03-ef172a6058a3/_tmp_space.db
19/10/28 22:27:10 INFO HiveClientImpl: Warehouse location for Hive client (version 1.2.2) is file:/home/double_happy/app/spark-2.4.4-bin-2.6.0-cdh5.15.1/sbin/spark-warehouse
19/10/28 22:27:10 INFO HiveMetaStore: 0: get_database: default
19/10/28 22:27:10 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_database: default
19/10/28 22:27:10 INFO StateStoreCoordinatorRef: Registered StateStoreCoordinator endpoint
19/10/28 22:27:10 INFO HiveUtils: Initializing execution hive, version 1.2.1
19/10/28 22:27:11 INFO HiveMetaStore: 0: Opening raw store with implemenation class:org.apache.hadoop.hive.metastore.ObjectStore
19/10/28 22:27:11 INFO ObjectStore: ObjectStore, initialize called
19/10/28 22:27:11 INFO Persistence: Property hive.metastore.integral.jdo.pushdown unknown - will be ignored
19/10/28 22:27:11 INFO Persistence: Property datanucleus.cache.level2 unknown - will be ignored
19/10/28 22:27:14 INFO ObjectStore: Setting MetaStore object pin classes with hive.metastore.cache.pinobjtypes="Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"
19/10/28 22:27:15 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:15 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:17 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:17 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:17 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is DERBY
19/10/28 22:27:17 INFO ObjectStore: Initialized ObjectStore
19/10/28 22:27:17 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 1.2.0
19/10/28 22:27:17 WARN ObjectStore: Failed to get database default, returning NoSuchObjectException
19/10/28 22:27:18 INFO HiveMetaStore: Added admin role in metastore
19/10/28 22:27:18 INFO HiveMetaStore: Added public role in metastore
19/10/28 22:27:18 INFO HiveMetaStore: No user is added in admin role, since config is empty
19/10/28 22:27:18 INFO HiveMetaStore: 0: get_all_databases
19/10/28 22:27:18 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_all_databases
19/10/28 22:27:18 INFO HiveMetaStore: 0: get_functions: db=default pat=*
19/10/28 22:27:18 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_functions: db=default pat=*
19/10/28 22:27:18 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MResourceUri" is tagged as "embedded-only" so does not have its own datastore table.
19/10/28 22:27:18 INFO SessionState: Created local directory: /tmp/5fe8af31-b32b-4788-a587-4fbf6cab7b1a_resources
19/10/28 22:27:18 INFO SessionState: Created HDFS directory: /tmp/hive/double_happy/5fe8af31-b32b-4788-a587-4fbf6cab7b1a
19/10/28 22:27:18 INFO SessionState: Created local directory: /tmp/double_happy/5fe8af31-b32b-4788-a587-4fbf6cab7b1a
19/10/28 22:27:18 INFO SessionState: Created HDFS directory: /tmp/hive/double_happy/5fe8af31-b32b-4788-a587-4fbf6cab7b1a/_tmp_space.db
19/10/28 22:27:18 INFO HiveClientImpl: Warehouse location for Hive client (version 1.2.2) is file:/home/double_happy/app/spark-2.4.4-bin-2.6.0-cdh5.15.1/sbin/spark-warehouse
19/10/28 22:27:18 INFO SessionManager: Operation log root directory is created: /tmp/double_happy/operation_logs
19/10/28 22:27:18 INFO SessionManager: HiveServer2: Background operation thread pool size: 100
19/10/28 22:27:18 INFO SessionManager: HiveServer2: Background operation thread wait queue size: 100
19/10/28 22:27:18 INFO SessionManager: HiveServer2: Background operation thread keepalive time: 10 seconds
19/10/28 22:27:18 INFO AbstractService: Service:OperationManager is inited.
19/10/28 22:27:18 INFO AbstractService: Service:SessionManager is inited.
19/10/28 22:27:18 INFO AbstractService: Service: CLIService is inited.
19/10/28 22:27:18 INFO AbstractService: Service:ThriftBinaryCLIService is inited.
19/10/28 22:27:18 INFO AbstractService: Service: HiveServer2 is inited.
19/10/28 22:27:18 INFO AbstractService: Service:OperationManager is started.
19/10/28 22:27:18 INFO AbstractService: Service:SessionManager is started.
19/10/28 22:27:18 INFO AbstractService: Service:CLIService is started.
19/10/28 22:27:18 INFO ObjectStore: ObjectStore, initialize called
19/10/28 22:27:18 INFO Query: Reading in results for query "org.datanucleus.store.rdbms.query.SQLQuery@0" since the connection used is closing
19/10/28 22:27:18 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is DERBY
19/10/28 22:27:18 INFO ObjectStore: Initialized ObjectStore
19/10/28 22:27:18 INFO HiveMetaStore: 0: get_databases: default
19/10/28 22:27:18 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_databases: default
19/10/28 22:27:18 INFO HiveMetaStore: 0: Shutting down the object store...
19/10/28 22:27:18 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=Shutting down the object store...
19/10/28 22:27:18 INFO HiveMetaStore: 0: Metastore shutdown complete.
19/10/28 22:27:18 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=Metastore shutdown complete.
19/10/28 22:27:18 INFO AbstractService: Service:ThriftBinaryCLIService is started.
19/10/28 22:27:18 INFO AbstractService: Service:HiveServer2 is started.
19/10/28 22:27:18 INFO HiveThriftServer2: HiveThriftServer2 started
19/10/28 22:27:18 INFO ThriftCLIService: Starting ThriftBinaryCLIService on port 10000 with 5...500 worker threads

说明 thriftserver 启动起来了 

sparkui端口 参数
spark.port.maxRetries 16 默认16 也就是同一时间点对一台机器 只能起16个spark-submit 

[double_happy@hadoop101 software]$ jps
2496 Jps
4289 NodeManager
4019 SecondaryNameNode
14999 AzkabanSingleServer
6633 HistoryServer
2297 SparkSubmit
3721 NameNode
4186 ResourceManager
3853 DataNode
[double_happy@hadoop101 software]$ 


也就是这个 2297 SparkSubmit  最多16个

在这里插入图片描述

这是 thriftserver 端起来了 说明服务端有了
所以接下来要通过客户端 连接一下 
客户端怎么链接呢？
使用beeline     用法跟Hive里是一毛一样的

[double_happy@hadoop101 bin]$ ./beeline -u jdbc:hive2://hadoop101:10000/ruozedata_g7 -n double_happy
Connecting to jdbc:hive2://hadoop101:10000/ruozedata_g7
19/10/28 22:40:56 INFO Utils: Supplied authorities: hadoop101:10000
19/10/28 22:40:56 INFO Utils: Resolved authority: hadoop101:10000
19/10/28 22:40:56 INFO HiveConnection: Will try to open client transport with JDBC Uri: jdbc:hive2://hadoop101:10000/ruozedata_g7
Error: Database 'ruozedata_g7' not found; (state=,code=0)
Beeline version 1.2.1.spark2 by Apache Hive
0: jdbc:hive2://hadoop101:10000/ruozedata_g7 (closed)> ^C^C[double_happy@hadoop101 bin]$ 
[double_happy@hadoop101 bin]$ ./beeline -u jdbc:hive2://hadoop101:10000/ -n double_happy            
Connecting to jdbc:hive2://hadoop101:10000/
19/10/28 22:42:15 INFO Utils: Supplied authorities: hadoop101:10000
19/10/28 22:42:15 INFO Utils: Resolved authority: hadoop101:10000
19/10/28 22:42:16 INFO HiveConnection: Will try to open client transport with JDBC Uri: jdbc:hive2://hadoop101:10000/
Connected to: Spark SQL (version 2.4.4)
Driver: Hive JDBC (version 1.2.1.spark2)
Transaction isolation: TRANSACTION_REPEATABLE_READ
Beeline version 1.2.1.spark2 by Apache Hive
0: jdbc:hive2://hadoop101:10000/> show databases;
+---------------+--+
| databaseName  |
+---------------+--+
| default       |
| homework      |
+---------------+--+
2 rows selected (1.206 seconds)
0: jdbc:hive2://hadoop101:10000/> use homework;
+---------+--+
| Result  |
+---------+--+
+---------+--+
No rows selected (0.181 seconds)
0: jdbc:hive2://hadoop101:10000/> show tables;
+-----------+------------------------------------+--------------+--+
| database  |             tableName              | isTemporary  |
+-----------+------------------------------------+--------------+--+
| homework  | access_wide                        | false        |
| homework  | dwd_platform_stat_info             | false        |
| homework  | jf_tmp                             | false        |
| homework  | ods_domain_traffic_info            | false        |
| homework  | ods_log_info                       | false        |
| homework  | ods_uid_pid_compression_info       | false        |
| homework  | ods_uid_pid_info                   | false        |
| homework  | ods_uid_pid_info_compression_test  | false        |
+-----------+------------------------------------+--------------+--+
8 rows selected (0.202 seconds)
0: jdbc:hive2://hadoop101:10000/>

[double_happy@hadoop101 bin]$ ./beeline -u jdbc:hive2://hadoop101:10000/homework -n double_happy
Connecting to jdbc:hive2://hadoop101:10000/homework
19/10/28 22:43:14 INFO Utils: Supplied authorities: hadoop101:10000
19/10/28 22:43:14 INFO Utils: Resolved authority: hadoop101:10000
19/10/28 22:43:14 INFO HiveConnection: Will try to open client transport with JDBC Uri: jdbc:hive2://hadoop101:10000/homework
Connected to: Spark SQL (version 2.4.4)
Driver: Hive JDBC (version 1.2.1.spark2)
Transaction isolation: TRANSACTION_REPEATABLE_READ
Beeline version 1.2.1.spark2 by Apache Hive
0: jdbc:hive2://hadoop101:10000/homework> show tables;
+-----------+------------------------------------+--------------+--+
| database  |             tableName              | isTemporary  |
+-----------+------------------------------------+--------------+--+
| homework  | access_wide                        | false        |
| homework  | dwd_platform_stat_info             | false        |
| homework  | jf_tmp                             | false        |
| homework  | ods_domain_traffic_info            | false        |
| homework  | ods_log_info                       | false        |
| homework  | ods_uid_pid_compression_info       | false        |
| homework  | ods_uid_pid_info                   | false        |
| homework  | ods_uid_pid_info_compression_test  | false        |
+-----------+------------------------------------+--------------+--+
8 rows selected (0.352 seconds)
0: jdbc:hive2://hadoop101:10000/homework>

这个东西适用在哪里呢？

你的数据是通过UI去访问的：eg：HUE/Zeppelin  (他们后台都有一个服务的 )
   
   之后可以写一个 jdbc代码 (跟hive里是一模一样的  把你的sql 发到服务 服务给你返回结果 
    通过你的ui界面 把数据结果渲染出来 )
    
    如果你发的SQL是一个计算/统计SQL：返回肯定是需要时间
    只拿结果，不计算

参考官网Distributed SQL Engine

Spark On Yarn
Running Spark on YARN

There are two deploy modes that can be used to launch Spark applications on YARN. In cluster mode, the Spark driver runs inside an application master process which is managed by YARN on the cluster, and the client can go away after initiating the application. In client mode, the driver runs in the client process, and the application master is only used for requesting resources from YARN.

在这里插入图片描述
client模式：

在这里插入图片描述

在Spark on YARN中  是没有Worker的概念，是Standalone中的

Spark on YARN client ：
   1.executor是运行在container中的
   2.driver是跑在本地的

cluster模式：

在这里插入图片描述

spark on yarn 总结：
Spark：Driver + Executors

spark on yarn
    cluster
        driver是运行在AM里面的
        AM：AM + Driver   既当爹又当妈 就是既要给executor发task和代码 也要申请资源
        客户端退出   ？作业是没事的 
        日志 是在YARN上的 ***  本地是看不见的 
            yarn logs -applicationId <app ID>

    client
        driver是运行在本地的      
        客户端退出  作业就退出了
        AM：负责从YARN上去申请资源
        日志是在本地的 ***   方便查看 

1.
  但是 日志在本地会有一个场景 本地的进程是有一定的限制的  
加入你提交多个作业 都是以yarn client模式 那么 进程可能扎堆出现 机器可能会挂掉 

eg：
[double_happy@hadoop101 ~]$ jps
4289 NodeManager
4019 SecondaryNameNode
14999 AzkabanSingleServer
17719 CoarseGrainedExecutorBackend
6633 HistoryServer
3721 NameNode
17689 CoarseGrainedExecutorBackend
4186 ResourceManager
17517 SparkSubmit
17645 ExecutorLauncher
3853 DataNode
17966 Jps
[double_happy@hadoop101 ~]$ 

这是在本地 client 就提交作业 CoarseGrainedExecutorBackend 扎堆出现 多了 机器可能会挂掉

2.driver 和 executor是有通信的 client模式 下 可能会有一种场景存在
driver可以在任意一台机器上面 但是如果这个机器 不是 集群里的机器 (跟yarn 没有关系哈 这里只讨论机器和集群)
如果这机器是在 集群外 这台机器一定是有集群的 gateway权限的 
driver 和 executor是有通信的 网络会用影响 
工作中在集群外的 很少哈 这里只是说一下这个场景 

集群内带宽 很高 上面的场景影响不大 

3.就是client模式就一个弱点 就是 本地进程太多

测试：

[double_happy@hadoop101 ~]$ spark-shell --help
Usage: ./bin/spark-shell [options]

Scala REPL options:
  -I <file>                   preload <file>, enforcing line-by-line interpretation

Options:
  --master MASTER_URL         spark://host:port, mesos://host:port, yarn,
                              k8s://https://host:port, or local (Default: local[*]).
  --deploy-mode DEPLOY_MODE   Whether to launch the driver program locally ("client") or
                              on one of the worker machines inside the cluster ("cluster")
                              (Default: client).
  --class CLASS_NAME          Your application's main class (for Java / Scala apps).
  --name NAME                 A name of your application.
  --jars JARS                 Comma-separated list of jars to include on the driver
                              and executor classpaths.
  --packages                  Comma-separated list of maven coordinates of jars to include
                              on the driver and executor classpaths. Will search the local
                              maven repo, then maven central and any additional remote
                              repositories given by --repositories. The format for the
                              coordinates should be groupId:artifactId:version.
  --exclude-packages          Comma-separated list of groupId:artifactId, to exclude while
                              resolving the dependencies provided in --packages to avoid
                              dependency conflicts.
  --repositories              Comma-separated list of additional remote repositories to
                              search for the maven coordinates given with --packages.
  --py-files PY_FILES         Comma-separated list of .zip, .egg, or .py files to place
                              on the PYTHONPATH for Python apps.
  --files FILES               Comma-separated list of files to be placed in the working
                              directory of each executor. File paths of these files
                              in executors can be accessed via SparkFiles.get(fileName).

  --conf PROP=VALUE           Arbitrary Spark configuration property.
  --properties-file FILE      Path to a file from which to load extra properties. If not
                              specified, this will look for conf/spark-defaults.conf.

  --driver-memory MEM         Memory for driver (e.g. 1000M, 2G) (Default: 1024M).
  --driver-java-options       Extra Java options to pass to the driver.
  --driver-library-path       Extra library path entries to pass to the driver.
  --driver-class-path         Extra class path entries to pass to the driver. Note that
                              jars added with --jars are automatically included in the
                              classpath.

  --executor-memory MEM       Memory per executor (e.g. 1000M, 2G) (Default: 1G).

  --proxy-user NAME           User to impersonate when submitting the application.
                              This argument does not work with --principal / --keytab.

  --help, -h                  Show this help message and exit.
  --verbose, -v               Print additional debug output.
  --version,                  Print the version of current Spark.

 Cluster deploy mode only:
  --driver-cores NUM          Number of cores used by the driver, only in cluster mode
                              (Default: 1).

 Spark standalone or Mesos with cluster deploy mode only:
  --supervise                 If given, restarts the driver on failure.
  --kill SUBMISSION_ID        If given, kills the driver specified.
  --status SUBMISSION_ID      If given, requests the status of the driver specified.

 Spark standalone and Mesos only:
  --total-executor-cores NUM  Total cores for all executors.

 Spark standalone and YARN only:
  --executor-cores NUM        Number of cores per executor. (Default: 1 in YARN mode,
                              or all available cores on the worker in standalone mode)

 YARN-only:
  --queue QUEUE_NAME          The YARN queue to submit to (Default: "default").
  --num-executors NUM         Number of executors to launch (Default: 2).
                              If dynamic allocation is enabled, the initial number of
                              executors will be at least NUM.
  --archives ARCHIVES         Comma separated list of archives to be extracted into the
                              working directory of each executor.
  --principal PRINCIPAL       Principal to be used to login to KDC, while running on
                              secure HDFS.
  --keytab KEYTAB             The full path to the file that contains the keytab for the
                              principal specified above. This keytab will be copied to
                              the node running the Application Master via the Secure
                              Distributed Cache, for renewing the login tickets and the
                              delegation tokens periodically.
      
[double_happy@hadoop101 ~]$

  --deploy-mode DEPLOY_MODE   Whether to launch the driver program locally ("client") or
                              on one of the worker machines inside the cluster ("cluster")
                              (Default: client).

spark-shell --master yarn  默认不写  --deploy-mode 是 client模式

client模式：测试

[double_happy@hadoop101 ~]$ spark-shell --master yarn
19/10/29 10:41:49 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
Setting default log level to "ERROR".
To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).
Spark context Web UI available at http://hadoop101:4040
Spark context available as 'sc' (master = yarn, app id = application_1570934113711_0037).
Spark session available as 'spark'.
Welcome to
      ____              __
     / __/__  ___ _____/ /__
    _\ \/ _ \/ _ `/ __/  '_/
   /___/ .__/\_,_/_/ /_/\_\   version 2.4.4
      /_/
         
Using Scala version 2.11.12 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_45)
Type in expressions to have them evaluated.
Type :help for more information.

scala>

在这里插入图片描述

注意：
写代码跟运行模式是没有关系的 
 --num-executors   默认是2 个
id 是application_xxx 开头的必然是 yarn 模式的 去historyserver看见这个开头的 就是yarn模式跑的任务

在这里插入图片描述

[double_happy@hadoop101 ~]$ spark-sql --jars ~/software/mysql-connector-java-5.1.47.jar --master yarn
19/10/29 10:54:17 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
19/10/29 10:54:19 INFO HiveMetaStore: 0: Opening raw store with implemenation class:org.apache.hadoop.hive.metastore.ObjectStore
19/10/29 10:54:19 INFO ObjectStore: ObjectStore, initialize called
19/10/29 10:54:20 INFO Persistence: Property hive.metastore.integral.jdo.pushdown unknown - will be ignored
19/10/29 10:54:20 INFO Persistence: Property datanucleus.cache.level2 unknown - will be ignored
19/10/29 10:54:21 INFO ObjectStore: Setting MetaStore object pin classes with hive.metastore.cache.pinobjtypes="Table,StorageDescriptor,SerDeInfo,Partition,Database,Type,FieldSchema,Order"
19/10/29 10:54:23 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not have its own datastore table.
19/10/29 10:54:23 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is tagged as "embedded-only" so does not have its own datastore table.
19/10/29 10:54:23 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MFieldSchema" is tagged as "embedded-only" so does not have its own datastore table.
19/10/29 10:54:23 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MOrder" is tagged as "embedded-only" so does not have its own datastore table.
19/10/29 10:54:23 INFO Query: Reading in results for query "org.datanucleus.store.rdbms.query.SQLQuery@0" since the connection used is closing
19/10/29 10:54:23 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is MYSQL
19/10/29 10:54:23 INFO ObjectStore: Initialized ObjectStore
19/10/29 10:54:24 INFO HiveMetaStore: Added admin role in metastore
19/10/29 10:54:24 INFO HiveMetaStore: Added public role in metastore
19/10/29 10:54:24 INFO HiveMetaStore: No user is added in admin role, since config is empty
19/10/29 10:54:24 INFO HiveMetaStore: 0: get_all_databases
19/10/29 10:54:24 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_all_databases
19/10/29 10:54:24 INFO HiveMetaStore: 0: get_functions: db=default pat=*
19/10/29 10:54:24 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_functions: db=default pat=*
19/10/29 10:54:24 INFO Datastore: The class "org.apache.hadoop.hive.metastore.model.MResourceUri" is tagged as "embedded-only" so does not have its own datastore table.
19/10/29 10:54:24 INFO HiveMetaStore: 0: get_functions: db=homework pat=*
19/10/29 10:54:24 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_functions: db=homework pat=*
19/10/29 10:54:24 INFO HiveMetaStore: 0: get_function: homework.add_prefix_new
19/10/29 10:54:24 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_function: homework.add_prefix_new
19/10/29 10:54:25 INFO HiveMetaStore: 0: get_function: homework.remove_prefix_new
19/10/29 10:54:25 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_function: homework.remove_prefix_new
19/10/29 10:54:25 INFO SessionState: Created local directory: /tmp/eed4bfce-e4a6-4683-81a4-9bda791d7822_resources
19/10/29 10:54:25 INFO SessionState: Created HDFS directory: /tmp/hive/double_happy/eed4bfce-e4a6-4683-81a4-9bda791d7822
19/10/29 10:54:25 INFO SessionState: Created local directory: /tmp/double_happy/eed4bfce-e4a6-4683-81a4-9bda791d7822
19/10/29 10:54:25 INFO SessionState: Created HDFS directory: /tmp/hive/double_happy/eed4bfce-e4a6-4683-81a4-9bda791d7822/_tmp_space.db
19/10/29 10:54:25 INFO SparkContext: Running Spark version 2.4.4
19/10/29 10:54:25 INFO SparkContext: Submitted application: SparkSQL::172.26.162.56
19/10/29 10:54:25 INFO SecurityManager: Changing view acls to: double_happy
19/10/29 10:54:25 INFO SecurityManager: Changing modify acls to: double_happy
19/10/29 10:54:25 INFO SecurityManager: Changing view acls groups to: 
19/10/29 10:54:25 INFO SecurityManager: Changing modify acls groups to: 
19/10/29 10:54:25 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(double_happy); groups with view permissions: Set(); users  with modify permissions: Set(double_happy); groups with modify permissions: Set()
19/10/29 10:54:26 INFO Utils: Successfully started service 'sparkDriver' on port 44153.
19/10/29 10:54:26 INFO SparkEnv: Registering MapOutputTracker
19/10/29 10:54:26 INFO SparkEnv: Registering BlockManagerMaster
19/10/29 10:54:26 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information
19/10/29 10:54:26 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up
19/10/29 10:54:26 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-73b7d763-b5a3-476a-a93c-d259c46eac97
19/10/29 10:54:26 INFO MemoryStore: MemoryStore started with capacity 366.3 MB
19/10/29 10:54:26 INFO SparkEnv: Registering OutputCommitCoordinator
19/10/29 10:54:26 INFO Utils: Successfully started service 'SparkUI' on port 4040.
19/10/29 10:54:26 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://hadoop101:4040
19/10/29 10:54:27 INFO RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
19/10/29 10:54:27 INFO Client: Requesting a new application from cluster with 1 NodeManagers
19/10/29 10:54:27 INFO Client: Verifying our application has not requested more than the maximum memory capability of the cluster (8192 MB per container)
19/10/29 10:54:27 INFO Client: Will allocate AM container, with 896 MB memory including 384 MB overhead
19/10/29 10:54:27 INFO Client: Setting up container launch context for our AM
19/10/29 10:54:27 INFO Client: Setting up the launch environment for our AM container
19/10/29 10:54:27 INFO Client: Preparing resources for our AM container
19/10/29 10:54:27 WARN Client: Neither spark.yarn.jars nor spark.yarn.archive is set, falling back to uploading libraries under SPARK_HOME.
19/10/29 10:54:33 INFO Client: Uploading resource file:/tmp/spark-84f5cd2c-d3ab-4e82-8520-7f45d7422e8c/__spark_libs__4819705212614105474.zip -> hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038/__spark_libs__4819705212614105474.zip
19/10/29 10:54:35 INFO Client: Uploading resource file:/home/double_happy/software/mysql-connector-java-5.1.47.jar -> hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038/mysql-connector-java-5.1.47.jar
19/10/29 10:54:35 INFO Client: Uploading resource file:/tmp/spark-84f5cd2c-d3ab-4e82-8520-7f45d7422e8c/__spark_conf__8084381853282513804.zip -> hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038/__spark_conf__.zip
19/10/29 10:54:35 INFO SecurityManager: Changing view acls to: double_happy
19/10/29 10:54:35 INFO SecurityManager: Changing modify acls to: double_happy
19/10/29 10:54:35 INFO SecurityManager: Changing view acls groups to: 
19/10/29 10:54:35 INFO SecurityManager: Changing modify acls groups to: 
19/10/29 10:54:35 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users  with view permissions: Set(double_happy); groups with view permissions: Set(); users  with modify permissions: Set(double_happy); groups with modify permissions: Set()
19/10/29 10:54:37 INFO Client: Submitting application application_1570934113711_0038 to ResourceManager
19/10/29 10:54:37 INFO YarnClientImpl: Submitted application application_1570934113711_0038
19/10/29 10:54:37 INFO SchedulerExtensionServices: Starting Yarn extension services with app application_1570934113711_0038 and attemptId None
19/10/29 10:54:38 INFO Client: Application report for application_1570934113711_0038 (state: ACCEPTED)
19/10/29 10:54:38 INFO Client: 
         client token: N/A
         diagnostics: N/A
         ApplicationMaster host: N/A
         ApplicationMaster RPC port: -1
         queue: root.double_happy
         start time: 1572317677273
         final status: UNDEFINED
         tracking URL: http://hadoop101:8088/proxy/application_1570934113711_0038/
         user: double_happy
19/10/29 10:54:39 INFO Client: Application report for application_1570934113711_0038 (state: ACCEPTED)
19/10/29 10:54:40 INFO Client: Application report for application_1570934113711_0038 (state: ACCEPTED)
19/10/29 10:54:41 INFO Client: Application report for application_1570934113711_0038 (state: ACCEPTED)
19/10/29 10:54:42 INFO Client: Application report for application_1570934113711_0038 (state: ACCEPTED)
19/10/29 10:54:43 INFO Client: Application report for application_1570934113711_0038 (state: ACCEPTED)
19/10/29 10:54:44 INFO YarnClientSchedulerBackend: Add WebUI Filter. org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter, Map(PROXY_HOSTS -> hadoop101, PROXY_URI_BASES -> http://hadoop101:8088/proxy/application_1570934113711_0038), /proxy/application_1570934113711_0038
19/10/29 10:54:44 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /jobs, /jobs/json, /jobs/job, /jobs/job/json, /stages, /stages/json, /stages/stage, /stages/stage/json, /stages/pool, /stages/pool/json, /storage, /storage/json, /storage/rdd, /storage/rdd/json, /environment, /environment/json, /executors, /executors/json, /executors/threadDump, /executors/threadDump/json, /static, /, /api, /jobs/job/kill, /stages/stage/kill.
19/10/29 10:54:44 INFO Client: Application report for application_1570934113711_0038 (state: RUNNING)
19/10/29 10:54:44 INFO Client: 
         client token: N/A
         diagnostics: N/A
         ApplicationMaster host: 172.26.162.56
         ApplicationMaster RPC port: -1
         queue: root.double_happy
         start time: 1572317677273
         final status: UNDEFINED
         tracking URL: http://hadoop101:8088/proxy/application_1570934113711_0038/
         user: double_happy
19/10/29 10:54:44 INFO YarnClientSchedulerBackend: Application application_1570934113711_0038 has started running.
19/10/29 10:54:44 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 38768.
19/10/29 10:54:44 INFO NettyBlockTransferService: Server created on hadoop101:38768
19/10/29 10:54:44 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy
19/10/29 10:54:44 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, hadoop101, 38768, None)
19/10/29 10:54:44 INFO BlockManagerMasterEndpoint: Registering block manager hadoop101:38768 with 366.3 MB RAM, BlockManagerId(driver, hadoop101, 38768, None)
19/10/29 10:54:44 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, hadoop101, 38768, None)
19/10/29 10:54:44 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, hadoop101, 38768, None)
19/10/29 10:54:44 INFO YarnSchedulerBackend$YarnSchedulerEndpoint: ApplicationMaster registered as NettyRpcEndpointRef(spark-client://YarnAM)
19/10/29 10:54:45 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /metrics/json.
19/10/29 10:54:45 INFO EventLoggingListener: Logging events to hdfs://hadoop101:8020/spark_directory/application_1570934113711_0038
19/10/29 10:54:50 INFO YarnSchedulerBackend$YarnDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (172.26.162.56:59428) with ID 1
19/10/29 10:54:50 INFO BlockManagerMasterEndpoint: Registering block manager hadoop101:36737 with 366.3 MB RAM, BlockManagerId(1, hadoop101, 36737, None)
19/10/29 10:54:52 INFO YarnSchedulerBackend$YarnDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (172.26.162.56:34236) with ID 2
19/10/29 10:54:52 INFO YarnClientSchedulerBackend: SchedulerBackend is ready for scheduling beginning after reached minRegisteredResourcesRatio: 0.8
19/10/29 10:54:52 INFO SharedState: loading hive config file: file:/home/double_happy/app/spark-2.4.4-bin-2.6.0-cdh5.15.1/conf/hive-site.xml
19/10/29 10:54:52 INFO SharedState: Setting hive.metastore.warehouse.dir ('null') to the value of spark.sql.warehouse.dir ('file:/home/double_happy/spark-warehouse').
19/10/29 10:54:52 INFO SharedState: Warehouse path is 'file:/home/double_happy/spark-warehouse'.
19/10/29 10:54:52 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /SQL.
19/10/29 10:54:52 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /SQL/json.
19/10/29 10:54:52 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /SQL/execution.
19/10/29 10:54:52 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /SQL/execution/json.
19/10/29 10:54:52 INFO JettyUtils: Adding filter org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter to /static/sql.
19/10/29 10:54:52 INFO BlockManagerMasterEndpoint: Registering block manager hadoop101:36948 with 366.3 MB RAM, BlockManagerId(2, hadoop101, 36948, None)
19/10/29 10:54:52 INFO HiveUtils: Initializing HiveMetastoreConnection version 1.2.1 using Spark classes.
19/10/29 10:54:52 INFO HiveClientImpl: Warehouse location for Hive client (version 1.2.2) is file:/home/double_happy/spark-warehouse
19/10/29 10:54:52 INFO metastore: Mestastore configuration hive.metastore.warehouse.dir changed from /user/hive/warehouse to file:/home/double_happy/spark-warehouse
19/10/29 10:54:52 INFO HiveMetaStore: 0: Shutting down the object store...
19/10/29 10:54:52 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=Shutting down the object store...
19/10/29 10:54:52 INFO HiveMetaStore: 0: Metastore shutdown complete.
19/10/29 10:54:52 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=Metastore shutdown complete.
19/10/29 10:54:52 INFO HiveMetaStore: 0: get_database: default
19/10/29 10:54:52 INFO audit: ugi=double_happy  ip=unknown-ip-addr      cmd=get_database: default
19/10/29 10:54:52 INFO HiveMetaStore: 0: Opening raw store with implemenation class:org.apache.hadoop.hive.metastore.ObjectStore
19/10/29 10:54:52 INFO ObjectStore: ObjectStore, initialize called
19/10/29 10:54:52 INFO Query: Reading in results for query "org.datanucleus.store.rdbms.query.SQLQuery@0" since the connection used is closing
19/10/29 10:54:52 INFO MetaStoreDirectSql: Using direct SQL, underlying DB is MYSQL
19/10/29 10:54:52 INFO ObjectStore: Initialized ObjectStore
19/10/29 10:54:53 INFO StateStoreCoordinatorRef: Registered StateStoreCoordinator endpoint
Spark master: yarn, Application Id: application_1570934113711_0038
19/10/29 10:54:53 INFO SparkSQLCLIDriver: Spark master: yarn, Application Id: application_1570934113711_0038
spark-sql (default)> 


注意：日志里
19/10/29 10:54:27 WARN Client:
 Neither spark.yarn.jars nor spark.yarn.archive is set, 
 falling back to uploading libraries under SPARK_HOME.

1. spark.yarn.jars nor spark.yarn.archive is set 
这个没有设置 会把SPARK_HOME相关的东西 全部传到hdfs上去 
不信看日志 

2.
19/10/29 10:54:33 INFO Client: Uploading resource file:/tmp/spark-84f5cd2c-d3ab-4e82-8520-7f45d7422e8c/__spark_libs__4819705212614105474.zip -> hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038/__spark_libs__4819705212614105474.zip
19/10/29 10:54:35 INFO Client: Uploading resource file:/home/double_happy/software/mysql-connector-java-5.1.47.jar -> hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038/mysql-connector-java-5.1.47.jar
19/10/29 10:54:35 INFO Client: Uploading resource file:/tmp/spark-84f5cd2c-d3ab-4e82-8520-7f45d7422e8c/__spark_conf__8084381853282513804.zip -> hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038/__spark_conf__.zip

打开这个地址看一眼：
 hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0038

这是我又重启了一个 spark-sql  之前的关掉了 
[double_happy@hadoop101 ~]$ hadoop fs -ls  hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0041
Found 3 items
-rw-r--r--   1 double_happy supergroup     211902 2019-10-29 11:18 hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0041/__spark_conf__.zip
-rw-r--r--   1 double_happy supergroup  298846294 2019-10-29 11:18 hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0041/__spark_libs__2528141214285970680.zip
-rw-r--r--   1 double_happy supergroup    1007502 2019-10-29 11:18 hdfs://hadoop101:8020/user/double_happy/.sparkStaging/application_1570934113711_0041/mysql-connector-java-5.1.47.jar
[double_happy@hadoop101 ~]$ 

__spark_conf__.zip
mysql-connector-java-5.1.47.jar
__spark_libs__2528141214285970680.zip    非常大这个 
作业跑完会把这些自动删掉

如果上面提到的两个参数没有设置 会把这些传到HDFS  上传是需要花费时间的
这个不解决 你的每个作业 都要这样

这就是一个调优点：
尽可能的让Spark快速的再yarn上运行起来   该怎么做的呢？

https://guguoyu.blog.csdn.net/article/details/102644376

在这里插入图片描述

spark-shell 和spark-sql 都可以 这不是主要的 主要的是下面的

[double_happy@hadoop101 ~]$ spark-shell --master yarn --deploy-mode cluster
Exception in thread "main" org.apache.spark.SparkException: Cluster deploy mode is not applicable to Spark shells.
        at org.apache.spark.deploy.SparkSubmit.error(SparkSubmit.scala:853)
        at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:281)
        at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:774)
        at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
        at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
        at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
        at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
[double_happy@hadoop101 ~]$ 

为什么Cluster deploy mode is not applicable to Spark shells.？

因为 spark-shell driver是在本地的 是可以交互代码的  而 yarn-claster  driver是在am里的  明白吗？