5. 提交之前 查看 k8s port :k8s-apiserver-port (看spark官网)
[root@container01 spark]# kubectl cluster-info Kubernetes master is running at https://172.21.230.89:6443 KubeDNS is running at https://172.21.230.89:6443/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. [root@container01 spark]#
[root@container01 spark]# bin/spark-submit \ > --master k8s://https://172.21.230.89:6443 \ > --deploy-mode cluster \ > --name spark-pi \ > --class org.apache.spark.examples.SparkPi \ > --conf spark.executor.instances=2 \ > --conf spark.kubernetes.container.image.pullSecrets=sparkkey \ > --conf spark.kubernetes.container.image=172.21.230.89/doublehappy/spark:v2.4.3 \ > --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark \ > --conf spark.kubernetes.driver.pod.name=spark-pi-driver \ > local:///opt/spark/examples/jars/spark-examples_2.11-2.4.3.jar log4j:WARN No appenders could be found for logger (io.fabric8.kubernetes.client.Config). log4j:WARN Please initialize the log4j system properly. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 20/02/11 16:16:49 INFO LoggingPodStatusWatcherImpl: State changed, new state: pod name: spark-pi-driver namespace: default labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece creation time: 2020-02-11T08:16:48Z service account name: spark volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn node name: N/A start time: N/A container images: N/A phase: Pending status: [] 20/02/11 16:16:49 INFO LoggingPodStatusWatcherImpl: State changed, new state: pod name: spark-pi-driver namespace: default labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece creation time: 2020-02-11T08:16:48Z service account name: spark volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn node name: container02 start time: N/A container images: N/A phase: Pending status: [] 20/02/11 16:16:49 INFO LoggingPodStatusWatcherImpl: State changed, new state: pod name: spark-pi-driver namespace: default labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece creation time: 2020-02-11T08:16:48Z service account name: spark volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn node name: container02 start time: 2020-02-11T08:16:49Z container images: 172.21.230.89/doublehappy/spark:v2.4.3 phase: Pending status: [ContainerStatus(containerID=null, image=172.21.230.89/doublehappy/spark:v2.4.3, imageID=, lastState=ContainerState(running=null, terminated=null, waiting=null, additionalProperties={}), name=spark-kubernetes-driver, ready=false, restartCount=0, state=ContainerState(running=null, terminated=null, waiting=ContainerStateWaiting(message=null, reason=ContainerCreating, additionalProperties={}), additionalProperties={}), additionalProperties={})] 20/02/11 16:16:49 INFO Client: Waiting for application spark-pi to finish... 20/02/11 16:16:59 INFO LoggingPodStatusWatcherImpl: State changed, new state: pod name: spark-pi-driver namespace: default labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece creation time: 2020-02-11T08:16:48Z service account name: spark volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn node name: container02 start time: 2020-02-11T08:16:49Z container images: 172.21.230.89/doublehappy/spark:v2.4.3 phase: Running status: [ContainerStatus(containerID=docker://80f7f9b25b92c8995b99de55ad1bcd47d46e04bbfe794f09b394a8516468ac2b, image=172.21.230.89/doublehappy/spark:v2.4.3, imageID=docker-pullable://172.21.230.89/doublehappy/spark@sha256:c8f97b50e7c325a9ad342afa7e796ae594f802306fae82b82d9622a462b32fa3, lastState=ContainerState(running=null, terminated=null, waiting=null, additionalProperties={}), name=spark-kubernetes-driver, ready=true, restartCount=0, state=ContainerState(running=ContainerStateRunning(startedAt=2020-02-11T08:16:59Z, additionalProperties={}), terminated=null, waiting=null, additionalProperties={}), additionalProperties={})] 20/02/11 16:17:34 INFO LoggingPodStatusWatcherImpl: State changed, new state: pod name: spark-pi-driver namespace: default labels: spark-app-selector -> spark-961842d41a564fc3ba906855f8ee4b57, spark-role -> driver pod uid: d9ab6302-4ca6-11ea-adad-00163e041ece creation time: 2020-02-11T08:16:48Z service account name: spark volumes: spark-local-dir-1, spark-conf-volume, spark-token-9hgdn node name: container02 start time: 2020-02-11T08:16:49Z container images: 172.21.230.89/doublehappy/spark:v2.4.3 phase: Succeeded status: [ContainerStatus(containerID=docker://80f7f9b25b92c8995b99de55ad1bcd47d46e04bbfe794f09b394a8516468ac2b, image=172.21.230.89/doublehappy/spark:v2.4.3, imageID=docker-pullable://172.21.230.89/doublehappy/spark@sha256:c8f97b50e7c325a9ad342afa7e796ae594f802306fae82b82d9622a462b32fa3, lastState=ContainerState(running=null, terminated=null, waiting=null, additionalProperties={}), name=spark-kubernetes-driver, ready=false, restartCount=0, state=ContainerState(running=null, terminated=ContainerStateTerminated(containerID=docker://80f7f9b25b92c8995b99de55ad1bcd47d46e04bbfe794f09b394a8516468ac2b, exitCode=0, finishedAt=2020-02-11T08:17:34Z, message=null, reason=Completed, signal=null, startedAt=2020-02-11T08:16:59Z, additionalProperties={}), waiting=null, additionalProperties={}), additionalProperties={})] 20/02/11 16:17:34 INFO LoggingPodStatusWatcherImpl: Container final statuses:
Container name: spark-kubernetes-driver Container image: 172.21.230.89/doublehappy/spark:v2.4.3 Container state: Terminated Exit code: 0 20/02/11 16:17:34 INFO Client: Application spark-pi finished. 20/02/11 16:17:34 INFO ShutdownHookManager: Shutdown hook called 20/02/11 16:17:34 INFO ShutdownHookManager: Deleting directory /tmp/spark-41d1f84f-677e-4476-9cdc-a277ebe96dfa [root@container01 spark]#
查看一下： [root@container01 ~]# kubectl get all NAME READY STATUS RESTARTS AGE pod/spark-pi-driver 0/1 Completed 0 92s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 26h service/spark-pi-1581409007077-driver-svc ClusterIP None <none> 7078/TCP,7079/TCP 91s [root@container01 ~]#
查看spark log：
[root@container01 ~]# kubectl logs -f spark-pi-driver ++ id -u + myuid=0 ++ id -g + mygid=0 + set +e ++ getent passwd 0 + uidentry=root:x:0:0:root:/root:/bin/ash + set -e + '[' -z root:x:0:0:root:/root:/bin/ash ']' + SPARK_K8S_CMD=driver + case "$SPARK_K8S_CMD" in + shift 1 + SPARK_CLASSPATH=':/opt/spark/jars/*' + grep SPARK_JAVA_OPT_ + sort -t_ -k4 -n + sed 's/[^=]*=\(.*\)/\1/g' + env + readarray -t SPARK_EXECUTOR_JAVA_OPTS + '[' -n '' ']' + '[' -n '' ']' + PYSPARK_ARGS= + '[' -n '' ']' + R_ARGS= + '[' -n '' ']' + '[' '' == 2 ']' + '[' '' == 3 ']' + case "$SPARK_K8S_CMD" in + CMD=("$SPARK_HOME/bin/spark-submit" --conf "spark.driver.bindAddress=$SPARK_DRIVER_BIND_ADDRESS" --deploy-mode client "$@") + exec /sbin/tini -s -- /opt/spark/bin/spark-submit --conf spark.driver.bindAddress=10.244.1.5 --deploy-mode client --properties-file /opt/spark/conf/spark.properties --class org.apache.spark.examples.SparkPi spark-internal 20/02/11 08:17:00 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties 20/02/11 08:17:01 INFO SparkContext: Running Spark version 2.4.3 20/02/11 08:17:01 INFO SparkContext: Submitted application: Spark Pi 20/02/11 08:17:01 INFO SecurityManager: Changing view acls to: root 20/02/11 08:17:01 INFO SecurityManager: Changing modify acls to: root 20/02/11 08:17:01 INFO SecurityManager: Changing view acls groups to: 20/02/11 08:17:01 INFO SecurityManager: Changing modify acls groups to: 20/02/11 08:17:01 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); groups with view permissions: Set(); users with modify permissions: Set(root); groups with modify permissions: Set() 20/02/11 08:17:01 INFO Utils: Successfully started service 'sparkDriver' on port 7078. 20/02/11 08:17:01 INFO SparkEnv: Registering MapOutputTracker 20/02/11 08:17:01 INFO SparkEnv: Registering BlockManagerMaster 20/02/11 08:17:01 INFO BlockManagerMasterEndpoint: Using org.apache.spark.storage.DefaultTopologyMapper for getting topology information 20/02/11 08:17:01 INFO BlockManagerMasterEndpoint: BlockManagerMasterEndpoint up 20/02/11 08:17:01 INFO DiskBlockManager: Created local directory at /var/data/spark-6753da05-ab60-4e01-bafd-70e0141b47f2/blockmgr-f9c8a8f5-32c4-42b4-bb3b-3ec9686c04f5 20/02/11 08:17:01 INFO MemoryStore: MemoryStore started with capacity 413.9 MB 20/02/11 08:17:01 INFO SparkEnv: Registering OutputCommitCoordinator 20/02/11 08:17:01 INFO Utils: Successfully started service 'SparkUI' on port 4040. 20/02/11 08:17:01 INFO SparkUI: Bound SparkUI to 0.0.0.0, and started at http://spark-pi-1581409007077-driver-svc.default.svc:4040 20/02/11 08:17:01 INFO SparkContext: Added JAR file:///opt/spark/examples/jars/spark-examples_2.11-2.4.3.jar at spark://spark-pi-1581409007077-driver-svc.default.svc:7078/jars/spark-examples_2.11-2.4.3.jar with timestamp 1581409021888 20/02/11 08:17:03 INFO ExecutorPodsAllocator: Going to request 2 executors from Kubernetes. 20/02/11 08:17:03 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 7079. 20/02/11 08:17:03 INFO NettyBlockTransferService: Server created on spark-pi-1581409007077-driver-svc.default.svc:7079 20/02/11 08:17:03 INFO BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 20/02/11 08:17:03 INFO BlockManagerMaster: Registering BlockManager BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None) 20/02/11 08:17:03 INFO BlockManagerMasterEndpoint: Registering block manager spark-pi-1581409007077-driver-svc.default.svc:7079 with 413.9 MB RAM, BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None) 20/02/11 08:17:03 INFO BlockManagerMaster: Registered BlockManager BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None) 20/02/11 08:17:03 INFO BlockManager: Initialized BlockManager: BlockManagerId(driver, spark-pi-1581409007077-driver-svc.default.svc, 7079, None) 20/02/11 08:17:14 INFO KubernetesClusterSchedulerBackend$KubernetesDriverEndpoint: Registered executor NettyRpcEndpointRef(spark-client://Executor) (10.244.2.4:56814) with ID 1 20/02/11 08:17:14 INFO BlockManagerMasterEndpoint: Registering block manager 10.244.2.4:37098 with 413.9 MB RAM, BlockManagerId(1, 10.244.2.4, 37098, None) 20/02/11 08:17:33 INFO KubernetesClusterSchedulerBackend: SchedulerBackend is ready for scheduling beginning after waiting maxRegisteredResourcesWaitingTime: 30000(ms) 20/02/11 08:17:33 INFO SparkContext: Starting job: reduce at SparkPi.scala:38 20/02/11 08:17:33 INFO DAGScheduler: Got job 0 (reduce at SparkPi.scala:38) with 2 output partitions 20/02/11 08:17:33 INFO DAGScheduler: Final stage: ResultStage 0 (reduce at SparkPi.scala:38) 20/02/11 08:17:33 INFO DAGScheduler: Parents of final stage: List() 20/02/11 08:17:33 INFO DAGScheduler: Missing parents: List() 20/02/11 08:17:33 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34), which has no missing parents 20/02/11 08:17:33 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 1936.0 B, free 413.9 MB) 20/02/11 08:17:33 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1256.0 B, free 413.9 MB) 20/02/11 08:17:33 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on spark-pi-1581409007077-driver-svc.default.svc:7079 (size: 1256.0 B, free: 413.9 MB) 20/02/11 08:17:33 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:1161 20/02/11 08:17:33 INFO DAGScheduler: Submitting 2 missing tasks from ResultStage 0 (MapPartitionsRDD[1] at map at SparkPi.scala:34) (first 15 tasks are for partitions Vector(0, 1)) 20/02/11 08:17:33 INFO TaskSchedulerImpl: Adding task set 0.0 with 2 tasks 20/02/11 08:17:33 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, 10.244.2.4, executor 1, partition 0, PROCESS_LOCAL, 7885 bytes) 20/02/11 08:17:34 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 10.244.2.4:37098 (size: 1256.0 B, free: 413.9 MB) 20/02/11 08:17:34 INFO TaskSetManager: Starting task 1.0 in stage 0.0 (TID 1, 10.244.2.4, executor 1, partition 1, PROCESS_LOCAL, 7885 bytes) 20/02/11 08:17:34 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 534 ms on 10.244.2.4 (executor 1) (1/2) 20/02/11 08:17:34 INFO TaskSetManager: Finished task 1.0 in stage 0.0 (TID 1) in 27 ms on 10.244.2.4 (executor 1) (2/2) 20/02/11 08:17:34 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool 20/02/11 08:17:34 INFO DAGScheduler: ResultStage 0 (reduce at SparkPi.scala:38) finished in 0.814 s 20/02/11 08:17:34 INFO DAGScheduler: Job 0 finished: reduce at SparkPi.scala:38, took 0.868531 s Pi is roughly 3.139475697378487 20/02/11 08:17:34 INFO SparkUI: Stopped Spark web UI at http://spark-pi-1581409007077-driver-svc.default.svc:4040 20/02/11 08:17:34 INFO KubernetesClusterSchedulerBackend: Shutting down all executors 20/02/11 08:17:34 INFO KubernetesClusterSchedulerBackend$KubernetesDriverEndpoint: Asking each executor to shut down 20/02/11 08:17:34 WARN ExecutorPodsWatchSnapshotSource: Kubernetes client has been closed (this is expected if the application is shutting down.) 20/02/11 08:17:34 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped! 20/02/11 08:17:34 INFO MemoryStore: MemoryStore cleared 20/02/11 08:17:34 INFO BlockManager: BlockManager stopped 20/02/11 08:17:34 INFO BlockManagerMaster: BlockManagerMaster stopped 20/02/11 08:17:34 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped! 20/02/11 08:17:34 INFO SparkContext: Successfully stopped SparkContext 20/02/11 08:17:34 INFO ShutdownHookManager: Shutdown hook called 20/02/11 08:17:34 INFO ShutdownHookManager: Deleting directory /var/data/spark-6753da05-ab60-4e01-bafd-70e0141b47f2/spark-67555fc1-109a-46a0-bff7-1c14b972cfe4 20/02/11 08:17:34 INFO ShutdownHookManager: Deleting directory /tmp/spark-7f09677c-5829-434e-b74d-af0f0c021571 [root@container01 ~]#
结果：使用kubectl logs -f spark-pi-driver 是可以看到的
Pi is roughly 3.1451757258786293 20/02/11 08:23:35 INFO SparkUI: Stopped Spark web UI at http://spark-pi-1581409376163-driver-svc.default.svc:4040 20/02/11 08:23:35 INFO KubernetesClusterSchedulerBacke
