Spark009--spark-shell执行流程

[double_happy@hadoop101 bin]$ cat spark-shell 
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Shell script for starting the Spark Shell REPL

cygwin=false     
case "$(uname)" in     
  CYGWIN*) cygwin=true;;
esac

# Enter posix mode for bash
set -o posix

if [ -z "${SPARK_HOME}" ]; then
  source "$(dirname "$0")"/find-spark-home
fi

export _SPARK_CMD_USAGE="Usage: ./bin/spark-shell [options]

Scala REPL options:
  -I <file>                   preload <file>, enforcing line-by-line interpretation"

# SPARK-4161: scala does not assume use of the java classpath,
# so we need to add the "-Dscala.usejavacp=true" flag manually. We
# do this specifically for the Spark shell because the scala REPL
# has its own class loader, and any additional classpath specified
# through spark.driver.extraClassPath is not automatically propagated.
SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Dscala.usejavacp=true"

function main() {
  if $cygwin; then
    # Workaround for issue involving JLine and Cygwin
    # (see http://sourceforge.net/p/jline/bugs/40/).
    # If you're using the Mintty terminal emulator in Cygwin, may need to set the
    # "Backspace sends ^H" setting in "Keys" section of the Mintty options
    # (see https://github.com/sbt/sbt/issues/562).
    stty -icanon min 1 -echo > /dev/null 2>&1
    export SPARK_SUBMIT_OPTS="$SPARK_SUBMIT_OPTS -Djline.terminal=unix"
    "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
    stty icanon echo > /dev/null 2>&1
  else
    export SPARK_SUBMIT_OPTS
    "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"
  fi
}

# Copy restore-TTY-on-exit functions from Scala script so spark-shell exits properly even in
# binary distribution of Spark where Scala is not installed
exit_status=127
saved_stty=""

# restore stty settings (echo in particular)
function restoreSttySettings() {
  stty $saved_stty
  saved_stty=""
}

function onExit() {
  if [[ "$saved_stty" != "" ]]; then
    restoreSttySettings
  fi
  exit $exit_status
}

# to reenable echo if we are interrupted before completing.
trap onExit INT

# save terminal settings
saved_stty=$(stty -g 2>/dev/null)
# clear on error so we don't later try to restore them
if [[ ! $? ]]; then
  saved_stty=""
fi

main "$@"

# record the exit status lest it be overwritten:
# then reenable echo and propagate the code.
exit_status=$?
onExit

[double_happy@hadoop101 bin]$

1.  cygwin=false  //windows 电脑操作linux东西 要安装cygwin  
2.  case "$(uname)" in       //uname 知道我们是什么操作系统

[double_happy@hadoop101 bin]$ uname -r        //内核的版本
3.10.0-514.26.2.el7.x86_64
[double_happy@hadoop101 bin]$ uname -a      //打印所有的信息
Linux hadoop101 3.10.0-514.26.2.el7.x86_64 #1 SMP Tue Jul 4 15:04:05 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
[double_happy@hadoop101 bin]$

case $变量名 in
      模式1)
      		command
      ;;
      模式2)
      		command
      ;;
      *)
      	command
      	;;
      	esac

[double_happy@hadoop101 script]$ cat test.sh 
#!/bin/bash
cygwin=false     
case "$(uname)" in     
  CYGWIN*) cygwin=true;;
esac
echo $cygwin
[double_happy@hadoop101 script]$ sh test.sh 
false
[double_happy@hadoop101 script]$

[double_happy@hadoop101 script]$ cat case.sh 
read -p "press key , then press return:" KEY
case $KEY in
        [a-z]|[A-Z])
echo "this is a letter..";;
        [0-9])
echo "this is a digit..." ;;
        *)
echo "other.." ;;
esac
[double_happy@hadoop101 script]$ sh case.sh 
press key , then press return:1
this is a digit...
[double_happy@hadoop101 script]$ sh case.sh 
press key , then press return:a
this is a letter..
[double_happy@hadoop101 script]$

[ -z STRING ]  “STRING” 的长度为零则为真。  
dirname 是什么？
获取当前的路径

find-spark-home是在bin目录下  这脚本里有 就是去export SPARK_HOME
 source "$(dirname "$0")"/find-spark-home    目的就是找 SPARK_HOME
 所以你环境里配置了 SPARK_HOME if那个shell 就直接跳过去 就不用找SPARK_HOME

[double_happy@hadoop101 script]$ cat dirname.sh 
HOME=`cd $(dirname "$0");pwd`
echo $HOME
[double_happy@hadoop101 script]$ sh dirname.sh 
/home/double_happy/script
[double_happy@hadoop101 script]$

main "$@"  main方法里 ：

 "${SPARK_HOME}"/bin/spark-submit --class org.apache.spark.repl.Main --name "Spark shell" "$@"

知道 spark-shell底层调用的是  spark-submit

[double_happy@hadoop101 script]$ cat main.sh 
function main(){
        echo "....."$@
}

main "$@"
[double_happy@hadoop101 script]$ sh main.sh abc 
.....abc
[double_happy@hadoop101 script]$ sh main.sh abc 123 456
.....abc 123 456
[double_happy@hadoop101 script]$ 

$@ :就是把一堆输入的参数  带走

[double_happy@hadoop101 script]$ cat main.sh 
function main(){
        echo "....."$@
}

main "$@"
[double_happy@hadoop101 script]$ mv main.sh spark-shell
[double_happy@hadoop101 script]$ chmod +x spark-shell 
[double_happy@hadoop101 script]$ ./spark-shell --master yarn --jars mysql.driver.jar
.....--master yarn --jars mysql.driver.jar
[double_happy@hadoop101 script]$ 


明白了吗 spark-shell的参数 就是这么传进去的

[double_happy@hadoop101 bin]$ cat spark-submit 
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

if [ -z "${SPARK_HOME}" ]; then
  source "$(dirname "$0")"/find-spark-home
fi

# disable randomized hash for string in Python 3.3+
export PYTHONHASHSEED=0

exec "${SPARK_HOME}"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"
[double_happy@hadoop101 bin]$

exec "${SPARK_HOME}"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"

exec 是做什么的？
就是一个执行的命令
例如在当前shell中执行 exec ls  表示执行ls这条命令来替换当前的shell ，即为执行完后会退出当前shell。

为了避免这个结果的影响，一般将exec命令放到一个shell脚本中，用主脚本调用这个脚本，调用处可以用bash  xx.sh(xx.sh为存放exec命令的脚本)，这样会为xx.sh建立一个子shell去执行，当执行exec后该子脚本进程就被替换成相应的exec的命令。

[double_happy@hadoop101 script]$ cat exec.sh 
exec ls
[double_happy@hadoop101 script]$ sh exec.sh 
azkaban-job  case.sh  dirname.sh  exec.sh  flume-agent  spark-shell  test.sh
[double_happy@hadoop101 script]$ 

明白了吗？把exec 封装在一个shell 里 去调用别的脚本

[double_happy@hadoop101 bin]$ cat spark-class 
#!/usr/bin/env bash

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

if [ -z "${SPARK_HOME}" ]; then
  source "$(dirname "$0")"/find-spark-home
fi

. "${SPARK_HOME}"/bin/load-spark-env.sh

# Find the java binary
if [ -n "${JAVA_HOME}" ]; then
  RUNNER="${JAVA_HOME}/bin/java"
else
  if [ "$(command -v java)" ]; then
    RUNNER="java"
  else
    echo "JAVA_HOME is not set" >&2
    exit 1
  fi
fi

# Find Spark jars.
if [ -d "${SPARK_HOME}/jars" ]; then
  SPARK_JARS_DIR="${SPARK_HOME}/jars"
else
  SPARK_JARS_DIR="${SPARK_HOME}/assembly/target/scala-$SPARK_SCALA_VERSION/jars"
fi

if [ ! -d "$SPARK_JARS_DIR" ] && [ -z "$SPARK_TESTING$SPARK_SQL_TESTING" ]; then
  echo "Failed to find Spark jars directory ($SPARK_JARS_DIR)." 1>&2
  echo "You need to build Spark with the target \"package\" before running this program." 1>&2
  exit 1
else
  LAUNCH_CLASSPATH="$SPARK_JARS_DIR/*"
fi

# Add the launcher build dir to the classpath if requested.
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
  LAUNCH_CLASSPATH="${SPARK_HOME}/launcher/target/scala-$SPARK_SCALA_VERSION/classes:$LAUNCH_CLASSPATH"
fi

# For tests
if [[ -n "$SPARK_TESTING" ]]; then
  unset YARN_CONF_DIR
  unset HADOOP_CONF_DIR
fi

# The launcher library will print arguments separated by a NULL character, to allow arguments with
# characters that would be otherwise interpreted by the shell. Read that in a while loop, populating
# an array that will be used to exec the final command.
#
# The exit code of the launcher is appended to the output, so the parent shell removes it from the
# command array and checks the value to see if the launcher succeeded.
build_command() {
  "$RUNNER" -Xmx128m -cp "$LAUNCH_CLASSPATH" org.apache.spark.launcher.Main "$@"
  printf "%d\0" $?
}

# Turn off posix mode since it does not allow process substitution
set +o posix
CMD=()
while IFS= read -d '' -r ARG; do
  CMD+=("$ARG")
done < <(build_command "$@")

COUNT=${#CMD[@]}
LAST=$((COUNT - 1))
LAUNCHER_EXIT_CODE=${CMD[$LAST]}

# Certain JVM failures result in errors being printed to stdout (instead of stderr), which causes
# the code that parses the output of the launcher to get confused. In those cases, check if the
# exit code is an integer, and if it's not, handle it as a special error case.
if ! [[ $LAUNCHER_EXIT_CODE =~ ^[0-9]+$ ]]; then
  echo "${CMD[@]}" | head -n-1 1>&2
  exit 1
fi

if [ $LAUNCHER_EXIT_CODE != 0 ]; then
  exit $LAUNCHER_EXIT_CODE
fi

CMD=("${CMD[@]:0:$LAST}")
exec "${CMD[@]}"
[double_happy@hadoop101 bin]$

. "${SPARK_HOME}"/bin/load-spark-env.sh  这是在做什么？
就是把环境里的scala版本找到

build_command() {
  "$RUNNER" -Xmx128m -cp "$LAUNCH_CLASSPATH" org.apache.spark.launcher.Main "$@"
  printf "%d\0" $?
}


RUNNER=java 去看一下就知道

spark-shell {

"${SPARK_HOME}"/bin/spark-submit \
--class org.apache.spark.repl.Main \
 --name "Spark shell" \
 "$@"
}

==> spark-submit{
	exec "${SPARK_HOME}"/bin/spark-class \
	 org.apache.spark.deploy.SparkSubmit \
	 "$@"
}

==>spark-class{
	build_command() {
	  "$RUNNER" -Xmx128m -cp "$LAUNCH_CLASSPATH" org.apache.spark.launcher.Main "$@"
	  printf "%d\0" $?
	}
}


spark-shell最底层就是使用Jave来启动 org.apache.spark.launcher.Main类
REPL  ==》交互式解释器

看org.apache.spark.launcher.Main 源码 需要这个依赖

 <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-launcher_2.11</artifactId>
            <version>${spark.version}</version>
 </dependency>

idea ：
control + shift +n 
control + shift +f