Spark运行的4种模式

Spark 运行的4种模式

1. 4种运行模式概述图

2. 不同的提交参数说明

  ./bin/spark-submit \

  //主类入口
  --class  \ 

  // 指定appname
  --name      
     \
  //pom依赖所需要的resource目录下的资源文件 
  --files     \

  //需要的jar包
  --jar      	  \

  //运行内存
  --executor-memory 1G \

  //运行内核数
  --num-executors 1 \

 //运行模式指定
  --master  \

  //指定client模式或者cluster模式，默认是client
  --deploy-mode  \

  //设置参数
  --conf = \

  //jar包路径
   \

  //main方法的参数
  [application-arguments]

  

# Run application locally on 8 cores
./bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master local[8] \
  /path/to/examples.jar \
  100

# Run on a Spark standalone cluster in client deploy mode
./bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master spark://207.184.161.138:7077 \
  --executor-memory 20G \
  --total-executor-cores 100 \
  /path/to/examples.jar \
  1000

# Run on a Spark standalone cluster in cluster deploy mode with supervise
./bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master spark://207.184.161.138:7077 \
  --deploy-mode cluster \
  --supervise \
  --executor-memory 20G \
  --total-executor-cores 100 \
  /path/to/examples.jar \
  1000

# Run on a YARN cluster
export HADOOP_CONF_DIR=XXX
./bin/spark-submit \
  --class org.apache.spark.examples.SparkPi \
  --master yarn \
  --deploy-mode cluster \  # can be client for client mode
  --executor-memory 20G \
  --num-executors 50 \
  /path/to/examples.jar \
  1000