Pacific-Design.com

    
Home Index

1. Spark

2. spark-submit

Spark / spark-submit /

Example 1: Local 4 CPU Submition

spark-submit                                        
  --packages com.databricks:spark-avro_2.11:3.0.0   
  --class "ExportData"                              
  --master local[4]                                 
  --driver-memory   16G                             
  --executor-memory 4G                              
  target/scala-2.11/exportdata_2.11-1.0.jar         
  $INPUT_S3_DIR                                     
  "0" "249" $OUTPUT_PREFIX &

Example 2: Cluster Submition

spark-submit                                                                
  --jars /opt/lib/aws-java-sdk-1.11.32.jar,/opt/lib/hadoop-aws-2.7.0.jar    
  --packages com.databricks:spark-avro_2.11:3.0.0                           
  --class "Avro2Cassandra"                                                  
  --driver-memory   16G                                                     
  --num-executors   4                                                       
  --executor-cores  4                                                       
  --executor-memory 4G                                                      
  --master spark://192.168.1.100:7077                                       
  target/scala-2.11/avro2cassandra_2.11-1.0.jar                             
  $INPUT_S3_DIR                                                             
  "0" "999" &