Pacific-Design.com

    
Home Index

1. Apache Spark

2. SBT Project

Apache Spark / SBT Project /

simple.sbt


name := "Simple Project"

version := "1.0"

scalaVersion := "2.10.4"

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.3.1"

SimpleApp.scala

/*
   SimpleApp.scala

   dse hadoop dfs -ls /data
   dse hadoop dfs -mkdir /data
   dse hadoop dfs -copyFromLocal war_and_peace.txt /data

   sbt package
   dse spark-submit --class SimpleApp --master spark://172.31.20.232:7077 target/scala-2.10/simple-project_2.10-1.0.jar
*/

import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf

object SimpleApp {

  def main(args: Array[String]) {

    val logFile = "/data/war_and_peace.txt"
    val conf = new SparkConf().setAppName("Search Application")
    val sc = new SparkContext(conf)
    val logData = sc.textFile(logFile, 2).cache()

    val numAs = logData.filter(line => line.contains("King")).count()
    val numBs = logData.filter(line => line.contains("tongue")).count()

    println("King = %s, tongue = %s ".format(numAs, numBs))

  }

}

/*
val counts = textFile.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _)
counts.saveAsTextFile("hdfs://data/output1")
*/

build project: $ sbt package

https://spark.apache.org/docs/latest/quick-start.html