Spark checkPoint Demo
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Seconds, StreamingContext}
import org.apache.spark.streaming.dstream.ReceiverInputDStream
object UpadateDemo {
def main(args: Array[String]): Unit = {
val conf =new SparkConf().setMaster("local[2]").setAppName(this.getClass.getSimpleName)
val ssc: StreamingContext = new StreamingContext(conf,Seconds(3))
val testUser: RDD[(String, Boolean)] = ssc.sparkContext.makeRDD(Array(("tom",true),("jim",true)))
ssc.checkpoint("./checkpoint")
val input: ReceiverInputDStream[String] = ssc.socketTextStream("b02master",1212)
// updateFunc: (Seq[V], Option[S]) => Option[S],
val func1=(newValue:Seq[Int],state:Option[Int])=>{
val value: Int = newValue.sum + state.getOrElse(0)
Some(value)
}
input.flatMap(_.split(" +")).map((_,1)).updateStateByKey(func1).print()
ssc.start()
ssc.awaitTermination()
}
}
作者：weixin_38677730
