Create An Spark Streaming App: 1. Architecture and Abstraction
Create An Spark Streaming App: 1. Architecture and Abstraction
Transformations
Transformations on DStreams can be grouped into either stateless or stateful:
Output Operations
Output operations specify what needs to be done with the final transformed data in a
stream (e.g., pushing it to an external database or printing it to the screen).
import org.apache.spark._
import org.apache.spark.SparkContext._
import org.apache.spark.streaming.StreamingContext
import org.apache.spark.streaming.StreamingContext._
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.Duration
import org.apache.spark.streaming.Seconds
object SocketStream {
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("Socket-Stream")
// Create a StreamingContext with a 1-second batch size from
a SparkConf
val ssc = new StreamingContext(conf, Seconds(1))
// Create a DStream using data received after connecting to
port 7777 on the
// local machine
val lines = ssc.socketTextStream("localhost", 7777)
// Filter our DStream for lines with "error"
val errorLines = lines.filter(_.contains("error"))
// Print out the lines with errors
errorLines.print()
// Start our streaming context and wait for it to "finish"
ssc.start()
// Wait for the job to finish
ssc.awaitTermination()
}
}
name := "socket-stream"
version := "0.0.1"
scalaVersion := "2.11.12"
// additional libraries
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % "2.4.1" % "provided",
"org.apache.spark" %% "spark-streaming" % "2.4.1"
)
- Build application :
$ sbt clean package
object ApacheAccessLog {
val PATTERN = """^(\S+) (\S+) (\S+) \[([\w:/]+\s[+\-]\d{4})\]
"(\S+) (\S+) (\S+)" (\d{3}) (\d+)""".r
/**
* Parse log entry from a string.
*
* @param log A string, typically a line from a log file
* @return An entry of Apache access log
* @throws RuntimeException Unable to parse the string
*/
def parseLogLine(log: String): ApacheAccessLog = {
log match {
case PATTERN(ipAddress, clientIdentd, userId, dateTime,
method, endpoint, protocol, responseCode, contentSize)
=> ApacheAccessLog(ipAddress, clientIdentd, userId,
dateTime, method, endpoint, protocol, responseCode.toInt,
contentSize.toLong)
case _ => throw new RuntimeException(s"""Cannot parse log
line: $log""")
}
}
}
import org.apache.spark.SparkConf
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.dstream.DStream
import org.apache.spark.streaming.{Seconds, StreamingContext}
object LogAnalyzerStreaming {
def main(args: Array[String]) {
val WINDOW_LENGTH = Seconds(30)
val SLIDE_INTERVAL = Seconds(10)
windowDStream.foreachRDD(accessLogs => {
if (accessLogs.count() == 0) {
println("No access logs received in this time interval")
} else {
// Calculate statistics based on the content size.
val contentSizes: RDD[Long] =
accessLogs.map(_.contentSize).cache()
println("Content Size Avg: %s, Min: %s, Max: %s".format(
contentSizes.reduce(_ + _) / contentSizes.count,
contentSizes.min,
contentSizes.max
))
// Top Endpoints.
val topEndpoints: Array[(String, Long)] = accessLogs
.map(_.endpoint -> 1L)
.reduceByKey(_ + _)
.top(10)(Ordering.by[(String, Long), Long](_._2))
println( s"""Top Endpoints: ${topEndpoints.mkString("[",
",", "]")}""")
}
})
name := "log-analyzer"
version := "0.0.1"
scalaVersion := "2.11.12"
// additional libraries
- Create the shell script named "stream.sh" that emulates network stream by
periodically sending portions of the sample log file to a network socket:
#!/bin/sh
set -o nounset
set -o errexit
file="$1"
network_port=9999
lines_in_batch=100
interval_sec=10
- Build application :
$ sbt clean package