Filter and Map

Filter data //create person.txt Name, Age Vinayak, 35 Nilesh, 37 Raju, 30 Karthik, 28 Shreshta,1 Siddhish, 2     val personfile = sc.textFile("C:/Users/Redirection/pateln7/Downloads/SPARK/person.txt") val first = personfile.first val person = personfile.filter(x=>x!=first) val personmap = person.map(x=>{   val splitted = x.split(",")   val name = splitted(0).trim   val age = splitted(1).trim.toInt   (name,age) }) personmap.foreach(println) // ========= Method 1 : Scala val output = personmap.filter(x=>x._2>30) output.foreach(println) // ========= Method 2 : DataFrame //option 1 : val persondf=personmap.toDF persondf.show val output=persondf.filter($"_2">30) output.show //option 2 : val persondf=personmap.toDF("name","age") persondf.show val output=persondf.filter("age>30") output.show // ========= Method 3 : Spark sql //(didn’t worked in wondows) persondf.registerTempTable("person") val sqlcontext = org.apache.spark.sql.SQLContext // ========= Method 4 : sqlContext val persondf=personmap.toDF("name","age") persondf.registerTempTable("Person") val sqlcontext = persondf.sqlContext val output = sqlcontext.sql("select * from person where age>30").show //OR val persondf=personmap.toDF persondf.registerTempTable("Person2") val sqlcontext = persondf.sqlContext val output = sqlcontext.sql("select * from person2 where _2>30").show