sbt_opts="-xms512m -xmx1536m -xss1m -xx:+cmsclassunloadingenabled -xx:maxpermsize=256m"
java $sbt_opts -jar `dirname $0`/sbt-launch.jar "$@"
chmod u+x ./sbt
vim ~/.sbt/repositories
[repositories] local aliyun-nexus: http://maven.aliyun.com/nexus/content/groups/public/ typesafe: http://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/(scala_[scalaversion]/)(sbt_[sbtversion]/)[revision]/[type]s/[artifact](-[classifier]).[ext], bootonly sonatype-oss-releases maven-central sonatype-oss-snapshots
vim sbtconfig.txt
vim sbtopts
-dsbt.override.build.repos=true
./sbt sbtversion
cd ~
mkdir sparkapp
mkdir -p ./sparkapp/src/main/scala
vim ./sparkapp/src/main/scala/simpleapp.scala
1 /* simpleapp.scala */ 2 import org.apache.spark.sparkcontext 3 import org.apache.spark.sparkcontext._ 4 import org.apache.spark.sparkconf 5 6 object simpleapp { 7 def main(args: array[string]) { 8 val logfile = "file:///usr/local/spark-2.4.3/readme.md" 9 val conf = new sparkconf().setappname("simple application") 10 val sc = new sparkcontext(conf) 11 val logdata = sc.textfile(logfile, 2).cache() 12 val numas = logdata.filter(line => line.contains("a")).count() 13 val numbs = logdata.filter(line => line.contains("b")).count() 14 println("lines with a: %s, lines with b: %s".format(numas, numbs)) 15 } 16 }
name := "simple project" version := "1.0" scalaversion := "2.11.12" librarydependencies += "org.apache.spark" %% "spark-core" % "2.4.3"
cd ~/sparkapp find .
/usr/local/sbt/sbt package
~/sparkapp/target/scala-2.11/simple-project_2.11-1.0.jar
/usr/local/spark-2.4.3/bin/spark-submit --class "simpleapp" ~/sparkapp/target/scala-2.11/simple-project_2.11-1.0.jar
/usr/local/spark-2.4.3/bin/spark-submit --class "simpleapp" ~/sparkapp/target/scala-2.11/simple-project_2.11-1.0.jar 2>&1 | grep "lines with a:"
如对本文有疑问, 点击进行留言回复!!
HBase Filter 过滤器之FamilyFilter详解
去 HBase,Kylin on Parquet 性能表现如何?
如何找到Hive提交的SQL相对应的Yarn程序的applicationId
网友评论