wget http://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz
tar -zxvf apache-maven-3.6.1-bin.tar.gz -c /usr/local rm apache-maven-3.6.1-bin.tar.gz -c /usr/local
cd /usr/local mv apache-maven-3.6.1 maven-3.6.1
vim /etc/profile
export maven_home=/usr/local/maven-3.6.1 export path=$java_home/bin:$maven_home/bin:$path
source /etc/profile
mvn -version
cd ~
mkdir sparkapp2
vim ./sparkapp2/src/main/java/simpleapp.java
1 /*** simpleapp.java ***/ 2 import org.apache.spark.api.java.*; 3 import org.apache.spark.api.java.function.function; 4 5 public class simpleapp { 6 public static void main(string[] args) { 7 string logfile = "file:///usr/local/spark-2.4.3/readme.md"; 8 javasparkcontext sc = new javasparkcontext("local", "simple app", 9 "file:///usr/local/spark-2.4.3/", new string[]{"target/simple-project-1.0.jar"}); 10 javardd<string> logdata = sc.textfile(logfile).cache(); 11 12 long numas = logdata.filter(new function<string, boolean>() { 13 public boolean call(string s) { 14 return s.contains("a"); 15 } 16 }).count(); 17 18 long numbs = logdata.filter(new function<string, boolean>() { 19 public boolean call(string s) { return s.contains("b"); } 20 }).count(); 21 22 system.out.println("lines with a: " + numas + ", lines with b: " + numbs); 23 } 24 }
<project> <groupid>edu.berkeley</groupid> <artifactid>simple-project</artifactid> <modelversion>4.0.0</modelversion> <name>simple project</name> <packaging>jar</packaging> <version>1.0</version> <repositories> <repository> <id>akka repository</id> <url>http://repo.akka.io/releases</url> </repository> </repositories> <dependencies> <dependency> <!-- spark dependency --> <groupid>org.apache.spark</groupid> <artifactid>spark-core_2.11</artifactid> <version>2.4.3</version> </dependency> </dependencies> </project>
cd ~/sparkapp2 find .
/usr/local/maven-3.6.1/bin.mvn package
/usr/local/spark-2.4.3/bin/spark-submit --class "simpleapp" ~/sparkapp2/target/simple-project-1.0.jar 2>&1 | grep "lines with a"
如对本文有疑问, 点击进行留言回复!!
去 HBase,Kylin on Parquet 性能表现如何?
如何找到Hive提交的SQL相对应的Yarn程序的applicationId
如何在 HBase Shell 命令行正常查看十六进制编码的中文?哈哈~
网友评论