From 95c588ef530c019a80d1594cb4e36dbf08149085 Mon Sep 17 00:00:00 2001 From: wah Date: Fri, 22 Mar 2019 16:14:55 +0800 Subject: [PATCH 1/4] Scala Test --- pom.xml | 139 ++++++++++++++++++++++++++++ src/main/resources/log4j.properties | 13 +++ src/main/scala/BaseTest.scala | 27 ++++++ 3 files changed, 179 insertions(+) create mode 100644 pom.xml create mode 100644 src/main/resources/log4j.properties create mode 100644 src/main/scala/BaseTest.scala diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..9dfebd5 --- /dev/null +++ b/pom.xml @@ -0,0 +1,139 @@ + + + 4.0.0 + + groupId + Learning + 1.0-SNAPSHOT + + + + 4.12 + 2.10.6 + 1.6.2 + 1.2.171.7.22 + + + + + junit + junit + ${junit.version} + + + + + org.scala-lang + scala-library + ${scala.version} + + + + + + org.slf4j + jcl-over-slf4j + ${slf4j.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + org.slf4j + slf4j-log4j12 + ${slf4j.version} + + + log4j + log4j + ${log4j.version} + + + + + + + org.apache.spark + spark-core_2.10 + ${spark.version} + + + org.apache.spark + spark-sql_2.10 + ${spark.version} + + + org.apache.spark + spark-streaming_2.10 + ${spark.version} + + + + + + + + + + src/main/java + + **/*.xml + **/*.properties + + + + src/main/resources + + **/*.xml + **/*.properties + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.2 + + 1.8 + 1.8 + UTF-8 + + + + compile + + compile + + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.1 + + + scala-compile-first + process-resources + + add-source + compile + + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..06cb3f6 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,13 @@ +#log4j.rootLogger=warn,stdout,R? ?#ʹ־ +log4j.rootLogger=info,stdout,R +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS}??%5p --- [%t]??%-c(line:%L) : %m%n + +log4j.appender.R=org.apache.log4j.RollingFileAppender +log4j.appender.R.File=spark.log +log4j.appender.R.MaxFileSize=1024KB +log4j.appender.R.MaxBackupIndex=1 + +log4j.appender.R.layout=org.apache.log4j.PatternLayout +log4j.appender.R.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS}??%5p --- [%t]??%-c(line:%L) : %m%n \ No newline at end of file diff --git a/src/main/scala/BaseTest.scala b/src/main/scala/BaseTest.scala new file mode 100644 index 0000000..90ba3be --- /dev/null +++ b/src/main/scala/BaseTest.scala @@ -0,0 +1,27 @@ + +import org.apache.spark.{SparkConf, SparkContext} +import org.slf4j.{Logger, LoggerFactory} + +object BaseTest { + val logger: Logger = LoggerFactory.getLogger(BaseTest.getClass) + + def main(args: Array[String]) { + //创建SparkConf()并设置App名称 + val conf = new SparkConf().setMaster("local[*]").setAppName("BaseTest") + + //创建SparkContext，该对象是提交spark App的入口 + val sc = new SparkContext(conf) + + //使用sc创建RDD并执行相应的transformation和action + val result = sc.textFile("C:\\Users\\Dell\\Desktop\\日志分析\\error-log24-2.log").flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _, 1).sortBy(_._2, false) + + //停止sc，结束该任务 + result.collect().foreach(println(_)) + + //result.saveAsTextFile("hdfs") + + logger.info("----complete!----") + + sc.stop() + } +} From 97e1082efcf036fec592137a018acc01d6274fd1 Mon Sep 17 00:00:00 2001 From: wah Date: Fri, 22 Mar 2019 16:20:24 +0800 Subject: [PATCH 2/4] Scala Test --- src/main/resources/log4j.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties index 06cb3f6..1994dbe 100644 --- a/src/main/resources/log4j.properties +++ b/src/main/resources/log4j.properties @@ -1,5 +1,6 @@ #log4j.rootLogger=warn,stdout,R? ?#ʹ־ -log4j.rootLogger=info,stdout,R +#log4j.rootLogger=info,stdout,R +log4j.rootLogger=warn,stdout,R log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS}??%5p --- [%t]??%-c(line:%L) : %m%n From 758f4e8227ee229db4a3bb84440395ef289d686e Mon Sep 17 00:00:00 2001 From: wah Date: Fri, 25 Oct 2019 13:44:40 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pom.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9dfebd5..645ce4c 100644 --- a/pom.xml +++ b/pom.xml @@ -13,7 +13,8 @@ 4.12 2.10.6 1.6.2 - 1.2.171.7.22 + 1.2.17 + 1.7.22 From f38c4dc8779afadafbf2f9d5e2c15df490e8d0e1 Mon Sep 17 00:00:00 2001 From: wah Date: Fri, 25 Oct 2019 13:52:45 +0800 Subject: [PATCH 4/4] word count --- src/main/scala/BaseTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/BaseTest.scala b/src/main/scala/BaseTest.scala index 90ba3be..86fe2b8 100644 --- a/src/main/scala/BaseTest.scala +++ b/src/main/scala/BaseTest.scala @@ -13,7 +13,7 @@ object BaseTest { val sc = new SparkContext(conf) //使用sc创建RDD并执行相应的transformation和action - val result = sc.textFile("C:\\Users\\Dell\\Desktop\\日志分析\\error-log24-2.log").flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _, 1).sortBy(_._2, false) + val result = sc.textFile("C:\\Users\\Dell\\Desktop\\word.txt").flatMap(_.split(" ")).map((_, 1)).reduceByKey(_ + _, 1).sortBy(_._2, false) //停止sc，结束该任务 result.collect().foreach(println(_))