电影推荐系统开发

创建一个maven项目MovieRecommendSystem，在MovieRecommendSystem下创建一个maven模块recommender，然后在recommender模块下创建DataLoader模块。

MovieRecommendSystem的依赖文件：

1.  
   <dependencyManagement>
2.  
   <dependencies>
3.  
   <!--引入Spark相关的Jar包-->
4.  
   <dependency>
5.  
   <groupId>org.apache.park</groupId>
6.  
   <artifactId>spark-core_2.11</artifactId>
7.  
   <version>${spark.version}</version>
8.  
   </dependency>
9.  
   <dependency>
10.  
    <groupId>org.apache.spark</groupId>
11.  
    <artifactId>spark-sql_2.11</artifactId>
12.  
    <version>${spark.version}</version>
13.  
    </dependency>
14.  
    <dependency>
15.  
    <groupId>org.apache.spark</groupId>
16.  
    <artifactId>spark-streaming2.11</artifactId>
17.  
    <version>${spark.version}</version>
18.  
    </dependency>
19.  
    <dependency>
20.  
    <groupId>org.apache.spark</groupId>
21.  
    <artifactId>spark-mllib_2.11</artifactId>
22.  
    <version>${spark.version}</version>
23.  
    </dependency>
24.  
    <dependency>
25.  
    <groupId>org.apache.spark</groupId>
26.  
    <artifactId>spark-graphx_2.11</artifactId>
27.  
    <version>${spark.version}</version>
28.  
    </dependency>
29.  
    <dependency>
30.  
    <groupId>org.scala-lang</groupId>
31.  
    <artifactId>scala-library</artifactId>
32.  
    <version>${scala.version}</version>
33.  
    </dependency>
34.  
    </dependencies>
35.  
    </dependencyManagement>
36.  
     
37.  
    <build>
38.  
    <plugins>
39.  
    <!--父项目己声明该plugin,子项目在引入的时候，不用声明版本和已经声明的配置-->
40.  
    <plugin>
41.  
    <groupId>net.alchim31.maven</groupId>
42.  
    <artifactId>scala-maven-plugin</artifactId>
43.  
    </plugin>
44.  
    </plugins>
45.  
     
46.  
    </build>

recommender的依赖文件：

1.  
   <dependencyManagement>
2.  
   <dependencies>
3.  
   <!--引入Spark相关的Jar包-->
4.  
   <dependency>
5.  
   <groupId>org.apache.park</groupId>
6.  
   <artifactId>spark-core_2.11</artifactId>
7.  
   <version>${spark.version}</version>
8.  
   </dependency>
9.  
   <dependency>
10.  
    <groupId>org.apache.spark</groupId>
11.  
    <artifactId>spark-sql_2.11</artifactId>
12.  
    <version>${spark.version}</version>
13.  
    </dependency>
14.  
    <dependency>
15.  
    <groupId>org.apache.spark</groupId>
16.  
    <artifactId>spark-streaming2.11</artifactId>
17.  
    <version>${spark.version}</version>
18.  
    </dependency>
19.  
    <dependency>
20.  
    <groupId>org.apache.spark</groupId>
21.  
    <artifactId>spark-mllib_2.11</artifactId>
22.  
    <version>${spark.version}</version>
23.  
    </dependency>
24.  
    <dependency>
25.  
    <groupId>org.apache.spark</groupId>
26.  
    <artifactId>spark-graphx_2.11</artifactId>
27.  
    <version>${spark.version}</version>
28.  
    </dependency>
29.  
    <dependency>
30.  
    <groupId>org.scala-lang</groupId>
31.  
    <artifactId>scala-library</artifactId>
32.  
    <version>${scala.version}</version>
33.  
    </dependency>
34.  
    </dependencies>
35.  
    </dependencyManagement>
36.  
     
37.  
    <build>
38.  
    <plugins>
39.  
    <!--父项目己声明该plugin,子项目在引入的时候，不用声明版本和已经声明的配置-->
40.  
    <plugin>
41.  
    <groupId>net.alchim31.maven</groupId>
42.  
    <artifactId>scala-maven-plugin</artifactId>
43.  
    </plugin>
44.  
    </plugins>
45.  
     
46.  
    </build>

DataLoader的依赖文件：

1.  
   <dependencies>
2.  
   <!-- Spark的依赖引入-->
3.  
   <dependency>
4.  
   <groupId>org.apache.spark</groupId>
5.  
   <artifactId>spark-core_2.11</artifactId>
6.  
   <version>2.1.1</version>
7.  
   </dependency>
8.  
   <dependency>
9.  
   <groupId>org.apache.spark</groupId>
10.  
    <artifactId>spark-sql_2.11</artifactId>
11.  
    </dependency>
12.  
    <!--引入Scala -->
13.  
    <dependency>
14.  
    <groupId>org.scala-lang</groupId>
15.  
    <artifactId>scala-library</artifactId>
16.  
    </dependency>
17.  
    <!--加入MongoDB的驱动-->
18.  
    <dependency>
19.  
    <groupId>org.mongodb</groupId>
20.  
    <artifactId>casbah-core_2.11</artifactId>
21.  
    <version>${casbah.version}</version>
22.  
    </dependency>
23.  
    <dependency>
24.  
    <groupId>org.mongodb.spark</groupId>
25.  
    <artifactId>mongo-spark-connector_2.11</artifactId>
26.  
    <version>${mongodb-spark.version}</version>
27.  
    </dependency>
28.  
    <!--加入ELasticSearch的驱动-->
29.  
    <dependency>
30.  
    <groupId>org.elasticsearch.client</groupId>
31.  
    <artifactId>transport</artifactId>
32.  
    <version>${elasticsearch.version}</version>
33.  
    </dependency>
34.  
    <dependency>
35.  
    <groupId>org.elasticsearch</groupId>
36.  
    <artifactId>elasticsearch-spark-20_2.11</artifactId>
37.  
    <version>${elasticsearch-spark.version}</version>
38.  
    <!--将不需要依赖的包从依赖路径中除去-->
39.  
    <exclusions>
40.  
    <exclusion>
41.  
    <groupId>org.apache.hive</groupId>
42.  
    <artifactId>hive-service</artifactId>
43.  
    </exclusion>
44.  
    </exclusions>
45.  
    </dependency>
46.  
    </dependencies>

在DataLoader文件的resources下创建log4j.properties文件，并将数据集复制进入

1.  
   log4j.rootLogger=info, stdout
2.  
   log4j.appender.stdout=org.apache.log4j.ConsoleAppender
3.  
   log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
4.  
   log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %5p --- [%50t] %-80c(line:%5L) : %m%n

接下来需要安装配置scala，并且在idea内创建 Scala Calss 

这篇好文章是转载于：学新通技术网