010-spark standalone模式Scala版本WordCount代码

#!/usr/bin/env bash
  
 SPARK_MASTER_IP=mycluster  
 export JAVA_HOME=/home/hadoop/app/jdk1.7.0_76  
 export SCALA_HOME=/home/hadoop/app/scala  
 export HADOOP_HOME=/home/hadoop/app/hadoop-2.6.0  
 export HIVE_HOME=/home/hadoop/app/hive  
 export SPARK_CLASSPATH=$HIVE_HOME/lib/mysql-connector-java-5.1.28.jar

slaves配置文件的内容
  
 # A Spark Worker will be started on each of the machines listed below.  
 192.168.2.20  
 192.168.2.33

<?xml version= "1.0" encoding ="UTF-8"?>
<project xmlns= "http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" >
    <modelVersion >4.0.0 </modelVersion>

    <groupId >testSpark </groupId>
    <artifactId >testSpark </artifactId>
    <version >1.0-SNAPSHOT </version>

    <repositories >
        <repository>
            <id> Akka repository </id>
            <url> http://repo.akka.io/releases</url >
        </repository>
        <repository>
            <id> cloudera</id >
            <url> https://repository.cloudera.com/artifactory/cloudera-repos/. </url>
        </repository>
        <repository>
            <id> jboss</id >
            <url> http://repository.jboss.org/nexus/content/groups/public-jboss </url>
        </repository>
        <repository>
            <id> Sonatype snapshots </id>
            <url> http://oss.sonatype.org/content/repositories/snapshots/</url >
        </repository>
    </repositories >

    <build >
        <sourceDirectory> src/spark/</sourceDirectory >
        <testSourceDirectory> src/test/</testSourceDirectory >
          <!-- 添加pluginManagement：解决 Maven报Plugin execution not covered by lifecycle configuration
         具体详见： 
          -->
          <pluginManagement>
            <plugins>
                <plugin>
                    <groupId> org.scala-tools</groupId >
                    <artifactId> maven- scala-plugin</artifactId >
                    <executions>
                        <execution>
                            <goals>
                                <goal> compile</ goal>
                                <goal> testCompile</goal >
                            </goals>
                        </execution>
                    </executions>
                    <configuration>
                        <scalaVersion> 2.10.5</ scalaVersion>
                    </configuration>
                </plugin>
    
                <plugin>
                    <groupId> org.apache.maven.plugins</groupId >
                    <artifactId> maven-shade- plugin</artifactId>
                    <version> 2.2</ version>
                    <executions>
                        <execution>
                            <phase> package</ phase>
                            <goals>
                                <goal> shade</ goal>
                            </goals>
                            <configuration>
                                <filters>
                                    <filter>
                                        <artifact> *:*</ artifact>
                                        <excludes>
                                            <exclude> *</ exclude>
                                        </excludes>
                                    </filter>
                                </filters>
                                <transformers>
    
                                    <transformer
                                            implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer" >
                                        <resource> reference.conf</resource >
                                    </transformer>
    
                                    <transformer
                                             implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer" >
                                    </transformer>
                                </transformers>
                            </configuration>
                        </execution>
                    </executions>
                </plugin>
            </plugins>
          </pluginManagement>
    </build >

    <dependencies >
          <!-- 解决 maven引用jdk中的tools.jar报Missing artifact的问题 ,指定maven去本地寻找 tools.jar -->
          <dependency>
            <groupId> jdk.tools</groupId >
            <artifactId> jdk.tools</artifactId >
            <version> 1.7</ version>
            <scope> system</ scope>
            <systemPath> ${JAVA_HOME}/lib/tools.jar </systemPath>
        </dependency>
        <dependency>
            <groupId> org.apache.spark</groupId >
            <artifactId> spark-core_2.10</artifactId >
            <version> 1.2.0-cdh5.3.2</version >
        </dependency>
        <dependency>
            <groupId> org.apache.hadoop</groupId >
            <artifactId> hadoop-client </artifactId>
            <version> 2.6.0-cdh5.3.6</version >
        </dependency>
        <dependency>
            <groupId> org.apache.spark</groupId >
            <artifactId> spark-streaming_2.10</artifactId >
            <version> 0.9.0-cdh5.0.0</version >
        </dependency>
        <dependency>
            <groupId> org.apache.spark</groupId >
            <artifactId> spark-yarn_2.10</artifactId >
            <version> 1.2.0-cdh5.3.3</version >
        </dependency>
        <dependency>
            <groupId> org.apache.spark</groupId >
            <artifactId> spark-tools_2.10</artifactId >
            <version> 0.9.0-cdh5.0.0</version >
        </dependency>
     
    </dependencies >

</project>