搭建一个IntelliJ的Spark项目

之前发现创建一个新项目之后，无法添加scala class

创建新项目
　　选择maven项目，然后选择simple或者quickstart；
　　进入项目后，在Project Structure里面，在global libraries面板中，删除已有的scala-sdk，然后再添加；
　　然后再工程中添加scala文件夹，添加一个package，在package里面就可以添加scala文件了。

Maven

 <?xml version="1.0" encoding="UTF-8"?>
 <project xmlns="http://maven.apache.org/POM/4.0.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>

     <groupId>com.cmiot</groupId>
     <artifactId>parquetToHbase</artifactId>
     <packaging>jar</packaging>
     <version>3.2.1-diy-thinrow</version>

     <repositories>
         <repository>
             <id>cloudera</id>
             <url>https://repository.cloudera.com/cloudera/cloudera-repos</url>
         </repository>
         <repository>
             <id>nexus</id>
             <url>https://maven.atlassian.com/3rdparty/</url>
         </repository>
     </repositories>

     <dependencies>
         <dependency>
             <groupId>com.typesafe</groupId>
             <artifactId>config</artifactId>
             <version>1.2.1</version>
         </dependency>

         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-core_${spark.tool.version}</artifactId>
             <version>${spark.version}</version>
             <!--scope>provided</scope-->
             <exclusions>
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>scala-library</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>scala-lang</artifactId>
                 </exclusion>
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>scalap</artifactId>
                 </exclusion>
                 <exclusion>
                     <artifactId>slf4j-log4j12</artifactId>
                     <groupId>org.slf4j</groupId>
                 </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_${spark.tool.version}</artifactId>
             <version>${spark.version}</version>
             <!--scope>provided</scope-->
         </dependency>
         <!--<dependency>-->
             <!--<groupId>com.oracle</groupId>-->
             <!--<artifactId>ojdbc6</artifactId>-->
             <!--<version>11.2.0.4.0-atlassian-hosted</version>-->
         <!--</dependency>
         <dependency>
             <groupId>oracle</groupId>
             <artifactId>ojdbc</artifactId>
             <version>8</version>
         </dependency>-->
         <dependency>
             <groupId>com.github.nscala-time</groupId>
             <artifactId>nscala-time_${scala.version.tools}</artifactId>
             <version>2.0.0</version>
         </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-library</artifactId>
             <version>${scala.version}</version>
         </dependency>

         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-reflect</artifactId>
             <version>${scala.version}</version>
         </dependency>

         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-compiler</artifactId>
             <version>${scala.version}</version>
         </dependency>
         <dependency>
             <groupId>org.scalatest</groupId>
             <artifactId>scalatest_${scala.version.tools}</artifactId>
             <version>${scala.test.version}</version>
             <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>scala-library</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.scalacheck</groupId>
             <artifactId>scalacheck_${scala.version.tools}</artifactId>
             <version>${scala.check.version}</version>
             <scope>test</scope>
             <exclusions>
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>scala-library</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
             <version>${junit.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-log4j12</artifactId>
             <version>${slf4j.log4j12.version}</version>
             <!--<scope>test</scope>-->
         </dependency>
         <dependency>
             <groupId>org.alluxio</groupId>
             <artifactId>alluxio-core-client</artifactId>
             <version>${alluxio.client.version}</version>
         </dependency>
         <dependency>
             <groupId>com.databricks</groupId>
             <artifactId>spark-xml_${spark.tool.version}</artifactId>
             <version>${spark.xml.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-client</artifactId>
             <version>${hbase.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>javax.servlet</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-server</artifactId>
             <version>${hbase.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>javax.servlet</groupId>
                     <artifactId>*</artifactId>
                 </exclusion>
             </exclusions>
         </dependency>
         <!--
         <dependency>
             <groupId>org.apache.hbase</groupId>
             <artifactId>hbase-spark</artifactId>
             <version>1.2.0-cdh5.10.2</version>
         </dependency>
         -->
     </dependencies>

     <properties>
         <hbase.version>1.2.0-cdh5.11.2</hbase.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <maven.compiler.encoding>UTF-8</maven.compiler.encoding>
         <maven.surefire.version>2.7</maven.surefire.version>
         <maven.compiler.version>3.1</maven.compiler.version>
         <maven.assembly.version>2.4</maven.assembly.version>
         <java.version>1.8</java.version>
         <scala.version>2.11.8</scala.version>
         <scala.version.tools>2.11</scala.version.tools>
         <scala.maven.version>3.2.0</scala.maven.version>
         <scala.check.version>1.11.4</scala.check.version>
         <scala.test.version>3.0.1</scala.test.version>
         <scala.test.maven.version>1.0</scala.test.maven.version>
         <junit.version>4.11</junit.version>
         <spark.version>2.1.0.cloudera2</spark.version>
         <spark.tool.version>2.11</spark.tool.version>
         <spark.xml.version>0.4.1</spark.xml.version>
         <alluxio.client.version>1.0.0</alluxio.client.version>
         <cobertura.version>2.7</cobertura.version>
         <slf4j.log4j12.version>1.7.10</slf4j.log4j12.version>
         <test.arg.line>-Xms1024m -Xmx1024m -XX:PermSize=512m -XX:MaxPermSize=1024m</test.arg.line>
     </properties>

     <profiles>
         <profile>
             <id>dev</id>
             <activation>
                 <activeByDefault>true</activeByDefault>
             </activation>
             <properties>
                 <filters.env>dev</filters.env>
             </properties>
         </profile>

         <profile>
             <id>test</id>

             <properties>
                 <filters.env>test</filters.env>
             </properties>
         </profile>

         <profile>
             <id>prod</id>
             <properties>
                 <filters.env>prod</filters.env>
             </properties>
         </profile>
     </profiles>

     <build>
         <resources>
             <resource>
                 <directory>${project.basedir}/src/main/resources</directory>
                 <filtering>true</filtering>
             </resource>
         </resources>
         <filters>
             <filter>${project.basedir}/filters/${filters.env}/conf.properties</filter>
         </filters>
         <plugins>

             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
                 <version>${scala.maven.version}</version>
                 <executions>
                     <execution>
                         <id>scala-compile</id>
                         <phase>process-resources</phase>
                         <goals>
                             <goal>add-source</goal>
                             <goal>compile</goal>
                         </goals>
                     </execution>
                     <execution>
                         <id>scala-test-compile</id>
                         <phase>process-test-resources</phase>
                         <goals>
                             <goal>testCompile</goal>
                         </goals>
                     </execution>
                 </executions>
             </plugin>
             <!---->
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <version>${maven.compiler.version}</version>
                 <configuration>
                     <source>${java.version}</source>
                     <target>${java.version}</target>
                 </configuration>
             </plugin>
             <!--
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>
                 <version>${maven.assembly.version}</version>
                 <configuration>
                     <descriptorRefs>
                         <descriptorRef>jar-with-dependencies</descriptorRef>
                     </descriptorRefs>
                     <archive>
                         <manifest>
                             <mainClass>com.cmiot.StatsApp</mainClass>
                         </manifest>
                     </archive>
                 </configuration>
                 <executions>
                     <execution>
                         <phase>package</phase>
                         <goals>
                             <goal>single</goal>
                         </goals>
                     </execution>
                 </executions>
             </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>${maven.surefire.version}</version>
                 <configuration>
                     <argLine>${test.arg.line}</argLine>
                 </configuration>
             </plugin>
             <plugin>
                 <groupId>org.scalatest</groupId>
                 <artifactId>scalatest-maven-plugin</artifactId>
                 <version>${scala.test.maven.version}</version>
                 <configuration>
                     <skipTests>false</skipTests>
                     <reportsDirectory>${project.build.directory}/test-reports</reportsDirectory>
                     <junitxml>.</junitxml>
                     <filereports>WDF TestSuite.txt</filereports>
                     <argLine>${test.arg.line}</argLine>
                 </configuration>
                 <executions>
                     <execution>
                         <id>test</id>
                         <goals>
                             <goal>test</goal>
                         </goals>
                     </execution>
                 </executions>
             </plugin>
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>cobertura-maven-plugin</artifactId>
                 <version>${cobertura.version}</version>
                 <configuration>
                     <formats>
                         <format>xml</format>
                         <format>html</format>
                     </formats>
                     <check/>
                     <instrumentation>
                         <ignores>
                             <ignore>com.cmiot.StatsApp</ignore>
                             <ignore>com.cmiot.app</ignore>
                         </ignores>
                         <excludes>
                             <exclude>com/cmiot/StatsApp*</exclude>
                             <exclude>com/cmiot/app/Stats*</exclude>
                         </excludes>
                     </instrumentation>
                 </configuration>
                 <executions>
                     <execution>
                         <phase>test</phase>
                         <goals>
                             <goal>cobertura</goal>
                         </goals>
                     </execution>
                 </executions>
             </plugin>
             -->
         </plugins>
     </build>
 </project>