Spark Streaming 002 统计单词的例子

1.准备 
事先在hdfs上创建两个目录：

保存上传数据的目录：hdfs://alamps:9000/library/SparkStreaming/data

checkpoint的目录：hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data

------------------------------------------------------

2.源码

package stream;

import java.util.Arrays;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.function.FlatMapFunction;
import org.apache.spark.api.java.function.Function2;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.streaming.Durations;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.api.java.JavaStreamingContextFactory;

import scala.Tuple2;

public class SparkStreamingOnHDFS {
    public static void main(String[] args) {
        /*
         * 第一步：配置SparkConf
         * 1.至少2条线程：因为Spark Streaming应用程序在运行的时候，至少有一条线程用于不断的循环接收数据，
         * 并且至少有一条线程用于处理接收数据（否则无法有线程用于处理数据，随着时间的推移，内存和磁盘都会不堪重负）
         * 2.对于集群而言，每个Executor一般肯定不止一个Thread，那对于处理spark streaming的应用程序而言，每个Executor
         * 一般分配多少个Core比较合适？根据经验，5个左右的Core是最佳的；
         */
//        final SparkConf conf = new SparkConf().setMaster("spark://alamps:7077")
          final SparkConf conf = new SparkConf().setMaster("local[2]")
                .setAppName("SparkStreamingOnHDFS");
        /*
         * 第二步：创建SparkStreamingContext,这个是SparkStreaming应用程序所有功能的起始点和程序调度核心
         * 1.SparkStreamingContext的构建可以基于SparkConf参数，也可以基于持久化的SparkStreamingContext的内容
         * 来恢复过来（典型的场景是Driver崩溃后重新启动，由于Spark Streaming具有连续7*24小时不间断运行的特征，
         * 所以需要在Driver重启后继续上一次的状态，此时的状态恢复需要基于曾经的Checkpoint）;
         * 2.在一个Spark Streaming应用程序中可以创建若干个SparkStreamingContext对象，使用一个SparkStream之前需要把前面
         * 运行的SparkStreamingContext对象关闭掉，由此，我们获得一个重大的启发SparkStreaming也只是Spark Core上的一个应用
         * 程序而已，只不过Spark Streaming框架运行的话需要Spark工程师写业务逻辑处理代码；
         */
        final String checkpointDirectory = "hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data";
        JavaStreamingContextFactory factory = new JavaStreamingContextFactory(){  

            @Override
            public JavaStreamingContext create() {
                return createContext(checkpointDirectory,conf);
            }  

        };
//        JavaStreamingContext jsc = new JavaStreamingContext(conf,Durations.seconds(5));
        /**
         * 可以从失败中恢复Driver，不过还需要指定Driver这个进程运行在Cluster,并且提交应用程序的时候指定--supervise
         */
        JavaStreamingContext jsc = JavaStreamingContext.getOrCreate(checkpointDirectory, factory);
        /*
         * 第三步：创建Spark Streaming输入数据来源input Stream
         * 1.数据输入来源可以基于File、HDFS、Flume、Kafka、Socket等;
         * 2.在这里我们指定数据来源于网络Socket端口，Spark Streaming连接上该端口并在运行的时候，一直监听该端口的数据
         * （当然该端口服务首先必须存在），并且在后续会根据业务需要不断的有数据产生（当然对于Spark Streaming应用程序
         * 的运行而言，有无数其处理流程都有一样）；
         * 3.如果经常在每间隔5秒钟没有数据的话，会不断的启动空的Job其实是会造成调度资源的浪费，因为并没有数据需要发生计算，所以
         *     实际的企业级生产环境的代码在具体提交Job前会判断是否有数据，如果没有的话就不再提交Job；
         * 4.此处没有Receiver,SparkStreaming应用程序只是按照时间间隔监控目录下每个Batch新增的内容（把新增的）作为RDD的数据来源生成原始RDD
         */
        JavaDStream<String> lines = jsc.textFileStream("hdfs://alamps:9000/library/SparkStreaming/data");//此处的是hdfs文件  

        /*
         * 第四步：接下来就像对于RDD编程一样基于DStream进行编程！！！原因是DStream是RDD产生的模板（或者说类），在Spark Streaming具体
         * 发生计算前，其实质是把每个Batch的DStream的操作翻译成为对RDD的操作！！！
         *对初始的DStream进行Transformation级别的处理，例如map、filter等高阶函数等的编程，来进行具体的数据计算
         *     第4.1步：将每一行的字符串拆分成单个的单词
         */
        JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() { //如果是Scala，由于SAM转换，所以可以写成val words = lines.flatMap { line => line.split(" ")}  

            @Override
            public Iterable<String> call(String line) throws Exception {
                return Arrays.asList(line.split(" "));
            }
        });  

         /*
           * 第四步：对初始的DStream进行Transformation级别的处理，例如map、filter等高阶函数等的编程，来进行具体的数据计算
           * 第4.2步：在单词拆分的基础上对每个单词实例计数为1，也就是word => (word, 1)
           */
        JavaPairDStream<String, Integer> pairs = words.mapToPair(new PairFunction<String, String, Integer>() {  

            /**
             *
             */
            private static final long serialVersionUID = 1L;  

            @Override
            public Tuple2<String, Integer> call(String word) throws Exception {
                return new Tuple2<String, Integer>(word, );
            }
        });  

         /*
           * 第四步：对初始的DStream进行Transformation级别的处理，例如map、filter等高阶函数等的编程，来进行具体的数据计算
           * 第4.3步：在每个单词实例计数为1基础之上统计每个单词在文件中出现的总次数
           */
        JavaPairDStream<String, Integer> wordsCount = pairs.reduceByKey(new Function2<Integer, Integer, Integer>() { //对相同的Key，进行Value的累计（包括Local和Reducer级别同时Reduce）  

            /**
             *
             */
            private static final long serialVersionUID = 1L;  

            public Integer call(Integer v1, Integer v2) throws Exception {
                return v1 + v2;
            }
        });  

        /*
         *    此处的print并不会直接出发Job的执行，因为现在的一切都是在Spark Streaming框架的控制之下的，对于Spark Streaming
         *    而言具体是否触发真正的Job运行是基于设置的Duration时间间隔的
         *
         *    诸位一定要注意的是Spark Streaming应用程序要想执行具体的Job，对Dtream就必须有output Stream操作，
         *    output Stream有很多类型的函数触发，类print、saveAsTextFile、saveAsHadoopFiles等，最为重要的一个
         *    方法是foraeachRDD,因为Spark Streaming处理的结果一般都会放在Redis、DB、DashBoard等上面，foreachRDD
         *    主要就是用来完成这些功能的，而且可以随意的自定义具体数据到底放在哪里！！！
         *
         */
        wordsCount.print();  

        /*
         * Spark Streaming执行引擎也就是Driver开始运行，Driver启动的时候是位于一条新的线程中的，当然其内部有消息循环体，用于
         * 接受应用程序本身或者Executor中的消息；
         */
        jsc.start();  

        jsc.awaitTermination();
        jsc.close();  

    }
    /**
     *
     * @Description:工厂模式创建JavaStreamingContext
     * @Author: lujinyong168
     * @Date: 2016年5月12日 下午10:01:40
     */
    private static JavaStreamingContext createContext(String checkpointDirectory,SparkConf sc){
        System.out.println("Creating new context");
        JavaStreamingContext ssc = new JavaStreamingContext(sc,Durations.seconds());
        ssc.checkpoint(checkpointDirectory);
        return ssc;
    }
}
3. 通过hdfs dfs -put命令上传文件到hdfs://master1:9000/library/SparkStreaming/data目录中，可以查看到隔10秒钟会处理一次。因为我设置的间隔时间是：Durations.seconds(10)

[root@alamps TestDir]# vi /usr/local/flume/tmp/TestDir/wc1.txt
hadoop hive hbase
spark stream flume
kafka spark sql
strom python[root@alamps TestDir]#

4.遇到的问题：

[root@alamps TestDir]# hadoop fs -chown hadoop:hadoop hdfs://alamps:9000/library/SparkStreaming/data/
[root@alamps TestDir]# hadoop fs -chown hadoop:hadoop hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data

//
//SLF4J: Class path contains multiple SLF4J bindings.
//SLF4J: Found binding in [jar:file:/home/hadoop/app/spark/lib/spark-assembly-1.2.0-hadoop2.4.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
//SLF4J: Found binding in [jar:file:/home/hadoop/app/spark/lib/spark-examples-1.2.0-hadoop2.4.0.jar!/org/slf4j/impl/StaticLoggerBinder.class]
//SLF4J: See http://www.slf4j.org/codes.html#multiple_bindings for an explanation.
//SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
//log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory).
//log4j:WARN Please initialize the log4j system properly.
//log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
//Creating new context
//Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
//18/01/12 23:40:44 INFO SecurityManager: Changing view acls to: hadoop
//18/01/12 23:40:44 INFO SecurityManager: Changing modify acls to: hadoop
//18/01/12 23:40:44 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(hadoop); users with modify permissions: Set(hadoop)
//18/01/12 23:40:44 INFO Slf4jLogger: Slf4jLogger started
//18/01/12 23:40:44 INFO Remoting: Starting remoting
//18/01/12 23:40:44 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@alamps:42898]
//18/01/12 23:40:44 INFO Utils: Successfully started service 'sparkDriver' on port 42898.
//18/01/12 23:40:44 INFO SparkEnv: Registering MapOutputTracker
//18/01/12 23:40:44 INFO SparkEnv: Registering BlockManagerMaster
//18/01/12 23:40:44 INFO DiskBlockManager: Created local directory at /tmp/spark-local-20180112234044-511d
//18/01/12 23:40:44 INFO MemoryStore: MemoryStore started with capacity 133.6 MB
//18/01/12 23:40:44 INFO HttpFileServer: HTTP File server directory is /tmp/spark-92d5a559-6100-48c0-8a1d-bcac165fe878
//18/01/12 23:40:44 INFO HttpServer: Starting HTTP Server
//18/01/12 23:40:44 INFO Utils: Successfully started service 'HTTP file server' on port 49358.
//18/01/12 23:40:44 INFO Utils: Successfully started service 'SparkUI' on port 4040.
//18/01/12 23:40:44 INFO SparkUI: Started SparkUI at http://alamps:4040
//18/01/12 23:40:45 INFO AkkaUtils: Connecting to HeartbeatReceiver: akka.tcp://sparkDriver@alamps:42898/user/HeartbeatReceiver
//18/01/12 23:40:45 INFO NettyBlockTransferService: Server created on 54231
//18/01/12 23:40:45 INFO BlockManagerMaster: Trying to register BlockManager
//18/01/12 23:40:45 INFO BlockManagerMasterActor: Registering block manager localhost:54231 with 133.6 MB RAM, BlockManagerId(<driver>, localhost, 54231)
//18/01/12 23:40:45 INFO BlockManagerMaster: Registered BlockManager
//18/01/12 23:40:45 INFO FileInputDStream: Duration for remembering RDDs set to 60000 ms for org.apache.spark.streaming.dstream.FileInputDStream@8ff85d
//18/01/12 23:40:45 INFO ForEachDStream: metadataCleanupDelay = -1
//18/01/12 23:40:45 INFO ShuffledDStream: metadataCleanupDelay = -1
//18/01/12 23:40:45 INFO MappedDStream: metadataCleanupDelay = -1
//18/01/12 23:40:45 INFO FlatMappedDStream: metadataCleanupDelay = -1
//18/01/12 23:40:45 INFO MappedDStream: metadataCleanupDelay = -1
//18/01/12 23:40:45 INFO FileInputDStream: metadataCleanupDelay = -1
//18/01/12 23:40:45 INFO FileInputDStream: Slide time = 10000 ms
//18/01/12 23:40:45 INFO FileInputDStream: Storage level = StorageLevel(false, false, false, false, 1)
//18/01/12 23:40:45 INFO FileInputDStream: Checkpoint interval = null
//18/01/12 23:40:45 INFO FileInputDStream: Remember duration = 60000 ms
//18/01/12 23:40:45 INFO FileInputDStream: Initialized and validated org.apache.spark.streaming.dstream.FileInputDStream@8ff85d
//18/01/12 23:40:45 INFO MappedDStream: Slide time = 10000 ms
//18/01/12 23:40:45 INFO MappedDStream: Storage level = StorageLevel(false, false, false, false, 1)
//18/01/12 23:40:45 INFO MappedDStream: Checkpoint interval = null
//18/01/12 23:40:45 INFO MappedDStream: Remember duration = 10000 ms
//18/01/12 23:40:45 INFO MappedDStream: Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@c3bcdf
//18/01/12 23:40:45 INFO FlatMappedDStream: Slide time = 10000 ms
//18/01/12 23:40:45 INFO FlatMappedDStream: Storage level = StorageLevel(false, false, false, false, 1)
//18/01/12 23:40:45 INFO FlatMappedDStream: Checkpoint interval = null
//18/01/12 23:40:45 INFO FlatMappedDStream: Remember duration = 10000 ms
//18/01/12 23:40:45 INFO FlatMappedDStream: Initialized and validated org.apache.spark.streaming.dstream.FlatMappedDStream@a4d9fe
//18/01/12 23:40:45 INFO MappedDStream: Slide time = 10000 ms
//18/01/12 23:40:45 INFO MappedDStream: Storage level = StorageLevel(false, false, false, false, 1)
//18/01/12 23:40:45 INFO MappedDStream: Checkpoint interval = null
//18/01/12 23:40:45 INFO MappedDStream: Remember duration = 10000 ms
//18/01/12 23:40:45 INFO MappedDStream: Initialized and validated org.apache.spark.streaming.dstream.MappedDStream@e3cfbf
//18/01/12 23:40:45 INFO ShuffledDStream: Slide time = 10000 ms
//18/01/12 23:40:45 INFO ShuffledDStream: Storage level = StorageLevel(false, false, false, false, 1)
//18/01/12 23:40:45 INFO ShuffledDStream: Checkpoint interval = null
//18/01/12 23:40:45 INFO ShuffledDStream: Remember duration = 10000 ms
//18/01/12 23:40:45 INFO ShuffledDStream: Initialized and validated org.apache.spark.streaming.dstream.ShuffledDStream@5c2d0f
//18/01/12 23:40:45 INFO ForEachDStream: Slide time = 10000 ms
//18/01/12 23:40:45 INFO ForEachDStream: Storage level = StorageLevel(false, false, false, false, 1)
//18/01/12 23:40:45 INFO ForEachDStream: Checkpoint interval = null
//18/01/12 23:40:45 INFO ForEachDStream: Remember duration = 10000 ms
//18/01/12 23:40:45 INFO ForEachDStream: Initialized and validated org.apache.spark.streaming.dstream.ForEachDStream@1925bfa
//18/01/12 23:40:46 INFO RecurringTimer: Started timer for JobGenerator at time 1515829250000
//18/01/12 23:40:46 INFO JobGenerator: Started JobGenerator at 1515829250000 ms
//18/01/12 23:40:46 INFO JobScheduler: Started JobScheduler
//18/01/12 23:40:50 INFO FileInputDStream: Finding new files took 5 ms
//18/01/12 23:40:50 INFO FileInputDStream: New files at time 1515829250000 ms:
//
//18/01/12 23:40:50 INFO JobScheduler: Added jobs for time 1515829250000 ms
//18/01/12 23:40:50 INFO JobGenerator: Checkpointing graph for time 1515829250000 ms
//18/01/12 23:40:50 INFO DStreamGraph: Updating checkpoint data for time 1515829250000 ms
//18/01/12 23:40:50 INFO JobScheduler: Starting job streaming job 1515829250000 ms.0 from job set of time 1515829250000 ms
//18/01/12 23:40:50 INFO DStreamGraph: Updated checkpoint data for time 1515829250000 ms
//18/01/12 23:40:50 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:40:50 INFO DAGScheduler: Registering RDD 3 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:40:50 INFO DAGScheduler: Got job 0 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:40:50 INFO DAGScheduler: Final stage: Stage 1(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:40:50 INFO DAGScheduler: Parents of final stage: List(Stage 0)
//18/01/12 23:40:50 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:40:50 INFO DAGScheduler: Submitting Stage 1 (ShuffledRDD[4] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:40:50 INFO CheckpointWriter: Saving checkpoint for time 1515829250000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829250000'
//18/01/12 23:40:50 WARN SizeEstimator: Failed to check whether UseCompressedOops is set; assuming yes
//18/01/12 23:40:50 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=0, maxMem=140142182
//18/01/12 23:40:50 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:40:50 INFO MemoryStore: ensureFreeSpace(1677) called with curMem=2264, maxMem=140142182
//18/01/12 23:40:50 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 1677.0 B, free 133.6 MB)
//18/01/12 23:40:50 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:54231 (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:40:50 INFO BlockManagerMaster: Updated info of block broadcast_0_piece0
//18/01/12 23:40:50 INFO SparkContext: Created broadcast 0 from broadcast at DAGScheduler.scala:838
//18/01/12 23:40:50 INFO DAGScheduler: Submitting 1 missing tasks from Stage 1 (ShuffledRDD[4] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:40:50 INFO TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
//18/01/12 23:40:50 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 0, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:40:50 INFO Executor: Running task 0.0 in stage 1.0 (TID 0)
//18/01/12 23:40:51 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:40:51 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 20 ms
//18/01/12 23:40:51 INFO Executor: Finished task 0.0 in stage 1.0 (TID 0). 820 bytes result sent to driver
//18/01/12 23:40:51 INFO DAGScheduler: Stage 1 (print at SparkStreamingOnHDFS.java:126) finished in 0.335 s
//18/01/12 23:40:51 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 0) in 308 ms on localhost (1/1)
//18/01/12 23:40:51 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
//18/01/12 23:40:51 INFO DAGScheduler: Job 0 finished: print at SparkStreamingOnHDFS.java:126, took 1.096348 s
//18/01/12 23:40:51 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:40:51 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 0 is 82 bytes
//18/01/12 23:40:51 INFO DAGScheduler: Got job 1 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:40:51 INFO DAGScheduler: Final stage: Stage 3(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:40:51 INFO DAGScheduler: Parents of final stage: List(Stage 2)
//18/01/12 23:40:51 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:40:51 INFO DAGScheduler: Submitting Stage 3 (ShuffledRDD[4] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:40:51 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=3941, maxMem=140142182
//18/01/12 23:40:51 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:40:51 INFO MemoryStore: ensureFreeSpace(1677) called with curMem=6205, maxMem=140142182
//18/01/12 23:40:51 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 1677.0 B, free 133.6 MB)
//18/01/12 23:40:51 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:54231 (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:40:51 INFO BlockManagerMaster: Updated info of block broadcast_1_piece0
//18/01/12 23:40:51 INFO SparkContext: Created broadcast 1 from getCallSite at DStream.scala:294
//18/01/12 23:40:51 INFO DAGScheduler: Submitting 1 missing tasks from Stage 3 (ShuffledRDD[4] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:40:51 INFO TaskSchedulerImpl: Adding task set 3.0 with 1 tasks
//18/01/12 23:40:51 INFO TaskSetManager: Starting task 0.0 in stage 3.0 (TID 1, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:40:51 INFO Executor: Running task 0.0 in stage 3.0 (TID 1)
//18/01/12 23:40:51 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:40:51 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
//18/01/12 23:40:51 INFO Executor: Finished task 0.0 in stage 3.0 (TID 1). 820 bytes result sent to driver
//18/01/12 23:40:51 INFO DAGScheduler: Stage 3 (print at SparkStreamingOnHDFS.java:126) finished in 0.023 s
//18/01/12 23:40:51 INFO DAGScheduler: Job 1 finished: print at SparkStreamingOnHDFS.java:126, took 0.077437 s
//18/01/12 23:40:51 INFO TaskSetManager: Finished task 0.0 in stage 3.0 (TID 1) in 21 ms on localhost (1/1)
//18/01/12 23:40:51 INFO TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool
//-------------------------------------------
//Time: 1515829250000 ms
//-------------------------------------------
//
//18/01/12 23:40:51 INFO JobScheduler: Finished job streaming job 1515829250000 ms.0 from job set of time 1515829250000 ms
//18/01/12 23:40:51 INFO JobScheduler: Total delay: 1.327 s for time 1515829250000 ms (execution: 1.240 s)
//18/01/12 23:40:51 INFO FileInputDStream: Cleared 0 old files that were older than 1515829190000 ms:
//18/01/12 23:40:51 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:40:51 INFO JobGenerator: Checkpointing graph for time 1515829250000 ms
//18/01/12 23:40:51 INFO DStreamGraph: Updating checkpoint data for time 1515829250000 ms
//18/01/12 23:40:51 INFO DStreamGraph: Updated checkpoint data for time 1515829250000 ms
//18/01/12 23:40:51 INFO CheckpointWriter: Checkpoint for time 1515829250000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829250000', took 5334 bytes and 1206 ms
//18/01/12 23:40:51 INFO DStreamGraph: Clearing checkpoint data for time 1515829250000 ms
//18/01/12 23:40:51 INFO CheckpointWriter: Saving checkpoint for time 1515829250000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829250000'
//18/01/12 23:40:51 INFO DStreamGraph: Cleared checkpoint data for time 1515829250000 ms
//18/01/12 23:40:51 INFO CheckpointWriter: Checkpoint for time 1515829250000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829250000', took 5330 bytes and 121 ms
//18/01/12 23:40:51 INFO DStreamGraph: Clearing checkpoint data for time 1515829250000 ms
//18/01/12 23:40:51 INFO DStreamGraph: Cleared checkpoint data for time 1515829250000 ms
//18/01/12 23:41:00 INFO FileInputDStream: Finding new files took 1 ms
//18/01/12 23:41:00 INFO FileInputDStream: New files at time 1515829260000 ms:
//
//18/01/12 23:41:00 INFO JobScheduler: Starting job streaming job 1515829260000 ms.0 from job set of time 1515829260000 ms
//18/01/12 23:41:00 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:00 INFO DAGScheduler: Registering RDD 8 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:00 INFO DAGScheduler: Got job 2 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:00 INFO DAGScheduler: Final stage: Stage 5(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:00 INFO DAGScheduler: Parents of final stage: List(Stage 4)
//18/01/12 23:41:00 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:00 INFO DAGScheduler: Submitting Stage 5 (ShuffledRDD[9] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:00 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=7882, maxMem=140142182
//18/01/12 23:41:00 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:00 INFO JobScheduler: Added jobs for time 1515829260000 ms
//18/01/12 23:41:00 INFO JobGenerator: Checkpointing graph for time 1515829260000 ms
//18/01/12 23:41:00 INFO DStreamGraph: Updating checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO DStreamGraph: Updated checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=10146, maxMem=140142182
//18/01/12 23:41:00 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:00 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:00 INFO BlockManagerMaster: Updated info of block broadcast_2_piece0
//18/01/12 23:41:00 INFO SparkContext: Created broadcast 2 from getCallSite at DStream.scala:294
//18/01/12 23:41:00 INFO DAGScheduler: Submitting 1 missing tasks from Stage 5 (ShuffledRDD[9] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:00 INFO TaskSchedulerImpl: Adding task set 5.0 with 1 tasks
//18/01/12 23:41:00 INFO CheckpointWriter: Saving checkpoint for time 1515829260000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829260000'
//18/01/12 23:41:00 INFO TaskSetManager: Starting task 0.0 in stage 5.0 (TID 2, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:00 INFO Executor: Running task 0.0 in stage 5.0 (TID 2)
//18/01/12 23:41:00 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:00 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
//18/01/12 23:41:00 INFO Executor: Finished task 0.0 in stage 5.0 (TID 2). 820 bytes result sent to driver
//18/01/12 23:41:00 INFO DAGScheduler: Stage 5 (print at SparkStreamingOnHDFS.java:126) finished in 0.014 s
//18/01/12 23:41:00 INFO DAGScheduler: Job 2 finished: print at SparkStreamingOnHDFS.java:126, took 0.059484 s
//18/01/12 23:41:00 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:00 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 1 is 82 bytes
//18/01/12 23:41:00 INFO DAGScheduler: Got job 3 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:00 INFO DAGScheduler: Final stage: Stage 7(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:00 INFO DAGScheduler: Parents of final stage: List(Stage 6)
//18/01/12 23:41:00 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:00 INFO DAGScheduler: Submitting Stage 7 (ShuffledRDD[9] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:00 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=11825, maxMem=140142182
//18/01/12 23:41:00 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:00 INFO TaskSetManager: Finished task 0.0 in stage 5.0 (TID 2) in 14 ms on localhost (1/1)
//18/01/12 23:41:00 INFO TaskSchedulerImpl: Removed TaskSet 5.0, whose tasks have all completed, from pool
//18/01/12 23:41:00 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=14089, maxMem=140142182
//18/01/12 23:41:00 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:00 INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:00 INFO BlockManagerMaster: Updated info of block broadcast_3_piece0
//18/01/12 23:41:00 INFO SparkContext: Created broadcast 3 from getCallSite at DStream.scala:294
//18/01/12 23:41:00 INFO DAGScheduler: Submitting 1 missing tasks from Stage 7 (ShuffledRDD[9] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:00 INFO TaskSchedulerImpl: Adding task set 7.0 with 1 tasks
//18/01/12 23:41:00 INFO TaskSetManager: Starting task 0.0 in stage 7.0 (TID 3, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:00 INFO Executor: Running task 0.0 in stage 7.0 (TID 3)
//18/01/12 23:41:00 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:00 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
//18/01/12 23:41:00 INFO Executor: Finished task 0.0 in stage 7.0 (TID 3). 820 bytes result sent to driver
//18/01/12 23:41:00 INFO DAGScheduler: Stage 7 (print at SparkStreamingOnHDFS.java:126) finished in 0.014 s
//-------------------------------------------
//Time: 1515829260000 ms
//-------------------------------------------
//
//18/01/12 23:41:00 INFO DAGScheduler: Job 3 finished: print at SparkStreamingOnHDFS.java:126, took 0.078915 s
//18/01/12 23:41:00 INFO JobScheduler: Finished job streaming job 1515829260000 ms.0 from job set of time 1515829260000 ms
//18/01/12 23:41:00 INFO JobScheduler: Total delay: 0.160 s for time 1515829260000 ms (execution: 0.149 s)
//18/01/12 23:41:00 INFO ShuffledRDD: Removing RDD 4 from persistence list
//18/01/12 23:41:00 INFO BlockManager: Removing RDD 4
//18/01/12 23:41:00 INFO TaskSetManager: Finished task 0.0 in stage 7.0 (TID 3) in 26 ms on localhost (1/1)
//18/01/12 23:41:00 INFO TaskSchedulerImpl: Removed TaskSet 7.0, whose tasks have all completed, from pool
//18/01/12 23:41:00 INFO MappedRDD: Removing RDD 3 from persistence list
//18/01/12 23:41:00 INFO FlatMappedRDD: Removing RDD 2 from persistence list
//18/01/12 23:41:00 INFO MappedRDD: Removing RDD 1 from persistence list
//18/01/12 23:41:00 INFO FileInputDStream: Cleared 0 old files that were older than 1515829200000 ms:
//18/01/12 23:41:00 INFO BlockManager: Removing RDD 3
//18/01/12 23:41:00 INFO BlockManager: Removing RDD 2
//18/01/12 23:41:00 INFO BlockManager: Removing RDD 1
//18/01/12 23:41:00 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:41:00 INFO JobGenerator: Checkpointing graph for time 1515829260000 ms
//18/01/12 23:41:00 INFO DStreamGraph: Updating checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO DStreamGraph: Updated checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO CheckpointWriter: Checkpoint for time 1515829260000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829260000', took 5345 bytes and 234 ms
//18/01/12 23:41:00 INFO DStreamGraph: Clearing checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO DStreamGraph: Cleared checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO CheckpointWriter: Saving checkpoint for time 1515829260000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829260000'
//18/01/12 23:41:00 INFO CheckpointWriter: Checkpoint for time 1515829260000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829260000', took 5341 bytes and 67 ms
//18/01/12 23:41:00 INFO DStreamGraph: Clearing checkpoint data for time 1515829260000 ms
//18/01/12 23:41:00 INFO DStreamGraph: Cleared checkpoint data for time 1515829260000 ms
//18/01/12 23:41:10 INFO FileInputDStream: Finding new files took 3 ms
//18/01/12 23:41:10 INFO FileInputDStream: New files at time 1515829270000 ms:
//
//18/01/12 23:41:10 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:10 INFO JobScheduler: Starting job streaming job 1515829270000 ms.0 from job set of time 1515829270000 ms
//18/01/12 23:41:10 INFO DAGScheduler: Registering RDD 13 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:10 INFO DAGScheduler: Got job 4 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:10 INFO DAGScheduler: Final stage: Stage 9(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:10 INFO DAGScheduler: Parents of final stage: List(Stage 8)
//18/01/12 23:41:10 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:10 INFO DAGScheduler: Submitting Stage 9 (ShuffledRDD[14] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:10 INFO JobScheduler: Added jobs for time 1515829270000 ms
//18/01/12 23:41:10 INFO JobGenerator: Checkpointing graph for time 1515829270000 ms
//18/01/12 23:41:10 INFO DStreamGraph: Updating checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO DStreamGraph: Updated checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=15768, maxMem=140142182
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:10 INFO BlockManager: Removing broadcast 3
//18/01/12 23:41:10 INFO BlockManager: Removing block broadcast_3_piece0
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_3_piece0 of size 1679 dropped from memory (free 140125829)
//18/01/12 23:41:10 INFO BlockManagerInfo: Removed broadcast_3_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:10 INFO BlockManagerMaster: Updated info of block broadcast_3_piece0
//18/01/12 23:41:10 INFO BlockManager: Removing block broadcast_3
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_3 of size 2264 dropped from memory (free 140128093)
//18/01/12 23:41:10 INFO MemoryStore: ensureFreeSpace(1677) called with curMem=14089, maxMem=140142182
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 1677.0 B, free 133.6 MB)
//18/01/12 23:41:10 INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:54231 (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:41:10 INFO BlockManagerMaster: Updated info of block broadcast_4_piece0
//18/01/12 23:41:10 INFO SparkContext: Created broadcast 4 from getCallSite at DStream.scala:294
//18/01/12 23:41:10 INFO DAGScheduler: Submitting 1 missing tasks from Stage 9 (ShuffledRDD[14] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:10 INFO TaskSchedulerImpl: Adding task set 9.0 with 1 tasks
//18/01/12 23:41:10 INFO TaskSetManager: Starting task 0.0 in stage 9.0 (TID 4, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:10 INFO Executor: Running task 0.0 in stage 9.0 (TID 4)
//18/01/12 23:41:10 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:10 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
//18/01/12 23:41:10 INFO Executor: Finished task 0.0 in stage 9.0 (TID 4). 820 bytes result sent to driver
//18/01/12 23:41:10 INFO ContextCleaner: Cleaned broadcast 3
//18/01/12 23:41:10 INFO CheckpointWriter: Saving checkpoint for time 1515829270000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829270000'
//18/01/12 23:41:10 INFO DAGScheduler: Stage 9 (print at SparkStreamingOnHDFS.java:126) finished in 0.005 s
//18/01/12 23:41:10 INFO DAGScheduler: Job 4 finished: print at SparkStreamingOnHDFS.java:126, took 0.082960 s
//18/01/12 23:41:10 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:10 INFO TaskSetManager: Finished task 0.0 in stage 9.0 (TID 4) in 11 ms on localhost (1/1)
//18/01/12 23:41:10 INFO TaskSchedulerImpl: Removed TaskSet 9.0, whose tasks have all completed, from pool
//18/01/12 23:41:10 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 2 is 82 bytes
//18/01/12 23:41:10 INFO DAGScheduler: Got job 5 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:10 INFO DAGScheduler: Final stage: Stage 11(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:10 INFO DAGScheduler: Parents of final stage: List(Stage 10)
//18/01/12 23:41:10 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:10 INFO DAGScheduler: Submitting Stage 11 (ShuffledRDD[14] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:10 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=15766, maxMem=140142182
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_5 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:10 INFO MemoryStore: ensureFreeSpace(1677) called with curMem=18030, maxMem=140142182
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 1677.0 B, free 133.6 MB)
//18/01/12 23:41:10 INFO BlockManagerInfo: Added broadcast_5_piece0 in memory on localhost:54231 (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:41:10 INFO BlockManagerMaster: Updated info of block broadcast_5_piece0
//18/01/12 23:41:10 INFO SparkContext: Created broadcast 5 from getCallSite at DStream.scala:294
//18/01/12 23:41:10 INFO DAGScheduler: Submitting 1 missing tasks from Stage 11 (ShuffledRDD[14] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:10 INFO TaskSchedulerImpl: Adding task set 11.0 with 1 tasks
//18/01/12 23:41:10 INFO TaskSetManager: Starting task 0.0 in stage 11.0 (TID 5, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:10 INFO Executor: Running task 0.0 in stage 11.0 (TID 5)
//18/01/12 23:41:10 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:10 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
//18/01/12 23:41:10 INFO Executor: Finished task 0.0 in stage 11.0 (TID 5). 820 bytes result sent to driver
//18/01/12 23:41:10 INFO TaskSetManager: Finished task 0.0 in stage 11.0 (TID 5) in 8 ms on localhost (1/1)
//18/01/12 23:41:10 INFO TaskSchedulerImpl: Removed TaskSet 11.0, whose tasks have all completed, from pool
//18/01/12 23:41:10 INFO DAGScheduler: Stage 11 (print at SparkStreamingOnHDFS.java:126) finished in 0.003 s
//-------------------------------------------
//Time: 1515829270000 ms
//-------------------------------------------
//
//18/01/12 23:41:10 INFO DAGScheduler: Job 5 finished: print at SparkStreamingOnHDFS.java:126, took 0.135786 s
//18/01/12 23:41:10 INFO JobScheduler: Finished job streaming job 1515829270000 ms.0 from job set of time 1515829270000 ms
//18/01/12 23:41:10 INFO JobScheduler: Total delay: 0.235 s for time 1515829270000 ms (execution: 0.221 s)
//18/01/12 23:41:10 INFO ShuffledRDD: Removing RDD 9 from persistence list
//18/01/12 23:41:10 INFO BlockManager: Removing RDD 9
//18/01/12 23:41:10 INFO MappedRDD: Removing RDD 8 from persistence list
//18/01/12 23:41:10 INFO BlockManager: Removing RDD 8
//18/01/12 23:41:10 INFO FlatMappedRDD: Removing RDD 7 from persistence list
//18/01/12 23:41:10 INFO BlockManager: Removing RDD 7
//18/01/12 23:41:10 INFO BlockManager: Removing broadcast 5
//18/01/12 23:41:10 INFO BlockManager: Removing block broadcast_5_piece0
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_5_piece0 of size 1677 dropped from memory (free 140124152)
//18/01/12 23:41:10 INFO BlockManagerInfo: Removed broadcast_5_piece0 on localhost:54231 in memory (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:41:10 INFO BlockManagerMaster: Updated info of block broadcast_5_piece0
//18/01/12 23:41:10 INFO BlockManager: Removing block broadcast_5
//18/01/12 23:41:10 INFO MemoryStore: Block broadcast_5 of size 2264 dropped from memory (free 140126416)
//18/01/12 23:41:10 INFO MappedRDD: Removing RDD 6 from persistence list
//18/01/12 23:41:10 INFO BlockManager: Removing RDD 6
//18/01/12 23:41:10 INFO ContextCleaner: Cleaned broadcast 5
//18/01/12 23:41:10 INFO FileInputDStream: Cleared 0 old files that were older than 1515829210000 ms:
//18/01/12 23:41:10 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:41:10 INFO JobGenerator: Checkpointing graph for time 1515829270000 ms
//18/01/12 23:41:10 INFO DStreamGraph: Updating checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO DStreamGraph: Updated checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO CheckpointWriter: Checkpoint for time 1515829270000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829270000', took 5352 bytes and 280 ms
//18/01/12 23:41:10 INFO DStreamGraph: Clearing checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO DStreamGraph: Cleared checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO CheckpointWriter: Saving checkpoint for time 1515829270000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829270000'
//18/01/12 23:41:10 INFO CheckpointWriter: Checkpoint for time 1515829270000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829270000', took 5348 bytes and 69 ms
//18/01/12 23:41:10 INFO DStreamGraph: Clearing checkpoint data for time 1515829270000 ms
//18/01/12 23:41:10 INFO DStreamGraph: Cleared checkpoint data for time 1515829270000 ms
//18/01/12 23:41:20 INFO FileInputDStream: Finding new files took 2 ms
//18/01/12 23:41:20 INFO FileInputDStream: New files at time 1515829280000 ms:
//
//18/01/12 23:41:20 INFO JobScheduler: Starting job streaming job 1515829280000 ms.0 from job set of time 1515829280000 ms
//18/01/12 23:41:20 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:20 INFO DAGScheduler: Registering RDD 18 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:20 INFO DAGScheduler: Got job 6 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:20 INFO DAGScheduler: Final stage: Stage 13(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:20 INFO DAGScheduler: Parents of final stage: List(Stage 12)
//18/01/12 23:41:20 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:20 INFO JobScheduler: Added jobs for time 1515829280000 ms
//18/01/12 23:41:20 INFO JobGenerator: Checkpointing graph for time 1515829280000 ms
//18/01/12 23:41:20 INFO DStreamGraph: Updating checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO DAGScheduler: Submitting Stage 13 (ShuffledRDD[19] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:20 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=15766, maxMem=140142182
//18/01/12 23:41:20 INFO DStreamGraph: Updated checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO MemoryStore: Block broadcast_6 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:20 INFO CheckpointWriter: Saving checkpoint for time 1515829280000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829280000'
//18/01/12 23:41:20 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=18030, maxMem=140142182
//18/01/12 23:41:20 INFO MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:20 INFO BlockManagerInfo: Added broadcast_6_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:20 INFO BlockManagerMaster: Updated info of block broadcast_6_piece0
//18/01/12 23:41:20 INFO SparkContext: Created broadcast 6 from getCallSite at DStream.scala:294
//18/01/12 23:41:20 INFO DAGScheduler: Submitting 1 missing tasks from Stage 13 (ShuffledRDD[19] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:20 INFO TaskSchedulerImpl: Adding task set 13.0 with 1 tasks
//18/01/12 23:41:20 INFO TaskSetManager: Starting task 0.0 in stage 13.0 (TID 6, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:20 INFO Executor: Running task 0.0 in stage 13.0 (TID 6)
//18/01/12 23:41:20 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:20 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
//18/01/12 23:41:20 INFO Executor: Finished task 0.0 in stage 13.0 (TID 6). 820 bytes result sent to driver
//18/01/12 23:41:20 INFO TaskSetManager: Finished task 0.0 in stage 13.0 (TID 6) in 6 ms on localhost (1/1)
//18/01/12 23:41:20 INFO TaskSchedulerImpl: Removed TaskSet 13.0, whose tasks have all completed, from pool
//18/01/12 23:41:20 INFO DAGScheduler: Stage 13 (print at SparkStreamingOnHDFS.java:126) finished in 0.002 s
//18/01/12 23:41:20 INFO DAGScheduler: Job 6 finished: print at SparkStreamingOnHDFS.java:126, took 0.040562 s
//18/01/12 23:41:20 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:20 INFO BlockManager: Removing broadcast 6
//18/01/12 23:41:20 INFO BlockManager: Removing block broadcast_6_piece0
//18/01/12 23:41:20 INFO MemoryStore: Block broadcast_6_piece0 of size 1679 dropped from memory (free 140124152)
//18/01/12 23:41:20 INFO BlockManagerInfo: Removed broadcast_6_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:20 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 3 is 82 bytes
//18/01/12 23:41:20 INFO DAGScheduler: Got job 7 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:20 INFO DAGScheduler: Final stage: Stage 15(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:20 INFO DAGScheduler: Parents of final stage: List(Stage 14)
//18/01/12 23:41:20 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:20 INFO DAGScheduler: Submitting Stage 15 (ShuffledRDD[19] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:20 INFO BlockManagerMaster: Updated info of block broadcast_6_piece0
//18/01/12 23:41:20 INFO BlockManager: Removing block broadcast_6
//18/01/12 23:41:20 INFO MemoryStore: Block broadcast_6 of size 2264 dropped from memory (free 140126416)
//18/01/12 23:41:20 INFO ContextCleaner: Cleaned broadcast 6
//18/01/12 23:41:20 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=15766, maxMem=140142182
//18/01/12 23:41:20 INFO MemoryStore: Block broadcast_7 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:20 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=18030, maxMem=140142182
//18/01/12 23:41:20 INFO MemoryStore: Block broadcast_7_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:20 INFO BlockManagerInfo: Added broadcast_7_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:20 INFO BlockManagerMaster: Updated info of block broadcast_7_piece0
//18/01/12 23:41:20 INFO SparkContext: Created broadcast 7 from getCallSite at DStream.scala:294
//18/01/12 23:41:20 INFO DAGScheduler: Submitting 1 missing tasks from Stage 15 (ShuffledRDD[19] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:20 INFO TaskSchedulerImpl: Adding task set 15.0 with 1 tasks
//18/01/12 23:41:20 INFO TaskSetManager: Starting task 0.0 in stage 15.0 (TID 7, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:20 INFO Executor: Running task 0.0 in stage 15.0 (TID 7)
//18/01/12 23:41:20 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:20 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
//18/01/12 23:41:20 INFO Executor: Finished task 0.0 in stage 15.0 (TID 7). 820 bytes result sent to driver
//18/01/12 23:41:20 INFO TaskSetManager: Finished task 0.0 in stage 15.0 (TID 7) in 7 ms on localhost (1/1)
//18/01/12 23:41:20 INFO TaskSchedulerImpl: Removed TaskSet 15.0, whose tasks have all completed, from pool
//18/01/12 23:41:20 INFO DAGScheduler: Stage 15 (print at SparkStreamingOnHDFS.java:126) finished in 0.001 s
//18/01/12 23:41:20 INFO DAGScheduler: Job 7 finished: print at SparkStreamingOnHDFS.java:126, took 0.084666 s
//-------------------------------------------
//Time: 1515829280000 ms
//-------------------------------------------
//
//18/01/12 23:41:20 INFO JobScheduler: Finished job streaming job 1515829280000 ms.0 from job set of time 1515829280000 ms
//18/01/12 23:41:20 INFO JobScheduler: Total delay: 0.141 s for time 1515829280000 ms (execution: 0.129 s)
//18/01/12 23:41:20 INFO ShuffledRDD: Removing RDD 14 from persistence list
//18/01/12 23:41:20 INFO BlockManager: Removing RDD 14
//18/01/12 23:41:20 INFO MappedRDD: Removing RDD 13 from persistence list
//18/01/12 23:41:20 INFO BlockManager: Removing RDD 13
//18/01/12 23:41:20 INFO FlatMappedRDD: Removing RDD 12 from persistence list
//18/01/12 23:41:20 INFO BlockManager: Removing RDD 12
//18/01/12 23:41:20 INFO MappedRDD: Removing RDD 11 from persistence list
//18/01/12 23:41:20 INFO BlockManager: Removing RDD 11
//18/01/12 23:41:20 INFO FileInputDStream: Cleared 0 old files that were older than 1515829220000 ms:
//18/01/12 23:41:20 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:41:20 INFO JobGenerator: Checkpointing graph for time 1515829280000 ms
//18/01/12 23:41:20 INFO DStreamGraph: Updating checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO DStreamGraph: Updated checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO CheckpointWriter: Checkpoint for time 1515829280000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829280000', took 5363 bytes and 121 ms
//18/01/12 23:41:20 INFO DStreamGraph: Clearing checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO DStreamGraph: Cleared checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO CheckpointWriter: Saving checkpoint for time 1515829280000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829280000'
//18/01/12 23:41:20 INFO CheckpointWriter: Checkpoint for time 1515829280000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829280000', took 5359 bytes and 80 ms
//18/01/12 23:41:20 INFO DStreamGraph: Clearing checkpoint data for time 1515829280000 ms
//18/01/12 23:41:20 INFO DStreamGraph: Cleared checkpoint data for time 1515829280000 ms
//18/01/12 23:41:30 INFO FileInputDStream: Finding new files took 2 ms
//18/01/12 23:41:30 INFO FileInputDStream: New files at time 1515829290000 ms:
//
//18/01/12 23:41:30 INFO JobScheduler: Starting job streaming job 1515829290000 ms.0 from job set of time 1515829290000 ms
//18/01/12 23:41:30 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:30 INFO JobScheduler: Added jobs for time 1515829290000 ms
//18/01/12 23:41:30 INFO JobGenerator: Checkpointing graph for time 1515829290000 ms
//18/01/12 23:41:30 INFO DAGScheduler: Registering RDD 23 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:30 INFO DAGScheduler: Got job 8 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:30 INFO DAGScheduler: Final stage: Stage 17(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:30 INFO DAGScheduler: Parents of final stage: List(Stage 16)
//18/01/12 23:41:30 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:30 INFO DStreamGraph: Updating checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO DAGScheduler: Submitting Stage 17 (ShuffledRDD[24] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:30 INFO DStreamGraph: Updated checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=19709, maxMem=140142182
//18/01/12 23:41:30 INFO CheckpointWriter: Saving checkpoint for time 1515829290000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829290000'
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_8 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:30 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=21973, maxMem=140142182
//18/01/12 23:41:30 INFO BlockManager: Removing broadcast 7
//18/01/12 23:41:30 INFO BlockManager: Removing block broadcast_7
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_7 of size 2264 dropped from memory (free 140122473)
//18/01/12 23:41:30 INFO BlockManager: Removing block broadcast_7_piece0
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_7_piece0 of size 1679 dropped from memory (free 140124152)
//18/01/12 23:41:30 INFO BlockManagerInfo: Removed broadcast_7_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:30 INFO BlockManagerMaster: Updated info of block broadcast_7_piece0
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_8_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:30 INFO BlockManagerInfo: Added broadcast_8_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:30 INFO BlockManagerMaster: Updated info of block broadcast_8_piece0
//18/01/12 23:41:30 INFO SparkContext: Created broadcast 8 from broadcast at DAGScheduler.scala:838
//18/01/12 23:41:30 INFO ContextCleaner: Cleaned broadcast 7
//18/01/12 23:41:30 INFO DAGScheduler: Submitting 1 missing tasks from Stage 17 (ShuffledRDD[24] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:30 INFO TaskSchedulerImpl: Adding task set 17.0 with 1 tasks
//18/01/12 23:41:30 INFO TaskSetManager: Starting task 0.0 in stage 17.0 (TID 8, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:30 INFO Executor: Running task 0.0 in stage 17.0 (TID 8)
//18/01/12 23:41:30 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 0 ms
//18/01/12 23:41:30 INFO Executor: Finished task 0.0 in stage 17.0 (TID 8). 820 bytes result sent to driver
//18/01/12 23:41:30 INFO TaskSetManager: Finished task 0.0 in stage 17.0 (TID 8) in 6 ms on localhost (1/1)
//18/01/12 23:41:30 INFO TaskSchedulerImpl: Removed TaskSet 17.0, whose tasks have all completed, from pool
//18/01/12 23:41:30 INFO DAGScheduler: Stage 17 (print at SparkStreamingOnHDFS.java:126) finished in 0.007 s
//18/01/12 23:41:30 INFO DAGScheduler: Job 8 finished: print at SparkStreamingOnHDFS.java:126, took 0.047397 s
//18/01/12 23:41:30 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:30 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 4 is 82 bytes
//18/01/12 23:41:30 INFO DAGScheduler: Got job 9 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:30 INFO DAGScheduler: Final stage: Stage 19(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:30 INFO DAGScheduler: Parents of final stage: List(Stage 18)
//18/01/12 23:41:30 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:30 INFO DAGScheduler: Submitting Stage 19 (ShuffledRDD[24] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:30 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=19709, maxMem=140142182
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_9 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:30 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=21973, maxMem=140142182
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_9_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:30 INFO BlockManagerInfo: Added broadcast_9_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:30 INFO BlockManagerMaster: Updated info of block broadcast_9_piece0
//18/01/12 23:41:30 INFO SparkContext: Created broadcast 9 from getCallSite at DStream.scala:294
//18/01/12 23:41:30 INFO DAGScheduler: Submitting 1 missing tasks from Stage 19 (ShuffledRDD[24] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:30 INFO TaskSchedulerImpl: Adding task set 19.0 with 1 tasks
//18/01/12 23:41:30 INFO TaskSetManager: Starting task 0.0 in stage 19.0 (TID 9, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:30 INFO Executor: Running task 0.0 in stage 19.0 (TID 9)
//18/01/12 23:41:30 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:30 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 3 ms
//18/01/12 23:41:30 INFO BlockManager: Removing broadcast 8
//18/01/12 23:41:30 INFO BlockManager: Removing block broadcast_8
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_8 of size 2264 dropped from memory (free 140120794)
//18/01/12 23:41:30 INFO BlockManager: Removing block broadcast_8_piece0
//18/01/12 23:41:30 INFO MemoryStore: Block broadcast_8_piece0 of size 1679 dropped from memory (free 140122473)
//18/01/12 23:41:30 INFO BlockManagerInfo: Removed broadcast_8_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:30 INFO BlockManagerMaster: Updated info of block broadcast_8_piece0
//18/01/12 23:41:30 INFO ContextCleaner: Cleaned broadcast 8
//18/01/12 23:41:30 INFO Executor: Finished task 0.0 in stage 19.0 (TID 9). 820 bytes result sent to driver
//18/01/12 23:41:30 INFO TaskSetManager: Finished task 0.0 in stage 19.0 (TID 9) in 9 ms on localhost (1/1)
//18/01/12 23:41:30 INFO TaskSchedulerImpl: Removed TaskSet 19.0, whose tasks have all completed, from pool
//18/01/12 23:41:30 INFO DAGScheduler: Stage 19 (print at SparkStreamingOnHDFS.java:126) finished in 0.000 s
//-------------------------------------------
//Time: 1515829290000 ms
//-------------------------------------------
//
//18/01/12 23:41:30 INFO DAGScheduler: Job 9 finished: print at SparkStreamingOnHDFS.java:126, took 0.044747 s
//18/01/12 23:41:30 INFO JobScheduler: Finished job streaming job 1515829290000 ms.0 from job set of time 1515829290000 ms
//18/01/12 23:41:30 INFO ShuffledRDD: Removing RDD 19 from persistence list
//18/01/12 23:41:30 INFO BlockManager: Removing RDD 19
//18/01/12 23:41:30 INFO JobScheduler: Total delay: 0.114 s for time 1515829290000 ms (execution: 0.098 s)
//18/01/12 23:41:30 INFO MappedRDD: Removing RDD 18 from persistence list
//18/01/12 23:41:30 INFO BlockManager: Removing RDD 18
//18/01/12 23:41:30 INFO FlatMappedRDD: Removing RDD 17 from persistence list
//18/01/12 23:41:30 INFO BlockManager: Removing RDD 17
//18/01/12 23:41:30 INFO MappedRDD: Removing RDD 16 from persistence list
//18/01/12 23:41:30 INFO BlockManager: Removing RDD 16
//18/01/12 23:41:30 INFO FileInputDStream: Cleared 0 old files that were older than 1515829230000 ms:
//18/01/12 23:41:30 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:41:30 INFO JobGenerator: Checkpointing graph for time 1515829290000 ms
//18/01/12 23:41:30 INFO DStreamGraph: Updating checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO DStreamGraph: Updated checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO CheckpointWriter: Checkpoint for time 1515829290000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829290000', took 5372 bytes and 503 ms
//18/01/12 23:41:30 INFO DStreamGraph: Clearing checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO DStreamGraph: Cleared checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO CheckpointWriter: Saving checkpoint for time 1515829290000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829290000'
//18/01/12 23:41:30 INFO CheckpointWriter: Checkpoint for time 1515829290000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829290000', took 5368 bytes and 115 ms
//18/01/12 23:41:30 INFO DStreamGraph: Clearing checkpoint data for time 1515829290000 ms
//18/01/12 23:41:30 INFO DStreamGraph: Cleared checkpoint data for time 1515829290000 ms
//18/01/12 23:41:40 INFO FileInputDStream: Finding new files took 2 ms
//18/01/12 23:41:40 INFO FileInputDStream: New files at time 1515829300000 ms:
//
//18/01/12 23:41:40 INFO JobScheduler: Starting job streaming job 1515829300000 ms.0 from job set of time 1515829300000 ms
//18/01/12 23:41:40 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:40 INFO DAGScheduler: Registering RDD 28 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:40 INFO DAGScheduler: Got job 10 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:40 INFO DAGScheduler: Final stage: Stage 21(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:40 INFO DAGScheduler: Parents of final stage: List(Stage 20)
//18/01/12 23:41:40 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:40 INFO JobScheduler: Added jobs for time 1515829300000 ms
//18/01/12 23:41:40 INFO JobGenerator: Checkpointing graph for time 1515829300000 ms
//18/01/12 23:41:40 INFO DStreamGraph: Updating checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO DStreamGraph: Updated checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO DAGScheduler: Submitting Stage 21 (ShuffledRDD[29] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:40 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=19709, maxMem=140142182
//18/01/12 23:41:40 INFO CheckpointWriter: Saving checkpoint for time 1515829300000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829300000'
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_10 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:40 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=21973, maxMem=140142182
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_10_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:40 INFO BlockManagerInfo: Added broadcast_10_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:40 INFO ContextCleaner: Cleaned shuffle 3
//18/01/12 23:41:40 INFO BlockManager: Removing broadcast 9
//18/01/12 23:41:40 INFO BlockManager: Removing block broadcast_9
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_9 of size 2264 dropped from memory (free 140120794)
//18/01/12 23:41:40 INFO BlockManager: Removing block broadcast_9_piece0
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_9_piece0 of size 1679 dropped from memory (free 140122473)
//18/01/12 23:41:40 INFO BlockManagerInfo: Removed broadcast_9_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:40 INFO BlockManagerMaster: Updated info of block broadcast_9_piece0
//18/01/12 23:41:40 INFO BlockManagerMaster: Updated info of block broadcast_10_piece0
//18/01/12 23:41:40 INFO SparkContext: Created broadcast 10 from broadcast at DAGScheduler.scala:838
//18/01/12 23:41:40 INFO DAGScheduler: Submitting 1 missing tasks from Stage 21 (ShuffledRDD[29] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:40 INFO TaskSchedulerImpl: Adding task set 21.0 with 1 tasks
//18/01/12 23:41:40 INFO TaskSetManager: Starting task 0.0 in stage 21.0 (TID 10, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:40 INFO Executor: Running task 0.0 in stage 21.0 (TID 10)
//18/01/12 23:41:40 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
//18/01/12 23:41:40 INFO Executor: Finished task 0.0 in stage 21.0 (TID 10). 820 bytes result sent to driver
//18/01/12 23:41:40 INFO ContextCleaner: Cleaned broadcast 9
//18/01/12 23:41:40 INFO TaskSetManager: Finished task 0.0 in stage 21.0 (TID 10) in 12 ms on localhost (1/1)
//18/01/12 23:41:40 INFO TaskSchedulerImpl: Removed TaskSet 21.0, whose tasks have all completed, from pool
//18/01/12 23:41:40 INFO DAGScheduler: Stage 21 (print at SparkStreamingOnHDFS.java:126) finished in 0.008 s
//18/01/12 23:41:40 INFO DAGScheduler: Job 10 finished: print at SparkStreamingOnHDFS.java:126, took 0.053828 s
//18/01/12 23:41:40 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:40 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 5 is 82 bytes
//18/01/12 23:41:40 INFO DAGScheduler: Got job 11 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:40 INFO DAGScheduler: Final stage: Stage 23(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:40 INFO DAGScheduler: Parents of final stage: List(Stage 22)
//18/01/12 23:41:40 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:40 INFO DAGScheduler: Submitting Stage 23 (ShuffledRDD[29] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:40 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=19709, maxMem=140142182
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_11 stored as values in memory (estimated size 2.2 KB, free 133.6 MB)
//18/01/12 23:41:40 INFO MemoryStore: ensureFreeSpace(1679) called with curMem=21973, maxMem=140142182
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_11_piece0 stored as bytes in memory (estimated size 1679.0 B, free 133.6 MB)
//18/01/12 23:41:40 INFO BlockManagerInfo: Added broadcast_11_piece0 in memory on localhost:54231 (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:40 INFO BlockManagerMaster: Updated info of block broadcast_11_piece0
//18/01/12 23:41:40 INFO SparkContext: Created broadcast 11 from getCallSite at DStream.scala:294
//18/01/12 23:41:40 INFO DAGScheduler: Submitting 1 missing tasks from Stage 23 (ShuffledRDD[29] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:40 INFO TaskSchedulerImpl: Adding task set 23.0 with 1 tasks
//18/01/12 23:41:40 INFO TaskSetManager: Starting task 0.0 in stage 23.0 (TID 11, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:40 INFO Executor: Running task 0.0 in stage 23.0 (TID 11)
//18/01/12 23:41:40 INFO BlockManager: Removing broadcast 10
//18/01/12 23:41:40 INFO BlockManager: Removing block broadcast_10
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_10 of size 2264 dropped from memory (free 140120794)
//18/01/12 23:41:40 INFO BlockManager: Removing block broadcast_10_piece0
//18/01/12 23:41:40 INFO MemoryStore: Block broadcast_10_piece0 of size 1679 dropped from memory (free 140122473)
//18/01/12 23:41:40 INFO BlockManagerInfo: Removed broadcast_10_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:40 INFO ShuffleBlockFetcherIterator: Getting 0 non-empty blocks out of 0 blocks
//18/01/12 23:41:40 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 5 ms
//18/01/12 23:41:40 INFO Executor: Finished task 0.0 in stage 23.0 (TID 11). 820 bytes result sent to driver
//18/01/12 23:41:40 INFO TaskSetManager: Finished task 0.0 in stage 23.0 (TID 11) in 16 ms on localhost (1/1)
//18/01/12 23:41:40 INFO TaskSchedulerImpl: Removed TaskSet 23.0, whose tasks have all completed, from pool
//18/01/12 23:41:40 INFO BlockManagerMaster: Updated info of block broadcast_10_piece0
//18/01/12 23:41:40 INFO ContextCleaner: Cleaned broadcast 10
//18/01/12 23:41:40 INFO CheckpointWriter: Deleting hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829250000.bk
//-------------------------------------------
//Time: 1515829300000 ms
//-------------------------------------------
//
//18/01/12 23:41:40 INFO DAGScheduler: Stage 23 (print at SparkStreamingOnHDFS.java:126) finished in 0.011 s
//18/01/12 23:41:40 INFO DAGScheduler: Job 11 finished: print at SparkStreamingOnHDFS.java:126, took 0.054029 s
//18/01/12 23:41:40 INFO JobScheduler: Finished job streaming job 1515829300000 ms.0 from job set of time 1515829300000 ms
//18/01/12 23:41:40 INFO ShuffledRDD: Removing RDD 24 from persistence list
//18/01/12 23:41:40 INFO BlockManager: Removing RDD 24
//18/01/12 23:41:40 INFO JobScheduler: Total delay: 0.121 s for time 1515829300000 ms (execution: 0.112 s)
//18/01/12 23:41:40 INFO MappedRDD: Removing RDD 23 from persistence list
//18/01/12 23:41:40 INFO BlockManager: Removing RDD 23
//18/01/12 23:41:40 INFO FlatMappedRDD: Removing RDD 22 from persistence list
//18/01/12 23:41:40 INFO BlockManager: Removing RDD 22
//18/01/12 23:41:40 INFO MappedRDD: Removing RDD 21 from persistence list
//18/01/12 23:41:40 INFO BlockManager: Removing RDD 21
//18/01/12 23:41:40 INFO FileInputDStream: Cleared 0 old files that were older than 1515829240000 ms:
//18/01/12 23:41:40 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:41:40 INFO JobGenerator: Checkpointing graph for time 1515829300000 ms
//18/01/12 23:41:40 INFO DStreamGraph: Updating checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO DStreamGraph: Updated checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO CheckpointWriter: Checkpoint for time 1515829300000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829300000', took 5374 bytes and 109 ms
//18/01/12 23:41:40 INFO CheckpointWriter: Saving checkpoint for time 1515829300000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829300000'
//18/01/12 23:41:40 INFO DStreamGraph: Clearing checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO DStreamGraph: Cleared checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO CheckpointWriter: Deleting hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829250000
//18/01/12 23:41:40 INFO CheckpointWriter: Checkpoint for time 1515829300000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829300000', took 5370 bytes and 114 ms
//18/01/12 23:41:40 INFO DStreamGraph: Clearing checkpoint data for time 1515829300000 ms
//18/01/12 23:41:40 INFO DStreamGraph: Cleared checkpoint data for time 1515829300000 ms
//18/01/12 23:41:50 INFO FileInputDStream: Finding new files took 6 ms
//18/01/12 23:41:50 INFO FileInputDStream: New files at time 1515829310000 ms:
//hdfs://alamps:9000/library/SparkStreaming/data/wc1.txt
//18/01/12 23:41:50 INFO BlockManager: Removing broadcast 11
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_11
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_11 of size 2264 dropped from memory (free 140124737)
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_11_piece0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_11_piece0 of size 1679 dropped from memory (free 140126416)
//18/01/12 23:41:50 INFO BlockManagerInfo: Removed broadcast_11_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_11_piece0
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned broadcast 11
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned shuffle 4
//18/01/12 23:41:50 INFO BlockManager: Removing broadcast 4
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_4
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_4 of size 2264 dropped from memory (free 140128680)
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_4_piece0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_4_piece0 of size 1677 dropped from memory (free 140130357)
//18/01/12 23:41:50 INFO BlockManagerInfo: Removed broadcast_4_piece0 on localhost:54231 in memory (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_4_piece0
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned broadcast 4
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned shuffle 2
//18/01/12 23:41:50 INFO BlockManager: Removing broadcast 2
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_2
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_2 of size 2264 dropped from memory (free 140132621)
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_2_piece0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_2_piece0 of size 1679 dropped from memory (free 140134300)
//18/01/12 23:41:50 INFO BlockManagerInfo: Removed broadcast_2_piece0 on localhost:54231 in memory (size: 1679.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_2_piece0
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned broadcast 2
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned shuffle 1
//18/01/12 23:41:50 INFO BlockManager: Removing broadcast 1
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_1_piece0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_1_piece0 of size 1677 dropped from memory (free 140135977)
//18/01/12 23:41:50 INFO BlockManagerInfo: Removed broadcast_1_piece0 on localhost:54231 in memory (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_1_piece0
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_1
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_1 of size 2264 dropped from memory (free 140138241)
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned broadcast 1
//18/01/12 23:41:50 INFO BlockManager: Removing broadcast 0
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_0 of size 2264 dropped from memory (free 140140505)
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_0_piece0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_0_piece0 of size 1677 dropped from memory (free 140142182)
//18/01/12 23:41:50 INFO BlockManagerInfo: Removed broadcast_0_piece0 on localhost:54231 in memory (size: 1677.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_0_piece0
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned broadcast 0
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned shuffle 0
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(206436) called with curMem=0, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_12 stored as values in memory (estimated size 201.6 KB, free 133.5 MB)
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(31473) called with curMem=206436, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_12_piece0 stored as bytes in memory (estimated size 30.7 KB, free 133.4 MB)
//18/01/12 23:41:50 INFO BlockManagerInfo: Added broadcast_12_piece0 in memory on localhost:54231 (size: 30.7 KB, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_12_piece0
//18/01/12 23:41:50 INFO SparkContext: Created broadcast 12 from textFileStream at SparkStreamingOnHDFS.java:67
//18/01/12 23:41:50 INFO FileInputFormat: Total input paths to process : 1
//18/01/12 23:41:50 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:50 INFO JobScheduler: Starting job streaming job 1515829310000 ms.0 from job set of time 1515829310000 ms
//18/01/12 23:41:50 INFO DAGScheduler: Registering RDD 34 (mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:50 INFO DAGScheduler: Got job 12 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:50 INFO DAGScheduler: Final stage: Stage 25(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:50 INFO DAGScheduler: Parents of final stage: List(Stage 24)
//18/01/12 23:41:50 INFO JobScheduler: Added jobs for time 1515829310000 ms
//18/01/12 23:41:50 INFO JobGenerator: Checkpointing graph for time 1515829310000 ms
//18/01/12 23:41:50 INFO DStreamGraph: Updating checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO DStreamGraph: Updated checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO DAGScheduler: Missing parents: List(Stage 24)
//18/01/12 23:41:50 INFO CheckpointWriter: Saving checkpoint for time 1515829310000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829310000'
//18/01/12 23:41:50 INFO DAGScheduler: Submitting Stage 24 (MappedRDD[34] at mapToPair at SparkStreamingOnHDFS.java:87), which has no missing parents
//18/01/12 23:41:50 INFO CheckpointWriter: Deleting hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829260000.bk
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(4488) called with curMem=237909, maxMem=140142182
//18/01/12 23:41:50 INFO CheckpointWriter: Checkpoint for time 1515829310000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829310000', took 5415 bytes and 36 ms
//18/01/12 23:41:50 INFO DStreamGraph: Clearing checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO DStreamGraph: Cleared checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_13 stored as values in memory (estimated size 4.4 KB, free 133.4 MB)
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(3075) called with curMem=242397, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_13_piece0 stored as bytes in memory (estimated size 3.0 KB, free 133.4 MB)
//18/01/12 23:41:50 INFO BlockManagerInfo: Added broadcast_13_piece0 in memory on localhost:54231 (size: 3.0 KB, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_13_piece0
//18/01/12 23:41:50 INFO SparkContext: Created broadcast 13 from getCallSite at DStream.scala:294
//18/01/12 23:41:50 INFO DAGScheduler: Submitting 1 missing tasks from Stage 24 (MappedRDD[34] at mapToPair at SparkStreamingOnHDFS.java:87)
//18/01/12 23:41:50 INFO TaskSchedulerImpl: Adding task set 24.0 with 1 tasks
//18/01/12 23:41:50 INFO TaskSetManager: Starting task 0.0 in stage 24.0 (TID 12, localhost, ANY, 1448 bytes)
//18/01/12 23:41:50 INFO Executor: Running task 0.0 in stage 24.0 (TID 12)
//18/01/12 23:41:50 INFO NewHadoopRDD: Input split: hdfs://alamps:9000/library/SparkStreaming/data/wc1.txt:0+65
//18/01/12 23:41:50 INFO Executor: Finished task 0.0 in stage 24.0 (TID 12). 1896 bytes result sent to driver
//18/01/12 23:41:50 INFO DAGScheduler: Stage 24 (mapToPair at SparkStreamingOnHDFS.java:87) finished in 0.318 s
//18/01/12 23:41:50 INFO DAGScheduler: looking for newly runnable stages
//18/01/12 23:41:50 INFO DAGScheduler: running: Set()
//18/01/12 23:41:50 INFO DAGScheduler: waiting: Set(Stage 25)
//18/01/12 23:41:50 INFO DAGScheduler: failed: Set()
//18/01/12 23:41:50 INFO TaskSetManager: Finished task 0.0 in stage 24.0 (TID 12) in 322 ms on localhost (1/1)
//18/01/12 23:41:50 INFO TaskSchedulerImpl: Removed TaskSet 24.0, whose tasks have all completed, from pool
//18/01/12 23:41:50 INFO DAGScheduler: Missing parents for Stage 25: List()
//18/01/12 23:41:50 INFO DAGScheduler: Submitting Stage 25 (ShuffledRDD[35] at reduceByKey at SparkStreamingOnHDFS.java:104), which is now runnable
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=245472, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_14 stored as values in memory (estimated size 2.2 KB, free 133.4 MB)
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(1680) called with curMem=247736, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_14_piece0 stored as bytes in memory (estimated size 1680.0 B, free 133.4 MB)
//18/01/12 23:41:50 INFO BlockManagerInfo: Added broadcast_14_piece0 in memory on localhost:54231 (size: 1680.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_14_piece0
//18/01/12 23:41:50 INFO SparkContext: Created broadcast 14 from getCallSite at DStream.scala:294
//18/01/12 23:41:50 INFO DAGScheduler: Submitting 1 missing tasks from Stage 25 (ShuffledRDD[35] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:50 INFO TaskSchedulerImpl: Adding task set 25.0 with 1 tasks
//18/01/12 23:41:50 INFO TaskSetManager: Starting task 0.0 in stage 25.0 (TID 13, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:50 INFO Executor: Running task 0.0 in stage 25.0 (TID 13)
//18/01/12 23:41:50 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
//18/01/12 23:41:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
//18/01/12 23:41:50 INFO Executor: Finished task 0.0 in stage 25.0 (TID 13). 1048 bytes result sent to driver
//18/01/12 23:41:50 INFO TaskSetManager: Finished task 0.0 in stage 25.0 (TID 13) in 34 ms on localhost (1/1)
//18/01/12 23:41:50 INFO TaskSchedulerImpl: Removed TaskSet 25.0, whose tasks have all completed, from pool
//18/01/12 23:41:50 INFO DAGScheduler: Stage 25 (print at SparkStreamingOnHDFS.java:126) finished in 0.029 s
//18/01/12 23:41:50 INFO DAGScheduler: Job 12 finished: print at SparkStreamingOnHDFS.java:126, took 0.442777 s
//18/01/12 23:41:50 INFO SparkContext: Starting job: print at SparkStreamingOnHDFS.java:126
//18/01/12 23:41:50 INFO MapOutputTrackerMaster: Size of output statuses for shuffle 6 is 147 bytes
//18/01/12 23:41:50 INFO DAGScheduler: Got job 13 (print at SparkStreamingOnHDFS.java:126) with 1 output partitions (allowLocal=true)
//18/01/12 23:41:50 INFO DAGScheduler: Final stage: Stage 27(print at SparkStreamingOnHDFS.java:126)
//18/01/12 23:41:50 INFO DAGScheduler: Parents of final stage: List(Stage 26)
//18/01/12 23:41:50 INFO DAGScheduler: Missing parents: List()
//18/01/12 23:41:50 INFO DAGScheduler: Submitting Stage 27 (ShuffledRDD[35] at reduceByKey at SparkStreamingOnHDFS.java:104), which has no missing parents
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(2264) called with curMem=249416, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_15 stored as values in memory (estimated size 2.2 KB, free 133.4 MB)
//18/01/12 23:41:50 INFO MemoryStore: ensureFreeSpace(1680) called with curMem=251680, maxMem=140142182
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_15_piece0 stored as bytes in memory (estimated size 1680.0 B, free 133.4 MB)
//18/01/12 23:41:50 INFO BlockManagerInfo: Added broadcast_15_piece0 in memory on localhost:54231 (size: 1680.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_15_piece0
//18/01/12 23:41:50 INFO SparkContext: Created broadcast 15 from getCallSite at DStream.scala:294
//18/01/12 23:41:50 INFO DAGScheduler: Submitting 1 missing tasks from Stage 27 (ShuffledRDD[35] at reduceByKey at SparkStreamingOnHDFS.java:104)
//18/01/12 23:41:50 INFO TaskSchedulerImpl: Adding task set 27.0 with 1 tasks
//18/01/12 23:41:50 INFO TaskSetManager: Starting task 0.0 in stage 27.0 (TID 14, localhost, PROCESS_LOCAL, 1056 bytes)
//18/01/12 23:41:50 INFO Executor: Running task 0.0 in stage 27.0 (TID 14)
//18/01/12 23:41:50 INFO ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
//18/01/12 23:41:50 INFO ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
//18/01/12 23:41:50 INFO Executor: Finished task 0.0 in stage 27.0 (TID 14). 1055 bytes result sent to driver
//18/01/12 23:41:50 INFO TaskSetManager: Finished task 0.0 in stage 27.0 (TID 14) in 8 ms on localhost (1/1)
//18/01/12 23:41:50 INFO TaskSchedulerImpl: Removed TaskSet 27.0, whose tasks have all completed, from pool
//18/01/12 23:41:50 INFO DAGScheduler: Stage 27 (print at SparkStreamingOnHDFS.java:126) finished in 0.004 s
//-------------------------------------------
//Time: 1515829310000 ms
//-------------------------------------------
//(stream,1)
//(hive,1)
//(python,1)
//(kafka,1)
//(sql,1)
//(spark,2)
//(hadoop,1)
//(flume,1)
//(strom,1)
//(hbase,1)
//
//18/01/12 23:41:50 INFO DAGScheduler: Job 13 finished: print at SparkStreamingOnHDFS.java:126, took 0.019272 s
//18/01/12 23:41:50 INFO JobScheduler: Finished job streaming job 1515829310000 ms.0 from job set of time 1515829310000 ms
//18/01/12 23:41:50 INFO JobScheduler: Total delay: 0.929 s for time 1515829310000 ms (execution: 0.464 s)
//18/01/12 23:41:50 INFO ShuffledRDD: Removing RDD 29 from persistence list
//18/01/12 23:41:50 INFO BlockManager: Removing RDD 29
//18/01/12 23:41:50 INFO MappedRDD: Removing RDD 28 from persistence list
//18/01/12 23:41:50 INFO BlockManager: Removing RDD 28
//18/01/12 23:41:50 INFO FlatMappedRDD: Removing RDD 27 from persistence list
//18/01/12 23:41:50 INFO BlockManager: Removing RDD 27
//18/01/12 23:41:50 INFO MappedRDD: Removing RDD 26 from persistence list
//18/01/12 23:41:50 INFO BlockManager: Removing RDD 26
//18/01/12 23:41:50 INFO UnionRDD: Removing RDD 0 from persistence list
//18/01/12 23:41:50 INFO BlockManager: Removing RDD 0
//18/01/12 23:41:50 INFO FileInputDStream: Cleared 0 old files that were older than 1515829250000 ms:
//18/01/12 23:41:50 INFO ReceivedBlockTracker: Deleting batches ArrayBuffer()
//18/01/12 23:41:50 INFO JobGenerator: Checkpointing graph for time 1515829310000 ms
//18/01/12 23:41:50 INFO DStreamGraph: Updating checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO DStreamGraph: Updated checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO CheckpointWriter: Saving checkpoint for time 1515829310000 ms to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829310000'
//18/01/12 23:41:50 INFO BlockManager: Removing broadcast 15
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_15_piece0
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_15_piece0 of size 1680 dropped from memory (free 139890502)
//18/01/12 23:41:50 INFO BlockManagerInfo: Removed broadcast_15_piece0 on localhost:54231 in memory (size: 1680.0 B, free: 133.6 MB)
//18/01/12 23:41:50 INFO BlockManagerMaster: Updated info of block broadcast_15_piece0
//18/01/12 23:41:50 INFO BlockManager: Removing block broadcast_15
//18/01/12 23:41:50 INFO MemoryStore: Block broadcast_15 of size 2264 dropped from memory (free 139892766)
//18/01/12 23:41:50 INFO ContextCleaner: Cleaned broadcast 15
//18/01/12 23:41:50 INFO CheckpointWriter: Deleting hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829260000
//18/01/12 23:41:50 INFO CheckpointWriter: Checkpoint for time 1515829310000 ms saved to file 'hdfs://alamps:9000/library/SparkStreaming/CheckPoint_data/checkpoint-1515829310000', took 5404 bytes and 39 ms
//18/01/12 23:41:50 INFO DStreamGraph: Clearing checkpoint data for time 1515829310000 ms
//18/01/12 23:41:50 INFO DStreamGraph: Cleared checkpoint data for time 1515829310000 ms