Dec 17, 2017 · Dec 17, 2017 · Dec 17, 2017 · Dec 17, 2017
diff --git a/build.gradle b/build.gradle
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
diff --git a/gradlew b/gradlew
diff --git a/gradlew.bat b/gradlew.bat
diff --git a/pairRdd/aggregation/reducebykey/WordCount.py b/pairRdd/aggregation/reducebykey/WordCount.py
 from pyspark import SparkContext
 from pyspark import SparkContext, SparkConf

 if __name__ == "__main__":

    sc = SparkContext("local", "wordCounts")
    sc.setLogLevel("ERROR")
    conf = SparkConf().setAppName("wordCounts").setMaster("local[3]")
    sc = SparkContext(conf = conf)

    lines = sc.textFile("in/word_count.text")
    wordRdd = lines.flatMap(lambda line: line.split(" "))
diff --git a/pairRdd/create/PairRddFromRegularRdd.py b/pairRdd/create/PairRddFromRegularRdd.py
 from pyspark import SparkContext
 from pyspark import SparkContext, SparkConf

 if __name__ == "__main__":

    sc = SparkContext("local", "create")
    sc.setLogLevel("ERROR")
    conf = SparkConf().setAppName("create").setMaster("local")
    sc = SparkContext(conf = conf)

    inputStrings = ["Lily 23", "Jack 29", "Mary 29", "James 8"]
    regularRDDs = sc.parallelize(inputStrings)
diff --git a/pairRdd/create/PairRddFromTupleList.py b/pairRdd/create/PairRddFromTupleList.py
 from pyspark import SparkContext
 from pyspark import SparkContext, SparkConf

 if __name__ == "__main__":

    sc = SparkContext("local", "create")
    sc.setLogLevel("ERROR")
    conf = SparkConf().setAppName("create").setMaster("local")
    sc = SparkContext(conf = conf)

    tuples = [("Lily", 23), ("Jack", 29), ("Mary", 29), ("James", 8)]
    pairRDD = sc.parallelize(tuples)
diff --git a/pairRdd/groupbykey/GroupByKeyVsReduceByKey.py b/pairRdd/groupbykey/GroupByKeyVsReduceByKey.py
 from pyspark import SparkContext
 from pyspark import SparkContext, SparkConf

 if __name__ == "__main__":

    sc = SparkContext("local", "GroupByKeyVsReduceByKey")
    sc.setLogLevel("ERROR")
    conf = SparkConf().setAppName('GroupByKeyVsReduceByKey').setMaster("local[*]")
    sc = SparkContext(conf = conf)

    words = ["one", "two", "two", "three", "three", "three"]
    wordsPairRdd = sc.parallelize(words).map(lambda word: (word, 1))

    wordCountsWithReduceByKey = wordsPairRdd.reduceByKey(lambda x, y: x + y).collect()
    wordCountsWithReduceByKey = wordsPairRdd \
        .reduceByKey(lambda x, y: x + y) \
        .collect()
    print("wordCountsWithReduceByKey: {}".format(list(wordCountsWithReduceByKey)))

    wordCountsWithGroupByKey = wordsPairRdd \
        .groupByKey() \
        .mapValues(lambda intIterable:len(intIterable)) \
        .mapValues(len) \
        .collect()
    print("wordCountsWithGroupByKey: {}".format(list(wordCountsWithGroupByKey)))


diff --git a/pairRdd/join/JoinOperations.py b/pairRdd/join/JoinOperations.py
 from pyspark import SparkContext
 from pyspark import SparkContext, SparkConf

 if __name__ == "__main__":

    sc = SparkContext("local", "JoinOperations")
    sc.setLogLevel("ERROR")
    conf = SparkConf().setAppName("JoinOperations").setMaster("local[1]")
    sc = SparkContext(conf = conf)

    ages = sc.parallelize([("Tom", 29), ("John", 22)])
    addresses = sc.parallelize([("James", "USA"), ("John", "UK")])
Original file line number	Diff line number	Diff line change
		@@ -1,9 +1,8 @@
		from pyspark import SparkContext
		from pyspark import SparkContext, SparkConf

		if __name__ == "__main__":

		sc = SparkContext("local", "wordCounts")
		sc.setLogLevel("ERROR")
		conf = SparkConf().setAppName("wordCounts").setMaster("local[3]")
		sc = SparkContext(conf = conf)

		lines = sc.textFile("in/word_count.text")
		wordRdd = lines.flatMap(lambda line: line.split(" "))
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,18 +1,21 @@
		from pyspark import SparkContext
		from pyspark import SparkContext, SparkConf

		if __name__ == "__main__":

		sc = SparkContext("local", "GroupByKeyVsReduceByKey")
		sc.setLogLevel("ERROR")
		conf = SparkConf().setAppName('GroupByKeyVsReduceByKey').setMaster("local[*]")
		sc = SparkContext(conf = conf)

		words = ["one", "two", "two", "three", "three", "three"]
		wordsPairRdd = sc.parallelize(words).map(lambda word: (word, 1))

		wordCountsWithReduceByKey = wordsPairRdd.reduceByKey(lambda x, y: x + y).collect()
		wordCountsWithReduceByKey = wordsPairRdd \
		.reduceByKey(lambda x, y: x + y) \
		.collect()
		print("wordCountsWithReduceByKey: {}".format(list(wordCountsWithReduceByKey)))

		wordCountsWithGroupByKey = wordsPairRdd \
		.groupByKey() \
		.mapValues(lambda intIterable:len(intIterable)) \
		.mapValues(len) \
		.collect()
		print("wordCountsWithGroupByKey: {}".format(list(wordCountsWithGroupByKey)))