Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit13b0ea0

Browse files
Initial Commit
1 parent08f84fa commit13b0ea0

File tree

6 files changed

+98
-1
lines changed

6 files changed

+98
-1
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Leave entire .ipython checkpoint folder
22
*/.ipynb*
33
*Untitled*
4-
4+
*/dataSink/*
55
*/.idea*
66
*/app-logs*
77
*__pycache__*

‎05-DataSinkDemo/DataSinkDemo.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
frompyspark.sqlimport*
2+
frompyspark.sql.functionsimportspark_partition_id
3+
4+
fromlib.loggerimportLog4j
5+
6+
if__name__=="__main__":
7+
spark=SparkSession \
8+
.builder \
9+
.master("local[3]") \
10+
.appName("SparkSchemaDemo") \
11+
.getOrCreate()
12+
13+
logger=Log4j(spark)
14+
15+
flightTimeParquetDF=spark.read \
16+
.format("parquet") \
17+
.load("dataSource/flight*.parquet")
18+
19+
logger.info("Num Partitions before: "+str(flightTimeParquetDF.rdd.getNumPartitions()))
20+
flightTimeParquetDF.groupBy(spark_partition_id()).count().show()
21+
22+
partitionedDF=flightTimeParquetDF.repartition(5)
23+
logger.info("Num Partitions after: "+str(partitionedDF.rdd.getNumPartitions()))
24+
partitionedDF.groupBy(spark_partition_id()).count().show()
25+
26+
partitionedDF.write \
27+
.format("avro") \
28+
.mode("overwrite") \
29+
.option("path","dataSink/avro/") \
30+
.save()
31+
32+
flightTimeParquetDF.write \
33+
.format("json") \
34+
.mode("overwrite") \
35+
.option("path","dataSink/json/") \
36+
.partitionBy("OP_CARRIER","ORIGIN") \
37+
.option("maxRecordsPerFile",10000) \
38+
.save()
4.77 MB
Binary file not shown.

‎05-DataSinkDemo/lib/__init__.py‎

Whitespace-only changes.

‎05-DataSinkDemo/lib/logger.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
classLog4j:
2+
def__init__(self,spark):
3+
log4j=spark._jvm.org.apache.log4j
4+
5+
root_class="guru.learningjournal.spark.examples"
6+
conf=spark.sparkContext.getConf()
7+
app_name=conf.get("spark.app.name")
8+
9+
self.logger=log4j.LogManager.getLogger(root_class+"."+app_name)
10+
11+
defwarn(self,message):
12+
self.logger.warn(message)
13+
14+
definfo(self,message):
15+
self.logger.info(message)
16+
17+
deferror(self,message):
18+
self.logger.error(message)
19+
20+
defdebug(self,message):
21+
self.logger.debug(message)

‎05-DataSinkDemo/log4j.properties‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Set everything to be logged to the console
2+
log4j.rootCategory=WARN, console
3+
4+
# define console appender
5+
log4j.appender.console=org.apache.log4j.ConsoleAppender
6+
log4j.appender.console.target=System.out
7+
log4j.appender.console.layout=org.apache.log4j.PatternLayout
8+
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
9+
10+
#application log
11+
log4j.logger.guru.learningjournal.spark.examples=INFO, console, file
12+
log4j.additivity.guru.learningjournal.spark.examples=false
13+
14+
#define rolling file appender
15+
log4j.appender.file=org.apache.log4j.RollingFileAppender
16+
log4j.appender.file.File=${spark.yarn.app.container.log.dir}/${logfile.name}.log
17+
#define following in Java System
18+
# -Dlog4j.configuration=file:log4j.properties
19+
# -Dlogfile.name=hello-spark
20+
# -Dspark.yarn.app.container.log.dir=app-logs
21+
log4j.appender.file.ImmediateFlush=true
22+
log4j.appender.file.Append=false
23+
log4j.appender.file.MaxFileSize=500MB
24+
log4j.appender.file.MaxBackupIndex=2
25+
log4j.appender.file.layout=org.apache.log4j.PatternLayout
26+
log4j.appender.file.layout.conversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
27+
28+
# Recommendations from Spark template
29+
log4j.logger.org.apache.spark.repl.Main=WARN
30+
log4j.logger.org.spark_project.jetty=WARN
31+
log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
32+
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
33+
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
34+
log4j.logger.org.apache.parquet=ERROR
35+
log4j.logger.parquet=ERROR
36+
log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
37+
log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
38+

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp