Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit860be30

Browse files
Coder2jCoder2j
Coder2j
authored and
Coder2j
committed
feature/add-source-code
* added README for the repo* added license* added all the jupyter notebooks used in the video
1 parentfa5f3ee commit860be30

File tree

46 files changed

+3360
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+3360
-0
lines changed

‎.gitignore

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
# Byte-compiled / optimized / DLL files
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
6+
# C extensions
7+
*.so
8+
9+
# Distribution / packaging
10+
.Python
11+
build/
12+
develop-eggs/
13+
dist/
14+
downloads/
15+
eggs/
16+
.eggs/
17+
lib/
18+
lib64/
19+
parts/
20+
sdist/
21+
var/
22+
wheels/
23+
share/python-wheels/
24+
*.egg-info/
25+
.installed.cfg
26+
*.egg
27+
MANIFEST
28+
29+
# PyInstaller
30+
# Usually these files are written by a python script from a template
31+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
32+
*.manifest
33+
*.spec
34+
35+
# Installer logs
36+
pip-log.txt
37+
pip-delete-this-directory.txt
38+
39+
# Unit test / coverage reports
40+
htmlcov/
41+
.tox/
42+
.nox/
43+
.coverage
44+
.coverage.*
45+
.cache
46+
nosetests.xml
47+
coverage.xml
48+
*.cover
49+
*.py,cover
50+
.hypothesis/
51+
.pytest_cache/
52+
cover/
53+
54+
# Translations
55+
*.mo
56+
*.pot
57+
58+
# Django stuff:
59+
*.log
60+
local_settings.py
61+
db.sqlite3
62+
db.sqlite3-journal
63+
64+
# Flask stuff:
65+
instance/
66+
.webassets-cache
67+
68+
# Scrapy stuff:
69+
.scrapy
70+
71+
# Sphinx documentation
72+
docs/_build/
73+
74+
# PyBuilder
75+
.pybuilder/
76+
target/
77+
78+
# Jupyter Notebook
79+
.ipynb_checkpoints
80+
81+
# IPython
82+
profile_default/
83+
ipython_config.py
84+
85+
# pyenv
86+
# For a library or package, you might want to ignore these files since the code is
87+
# intended to run in multiple environments; otherwise, check them in:
88+
# .python-version
89+
90+
# pipenv
91+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
93+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
94+
# install all needed dependencies.
95+
#Pipfile.lock
96+
97+
# poetry
98+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99+
# This is especially recommended for binary packages to ensure reproducibility, and is more
100+
# commonly ignored for libraries.
101+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102+
#poetry.lock
103+
104+
# pdm
105+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106+
#pdm.lock
107+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108+
# in version control.
109+
# https://pdm.fming.dev/#use-with-ide
110+
.pdm.toml
111+
112+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113+
__pypackages__/
114+
115+
# Celery stuff
116+
celerybeat-schedule
117+
celerybeat.pid
118+
119+
# SageMath parsed files
120+
*.sage.py
121+
122+
# Environments
123+
.env
124+
.venv
125+
env/
126+
venv/
127+
ENV/
128+
env.bak/
129+
venv.bak/
130+
131+
# Spyder project settings
132+
.spyderproject
133+
.spyproject
134+
135+
# Rope project settings
136+
.ropeproject
137+
138+
# mkdocs documentation
139+
/site
140+
141+
# mypy
142+
.mypy_cache/
143+
.dmypy.json
144+
dmypy.json
145+
146+
# Pyre type checker
147+
.pyre/
148+
149+
# pytype static type analyzer
150+
.pytype/
151+
152+
# Cython debug symbols
153+
cython_debug/
154+
155+
# PyCharm
156+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158+
# and can be added to the global gitignore or merged into this file. For a more nuclear
159+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
160+
#.idea/

‎01-PySpark-Get-Started.ipynb

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type":"code",
5+
"execution_count":1,
6+
"id":"fe9fd1c0-db30-47b1-bbe2-0b1cbd97a9e2",
7+
"metadata": {},
8+
"outputs": [],
9+
"source": [
10+
"# Set the PySpark environment variables\n",
11+
"import os\n",
12+
"os.environ['SPARK_HOME'] =\"/Users/coder2j/Apps/Spark\"\n",
13+
"os.environ['PYSPARK_DRIVER_PYTHON'] = 'jupyter'\n",
14+
"os.environ['PYSPARK_DRIVER_PYTHON_OPTS'] = 'lab'\n",
15+
"os.environ['PYSPARK_PYTHON'] = 'python'"
16+
]
17+
},
18+
{
19+
"cell_type":"code",
20+
"execution_count":2,
21+
"id":"4a915758-1498-4831-820b-a44fd888e87b",
22+
"metadata": {},
23+
"outputs": [],
24+
"source": [
25+
"# Import PySpark\n",
26+
"from pyspark.sql import SparkSession"
27+
]
28+
},
29+
{
30+
"cell_type":"code",
31+
"execution_count":3,
32+
"id":"bb53020b-1e79-4893-a13a-4968fa120fa3",
33+
"metadata": {},
34+
"outputs": [
35+
{
36+
"name":"stderr",
37+
"output_type":"stream",
38+
"text": [
39+
"Setting default log level to\"WARN\".\n",
40+
"To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n",
41+
"23/06/25 21:26:04 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
42+
]
43+
}
44+
],
45+
"source": [
46+
"# Create a SparkSession\n",
47+
"spark = SparkSession.builder\\\n",
48+
" .appName(\"PySpark-Get-Started\")\\\n",
49+
" .getOrCreate()"
50+
]
51+
},
52+
{
53+
"cell_type":"code",
54+
"execution_count":4,
55+
"id":"dde43975-a1f5-4ad1-88a3-76eb84215f2b",
56+
"metadata": {},
57+
"outputs": [
58+
{
59+
"name":"stderr",
60+
"output_type":"stream",
61+
"text": [
62+
"\r"
63+
]
64+
},
65+
{
66+
"name":"stdout",
67+
"output_type":"stream",
68+
"text": [
69+
"+-------+---+\n",
70+
"| Name|Age|\n",
71+
"+-------+---+\n",
72+
"| Alice| 25|\n",
73+
"| Bob| 30|\n",
74+
"|Charlie| 35|\n",
75+
"+-------+---+\n",
76+
"\n"
77+
]
78+
}
79+
],
80+
"source": [
81+
"# Test the setup\n",
82+
"data = [(\"Alice\", 25), (\"Bob\", 30), (\"Charlie\", 35)]\n",
83+
"df = spark.createDataFrame(data, [\"Name\",\"Age\"])\n",
84+
"df.show()"
85+
]
86+
},
87+
{
88+
"cell_type":"code",
89+
"execution_count":null,
90+
"id":"096c5a89-058c-488a-9d9e-146fdb6a44dd",
91+
"metadata": {},
92+
"outputs": [],
93+
"source": []
94+
}
95+
],
96+
"metadata": {
97+
"kernelspec": {
98+
"display_name":"Python 3 (ipykernel)",
99+
"language":"python",
100+
"name":"python3"
101+
},
102+
"language_info": {
103+
"codemirror_mode": {
104+
"name":"ipython",
105+
"version":3
106+
},
107+
"file_extension":".py",
108+
"mimetype":"text/x-python",
109+
"name":"python",
110+
"nbconvert_exporter":"python",
111+
"pygments_lexer":"ipython3",
112+
"version":"3.10.8"
113+
}
114+
},
115+
"nbformat":4,
116+
"nbformat_minor":5
117+
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp