Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0645c74

Browse files
authored
Data uploader (#309)
1 parent065c6d4 commit0645c74

File tree

8 files changed

+250
-2
lines changed

8 files changed

+250
-2
lines changed
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Generated by Django 4.0.7 on 2022-09-08 20:07
2+
3+
fromdjango.dbimportmigrations,models
4+
5+
6+
classMigration(migrations.Migration):
7+
8+
dependencies= [
9+
("app","0002_notebook_notebookcell"),
10+
]
11+
12+
operations= [
13+
migrations.CreateModel(
14+
name="UploadedData",
15+
fields=[
16+
("id",models.BigAutoField(auto_created=True,primary_key=True,serialize=False,verbose_name="ID")),
17+
("file_type",models.IntegerField(choices=[(1,"CSV"), (2,"JSON")],default=1)),
18+
("created_at",models.DateTimeField(auto_now_add=True)),
19+
("updated_at",models.DateTimeField(auto_now=True)),
20+
],
21+
),
22+
]

‎pgml-dashboard/app/models.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
1-
fromdjango.dbimportmodels,connection
1+
fromdjango.dbimportmodels,connection,transaction
22
fromdjango.template.loaderimportrender_to_string
33
fromdjango.utils.safestringimportmark_safe
44
fromdjango.db.utilsimportProgrammingError
55
fromdjango.utilsimporttimezone
66
fromdjango.utils.htmlimportstrip_tags
77

88
importmarkdown
9+
importcodecs
10+
importcsv
911

1012

1113
classProject(models.Model):
@@ -298,3 +300,35 @@ def code(self):
298300

299301
def__str__(self):
300302
returnf"{self.notebook} -{self.pk}"
303+
304+
305+
classUploadedData(models.Model):
306+
"""Data uploaded by the user through the dashboard."""
307+
308+
file_type=models.IntegerField(
309+
choices=(
310+
(
311+
1,
312+
"CSV",
313+
),
314+
(2,"JSON"),
315+
),
316+
default=1,
317+
)
318+
created_at=models.DateTimeField(auto_now_add=True)
319+
updated_at=models.DateTimeField(auto_now=True)
320+
321+
defcreate_table(self,file):
322+
iffile.content_type=="text/csv":
323+
reader=csv.reader(codecs.iterdecode(file,"utf-8"))
324+
headers=next(reader)
325+
columns=", ".join(map(lambdax:f"{x.replace(' ','_').lower()} FLOAT4",headers))
326+
327+
withtransaction.atomic():
328+
sql=f"CREATE TABLE data_{self.pk} ("+columns+")"
329+
330+
withconnection.cursor()ascursor:
331+
cursor.execute(sql)
332+
333+
file.seek(0)
334+
cursor.copy_expert(f"COPY data_{self.pk} FROM STDIN CSV HEADER",file)

‎pgml-dashboard/app/static/css/base.css‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -652,3 +652,36 @@ main turbo-frame:first-of-type .notebook-cell {
652652
.CodeMirror {
653653
font-size:1rem;
654654
}
655+
656+
/*
657+
* Uploader
658+
*/
659+
body.uploadersectionp,body.uploadersectionli {
660+
margin:0.5rem0;
661+
}
662+
663+
body.uploadersectionol,body.uploadersectionul {
664+
margin:1rem0;
665+
}
666+
667+
body.uploadersection .markdown-body{
668+
margin:1rem0;
669+
}
670+
671+
body.uploaderul {
672+
list-style-type: disc;
673+
list-style-position: inside;
674+
}
675+
676+
body.uploaderol {
677+
list-style-type: decimal;
678+
list-style-position: inside;
679+
}
680+
681+
body.uploadersectionli {
682+
margin-left:1rem;
683+
}
684+
685+
body.uploaderstrong {
686+
font-weight: bold;
687+
}

‎pgml-dashboard/app/templates/base.html‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@
3232
<linkrel="stylesheet"href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/codemirror.min.css"integrity="sha512-uf06llspW44/LZpHzHT6qBOIVODjWtv4MxCricRxkzvopAlSWnTf6hpZTFxuuZcuNE9CBQhqE0Seu1CoRk84nQ=="crossorigin="anonymous"referrerpolicy="no-referrer"/>
3333
<linkrel="stylesheet"href="https://cdnjs.cloudflare.com/ajax/libs/codemirror/6.65.7/addon/hint/show-hint.min.css"integrity="sha512-OmcLQEy8iGiD7PSm85s06dnR7G7C9C0VqahIPAj/KHk5RpOCmnC6R2ob1oK4/uwYhWa9BF1GC6tzxsC8TIx7Jg=="crossorigin="anonymous"referrerpolicy="no-referrer"/>
3434

35+
<!-- Papa Parse <3 -->
36+
<!-- <script defer async src="https://cdn.jsdelivr.net/npm/papaparse@5.3.2/papaparse.min.js"></script> -->
37+
38+
<!-- CSV preview -->
39+
<!-- <link rel="stylesheet" href="https://cdn.jsdelivr.net/gh/kktsvetkov/heiho@latest/heiho.css" /> -->
40+
<!-- <script async defer src="https://cdn.jsdelivr.net/gh/kktsvetkov/heiho@latest/heiho.js"></script> -->
41+
3542
<scriptdefersrc="https://unpkg.com/es-module-shims@1.2.0/dist/es-module-shims.js"></script>
3643
<scripttype="importmap-shim">
3744
{
@@ -92,6 +99,7 @@
9299
<li{%iftopic == "deployments"%}class="selected"{%endif%}><ahref="{% url 'deployments' %}"><spanclass="material-symbols-outlined">inventory</span>Deployments</a></li>
93100
<li{%iftopic == "snapshots"%}class="selected"{%endif%}><ahref="{% url 'snapshots' %}"><spanclass="material-symbols-outlined">storage</span>Snapshots</a></li>
94101
<li{%iftopic == "console"%}class="selected"{%endif%}><ahref="{% url 'console' %}"><spanclass="material-symbols-outlined">terminal</span>Console</a></li>
102+
<li{%iftopic == "uploader"%}class="selected"{%endif%}><ahref="{% url 'uploader' %}"><spanclass="material-symbols-outlined">cloud_upload</span>Upload Data</a></li>
95103
<li><ahref="https://postgresml.org/user_guides/training/overview/"data-turbo="false"target="_blank"><spanclass="material-symbols-outlined">menu_book</span>Docs</a></li>
96104
</ul>
97105
</nav>
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{% extends "base.html" %}
2+
{% load humanize %}
3+
4+
{% block main %}
5+
6+
<section>
7+
{% if error %}
8+
<h1><spanclass="material-symbols-outlined"style="color: var(--highlite-red)">cloud_upload</span>Error</h1>
9+
{% else %}
10+
<h1><spanclass="material-symbols-outlined">cloud_upload</span>Upload Data</h1>
11+
{% endif %}
12+
13+
{% if error %}
14+
<pstyle="margin-bottom: 1rem;">Hmm, something went wrong. Please make sure:</p>
15+
{% else %}
16+
<pstyle="margin-bottom: 1rem;">You can upload your datasets using the CSV format. Before uploading, please make sure:</p>
17+
{% endif %}
18+
19+
<ol>
20+
<li>The data is numeric (i.e. only floats or integers and no text)</li>
21+
<li>The CSV includes headers on the first line</li>
22+
<li>The headers are alphanumeric, contain no spaces and don't start with a number</li>
23+
<li>The CSV is comma (<code>,</code>) delimited</li>
24+
</ol>
25+
26+
{% if error %}
27+
<h4>Error:</h4>
28+
<divclass="markdown-body">
29+
<pre><code>{{ error }}</code></pre>
30+
</div>
31+
{% endif %}
32+
33+
<p>If you are exporting data from a PostgreSQL database, you can use<code>psql</code> to generate a valid CSV file:</p>
34+
<divclass="markdown-body">
35+
<pre><codeclass="language-sql">\copy your_table_name TO 'output.csv' CSV HEADER</code></pre>
36+
</div>
37+
38+
</section>
39+
40+
<section>
41+
<formaction="{% url 'uploader' %}"method="post"enctype="multipart/form-data">
42+
{% csrf_token %}
43+
<divclass="flex">
44+
<inputid="file"type="file"name="file"accept="text/csv,application/json"required="true"/>
45+
</div>
46+
47+
<divclass="button-container">
48+
<buttontype="submit">Upload</button>
49+
</div>
50+
</form>
51+
</section>
52+
{% endblock %}
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{% extends "base.html" %}
2+
{% load humanize %}
3+
4+
{% block main %}
5+
6+
<section>
7+
<h1><spanclass="material-symbols-outlined"style="color: var(--highlite-green)">cloud_upload</span>Upload Successful</h1>
8+
</section>
9+
10+
<section>
11+
<h2><spanclass="material-symbols-outlined">data_array</span>Preview</h2>
12+
13+
{% include 'projects/sample.html' %}
14+
</section>
15+
16+
<section>
17+
<h2><spanclass="material-symbols-outlined">table_rows</span>Next Steps</h2>
18+
<p>Your data has been saved in<strong>pgml.{{ table_name }}</strong> table.</p>
19+
<p>You can now build a model using a<ahref="{% url 'notebooks' %}">Notebook</a> or browse the data in the<ahref="{% url 'console' %}">Console</a>:</p>
20+
<divclass="markdown-body">
21+
<pre><codeclass="language-sql">SELECT * FROM pgml.{{ table_name }}
22+
LIMIT 10</code></pre>
23+
</section>
24+
25+
{% endblock %}

‎pgml-dashboard/app/urls.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
fromdjango.urlsimportpath
22
fromrest_frameworkimportrouters
33

4-
fromapp.viewsimportroot,projects,models,snapshots,deployments,console,notebooks
4+
fromapp.viewsimportroot,projects,models,snapshots,deployments,console,notebooks,uploader
55

66

77
router=routers.DefaultRouter()
@@ -26,6 +26,8 @@
2626
path("projects/<int:pk>",projects.ProjectView.as_view(),name="project"),
2727
path("snapshots/",snapshots.index,name="snapshots"),
2828
path("snapshots/<int:id>",snapshots.snapshot,name="snapshot"),
29+
path("uploader/",uploader.index,name="uploader"),
30+
path("uploader/uploaded/<int:pk>/",uploader.uploaded,name="uploader/uploaded"),
2931
path("console/",console.ConsoleView.as_view(),name="console"),
3032
path("console/run/",console.run_sql,name="console/run-sql"),
3133
path("set-auth-cookie/",root.set_auth_cookie,name="set-auth-cookie"),
Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
fromdjango.dbimportconnection
2+
fromdjango.shortcutsimportrender,get_object_or_404
3+
fromdjango.utils.safestringimportSafeString
4+
fromdjango.httpimportHttpResponse,HttpResponseRedirect
5+
fromdjango.urlsimportreverse_lazy
6+
fromdjangoimportforms
7+
8+
fromapp.modelsimportUploadedData
9+
10+
importcsv
11+
importjson
12+
importcodecs
13+
14+
15+
classUploadForm(forms.Form):
16+
file=forms.FileField()
17+
18+
19+
defindex(request):
20+
ifrequest.method=="POST":
21+
form=UploadForm(request.POST,request.FILES)
22+
ifnotform.is_valid():
23+
returnHttpResponse(status=400)
24+
25+
file=request.FILES.get("file")
26+
iffile.content_typenotin ["text/csv","application/json"]:
27+
returnHttpResponse(status=400)
28+
else:
29+
try:
30+
upload=UploadedData.objects.create(
31+
file_type=1iffile.content_type=="text/csv"else2,
32+
)
33+
34+
upload.create_table(file)
35+
exceptExceptionase:
36+
returnrender(
37+
request,
38+
"uploader/index.html",
39+
{
40+
"error":str(e),
41+
"topic":"uploader",
42+
},
43+
status=400,
44+
)
45+
returnHttpResponseRedirect(reverse_lazy("uploader/uploaded",kwargs={"pk":upload.pk}))
46+
else:
47+
returnrender(
48+
request,
49+
"uploader/index.html",
50+
{
51+
"topic":"uploader",
52+
},
53+
)
54+
55+
56+
defuploaded(request,pk):
57+
upload=UploadedData.objects.get(pk=pk)
58+
withconnection.cursor()ascursor:
59+
cursor.execute(f"SELECT * FROM data_{upload.pk} LIMIT 11")
60+
columns= [col[0]forcolincursor.description]
61+
rows=cursor.fetchall()
62+
returnrender(
63+
request,
64+
"uploader/uploaded.html",
65+
{
66+
"columns":columns,
67+
"rows":rows[:10],
68+
"table_name":f"data_{upload.pk}",
69+
"redacted":len(rows)>10,
70+
"topic":"uploader",
71+
},
72+
)

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp