Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitfa3fde4

Browse files
authored
Merge pull request#47 from MozillaSecurity/triage-during-import
A few performance/UX improvements for the updater
2 parentsdf481a6 +7f3953b commitfa3fde4

File tree

4 files changed

+61
-60
lines changed

4 files changed

+61
-60
lines changed

‎server/reportmanager/management/commands/cleanup_old_reports.py‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,13 @@ def handle(self, *args, **options):
7171

7272
bug.delete()
7373

74-
# Select all buckets that are empty and delete them
75-
forbucketinBucket.objects.annotate(size=Count("reportentry")).filter(
76-
size=0,bug=None
77-
):
78-
LOG.info("Removing empty bucket %d",bucket.id)
79-
bucket.delete()
74+
ifnotoptions["leave_empty_buckets"]:
75+
# Select all buckets that are empty and delete them
76+
forbucketinBucket.objects.annotate(size=Count("reportentry")).filter(
77+
size=0,bug=None
78+
):
79+
LOG.info("Removing empty bucket %d",bucket.id)
80+
bucket.delete()
8081

8182
# Select all entries that are older than x days
8283
# Again, for the same reason as mentioned above, we have to delete entries in
@@ -97,3 +98,12 @@ def handle(self, *args, **options):
9798
iforphan_bug_count:
9899
LOG.info("Removing %d orphaned Bug objects",orphan_bug_count)
99100
orphan_bugs.delete()
101+
102+
defadd_arguments(self,parser):
103+
parser.add_argument(
104+
"--leave-empty-buckets",
105+
help="skip deleting empty buckets, for example when re-importing reports",
106+
action="store_true",
107+
required=False,
108+
default=False,
109+
)

‎server/reportmanager/management/commands/import_reports.py‎

Lines changed: 0 additions & 47 deletions
This file was deleted.

‎server/reportmanager/management/commands/import_reports_from_bigquery.py‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,49 @@
88
fromdateutil.parserimportisoparse
99
fromdjango.confimportsettings
1010
fromdjango.core.managementimportBaseCommand
11+
fromdjango.db.modelsimportQ
1112
fromdjango.db.utilsimportIntegrityError
1213
fromdjango.utilsimporttimezone
1314
fromgoogle.cloudimportbigquery
1415
fromgoogle.oauth2importservice_account
1516

16-
fromreportmanager.modelsimportReportEntry
17+
fromreportmanager.modelsimportBucket,ReportEntry
1718
fromwebcompat.modelsimportReport
1819

1920
LOG=getLogger("reportmanager.import")
2021

22+
# Little dict that maps hostnames to bucket IDs, used to memoize the results of
23+
# the find_bucket_for_report function below, since DB lookups are slow.
24+
KNOWN_BUCKET_IDS:dict[str,int]= {}
25+
26+
27+
# This is only returning a bucket if there is exactly one matching bucket, or
28+
# if there is absolutely no matching bucket and we have to create one. If there
29+
# are multiple buckets matching, we return none, and effectively leave the
30+
# report in the "untriaged", i.e. not assigned to any bucket, state. The cronjob
31+
# can then pick the report up and run the more comprehensive full-signature
32+
# check.
33+
deffind_bucket_for_report(report_info:Report)->int|None:
34+
hostname=report_info.url.hostname
35+
if (known_bucket:=KNOWN_BUCKET_IDS.get(hostname))isnotNone:
36+
returnknown_bucket
37+
38+
candidates=Bucket.objects.filter(Q(domain=hostname)).values_list("id",flat=True)
39+
40+
iflen(candidates)==1:
41+
KNOWN_BUCKET_IDS[hostname]=candidates[0]
42+
returncandidates[0]
43+
44+
iflen(candidates)==0:
45+
bucket=Bucket.objects.create(
46+
description=f"domain is{report_info.url.hostname}",
47+
signature=report_info.create_signature().raw_signature,
48+
)
49+
KNOWN_BUCKET_IDS[hostname]=bucket.id
50+
returnbucket.id
51+
52+
returnNone
53+
2154

2255
classCommand(BaseCommand):
2356
help="Import reports from BigQuery"
@@ -37,7 +70,7 @@ def handle(self, *args, **options):
3770
# For importing, we ignore reports that have a NULL URL or comment.
3871
# These shouldn't even exist, but we have quite a few rows like that
3972
# anyway. Since they're most likely just broken reports, we don't care.
40-
result=client.query_and_wait(
73+
result=client.query(
4174
f"""SELECT
4275
r.*, t.language_code, t.translated_text,
4376
c.label as ml_label, c.probability as ml_probability
@@ -87,7 +120,9 @@ def handle(self, *args, **options):
87120
ml_valid_probability=ml_valid_probability,
88121
)
89122
withsuppress(IntegrityError):
90-
ReportEntry.objects.create_from_report(report_obj)
123+
ReportEntry.objects.create_from_report(
124+
report_obj,find_bucket_for_report(report_obj)
125+
)
91126
created+=1
92127
LOG.info("imported %d report entries",created)
93128

‎server/reportmanager/models.py‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -427,7 +427,7 @@ class Meta:
427427

428428
classReportEntryManager(models.Manager):
429429
@transaction.atomic
430-
defcreate_from_report(self,report):
430+
defcreate_from_report(self,report,bucket_id=None):
431431
app=App.objects.get_or_create(
432432
channel=report.app_channel,
433433
name=report.app_name,
@@ -452,6 +452,7 @@ def create_from_report(self, report):
452452
comments_translated=report.comments_translated,
453453
comments_original_language=report.comments_original_language,
454454
ml_valid_probability=report.ml_valid_probability,
455+
bucket_id=bucket_id,
455456
)
456457

457458

@@ -523,9 +524,11 @@ def get_report(self):
523524
reported_at=self.reported_at,
524525
uuid=self.uuid,
525526
url=urlsplit(self.url),
526-
breakage_category=self.breakage_category.value
527-
ifself.breakage_categoryisnotNone
528-
elseNone,
527+
breakage_category=(
528+
self.breakage_category.value
529+
ifself.breakage_categoryisnotNone
530+
elseNone
531+
),
529532
ml_valid_probability=self.ml_valid_probability,
530533
)
531534
returnself._cached_report

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp