Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitedaef00

Browse files
authored
Merge pull requestalexmojaki#14 from alexmojaki/monitor
Add process monitor
2 parents1f84a17 +f4d1d90 commitedaef00

File tree

7 files changed

+195
-43
lines changed

7 files changed

+195
-43
lines changed

‎backend/book/settings.py‎

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,25 +16,11 @@
1616

1717
importbirdseye
1818
importdj_database_url
19-
importsnoop
2019
fromdjango.contrib.messagesimportconstantsasmessages
21-
frommainimportsimple_settings
20+
frommain.simple_settingsimport*
2221

2322
BASE_DIR=Path(__file__).parent.parent
2423

25-
SECRET_KEY=os.environ.get(
26-
'SECRET_KEY',
27-
'kt1+4_u=ga%3v3@fy0@7c(&lq%)6tt=c+f-(ihd32@t$)i6gjm',
28-
)
29-
30-
DEBUG=os.environ.get('DEBUG','True')[0].upper()=='T'
31-
32-
SAVE_CODE_ENTRIES=os.environ.get('SAVE_CODE_ENTRIES','True')[0].upper()=='T'
33-
34-
snoop.install(enabled=DEBUG,out=sys.__stderr__,columns=['thread'])
35-
36-
GITHUB_TOKEN=os.environ.get('GITHUB_TOKEN')
37-
3824
ALLOWED_HOSTS= [
3925
'alexmojaki.pythonanywhere.com',
4026
os.environ.get('HEROKU_APP_NAME','')+'.herokuapp.com',

‎backend/main/simple_settings.py‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,41 @@
1-
importos
1+
importsys
22

33
importsentry_sdk
4+
importsnoop
5+
fromdryenvimportDryEnv,populate_globals
46
fromlittleutilsimportsetup_quick_console_logging
57
fromsentry_sdk.integrations.djangoimportDjangoIntegration
68

79
setup_quick_console_logging()
810

11+
12+
classRoot(DryEnv):
13+
DEBUG=True
14+
15+
SEPARATE_WORKER_PROCESS=False
16+
MASTER_URL="http://localhost:5000/"
17+
18+
SAVE_CODE_ENTRIES=True
19+
20+
SENTRY_DSN=""
21+
SECRET_KEY='kt1+4_u=ga%3v3@fy0@7c(&lq%)6tt=c+f-(ihd32@t$)i6gjm'
22+
GITHUB_TOKEN=""
23+
24+
25+
classMONITOR(DryEnv):
26+
ACTIVE=False
27+
THRESHOLD=90
28+
MIN_PROCESSES=1
29+
NUM_MEASUREMENTS=3
30+
SLEEP_TIME=5
31+
32+
33+
snoop.install(enabled=Root.DEBUG,out=sys.__stderr__,columns=['thread'])
34+
935
sentry_sdk.init(
10-
dsn=os.environ.get("SENTRY_DSN"),
36+
dsn=Root.SENTRY_DSN,
1137
integrations=[DjangoIntegration()],
1238
send_default_pii=True
1339
)
1440

15-
SEPARATE_WORKER_PROCESS=os.environ.get('SEPARATE_WORKER_PROCESS','False')[0].upper()=='T'
16-
17-
MASTER_URL=os.environ.get('MASTER_URL',"http://localhost:5000/")
41+
populate_globals()

‎backend/main/workers/master.py‎

Lines changed: 66 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,54 @@
11
importatexit
2+
importlogging
23
importmultiprocessing
34
importqueue
4-
fromcollectionsimportdefaultdict
5+
fromcollectionsimportdefaultdict,deque
56
fromfunctoolsimportlru_cache
67
frommultiprocessingimportQueue,Process
78
fromthreadingimportThread
8-
fromtimeimportsleep
9+
fromtimeimportsleep,time
910

1011
importflask
12+
importpsutil
13+
importsentry_sdk
1114

1215
frommainimportsimple_settings
16+
frommain.simple_settingsimportMONITOR
1317
frommain.workers.utilsimportinternal_error_result,make_result
1418
frommain.workers.workerimportworker_loop_in_thread
1519

1620
TESTING=False
1721

22+
log=logging.getLogger(__name__)
23+
1824

1925
classUserProcess:
2026
def__init__(self):
27+
self.user_id=None
2128
self.task_queue=Queue()
2229
self.input_queue=Queue()
2330
self.result_queue=Queue()
2431
self.awaiting_input=False
2532
self.process=None
2633
self.fresh_process=True
34+
self.last_used=float('inf')
2735
self.start_process()
2836

29-
@atexit.register
30-
defcleanup():
31-
ifself.process:
32-
self.process.terminate()
37+
atexit.register(self.atexit_cleanup)
38+
39+
defatexit_cleanup(self):
40+
ifself.process:
41+
self.process.terminate()
42+
43+
defclose(self):
44+
atexit.unregister(self.atexit_cleanup)
45+
forqin [self.task_queue,self.input_queue,self.result_queue]:
46+
q.close()
47+
self.process.terminate()
48+
49+
@property
50+
defps(self):
51+
returnpsutil.Process(self.process.pid)
3352

3453
defstart_process(self):
3554
self.fresh_process=True
@@ -41,6 +60,7 @@ def start_process(self):
4160
self.process.start()
4261

4362
defhandle_entry(self,entry):
63+
self.last_used=time()
4464
ifentry["source"]=="shell":
4565
ifself.awaiting_input:
4666
self.input_queue.put(entry["input"])
@@ -55,9 +75,8 @@ def handle_entry(self, entry):
5575

5676
defawait_result(self):
5777
result=self._await_result()
58-
# if result["error"] and result["error"]["sentry_event"]:
59-
# event, hint = result["error"]["sentry_event"]
60-
# capture_event(event, hint)
78+
ifresult["error"]andresult["error"]["sentry_event"]:
79+
sentry_sdk.capture_event(result["error"]["sentry_event"])
6180
self.awaiting_input=result["awaiting_input"]
6281
returnresult
6382

@@ -71,7 +90,7 @@ def _await_result(self):
7190
assert (resultisNone)==self.fresh_process
7291
exceptqueue.Empty:
7392
alive=self.process.is_alive()
74-
print(f"Process{alive=}")
93+
log.info(f"Process{alive=}")
7594
ifalive:
7695
self.process.terminate()
7796
self.start_process()
@@ -98,11 +117,44 @@ def _await_result(self):
98117
assertmultiprocessing.get_start_method()=="spawn"
99118

100119

120+
defmonitor_processes():
121+
history=deque([],MONITOR.NUM_MEASUREMENTS)
122+
whileTrue:
123+
sleep(MONITOR.SLEEP_TIME)
124+
percent=psutil.virtual_memory().percent
125+
history.append(percent)
126+
log.info(f"Recent memory usage:{history}")
127+
log.info(f"Number of user processes:{len(user_processes)}")
128+
if (
129+
len(history)==history.maxlen
130+
andmin(history)>MONITOR.THRESHOLD
131+
andlen(user_processes)>MONITOR.MIN_PROCESSES
132+
):
133+
oldest=min(user_processes.values(),key=lambdap:p.last_used)
134+
log.info(f"Terminating process last used{int(time()-oldest.last_used)} seconds ago")
135+
deluser_processes[oldest.user_id]
136+
oldest.close()
137+
history.clear()
138+
139+
140+
@lru_cache()
141+
defstart_monitor():
142+
ifMONITOR.ACTIVE:
143+
Thread(
144+
target=monitor_processes,
145+
name=monitor_processes.__name__,
146+
daemon=True,
147+
).start()
148+
149+
101150
@app.route("/run",methods=["POST"])
102151
defrun():
152+
start_monitor()
103153
try:
104154
entry=flask.request.json
105-
user_process=user_processes[entry["user_id"]]
155+
user_id=entry["user_id"]
156+
user_process=user_processes[user_id]
157+
user_process.user_id=user_id
106158
user_process.handle_entry(entry)
107159
returnuser_process.await_result()
108160
exceptException:
@@ -123,7 +175,7 @@ def master_session():
123175
importrequests
124176
session=requests.Session()
125177

126-
ifnotsimple_settings.SEPARATE_WORKER_PROCESS:
178+
ifnotsimple_settings.Root.SEPARATE_WORKER_PROCESS:
127179
Thread(
128180
target=run_server,
129181
daemon=True,
@@ -133,7 +185,7 @@ def master_session():
133185
# Wait until alive
134186
whileTrue:
135187
try:
136-
session.get(simple_settings.MASTER_URL+"health")
188+
session.get(simple_settings.Root.MASTER_URL+"health")
137189
break
138190
exceptrequests.exceptions.ConnectionError:
139191
sleep(1)
@@ -143,7 +195,7 @@ def master_session():
143195

144196
defworker_result(entry):
145197
session=master_session()
146-
returnsession.post(simple_settings.MASTER_URL+"run",json=entry).json()
198+
returnsession.post(simple_settings.Root.MASTER_URL+"run",json=entry).json()
147199

148200

149201
if__name__=='__main__':

‎backend/main/workers/utils.py‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1+
importjson
2+
importmultiprocessing.queues
13
importsys
24
importtraceback
3-
importmultiprocessing.queues
4-
importjson
55

6+
importsentry_sdk
67
fromlittleutilsimportDecentJSONEncoder
7-
fromsentry_sdkimportcapture_exception
88

99

1010
classSysStream:
@@ -72,15 +72,31 @@ def make_result(
7272
returnresult
7373

7474

75+
# Import eagerly
76+
sentry_sdk.Hub(sentry_sdk.Client(transport=lambdae:None))
77+
78+
79+
defget_exception_event():
80+
event= {}
81+
82+
deftransport(e):
83+
nonlocalevent
84+
event=e
85+
86+
client=sentry_sdk.Client(transport=transport)
87+
hub=sentry_sdk.Hub(client)
88+
hub.capture_exception()
89+
90+
assertevent
91+
returnevent
92+
93+
7594
definternal_error_result(sentry_offline=False):
7695
ifsentry_offline:
77-
sentry_event=None
78-
# TODO https://stackoverflow.com/questions/60801638/how-to-capture-an-easily-serialisable-exception-event-with-sentry
79-
# exc_info = sys.exc_info()
80-
# sentry_event = event_from_exception(exc_info)
96+
sentry_event=get_exception_event()
8197
else:
8298
sentry_event=None
83-
capture_exception()
99+
sentry_sdk.capture_exception()
84100

85101
tb=traceback.format_exc()
86102
output=f"""

‎docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ services:
2121
image:python-init
2222
build:.
2323
env_file:.env
24+
environment:
25+
MONITOR_ACTIVE:'True'
2426
stdin_open:true
2527
tty:true
2628
entrypoint:./master_server.sh

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp