json-logging

Content

1. Features

2. Usage

2.1 Non-web application log

importjson_logging,logging,sys# log is initialized without a web framework namejson_logging.init_non_web(enable_json=True)logger=logging.getLogger("test-logger")logger.setLevel(logging.DEBUG)logger.addHandler(logging.StreamHandler(sys.stdout))logger.info("test logging statement")

2.2 Web application log

FastAPI

importdatetime,logging,sys,json_logging,fastapi,uvicornapp=fastapi.FastAPI()json_logging.init_fastapi(enable_json=True)json_logging.init_request_instrument(app)# init the logger as usuallogger=logging.getLogger("test-logger")logger.setLevel(logging.DEBUG)logger.addHandler(logging.StreamHandler(sys.stdout))@app.get('/')defhome():logger.info("test log statement")logger.info("test log statement with extra props",extra={'props': {"extra_property":'extra_value'}})correlation_id=json_logging.get_correlation_id()return"Hello world : "+str(datetime.datetime.now())if__name__=="__main__":uvicorn.run(app,host='0.0.0.0',port=5000)

Flask

importdatetime,logging,sys,json_logging,flaskapp=flask.Flask(__name__)json_logging.init_flask(enable_json=True)json_logging.init_request_instrument(app)# init the logger as usuallogger=logging.getLogger("test-logger")logger.setLevel(logging.DEBUG)logger.addHandler(logging.StreamHandler(sys.stdout))@app.route('/')defhome():logger.info("test log statement")logger.info("test log statement with extra props",extra={'props': {"extra_property":'extra_value'}})correlation_id=json_logging.get_correlation_id()return"Hello world : "+str(datetime.datetime.now())if__name__=="__main__":app.run(host='0.0.0.0',port=int(5000),use_reloader=False)

Sanic

importlogging,sys,json_logging,sanicapp=sanic.Sanic(name="sanic-web-app")json_logging.init_sanic(enable_json=True)json_logging.init_request_instrument(app)# init the logger as usuallogger=logging.getLogger("sanic-integration-test-app")logger.setLevel(logging.DEBUG)logger.addHandler(logging.StreamHandler(sys.stdout))@app.route("/")asyncdefhome(request):logger.info("test log statement")logger.info("test log statement with extra props",extra={'props': {"extra_property":'extra_value'}})# this will be fastercorrelation_id=json_logging.get_correlation_id(request=request)# this will be slower, but will work in context you cant get a reference of request objectcorrelation_id_without_request_obj=json_logging.get_correlation_id()returnsanic.response.text("hello world")if__name__=="__main__":app.run(host="0.0.0.0",port=5000)

Quart

importasyncio,logging,sys,json_logging,quartapp=quart.Quart(__name__)json_logging.init_quart(enable_json=True)json_logging.init_request_instrument(app)# init the logger as usuallogger=logging.getLogger("test logger")logger.setLevel(logging.DEBUG)logger.addHandler(logging.StreamHandler(sys.stdout))@app.route('/')asyncdefhome():logger.info("test log statement")logger.info("test log statement with extra props",extra={'props': {"extra_property":'extra_value'}})correlation_id=json_logging.get_correlation_id()return"Hello world"if__name__=="__main__":loop=asyncio.get_event_loop()app.run(host='0.0.0.0',port=int(5000),use_reloader=False,loop=loop)

Connexion

importlogging,sys,json_logging,connexionapp=connexion.FlaskApp(__name__)json_logging.init_connexion(enable_json=True)json_logging.init_request_instrument(app)app.add_api('api.yaml')# init the logger as usuallogger=logging.getLogger("test-logger")logger.setLevel(logging.DEBUG)logger.addHandler(logging.StreamHandler(sys.stdout))if__name__=="__main__":app.run()

Custom handler for request instrumentation

importjson_logging.formattersrequest_logger=json_logging.get_request_logger()handler=logging.handlers.RotatingFileHandler(filename='log_req.log',maxBytes=5000000,backupCount=10)handler.setFormatter(json_logging.formatters.JSONRequestLogFormatter())request_logger.addHandler(handler)

2.3 Get current correlation-id

# this will be fastercorrelation_id=json_logging.get_correlation_id(request=request)# this will be slower, but will work in context where you couldn't get a reference of request objectcorrelation_id_without_request_obj=json_logging.get_correlation_id()# use correlation id for upstream service calls here

2.4 Log extra properties

logger.info("test log statement",extra={'props': {'extra_property':'extra_value'},'tags': ['app:name']})

Forced overriding correlation-id

logger.info("test log statement", extra={'props': {'correlation_id': 'custom_correlation_id'}})

2.5 Root logger

logging.basicConfig()json_logging.init_<framework_name> ()json_logging.config_root_logger()

2.6 Custom log formatter

2.7 Exclude certain URl from request instrumentation

json_logging.init_request_instrument(app,exclude_url_patterns=[r'/exclude_from_request_instrumentation'])

3. Configuration

4. Python References

5. Framework support plugin development

6. FAQ & Troubleshooting

7. References

[0] Full logging format references

{"type": "log","written_at": "2017-12-23T16:55:37.280Z","written_ts": 1514048137280721000,"component_id": "1d930c0xd-19-s3213","component_name": "ny-component_name","component_instance_idx": 0,"logger": "test logger","thread": "MainThread","level": "INFO","line_no": 22,"filename": "/path/to/foo.py""exc_info": "Traceback (most recent call last): \n  File "<stdin>", line 1, in <module>\n ValueError: There is something wrong with your input","correlation_id": "1975a02e-e802-11e7-8971-28b2bd90b19a","extra_property": "extra_value","msg": "This is a message"}

{"type": "request","written_at": "2017-12-23T16:55:37.280Z","written_ts": 1514048137280721000,"component_id": "-","component_name": "-","component_instance_idx": 0,"correlation_id": "1975a02e-e802-11e7-8971-28b2bd90b19a","remote_user": "user_a","request": "/index.html","referer": "-","x_forwarded_for": "-","protocol": "HTTP/1.1","method": "GET","remote_ip": "127.0.0.1","request_size_b": 1234,"remote_host": "127.0.0.1","remote_port": 50160,"request_received_at": "2017-12-23T16:55:37.280Z","response_time_ms": 0,"response_status": 200,"response_size_b": "122","response_content_type": "text/html; charset=utf-8","response_sent_at": "2017-12-23T16:55:37.280Z"}

[1] What is correlation-id/request id

[2] Python logging propagate

[3] more on flask use_reloader