diff --git a/backend/app/api/v1/login_log.py b/backend/app/api/v1/login_log.py index efe4396b..6477cf16 100644 --- a/backend/app/api/v1/login_log.py +++ b/backend/app/api/v1/login_log.py @@ -20,9 +20,9 @@ async def get_all_login_logs( db: CurrentSession, username: Annotated[str | None, Query()] = None, status: Annotated[bool | None, Query()] = None, - ipaddr: Annotated[str | None, Query()] = None, + ip: Annotated[str | None, Query()] = None, ): - log_select = await LoginLogService.get_select(username=username, status=status, ipaddr=ipaddr) + log_select = await LoginLogService.get_select(username=username, status=status, ip=ip) page_data = await paging_data(db, log_select, GetAllLoginLog) return await response_base.success(data=page_data) diff --git a/backend/app/api/v1/opera_log.py b/backend/app/api/v1/opera_log.py index dd1793ca..184ca2df 100644 --- a/backend/app/api/v1/opera_log.py +++ b/backend/app/api/v1/opera_log.py @@ -20,9 +20,9 @@ async def get_all_opera_logs( db: CurrentSession, username: Annotated[str | None, Query()] = None, status: Annotated[bool | None, Query()] = None, - ipaddr: Annotated[str | None, Query()] = None, + ip: Annotated[str | None, Query()] = None, ): - log_select = await OperaLogService.get_select(username=username, status=status, ipaddr=ipaddr) + log_select = await OperaLogService.get_select(username=username, status=status, ip=ip) page_data = await paging_data(db, log_select, GetAllOperaLog) return await response_base.success(data=page_data) diff --git a/backend/app/crud/crud_login_log.py b/backend/app/crud/crud_login_log.py index 577b47eb..e943ee22 100644 --- a/backend/app/crud/crud_login_log.py +++ b/backend/app/crud/crud_login_log.py @@ -11,17 +11,15 @@ class CRUDLoginLog(CRUDBase[LoginLog, CreateLoginLog, UpdateLoginLog]): - async def get_all( - self, username: str | None = None, status: bool | None = None, ipaddr: str | None = None - ) -> Select: + async def get_all(self, username: str | None = None, status: bool | None = None, ip: str | None = None) -> Select: se = select(self.model).order_by(desc(self.model.create_time)) where_list = [] if username: where_list.append(self.model.username.like(f'%{username}%')) if status is not None: where_list.append(self.model.status == status) - if ipaddr: - where_list.append(self.model.ipaddr.like(f'%{ipaddr}%')) + if ip: + where_list.append(self.model.ip.like(f'%{ip}%')) if where_list: se = se.where(and_(*where_list)) return se diff --git a/backend/app/crud/crud_opera_log.py b/backend/app/crud/crud_opera_log.py index 6ac5d802..2a308652 100644 --- a/backend/app/crud/crud_opera_log.py +++ b/backend/app/crud/crud_opera_log.py @@ -11,17 +11,15 @@ class CRUDOperaLogDao(CRUDBase[OperaLog, CreateOperaLog, UpdateOperaLog]): - async def get_all( - self, username: str | None = None, status: bool | None = None, ipaddr: str | None = None - ) -> Select: + async def get_all(self, username: str | None = None, status: bool | None = None, ip: str | None = None) -> Select: se = select(self.model).order_by(desc(self.model.create_time)) where_list = [] if username: where_list.append(self.model.username.like(f'%{username}%')) if status is not None: where_list.append(self.model.status == status) - if ipaddr: - where_list.append(self.model.ipaddr.like(f'%{ipaddr}%')) + if ip: + where_list.append(self.model.ip.like(f'%{ip}%')) if where_list: se = se.where(and_(*where_list)) return se diff --git a/backend/app/middleware/opera_log_middleware.py b/backend/app/middleware/opera_log_middleware.py index cfe9deed..c9d74025 100644 --- a/backend/app/middleware/opera_log_middleware.py +++ b/backend/app/middleware/opera_log_middleware.py @@ -1,21 +1,22 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- +import traceback from datetime import datetime from typing import Any +from asgiref.sync import sync_to_async from fastapi import UploadFile from starlette.background import BackgroundTask from starlette.requests import Request from starlette.types import ASGIApp, Scope, Receive, Send -from user_agents import parse from backend.app.common.enums import OperaLogCipherType from backend.app.common.log import log from backend.app.core.conf import settings from backend.app.schemas.opera_log import CreateOperaLog from backend.app.services.opera_log_service import OperaLogService -from backend.app.utils import request_parse from backend.app.utils.encrypt import AESCipher, Md5Cipher +from backend.app.utils.request_parse import parse_user_agent_info, parse_ip_info class OperaLogMiddleware: @@ -38,47 +39,72 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: return # 请求信息解析 - ip = await request_parse.get_request_ip(request) - user_agent = request.headers.get('User-Agent') - user_agent_parsed = parse(user_agent) - os = user_agent_parsed.get_os() - browser = user_agent_parsed.get_browser() - if settings.LOCATION_PARSE == 'online': - location = await request_parse.get_location_online(ip, user_agent) - elif settings.LOCATION_PARSE == 'offline': - location = await request_parse.get_location_offline(ip) - else: - location = '未知' + user_agent, device, os, browser = await parse_user_agent_info(request) + ip, country, region, city = await parse_ip_info(request) try: # 此信息依赖于 jwt 中间件 username = request.user.username except AttributeError: username = None method = request.method - args = dict(request.query_params) - form_data = await request.form() - if len(form_data) > 0: - args.update({k: v.filename if isinstance(v, UploadFile) else v for k, v in form_data.items()}) - else: - body_data = await request.body() - if body_data: - json_data = await request.json() - args.update(json_data) + args = await self.get_request_args(request) - # 设置附加请求信息 + # 设置附加请求信息(可选) request.state.ip = ip - request.state.location = location + request.state.country = country + request.state.region = region + request.state.city = city + request.state.user_agent = user_agent request.state.os = os request.state.browser = browser + request.state.device = device + # 执行请求 + start_time = datetime.now() + code, msg, status, err = await self.execute_request(request, send) + end_time = datetime.now() + cost_time = (end_time - start_time).total_seconds() * 1000.0 + + router = request.scope.get('route') + summary = getattr(router, 'summary', '') + args.update(request.path_params) + # 脱敏处理 + args = await self.desensitization(args) + + # 日志创建 + opera_log_in = CreateOperaLog( + username=username, + method=method, + title=summary, + path=path, + ip=ip, + country=country, + region=region, + city=city, + user_agent=user_agent, + os=os, + browser=browser, + device=device, + args=args, + status=status, + code=code, + msg=msg, + cost_time=cost_time, + opera_time=start_time, + ) + back = BackgroundTask(OperaLogService.create, obj_in=opera_log_in) + await back() + + # 错误抛出 + if err: + raise err from None + + async def execute_request(self, request: Request, send: Send) -> tuple: # 预置响应信息 code: int = 200 msg: str = 'Success' status: bool = True err: Any = None - - # 执行请求 - start_time = datetime.now() try: # 详见 https://github.com/tiangolo/fastapi/discussions/8385#discussioncomment-6117967 async def wrapped_rcv_gen(): @@ -96,13 +122,30 @@ async def wrapped_rcv_gen(): except Exception as e: log.exception(e) code = getattr(e, 'code', 500) - msg = getattr(e, 'msg', str(e) or 'Internal Server Error') + msg = getattr( + e, 'msg', traceback.format_exc() if settings.ENVIRONMENT == 'dev' else 'Internal Server Error' + ) status = False err = e - end_time = datetime.now() - summary = request.scope.get('route').summary - title = summary if summary != '' else request.scope.get('route').summary - args.update(request.path_params) + + return code, msg, status, err + + @staticmethod + async def get_request_args(request: Request) -> dict: + args = dict(request.query_params) + form_data = await request.form() + if len(form_data) > 0: + args.update({k: v.filename if isinstance(v, UploadFile) else v for k, v in form_data.items()}) + else: + body_data = await request.body() + if body_data: + json_data = await request.json() + args.update(json_data) + return args + + @staticmethod + @sync_to_async + def desensitization(args: dict): if len(args) > 0: match settings.OPERA_LOG_ENCRYPT: case OperaLogCipherType.aes: @@ -123,29 +166,4 @@ async def wrapped_rcv_gen(): for key in args.keys(): if key in settings.OPERA_LOG_ENCRYPT_INCLUDE: args[key] = '******' - args = args if len(args) > 0 else None - cost_time = (end_time - start_time).total_seconds() * 1000.0 - - # 日志创建 - opera_log_in = CreateOperaLog( - username=username, - method=method, - title=title, - path=path, - ipaddr=ip, - location=location, - os=os, - browser=browser, - args=args, - status=status, - code=code, - msg=msg, - cost_time=cost_time, - opera_time=start_time, - ) - back = BackgroundTask(OperaLogService.create, obj_in=opera_log_in) - await back() - - # 错误抛出 - if err: - raise err from None + return args if len(args) > 0 else None diff --git a/backend/app/models/sys_login_log.py b/backend/app/models/sys_login_log.py index 2bc2a4e9..966aee3e 100644 --- a/backend/app/models/sys_login_log.py +++ b/backend/app/models/sys_login_log.py @@ -17,10 +17,14 @@ class LoginLog(DataClassBase): user_uuid: Mapped[str] = mapped_column(String(50), comment='用户UUID') username: Mapped[str] = mapped_column(String(20), comment='用户名') status: Mapped[bool] = mapped_column(insert_default=0, comment='登录状态(0失败 1成功)') - ipaddr: Mapped[str] = mapped_column(String(50), comment='登录IP地址') - location: Mapped[str] = mapped_column(String(50), comment='归属地') - browser: Mapped[str] = mapped_column(String(50), comment='浏览器') - os: Mapped[str] = mapped_column(String(50), comment='操作系统') + ip: Mapped[str] = mapped_column(String(50), comment='登录IP地址') + country: Mapped[str | None] = mapped_column(String(50), comment='国家') + region: Mapped[str | None] = mapped_column(String(50), comment='地区') + city: Mapped[str | None] = mapped_column(String(50), comment='城市') + user_agent: Mapped[str] = mapped_column(String(255), comment='请求头') + os: Mapped[str | None] = mapped_column(String(50), comment='操作系统') + browser: Mapped[str | None] = mapped_column(String(50), comment='浏览器') + device: Mapped[str | None] = mapped_column(String(50), comment='设备') msg: Mapped[str] = mapped_column(String(255), comment='提示消息') login_time: Mapped[datetime] = mapped_column(comment='登录时间') create_time: Mapped[datetime] = mapped_column(init=False, default=func.now(), comment='创建时间') diff --git a/backend/app/models/sys_opera_log.py b/backend/app/models/sys_opera_log.py index 40d1b1fb..5d007a30 100644 --- a/backend/app/models/sys_opera_log.py +++ b/backend/app/models/sys_opera_log.py @@ -19,10 +19,14 @@ class OperaLog(DataClassBase): method: Mapped[str] = mapped_column(String(20), comment='请求类型') title: Mapped[str] = mapped_column(String(255), comment='操作模块') path: Mapped[str] = mapped_column(String(500), comment='请求路径') - ipaddr: Mapped[str] = mapped_column(String(50), comment='IP地址') - location: Mapped[str] = mapped_column(String(50), comment='归属地') - os: Mapped[str] = mapped_column(String(50), comment='操作系统') - browser: Mapped[str] = mapped_column(String(50), comment='浏览器') + ip: Mapped[str] = mapped_column(String(50), comment='IP地址') + country: Mapped[str | None] = mapped_column(String(50), comment='国家') + region: Mapped[str | None] = mapped_column(String(50), comment='地区') + city: Mapped[str | None] = mapped_column(String(50), comment='城市') + user_agent: Mapped[str] = mapped_column(String(255), comment='请求头') + os: Mapped[str | None] = mapped_column(String(50), comment='操作系统') + browser: Mapped[str | None] = mapped_column(String(50), comment='浏览器') + device: Mapped[str | None] = mapped_column(String(50), comment='设备') args: Mapped[str | None] = mapped_column(JSON(), comment='请求参数') status: Mapped[bool] = mapped_column(comment='操作状态（0异常 1正常）') code: Mapped[int] = mapped_column(insert_default=200, comment='操作状态码') diff --git a/backend/app/schemas/login_log.py b/backend/app/schemas/login_log.py index ef929cc7..21119217 100644 --- a/backend/app/schemas/login_log.py +++ b/backend/app/schemas/login_log.py @@ -9,10 +9,14 @@ class LoginLogBase(BaseModel): user_uuid: str username: str status: bool - ipaddr: str - location: str - browser: str - os: str + ip: str + country: str | None + region: str | None + city: str | None + user_agent: str + browser: str | None + os: str | None + device: str | None msg: str login_time: datetime diff --git a/backend/app/schemas/opera_log.py b/backend/app/schemas/opera_log.py index 7455115c..67c3bba9 100644 --- a/backend/app/schemas/opera_log.py +++ b/backend/app/schemas/opera_log.py @@ -10,10 +10,14 @@ class OperaLogBase(BaseModel): method: str title: str path: str - ipaddr: str - location: str - os: str - browser: str + ip: str + country: str | None = None + region: str | None = None + city: str | None = None + user_agent: str + os: str | None = None + browser: str | None = None + device: str | None = None args: dict | None = None status: bool code: int diff --git a/backend/app/services/login_log_service.py b/backend/app/services/login_log_service.py index 76e637e2..6e0436c5 100644 --- a/backend/app/services/login_log_service.py +++ b/backend/app/services/login_log_service.py @@ -12,32 +12,31 @@ from backend.app.database.db_mysql import async_db_session from backend.app.models import User from backend.app.schemas.login_log import CreateLoginLog -from backend.app.utils import request_parse class LoginLogService: @staticmethod - async def get_select(*, username: str, status: bool, ipaddr: str) -> Select: - return await LoginLogDao.get_all(username=username, status=status, ipaddr=ipaddr) + async def get_select(*, username: str, status: bool, ip: str) -> Select: + return await LoginLogDao.get_all(username=username, status=status, ip=ip) @staticmethod async def create( *, db: AsyncSession, request: Request, user: User, login_time: datetime, status: bool, msg: str ) -> NoReturn: try: - ip = await request_parse.get_request_ip(request) - # 来自 opera log 中间件定义的扩展参数，详见 opera_log_middleware.py - location = request.state.location - browser = request.state.browser - os = request.state.os + # request.state 来自 opera log 中间件定义的扩展参数，详见 opera_log_middleware.py obj_in = CreateLoginLog( user_uuid=user.user_uuid, username=user.username, status=status, - ipaddr=ip, - location=location, - browser=browser, - os=os, + ip=request.state.ip, + country=request.state.country, + region=request.state.region, + city=request.state.city, + user_agent=request.state.user_agent, + browser=request.state.browser, + os=request.state.os, + device=request.state.device, msg=msg, login_time=login_time, ) diff --git a/backend/app/services/opera_log_service.py b/backend/app/services/opera_log_service.py index e47bdda1..d55c8d31 100644 --- a/backend/app/services/opera_log_service.py +++ b/backend/app/services/opera_log_service.py @@ -7,8 +7,8 @@ class OperaLogService: @staticmethod - async def get_select(*, username: str | None = None, status: bool | None = None, ipaddr: str | None = None): - return await OperaLogDao.get_all(username=username, status=status, ipaddr=ipaddr) + async def get_select(*, username: str | None = None, status: bool | None = None, ip: str | None = None): + return await OperaLogDao.get_all(username=username, status=status, ip=ip) @staticmethod async def create(*, obj_in: CreateOperaLog): diff --git a/backend/app/utils/request_parse.py b/backend/app/utils/request_parse.py index 59a042a6..0b29e10d 100644 --- a/backend/app/utils/request_parse.py +++ b/backend/app/utils/request_parse.py @@ -3,9 +3,11 @@ import httpx from XdbSearchIP.xdbSearcher import XdbSearcher from asgiref.sync import sync_to_async -from httpx import HTTPError from fastapi import Request +from user_agents import parse +from backend.app.common.log import log +from backend.app.core.conf import settings from backend.app.core.path_conf import IP2REGION_XDB @@ -24,44 +26,69 @@ def get_request_ip(request: Request) -> str: return ip -async def get_location_online(ipaddr: str, user_agent: str) -> str: +async def get_location_online(ip: str, user_agent: str) -> dict | None: """ 在线获取 ip 地址属地，无法保证可用性，准确率较高 - :param ipaddr: + :param ip: :param user_agent: :return: """ async with httpx.AsyncClient(timeout=3) as client: - ip_api_url = f'http://ip-api.com/json/{ipaddr}?lang=zh-CN' - whois_url = f'http://whois.pconline.com.cn/ipJson.jsp?ip={ipaddr}&json=true' + ip_api_url = f'http://ip-api.com/json/{ip}?lang=zh-CN' headers = {'User-Agent': user_agent} try: - resp1 = await client.get(ip_api_url, headers=headers) - city = resp1.json()['city'] - except (HTTPError, KeyError): - try: - resp2 = await client.get(whois_url, headers=headers) - city = resp2.json()['city'] - except (HTTPError, KeyError): - city = None - return city or '未知' if city != '' else '未知' + response = await client.get(ip_api_url, headers=headers) + if response.status_code == 200: + return response.json() + except Exception as e: + log.error(f'在线获取 ip 地址属地失败，错误信息：{e}') + return None @sync_to_async -def get_location_offline(ipaddr: str) -> str: +def get_location_offline(ip: str) -> str | None: """ 离线获取 ip 地址属地，无法保证准确率，100%可用 - :param ipaddr: + :param ip: :return: """ - cb = XdbSearcher.loadContentFromFile(dbfile=IP2REGION_XDB) - searcher = XdbSearcher(contentBuff=cb) - data = searcher.search(ipaddr) - searcher.close() - location_info = data.split('|') - country = location_info[0] - province = location_info[2] - city = location_info[3] - return city if city != '0' else province if province != '0' else country if country != '0' else '未知' + try: + cb = XdbSearcher.loadContentFromFile(dbfile=IP2REGION_XDB) + searcher = XdbSearcher(contentBuff=cb) + data = searcher.search(ip) + searcher.close() + location_info = data.split('|') + return location_info + except Exception as e: + log.error(f'离线获取 ip 地址属地失败，错误信息：{e}') + return None + + +async def parse_ip_info(request: Request) -> tuple[str, str, str, str]: + country, region, city = None, None, None + ip = await get_request_ip(request) + if settings.LOCATION_PARSE == 'online': + location_info = await get_location_online(ip, request.headers.get('User-Agent')) + if location_info: + country = location_info.get('country') + region = location_info.get('regionName') + city = location_info.get('city') + elif settings.LOCATION_PARSE == 'offline': + location_info = await get_location_offline(ip) + if location_info: + country = location_info[0] if location_info[0] != '0' else None + region = location_info[1] if location_info[1] != '0' else None + city = location_info[2] if location_info[2] != '0' else None + return ip, country, region, city + + +@sync_to_async +def parse_user_agent_info(request: Request) -> tuple[str, str, str, str]: + user_agent = request.headers.get('User-Agent') + _user_agent = parse(user_agent) + device = _user_agent.get_device() + os = _user_agent.get_os() + browser = _user_agent.get_browser() + return user_agent, device, os, browser