feat: Phase 1+2 — project setup and core infrastructure

- requirements.txt, config.yaml, .env, Dockerfile, docker-compose.yml
- app/core: config (YAML+env override), logging (JSON structured),
  exceptions (typed hierarchy), json_utils (Markdown fence stripping)
- app/clients: LLMClient ABC + ZhipuAIClient (run_in_executor),
  StorageClient ABC + RustFSClient (boto3 head_object for size check)
- app/main.py: FastAPI app with health endpoint and router registration
- app/core/dependencies.py: lru_cache singleton factories
- tests/conftest.py: mock_llm, mock_storage, test_app, client fixtures
- pytest.ini: asyncio_mode=auto
- 11 unit tests passing
This commit is contained in:
wh
2026-04-10 15:22:45 +08:00
parent 4162d9f4e6
commit e1eb5e47b1
54 changed files with 716 additions and 0 deletions

0
app/__init__.py Normal file
View File

Binary file not shown.

0
app/clients/__init__.py Normal file
View File

Binary file not shown.

View File

Binary file not shown.

Binary file not shown.

11
app/clients/llm/base.py Normal file
View File

@@ -0,0 +1,11 @@
from abc import ABC, abstractmethod
class LLMClient(ABC):
@abstractmethod
async def chat(self, model: str, messages: list[dict]) -> str:
"""Send a text chat request and return the response content string."""
@abstractmethod
async def chat_vision(self, model: str, messages: list[dict]) -> str:
"""Send a multimodal (vision) chat request and return the response content string."""

View File

@@ -0,0 +1,37 @@
import asyncio
from zhipuai import ZhipuAI
from app.clients.llm.base import LLMClient
from app.core.exceptions import LLMCallError
from app.core.logging import get_logger
logger = get_logger(__name__)
class ZhipuAIClient(LLMClient):
def __init__(self, api_key: str) -> None:
self._client = ZhipuAI(api_key=api_key)
async def chat(self, model: str, messages: list[dict]) -> str:
return await self._call(model, messages)
async def chat_vision(self, model: str, messages: list[dict]) -> str:
return await self._call(model, messages)
async def _call(self, model: str, messages: list[dict]) -> str:
loop = asyncio.get_event_loop()
try:
response = await loop.run_in_executor(
None,
lambda: self._client.chat.completions.create(
model=model,
messages=messages,
),
)
content = response.choices[0].message.content
logger.info("llm_call", extra={"model": model, "response_len": len(content)})
return content
except Exception as exc:
logger.error("llm_call_error", extra={"model": model, "error": str(exc)})
raise LLMCallError(f"大模型调用失败: {exc}") from exc

View File

Binary file not shown.

View File

@@ -0,0 +1,21 @@
from abc import ABC, abstractmethod
class StorageClient(ABC):
@abstractmethod
async def download_bytes(self, bucket: str, path: str) -> bytes:
"""Download an object and return its raw bytes."""
@abstractmethod
async def upload_bytes(
self, bucket: str, path: str, data: bytes, content_type: str = "application/octet-stream"
) -> None:
"""Upload raw bytes to the given bucket/path."""
@abstractmethod
async def get_presigned_url(self, bucket: str, path: str, expires: int = 3600) -> str:
"""Return a presigned GET URL valid for `expires` seconds."""
@abstractmethod
async def get_object_size(self, bucket: str, path: str) -> int:
"""Return the object size in bytes without downloading it."""

View File

@@ -0,0 +1,70 @@
import asyncio
import io
import boto3
from botocore.exceptions import ClientError
from app.clients.storage.base import StorageClient
from app.core.exceptions import StorageError
from app.core.logging import get_logger
logger = get_logger(__name__)
class RustFSClient(StorageClient):
def __init__(self, endpoint: str, access_key: str, secret_key: str) -> None:
self._s3 = boto3.client(
"s3",
endpoint_url=endpoint,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
)
async def download_bytes(self, bucket: str, path: str) -> bytes:
loop = asyncio.get_event_loop()
try:
resp = await loop.run_in_executor(
None, lambda: self._s3.get_object(Bucket=bucket, Key=path)
)
return resp["Body"].read()
except ClientError as exc:
raise StorageError(f"存储下载失败 [{bucket}/{path}]: {exc}") from exc
async def upload_bytes(
self, bucket: str, path: str, data: bytes, content_type: str = "application/octet-stream"
) -> None:
loop = asyncio.get_event_loop()
try:
await loop.run_in_executor(
None,
lambda: self._s3.put_object(
Bucket=bucket, Key=path, Body=io.BytesIO(data), ContentType=content_type
),
)
except ClientError as exc:
raise StorageError(f"存储上传失败 [{bucket}/{path}]: {exc}") from exc
async def get_presigned_url(self, bucket: str, path: str, expires: int = 3600) -> str:
loop = asyncio.get_event_loop()
try:
url = await loop.run_in_executor(
None,
lambda: self._s3.generate_presigned_url(
"get_object",
Params={"Bucket": bucket, "Key": path},
ExpiresIn=expires,
),
)
return url
except ClientError as exc:
raise StorageError(f"生成预签名 URL 失败 [{bucket}/{path}]: {exc}") from exc
async def get_object_size(self, bucket: str, path: str) -> int:
loop = asyncio.get_event_loop()
try:
resp = await loop.run_in_executor(
None, lambda: self._s3.head_object(Bucket=bucket, Key=path)
)
return resp["ContentLength"]
except ClientError as exc:
raise StorageError(f"获取文件大小失败 [{bucket}/{path}]: {exc}") from exc

0
app/core/__init__.py Normal file
View File

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

46
app/core/config.py Normal file
View File

@@ -0,0 +1,46 @@
import os
from functools import lru_cache
from pathlib import Path
from typing import Any
import yaml
from dotenv import load_dotenv
load_dotenv()
# Maps environment variable names to nested YAML key paths
_ENV_OVERRIDES: dict[str, list[str]] = {
"ZHIPUAI_API_KEY": ["zhipuai", "api_key"],
"STORAGE_ACCESS_KEY": ["storage", "access_key"],
"STORAGE_SECRET_KEY": ["storage", "secret_key"],
"STORAGE_ENDPOINT": ["storage", "endpoint"],
"BACKEND_CALLBACK_URL": ["backend", "callback_url"],
"LOG_LEVEL": ["server", "log_level"],
"MAX_VIDEO_SIZE_MB": ["video", "max_file_size_mb"],
}
_CONFIG_PATH = Path(__file__).parent.parent.parent / "config.yaml"
def _set_nested(cfg: dict, keys: list[str], value: Any) -> None:
for key in keys[:-1]:
cfg = cfg.setdefault(key, {})
# Coerce numeric env vars
try:
value = int(value)
except (TypeError, ValueError):
pass
cfg[keys[-1]] = value
@lru_cache(maxsize=1)
def get_config() -> dict:
with open(_CONFIG_PATH, "r", encoding="utf-8") as f:
cfg: dict = yaml.safe_load(f)
for env_var, key_path in _ENV_OVERRIDES.items():
value = os.environ.get(env_var)
if value is not None:
_set_nested(cfg, key_path, value)
return cfg

23
app/core/dependencies.py Normal file
View File

@@ -0,0 +1,23 @@
from functools import lru_cache
from app.clients.llm.base import LLMClient
from app.clients.llm.zhipuai_client import ZhipuAIClient
from app.clients.storage.base import StorageClient
from app.clients.storage.rustfs_client import RustFSClient
from app.core.config import get_config
@lru_cache(maxsize=1)
def get_llm_client() -> LLMClient:
cfg = get_config()
return ZhipuAIClient(api_key=cfg["zhipuai"]["api_key"])
@lru_cache(maxsize=1)
def get_storage_client() -> StorageClient:
cfg = get_config()
return RustFSClient(
endpoint=cfg["storage"]["endpoint"],
access_key=cfg["storage"]["access_key"],
secret_key=cfg["storage"]["secret_key"],
)

50
app/core/exceptions.py Normal file
View File

@@ -0,0 +1,50 @@
from fastapi import Request
from fastapi.responses import JSONResponse
class AIServiceError(Exception):
status_code: int = 500
code: str = "INTERNAL_ERROR"
def __init__(self, message: str) -> None:
self.message = message
super().__init__(message)
class UnsupportedFileTypeError(AIServiceError):
status_code = 400
code = "UNSUPPORTED_FILE_TYPE"
class VideoTooLargeError(AIServiceError):
status_code = 400
code = "VIDEO_TOO_LARGE"
class StorageError(AIServiceError):
status_code = 502
code = "STORAGE_ERROR"
class LLMParseError(AIServiceError):
status_code = 502
code = "LLM_PARSE_ERROR"
class LLMCallError(AIServiceError):
status_code = 503
code = "LLM_CALL_ERROR"
async def ai_service_exception_handler(request: Request, exc: AIServiceError) -> JSONResponse:
return JSONResponse(
status_code=exc.status_code,
content={"code": exc.code, "message": exc.message},
)
async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse:
return JSONResponse(
status_code=500,
content={"code": "INTERNAL_ERROR", "message": str(exc)},
)

19
app/core/json_utils.py Normal file
View File

@@ -0,0 +1,19 @@
import json
import re
from app.core.exceptions import LLMParseError
def extract_json(text: str) -> any:
"""Parse JSON from LLM response, stripping Markdown code fences if present."""
text = text.strip()
# Strip ```json ... ``` or ``` ... ``` fences
fence_match = re.search(r"```(?:json)?\s*([\s\S]+?)\s*```", text)
if fence_match:
text = fence_match.group(1).strip()
try:
return json.loads(text)
except json.JSONDecodeError as e:
raise LLMParseError(f"大模型返回非合法 JSON: {e}") from e

62
app/core/logging.py Normal file
View File

@@ -0,0 +1,62 @@
import json
import logging
import time
from typing import Callable
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.requests import Request
from starlette.responses import Response
def get_logger(name: str) -> logging.Logger:
logger = logging.getLogger(name)
if not logger.handlers:
handler = logging.StreamHandler()
handler.setFormatter(_JsonFormatter())
logger.addHandler(handler)
logger.propagate = False
return logger
class _JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
payload = {
"time": self.formatTime(record, datefmt="%Y-%m-%dT%H:%M:%S"),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
}
if record.exc_info:
payload["exc_info"] = self.formatException(record.exc_info)
# Merge any extra fields passed via `extra=`
for key, value in record.__dict__.items():
if key not in (
"name", "msg", "args", "levelname", "levelno", "pathname",
"filename", "module", "exc_info", "exc_text", "stack_info",
"lineno", "funcName", "created", "msecs", "relativeCreated",
"thread", "threadName", "processName", "process", "message",
"taskName",
):
payload[key] = value
return json.dumps(payload, ensure_ascii=False)
class RequestLoggingMiddleware(BaseHTTPMiddleware):
def __init__(self, app, logger: logging.Logger | None = None) -> None:
super().__init__(app)
self._logger = logger or get_logger("request")
async def dispatch(self, request: Request, call_next: Callable) -> Response:
start = time.perf_counter()
response = await call_next(request)
duration_ms = round((time.perf_counter() - start) * 1000, 1)
self._logger.info(
"request",
extra={
"method": request.method,
"path": request.url.path,
"status": response.status_code,
"duration_ms": duration_ms,
},
)
return response

46
app/main.py Normal file
View File

@@ -0,0 +1,46 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI
from app.core.exceptions import (
AIServiceError,
ai_service_exception_handler,
unhandled_exception_handler,
)
from app.core.logging import RequestLoggingMiddleware, get_logger
logger = get_logger(__name__)
@asynccontextmanager
async def lifespan(app: FastAPI):
logger.info("startup", extra={"message": "AI service starting"})
yield
logger.info("shutdown", extra={"message": "AI service stopping"})
app = FastAPI(
title="Label AI Service",
description="知识图谱标注平台 AI 计算服务",
version="1.0.0",
lifespan=lifespan,
)
app.add_middleware(RequestLoggingMiddleware)
app.add_exception_handler(AIServiceError, ai_service_exception_handler)
app.add_exception_handler(Exception, unhandled_exception_handler)
@app.get("/health", tags=["Health"])
async def health():
return {"status": "ok"}
# Routers registered after implementation (imported lazily to avoid circular deps)
from app.routers import text, image, video, qa, finetune # noqa: E402
app.include_router(text.router, prefix="/api/v1")
app.include_router(image.router, prefix="/api/v1")
app.include_router(video.router, prefix="/api/v1")
app.include_router(qa.router, prefix="/api/v1")
app.include_router(finetune.router, prefix="/api/v1")

0
app/models/__init__.py Normal file
View File

0
app/routers/__init__.py Normal file
View File

3
app/routers/finetune.py Normal file
View File

@@ -0,0 +1,3 @@
from fastapi import APIRouter
router = APIRouter(tags=["Finetune"])

3
app/routers/image.py Normal file
View File

@@ -0,0 +1,3 @@
from fastapi import APIRouter
router = APIRouter(tags=["Image"])

3
app/routers/qa.py Normal file
View File

@@ -0,0 +1,3 @@
from fastapi import APIRouter
router = APIRouter(tags=["QA"])

3
app/routers/text.py Normal file
View File

@@ -0,0 +1,3 @@
from fastapi import APIRouter
router = APIRouter(tags=["Text"])

3
app/routers/video.py Normal file
View File

@@ -0,0 +1,3 @@
from fastapi import APIRouter
router = APIRouter(tags=["Video"])

0
app/services/__init__.py Normal file
View File