docs: update project docs and clean up redundant files
- PROGRESS.md: update to 2026-05-29 with security hardening (T-005), 4-frontend architecture, AI provider refactoring, discovery features, landing page/referral/quota, desktop layout, admin AI management - AGENTS.md: add AI provider list (Alibaba/NVIDIA, removed Claude/DeepL/Local), DB-driven config, CSRF/rate-limit/CORS notes, admin_ai reload quirk - .env.example: sync with actual config, replace deprecated providers with current Sensenova/OpencodeGo/NVIDIA/Spark/Alibaba - docs/PROJECT_STATUS.md: archive (fully superseded by PROGRESS.md) - Remove generated JS files (_bing_search.js, _batch_search.js) - Remove empty directories (data/corpus, data/models) - Remove backend/.coverage (test artifact) - Fix services/.gitignore to cover _bing_search.js - Include pending AI provider DB admin feature (admin_ai, AIProvider model, AIProviders.vue, migration) and T-008 test report
This commit is contained in:
TradeMate Dev
@@ -1,11 +1,8 @@
|
||||
from .openai import OpenAIProvider
|
||||
from .claude import ClaudeProvider
|
||||
from .deepl import DeepLProvider
|
||||
from .local import LocalProvider
|
||||
from .spark import SparkProvider
|
||||
from .sensenova import SensenovaProvider
|
||||
from .opencode_go import OpencodeGoProvider
|
||||
from .nvidia import NvidiaProvider
|
||||
from .alibaba import AlibabaMTProvider
|
||||
|
||||
__all__ = ["OpenAIProvider", "ClaudeProvider", "DeepLProvider", "LocalProvider", "SparkProvider", "SensenovaProvider", "OpencodeGoProvider", "NvidiaProvider", "AlibabaMTProvider"]
|
||||
__all__ = ["OpenAIProvider", "SparkProvider", "SensenovaProvider", "OpencodeGoProvider", "NvidiaProvider", "AlibabaMTProvider"]
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
from typing import Dict, Any, Optional
|
||||
import json
|
||||
from app.ai.base import AIProvider
|
||||
|
||||
|
||||
SYSTEM_PROMPTS = {
|
||||
"marketing": "You are a world-class copywriter for international trade. Write persuasive, "
|
||||
"culturally-adapted marketing content that converts. You excel at storytelling "
|
||||
"and emotional appeal in business contexts.",
|
||||
"reply": "You are a senior international sales representative with 20 years of experience. "
|
||||
"Your replies are warm, professional, and strategically move the conversation "
|
||||
"toward closing the deal.",
|
||||
"translate": "You are a professional translator specializing in trade documents. "
|
||||
"Preserve all numbers, terms, and formatting. Translate meaning, not words.",
|
||||
"extract": "Extract structured data from text. Return ONLY valid JSON.",
|
||||
}
|
||||
|
||||
|
||||
class ClaudeProvider(AIProvider):
|
||||
def __init__(self, api_key: str, model: str = "claude-sonnet-4-20250514"):
|
||||
try:
|
||||
from anthropic import AsyncAnthropic
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"anthropic SDK is required for ClaudeProvider. "
|
||||
"Install it with: pip install anthropic"
|
||||
)
|
||||
self.client = AsyncAnthropic(api_key=api_key)
|
||||
self.model = model
|
||||
self._name = f"claude-sonnet"
|
||||
self._pricing = {"input": 0.003, "output": 0.015}
|
||||
|
||||
async def translate(self, text: str, source_lang: Optional[str], target_lang: str, context: Optional[str] = None) -> Dict[str, Any]:
|
||||
system = SYSTEM_PROMPTS["translate"]
|
||||
if context:
|
||||
system += f"\nContext: {context}"
|
||||
prompt = f"Translate to {target_lang}:\n\n{text}"
|
||||
content = await self._call(system, prompt)
|
||||
return {"translated_text": content, "provider": self.name}
|
||||
|
||||
async def reply(self, inquiry: str, context: Optional[Dict[str, Any]] = None, tone: str = "professional", preference_context: Optional[str] = None) -> Dict[str, Any]:
|
||||
system = SYSTEM_PROMPTS["reply"]
|
||||
if preference_context:
|
||||
system += f"\nUser writing preference: {preference_context}"
|
||||
context_str = ""
|
||||
if context:
|
||||
for k, v in context.items():
|
||||
if v:
|
||||
context_str += f"{k}: {v}\n"
|
||||
prompt = f"{context_str}\nCustomer says:\n{inquiry}\n\nYour reply ({tone} tone):"
|
||||
content = await self._call(system, prompt)
|
||||
return {"reply": content, "provider": self.name}
|
||||
|
||||
async def generate_marketing(self, product_info: Dict[str, Any], target: str, style: str = "professional", language: str = "en", preference_context: Optional[str] = None) -> Dict[str, Any]:
|
||||
system = SYSTEM_PROMPTS["marketing"]
|
||||
if preference_context:
|
||||
system += f"\nUser preference: {preference_context}"
|
||||
info = json.dumps(product_info, ensure_ascii=False, indent=2)
|
||||
prompt = f"Product:\n{info}\n\nTarget: {target}\nStyle: {style}\nLanguage: {language}\n\nWrite marketing copy:"
|
||||
content = await self._call(system, prompt, max_tokens=1500)
|
||||
return {"content": content, "provider": self.name}
|
||||
|
||||
async def extract_info(self, text: str, schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
system = SYSTEM_PROMPTS["extract"]
|
||||
prompt = f"Schema:\n{json.dumps(schema, indent=2)}\n\nText:\n{text}\n\nJSON:"
|
||||
content = await self._call(system, prompt, max_tokens=1000)
|
||||
try:
|
||||
data = json.loads(content)
|
||||
return {"data": data, "confidence": 0.9, "provider": self.name}
|
||||
except json.JSONDecodeError:
|
||||
return {"data": {}, "confidence": 0.0, "provider": self.name, "error": "parse_failed"}
|
||||
|
||||
async def _call(self, system: str, prompt: str, max_tokens: int = 1000) -> str:
|
||||
resp = await self.client.messages.create(
|
||||
model=self.model,
|
||||
system=system,
|
||||
messages=[{"role": "user", "content": prompt}],
|
||||
max_tokens=max_tokens,
|
||||
temperature=0.7,
|
||||
)
|
||||
return resp.content[0].text
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def cost_per_1k_tokens(self) -> float:
|
||||
return (self._pricing["input"] + self._pricing["output"]) / 2
|
||||
|
||||
@property
|
||||
def supports_streaming(self) -> bool:
|
||||
return True
|
||||
@@ -1,51 +0,0 @@
|
||||
from typing import Dict, Any, Optional
|
||||
import httpx
|
||||
from app.ai.base import AIProvider
|
||||
|
||||
|
||||
class DeepLProvider(AIProvider):
|
||||
def __init__(self, api_key: str, endpoint: str = "https://api.deepl.com/v2"):
|
||||
self.api_key = api_key
|
||||
self.endpoint = endpoint
|
||||
self._name = "deepl"
|
||||
self._cost_per_char = 0.000006
|
||||
|
||||
async def translate(self, text: str, source_lang: Optional[str], target_lang: str, context: Optional[str] = None) -> Dict[str, Any]:
|
||||
params = {
|
||||
"auth_key": self.api_key,
|
||||
"text": text,
|
||||
"target_lang": target_lang.upper()[:2],
|
||||
}
|
||||
if source_lang and source_lang != "auto":
|
||||
params["source_lang"] = source_lang.upper()[:2]
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(f"{self.endpoint}/translate", data=params, timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
t = data["translations"][0]
|
||||
return {
|
||||
"translated_text": t["text"],
|
||||
"provider": self.name,
|
||||
"detected_source_lang": t.get("detected_source_language", source_lang),
|
||||
"char_count": len(text),
|
||||
"cost": len(text) * self._cost_per_char,
|
||||
}
|
||||
|
||||
async def reply(self, inquiry: str, context: Optional[Dict[str, Any]] = None, tone: str = "professional") -> Dict[str, Any]:
|
||||
raise NotImplementedError("DeepL does not support reply generation")
|
||||
|
||||
async def generate_marketing(self, product_info: Dict[str, Any], target: str, style: str = "professional", language: str = "en") -> Dict[str, Any]:
|
||||
raise NotImplementedError("DeepL does not support marketing generation")
|
||||
|
||||
async def extract_info(self, text: str, schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
raise NotImplementedError("DeepL does not support info extraction")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def cost_per_1k_tokens(self) -> float:
|
||||
return self._cost_per_char * 1000
|
||||
@@ -1,60 +0,0 @@
|
||||
from typing import Dict, Any, Optional
|
||||
import json, httpx
|
||||
from app.ai.base import AIProvider
|
||||
|
||||
|
||||
class LocalProvider(AIProvider):
|
||||
def __init__(self, model_url: str = "http://localhost:8001", model_name: str = "gemma-3-8b"):
|
||||
self.model_url = model_url.rstrip("/")
|
||||
self.model_name = model_name
|
||||
self._name = f"local-{model_name}"
|
||||
|
||||
async def translate(self, text: str, source_lang: Optional[str], target_lang: str, context: Optional[str] = None) -> Dict[str, Any]:
|
||||
prompt = f"Translate{ f' from {source_lang}' if source_lang else ''} to {target_lang}:\n{text}\n\nTranslation:"
|
||||
result = await self._generate(prompt)
|
||||
return {"translated_text": result, "provider": self.name, "cost": 0.0}
|
||||
|
||||
async def reply(self, inquiry: str, context: Optional[Dict[str, Any]] = None, tone: str = "professional", preference_context: Optional[str] = None) -> Dict[str, Any]:
|
||||
prompt = ""
|
||||
if preference_context:
|
||||
prompt += f"[User prefers: {preference_context}]\n"
|
||||
if context:
|
||||
prompt += "\n".join(f"{k}: {v}" for k, v in context.items() if v) + "\n"
|
||||
prompt += f"Customer: {inquiry}\n\nWrite a {tone} reply:"
|
||||
result = await self._generate(prompt)
|
||||
return {"reply": result, "provider": self.name, "cost": 0.0}
|
||||
|
||||
async def generate_marketing(self, product_info: Dict[str, Any], target: str, style: str = "professional", language: str = "en", preference_context: Optional[str] = None) -> Dict[str, Any]:
|
||||
info = json.dumps(product_info, ensure_ascii=False)
|
||||
prompt = ""
|
||||
if preference_context:
|
||||
prompt += f"[User prefers: {preference_context}]\n"
|
||||
prompt += f"Product: {info}\nTarget: {target}\nStyle: {style}\nLanguage: {language}\n\nMarketing copy:"
|
||||
result = await self._generate(prompt, max_tokens=800)
|
||||
return {"content": result, "provider": self.name, "cost": 0.0}
|
||||
|
||||
async def extract_info(self, text: str, schema: Dict[str, Any]) -> Dict[str, Any]:
|
||||
prompt = f"Extract JSON from text matching schema:\nSchema: {json.dumps(schema)}\n\nText: {text}\n\nJSON:"
|
||||
result = await self._generate(prompt, max_tokens=500)
|
||||
try:
|
||||
return {"data": json.loads(result), "confidence": 0.7, "provider": self.name, "cost": 0.0}
|
||||
except json.JSONDecodeError:
|
||||
return {"data": {}, "confidence": 0.0, "provider": self.name, "cost": 0.0, "error": "parse_failed"}
|
||||
|
||||
async def _generate(self, prompt: str, max_tokens: int = 500) -> str:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.model_url}/v1/completions",
|
||||
json={"model": self.model_name, "prompt": prompt, "max_tokens": max_tokens, "temperature": 0.7, "stream": False},
|
||||
timeout=60,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["choices"][0]["text"].strip()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._name
|
||||
|
||||
@property
|
||||
def cost_per_1k_tokens(self) -> float:
|
||||
return 0.0
|
||||
+94
-78
@@ -1,6 +1,6 @@
|
||||
from typing import Dict, Any, Optional, List
|
||||
from app.ai.base import AIProvider
|
||||
from app.ai.providers import OpenAIProvider, ClaudeProvider, DeepLProvider, LocalProvider, SparkProvider, SensenovaProvider, OpencodeGoProvider, NvidiaProvider, AlibabaMTProvider
|
||||
from app.ai.providers import SparkProvider, SensenovaProvider, OpencodeGoProvider, NvidiaProvider, AlibabaMTProvider
|
||||
from app.config import settings
|
||||
from app.ai.trade_corpus import TradeCorpus
|
||||
import logging
|
||||
@@ -13,95 +13,111 @@ class AIRouter:
|
||||
self.providers: Dict[str, AIProvider] = {}
|
||||
self.routing_rules = settings.AI_ROUTING
|
||||
self.corpus = TradeCorpus()
|
||||
self._init_providers()
|
||||
|
||||
def _init_providers(self):
|
||||
if settings.OPENAI_API_KEY:
|
||||
try:
|
||||
self.providers["openai"] = OpenAIProvider(api_key=settings.OPENAI_API_KEY)
|
||||
logger.info("OpenAI provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"OpenAI init failed: {e}")
|
||||
async def reload_from_db(self, db_session) -> int:
|
||||
from app.models.ai_provider import AIProvider
|
||||
from sqlalchemy import select
|
||||
|
||||
result = await db_session.execute(
|
||||
select(AIProvider).where(AIProvider.enabled == True).order_by(AIProvider.priority)
|
||||
)
|
||||
rows = result.scalars().all()
|
||||
|
||||
new_providers: Dict[str, AIProvider] = {}
|
||||
for p in rows:
|
||||
inst = self._build_provider(p)
|
||||
if inst:
|
||||
key = p.id.hex if hasattr(p.id, 'hex') else str(p.id)
|
||||
new_providers[key] = inst
|
||||
new_providers[p.name] = inst
|
||||
new_providers[p.provider_type] = inst
|
||||
|
||||
if new_providers:
|
||||
self.providers = new_providers
|
||||
logger.info(f"Loaded {len(rows)} AI providers from DB")
|
||||
else:
|
||||
logger.warning("No enabled AI providers found in DB")
|
||||
|
||||
return len(rows)
|
||||
|
||||
async def seed_from_env(self, db_session) -> int:
|
||||
from app.models.ai_provider import AIProvider
|
||||
|
||||
count = 0
|
||||
seeds = []
|
||||
|
||||
if settings.SENSENOVA_API_KEY:
|
||||
try:
|
||||
self.providers["sensenova"] = SensenovaProvider(
|
||||
api_key=settings.SENSENOVA_API_KEY,
|
||||
model=settings.SENSENOVA_MODEL,
|
||||
base_url=settings.SENSENOVA_BASE_URL,
|
||||
)
|
||||
logger.info("Sensenova provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"Sensenova init failed: {e}")
|
||||
|
||||
seeds.append(AIProvider(
|
||||
name="Sensenova (商汤)", provider_type="sensenova",
|
||||
api_key=settings.SENSENOVA_API_KEY,
|
||||
base_url=settings.SENSENOVA_BASE_URL,
|
||||
model_name=settings.SENSENOVA_MODEL, priority=0, enabled=True,
|
||||
))
|
||||
if settings.OPENCODE_GO_API_KEY:
|
||||
try:
|
||||
self.providers["opencode_go"] = OpencodeGoProvider(
|
||||
api_key=settings.OPENCODE_GO_API_KEY,
|
||||
model=settings.OPENCODE_GO_MODEL,
|
||||
base_url=settings.OPENCODE_GO_BASE_URL,
|
||||
)
|
||||
logger.info("OpencodeGo provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"OpencodeGo init failed: {e}")
|
||||
|
||||
seeds.append(AIProvider(
|
||||
name="OpencodeGo", provider_type="opencode_go",
|
||||
api_key=settings.OPENCODE_GO_API_KEY,
|
||||
base_url=settings.OPENCODE_GO_BASE_URL,
|
||||
model_name=settings.OPENCODE_GO_MODEL, priority=1, enabled=True,
|
||||
))
|
||||
if settings.NVIDIA_API_KEY:
|
||||
try:
|
||||
self.providers["nvidia"] = NvidiaProvider(
|
||||
api_key=settings.NVIDIA_API_KEY,
|
||||
model=settings.NVIDIA_MODEL,
|
||||
base_url=settings.NVIDIA_BASE_URL,
|
||||
)
|
||||
logger.info("Nvidia provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"Nvidia init failed: {e}")
|
||||
|
||||
if settings.ANTHROPIC_API_KEY:
|
||||
try:
|
||||
self.providers["anthropic"] = ClaudeProvider(api_key=settings.ANTHROPIC_API_KEY)
|
||||
logger.info("Claude provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"Claude init failed: {e}")
|
||||
|
||||
if settings.DEEPL_API_KEY:
|
||||
try:
|
||||
self.providers["deepl"] = DeepLProvider(api_key=settings.DEEPL_API_KEY)
|
||||
logger.info("DeepL provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"DeepL init failed: {e}")
|
||||
|
||||
seeds.append(AIProvider(
|
||||
name="NVIDIA", provider_type="nvidia",
|
||||
api_key=settings.NVIDIA_API_KEY,
|
||||
base_url=settings.NVIDIA_BASE_URL,
|
||||
model_name=settings.NVIDIA_MODEL, priority=2, enabled=True,
|
||||
))
|
||||
if settings.IFLYTEK_API_KEY:
|
||||
try:
|
||||
self.providers["spark"] = SparkProvider(
|
||||
api_key=settings.IFLYTEK_API_KEY,
|
||||
model=settings.IFLYTEK_MODEL,
|
||||
base_url=settings.IFLYTEK_API_BASE,
|
||||
)
|
||||
logger.info("Spark provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"Spark init failed: {e}")
|
||||
|
||||
seeds.append(AIProvider(
|
||||
name="讯飞 Spark", provider_type="spark",
|
||||
api_key=settings.IFLYTEK_API_KEY,
|
||||
base_url=settings.IFLYTEK_API_BASE,
|
||||
model_name=settings.IFLYTEK_MODEL, priority=3, enabled=True,
|
||||
))
|
||||
if settings.ALIBABA_ACCESS_KEY_ID and settings.ALIBABA_ACCESS_KEY_SECRET:
|
||||
try:
|
||||
self.providers["alibaba-mt"] = AlibabaMTProvider(
|
||||
access_key_id=settings.ALIBABA_ACCESS_KEY_ID,
|
||||
access_key_secret=settings.ALIBABA_ACCESS_KEY_SECRET,
|
||||
)
|
||||
logger.info("Alibaba MT provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"Alibaba MT init failed: {e}")
|
||||
seeds.append(AIProvider(
|
||||
name="阿里翻译", provider_type="alibaba-mt",
|
||||
api_key=settings.ALIBABA_ACCESS_KEY_ID,
|
||||
api_secret=settings.ALIBABA_ACCESS_KEY_SECRET,
|
||||
model_name="alibaba-mt", priority=4, enabled=True,
|
||||
))
|
||||
|
||||
if settings.LOCAL_MODEL_ENABLED:
|
||||
try:
|
||||
self.providers["local"] = LocalProvider(model_url=settings.LOCAL_MODEL_URL)
|
||||
logger.info("Local provider ready")
|
||||
except Exception as e:
|
||||
logger.warning(f"Local init failed: {e}")
|
||||
for p in seeds:
|
||||
db_session.add(p)
|
||||
count += 1
|
||||
if count:
|
||||
await db_session.commit()
|
||||
logger.info(f"Seeded {count} AI providers from .env into DB")
|
||||
return count
|
||||
|
||||
def schedule_reload(self):
|
||||
self._needs_reload = True
|
||||
logger.info("AI router scheduled for reload on next call")
|
||||
|
||||
def _build_provider(self, p) -> Optional[AIProvider]:
|
||||
try:
|
||||
t = p.provider_type
|
||||
if t == "sensenova":
|
||||
return SensenovaProvider(api_key=p.api_key, model=p.model_name, base_url=p.base_url)
|
||||
elif t == "opencode_go":
|
||||
return OpencodeGoProvider(api_key=p.api_key, model=p.model_name, base_url=p.base_url)
|
||||
elif t == "nvidia":
|
||||
return NvidiaProvider(api_key=p.api_key, model=p.model_name, base_url=p.base_url)
|
||||
elif t == "spark":
|
||||
return SparkProvider(api_key=p.api_key, model=p.model_name, base_url=p.base_url)
|
||||
elif t == "alibaba-mt":
|
||||
return AlibabaMTProvider(access_key_id=p.api_key, access_key_secret=p.api_secret or "")
|
||||
else:
|
||||
logger.warning(f"Unknown provider type: {t}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to build provider {p.name}: {e}")
|
||||
return None
|
||||
|
||||
def get_providers_for_task(self, task_type: str) -> List[AIProvider]:
|
||||
rules = self.routing_rules.get(
|
||||
task_type,
|
||||
{"primary": "openai", "fallback": ["local"]},
|
||||
{"primary": "sensenova", "fallback": ["opencode_go"]},
|
||||
)
|
||||
ordered = []
|
||||
seen = set()
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
from . import auth
|
||||
from . import marketing
|
||||
from . import translate
|
||||
from . import customer
|
||||
from . import quotation
|
||||
from . import whatsapp
|
||||
from . import product
|
||||
from . import exchange
|
||||
from . import push
|
||||
from . import admin
|
||||
from . import analytics
|
||||
from . import teams
|
||||
from . import onboarding
|
||||
from . import notification
|
||||
from . import feedback
|
||||
from . import payment
|
||||
from . import interaction
|
||||
from . import silent_pattern
|
||||
from . import training
|
||||
from . import followup
|
||||
from . import ai_assistant
|
||||
from . import discovery
|
||||
from . import discovery_record
|
||||
from . import certification
|
||||
from . import invoice
|
||||
from . import usage
|
||||
from . import referral
|
||||
from . import admin_search
|
||||
from . import search
|
||||
from . import admin_ai
|
||||
|
||||
__all__ = [
|
||||
'auth', 'marketing', 'translate', 'customer', 'quotation', 'whatsapp',
|
||||
'product', 'exchange', 'push', 'admin', 'analytics', 'teams',
|
||||
'onboarding', 'notification', 'feedback', 'payment', 'interaction',
|
||||
'silent_pattern', 'training', 'followup', 'ai_assistant', 'discovery',
|
||||
'discovery_record', 'certification', 'invoice', 'usage', 'referral',
|
||||
'admin_search', 'search', 'admin_ai'
|
||||
]
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select
|
||||
from app.database import get_db
|
||||
from app.api.v1.deps import get_current_user
|
||||
from app.models.ai_provider import AIProvider
|
||||
from app.ai.router import get_ai_router
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
async def require_admin(current_user: dict = Depends(get_current_user)) -> dict:
|
||||
if current_user.get("role") != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin access required")
|
||||
return current_user
|
||||
|
||||
|
||||
class AIProviderCreate(BaseModel):
|
||||
name: str
|
||||
provider_type: str
|
||||
api_key: Optional[str] = None
|
||||
api_secret: Optional[str] = None
|
||||
base_url: Optional[str] = None
|
||||
model_name: str = "deepseek-v4-flash"
|
||||
extra_config: Optional[dict] = None
|
||||
priority: int = 0
|
||||
enabled: bool = True
|
||||
|
||||
|
||||
class AIProviderUpdate(BaseModel):
|
||||
name: Optional[str] = None
|
||||
api_key: Optional[str] = None
|
||||
api_secret: Optional[str] = None
|
||||
base_url: Optional[str] = None
|
||||
model_name: Optional[str] = None
|
||||
extra_config: Optional[dict] = None
|
||||
priority: Optional[int] = None
|
||||
enabled: Optional[bool] = None
|
||||
|
||||
|
||||
PROVIDER_TYPE_LABELS = {
|
||||
"sensenova": "Sensenova (商汤)",
|
||||
"opencode_go": "OpencodeGo",
|
||||
"nvidia": "NVIDIA",
|
||||
"spark": "讯飞 Spark",
|
||||
"alibaba-mt": "阿里翻译",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/ai-providers")
|
||||
async def list_providers(
|
||||
page: int = Query(1, ge=1),
|
||||
size: int = Query(50, ge=1, le=100),
|
||||
_: dict = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
result = await db.execute(
|
||||
select(AIProvider).order_by(AIProvider.priority).offset((page - 1) * size).limit(size)
|
||||
)
|
||||
providers = result.scalars().all()
|
||||
total_result = await db.execute(select(AIProvider))
|
||||
total = len(total_result.scalars().all())
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"id": str(p.id),
|
||||
"name": p.name,
|
||||
"provider_type": p.provider_type,
|
||||
"type_label": PROVIDER_TYPE_LABELS.get(p.provider_type, p.provider_type),
|
||||
"api_key": p.api_key[:8] + "..." if p.api_key and len(p.api_key) > 8 else (p.api_key or ""),
|
||||
"api_secret": bool(p.api_secret),
|
||||
"base_url": p.base_url,
|
||||
"model_name": p.model_name,
|
||||
"extra_config": p.extra_config,
|
||||
"priority": p.priority,
|
||||
"enabled": p.enabled,
|
||||
"created_at": p.created_at.isoformat() if p.created_at else None,
|
||||
"updated_at": p.updated_at.isoformat() if p.updated_at else None,
|
||||
}
|
||||
for p in providers
|
||||
],
|
||||
"total": total,
|
||||
"page": page,
|
||||
"size": size,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/ai-providers")
|
||||
async def create_provider(
|
||||
data: AIProviderCreate,
|
||||
_: dict = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
provider = AIProvider(
|
||||
name=data.name,
|
||||
provider_type=data.provider_type,
|
||||
api_key=data.api_key,
|
||||
api_secret=data.api_secret,
|
||||
base_url=data.base_url,
|
||||
model_name=data.model_name,
|
||||
extra_config=data.extra_config or {},
|
||||
priority=data.priority,
|
||||
enabled=data.enabled,
|
||||
)
|
||||
db.add(provider)
|
||||
await db.commit()
|
||||
await db.refresh(provider)
|
||||
await get_ai_router().reload_from_db(db)
|
||||
return {"id": str(provider.id), "message": "AI provider created"}
|
||||
|
||||
|
||||
@router.put("/ai-providers/{provider_id}")
|
||||
async def update_provider(
|
||||
provider_id: str,
|
||||
data: AIProviderUpdate,
|
||||
_: dict = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
result = await db.execute(select(AIProvider).where(AIProvider.id == provider_id))
|
||||
provider = result.scalar_one_or_none()
|
||||
if not provider:
|
||||
raise HTTPException(status_code=404, detail="AI provider not found")
|
||||
|
||||
update_data = data.model_dump(exclude_unset=True)
|
||||
for key, value in update_data.items():
|
||||
setattr(provider, key, value)
|
||||
await db.commit()
|
||||
await db.refresh(provider)
|
||||
|
||||
await get_ai_router().reload_from_db(db)
|
||||
return {"id": str(provider.id), "message": "AI provider updated"}
|
||||
|
||||
|
||||
@router.delete("/ai-providers/{provider_id}")
|
||||
async def delete_provider(
|
||||
provider_id: str,
|
||||
_: dict = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
result = await db.execute(select(AIProvider).where(AIProvider.id == provider_id))
|
||||
provider = result.scalar_one_or_none()
|
||||
if not provider:
|
||||
raise HTTPException(status_code=404, detail="AI provider not found")
|
||||
await db.delete(provider)
|
||||
await db.commit()
|
||||
|
||||
await get_ai_router().reload_from_db(db)
|
||||
return {"message": "AI provider deleted"}
|
||||
|
||||
|
||||
@router.post("/ai-providers/reload")
|
||||
async def reload_providers(
|
||||
_: dict = Depends(require_admin),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
count = await get_ai_router().reload_from_db(db)
|
||||
return {"message": f"AI providers reloaded, {count} providers active"}
|
||||
|
||||
|
||||
@router.get("/ai-providers/status")
|
||||
async def get_provider_status(
|
||||
_: dict = Depends(require_admin),
|
||||
):
|
||||
router = get_ai_router()
|
||||
active = list(router.providers.keys())
|
||||
routing = router.routing_rules
|
||||
return {"active_providers": active, "routing_rules": routing, "provider_count": len(active)}
|
||||
@@ -17,6 +17,7 @@ from .invoice import Invoice, InvoiceType, InvoiceStatus
|
||||
from .referral import ReferralCode, Referral
|
||||
from .search_provider import SearchProvider
|
||||
from .discovery_record import DiscoveryRecord
|
||||
from .ai_provider import AIProvider
|
||||
|
||||
__all__ = [
|
||||
"User", "Product",
|
||||
@@ -38,4 +39,5 @@ __all__ = [
|
||||
"ReferralCode", "Referral",
|
||||
"SearchProvider",
|
||||
"DiscoveryRecord",
|
||||
"AIProvider",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
from sqlalchemy import Column, String, Integer, DateTime, Boolean, Text
|
||||
from sqlalchemy.dialects.postgresql import UUID, JSONB
|
||||
from datetime import datetime
|
||||
from app.database import Base
|
||||
import uuid
|
||||
|
||||
|
||||
class AIProvider(Base):
|
||||
__tablename__ = "ai_providers"
|
||||
|
||||
id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
name = Column(String(100), nullable=False)
|
||||
provider_type = Column(String(50), nullable=False)
|
||||
api_key = Column(Text, nullable=True)
|
||||
api_secret = Column(Text, nullable=True)
|
||||
base_url = Column(String(500), nullable=True)
|
||||
model_name = Column(String(100), nullable=False)
|
||||
extra_config = Column(JSONB, default={})
|
||||
priority = Column(Integer, default=0)
|
||||
enabled = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
@@ -1 +1,2 @@
|
||||
_batch_search.js
|
||||
_bing_search.js
|
||||
|
||||
@@ -288,11 +288,11 @@ class AdminService:
|
||||
|
||||
async def _seed_default_configs(self):
|
||||
defaults = [
|
||||
SystemConfig(key="ai_provider_translate", value={"primary": "sensenova", "fallback": ["openai", "local"]}, description="翻译任务 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_reply", value={"primary": "sensenova", "fallback": ["anthropic", "local"]}, description="回复建议 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_marketing", value={"primary": "sensenova", "fallback": ["openai", "local"]}, description="营销文案 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_extract", value={"primary": "sensenova", "fallback": ["openai"]}, description="信息提取 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_quotation", value={"primary": "sensenova", "fallback": ["openai"]}, description="报价单 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_translate", value={"primary": "sensenova", "fallback": ["alibaba-mt", "opencode_go"]}, description="翻译任务 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_reply", value={"primary": "sensenova", "fallback": ["opencode_go"]}, description="回复建议 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_marketing", value={"primary": "sensenova", "fallback": ["opencode_go"]}, description="营销文案 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_extract", value={"primary": "sensenova", "fallback": ["opencode_go"]}, description="信息提取 AI 模型选择"),
|
||||
SystemConfig(key="ai_provider_quotation", value={"primary": "sensenova", "fallback": ["opencode_go"]}, description="报价单 AI 模型选择"),
|
||||
SystemConfig(key="feature_guest_mode", value={"enabled": True}, description="游客模式开关"),
|
||||
SystemConfig(key="feature_wechat_login", value={"enabled": False}, description="微信登录开关"),
|
||||
SystemConfig(key="feature_registration", value={"enabled": True}, description="新用户注册开关"),
|
||||
|
||||
@@ -1,122 +1,130 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import re
|
||||
from typing import List, Dict
|
||||
import functools
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
||||
NODE_BIN = "/usr/bin/node"
|
||||
|
||||
BATCH_SCRIPT = r"""
|
||||
const p = require('puppeteer');
|
||||
(async () => {
|
||||
const queries = JSON.parse(process.argv[process.argv.length - 2]);
|
||||
const max = parseInt(process.argv[process.argv.length - 1] || '6', 10);
|
||||
const sk = ['bing.com','google.com','facebook.com','twitter.com','instagram.com','youtube.com','reddit.com','amazon.com','walmart.com','w3.org','whatsapp.com','wechat.com','qq.com','taobao.com','tmall.com','alipay.com','zhihu.com','baike.baidu.com','sogou.com','163.com','sohu.com','sina.com','iciba.com','cambridge','britannica','sciencedirect','mdpi.com','springer','wiley.com','acm.org','ieee.org','researchgate','semanticscholar','ncbi.nlm.nih','nature.com','oup.com','sagepub','tandfonline','pinterest','ebay','dictionary','translate'];
|
||||
HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
|
||||
}
|
||||
|
||||
try {
|
||||
const b = await p.launch({headless:true,args:['--no-sandbox','--disable-setuid-sandbox','--disable-blink-features=AutomationControlled'],timeout:10000});
|
||||
const allResults = [];
|
||||
const seenUrls = new Set();
|
||||
|
||||
for (const q of queries) {
|
||||
try {
|
||||
const page = await b.newPage();
|
||||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
|
||||
await page.setExtraHTTPHeaders({'Accept-Language':'en-US,en;q=0.9'});
|
||||
|
||||
const url = 'https://www.bing.com/search?q=' + encodeURIComponent(q) + '&setlang=en-US&cc=US';
|
||||
await page.goto(url, {waitUntil:'domcontentloaded',timeout:8000});
|
||||
await page.waitForSelector('.b_algo', {timeout:4000}).catch(()=>{});
|
||||
|
||||
const results = await page.evaluate((m, sk) => {
|
||||
const found = []; const seen = new Set();
|
||||
document.querySelectorAll('li.b_algo').forEach(li => {
|
||||
const a = li.querySelector('h2 a'); if (!a) return;
|
||||
let url = (a.href || '').replace(/\/$/,'');
|
||||
if (!url.startsWith('http') || seen.has(url)) return;
|
||||
seen.add(url);
|
||||
if (sk.some(d => url.includes(d))) return;
|
||||
const hostname = url.replace(/^https?:\/\//,'').split('/')[0];
|
||||
if (hostname.endsWith('.edu') || hostname.endsWith('.ac') || hostname.endsWith('.gov')) return;
|
||||
const title = (a.textContent||'').trim().substring(0,100);
|
||||
const s = li.querySelector('.b_caption p, .b_lineclamp2');
|
||||
found.push({title, url, snippet:s?s.textContent.trim().substring(0,200):''});
|
||||
});
|
||||
return found.slice(0,m);
|
||||
}, max, sk);
|
||||
|
||||
for (const r of results) {
|
||||
if (!seenUrls.has(r.url)) {
|
||||
seenUrls.add(r.url);
|
||||
allResults.push(r);
|
||||
}
|
||||
}
|
||||
await page.close();
|
||||
} catch(e) { /* skip failed query */ }
|
||||
}
|
||||
console.log(JSON.stringify(allResults.slice(0, max * queries.length)));
|
||||
await b.close();
|
||||
} catch(e) { console.log('[]'); }
|
||||
})();
|
||||
"""
|
||||
SKIP_DOMAINS = {
|
||||
"iciba.com", "baike.baidu.com", "cambridge.org", "dictionary.cambridge.org",
|
||||
"collinsdictionary.com", "dictionary.com", "merriam-webster.com",
|
||||
"thesaurus.com", "britannica.com", "wikipedia.org", "wikihow.com",
|
||||
"facebook.com", "twitter.com", "instagram.com", "youtube.com",
|
||||
"reddit.com", "pinterest.com", "amazon.com", "ebay.com",
|
||||
"walmart.com", "target.com", "bestbuy.com", "homedepot.com",
|
||||
"linkedin.com", "bing.com", "google.com",
|
||||
}
|
||||
SKIP_TITLE_PATTERNS = [
|
||||
r'^是什么意思$', r'^翻译$', r'^词典$', r'^字典$',
|
||||
r'翻译$', r'^百度百科', r'^维基百科',
|
||||
]
|
||||
|
||||
|
||||
BATCH_SCRIPT_FILE = os.path.join(os.path.dirname(__file__), "_batch_search.js")
|
||||
NODE_MODULES = os.path.join(PROJECT_ROOT, "node_modules")
|
||||
def _is_junk(item: Dict[str, str]) -> bool:
|
||||
url = item.get("url", "")
|
||||
title = item.get("title", "")
|
||||
hostname = url.replace("https://", "").replace("http://", "").split("/")[0]
|
||||
if any(d in hostname for d in SKIP_DOMAINS):
|
||||
return True
|
||||
if any(d in url for d in SKIP_DOMAINS):
|
||||
return True
|
||||
for p in SKIP_TITLE_PATTERNS:
|
||||
if re.search(p, title):
|
||||
return True
|
||||
if hostname.endswith(".edu") or hostname.endswith(".ac") or hostname.endswith(".gov"):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _search_bing(query: str, count: int = 6) -> List[Dict[str, str]]:
|
||||
try:
|
||||
is_cjk = bool(re.search(r'[\u4e00-\u9fff]', query))
|
||||
params = {"q": query, "count": count}
|
||||
if not is_cjk:
|
||||
params.update({"setlang": "en-US", "cc": "US"})
|
||||
url = "https://www.bing.com/search"
|
||||
resp = requests.get(url, params=params, headers=HEADERS, timeout=10)
|
||||
resp.raise_for_status()
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
results = []
|
||||
seen = set()
|
||||
for li in soup.select("li.b_algo"):
|
||||
a = li.select_one("h2 a")
|
||||
if not a:
|
||||
continue
|
||||
href = a.get("href", "")
|
||||
if not href.startswith("http") or href in seen:
|
||||
continue
|
||||
seen.add(href)
|
||||
title = a.get_text(strip=True)[:120]
|
||||
snippet_el = li.select_one(".b_caption p, .b_lineclamp2")
|
||||
snippet = snippet_el.get_text(strip=True)[:300] if snippet_el else ""
|
||||
entry = {"title": title, "url": href, "snippet": snippet, "engine": "bing"}
|
||||
if not _is_junk(entry):
|
||||
results.append(entry)
|
||||
if len(results) >= count:
|
||||
break
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"Bing search failed: {e}")
|
||||
return []
|
||||
|
||||
|
||||
def _search_360(query: str, count: int = 6) -> List[Dict[str, str]]:
|
||||
try:
|
||||
resp = requests.get("https://www.so.com/s", params={"q": query}, headers=HEADERS, timeout=10)
|
||||
resp.raise_for_status()
|
||||
soup = BeautifulSoup(resp.text, "html.parser")
|
||||
results = []
|
||||
seen = set()
|
||||
for li in soup.select(".result-list li, .result"):
|
||||
a = li.select_one("h3 a")
|
||||
if not a:
|
||||
continue
|
||||
href = a.get("href", "")
|
||||
if not href or href in seen:
|
||||
continue
|
||||
seen.add(href)
|
||||
title = a.get_text(strip=True)[:120]
|
||||
snippet_el = li.select_one(".masonry-text, .res-desc")
|
||||
snippet = snippet_el.get_text(strip=True)[:300] if snippet_el else ""
|
||||
entry = {"title": title, "url": href, "snippet": snippet, "engine": "360"}
|
||||
if not _is_junk(entry):
|
||||
results.append(entry)
|
||||
if len(results) >= count:
|
||||
break
|
||||
return results
|
||||
except Exception as e:
|
||||
logger.warning(f"360 search failed: {e}")
|
||||
return []
|
||||
|
||||
|
||||
async def search_bing_batch(queries: List[str], max_per_query: int = 6) -> List[Dict[str, str]]:
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
with open(BATCH_SCRIPT_FILE, "w") as f:
|
||||
f.write(BATCH_SCRIPT)
|
||||
env = os.environ.copy()
|
||||
env["NODE_PATH"] = NODE_MODULES
|
||||
fn = functools.partial(
|
||||
subprocess.run,
|
||||
[NODE_BIN, BATCH_SCRIPT_FILE, json.dumps(queries), str(max_per_query)],
|
||||
capture_output=True, text=True, timeout=120, cwd=PROJECT_ROOT, env=env,
|
||||
)
|
||||
result = await loop.run_in_executor(None, fn)
|
||||
for line in result.stdout.strip().split("\n"):
|
||||
line = line.strip()
|
||||
if line.startswith("["):
|
||||
return json.loads(line)
|
||||
return []
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("Bing batch search timed out")
|
||||
return []
|
||||
except (json.JSONDecodeError, Exception) as e:
|
||||
logger.warning(f"Bing batch search error: {e}")
|
||||
return []
|
||||
all_results = []
|
||||
seen_urls = set()
|
||||
|
||||
for query in queries:
|
||||
loop = asyncio.get_running_loop()
|
||||
bing_task = loop.run_in_executor(None, _search_bing, query, max_per_query)
|
||||
so_task = loop.run_in_executor(None, _search_360, query, max_per_query)
|
||||
bing_results, so_results = await asyncio.gather(bing_task, so_task)
|
||||
|
||||
for entry in bing_results + so_results:
|
||||
url = entry["url"].rstrip("/")
|
||||
if url not in seen_urls:
|
||||
seen_urls.add(url)
|
||||
all_results.append(entry)
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
async def search_bing(query: str, max_results: int = 10) -> List[Dict[str, str]]:
|
||||
return await search_bing_batch([query], max_per_query=max_results)
|
||||
|
||||
|
||||
mcp = FastMCP("trade-search", log_level="WARNING")
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
name="web_search",
|
||||
description="Search the web for companies, buyers, or business information. Returns title, URL, and snippet for each result. Useful for finding potential customers, researching companies, or gathering market intelligence.",
|
||||
)
|
||||
async def web_search(query: str, max_results: int = 10) -> str:
|
||||
results = await search_bing(query, max_results)
|
||||
if not results:
|
||||
return json.dumps({"results": [], "error": None})
|
||||
return json.dumps({"results": results, "error": None})
|
||||
|
||||
|
||||
def main():
|
||||
asyncio.run(mcp.run_stdio_async())
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -3,6 +3,7 @@ from sqlalchemy import select, func
|
||||
from fastapi import HTTPException, Depends
|
||||
from datetime import datetime, date
|
||||
from sqlalchemy import Date
|
||||
from typing import Tuple
|
||||
import logging
|
||||
|
||||
from app.models import UsageLog, SystemConfig, User, Customer, Product
|
||||
@@ -75,7 +76,7 @@ class UsageService:
|
||||
result = await self.db.execute(stmt)
|
||||
return result.scalar() or 0
|
||||
|
||||
async def check_quota(self, user_id: str, action: str, chars: int = 0) -> tuple[bool, str]:
|
||||
async def check_quota(self, user_id: str, action: str, chars: int = 0) -> Tuple[bool, str]:
|
||||
tier = await self.get_tier(user_id)
|
||||
limits = await self.get_limits(tier)
|
||||
limit_key = ACTION_MAP.get(action)
|
||||
|
||||
Reference in New Issue
Block a user