diff --git a/agents/lifecycle.py b/agents/lifecycle.py index d92c142..205ba4a 100644 --- a/agents/lifecycle.py +++ b/agents/lifecycle.py @@ -74,7 +74,7 @@ class ProjectContextMixin: def _json_from_sol(self, router: ModelRouter | None, prompt: str, *, fallback: dict[str, object], project: Project | None = None) -> dict[str, object]: if router is None: return fallback - response = router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, prompt=prompt, model_hint="sol", project=project)) + response = router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, prompt=prompt, project=project)) try: payload = json.loads(response.content) except json.JSONDecodeError as exc: diff --git a/agents/roadmap.py b/agents/roadmap.py index bae71c0..d1d3ba0 100644 --- a/agents/roadmap.py +++ b/agents/roadmap.py @@ -66,7 +66,7 @@ class RoadmapService(ProjectContextMixin): return fallback payload = {"context": self.project_context(project), "roadmap_items": list(project.roadmap_items.values("id", "title", "description", "horizon", "status", "target_action", "value_score", "effort_score", "risk_score", "confidence", "strategic_fit", "technical_fit", "urgency", "composite_score"))} try: - response = self.router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, model_hint="sol", project=project, prompt="Review this existing project roadmap. Return JSON with recommendations: item_id, recommendation, rationale, optional horizon/status. Do not execute work.\n" + json.dumps(payload, default=str))) + response = self.router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, project=project, prompt="Review this existing project roadmap. Return JSON with recommendations: item_id, recommendation, rationale, optional horizon/status. Do not execute work.\n" + json.dumps(payload, default=str))) parsed = json.loads(response.content) return parsed if isinstance(parsed, dict) else fallback except Exception as exc: diff --git a/agents/scenario_lab.py b/agents/scenario_lab.py index 8b0884a..b01830a 100644 --- a/agents/scenario_lab.py +++ b/agents/scenario_lab.py @@ -60,7 +60,7 @@ class ScenarioLabService(ProjectContextMixin): payload = fallback if self.router is not None: try: - response = self.router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, model_hint="sol", project=suite.project, prompt="Design safe Scenario Lab candidates for this existing project. Return JSON with scenarios. Each scenario needs type, injected_condition, expected_invariants, success_criteria, and resource_budget. Do not create executable work.\n" + json.dumps(self.project_context(suite.project), default=str))) + response = self.router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, project=suite.project, prompt="Design safe Scenario Lab candidates for this existing project. Return JSON with scenarios. Each scenario needs type, injected_condition, expected_invariants, success_criteria, and resource_budget. Do not create executable work.\n" + json.dumps(self.project_context(suite.project), default=str))) parsed = json.loads(response.content) if isinstance(parsed, dict) and isinstance(parsed.get("scenarios"), list): payload = parsed diff --git a/agents/venture_discovery.py b/agents/venture_discovery.py index 0602c49..90e8c4f 100644 --- a/agents/venture_discovery.py +++ b/agents/venture_discovery.py @@ -18,7 +18,9 @@ from control_plane.events.bus import EventBus from control_plane.ventures.models import CapabilityPriority, CapabilityStatus, CompanyBoardReview, CompanyCapabilityRequirement, CompanyMandate, CompanyProposal, CompanyProposalStatus, EvidenceTier, ICDecision, ICDecisionType, ICDiligence, ICQuestion, ICResponse, OverlapClassification, PortfolioCapabilityGap, PortfolioICReview, VentureArtifact, VentureCapabilityDemand, VentureCohort, VentureCohortMember, VentureCollision, VentureThesis, VentureThesisFingerprint from graph.models import GraphRun, GraphRunStatus from model_router.providers import extract_json_object +from model_router.policy import model_for_role from model_router.router import ModelCapability, ModelRequestContract, ModelRouter +from research.searxng import SearxngSearchClient, WebPageFetcher PITCH_SECTIONS = ["Company name", "One-line thesis", "Problem", "ICP", "Why now", "Product / service", "Business model", "Pricing", "Route to first customer", "Validation plan", "$50 capital allocation proposal", "Time to first dollar", "Path to $500 net cash", "Competition", "Differentiation", "Build requirements", "Distribution requirements", "Risks", "What would falsify the thesis", "Confidence"] @@ -29,11 +31,14 @@ AI_NATIVE_POLICY = {"preference": "Prefer AI-native opportunities where Artifex class VentureDiscoveryService: - def __init__(self, router: ModelRouter | None = None, bus: EventBus | None = None, web_research_available: bool = False, research_model_hint: str = "luna") -> None: + def __init__(self, router: ModelRouter | None = None, bus: EventBus | None = None, web_research_available: bool = False, research_model_hint: str | None = None, ideation_model_hint: str | None = None, search_client: Any | None = None, page_fetcher: Any | None = None) -> None: self.router = router self.bus = bus or EventBus() self.web_research_available = web_research_available - self.research_model_hint = research_model_hint + self.research_model_hint = research_model_hint or model_for_role("venture_research") + self.ideation_model_hint = ideation_model_hint or model_for_role("venture_ideation") + self.search_client = search_client + self.page_fetcher = page_fetcher or WebPageFetcher() self._last_bounded_map_peak = 0 def create_v0_mandate(self) -> CompanyMandate: @@ -48,7 +53,7 @@ class VentureDiscoveryService: def generate_single_company(self, mandate: CompanyMandate, *, graph_run=None, ideation_index: int | None = None) -> CompanyProposal: payload, source = self._company_payload(mandate, ideation_index=ideation_index) - pitch = self._pitch(payload, fallback=source != "sol") + pitch = self._pitch(payload, fallback=source == "deterministic_fallback") confidence = self._confidence(payload.get("confidence", 0.55)) evidence = self._as_list(payload.get("market_evidence", [])) if not self.web_research_available: @@ -77,19 +82,21 @@ class VentureDiscoveryService: confidence=confidence, status=CompanyProposalStatus.SUBMITTED, pitch=pitch, - metadata={"generation_source": source, "fallback_evidence": source != "sol", "web_research_available": self.web_research_available, "real_spend": 0, "real_customer_outreach": False}, + metadata={"generation_source": source, "fallback_evidence": source == "deterministic_fallback", "web_research_available": self.web_research_available, "real_spend": 0, "real_customer_outreach": False}, evidence_tier=EvidenceTier.TIER_0_THESIS, ) - self._artifact(proposal, mandate, "STANDARDIZED_COMPANY_PITCH", "Standardized Company Pitch", pitch, self.readable_pitch(pitch), "Company Brain/Sol" if source == "sol" else "deterministic_fallback", graph_run=graph_run) + self._artifact(proposal, mandate, "STANDARDIZED_COMPANY_PITCH", "Standardized Company Pitch", pitch, self.readable_pitch(pitch), f"Company Brain/{source}" if source != "deterministic_fallback" else "deterministic_fallback", graph_run=graph_run) self.bus.publish("VENTURE_COMPANY_PROPOSED", payload={"proposal_id": str(proposal.id), "source": source}) return proposal def conduct_market_research(self, proposal: CompanyProposal, *, graph_run=None) -> dict[str, Any]: required = ["competitors", "pricing", "customer_pain", "market_alternatives", "regulatory_platform_risks"] research = {"coverage": {category: False for category in required}, "sources": [], "findings": {}, "unverified_categories": required, "research_available": False} + search_sources = self._searxng_sources(proposal, required) if self.web_research_available else [] + page_corpus = self._page_corpus(search_sources) if search_sources else [] if self.web_research_available and self.router is not None: try: - response = self.router.complete(ModelRequestContract(purpose=ModelCapability.REASONING, model_hint=self.research_model_hint, prompt="Bounded public web market research for exactly one startup pitch. Return JSON with keys: sources (list of {url,title,category,summary}), findings (object keyed by competitors, pricing, customer_pain, market_alternatives, regulatory_platform_risks), and coverage (object with each required key true/false). Do not fabricate URLs. If a category has no source, mark coverage false. Pitch: " + json.dumps(proposal.pitch, default=str))) + response = self.router.complete(ModelRequestContract(purpose=ModelCapability.REASONING, model_hint=self.research_model_hint, prompt="Bounded public web market research for exactly one startup pitch. Return JSON with keys: sources (list of {url,title,category,summary}), findings (object keyed by competitors, pricing, customer_pain, market_alternatives, regulatory_platform_risks), and coverage (object with each required key true/false). Use only the provided SearXNG search context and fetched page excerpts for source URLs; do not fabricate URLs. If a category has no source, mark coverage false. Search context: " + json.dumps(search_sources, default=str) + " Page excerpts: " + json.dumps(page_corpus, default=str) + " Pitch: " + json.dumps(proposal.pitch, default=str))) parsed = extract_json_object(response.content) if isinstance(parsed, dict): research = self._normalize_research(parsed, required) @@ -97,6 +104,10 @@ class VentureDiscoveryService: research["provider"] = self.research_model_hint except Exception as exc: research["failure"] = str(exc) + if search_sources: + research = self._merge_search_sources(research, search_sources, required) + if page_corpus: + research = {**research, "page_corpus": page_corpus, "page_fetch_count": len(page_corpus)} if not research.get("sources"): research = self._missing_research(required, research.get("failure", "public web research unavailable or returned no source-linked evidence")) source_categories = {str(source.get("category", "")) for source in research.get("sources", []) if source.get("url")} @@ -108,14 +119,14 @@ class VentureDiscoveryService: coverage_ratio = (len(required) - len(research["unverified_categories"])) / len(required) proposal.confidence = round(min(float(proposal.confidence), 0.45 + 0.35 * coverage_ratio), 2) proposal.market_evidence = [*self._as_list(proposal.market_evidence), *research.get("sources", []), {"type": "research_coverage", "source": "venture_research", "summary": f"Source-linked research coverage: {round(coverage_ratio * 100)}%", "coverage": coverage, "unverified_categories": research["unverified_categories"]}] - proposal.metadata = {**proposal.metadata, "research": {"coverage_ratio": coverage_ratio, "unverified_categories": research["unverified_categories"], "source_count": len(research.get("sources", [])), "provider": research.get("provider", "none")}} + proposal.metadata = {**proposal.metadata, "research": {"coverage_ratio": coverage_ratio, "unverified_categories": research["unverified_categories"], "source_count": len(research.get("sources", [])), "page_fetch_count": research.get("page_fetch_count", 0), "provider": research.get("provider", "none"), "search_provider": research.get("search_provider", "none")}} proposal.evidence_tier = EvidenceTier.TIER_1_PUBLIC_EVIDENCE if coverage_ratio > 0 else EvidenceTier.TIER_0_THESIS if proposal.thesis: proposal.thesis.evidence_tier = proposal.evidence_tier proposal.thesis.save(update_fields=["evidence_tier", "updated_at"]) proposal.pitch = {**proposal.pitch, "Research evidence": research} proposal.save(update_fields=["confidence", "market_evidence", "metadata", "pitch", "evidence_tier", "updated_at"]) - self._artifact(proposal, proposal.mandate, "MARKET_RESEARCH", "Bounded Public Market Research", research, self._readable_research(research), "Luna/web research" if research.get("provider") else "research_gap", graph_run=graph_run) + self._artifact(proposal, proposal.mandate, "MARKET_RESEARCH", "Bounded Public Market Research", research, self._readable_research(research), f"{research.get('provider', 'search')}/web research" if research.get("sources") else "research_gap", graph_run=graph_run) return research def board_review(self, proposal: CompanyProposal, *, graph_run=None) -> CompanyBoardReview: @@ -466,10 +477,10 @@ class VentureDiscoveryService: if self.router is not None: try: slot = f" Independent cohort slot: {ideation_index}. Do not use or imitate other cohort ideas; no other ideas are visible." if ideation_index else "" - response = self.router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, model_hint="sol", prompt="Generate exactly ONE startup idea for Venture Discovery V0. Return a single JSON object, not a list. Respect no spend and no outreach in V0. Keep the $50 to $500 in 30 days mandate. Prefer, but do not require, AI-native businesses where Artifex can deliver most value using agents/local inference: AI wrapper platforms, AI-enabled services, AI infrastructure/developer tools, AI intelligence products, or SMB/enterprise workflow automation. Do not let AI novelty outweigh customer demand. Penalize generic audits, emergency fix services, one-off consulting, Shopify/ecommerce concentration, and identical outbound-led service models unless exceptional. Include title, one_line_thesis, description, problem, target_customer, proposed_solution, business_model, pricing_hypothesis, acquisition_strategy, validation_plan, capital_requested, time_to_first_dollar_estimate, expected_margin, build_complexity, market_evidence, differentiation, major_risks, confidence." + slot + " Mandate: " + json.dumps({"objective": mandate.objective, "constraints": mandate.constraints, "optimization_targets": mandate.optimization_targets, "ai_native_policy": AI_NATIVE_POLICY}))) + response = self.router.complete(ModelRequestContract(purpose=ModelCapability.PLANNING, model_hint=self.ideation_model_hint, prompt="Generate exactly ONE startup idea for Venture Discovery V0. Return a single JSON object, not a list. Respect no spend and no outreach in V0. Keep the $50 to $500 in 30 days mandate. Prefer, but do not require, AI-native businesses where Artifex can deliver most value using agents/local inference: AI wrapper platforms, AI-enabled services, AI infrastructure/developer tools, AI intelligence products, or SMB/enterprise workflow automation. Do not let AI novelty outweigh customer demand. Penalize generic audits, emergency fix services, one-off consulting, Shopify/ecommerce concentration, and identical outbound-led service models unless exceptional. Include title, one_line_thesis, description, problem, target_customer, proposed_solution, business_model, pricing_hypothesis, acquisition_strategy, validation_plan, capital_requested, time_to_first_dollar_estimate, expected_margin, build_complexity, market_evidence, differentiation, major_risks, confidence." + slot + " Mandate: " + json.dumps({"objective": mandate.objective, "constraints": mandate.constraints, "optimization_targets": mandate.optimization_targets, "ai_native_policy": AI_NATIVE_POLICY}))) parsed = extract_json_object(response.content) if isinstance(parsed, dict) and parsed.get("title"): - return self._normalize_payload(parsed), "sol" + return self._normalize_payload(parsed), self.ideation_model_hint except Exception: pass payload = self._fallback_company_payload() @@ -495,6 +506,49 @@ class VentureDiscoveryService: coverage = {category: bool(dict(payload.get("coverage", {})).get(category)) for category in required} return {"coverage": coverage, "sources": sources, "findings": dict(payload.get("findings", {})), "unverified_categories": []} + def _searxng_sources(self, proposal: CompanyProposal, required: list[str]) -> list[dict[str, Any]]: + client = self.search_client or SearxngSearchClient.from_resources() + if client is None: + return [] + sources = [] + query_terms = { + "competitors": "competitors alternatives", + "pricing": "pricing cost", + "customer_pain": "customer pain problem forum", + "market_alternatives": "alternatives tools services", + "regulatory_platform_risks": "regulatory platform risk compliance", + } + for category in required: + query = f"{proposal.title} {proposal.target_customer} {query_terms.get(category, category)}" + try: + sources.extend(client.search(query, category=category, limit=3)) + except Exception: + continue + return sources[:15] + + def _page_corpus(self, search_sources: list[dict[str, Any]]) -> list[dict[str, Any]]: + if self.page_fetcher is None: + return [] + try: + return self.page_fetcher.fetch_many(search_sources, max_pages=8) + except Exception: + return [] + + def _merge_search_sources(self, research: dict[str, Any], search_sources: list[dict[str, Any]], required: list[str]) -> dict[str, Any]: + seen = {source.get("url") for source in research.get("sources", [])} + merged_sources = [*research.get("sources", [])] + for source in search_sources: + if source.get("url") in seen: + continue + seen.add(source.get("url")) + merged_sources.append(source) + coverage = {category: bool(dict(research.get("coverage", {})).get(category)) for category in required} + for source in merged_sources: + category = str(source.get("category", "")) + if category in coverage and source.get("url"): + coverage[category] = True + return {**research, "coverage": coverage, "sources": merged_sources, "research_available": True, "search_provider": "searxng"} + def _missing_research(self, required: list[str], reason: str) -> dict[str, Any]: return {"coverage": {category: False for category in required}, "sources": [], "findings": {}, "unverified_categories": required, "research_available": False, "failure": reason} diff --git a/control_plane/resources/management/commands/model_provider_health.py b/control_plane/resources/management/commands/model_provider_health.py index 1401211..3d27793 100644 --- a/control_plane/resources/management/commands/model_provider_health.py +++ b/control_plane/resources/management/commands/model_provider_health.py @@ -5,18 +5,21 @@ from django.utils import timezone from control_plane.resources.models import Resource from model_router.providers import QwenProvider, SolProvider +from research.searxng import SearxngSearchClient class Command(BaseCommand): help = "Check configured model provider health without crashing the control plane." def handle(self, *args, **options): - resources = sorted(Resource.objects.filter(is_active=True, kind="MODEL"), key=self.sort_key) + resources = sorted(Resource.objects.filter(is_active=True, provider__in=["local_inference", "opencode", "searxng"]), key=self.sort_key) for resource in resources: if resource.provider == "opencode": status = SolProvider(resource).health() elif resource.provider == "local_inference": status = QwenProvider(resource).health() + elif resource.provider == "searxng": + status = SearxngSearchClient(endpoint_url=str(resource.config.get("endpoint_url", "http://127.0.0.1:8080"))).health() else: status = "UNAVAILABLE" resource.health_status = status @@ -28,5 +31,7 @@ class Command(BaseCommand): model_key = str(resource.config.get("model_key") or "").lower() if resource.provider == "local_inference": model_key = "qwen" - order = {"qwen": 0, "sol": 1, "terra": 2, "luna": 3} + if resource.provider == "searxng": + model_key = "searxng" + order = {"qwen": 0, "sol": 1, "terra": 2, "luna": 3, "searxng": 4} return order.get(model_key, 100), resource.name diff --git a/control_plane/resources/management/commands/seed_spark_resources.py b/control_plane/resources/management/commands/seed_spark_resources.py index 1436b74..ba739ac 100644 --- a/control_plane/resources/management/commands/seed_spark_resources.py +++ b/control_plane/resources/management/commands/seed_spark_resources.py @@ -93,4 +93,17 @@ class Command(BaseCommand): }, }, ) + Resource.objects.update_or_create( + name=os.environ.get("ARTIFEX_SEARXNG_RESOURCE_NAME", "SearXNG Search"), + defaults={ + "kind": ResourceKind.COMPUTE, + "provider": "searxng", + "compute": spark, + "roles": ["WEB_RESEARCH", "MARKET_RESEARCH", "SEARCH"], + "config": { + "endpoint_url": os.environ.get("ARTIFEX_SEARXNG_ENDPOINT_URL", "http://127.0.0.1:8080"), + "timeout_seconds": int(os.environ.get("ARTIFEX_SEARXNG_TIMEOUT_SECONDS", "20")), + }, + }, + ) self.stdout.write(self.style.SUCCESS("Seeded Spark model resources.")) diff --git a/model_router/policy.py b/model_router/policy.py new file mode 100644 index 0000000..3217958 --- /dev/null +++ b/model_router/policy.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import os + + +DEFAULT_MODEL_POLICY = { + "planning": "sol", + "project_brain": "sol", + "archaeology_interpretation": "sol", + "agent_design": "sol", + "escalation": "sol", + "coding": "qwen", + "review": "qwen", + "reasoning": "qwen", + "venture_ideation": "sol", + "venture_research": "luna", + "venture_portfolio_ic": "terra", +} + + +ENV_BY_ROLE = { + "planning": "ARTIFEX_PLANNING_MODEL", + "project_brain": "ARTIFEX_PROJECT_BRAIN_MODEL", + "archaeology_interpretation": "ARTIFEX_ARCHAEOLOGY_MODEL", + "agent_design": "ARTIFEX_AGENT_DESIGN_MODEL", + "escalation": "ARTIFEX_ESCALATION_MODEL", + "coding": "ARTIFEX_CODING_MODEL", + "review": "ARTIFEX_REVIEW_MODEL", + "reasoning": "ARTIFEX_REASONING_MODEL", + "venture_ideation": "ARTIFEX_VENTURE_IDEATION_MODEL", + "venture_research": "ARTIFEX_VENTURE_RESEARCH_MODEL", + "venture_portfolio_ic": "ARTIFEX_VENTURE_PORTFOLIO_IC_MODEL", +} + + +PURPOSE_TO_ROLE = { + "PROJECT_BRAIN": "project_brain", + "PLANNING": "planning", + "ARCHAEOLOGY_INTERPRETATION": "archaeology_interpretation", + "AGENT_DESIGN": "agent_design", + "ESCALATION": "escalation", + "CODING": "coding", + "REVIEW": "review", + "REASONING": "reasoning", +} + + +def model_for_role(role: str) -> str: + normalized = role.lower() + env_name = ENV_BY_ROLE.get(normalized) + if env_name: + configured = os.environ.get(env_name) + if configured: + return configured.strip().lower() + return DEFAULT_MODEL_POLICY.get(normalized, DEFAULT_MODEL_POLICY["reasoning"]) + + +def model_for_purpose(purpose: str) -> str: + role = PURPOSE_TO_ROLE.get(str(purpose).upper(), "reasoning") + return model_for_role(role) diff --git a/model_router/router.py b/model_router/router.py index 380b021..b41a9c0 100644 --- a/model_router/router.py +++ b/model_router/router.py @@ -11,6 +11,7 @@ from django.utils import timezone from control_plane.agents.models import AgentVersion from control_plane.projects.models import Project from control_plane.resources.models import ModelRequest, Resource +from model_router.policy import model_for_purpose class ModelCapability(StrEnum): @@ -53,18 +54,7 @@ class ModelRouter: self.persist_requests = persist_requests def route(self, purpose: str) -> str: - normalized = purpose.upper() - if normalized in { - ModelCapability.PROJECT_BRAIN, - ModelCapability.PLANNING, - ModelCapability.ARCHAEOLOGY_INTERPRETATION, - "PLANNING", - "ARCHAEOLOGY_INTERPRETATION", - "AGENT_DESIGN", - "ESCALATION", - }: - return "sol" - return "qwen" + return model_for_purpose(str(purpose)) def complete(self, request: ModelRequestContract) -> ModelResponseContract: provider_key = request.model_hint or self.route(request.purpose) diff --git a/project_brain/sol.py b/project_brain/sol.py index 7f27f52..92076b5 100644 --- a/project_brain/sol.py +++ b/project_brain/sol.py @@ -19,10 +19,10 @@ class SolProjectBrain: def plan_project(self, prompt: PlanningPrompt) -> str: response = self.router.complete( - ModelRequestContract(purpose="PLANNING", prompt=prompt.idea, model_hint="sol") + ModelRequestContract(purpose="PLANNING", prompt=prompt.idea) ) return response.content def create_project_plan_contract(self, prompt: PlanningPrompt) -> ProjectPlanContract: - response = self.router.complete(ModelRequestContract(purpose="PLANNING", prompt=prompt.idea, model_hint="sol")) + response = self.router.complete(ModelRequestContract(purpose="PLANNING", prompt=prompt.idea)) return parse_project_plan_response(response.content) diff --git a/research/__init__.py b/research/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/research/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/research/searxng.py b/research/searxng.py new file mode 100644 index 0000000..d24b48e --- /dev/null +++ b/research/searxng.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import json +import re +import urllib.parse +import urllib.request +from dataclasses import dataclass +from html import unescape +from typing import Any + +from control_plane.resources.models import Resource + + +@dataclass +class SearxngSearchClient: + endpoint_url: str + timeout_seconds: int = 20 + + @classmethod + def from_resources(cls) -> "SearxngSearchClient | None": + resource = Resource.objects.filter(is_active=True, provider="searxng").first() + if resource is None: + return None + return cls(endpoint_url=str(resource.config.get("endpoint_url", "http://127.0.0.1:8080")), timeout_seconds=int(resource.config.get("timeout_seconds", 20))) + + def health(self) -> str: + try: + with urllib.request.urlopen(self.endpoint_url.rstrip("/") + "/", timeout=5) as response: + return "AVAILABLE" if 200 <= response.status < 500 else "DEGRADED" + except Exception: + return "UNAVAILABLE" + + def search(self, query: str, *, category: str = "general", limit: int = 5) -> list[dict[str, Any]]: + params = urllib.parse.urlencode({"q": query, "format": "json", "categories": "general", "language": "en"}) + url = self.endpoint_url.rstrip("/") + "/search?" + params + request = urllib.request.Request(url, headers={"Accept": "application/json"}, method="GET") + with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response: + payload = json.loads(response.read().decode("utf-8")) + results = [] + for item in payload.get("results", [])[:limit]: + if not isinstance(item, dict) or not item.get("url"): + continue + results.append( + { + "type": "public_web", + "source": "searxng", + "url": str(item["url"]), + "title": str(item.get("title", "")), + "category": category, + "summary": str(item.get("content", item.get("snippet", ""))), + "fallback_evidence": False, + } + ) + return results + + +@dataclass +class WebPageFetcher: + timeout_seconds: int = 12 + max_bytes: int = 200_000 + max_text_chars: int = 4_000 + + def fetch_many(self, sources: list[dict[str, Any]], *, max_pages: int = 8) -> list[dict[str, Any]]: + pages = [] + seen = set() + for source in sources: + url = str(source.get("url", "")) + if url in seen or not self._allowed_url(url): + continue + seen.add(url) + page = self.fetch(url) + if page is None: + continue + pages.append({**page, "category": source.get("category", ""), "source_title": source.get("title", "")}) + if len(pages) >= max_pages: + break + return pages + + def fetch(self, url: str) -> dict[str, Any] | None: + if not self._allowed_url(url): + return None + request = urllib.request.Request(url, headers={"User-Agent": "ArtifexResearchBot/0.1 (+local bounded research)"}, method="GET") + try: + with urllib.request.urlopen(request, timeout=self.timeout_seconds) as response: + content_type = response.headers.get("Content-Type", "") + raw = response.read(self.max_bytes) + except Exception: + return None + text = raw.decode("utf-8", errors="ignore") + if "html" in content_type.lower() or " bool: + parsed = urllib.parse.urlparse(url) + return parsed.scheme in {"http", "https"} and bool(parsed.netloc) + + def _html_to_text(self, html: str) -> str: + html = re.sub(r"(?is)<(script|style|noscript|svg).*?", " ", html) + html = re.sub(r"(?s)", " ", html) + html = re.sub(r"(?is)", "\n", html) + html = re.sub(r"(?is)", "\n", html) + text = re.sub(r"(?s)<[^>]+>", " ", html) + return self._clean_text(unescape(text)) + + def _clean_text(self, text: str) -> str: + lines = [re.sub(r"\s+", " ", line).strip() for line in text.splitlines()] + return "\n".join(line for line in lines if line) diff --git a/tests/test_interfaces.py b/tests/test_interfaces.py index 714e55a..476e2c4 100644 --- a/tests/test_interfaces.py +++ b/tests/test_interfaces.py @@ -15,6 +15,15 @@ def test_model_router_keeps_sol_for_planning_and_qwen_for_work() -> None: assert router.route("review") == "qwen" +def test_model_router_uses_env_model_policy(monkeypatch) -> None: + monkeypatch.setenv("ARTIFEX_PLANNING_MODEL", "terra") + monkeypatch.setenv("ARTIFEX_CODING_MODEL", "qwen") + router = ModelRouter() + + assert router.route("planning") == "terra" + assert router.route("coding") == "qwen" + + def test_langgraph_runtime_is_hidden_behind_runtime_interface() -> None: runtime = LangGraphRuntime() diff --git a/tests/test_venture_discovery_v0.py b/tests/test_venture_discovery_v0.py index ce69b61..bec2f99 100644 --- a/tests/test_venture_discovery_v0.py +++ b/tests/test_venture_discovery_v0.py @@ -71,10 +71,44 @@ class ResearchProvider(ModelProvider): return "AVAILABLE" +class FakeSearchClient: + def search(self, query: str, *, category: str = "general", limit: int = 5) -> list[dict[str, object]]: + return [ + { + "type": "public_web", + "source": "searxng", + "url": f"https://search.example/{category}", + "title": f"{category} source", + "category": category, + "summary": f"Search result for {query}", + "fallback_evidence": False, + } + ] + + +class FakePageFetcher: + def fetch_many(self, sources: list[dict[str, object]], *, max_pages: int = 8) -> list[dict[str, object]]: + return [{"url": source["url"], "category": source["category"], "source_title": source["title"], "text": f"Fetched page text for {source['category']} with pricing, alternatives, and customer pain evidence.", "fetched_chars": 80} for source in sources[:max_pages]] + + def service(*, web_research_available: bool = False, complete_research: bool = True) -> VentureDiscoveryService: return VentureDiscoveryService(ModelRouter({"sol": SolOneCompanyProvider(), "luna": ResearchProvider(complete_research)}), web_research_available=web_research_available) +def test_venture_model_policy_overrides_ideation_and_research(monkeypatch) -> None: + monkeypatch.setenv("ARTIFEX_VENTURE_IDEATION_MODEL", "terra") + monkeypatch.setenv("ARTIFEX_VENTURE_RESEARCH_MODEL", "qwen") + router = ModelRouter({"terra": SolOneCompanyProvider(), "qwen": ResearchProvider()}) + svc = VentureDiscoveryService(router, web_research_available=True) + + proposal = svc.generate_single_company(svc.create_v0_mandate()) + research = svc.conduct_market_research(proposal) + + assert proposal.metadata["generation_source"] == "terra" + assert proposal.metadata["fallback_evidence"] is False + assert research["provider"] == "qwen" + + def test_company_proposal_lifecycle_mandate_limits_and_pitch_schema() -> None: svc = service() mandate = svc.create_v0_mandate() @@ -145,6 +179,20 @@ def test_ic_scoring_decision_capability_gap_and_memo_artifact() -> None: assert "Capability gap" in memo.content +def test_searxng_search_and_page_fetch_sources_can_supply_market_research_evidence() -> None: + svc = VentureDiscoveryService(ModelRouter({}), web_research_available=True, search_client=FakeSearchClient(), page_fetcher=FakePageFetcher()) + proposal = svc.generate_single_company(svc.create_v0_mandate()) + + research = svc.conduct_market_research(proposal) + + assert research["search_provider"] == "searxng" + assert len(research["sources"]) == 5 + assert research["page_fetch_count"] == 5 + assert "Fetched page text" in research["page_corpus"][0]["text"] + assert proposal.metadata["research"]["page_fetch_count"] == 5 + assert all(research["coverage"].values()) + + def test_fallback_is_marked_and_reduces_confidence_when_research_unavailable() -> None: svc = VentureDiscoveryService(web_research_available=False) proposal = svc.generate_single_company(svc.create_v0_mandate())