Source code for biblicus.api.web
"""
Typed requests and functions for the web extraction and bounded transform commands.
Covers ``extract url-text``, ``extract web-metadata``, ``extract article-text`` and
``summarize map-reduce``.
"""
from __future__ import annotations
from typing import Any, Dict, List, Literal, Optional
from pydantic import Field, model_validator
from .. import url_text as url_text_module
from .. import web_reference_metadata as web_reference_metadata_module
from ..url_text import URL_TEXT_DEFAULT_TIMEOUT_SECONDS
from ..web_transform import ArticleInput, SourceInput, TransformResult, transform_source
from ._common import ApiRequestModel, ApiResponseModel
[docs]
class UrlTextRequest(ApiRequestModel):
"""
Request to extract readable text from a source URL.
:ivar source_uri: Source URL.
:vartype source_uri: str
:ivar reference_title: Optional reference title context.
:vartype reference_title: str
:ivar timeout_seconds: Per-attempt fetch timeout in seconds.
:vartype timeout_seconds: float
"""
source_uri: str = Field(min_length=1)
reference_title: str = ""
timeout_seconds: float = Field(default=URL_TEXT_DEFAULT_TIMEOUT_SECONDS, gt=0)
[docs]
class UrlTextResponse(ApiResponseModel):
"""
Result of a URL text extraction.
Fields that the extraction did not produce are absent from the serialized JSON, so a
failed extraction omits ``method``, ``grobid`` and ``structured``.
:ivar status: ``ok`` or ``failed``.
:vartype status: str
:ivar source_kind: ``web``, ``pdf`` or ``youtube``.
:vartype source_kind: str
:ivar strategy: Fetch strategy that was applied.
:vartype strategy: str
:ivar text: Extracted plain text.
:vartype text: str
:ivar markdown: Extracted markdown.
:vartype markdown: str
:ivar title: Extracted title or None.
:vartype title: str or None
:ivar content_type: Fetched content type or None.
:vartype content_type: str or None
:ivar method: Extraction method or None.
:vartype method: str or None
:ivar grobid: Scholarly parse payload or None.
:vartype grobid: dict[str, Any] or None
:ivar structured: Structured web payload or None.
:vartype structured: dict[str, Any] or None
:ivar prompt_version: Extraction contract version.
:vartype prompt_version: str
:ivar attempts: Ordered fetch attempts.
:vartype attempts: list[dict[str, Any]]
:ivar error: Structured failure or None.
:vartype error: dict[str, Any] or None
"""
status: Literal["ok", "failed"]
source_kind: Literal["web", "pdf", "youtube"]
strategy: str
text: str
markdown: str
title: Optional[str]
content_type: Optional[str]
method: Optional[str] = None
grobid: Optional[Dict[str, Any]] = None
structured: Optional[Dict[str, Any]] = None
prompt_version: str
attempts: List[Dict[str, Any]]
error: Optional[Dict[str, Any]]
[docs]
class UrlTextFailureDetails(ApiResponseModel):
"""
Diagnostics a failed URL text extraction carries in the command failure ``details``.
:ivar source_kind: ``web``, ``pdf`` or ``youtube``.
:vartype source_kind: str
:ivar strategy: Fetch strategy that was applied.
:vartype strategy: str
:ivar content_type: Fetched content type or None.
:vartype content_type: str or None
:ivar prompt_version: Extraction contract version.
:vartype prompt_version: str
:ivar attempts: Ordered fetch attempts made before the failure.
:vartype attempts: list[dict[str, Any]]
"""
source_kind: Literal["web", "pdf", "youtube"]
strategy: str
content_type: Optional[str]
prompt_version: str
attempts: List[Dict[str, Any]]
[docs]
@classmethod
def from_response(cls, response: "UrlTextResponse") -> "UrlTextFailureDetails":
"""
Copy the diagnostics out of a failed response.
:param response: Failed URL text response.
:type response: UrlTextResponse
:return: The diagnostics.
:rtype: UrlTextFailureDetails
"""
return cls(
source_kind=response.source_kind,
strategy=response.strategy,
content_type=response.content_type,
prompt_version=response.prompt_version,
attempts=response.attempts,
)
[docs]
class WebMetadataRequest(ApiRequestModel):
"""
Request to extract reference metadata from a page URL or supplied HTML.
:ivar source_uri: Page URL; fetched when ``html_content`` is absent.
:vartype source_uri: str
:ivar html_content: Supplied HTML document, which takes precedence over fetching.
:vartype html_content: str
:ivar reference_title: Optional title hint for model enrichment.
:vartype reference_title: str
:ivar use_llm_fallback: True to enrich sparse heuristics with a model.
:vartype use_llm_fallback: bool
:ivar model: Model identifier for enrichment.
:vartype model: str
:ivar timeout_seconds: Fetch timeout in seconds.
:vartype timeout_seconds: float
"""
source_uri: str = ""
html_content: str = ""
reference_title: str = ""
use_llm_fallback: bool = False
model: str = ""
timeout_seconds: float = Field(default=URL_TEXT_DEFAULT_TIMEOUT_SECONDS, gt=0)
[docs]
@model_validator(mode="after")
def require_source_uri_or_html_content(self) -> "WebMetadataRequest":
"""
Require a page URL or supplied HTML.
:return: The validated request.
:rtype: WebMetadataRequest
:raises ValueError: If both ``source_uri`` and ``html_content`` are blank.
"""
if not self.source_uri.strip() and not self.html_content.strip():
raise ValueError("Provide source_uri and/or html_content.")
return self
[docs]
class WebMetadataResponse(ApiResponseModel):
"""
Reference metadata extracted from a web page.
:ivar status: Always ``ok``; failures are raised as errors.
:vartype status: str
:ivar schemaVersion: Metadata contract version.
:vartype schemaVersion: str
:ivar sourceUri: Page URL used for provenance.
:vartype sourceUri: str
:ivar title: Page title or None.
:vartype title: str or None
:ivar subtitle: Page subtitle or None.
:vartype subtitle: str or None
:ivar authors: Author names.
:vartype authors: list[str]
:ivar authorsStructured: Structured author records.
:vartype authorsStructured: list[dict[str, Any]]
:ivar publicationDate: Normalized publication date or None.
:vartype publicationDate: str or None
:ivar publicationDateRaw: Publication date as found or None.
:vartype publicationDateRaw: str or None
:ivar updatedAt: Last update timestamp or None.
:vartype updatedAt: str or None
:ivar structured: Structured web payload.
:vartype structured: dict[str, Any]
:ivar method: Extraction method.
:vartype method: str
:ivar layers: Heuristic layers that contributed.
:vartype layers: list[Any]
:ivar rawMetadata: Raw page metadata.
:vartype rawMetadata: dict[str, Any]
:ivar warnings: Extraction warnings.
:vartype warnings: list[Any]
:ivar referenceCandidates: Candidate cited references.
:vartype referenceCandidates: list[Any]
:ivar citationCount: Number of structured citations.
:vartype citationCount: int
"""
status: Literal["ok"]
schemaVersion: str
sourceUri: str
title: Optional[str]
subtitle: Optional[str]
authors: List[str]
authorsStructured: List[Dict[str, Any]]
publicationDate: Optional[str]
publicationDateRaw: Optional[str]
updatedAt: Optional[str]
structured: Dict[str, Any]
method: str
layers: List[Any]
rawMetadata: Dict[str, Any]
warnings: List[Any]
referenceCandidates: List[Any]
citationCount: int
[docs]
class SummarizeMapReduceRequest(SourceInput):
"""
Request to map-reduce one bounded metadata field from extracted source text.
:ivar task: Metadata field to generate.
:vartype task: str
:ivar model: Pinned model identifier.
:vartype model: str
"""
task: Literal["title", "subtitle", "summary"]
model: str = Field(min_length=1)
ArticleTextRequest = ArticleInput
ArticleTextResponse = TransformResult
SummarizeMapReduceResponse = TransformResult
[docs]
def extract_url_text(request: UrlTextRequest) -> UrlTextResponse:
"""
Extract readable text from a source URL using the source-aware fetch strategy.
:param request: Validated request.
:type request: UrlTextRequest
:return: Extraction result, with ``status`` ``failed`` and a structured error on failure.
:rtype: UrlTextResponse
"""
payload = url_text_module.extract_url_text(
source_uri=request.source_uri,
reference_title=request.reference_title,
timeout_seconds=request.timeout_seconds,
)
return UrlTextResponse.model_validate(payload)
[docs]
def extract_web_metadata(request: WebMetadataRequest) -> WebMetadataResponse:
"""
Extract reference metadata from supplied HTML or a fetched page.
:param request: Validated request.
:type request: WebMetadataRequest
:return: Reference metadata.
:rtype: WebMetadataResponse
:raises ValueError: If extraction rejects the supplied page.
"""
source_uri = request.source_uri.strip()
html_content = request.html_content.strip()
if html_content:
payload = web_reference_metadata_module.extract_web_reference_metadata_from_html(
html_content,
source_uri=source_uri or web_reference_metadata_module.LOCAL_HTML_SOURCE_URI,
reference_title=request.reference_title,
use_llm_fallback=request.use_llm_fallback,
model=request.model,
)
else:
payload = web_reference_metadata_module.extract_web_reference_metadata_from_url(
source_uri,
reference_title=request.reference_title,
use_llm_fallback=request.use_llm_fallback,
model=request.model,
timeout_seconds=request.timeout_seconds,
)
return WebMetadataResponse.model_validate({"status": "ok", **payload})
[docs]
def extract_article_text(request: ArticleTextRequest) -> ArticleTextResponse:
"""
Select grounded article paragraphs from extracted source text.
:param request: Validated article input.
:type request: ArticleInput
:return: Transform result, with ``status`` ``failed`` and a sanitized error on failure.
:rtype: TransformResult
"""
return transform_source(request, article=True, task=None, model=None)
[docs]
def summarize_map_reduce(request: SummarizeMapReduceRequest) -> SummarizeMapReduceResponse:
"""
Map source evidence and reduce one metadata field.
:param request: Validated request.
:type request: SummarizeMapReduceRequest
:return: Transform result, with ``status`` ``failed`` and a sanitized error on failure.
:rtype: TransformResult
"""
source = SourceInput.model_validate(request.model_dump(exclude={"task", "model"}))
return transform_source(source, article=False, task=request.task, model=request.model)