diff --git a/cli/llms_txt_detector.py b/cli/llms_txt_detector.py index 6a4a832..688fdb7 100644 --- a/cli/llms_txt_detector.py +++ b/cli/llms_txt_detector.py @@ -2,7 +2,7 @@ # ABOUTME: Supports llms-full.txt, llms.txt, and llms-small.txt variants import requests -from typing import Optional, Dict +from typing import Optional, Dict, List from urllib.parse import urlparse class LlmsTxtDetector: @@ -35,6 +35,28 @@ class LlmsTxtDetector: return None + def detect_all(self) -> List[Dict[str, str]]: + """ + Detect all available llms.txt variants. + + Returns: + List of dicts with 'url' and 'variant' keys for each found variant + """ + found_variants = [] + + for filename, variant in self.VARIANTS: + parsed = urlparse(self.base_url) + root_url = f"{parsed.scheme}://{parsed.netloc}" + url = f"{root_url}/{filename}" + + if self._check_url_exists(url): + found_variants.append({ + 'url': url, + 'variant': variant + }) + + return found_variants + def _check_url_exists(self, url: str) -> bool: """Check if URL returns 200 status""" try: diff --git a/tests/test_llms_txt_detector.py b/tests/test_llms_txt_detector.py index fc9481e..d5934d8 100644 --- a/tests/test_llms_txt_detector.py +++ b/tests/test_llms_txt_detector.py @@ -50,3 +50,28 @@ def test_url_parsing_with_complex_paths(): timeout=5, allow_redirects=True ) + +def test_detect_all_variants(): + """Test detecting all llms.txt variants""" + detector = LlmsTxtDetector("https://hono.dev/docs") + + with patch('cli.llms_txt_detector.requests.head') as mock_head: + # Mock responses for different variants + def mock_response(url, **kwargs): + response = Mock() + # All 3 variants exist for Hono + if 'llms-full.txt' in url or 'llms.txt' in url or 'llms-small.txt' in url: + response.status_code = 200 + else: + response.status_code = 404 + return response + + mock_head.side_effect = mock_response + + variants = detector.detect_all() + + assert len(variants) == 3 + assert any(v['variant'] == 'full' for v in variants) + assert any(v['variant'] == 'standard' for v in variants) + assert any(v['variant'] == 'small' for v in variants) + assert all('url' in v for v in variants)