✨ feat: add skip_llms_txt config option to bypass llms.txt detection

- Add skip_llms_txt config option (default: False)
- Validate value is boolean, warn and default to False if not
- Support in both sync and async scraping modes
- Add 17 tests for config, behavior, and edge cases

tests/test_config_validation.py

@@ -307,6 +307,30 @@ class TestConfigValidation(unittest.TestCase):
         # Should be valid
         self.assertEqual(config.get('llms_txt_url'), 'https://example.com/llms-full.txt')
 
+    def test_config_with_skip_llms_txt(self):
+        """Test config validation accepts skip_llms_txt"""
+        config = {
+            'name': 'test',
+            'base_url': 'https://example.com/docs',
+            'skip_llms_txt': True
+        }
+
+        errors, warnings = validate_config(config)
+        self.assertEqual(errors, [])
+        self.assertTrue(config.get('skip_llms_txt'))
+
+    def test_config_with_skip_llms_txt_false(self):
+        """Test config validation accepts skip_llms_txt as False"""
+        config = {
+            'name': 'test',
+            'base_url': 'https://example.com/docs',
+            'skip_llms_txt': False
+        }
+
+        errors, warnings = validate_config(config)
+        self.assertEqual(errors, [])
+        self.assertFalse(config.get('skip_llms_txt'))
+
 
 if __name__ == '__main__':
     unittest.main()