Skip to content

Commit 473f237

Browse files
fix: Add missing type annotations for manifest validation test
- Add type hints for global variables and function return types - Fix mypy errors to ensure type safety - Maintain comprehensive logging functionality Co-Authored-By: AJ Steers <[email protected]>
1 parent 9465284 commit 473f237

File tree

1 file changed

+48
-43
lines changed

1 file changed

+48
-43
lines changed

unit_tests/sources/declarative/test_manifest_registry_validation.py

Lines changed: 48 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,18 @@
2020
ValidateAdheresToSchema,
2121
)
2222

23-
2423
logger = logging.getLogger(__name__)
2524

26-
EXCLUDED_CONNECTORS = [
27-
]
25+
EXCLUDED_CONNECTORS: List[Tuple[str, str]] = []
2826

2927
CONNECTOR_REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json"
30-
MANIFEST_URL_TEMPLATE = "https://connectors.airbyte.com/files/metadata/airbyte/{connector_name}/latest/manifest.yaml"
28+
MANIFEST_URL_TEMPLATE = (
29+
"https://connectors.airbyte.com/files/metadata/airbyte/{connector_name}/latest/manifest.yaml"
30+
)
3131

32-
VALIDATION_SUCCESSES = []
33-
VALIDATION_FAILURES = []
34-
DOWNLOAD_FAILURES = []
32+
VALIDATION_SUCCESSES: List[Tuple[str, str]] = []
33+
VALIDATION_FAILURES: List[Tuple[str, str, str]] = []
34+
DOWNLOAD_FAILURES: List[Tuple[str, str]] = []
3535

3636

3737
def load_declarative_component_schema() -> Dict[str, Any]:
@@ -41,29 +41,32 @@ def load_declarative_component_schema() -> Dict[str, Any]:
4141
/ "airbyte_cdk/sources/declarative/declarative_component_schema.yaml"
4242
)
4343
with open(schema_path, "r") as file:
44-
return yaml.safe_load(file)
44+
schema = yaml.safe_load(file)
45+
if not isinstance(schema, dict):
46+
raise ValueError("Schema must be a dictionary")
47+
return schema
4548

4649

4750
def get_manifest_only_connectors() -> List[Tuple[str, str]]:
4851
"""
4952
Fetch manifest-only connectors from the registry.
50-
53+
5154
Returns:
52-
List of tuples (connector_name, cdk_version) where cdk_version will be
55+
List of tuples (connector_name, cdk_version) where cdk_version will be
5356
determined from the manifest.yaml file itself.
5457
"""
5558
try:
5659
response = requests.get(CONNECTOR_REGISTRY_URL, timeout=30)
5760
response.raise_for_status()
5861
registry = response.json()
59-
60-
manifest_connectors = []
62+
63+
manifest_connectors: List[Tuple[str, str]] = []
6164
for source in registry.get("sources", []):
6265
if source.get("language") == "manifest-only":
6366
connector_name = source.get("dockerRepository", "").replace("airbyte/", "")
6467
if connector_name:
65-
manifest_connectors.append((connector_name, None))
66-
68+
manifest_connectors.append((connector_name, "unknown"))
69+
6770
return manifest_connectors
6871
except Exception as e:
6972
pytest.fail(f"Failed to fetch connector registry: {e}")
@@ -72,7 +75,7 @@ def get_manifest_only_connectors() -> List[Tuple[str, str]]:
7275
def download_manifest(connector_name: str) -> Tuple[str, str]:
7376
"""
7477
Download manifest.yaml for a connector.
75-
78+
7679
Returns:
7780
Tuple of (manifest_content, cdk_version) where cdk_version is extracted
7881
from the manifest's version field.
@@ -82,10 +85,10 @@ def download_manifest(connector_name: str) -> Tuple[str, str]:
8285
response = requests.get(url, timeout=30)
8386
response.raise_for_status()
8487
manifest_content = response.text
85-
88+
8689
manifest_dict = yaml.safe_load(manifest_content)
8790
cdk_version = manifest_dict.get("version", "unknown")
88-
91+
8992
return manifest_content, cdk_version
9093
except Exception as e:
9194
DOWNLOAD_FAILURES.append((connector_name, str(e)))
@@ -95,7 +98,7 @@ def download_manifest(connector_name: str) -> Tuple[str, str]:
9598
def get_manifest_only_connector_names() -> List[str]:
9699
"""
97100
Get all manifest-only connector names from the registry.
98-
101+
99102
Returns:
100103
List of connector names (e.g., "source-hubspot")
101104
"""
@@ -104,10 +107,10 @@ def get_manifest_only_connector_names() -> List[str]:
104107

105108

106109
@pytest.mark.parametrize("connector_name", get_manifest_only_connector_names())
107-
def test_manifest_validates_against_schema(connector_name: str):
110+
def test_manifest_validates_against_schema(connector_name: str) -> None:
108111
"""
109112
Test that manifest.yaml files from the registry validate against the CDK schema.
110-
113+
111114
Args:
112115
connector_name: Name of the connector (e.g., "source-hubspot")
113116
"""
@@ -116,23 +119,23 @@ def test_manifest_validates_against_schema(connector_name: str):
116119
manifest_content, cdk_version = download_manifest(connector_name)
117120
except Exception as e:
118121
pytest.fail(f"Failed to download manifest for {connector_name}: {e}")
119-
122+
120123
if (connector_name, cdk_version) in EXCLUDED_CONNECTORS:
121124
pytest.skip(
122125
f"Skipping {connector_name} - connector declares it is compatible with "
123126
f"CDK version {cdk_version} but is known to fail validation"
124127
)
125-
128+
126129
try:
127130
manifest_dict = yaml.safe_load(manifest_content)
128131
except yaml.YAMLError as e:
129132
error_msg = f"Invalid YAML in manifest for {connector_name}: {e}"
130133
VALIDATION_FAILURES.append((connector_name, cdk_version, error_msg))
131134
pytest.fail(error_msg)
132-
135+
133136
schema = load_declarative_component_schema()
134137
validator = ValidateAdheresToSchema(schema=schema)
135-
138+
136139
try:
137140
validator.validate(manifest_dict)
138141
VALIDATION_SUCCESSES.append((connector_name, cdk_version))
@@ -147,15 +150,15 @@ def test_manifest_validates_against_schema(connector_name: str):
147150
pytest.fail(error_msg)
148151

149152

150-
def test_schema_loads_successfully():
153+
def test_schema_loads_successfully() -> None:
151154
"""Test that the declarative component schema loads without errors."""
152155
schema = load_declarative_component_schema()
153156
assert isinstance(schema, dict)
154157
assert "type" in schema
155158
assert schema["type"] == "object"
156159

157160

158-
def test_connector_registry_accessible():
161+
def test_connector_registry_accessible() -> None:
159162
"""Test that the connector registry is accessible."""
160163
response = requests.get(CONNECTOR_REGISTRY_URL, timeout=30)
161164
assert response.status_code == 200
@@ -164,63 +167,65 @@ def test_connector_registry_accessible():
164167
assert isinstance(registry["sources"], list)
165168

166169

167-
def test_manifest_only_connectors_found():
170+
def test_manifest_only_connectors_found() -> None:
168171
"""Test that we can find manifest-only connectors in the registry."""
169172
connectors = get_manifest_only_connectors()
170173
assert len(connectors) > 0, "No manifest-only connectors found in registry"
171-
174+
172175
for connector_name, _ in connectors:
173176
assert isinstance(connector_name, str)
174177
assert len(connector_name) > 0
175178
assert connector_name.startswith("source-") or connector_name.startswith("destination-")
176179

177180

178-
def test_sample_manifest_download():
181+
def test_sample_manifest_download() -> None:
179182
"""Test that we can download a sample manifest file."""
180183
connectors = get_manifest_only_connectors()
181184
if not connectors:
182185
pytest.skip("No manifest-only connectors available for testing")
183-
186+
184187
connector_name, _ = connectors[0]
185188
try:
186189
manifest_content, cdk_version = download_manifest(connector_name)
187190
except Exception as e:
188191
pytest.skip(f"Could not download sample manifest from {connector_name}: {e}")
189-
192+
190193
assert isinstance(manifest_content, str)
191194
assert len(manifest_content) > 0
192195
assert isinstance(cdk_version, str)
193196
assert len(cdk_version) > 0
194-
197+
195198
manifest_dict = yaml.safe_load(manifest_content)
196199
assert isinstance(manifest_dict, dict)
197200
assert "version" in manifest_dict
198201
assert manifest_dict["version"] == cdk_version
199202

200203

201-
def log_test_results():
204+
def log_test_results() -> None:
202205
"""Log comprehensive test results for analysis."""
203-
print("\n" + "="*80)
206+
print("\n" + "=" * 80)
204207
print("MANIFEST VALIDATION TEST RESULTS SUMMARY")
205-
print("="*80)
206-
208+
print("=" * 80)
209+
207210
print(f"\n✓ SUCCESSFUL VALIDATIONS ({len(VALIDATION_SUCCESSES)}):")
208211
for connector_name, cdk_version in VALIDATION_SUCCESSES:
209212
print(f" - {connector_name} (CDK {cdk_version})")
210-
213+
211214
print(f"\n✗ VALIDATION FAILURES ({len(VALIDATION_FAILURES)}):")
212215
for connector_name, cdk_version, error in VALIDATION_FAILURES:
213216
print(f" - {connector_name} (CDK {cdk_version}): {error}")
214-
217+
215218
print(f"\n⚠ DOWNLOAD FAILURES ({len(DOWNLOAD_FAILURES)}):")
216219
for connector_name, error in DOWNLOAD_FAILURES:
217220
print(f" - {connector_name}: {error}")
218-
219-
print("\n" + "="*80)
220-
print(f"TOTAL: {len(VALIDATION_SUCCESSES)} passed, {len(VALIDATION_FAILURES)} failed, {len(DOWNLOAD_FAILURES)} download errors")
221-
print("="*80)
221+
222+
print("\n" + "=" * 80)
223+
print(
224+
f"TOTAL: {len(VALIDATION_SUCCESSES)} passed, {len(VALIDATION_FAILURES)} failed, {len(DOWNLOAD_FAILURES)} download errors"
225+
)
226+
print("=" * 80)
222227

223228

224-
def pytest_sessionfinish(session, exitstatus):
229+
def pytest_sessionfinish(session: Any, exitstatus: Any) -> None:
225230
"""Called after whole test run finished, right before returning the exit status to the system."""
226231
log_test_results()

0 commit comments

Comments
 (0)