tests: Add parametrized validation test for manifest-only connectors

airbyte_cdk/models/__init__.py

@@ -19,7 +19,6 @@
     AirbyteMessage,
     AirbyteProtocol,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteStateBlob,
     AirbyteStateMessage,
     AirbyteStateStats,

airbyte_cdk/sources/declarative/retrievers/file_uploader/default_file_uploader.py

@@ -9,7 +9,6 @@
 from pathlib import Path
 from typing import Any, Mapping, Optional, Union
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import (
     InterpolatedString,
@@ -90,8 +89,8 @@ def upload(self, record: Record) -> None:
         logger.info(f"File size: {file_size_bytes / 1024} KB")
         logger.info(f"File relative path: {str(file_relative_path)}")
 
-        record.file_reference = AirbyteRecordMessageFileReference(
-            staging_file_url=str(full_path),
-            source_file_relative_path=str(file_relative_path),
-            file_size_bytes=file_size_bytes,
-        )
+        # record.file_reference = AirbyteRecordMessageFileReference(
+        #     staging_file_url=str(full_path),
+        #     source_file_relative_path=str(file_relative_path),
+        #     file_size_bytes=file_size_bytes,
+        # )

airbyte_cdk/sources/file_based/file_based_stream_reader.py

@@ -12,7 +12,6 @@
 
 from wcmatch.glob import GLOBSTAR, globmatch
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec
 from airbyte_cdk.sources.file_based.config.validate_config_transfer_modes import (
     include_identities_stream,
@@ -156,7 +155,7 @@ def include_identities_stream(self) -> bool:
     @abstractmethod
     def upload(
         self, file: RemoteFile, local_directory: str, logger: logging.Logger
-    ) -> Tuple[FileRecordData, AirbyteRecordMessageFileReference]:
+    ) -> Tuple[FileRecordData, Any]:
         """
         This is required for connectors that will support writing to
         files. It will handle the logic to download,get,read,acquire or
@@ -168,7 +167,7 @@ def upload(
                logger (logging.Logger): Logger for logging information and errors.
 
            Returns:
-               AirbyteRecordMessageFileReference: A file reference object containing:
+               Any: A file reference object containing:
                    - staging_file_url (str): The absolute path to the referenced file in the staging area.
                    - file_size_bytes (int): The size of the referenced file in bytes.
                    - source_file_relative_path (str): The relative path to the referenced file in source.

airbyte_cdk/sources/file_based/file_types/file_transfer.py

@@ -2,9 +2,8 @@
 # Copyright (c) 2024 Airbyte, Inc., all rights reserved.
 #
 import logging
-from typing import Iterable, Tuple
+from typing import Any, Iterable, Optional, Tuple
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader
 from airbyte_cdk.sources.file_based.file_record_data import FileRecordData
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
@@ -20,7 +19,7 @@ def upload(
         file: RemoteFile,
         stream_reader: AbstractFileBasedStreamReader,
         logger: logging.Logger,
-    ) -> Iterable[Tuple[FileRecordData, AirbyteRecordMessageFileReference]]:
+    ) -> Iterable[Tuple[FileRecordData, Optional[Any]]]:
         try:
             yield stream_reader.upload(
                 file=file, local_directory=self._local_directory, logger=logger

airbyte_cdk/sources/types.py

@@ -6,7 +6,6 @@
 
 from typing import Any, ItemsView, Iterator, KeysView, List, Mapping, Optional, ValuesView
 
-from airbyte_cdk.models import AirbyteRecordMessageFileReference
 from airbyte_cdk.utils.slice_hasher import SliceHasher
 
 # A FieldPointer designates a path to a field inside a mapping. For example, retrieving ["k1", "k1.2"] in the object {"k1" :{"k1.2":
@@ -24,7 +23,7 @@ def __init__(
         data: Mapping[str, Any],
         stream_name: str,
         associated_slice: Optional[StreamSlice] = None,
-        file_reference: Optional[AirbyteRecordMessageFileReference] = None,
+        file_reference: Optional[Any] = None,
     ):
         self._data = data
         self._associated_slice = associated_slice
@@ -40,11 +39,11 @@ def associated_slice(self) -> Optional[StreamSlice]:
         return self._associated_slice
 
     @property
-    def file_reference(self) -> AirbyteRecordMessageFileReference:
+    def file_reference(self) -> Optional[Any]:
         return self._file_reference
 
     @file_reference.setter
-    def file_reference(self, value: AirbyteRecordMessageFileReference) -> None:
+    def file_reference(self, value: Optional[Any]) -> None:
         self._file_reference = value
 
     def __repr__(self) -> str:

airbyte_cdk/sources/utils/record_helper.py

@@ -9,7 +9,6 @@
     AirbyteLogMessage,
     AirbyteMessage,
     AirbyteRecordMessage,
-    AirbyteRecordMessageFileReference,
     AirbyteTraceMessage,
 )
 from airbyte_cdk.models import Type as MessageType
@@ -22,7 +21,7 @@ def stream_data_to_airbyte_message(
     data_or_message: StreamData,
     transformer: TypeTransformer = TypeTransformer(TransformConfig.NoTransform),
     schema: Optional[Mapping[str, Any]] = None,
-    file_reference: Optional[AirbyteRecordMessageFileReference] = None,
+    file_reference: Optional[Any] = None,
 ) -> AirbyteMessage:
     if schema is None:
         schema = {}