Skip to content

feat(cdk): connector builder support for file uploader #503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Apr 28, 2025
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
file-api: run ruff format .
  • Loading branch information
aldogonzalez8 committed Apr 23, 2025
commit c8e29d1db01688a92021d38c0deb95987ef61593
2 changes: 1 addition & 1 deletion airbyte_cdk/sources/declarative/yaml_declarative_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def __init__(
catalog=catalog or ConfiguredAirbyteCatalog(streams=[]),
config=config or {},
state=state or [],
source_config=source_config
source_config=source_config,
)

def _read_and_parse_yaml_file(self, path_to_yaml_file: str) -> ConnectionDefinition:
Expand Down
17 changes: 10 additions & 7 deletions unit_tests/sources/declarative/file/test_file_stream.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@
from unittest.mock import Mock, patch

from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, Status
from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory as OriginalModelToComponentFactory
from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import (
ModelToComponentFactory as OriginalModelToComponentFactory,
)
from airbyte_cdk.sources.declarative.retrievers.file_uploader.noop_file_writer import NoopFileWriter
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
from airbyte_cdk.test.catalog_builder import CatalogBuilder, ConfiguredAirbyteStreamBuilder
Expand Down Expand Up @@ -43,7 +45,7 @@ def _source(
path_to_yaml=str(Path(__file__).parent / yaml_file),
catalog=catalog,
config=config,
state=state
state=state,
)


Expand All @@ -52,7 +54,7 @@ def read(
catalog: ConfiguredAirbyteCatalog,
state_builder: Optional[StateBuilder] = None,
expecting_exception: bool = False,
yaml_file: Optional[str] = None
yaml_file: Optional[str] = None,
) -> EntrypointOutput:
config = config_builder.build()
state = state_builder.build() if state_builder else StateBuilder().build()
Expand Down Expand Up @@ -218,20 +220,20 @@ def test_get_article_attachments_messages_for_connector_builder(self) -> None:
# Define a mock factory that forces emit_connector_builder_messages=True
class MockModelToComponentFactory(OriginalModelToComponentFactory):
def __init__(self, *args, **kwargs):
kwargs['emit_connector_builder_messages'] = True
kwargs["emit_connector_builder_messages"] = True
super().__init__(*args, **kwargs)

# Patch the factory class where ConcurrentDeclarativeSource (parent of YamlDeclarativeSource) imports it
with patch(
"airbyte_cdk.sources.declarative.concurrent_declarative_source.ModelToComponentFactory",
new=MockModelToComponentFactory
new=MockModelToComponentFactory,
):
output = read(
self._config(),
CatalogBuilder()
.with_stream(ConfiguredAirbyteStreamBuilder().with_name("article_attachments"))
.build(),
yaml_file="test_file_stream_with_filename_extractor.yaml"
yaml_file="test_file_stream_with_filename_extractor.yaml",
)

assert len(output.records) == 1
Expand All @@ -246,7 +248,8 @@ def __init__(self, *args, **kwargs):
record_data = output.records[0].record.data
assert record_data["staging_file_url"] == file_reference.staging_file_url
assert (
record_data["source_file_relative_path"] == file_reference.source_file_relative_path
record_data["source_file_relative_path"]
== file_reference.source_file_relative_path
)
assert record_data["file_size_bytes"] == file_reference.file_size_bytes

Expand Down
Loading