diff --git a/poetry.lock b/poetry.lock index e6c6898f..6f45500f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "aioftp" @@ -199,6 +199,18 @@ tzdata = {version = "*", markers = "python_version >= \"3.9\""} doc = ["doc8", "sphinx (>=7.0.0)", "sphinx-autobuild", "sphinx-autodoc-typehints", "sphinx_rtd_theme (>=1.3.0)"] test = ["dateparser (==1.*)", "pre-commit", "pytest", "pytest-cov", "pytest-mock", "pytz (==2025.2)", "simplejson (==3.*)"] +[[package]] +name = "astroid" +version = "4.0.4" +description = "An abstract syntax tree for Python with inference support." +optional = false +python-versions = ">=3.10.0" +groups = ["dev"] +files = [ + {file = "astroid-4.0.4-py3-none-any.whl", hash = "sha256:52f39653876c7dec3e3afd4c2696920e05c83832b9737afc21928f2d2eb7a753"}, + {file = "astroid-4.0.4.tar.gz", hash = "sha256:986fed8bcf79fb82c78b18a53352a0b287a73817d6dbcfba3162da36667c49a0"}, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -1271,6 +1283,22 @@ files = [ {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, ] +[[package]] +name = "dill" +version = "0.4.1" +description = "serialize all of Python" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d"}, + {file = "dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa"}, +] + +[package.extras] +graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] + [[package]] name = "distlib" version = "0.4.0" @@ -3247,8 +3275,8 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.23.2", markers = "python_version == \"3.11\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -3895,6 +3923,34 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pylint" +version = "4.0.8" +description = "python code static checker" +optional = false +python-versions = ">=3.10.0" +groups = ["dev"] +files = [ + {file = "pylint-4.0.8-py3-none-any.whl", hash = "sha256:3341c08c0aabaa4adc71516de0969f3ba5c692b56c75af4dcb4d242823fbe363"}, + {file = "pylint-4.0.8.tar.gz", hash = "sha256:1c1b2128bde5ff5e966801413080b6384d42a5782718d528c906dbb6beab94ed"}, +] + +[package.dependencies] +astroid = ">=4.0.2,<=4.1.dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version == \"3.11\""}, +] +isort = ">=5,<5.13 || >5.13,<10" +mccabe = ">=0.6,<0.8" +platformdirs = ">=2.2" +tomlkit = ">=0.10.1" + +[package.extras] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] + [[package]] name = "pyparsing" version = "3.3.2" @@ -5332,6 +5388,18 @@ files = [ {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, ] +[[package]] +name = "tomlkit" +version = "0.15.1" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "tomlkit-0.15.1-py3-none-any.whl", hash = "sha256:177a05aece5a8ca5266fd3c448abb47b8d352f09d477d3ca8332db4d89b24304"}, + {file = "tomlkit-0.15.1.tar.gz", hash = "sha256:e25bbf38843005246210a12982776f27f99cb9be67160e14434d0c0d21ee1e97"}, +] + [[package]] name = "tornado" version = "6.5.7" @@ -5802,4 +5870,4 @@ web = ["streamlit"] [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.14" -content-hash = "e977e2830aeca92c8938c314aa38478261d87d1ab172aa480b87389ab20add4b" +content-hash = "c7b74c65dd4997b3f7ce0031157af5d2f604f82c7485a9e0d5fd6818d1bfa73b" diff --git a/pyproject.toml b/pyproject.toml index 72671b0a..8fdd021c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ pytest-retry = "1.7.0" datamodel-code-generator = {extras = ["http"], version = ">=0.54,<0.65"} pytest-asyncio = "^1.3.0" pytest-cov = "^7.1.0" +pylint = "^4.0.8" [tool.poetry.group.docs.dependencies] sphinx = "^5.1.1" diff --git a/pysus/management/catalog.py b/pysus/management/catalog.py index 7b0eed3a..a66cd87e 100644 --- a/pysus/management/catalog.py +++ b/pysus/management/catalog.py @@ -10,6 +10,7 @@ from __future__ import annotations import hashlib +from dataclasses import dataclass from datetime import datetime from pathlib import Path from typing import TYPE_CHECKING, Any @@ -64,6 +65,15 @@ def sha256_of(path: Path) -> str: return digest.hexdigest() +@dataclass(frozen=True) +class FileOriginMeta: + """Encapsulates file origin metadata for updates.""" + + modified: datetime | None + size: int + source_sha256: str | None = None + + class CatalogWriter: """Upsert dataset/group/file/column metadata into the DuckLake catalogs.""" @@ -228,22 +238,20 @@ def touch_file( self, cursor, file_id: int, - origin_modified: datetime | None, - origin_size: int, - source_sha256: str | None = None, + meta: FileOriginMeta, ) -> None: """Update origin metadata without replacing the artifact.""" - if source_sha256 is not None: + if meta.source_sha256 is not None: cursor.execute( "UPDATE pysus.files SET origin_modified = ?, " "origin_size = ?, source_sha256 = ? WHERE id = ?", - (origin_modified, origin_size, source_sha256, file_id), + (meta.modified, meta.size, meta.source_sha256, file_id), ) else: cursor.execute( "UPDATE pysus.files SET origin_modified = ?, " "origin_size = ? WHERE id = ?", - (origin_modified, origin_size, file_id), + (meta.modified, meta.size, file_id), ) def delete_file(self, cursor, file_id: int) -> None: diff --git a/pysus/management/sync.py b/pysus/management/sync.py index 0d73d7e9..576b54a1 100644 --- a/pysus/management/sync.py +++ b/pysus/management/sync.py @@ -34,7 +34,7 @@ from pysus.api.errors import AuthenticationError, ConnectionError from pysus.api.models import BaseRemoteFile -from .catalog import CatalogWriter, sha256_of +from .catalog import CatalogWriter, FileOriginMeta, sha256_of from .compare import Comparator from .inventory import Inventory from .records import ( @@ -365,8 +365,10 @@ async def upload_file( writer.touch_file( dataset_cursor, file_id, - self._safe_modify(file), - self._safe_size(file), + FileOriginMeta( + modified=self._safe_modify(file), + size=self._safe_size(file), + ) ) dataset_conn.commit() dataset_cursor.execute("CHECKPOINT") @@ -394,9 +396,11 @@ async def upload_file( writer.touch_file( dataset_cursor, file_id, - self._safe_modify(file), - self._safe_size(file), - source_sha256=raw_digest, + FileOriginMeta( + modified=self._safe_modify(file), + size=self._safe_size(file), + source_sha256=raw_digest, + ) ) dataset_conn.commit() dataset_cursor.execute("CHECKPOINT") diff --git a/pysus/tests/management/test_catalog.py b/pysus/tests/management/test_catalog.py index fb763e8b..fffe6f75 100644 --- a/pysus/tests/management/test_catalog.py +++ b/pysus/tests/management/test_catalog.py @@ -6,7 +6,7 @@ import duckdb import pyarrow as pa import pytest -from pysus.management.catalog import CatalogWriter +from pysus.management.catalog import CatalogWriter, FileOriginMeta _SCHEMA = """ CREATE SCHEMA pysus; @@ -120,7 +120,13 @@ def test_updates_origin_metadata(self, writer_and_cursor): writer, cursor, _ = writer_and_cursor _insert(cursor, "public/data/x.parquet") writer.touch_file( - cursor, 1, datetime(2026, 2, 2), 99, source_sha256="cc" * 32 + cursor, + 1, + FileOriginMeta( + modified=datetime(2026, 2, 2), + size=99, + source_sha256="cc" * 32, + ) ) result = writer.get_file_full(cursor, "public/data/x.parquet") assert result is not None @@ -132,7 +138,14 @@ def test_updates_origin_metadata(self, writer_and_cursor): def test_without_source_sha256(self, writer_and_cursor): writer, cursor, _ = writer_and_cursor _insert(cursor, "public/data/x.parquet") - writer.touch_file(cursor, 1, datetime(2026, 2, 2), 99) + writer.touch_file( + cursor, + 1, + FileOriginMeta( + modified=datetime(2026, 2, 2), + size=99, + ) + ) result = writer.get_file_full(cursor, "public/data/x.parquet") assert result is not None assert result[1] == datetime(2026, 2, 2)