diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 707c97fc..21660816 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -17,8 +17,8 @@ "PATH": "${containerEnv:PATH}:/home/vscode/.local/bin" }, - // Mirror the full CI test environment and install this checkout in editable mode. - "postCreateCommand": "python -m pip install --user --upgrade pip && python -m pip install --user -r requirements-full.txt pytest==9.0.2 pytest-mock -e .", + // Mirror the complete local test/tooling environment and install this checkout in editable mode. + "postCreateCommand": "python -m pip install --user --upgrade pip && python -m pip install --user -r requirements-dev.txt -e . && python -m pip check", // Configure tool-specific properties. "customizations": { @@ -39,7 +39,7 @@ "https://public.wrangle.works/schema/recipes/schema.json": ["*.wrgl.yml", "*.wrgl.yaml", "*.recipe"] }, "python.defaultInterpreterPath": "/usr/local/bin/python", - "python.testing.pytestArgs": ["tests"], + "python.testing.pytestArgs": ["-c", "pytest-local.ini"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml index 4b3dbc9e..79eee0a8 100644 --- a/.github/workflows/deploy-dev.yml +++ b/.github/workflows/deploy-dev.yml @@ -116,7 +116,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.10" + python-version: "3.13" - name: Test pip install run: | @@ -139,7 +139,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.12' + python-version: '3.13' - name: Install Dependencies run: | @@ -198,7 +198,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: "3.10" + python-version: "3.13" - name: Patch version in setup.py run: | diff --git a/.gitignore b/.gitignore index f9ae0d04..580fbfe5 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.test-local/ # Translations *.mo @@ -138,4 +139,4 @@ docs/ dmypy.json # Pyre type checker -.pyre/ \ No newline at end of file +.pyre/ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..82c51f63 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE requirements.txt diff --git a/README.md b/README.md index 1469b7d1..d071f7c1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,40 @@ Full documentation available at [wrangles.io](https://wrangles.io/python). +## Local development + +Supported local development uses Python 3.13. On Windows, create or refresh the +complete test and tooling environment with one command from the repository +root: + +```powershell +.\scripts\bootstrap-dev.ps1 +``` + +Add `-RunTests` to run the self-contained local test suite after installation. The script +creates `.venv`, installs `requirements-dev.txt` and this checkout in editable +mode, runs `pip check`, and verifies representative core and SQL connector +imports. It will not replace an existing environment created with another +Python version. + +If `.venv` was created with an older Python version and can be discarded, +deactivate it and recreate it from the committed declaration: + +```powershell +deactivate +Remove-Item -LiteralPath .\.venv -Recurse -Force +.\scripts\bootstrap-dev.ps1 -RunTests +.\.venv\Scripts\Activate.ps1 +``` + +Dev Containers and Codespaces use the same Python 3.13 developer declaration +automatically. `pytest-local.ini` is the shared credential-safe test selection. +`scripts/test-local.ps1` clears credential variables before running it and +isolates generated pytest state by Windows identity. The configuration excludes +the intentionally live database, AWS, WrangleWorks, AI, and search-provider +checks. The complete credentialed suite remains a CI validation and a local +dependency/import pass does not claim live-service validation. + ## What are Wrangles? Wrangles are a set of modular transformations for data cleaning and enrichment. Each Wrangle is optimized for a particular job, many of which are backed by sophisticated machine learning models. @@ -36,7 +70,7 @@ Connectors for databases, cloud storage, and external services require additiona | DuckDB | `pip install duckdb` | | PostgreSQL | `pip install psycopg2-binary sqlalchemy` | | MySQL | `pip install pymysql sqlalchemy` | -| MongoDB | `pip install pymongo[srv]` | +| MongoDB | `pip install pymongo` | | AWS S3 | `pip install boto3` | | Salesforce | `pip install simple-salesforce` | | SFTP / SSH | `pip install fabric` | diff --git a/pytest-local.ini b/pytest-local.ini new file mode 100644 index 00000000..02f160ad --- /dev/null +++ b/pytest-local.ini @@ -0,0 +1,53 @@ +[pytest] +testpaths = + tests/test_ai_cache.py + tests/test_ai_definition.py + tests/test_data.py + tests/test_dataframe.py + tests/test_openai_extract_ai.py + tests/recipes + tests/connectors/test_access.py + tests/connectors/test_concurrent.py + tests/connectors/test_duckdb.py + tests/connectors/test_excel.py + tests/connectors/test_file.py + tests/connectors/test_input.py + tests/connectors/test_jinja.py + tests/connectors/test_matrix.py + tests/connectors/test_memory.py + tests/connectors/test_mongodb.py + tests/connectors/test_mssql.py + tests/connectors/test_mysql.py + tests/connectors/test_notifications.py + tests/connectors/test_postgres.py + tests/connectors/test_recipe.py + tests/connectors/test_sqlite.py + tests/connectors/test_ssh.py + tests/connectors/test_test.py +addopts = + -p no:cacheprovider + --ignore=tests/recipes/wrangles/test_extract.py + --ignore=tests/recipes/wrangles/test_extract_unit_conversion.py + --ignore=tests/recipes/wrangles/test_generate.py + --ignore=tests/recipes/wrangles/test_search.py + --deselect=tests/recipes/wrangles/test_create.py::TestCreateEmbeddings + --deselect=tests/recipes/wrangles/test_main.py::TestClassify + --deselect=tests/recipes/wrangles/test_main.py::TestLookup + --deselect=tests/recipes/wrangles/test_main.py::TestMatrix::test_extract_ai + --deselect=tests/recipes/wrangles/test_main.py::TestStandardize + --deselect=tests/recipes/wrangles/test_main.py::TestTranslate + --deselect=tests/recipes/test_custom_functions.py::test_local_takes_priority + --deselect=tests/recipes/test_custom_functions.py::test_model_with_custom_functions + --deselect=tests/recipes/test_recipes.py::test_recipe_by_latest_version + --deselect=tests/recipes/test_recipes.py::test_recipe_by_production_version + --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_id + --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_latest + --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_tag + --deselect=tests/recipes/test_recipes.py::test_recipe_from_url + --deselect=tests/recipes/test_recipes.py::test_recipe_from_url_not_found + --deselect=tests/recipes/test_recipes.py::test_recipe_model + --deselect=tests/recipes/test_recipes.py::test_recipe_wrong_model + --deselect=tests/connectors/test_recipe.py::test_model_id + --deselect=tests/connectors/test_recipe.py::test_model_with_custom_functions +filterwarnings = + ignore:'imghdr' is deprecated and slated for removal in Python 3.13:DeprecationWarning:apprise\.utils\.pgp diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..5444f46e --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,9 @@ +# Complete Python 3.13 development and test environment. +-r requirements-full.txt + +pytest==9.0.2 +pytest-mock==3.15.1 +pytest-cov==7.1.0 +jsonschema==4.26.0 +build==1.5.0 +twine==7.0.0 diff --git a/requirements.txt b/requirements.txt index a98042a6..a1a978b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ xlsxwriter boto3 # NoSQL -pymongo[srv] +pymongo # CRM simple-salesforce diff --git a/scripts/bootstrap-dev.ps1 b/scripts/bootstrap-dev.ps1 new file mode 100644 index 00000000..2173e337 --- /dev/null +++ b/scripts/bootstrap-dev.ps1 @@ -0,0 +1,72 @@ +[CmdletBinding()] +param( + [string]$VenvPath = ".venv", + [switch]$RunTests +) + +$ErrorActionPreference = "Stop" +$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) + +if (-not [System.IO.Path]::IsPathRooted($VenvPath)) { + $VenvPath = Join-Path $repoRoot $VenvPath +} + +$launcher = Get-Command py.exe -ErrorAction SilentlyContinue +if (-not $launcher) { + throw "Python Launcher for Windows (py.exe) is required. Install Python 3.13 and retry." +} + +$bootstrapVersion = & $launcher.Source -3.13 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" +if ($LASTEXITCODE -ne 0 -or $bootstrapVersion -ne "3.13") { + throw "Python 3.13 is required, but py -3.13 did not resolve to it." +} + +$python = Join-Path $VenvPath "Scripts\python.exe" +if (-not (Test-Path -LiteralPath $python)) { + Write-Host "Creating Python 3.13 environment at $VenvPath..." + & $launcher.Source -3.13 -m venv $VenvPath + if ($LASTEXITCODE -ne 0) { + throw "Unable to create the virtual environment." + } +} + +$venvVersion = & $python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')" +if ($LASTEXITCODE -ne 0 -or $venvVersion -ne "3.13") { + throw "$VenvPath is not a Python 3.13 environment. Choose a new -VenvPath; this script will not replace it." +} + +Push-Location $repoRoot +try { + & $python -m pip install --upgrade pip + if ($LASTEXITCODE -ne 0) { + throw "pip upgrade failed." + } + + & $python -m pip install --requirement requirements-dev.txt --editable . + if ($LASTEXITCODE -ne 0) { + throw "Development dependency installation failed." + } + + & $python -m pip check + if ($LASTEXITCODE -ne 0) { + throw "Installed packages have incompatible requirements." + } + + & $python -c "from importlib.metadata import version; import duckdb, numpy, pandas, pyarrow, pytest, sqlalchemy, wrangles; print('Python development imports OK: wrangles={}, pandas={}, numpy={}, pyarrow={}, duckdb={}, sqlalchemy={}, pytest={}'.format(version('wrangles'), pandas.__version__, numpy.__version__, pyarrow.__version__, duckdb.__version__, sqlalchemy.__version__, pytest.__version__))" + if ($LASTEXITCODE -ne 0) { + throw "Development dependency import check failed." + } + + if ($RunTests) { + & (Join-Path $PSScriptRoot "test-local.ps1") -Python $python + if ($LASTEXITCODE -ne 0) { + throw "The self-contained local test suite failed." + } + } +} +finally { + Pop-Location +} + +Write-Host "WranglesPY Python 3.13 development environment is ready." +Write-Host "Interpreter: $python" diff --git a/scripts/test-local.ps1 b/scripts/test-local.ps1 new file mode 100644 index 00000000..c1a87c8a --- /dev/null +++ b/scripts/test-local.ps1 @@ -0,0 +1,66 @@ +[CmdletBinding()] +param( + [string]$Python = ".venv\Scripts\python.exe" +) + +$ErrorActionPreference = "Stop" +$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) + +if (-not [System.IO.Path]::IsPathRooted($Python)) { + $Python = Join-Path $repoRoot $Python +} +if (-not (Test-Path -LiteralPath $Python)) { + throw "Python interpreter not found: $Python" +} + +# The complete CI suite intentionally exercises credentialed databases, AWS, +# WrangleWorks services, and AI/search providers. Local tests must not inherit +# credentials and accidentally make live calls or echo a secret in a traceback. +foreach ($name in @( + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GEMINI_API_KEY", + "HUGGINGFACE_TOKEN", + "OPENAI_API_KEY", + "SERPAPI_API_KEY", + "WRANGLES_PASSWORD", + "WRANGLES_USER" +)) { + [Environment]::SetEnvironmentVariable($name, $null, "Process") +} + +$localConfig = Join-Path $repoRoot "pytest-local.ini" +if (-not (Test-Path -LiteralPath $localConfig)) { + throw "Local pytest configuration not found: $localConfig" +} + +# Pytest's default per-user temp root and repository cache can become +# inaccessible when alternating between the restricted sandbox and Eric's +# normal Windows identity. Keep each identity's generated state separate. +$identity = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name +$identitySlug = $identity -replace "[^A-Za-z0-9_.-]", "-" +$testTemp = Join-Path $repoRoot ".test-local\$identitySlug\pytest" +$testTempParent = Split-Path -Parent $testTemp +New-Item -ItemType Directory -Path $testTempParent -Force | Out-Null +$localState = @( + "--basetemp=$testTemp" +) + +Push-Location $repoRoot +try { + foreach ($relativePath in @("tests/temp/temp.db", "tests/temp/temp_run.db")) { + $generatedDatabase = Join-Path $repoRoot $relativePath + if (Test-Path -LiteralPath $generatedDatabase) { + Remove-Item -LiteralPath $generatedDatabase + } + } + + & $Python -m pytest -c $localConfig @localState + if ($LASTEXITCODE -ne 0) { + throw "The self-contained local test suite failed." + } +} +finally { + Pop-Location +} diff --git a/setup.py b/setup.py index f5b18480..5762a468 100644 --- a/setup.py +++ b/setup.py @@ -22,13 +22,17 @@ description = 'Wrangle your data into shape with AI', long_description = long_description, long_description_content_type = 'text/markdown', - license_files = ('LICENSE.txt',), + license_files = ('LICENSE',), license = 'Apache License 2.0', classifiers = [ 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent' ], + python_requires = '>=3.11', version = '1.19.0', url = 'https://github.com/wrangleworks/WranglesPy', author = 'WrangleWorks',