Standardize local development on Python 3.13

.devcontainer/devcontainer.json

@@ -17,8 +17,8 @@
 		"PATH": "${containerEnv:PATH}:/home/vscode/.local/bin"
 	},
 
-	// Mirror the full CI test environment and install this checkout in editable mode.
-	"postCreateCommand": "python -m pip install --user --upgrade pip && python -m pip install --user -r requirements-full.txt pytest==9.0.2 pytest-mock -e .",
+	// Mirror the complete local test/tooling environment and install this checkout in editable mode.
+	"postCreateCommand": "python -m pip install --user --upgrade pip && python -m pip install --user -r requirements-dev.txt -e . && python -m pip check",
 
 	// Configure tool-specific properties.
 	"customizations": {
@@ -39,7 +39,7 @@
 					"https://public.wrangle.works/schema/recipes/schema.json": ["*.wrgl.yml", "*.wrgl.yaml", "*.recipe"]
 				},
 				"python.defaultInterpreterPath": "/usr/local/bin/python",
-				"python.testing.pytestArgs": ["tests"],
+				"python.testing.pytestArgs": ["-c", "pytest-local.ini"],
 				"python.testing.unittestEnabled": false,
 				"python.testing.pytestEnabled": true
 			}

.github/workflows/deploy-dev.yml

@@ -116,7 +116,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.13"
 
       - name: Test pip install
         run: |
@@ -139,7 +139,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: '3.12'
+          python-version: '3.13'
 
       - name: Install Dependencies
         run: |
@@ -198,7 +198,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.13"
 
       - name: Patch version in setup.py
         run: |

.gitignore

@@ -61,6 +61,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+.test-local/
 
 # Translations
 *.mo
@@ -138,4 +139,4 @@ docs/
 dmypy.json
 
 # Pyre type checker
-.pyre/
+.pyre/

MANIFEST.in

@@ -0,0 +1 @@
+include LICENSE requirements.txt

README.md

@@ -2,6 +2,40 @@
 
 Full documentation available at [wrangles.io](https://wrangles.io/python).
 
+## Local development
+
+Supported local development uses Python 3.13. On Windows, create or refresh the
+complete test and tooling environment with one command from the repository
+root:
+
+```powershell
+.\scripts\bootstrap-dev.ps1
+```
+
+Add `-RunTests` to run the self-contained local test suite after installation. The script
+creates `.venv`, installs `requirements-dev.txt` and this checkout in editable
+mode, runs `pip check`, and verifies representative core and SQL connector
+imports. It will not replace an existing environment created with another
+Python version.
+
+If `.venv` was created with an older Python version and can be discarded,
+deactivate it and recreate it from the committed declaration:
+
+```powershell
+deactivate
+Remove-Item -LiteralPath .\.venv -Recurse -Force
+.\scripts\bootstrap-dev.ps1 -RunTests
+.\.venv\Scripts\Activate.ps1
+```
+
+Dev Containers and Codespaces use the same Python 3.13 developer declaration
+automatically. `pytest-local.ini` is the shared credential-safe test selection.
+`scripts/test-local.ps1` clears credential variables before running it and
+isolates generated pytest state by Windows identity. The configuration excludes
+the intentionally live database, AWS, WrangleWorks, AI, and search-provider
+checks. The complete credentialed suite remains a CI validation and a local
+dependency/import pass does not claim live-service validation.
+
 ## What are Wrangles?
 
 Wrangles are a set of modular transformations for data cleaning and enrichment. Each Wrangle is optimized for a particular job, many of which are backed by sophisticated machine learning models.
@@ -36,7 +70,7 @@ Connectors for databases, cloud storage, and external services require additiona
 | DuckDB | `pip install duckdb` |
 | PostgreSQL | `pip install psycopg2-binary sqlalchemy` |
 | MySQL | `pip install pymysql sqlalchemy` |
-| MongoDB | `pip install pymongo[srv]` |
+| MongoDB | `pip install pymongo` |
 | AWS S3 | `pip install boto3` |
 | Salesforce | `pip install simple-salesforce` |
 | SFTP / SSH | `pip install fabric` |

pytest-local.ini

@@ -0,0 +1,53 @@
+[pytest]
+testpaths =
+    tests/test_ai_cache.py
+    tests/test_ai_definition.py
+    tests/test_data.py
+    tests/test_dataframe.py
+    tests/test_openai_extract_ai.py
+    tests/recipes
+    tests/connectors/test_access.py
+    tests/connectors/test_concurrent.py
+    tests/connectors/test_duckdb.py
+    tests/connectors/test_excel.py
+    tests/connectors/test_file.py
+    tests/connectors/test_input.py
+    tests/connectors/test_jinja.py
+    tests/connectors/test_matrix.py
+    tests/connectors/test_memory.py
+    tests/connectors/test_mongodb.py
+    tests/connectors/test_mssql.py
+    tests/connectors/test_mysql.py
+    tests/connectors/test_notifications.py
+    tests/connectors/test_postgres.py
+    tests/connectors/test_recipe.py
+    tests/connectors/test_sqlite.py
+    tests/connectors/test_ssh.py
+    tests/connectors/test_test.py
+addopts =
+    -p no:cacheprovider
+    --ignore=tests/recipes/wrangles/test_extract.py
+    --ignore=tests/recipes/wrangles/test_extract_unit_conversion.py
+    --ignore=tests/recipes/wrangles/test_generate.py
+    --ignore=tests/recipes/wrangles/test_search.py
+    --deselect=tests/recipes/wrangles/test_create.py::TestCreateEmbeddings
+    --deselect=tests/recipes/wrangles/test_main.py::TestClassify
+    --deselect=tests/recipes/wrangles/test_main.py::TestLookup
+    --deselect=tests/recipes/wrangles/test_main.py::TestMatrix::test_extract_ai
+    --deselect=tests/recipes/wrangles/test_main.py::TestStandardize
+    --deselect=tests/recipes/wrangles/test_main.py::TestTranslate
+    --deselect=tests/recipes/test_custom_functions.py::test_local_takes_priority
+    --deselect=tests/recipes/test_custom_functions.py::test_model_with_custom_functions
+    --deselect=tests/recipes/test_recipes.py::test_recipe_by_latest_version
+    --deselect=tests/recipes/test_recipes.py::test_recipe_by_production_version
+    --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_id
+    --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_latest
+    --deselect=tests/recipes/test_recipes.py::test_recipe_by_version_tag
+    --deselect=tests/recipes/test_recipes.py::test_recipe_from_url
+    --deselect=tests/recipes/test_recipes.py::test_recipe_from_url_not_found
+    --deselect=tests/recipes/test_recipes.py::test_recipe_model
+    --deselect=tests/recipes/test_recipes.py::test_recipe_wrong_model
+    --deselect=tests/connectors/test_recipe.py::test_model_id
+    --deselect=tests/connectors/test_recipe.py::test_model_with_custom_functions
+filterwarnings =
+    ignore:'imghdr' is deprecated and slated for removal in Python 3.13:DeprecationWarning:apprise\.utils\.pgp

requirements-dev.txt

@@ -0,0 +1,9 @@
+# Complete Python 3.13 development and test environment.
+-r requirements-full.txt
+
+pytest==9.0.2
+pytest-mock==3.15.1
+pytest-cov==7.1.0
+jsonschema==4.26.0
+build==1.5.0
+twine==7.0.0

requirements.txt

@@ -26,7 +26,7 @@ xlsxwriter
 boto3
 
 # NoSQL
-pymongo[srv]
+pymongo
 
 # CRM
 simple-salesforce

scripts/bootstrap-dev.ps1

@@ -0,0 +1,72 @@
+[CmdletBinding()]
+param(
+    [string]$VenvPath = ".venv",
+    [switch]$RunTests
+)
+
+$ErrorActionPreference = "Stop"
+$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot ".."))
+
+if (-not [System.IO.Path]::IsPathRooted($VenvPath)) {
+    $VenvPath = Join-Path $repoRoot $VenvPath
+}
+
+$launcher = Get-Command py.exe -ErrorAction SilentlyContinue
+if (-not $launcher) {
+    throw "Python Launcher for Windows (py.exe) is required. Install Python 3.13 and retry."
+}
+
+$bootstrapVersion = & $launcher.Source -3.13 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
+if ($LASTEXITCODE -ne 0 -or $bootstrapVersion -ne "3.13") {
+    throw "Python 3.13 is required, but py -3.13 did not resolve to it."
+}
+
+$python = Join-Path $VenvPath "Scripts\python.exe"
+if (-not (Test-Path -LiteralPath $python)) {
+    Write-Host "Creating Python 3.13 environment at $VenvPath..."
+    & $launcher.Source -3.13 -m venv $VenvPath
+    if ($LASTEXITCODE -ne 0) {
+        throw "Unable to create the virtual environment."
+    }
+}
+
+$venvVersion = & $python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')"
+if ($LASTEXITCODE -ne 0 -or $venvVersion -ne "3.13") {
+    throw "$VenvPath is not a Python 3.13 environment. Choose a new -VenvPath; this script will not replace it."
+}
+
+Push-Location $repoRoot
+try {
+    & $python -m pip install --upgrade pip
+    if ($LASTEXITCODE -ne 0) {
+        throw "pip upgrade failed."
+    }
+
+    & $python -m pip install --requirement requirements-dev.txt --editable .
+    if ($LASTEXITCODE -ne 0) {
+        throw "Development dependency installation failed."
+    }
+
+    & $python -m pip check
+    if ($LASTEXITCODE -ne 0) {
+        throw "Installed packages have incompatible requirements."
+    }
+
+    & $python -c "from importlib.metadata import version; import duckdb, numpy, pandas, pyarrow, pytest, sqlalchemy, wrangles; print('Python development imports OK: wrangles={}, pandas={}, numpy={}, pyarrow={}, duckdb={}, sqlalchemy={}, pytest={}'.format(version('wrangles'), pandas.__version__, numpy.__version__, pyarrow.__version__, duckdb.__version__, sqlalchemy.__version__, pytest.__version__))"
+    if ($LASTEXITCODE -ne 0) {
+        throw "Development dependency import check failed."
+    }
+
+    if ($RunTests) {
+        & (Join-Path $PSScriptRoot "test-local.ps1") -Python $python
+        if ($LASTEXITCODE -ne 0) {
+            throw "The self-contained local test suite failed."
+        }
+    }
+}
+finally {
+    Pop-Location
+}
+
+Write-Host "WranglesPY Python 3.13 development environment is ready."
+Write-Host "Interpreter: $python"

scripts/test-local.ps1

@@ -0,0 +1,66 @@
+[CmdletBinding()]
+param(
+    [string]$Python = ".venv\Scripts\python.exe"
+)
+
+$ErrorActionPreference = "Stop"
+$repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot ".."))
+
+if (-not [System.IO.Path]::IsPathRooted($Python)) {
+    $Python = Join-Path $repoRoot $Python
+}
+if (-not (Test-Path -LiteralPath $Python)) {
+    throw "Python interpreter not found: $Python"
+}
+
+# The complete CI suite intentionally exercises credentialed databases, AWS,
+# WrangleWorks services, and AI/search providers. Local tests must not inherit
+# credentials and accidentally make live calls or echo a secret in a traceback.
+foreach ($name in @(
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "AWS_SESSION_TOKEN",
+    "GEMINI_API_KEY",
+    "HUGGINGFACE_TOKEN",
+    "OPENAI_API_KEY",
+    "SERPAPI_API_KEY",
+    "WRANGLES_PASSWORD",
+    "WRANGLES_USER"
+)) {
+    [Environment]::SetEnvironmentVariable($name, $null, "Process")
+}
+
+$localConfig = Join-Path $repoRoot "pytest-local.ini"
+if (-not (Test-Path -LiteralPath $localConfig)) {
+    throw "Local pytest configuration not found: $localConfig"
+}
+
+# Pytest's default per-user temp root and repository cache can become
+# inaccessible when alternating between the restricted sandbox and Eric's
+# normal Windows identity. Keep each identity's generated state separate.
+$identity = [System.Security.Principal.WindowsIdentity]::GetCurrent().Name
+$identitySlug = $identity -replace "[^A-Za-z0-9_.-]", "-"
+$testTemp = Join-Path $repoRoot ".test-local\$identitySlug\pytest"
+$testTempParent = Split-Path -Parent $testTemp
+New-Item -ItemType Directory -Path $testTempParent -Force | Out-Null
+$localState = @(
+    "--basetemp=$testTemp"
+)
+
+Push-Location $repoRoot
+try {
+    foreach ($relativePath in @("tests/temp/temp.db", "tests/temp/temp_run.db")) {
+        $generatedDatabase = Join-Path $repoRoot $relativePath
+        if (Test-Path -LiteralPath $generatedDatabase) {
+            Remove-Item -LiteralPath $generatedDatabase
+        }
+    }
+
+    & $Python -m pytest -c $localConfig @localState
+    if ($LASTEXITCODE -ne 0) {
+        throw "The self-contained local test suite failed."
+    }
+}
+finally {
+    Pop-Location
+}

setup.py

@@ -22,13 +22,17 @@
     description = 'Wrangle your data into shape with AI',
     long_description = long_description,
     long_description_content_type = 'text/markdown',
-    license_files = ('LICENSE.txt',),
+    license_files = ('LICENSE',),
     license = 'Apache License 2.0',
     classifiers = [
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
+        'Programming Language :: Python :: 3.13',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: OS Independent'
     ],
+    python_requires = '>=3.11',
     version = '1.19.0',
     url = 'https://github.com/wrangleworks/WranglesPy',
     author = 'WrangleWorks',