diff --git a/.gitignore b/.gitignore index 339aaa0d..c727841c 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,8 @@ pyrightconfig.json # End of https://www.toptal.com/developers/gitignore/api/python,visualstudiocode *.err *.out + +# iterate2 local run artifacts +experiment_logs/ +iterate_study.db +*.db diff --git a/setup.cfg b/setup.cfg index d266050d..3185af3e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,3 +27,7 @@ exclude = [flake8] max-line-length = 80 ignore = E203, E501, W503 +# C3 code-generation templates are string.Template files containing +# ${placeholder} substitutions; they are not valid standalone Python and +# must not be linted (they trip E999 SyntaxError). +extend-exclude = src/claimed/c3/templates diff --git a/src/claimed/c3/operator_utils.py b/src/claimed/c3/operator_utils.py index 5f524872..93049680 100644 --- a/src/claimed/c3/operator_utils.py +++ b/src/claimed/c3/operator_utils.py @@ -1,6 +1,9 @@ import contextlib import logging import os +import re +import subprocess +import time # converts string in form [cos|s3]://access_key_id:secret_access_key@endpoint/bucket/path to # access_key_id, secret_access_key, endpoint, path - path includes bucket name diff --git a/src/claimed/components/examples/fibonacci.py b/src/claimed/components/examples/fibonacci.py index 377c34ff..50240730 100644 --- a/src/claimed/components/examples/fibonacci.py +++ b/src/claimed/components/examples/fibonacci.py @@ -1,3 +1,6 @@ +import os + + def fib(b): n = int(b) if n == 0: diff --git a/src/claimed/components/util/cosutils.py b/src/claimed/components/util/cosutils.py index 8dee5445..a4913a06 100644 --- a/src/claimed/components/util/cosutils.py +++ b/src/claimed/components/util/cosutils.py @@ -65,7 +65,7 @@ def _download(s3, cos_file, local_file): desc=f'↓ {desc}', leave=True) as pbar: with s3.open(cos_file, 'rb') as f_in, open(local_file, 'wb') as f_out: while True: - chunk = f_in.read(CHUNK_SIZE) + chunk = f_in.read(MIN_CHUNK_SIZE) if not chunk: break f_out.write(chunk) diff --git a/src/claimed/mlx/s3_kv_store.py b/src/claimed/mlx/s3_kv_store.py index 40b327cd..fdb250fa 100644 --- a/src/claimed/mlx/s3_kv_store.py +++ b/src/claimed/mlx/s3_kv_store.py @@ -232,7 +232,7 @@ def main(): sp.add_argument("--filter", type=json.loads, required=True) args = parser.parse_args() - store = MLX(bucket=args.bucket, store_name=args.store, endpoint_url=args.endpoint) + store = S3KVStore(bucket=args.bucket, store_name=args.store, endpoint_url=args.endpoint) if args.cmd == "put": if args.value_file: