From ce29fdeee5f377cd42c54bcb88eae44c42990a84 Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Tue, 12 May 2026 10:11:53 -0400 Subject: [PATCH 1/9] dev mode architecture for #159 --- SConstruct | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/SConstruct b/SConstruct index 3e37858..de5e241 100644 --- a/SConstruct +++ b/SConstruct @@ -7,6 +7,15 @@ import source.lib.JMSLab as jms sys.path.append('config') sys.dont_write_bytecode = True # Don't write .pyc files +AddOption('--mode', dest='build_mode', type='string', default='full') +mode = GetOption('build_mode') + +def resolve_config(config, mode): + overrides = config.pop('dev', {}) + if mode == 'dev': + config.update(overrides) + return config + os.environ['PYTHONPATH'] = '.' env = Environment(ENV = {'PATH' : os.environ['PATH']}, IMPLICIT_COMMAND_DEPENDENCIES = 0, @@ -19,7 +28,7 @@ env = Environment(ENV = {'PATH' : os.environ['PATH']}, 'Latex' : Builder(action = jms.build_latex)}) env.Decider('MD5-timestamp') # Only computes hash if time-stamp changed -Export('env') +Export('env', 'mode', 'resolve_config') jms.start_log('develop', '') From d027bdb2fe462250b5a7081986f7adf4e441d396 Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Tue, 12 May 2026 10:28:49 -0400 Subject: [PATCH 2/9] MWE for #159 --- source/derived/SConscript | 9 +++++++++ source/derived/wb_clean/build.py | 12 ++++++++---- source/derived/wb_clean/wb_clean.json | 6 ++++++ 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 source/derived/wb_clean/wb_clean.json diff --git a/source/derived/SConscript b/source/derived/SConscript index 965b855..877fad3 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -1,9 +1,18 @@ Import('*') +import os +import json + +with open(File('#source/derived/wb_clean/wb_clean.json')) as f: + config = resolve_config(json.load(f), mode) +os.makedirs(Dir('#temp/derived/wb_clean'), exist_ok=True) +with open(File('#temp/derived/wb_clean/wb_clean.json'), 'w') as f: + json.dump(config, f) target = ['#output/derived/wb_clean/gdp_education.csv'] source = ['#source/derived/wb_clean/build.py', '#source/lib/SaveData.py', + '#temp/derived/wb_clean/wb_clean.json', '#datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv', '#datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv'] env.Python(target, source) diff --git a/source/derived/wb_clean/build.py b/source/derived/wb_clean/build.py index b0aecfd..ea059e8 100644 --- a/source/derived/wb_clean/build.py +++ b/source/derived/wb_clean/build.py @@ -1,3 +1,4 @@ +import json import pandas as pd from source.lib.SaveData import SaveData from pathlib import Path @@ -7,7 +8,10 @@ def Main(): raw_dir = Path("datastore/raw/world_bank/orig") out_dir = Path("output/derived/wb_clean") - df = PrepareData(raw_dir) + with open("temp/derived/wb_clean/wb_clean.json") as f: + config = json.load(f) + + df = PrepareData(raw_dir, config["nrows"]) SaveData( df=df, keys=["Country Name"], @@ -18,16 +22,16 @@ def Main(): ) -def PrepareData(infolder): +def PrepareData(infolder, nrows): gdp_df = pd.read_csv( - infolder / "API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", header=2 + infolder / "API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", header=2, nrows=nrows ) gdp_df = gdp_df[["Country Name", "2010"]] gdp_df.rename(columns={"2010": "GDP_2010"}, inplace=True) educ_df = pd.read_csv( - infolder / "API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv", header=2 + infolder / "API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv", header=2, nrows=nrows ) educ_df = educ_df[["Country Name", "2010"]] diff --git a/source/derived/wb_clean/wb_clean.json b/source/derived/wb_clean/wb_clean.json new file mode 100644 index 0000000..999d75f --- /dev/null +++ b/source/derived/wb_clean/wb_clean.json @@ -0,0 +1,6 @@ +{ + "nrows": null, + "dev": { + "nrows": 20 + } +} From 76a783556978346725a4731e13d04d2206710c39 Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Tue, 12 May 2026 10:32:44 -0400 Subject: [PATCH 3/9] documentation for #159 --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c569717..f72c12f 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ In addition, each project may use other specialized tools. For the working examp - To run a given script or create a given file, run `scons path/to/script_or_file`; this will recursively run all the dependencies required to run the script or create the file. e.g. `scons output/derived/wb_clean/gdp_education.csv`. + - For a faster, lower-fidelity build for development, run `scons --mode=dev`. Each mode-sensitive module reads its dev overrides from the `dev` key of its config JSON (see `source/derived/wb_clean/wb_clean.json` for an example). + ### SConscript files In order to integrate a new script into the SCons build, you need to modify the SConscript file in the corresponding `source/` sub-folder. For example, to add `source/derived/wb_clean/takelogs.do` to the SCons build, add an entry to `source/derived/SConscript`. In this case: From 9e425c8e27a4dae330cf7ee5801f9c720c42fbed Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Tue, 12 May 2026 10:43:24 -0400 Subject: [PATCH 4/9] streamline for #159 --- SConstruct | 8 +++++++- source/derived/SConscript | 9 +-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/SConstruct b/SConstruct index de5e241..3d8de73 100644 --- a/SConstruct +++ b/SConstruct @@ -1,6 +1,7 @@ # Preliminaries import os import sys +import json import atexit import source.lib.JMSLab as jms @@ -10,10 +11,15 @@ sys.dont_write_bytecode = True # Don't write .pyc files AddOption('--mode', dest='build_mode', type='string', default='full') mode = GetOption('build_mode') -def resolve_config(config, mode): +def resolve_config(source_path, mode): + src = File(source_path).abspath + dst = File('#temp/' + source_path.removeprefix('#source/')).abspath + config = json.load(open(src)) overrides = config.pop('dev', {}) if mode == 'dev': config.update(overrides) + os.makedirs(os.path.dirname(dst), exist_ok=True) + json.dump(config, open(dst, 'w')) return config os.environ['PYTHONPATH'] = '.' diff --git a/source/derived/SConscript b/source/derived/SConscript index 877fad3..a0bfe93 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -1,13 +1,6 @@ Import('*') -import os -import json - -with open(File('#source/derived/wb_clean/wb_clean.json')) as f: - config = resolve_config(json.load(f), mode) -os.makedirs(Dir('#temp/derived/wb_clean'), exist_ok=True) -with open(File('#temp/derived/wb_clean/wb_clean.json'), 'w') as f: - json.dump(config, f) +resolve_config('#source/derived/wb_clean/wb_clean.json', mode) target = ['#output/derived/wb_clean/gdp_education.csv'] source = ['#source/derived/wb_clean/build.py', From d178a8cea7822a98c13ed4f87bcd61c49b3e1f5a Mon Sep 17 00:00:00 2001 From: zhizhongpu <84325421+zhizhongpu@users.noreply.github.com> Date: Tue, 12 May 2026 21:33:06 -0400 Subject: [PATCH 5/9] #159 fx missing scons log file --- source/derived/SConscript | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/derived/SConscript b/source/derived/SConscript index a0bfe93..0d5383a 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -2,7 +2,10 @@ Import('*') resolve_config('#source/derived/wb_clean/wb_clean.json', mode) -target = ['#output/derived/wb_clean/gdp_education.csv'] +target = [ + '#output/derived/wb_clean/gdp_education.csv', + '#output/derived/wb_clean/gdp_education.log' +] source = ['#source/derived/wb_clean/build.py', '#source/lib/SaveData.py', '#temp/derived/wb_clean/wb_clean.json', From cc7a75e2d8fb9a21818ac6e243ed622d3c3455a2 Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Wed, 13 May 2026 11:56:13 -0400 Subject: [PATCH 6/9] refactor for #162 --- .sconsign.dblite | Bin 29177 -> 30101 bytes README.md | 2 +- SConstruct | 18 +++--------------- sconstruct.log | 4 ++-- source/derived/SConscript | 7 +++---- source/derived/wb_clean/build.py | 9 ++++++++- 6 files changed, 17 insertions(+), 23 deletions(-) diff --git a/.sconsign.dblite b/.sconsign.dblite index 63381820ea151693d2847e533b1f39e4dfcec6fb..5143031b4546bf2182cff21ed81716ff4635f69e 100644 GIT binary patch delta 2370 zcmb_eU2GIp6n4ulw%7^bqdjZ^xw)wU81 z1~nK0tUh>_#P9&Tsc+&Y8lxDE@zEHgPn!4>^+Al{gT@$*dS~vg0bkI>O=frY+_~pF z-*?Z?{Q6bm^mmEW6Uoyj7cZnXByQYwHhHL=oS%G5Rf?Ud>TqH-nLL$yc@j>gj+vWP zOE)(D+;H7>de5cm=+dnTV}AB}KFA&9Gmm;gF`G-*AvX09b1aMqL4wOl*gHC|0m5wY z=@fJ|!t!#j=yG+&y9p49f7$6X)WHbqonE8>8gUHJV?h@^# zumxszwZj{YtD0U}8b|cPQaHcsrJ-vEQf^w9oKf=&Y-X+!hg9dR<9f5{=A+EADJGn# zfFQy+79PVw1=O;rL%1s{VcTdVOPDUcwlZulG%WIP#rh{Jk5|@(tIFNirqE_6qzhZj z?sDtyk<#pZ{;rZ#`Fr{wT?k9Fm2%5oHod4K+pgK5RDsIh5XoS1db$GFFLN8lPQq== zZfkh!Bs{Tf3*1@Q4XrnF-`QF^=4It8zw4YPG5PLEcPW3JX^dk<>~)ETq^e6uvy<3MTX(U7Sum& zwtt+jS>v^G>w|LcfRc-XElieX^V22hPIshR!%Vm)>`EUY}uCZ-N3PZg)ySU6GZ2?D-n&nJra{GK+nAyXdb^8zF)C3 z=UXU1(zi6E#i{Khi75(%u(*dkCy1jk`-!@Q>*O5{RjCJn0z9$^& z>5Oz;j<_LxXKb8Gj}2PNVMO>!mnXNm z>tLc?>sTm7jOwJ1d`6q}fghQJ@!jp!-_i$avh2@16QfiwXPRrKKUa^{(x_tyruKfG z(>`Qi>1ub0NJa(-JC)(^VW;8H-1U58 z?a?fpyT85qZr6AXHgk(G&mO(ycw?Ns*&>2-`zBzjyU)Z3-7|@nXsKS>8=jlkXEsgs z{$XxrvSSAOV%CRm_kRSN2C~NRW8X33H(lHdrGuGjW$o@7_IJ0QY^p(~6OfpAz~Ft_ zKeEJl?H$-`oUr=M`GHMl=`(GK_Ci-K1LdinhOiHgJh%o*st3q{UU)TkG$)YHkrWQn zf-%gkKnf4pj=~B_N&L9?JQgcB`U|kZ&^Ua5u;#frKWr?;d`DK=7CROcUO!giHl zzRp}mv9BAo?xhj_W_T|za|Am+v1}rulyj*Rf#(K+v;^ja2EG%W+*1ZOQ{A(1n~|e> z2JK9=6t%ss^ZV5wtTTzIEuq^l=(bcHc0Q>|x5hnHm+ss3(Cv1?7egsH-q{9krv{+a z>#MJJCSmKL_Uaz*L`~D1x3&MbI%3;yiUfwYhI--1=E)rL14W7OJnjTIklYf|5rJSX z5-xMyxCoEO?1~s#hi13LN1QHpCTneLrwTK=vvugJv)p|xVd3J?)SwqoW-(plzN-+` zod^Y3V2?-Bj0RqedCDj{wxYUc_`Uy&tP}Xy;{wDEY&pKibT`3_xrFlQD32Mk z`=Pjf&Lqq`+dKXZuXT_e85#5}W>KarPuWUIJrx6DBjxHj?`zI7hB#rkJ+-L{Bdhb* z9;#POU_C`AWHMLCwr9(E6lTg4J3<>o+i6QX^bh)Dr@epg-FNK|Z3?tbJA$Hu3ex-QRH%T3#3%_Y z2#OR;xFsYcNJuosghcDZ#9%PdMAt4zG`c{7E)@AOaYN$5`{uPoSCy5EJGnV?-*?XW z&iT$cbt7^2dZP4D^3<)pv!xx0ihk7e(sXia;&nZfzmyF761imZ`)#R$Jyv=+dHdA| z3-;k9-DSH<7AV2S10u`0&_Drg4CB5}fj|Uv;B#n9AQe+Hg{Iu; z;d6yTbG|ld>&vWVsZ6TAv|GFJI<;?NQcvyUlY3^0X!GGZ$hXEb-C9zi zg`w0yAh?lKb0)DTJ)xDx+yzmYdw;|QQI$WxqL8+;W2EH3MLW23ZON^Rwk_XpH&!*; zYfDEHIh!H%_PhL5yS}a^x$^OI1^Y(%-Z6i9GdbDJ}(1phBGlgb% zR!=t|Um?vgz?cODD&=#^F_i|904aq0nL?|*uyIRi);;gGWt&=T@5^cXQ$*1l9_95P5(FMqt4x2b5B0^X=88HMZxe3!RXt z03)G^2p}OwCz|(2lvAMHj0dO`GCEHeLF0|NhHxFe2 zzy=c_0YFAD=fMXa@}W zy9-H#;!;ozsJ7qKebvSVGe$G!VPQN<1dxs~ECR&2!huI3y3RVW*pKT+7J#eFOeTp- z+#bp3VpR5aCd#|jf>kHW4qbXTs%x{o{CqmRw(7}5{KRjR@#Spen`M!NohpI7yluNZ z+SC%~Hrq6rsQCPYoA)L{#|k^p_}tv<(U0t*9clYs%k^aR#vWTeRu)C!e4VteIUAJl z5hL6Sln*)A5K#(U;XE9;4i#A0Rvy}|+2%O*iq^w1a8>?Udt6k9pR{gH#K@`kw^MTw zRwbeciW&^>wht|iQy9qh#2^RVCCb-+k$o)w_OEQu+TyoWc~CMh(u`AFgF8*r4TNgS zG{R7Oz$nR;a?=Ay=)-_83Y>rQz+VM3__Da{mPK-V7X(O=$z#xhV0vc#c84(%- z>P2OHr^tA2%pUA6!~x#u?uq9{2k470J3(zT;ZSkU2Z(VVP=cgF0T6*8KH$cwC-%M4 zemc=$uLITTD=D>mhlTPOqYOJC2a>tDf}ALr6J%xXbnH7FPQk&r(&1-lI8p8`ue(>q z&TttylZd9?6T`F7)Ju%ig!}w2=Ng#>$L8t&wcYjy_4c=5V%}1hv@h;*=!KM4JtsSl3`K|4CaBZ|XF AE&u=k diff --git a/README.md b/README.md index f72c12f..762ad08 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ In addition, each project may use other specialized tools. For the working examp - To run a given script or create a given file, run `scons path/to/script_or_file`; this will recursively run all the dependencies required to run the script or create the file. e.g. `scons output/derived/wb_clean/gdp_education.csv`. - - For a faster, lower-fidelity build for development, run `scons --mode=dev`. Each mode-sensitive module reads its dev overrides from the `dev` key of its config JSON (see `source/derived/wb_clean/wb_clean.json` for an example). + - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each analysis script as a command-line argument; the script reads its config JSON and merges in the `dev` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.json` and `source/derived/wb_clean/build.py` for an example). ### SConscript files diff --git a/SConstruct b/SConstruct index 3d8de73..74383ad 100644 --- a/SConstruct +++ b/SConstruct @@ -1,26 +1,14 @@ # Preliminaries import os import sys -import json import atexit import source.lib.JMSLab as jms sys.path.append('config') sys.dont_write_bytecode = True # Don't write .pyc files -AddOption('--mode', dest='build_mode', type='string', default='full') -mode = GetOption('build_mode') - -def resolve_config(source_path, mode): - src = File(source_path).abspath - dst = File('#temp/' + source_path.removeprefix('#source/')).abspath - config = json.load(open(src)) - overrides = config.pop('dev', {}) - if mode == 'dev': - config.update(overrides) - os.makedirs(os.path.dirname(dst), exist_ok=True) - json.dump(config, open(dst, 'w')) - return config +AddOption('--mode', dest='mode', type='string', default='full') +mode = GetOption('mode') os.environ['PYTHONPATH'] = '.' env = Environment(ENV = {'PATH' : os.environ['PATH']}, @@ -34,7 +22,7 @@ env = Environment(ENV = {'PATH' : os.environ['PATH']}, 'Latex' : Builder(action = jms.build_latex)}) env.Decider('MD5-timestamp') # Only computes hash if time-stamp changed -Export('env', 'mode', 'resolve_config') +Export('env', 'mode') jms.start_log('develop', '') diff --git a/sconstruct.log b/sconstruct.log index 3e9afe6..4e69906 100644 --- a/sconstruct.log +++ b/sconstruct.log @@ -1,5 +1,5 @@ -*** New build: {2025-08-15 10:50:03} *** +*** New build: {2026-05-13 11:54:21} *** scons: done reading SConscript files. scons: Building targets ... -build_python(["output/derived/wb_clean/gdp_education.csv"], ["source/derived/wb_clean/build.py", "source/lib/SaveData.py", "datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", "datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv"]) +scons: `.' is up to date. scons: done building targets. diff --git a/source/derived/SConscript b/source/derived/SConscript index 0d5383a..977c77e 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -1,17 +1,16 @@ Import('*') -resolve_config('#source/derived/wb_clean/wb_clean.json', mode) - target = [ '#output/derived/wb_clean/gdp_education.csv', '#output/derived/wb_clean/gdp_education.log' ] source = ['#source/derived/wb_clean/build.py', '#source/lib/SaveData.py', - '#temp/derived/wb_clean/wb_clean.json', + '#source/derived/wb_clean/wb_clean.json', + env.Value(mode), '#datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv', '#datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv'] -env.Python(target, source) +env.Python(target, source, CL_ARG = f'--mode={mode}') target = ['#output/derived/wb_clean/gdp_education_logs.csv'] source = ['#source/derived/wb_clean/takelogs.do', diff --git a/source/derived/wb_clean/build.py b/source/derived/wb_clean/build.py index ea059e8..870c75a 100644 --- a/source/derived/wb_clean/build.py +++ b/source/derived/wb_clean/build.py @@ -1,3 +1,4 @@ +import argparse import json import pandas as pd from source.lib.SaveData import SaveData @@ -8,8 +9,14 @@ def Main(): raw_dir = Path("datastore/raw/world_bank/orig") out_dir = Path("output/derived/wb_clean") - with open("temp/derived/wb_clean/wb_clean.json") as f: + parser = argparse.ArgumentParser() + parser.add_argument("--mode", default="full") + mode = parser.parse_args().mode + + with open("source/derived/wb_clean/wb_clean.json") as f: config = json.load(f) + if mode == "dev": + config.update(config.pop("dev", {})) df = PrepareData(raw_dir, config["nrows"]) SaveData( From 20a239c608607664b3764b26364732fa274572f3 Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Wed, 13 May 2026 12:02:48 -0400 Subject: [PATCH 7/9] json to toml for #162 --- .sconsign.dblite | Bin 30101 -> 30238 bytes README.md | 2 +- sconstruct.log | 2 +- source/derived/SConscript | 2 +- source/derived/wb_clean/build.py | 8 ++++---- source/derived/wb_clean/wb_clean.json | 6 ------ source/derived/wb_clean/wb_clean.toml | 2 ++ 7 files changed, 9 insertions(+), 13 deletions(-) delete mode 100644 source/derived/wb_clean/wb_clean.json create mode 100644 source/derived/wb_clean/wb_clean.toml diff --git a/.sconsign.dblite b/.sconsign.dblite index 5143031b4546bf2182cff21ed81716ff4635f69e..688546062d199c57641f7da0332ad649a8aff718 100644 GIT binary patch delta 990 zcmb7DJ7`o<5cTb5qw7C@qC&EMA}c~x=e_%VV4{d3BrK>Xh!%Q3b4@g>fdqfx2eGh7 zqj!Ui=xb~&1bJYkt#&pF778L37Ft+|VB>weSgX}Ab7#(+IWu?uYxLk-RJqYJzxu+{ z%1CeZT6w{O;LVit@!m(#uw{-h3WkG3rO;XG93n`SNaQJq_QaO46AydNObj*W^MOT! z(Tcfc7xhUqh3edn+4HR_zqzt^*ifxdI&CGoLt#pdS)iCV7obA@wZRayji^aAG)sXcI_YU%U>_P5mgJq?TMke zF<+yD{$2B<-#@UmWHWzx*`2~HUH-F#neScky=n`({moyIpFX%Q?_W6@Rk0g-yTkXL z=Dz3f=QVlDP`@6wPRD6zrX5PgFsNm0ZEL8@CKYP2*rpF ziHNo7rgnO6vVPBzNxx+Efx0xtnpjH05sh}K)_^<~C=`(iW4c?FA6@gRTymMOt$!a( z{%ifYau_E+J1`j4sbDMyOsHduhh?V;8k_2b0gX9HRRPh;{FA}ArB;^Ts*OgATAe}1 zhy07<>+(+QTNCD@Rw@xQYYTFhvb+`FEtL*E%p=-g zt^b>neq`ijT?i{Vf-sbzwUlcPsu-Ue0n{KV;IC%+_R*jdC=Fa_BU2mBH;_-jDgqiA z5FuD+K{vB}YWJ7&e^3~fBz!6EBo8uZVmMVvjLaymrPYvhV6Su!>v(>4--f8300R?S d;9Lq~1jA4jgh>n|EZhPXq_1cBYB(h|*y#cA}-h1sgt%YIpH_vGTB zf3(x(W|OdSSAUe9#obNy4SzuQ!zI}e`78T7{@fJD{5>_~?*bgU4!o^P@xH#dFQc94 zeGv6%AQUhfgH%o;1)b3h#MmMWj3Gj2`g0Zi!jJEMxTn=Ok937IU?J)x2nzyZqO&k4 zH`+1koPioC!kxew^=1;j!~3~hF$s5YOa5O|e%bf369{KH(TpkVkTVKugSFF=0!<~Q zl|pVM;Yojc)X5x|P^Lu*Y5_Nv0f#hV3J4^*V-@V(B&;3zRP!Gey0H@os+h(KVQ4Hs z5DO;L`jM2BaFuPiP;GcPd^^4)H3B8%(n4lqYYoT5kr-H-{vweYG!@3}B%qV4hkgT= CCnSOZ diff --git a/README.md b/README.md index 762ad08..4a28a56 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ In addition, each project may use other specialized tools. For the working examp - To run a given script or create a given file, run `scons path/to/script_or_file`; this will recursively run all the dependencies required to run the script or create the file. e.g. `scons output/derived/wb_clean/gdp_education.csv`. - - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each analysis script as a command-line argument; the script reads its config JSON and merges in the `dev` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.json` and `source/derived/wb_clean/build.py` for an example). + - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each analysis script as a command-line argument; the script reads its TOML config and merges in the `[dev]` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.toml` and `source/derived/wb_clean/build.py` for an example). ### SConscript files diff --git a/sconstruct.log b/sconstruct.log index 4e69906..af7028a 100644 --- a/sconstruct.log +++ b/sconstruct.log @@ -1,4 +1,4 @@ -*** New build: {2026-05-13 11:54:21} *** +*** New build: {2026-05-13 12:02:17} *** scons: done reading SConscript files. scons: Building targets ... scons: `.' is up to date. diff --git a/source/derived/SConscript b/source/derived/SConscript index 977c77e..af36aff 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -6,7 +6,7 @@ target = [ ] source = ['#source/derived/wb_clean/build.py', '#source/lib/SaveData.py', - '#source/derived/wb_clean/wb_clean.json', + '#source/derived/wb_clean/wb_clean.toml', env.Value(mode), '#datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv', '#datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv'] diff --git a/source/derived/wb_clean/build.py b/source/derived/wb_clean/build.py index 870c75a..556436a 100644 --- a/source/derived/wb_clean/build.py +++ b/source/derived/wb_clean/build.py @@ -1,5 +1,5 @@ import argparse -import json +import tomllib import pandas as pd from source.lib.SaveData import SaveData from pathlib import Path @@ -13,12 +13,12 @@ def Main(): parser.add_argument("--mode", default="full") mode = parser.parse_args().mode - with open("source/derived/wb_clean/wb_clean.json") as f: - config = json.load(f) + with open("source/derived/wb_clean/wb_clean.toml", "rb") as f: + config = tomllib.load(f) if mode == "dev": config.update(config.pop("dev", {})) - df = PrepareData(raw_dir, config["nrows"]) + df = PrepareData(raw_dir, config.get("nrows")) SaveData( df=df, keys=["Country Name"], diff --git a/source/derived/wb_clean/wb_clean.json b/source/derived/wb_clean/wb_clean.json deleted file mode 100644 index 999d75f..0000000 --- a/source/derived/wb_clean/wb_clean.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "nrows": null, - "dev": { - "nrows": 20 - } -} diff --git a/source/derived/wb_clean/wb_clean.toml b/source/derived/wb_clean/wb_clean.toml new file mode 100644 index 0000000..b87e0d5 --- /dev/null +++ b/source/derived/wb_clean/wb_clean.toml @@ -0,0 +1,2 @@ +[dev] +nrows = 20 From ecba3645480c6e491f62c4ede960ed40f34e822e Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Wed, 13 May 2026 15:45:44 -0400 Subject: [PATCH 8/9] python builder for #162 --- .sconsign.dblite | Bin 30238 -> 30238 bytes SConstruct | 1 + sconstruct.log | 2 +- source/derived/SConscript | 2 +- source/lib/JMSLab/builders/build_python.py | 12 +++++++++--- source/lib/JMSLab/builders/jmslab_builder.py | 1 + 6 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.sconsign.dblite b/.sconsign.dblite index 688546062d199c57641f7da0332ad649a8aff718..354013d24723d2746cbfc8ea0cdfa93ae2c0675a 100644 GIT binary patch delta 773 zcmb7?yH8X>6vi_P5{NEpV0k2h7?TylvdsGt1ry^d=*AE_CB%6K(D;a*oe7;r5@v~w zp|dwmp^cs5AK-0lv@#Y_SRh(*?=FRZxR%D@y~7Z>Qs(>sR7j@A*(Mb#v&=ZG zor6L!tdJu?Qxg<{)F_IbYc5r0M_M(n-it5hmt(Ws3(Ep~&F%c$Use|#95v39Gk_hRwgu%+t4ozSBKX!BvwAp!lFgO%H481E2WNCHy zZbt#Tb8H0El-7hw7xGhKW-Zpj5M)!N_jR#h0U@G@7OI@flEP zW2~>Mdp~GB-PGM6-amD6|D|y|J~IpP38ZOa{PTf4K6cfoi?oaN=U~!1jD=@TLgX|y zK3J?wfJhj`Jf^VlKhg4^{kEXc_LkdtDtHi3LJ692xH7BsV&sZNKYJ*94b_# zj%R5Xt(VFRe~tBZ+Mow}{!c~t+zWZ)7zOr zQ03fEf*}T>3G!2U;QcmH+?% delta 779 zcmb7COG}ke6y;n^l+p^ds}Lmwx5C#x@AE+#L{IAy;v`I>^F&R{Xc9t1;jBpTC{4n? zN%PznB1FlxWB-aPxb`OeIu zU#%WYC(zqHZ7y$~ojy?gdOXfuJ|D+BPL*69*E;@`d>)tg{wS*SVBZuf%JgM_9~$72 zFs(S|kcli7OcW+0wcVT1i)RD~bz8k8J-`Z^S2NDn0_1`j36wRp4XG&`7QA%+OOb7`RbEhpZT7 zBp96u<(&~2q&FNw-O@sR{@CxWcx|vJcF**uOR$70)=6W$VwwwPl_FR;Tcp68(h_s$ zYSQoKX$!BE+Q*;oEcZiNW4qh_HTR8?jR7g05+0;w)H}z75uj^12`hnFPivFzHS(tE z^=KJYKpPO!Se3beU15%Z(-e&7K&9mMr9RBluan=J{zr(Sjs+2vC}J|y3Z@D6+-hNz uGeGLJiwz-@>F4QQR3X5@Vh2JfNv-6*=qi#TmQ!AwlCMd>pC@|u`|w}EiurH= diff --git a/SConstruct b/SConstruct index 74383ad..9b9d3f5 100644 --- a/SConstruct +++ b/SConstruct @@ -13,6 +13,7 @@ mode = GetOption('mode') os.environ['PYTHONPATH'] = '.' env = Environment(ENV = {'PATH' : os.environ['PATH']}, IMPLICIT_COMMAND_DEPENDENCIES = 0, + MODE = mode, BUILDERS = {'R' : Builder(action = jms.build_r), 'Tablefill' : Builder(action = jms.build_tables), 'Stata' : Builder(action = jms.build_stata), diff --git a/sconstruct.log b/sconstruct.log index af7028a..faaf9eb 100644 --- a/sconstruct.log +++ b/sconstruct.log @@ -1,4 +1,4 @@ -*** New build: {2026-05-13 12:02:17} *** +*** New build: {2026-05-13 15:42:44} *** scons: done reading SConscript files. scons: Building targets ... scons: `.' is up to date. diff --git a/source/derived/SConscript b/source/derived/SConscript index af36aff..be5a97d 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -10,7 +10,7 @@ source = ['#source/derived/wb_clean/build.py', env.Value(mode), '#datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv', '#datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv'] -env.Python(target, source, CL_ARG = f'--mode={mode}') +env.Python(target, source) target = ['#output/derived/wb_clean/gdp_education_logs.csv'] source = ['#source/derived/wb_clean/takelogs.do', diff --git a/source/lib/JMSLab/builders/build_python.py b/source/lib/JMSLab/builders/build_python.py index f623c45..871d21c 100644 --- a/source/lib/JMSLab/builders/build_python.py +++ b/source/lib/JMSLab/builders/build_python.py @@ -35,8 +35,14 @@ class PythonBuilder(JMSLabBuilder): def add_call_args(self): ''' ''' - args = '%s %s > %s' % (os.path.normpath(self.source_file), - self.cl_arg, - os.path.normpath(self.log_file)) + mode = self.env.get('MODE') + mode_in_source = mode and any( + getattr(s, 'value', None) == mode for s in self.source + ) + mode_arg = f"--mode={mode} " if mode_in_source else '' + args = '%s %s%s > %s' % (os.path.normpath(self.source_file), + mode_arg, + self.cl_arg, + os.path.normpath(self.log_file)) self.call_args = args return None diff --git a/source/lib/JMSLab/builders/jmslab_builder.py b/source/lib/JMSLab/builders/jmslab_builder.py index 60a63da..19592d5 100644 --- a/source/lib/JMSLab/builders/jmslab_builder.py +++ b/source/lib/JMSLab/builders/jmslab_builder.py @@ -44,6 +44,7 @@ def __init__(self, target, source, env, name = 'JMSLab Builder', self.exec_opts = exec_opts # Build system call and store components + self.source = misc.make_list_if_string(source) if source else [] self.add_source_file(source) self.target = [str(t) for t in misc.make_list_if_string(target)] self.target_dir = misc.get_directory(self.target[0]) From e5a064bda984a0f0a58981c86465c6cd4b73a5b4 Mon Sep 17 00:00:00 2001 From: simonessigaberg <95505173+simonessigaberg@users.noreply.github.com> Date: Wed, 20 May 2026 11:22:30 -0400 Subject: [PATCH 9/9] update readme for #162 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a28a56..1d75a1e 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ In addition, each project may use other specialized tools. For the working examp - To run a given script or create a given file, run `scons path/to/script_or_file`; this will recursively run all the dependencies required to run the script or create the file. e.g. `scons output/derived/wb_clean/gdp_education.csv`. - - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each analysis script as a command-line argument; the script reads its TOML config and merges in the `[dev]` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.toml` and `source/derived/wb_clean/build.py` for an example). + - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each python script as a command-line argument; the script reads its TOML config and merges in the `[dev]` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.toml` and `source/derived/wb_clean/build.py` for an example). ### SConscript files