diff --git a/.sconsign.dblite b/.sconsign.dblite index c90be80..354013d 100644 Binary files a/.sconsign.dblite and b/.sconsign.dblite differ diff --git a/README.md b/README.md index c569717..1d75a1e 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,8 @@ In addition, each project may use other specialized tools. For the working examp - To run a given script or create a given file, run `scons path/to/script_or_file`; this will recursively run all the dependencies required to run the script or create the file. e.g. `scons output/derived/wb_clean/gdp_education.csv`. + - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each python script as a command-line argument; the script reads its TOML config and merges in the `[dev]` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.toml` and `source/derived/wb_clean/build.py` for an example). + ### SConscript files In order to integrate a new script into the SCons build, you need to modify the SConscript file in the corresponding `source/` sub-folder. For example, to add `source/derived/wb_clean/takelogs.do` to the SCons build, add an entry to `source/derived/SConscript`. In this case: diff --git a/SConstruct b/SConstruct index 3e37858..9b9d3f5 100644 --- a/SConstruct +++ b/SConstruct @@ -7,9 +7,13 @@ import source.lib.JMSLab as jms sys.path.append('config') sys.dont_write_bytecode = True # Don't write .pyc files +AddOption('--mode', dest='mode', type='string', default='full') +mode = GetOption('mode') + os.environ['PYTHONPATH'] = '.' env = Environment(ENV = {'PATH' : os.environ['PATH']}, IMPLICIT_COMMAND_DEPENDENCIES = 0, + MODE = mode, BUILDERS = {'R' : Builder(action = jms.build_r), 'Tablefill' : Builder(action = jms.build_tables), 'Stata' : Builder(action = jms.build_stata), @@ -19,7 +23,7 @@ env = Environment(ENV = {'PATH' : os.environ['PATH']}, 'Latex' : Builder(action = jms.build_latex)}) env.Decider('MD5-timestamp') # Only computes hash if time-stamp changed -Export('env') +Export('env', 'mode') jms.start_log('develop', '') diff --git a/sconstruct.log b/sconstruct.log index 416c79d..faaf9eb 100644 --- a/sconstruct.log +++ b/sconstruct.log @@ -1,7 +1,5 @@ -*** New build: {2026-05-21 13:22:36} *** +*** New build: {2026-05-13 15:42:44} *** scons: done reading SConscript files. scons: Building targets ... -build_tables(["output/tables/top_gdp.lyx"], ["source/tables/top_gdp.lyx", "output/analysis/top_gdp/top_gdp.txt", "output/analysis/top_gdp/top_gdp_gap.txt"]) -source/tables/top_gdp.lyx filled successfully by tablefill -build_lyx(["output/paper/TemplateLyx.pdf"], ["source/paper/Template.lyx", "output/tables/top_gdp.lyx", "output/analysis/top_gdp/top_gdp.tex", "source/figures/gdp_educ.lyx", "source/paper/References.bib"]) +scons: `.' is up to date. scons: done building targets. diff --git a/source/derived/SConscript b/source/derived/SConscript index 965b855..be5a97d 100644 --- a/source/derived/SConscript +++ b/source/derived/SConscript @@ -1,9 +1,13 @@ Import('*') - -target = ['#output/derived/wb_clean/gdp_education.csv'] +target = [ + '#output/derived/wb_clean/gdp_education.csv', + '#output/derived/wb_clean/gdp_education.log' +] source = ['#source/derived/wb_clean/build.py', '#source/lib/SaveData.py', + '#source/derived/wb_clean/wb_clean.toml', + env.Value(mode), '#datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv', '#datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv'] env.Python(target, source) diff --git a/source/derived/wb_clean/build.py b/source/derived/wb_clean/build.py index b0aecfd..556436a 100644 --- a/source/derived/wb_clean/build.py +++ b/source/derived/wb_clean/build.py @@ -1,3 +1,5 @@ +import argparse +import tomllib import pandas as pd from source.lib.SaveData import SaveData from pathlib import Path @@ -7,7 +9,16 @@ def Main(): raw_dir = Path("datastore/raw/world_bank/orig") out_dir = Path("output/derived/wb_clean") - df = PrepareData(raw_dir) + parser = argparse.ArgumentParser() + parser.add_argument("--mode", default="full") + mode = parser.parse_args().mode + + with open("source/derived/wb_clean/wb_clean.toml", "rb") as f: + config = tomllib.load(f) + if mode == "dev": + config.update(config.pop("dev", {})) + + df = PrepareData(raw_dir, config.get("nrows")) SaveData( df=df, keys=["Country Name"], @@ -18,16 +29,16 @@ def Main(): ) -def PrepareData(infolder): +def PrepareData(infolder, nrows): gdp_df = pd.read_csv( - infolder / "API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", header=2 + infolder / "API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", header=2, nrows=nrows ) gdp_df = gdp_df[["Country Name", "2010"]] gdp_df.rename(columns={"2010": "GDP_2010"}, inplace=True) educ_df = pd.read_csv( - infolder / "API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv", header=2 + infolder / "API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv", header=2, nrows=nrows ) educ_df = educ_df[["Country Name", "2010"]] diff --git a/source/derived/wb_clean/wb_clean.toml b/source/derived/wb_clean/wb_clean.toml new file mode 100644 index 0000000..b87e0d5 --- /dev/null +++ b/source/derived/wb_clean/wb_clean.toml @@ -0,0 +1,2 @@ +[dev] +nrows = 20 diff --git a/source/lib/JMSLab/builders/build_python.py b/source/lib/JMSLab/builders/build_python.py index f623c45..871d21c 100644 --- a/source/lib/JMSLab/builders/build_python.py +++ b/source/lib/JMSLab/builders/build_python.py @@ -35,8 +35,14 @@ class PythonBuilder(JMSLabBuilder): def add_call_args(self): ''' ''' - args = '%s %s > %s' % (os.path.normpath(self.source_file), - self.cl_arg, - os.path.normpath(self.log_file)) + mode = self.env.get('MODE') + mode_in_source = mode and any( + getattr(s, 'value', None) == mode for s in self.source + ) + mode_arg = f"--mode={mode} " if mode_in_source else '' + args = '%s %s%s > %s' % (os.path.normpath(self.source_file), + mode_arg, + self.cl_arg, + os.path.normpath(self.log_file)) self.call_args = args return None diff --git a/source/lib/JMSLab/builders/jmslab_builder.py b/source/lib/JMSLab/builders/jmslab_builder.py index 60a63da..19592d5 100644 --- a/source/lib/JMSLab/builders/jmslab_builder.py +++ b/source/lib/JMSLab/builders/jmslab_builder.py @@ -44,6 +44,7 @@ def __init__(self, target, source, env, name = 'JMSLab Builder', self.exec_opts = exec_opts # Build system call and store components + self.source = misc.make_list_if_string(source) if source else [] self.add_source_file(source) self.target = [str(t) for t in misc.make_list_if_string(target)] self.target_dir = misc.get_directory(self.target[0])