Pull Request for #159: dev mode

README.md

@@ -72,6 +72,8 @@ In addition, each project may use other specialized tools. For the working examp
 
     - To run a given script or create a given file, run `scons path/to/script_or_file`; this will recursively run all the dependencies required to run the script or create the file.  e.g.  `scons output/derived/wb_clean/gdp_education.csv`.
 
+    - For a faster, lower-fidelity build for development, run `scons --mode=dev`. SCons passes the mode to each python script as a command-line argument; the script reads its TOML config and merges in the `[dev]` overrides when the mode is `dev` (see `source/derived/wb_clean/wb_clean.toml` and `source/derived/wb_clean/build.py` for an example).
+
 ### SConscript files
 
 In order to integrate a new script into the SCons build, you need to modify the SConscript file in the corresponding `source/` sub-folder.  For example, to add `source/derived/wb_clean/takelogs.do` to the SCons build, add an entry to `source/derived/SConscript`. In this case:

SConstruct

@@ -7,9 +7,13 @@ import source.lib.JMSLab as jms
 sys.path.append('config')
 sys.dont_write_bytecode = True # Don't write .pyc files
 
+AddOption('--mode', dest='mode', type='string', default='full')
+mode = GetOption('mode')
+
 os.environ['PYTHONPATH'] = '.'
 env = Environment(ENV = {'PATH' : os.environ['PATH']},
                   IMPLICIT_COMMAND_DEPENDENCIES = 0,
+                  MODE = mode,
                   BUILDERS = {'R'         : Builder(action = jms.build_r),
                               'Tablefill' : Builder(action = jms.build_tables),
                               'Stata'     : Builder(action = jms.build_stata),
@@ -19,7 +23,7 @@ env = Environment(ENV = {'PATH' : os.environ['PATH']},
                               'Latex'     : Builder(action = jms.build_latex)})
 
 env.Decider('MD5-timestamp') # Only computes hash if time-stamp changed
-Export('env')
+Export('env', 'mode')
 
 jms.start_log('develop', '')
 

sconstruct.log

@@ -1,7 +1,5 @@
-*** New build: {2026-05-21 13:22:36} ***
+*** New build: {2026-05-13 15:42:44} ***
 scons: done reading SConscript files.
 scons: Building targets ...
-build_tables(["output/tables/top_gdp.lyx"], ["source/tables/top_gdp.lyx", "output/analysis/top_gdp/top_gdp.txt", "output/analysis/top_gdp/top_gdp_gap.txt"])
-source/tables/top_gdp.lyx filled successfully by tablefill
-build_lyx(["output/paper/TemplateLyx.pdf"], ["source/paper/Template.lyx", "output/tables/top_gdp.lyx", "output/analysis/top_gdp/top_gdp.tex", "source/figures/gdp_educ.lyx", "source/paper/References.bib"])
+scons: `.' is up to date.
 scons: done building targets.

source/derived/SConscript

@@ -1,9 +1,13 @@
 Import('*')
 
-
-target = ['#output/derived/wb_clean/gdp_education.csv']
+target = [
+    '#output/derived/wb_clean/gdp_education.csv',
+    '#output/derived/wb_clean/gdp_education.log'
+]
 source = ['#source/derived/wb_clean/build.py',
           '#source/lib/SaveData.py',
+          '#source/derived/wb_clean/wb_clean.toml',
+          env.Value(mode),
           '#datastore/raw/world_bank/orig/API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv',
           '#datastore/raw/world_bank/orig/API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv']
 env.Python(target, source)

source/derived/wb_clean/build.py

@@ -1,3 +1,5 @@
+import argparse
+import tomllib
 import pandas as pd
 from source.lib.SaveData import SaveData
 from pathlib import Path
@@ -7,7 +9,16 @@ def Main():
     raw_dir = Path("datastore/raw/world_bank/orig")
     out_dir = Path("output/derived/wb_clean")
 
-    df = PrepareData(raw_dir)
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--mode", default="full")
+    mode = parser.parse_args().mode
+
+    with open("source/derived/wb_clean/wb_clean.toml", "rb") as f:
+        config = tomllib.load(f)
+    if mode == "dev":
+        config.update(config.pop("dev", {}))
+
+    df = PrepareData(raw_dir, config.get("nrows"))
     SaveData(
         df=df,
         keys=["Country Name"],
@@ -18,16 +29,16 @@ def Main():
     )
 
 
-def PrepareData(infolder):
+def PrepareData(infolder, nrows):
     gdp_df = pd.read_csv(
-        infolder / "API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", header=2
+        infolder / "API_NY.GDP.PCAP.CD_DS2_en_csv_v2_1740213.csv", header=2, nrows=nrows
     )
 
     gdp_df = gdp_df[["Country Name", "2010"]]
     gdp_df.rename(columns={"2010": "GDP_2010"}, inplace=True)
 
     educ_df = pd.read_csv(
-        infolder / "API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv", header=2
+        infolder / "API_SE.XPD.TOTL.GD.ZS_DS2_en_csv_v2_1740282.csv", header=2, nrows=nrows
     )
 
     educ_df = educ_df[["Country Name", "2010"]]

source/derived/wb_clean/wb_clean.toml

@@ -0,0 +1,2 @@
+[dev]
+nrows = 20

source/lib/JMSLab/builders/build_python.py

@@ -35,8 +35,14 @@ class PythonBuilder(JMSLabBuilder):
     def add_call_args(self):
         '''
         '''
-        args = '%s %s > %s' % (os.path.normpath(self.source_file),
-                               self.cl_arg,
-                               os.path.normpath(self.log_file))
+        mode = self.env.get('MODE')
+        mode_in_source = mode and any(
+            getattr(s, 'value', None) == mode for s in self.source
+        )
+        mode_arg = f"--mode={mode} " if mode_in_source else ''
+        args = '%s %s%s > %s' % (os.path.normpath(self.source_file),
+                                 mode_arg,
+                                 self.cl_arg,
+                                 os.path.normpath(self.log_file))
         self.call_args = args
         return None

source/lib/JMSLab/builders/jmslab_builder.py

@@ -44,6 +44,7 @@ def __init__(self, target, source, env, name = 'JMSLab Builder',
         self.exec_opts        = exec_opts
 
         # Build system call and store components
+        self.source           = misc.make_list_if_string(source) if source else []
         self.add_source_file(source)
         self.target           = [str(t) for t in misc.make_list_if_string(target)]
         self.target_dir       = misc.get_directory(self.target[0])