[hls-fuzzer] Add reproducers for test cases

test/CMakeLists.txt

@@ -10,6 +10,7 @@ set(DYNAMATIC_TEST_DEPENDS
   split-file
   dynamatic-opt
   export-rtl
+  hls-fuzzer
   hls-fuzzer-check-bitwidth
   translate-llvm-to-std
   source-rewriter

test/lit.cfg.py

@@ -43,7 +43,7 @@
 
 tool_dirs = [config.dynamatic_tools_dir,
              config.mlir_tools_dir, config.llvm_tools_dir]
-tools = ["dynamatic-opt", "hls-fuzzer-check-bitwidth",
+tools = ["dynamatic-opt", "hls-fuzzer", "hls-fuzzer-check-bitwidth",
          ToolSubst("%source-rewriter",
                    command=f"cp %s %t.c && {config.dynamatic_tools_dir}/source-rewriter %t.c --"),
          ToolSubst("%export-vhdl",

test/tools/hls-fuzzer/lit.local.cfg

@@ -0,0 +1 @@
+config.suffixes = [".mlir", ".test"]

test/tools/hls-fuzzer/reproduce.test

@@ -0,0 +1,8 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: hls-fuzzer --target random-c --single-program %t --no-verify
+// RUN: mv %t/test.c %t/expected.c
+
+// Reproducing reads the seed and target options back from 'reproducer.json' and
+// regenerates the program, which must match the one generated above.
+// RUN: hls-fuzzer --reproduce %t --no-verify
+// RUN: diff %t/test.c %t/expected.c

tools/hls-fuzzer/CMakeLists.txt

@@ -12,6 +12,7 @@ add_llvm_library(DynamaticHLSFuzzer
         AST.cpp
         BasicCGenerator.cpp
         Randomly.cpp
+        Reproducer.cpp
         TypeSystem.cpp
         PARTIAL_SOURCES_INTENDED
 )

tools/hls-fuzzer/Options.h

@@ -16,8 +16,18 @@ struct Options {
   // Path of this executable.
   std::string executablePath;
   std::string dynamaticExecutablePath;
+  // Arguments passed for options in the target options group, rendered so they
+  // can be parsed again. Stored in the reproducer of every program so it can be
+  // regenerated with the exact same options. See
+  // 'OptionsParser::getTargetArguments'.
+  std::vector<std::string> targetArguments;
   OracleKind kind = OracleKind::Functional;
 
+  // Whether to skip verifying the generated program, as requested via the
+  // internal '--no-verify' option. Used by tests to exercise generation and
+  // reproduction without the heavy verification flow.
+  bool noVerify = false;
+
   // Controls statistics reporting as requested via the '--statistics' option.
   // - If empty (nullopt), statistics collection is disabled.
   // - If present but the list is empty, all statistics are reported.

tools/hls-fuzzer/OptionsParser.cpp

@@ -2,6 +2,7 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Option/Arg.h"
 #include "llvm/Option/Option.h"
 #include "llvm/Support/raw_ostream.h"
 
@@ -83,10 +84,40 @@ dynamatic::OptionsParser::getSingleProgramDirectory() const {
   return args.getLastArgValue(OPT_single_program).str();
 }
 
+std::optional<std::string>
+dynamatic::OptionsParser::getReproduceDirectory() const {
+  if (!args.hasArg(OPT_reproduce))
+    return std::nullopt;
+
+  return args.getLastArgValue(OPT_reproduce).str();
+}
+
 std::string dynamatic::OptionsParser::getTargetName() const {
   return args.getLastArgValue(OPT_target).str();
 }
 
+std::vector<std::string>
+dynamatic::OptionsParser::getTargetArguments() const {
+  // Whether 'option' belongs to the target options group, directly or through
+  // one of its nested groups (e.g. the oracle options).
+  auto inTargetGroup = [](llvm::opt::Option option) {
+    for (llvm::opt::Option group = option.getGroup(); group.isValid();
+         group = group.getGroup()) {
+      if (group.getID() == OPT_grp_target)
+        return true;
+    }
+    return false;
+  };
+
+  llvm::opt::ArgStringList rendered;
+  for (const llvm::opt::Arg *arg : args) {
+    if (inTargetGroup(arg->getOption()))
+      arg->render(args, rendered);
+  }
+
+  return std::vector<std::string>(rendered.begin(), rendered.end());
+}
+
 std::optional<std::vector<std::string>>
 dynamatic::OptionsParser::getStatistics() const {
   if (!args.hasArg(OPT_statistics) && !args.hasArg(OPT_statistics_flag))
@@ -112,6 +143,8 @@ dynamatic::Options dynamatic::OptionsParser::apply(Options defaults) {
   if (arguments.size() == 1)
     defaults.dynamaticExecutablePath = getPositionalArguments()[0];
 
+  defaults.targetArguments = getTargetArguments();
+  defaults.noVerify = args.hasArg(OPT_no_verify);
   defaults.kind = args.hasFlag(OPT_functional, OPT_non_functional,
                                defaults.kind == OracleKind::Functional)
                       ? OracleKind::Functional

tools/hls-fuzzer/OptionsParser.h

@@ -40,9 +40,22 @@ class OptionsParser : llvm::opt::GenericOptTable {
   /// fuzz rather than work on the single program a fuzzer asked it for.
   std::optional<std::string> getSingleProgramDirectory() const;
 
+  /// Returns the directory holding the 'reproducer.json' whose program should
+  /// be regenerated and verified in this process, as requested via
+  /// '--reproduce'. Returns 'std::nullopt' if the option was not specified,
+  /// i.e. this process should fuzz rather than reproduce a single program.
+  std::optional<std::string> getReproduceDirectory() const;
+
   /// Returns the name of the target fuzzer.
   std::string getTargetName() const;
 
+  /// Returns the arguments that were passed for options in the target options
+  /// group, rendered so that feeding them back to a new parser reconstructs the
+  /// exact same options. This is what a reproducer stores, which keeps it in
+  /// sync with the option definitions: any option added to the group is picked
+  /// up here automatically.
+  std::vector<std::string> getTargetArguments() const;
+
   /// Returns the statistics selection requested on the command line.
   /// Returns 'std::nullopt' if '--statistics' was not specified, an empty
   /// vector if it was specified without an explicit list (i.e. report all

tools/hls-fuzzer/Opts.td

@@ -2,7 +2,14 @@ include "llvm/Option/OptParser.td"
 
 class F<string letter, string help> : Flag<["-"], letter>, HelpText<help>;
 
-def target : Separate<["--"], "target">, MetaVarName<"<target>">;
+// Options that influence the program that is generated and how it is verified.
+// Every option in this group (directly or through a nested group) is captured
+// in a reproducer and replayed by '--reproduce', so adding a new option here is
+// all that is needed for it to become part of reproducers.
+def grp_target : OptionGroup<"Target options">;
+
+def target : Separate<["--"], "target">, MetaVarName<"<target>">,
+             Group<grp_target>;
 def num_threads : JoinedOrSeparate<["-"], "j">,
                   MetaVarName<"<num-threads>">,
                   HelpText<"Number of threads to use">;
@@ -23,6 +30,18 @@ def inplace : Flag<["--"], "inplace">,
 // the same in this process.
 def single_program : Separate<["--"], "single-program">, MetaVarName<"<dir>">,
                      Flags<[HelpHidden]>;
+// Internal option that generates a program but skips its verification. Only the
+// generation is exercised, which lets tests check that a program can be
+// regenerated from its reproducer without running the heavy verification flow.
+// Hidden as it is not meant for normal use. Deliberately not a target option:
+// it does not influence the generated program and is therefore not part of a
+// reproducer.
+def no_verify : Flag<["--"], "no-verify">, Flags<[HelpHidden]>;
+def reproduce : Separate<["--"], "reproduce">, MetaVarName<"<dir>">,
+                HelpText<"Regenerate and verify, in this very process, the exact "
+                         "program described by the 'reproducer.json' in <dir>. "
+                         "Used to reproduce and debug a crash of the generation "
+                         "or verification">;
 def help : Flag<["--"], "help">, HelpText<"displays this help text">;
 
 // Bare '--statistics' enables reporting of all statistics, while
@@ -39,7 +58,7 @@ def json_output : Separate<["--"], "json-output">,
                            "all requested statistics to <file> as JSON instead "
                            "of reporting them on the console">;
 
-def grp_oracle : OptionGroup<"Oracle options">;
+def grp_oracle : OptionGroup<"Oracle options">, Group<grp_target>;
 
 def functional : Flag<["--"], "functional">,
                  HelpText<"Switch to functional testing">,

tools/hls-fuzzer/Reproducer.cpp

@@ -0,0 +1,55 @@
+#include "Reproducer.h"
+
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace dynamatic;
+
+llvm::Error dynamatic::writeReproducer(const std::filesystem::path &file,
+                                       const ReproducerInfo &info) {
+  llvm::json::Array arguments;
+  for (const std::string &argument : info.arguments)
+    arguments.push_back(argument);
+  llvm::json::Value reproducer = llvm::json::Object{
+      {"seed", info.seed},
+      {"arguments", std::move(arguments)},
+  };
+
+  return llvm::writeToOutput(file.string(), [&](llvm::raw_ostream &os) {
+    llvm::json::OStream(os, /*IndentSize=*/2).value(reproducer);
+    os << '\n';
+    return llvm::Error::success();
+  });
+}
+
+std::optional<ReproducerInfo>
+dynamatic::readReproducer(const std::filesystem::path &file) {
+  auto fail = [&](llvm::Error error) -> std::optional<ReproducerInfo> {
+    llvm::errs() << "Failed to read '" << file.string()
+                 << "': " << llvm::toString(std::move(error)) << '\n';
+    return std::nullopt;
+  };
+
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> buffer =
+      llvm::MemoryBuffer::getFile(file.string());
+  if (std::error_code error = buffer.getError())
+    return fail(llvm::errorCodeToError(error));
+
+  llvm::Expected<llvm::json::Value> value =
+      llvm::json::parse((*buffer)->getBuffer());
+  if (!value)
+    return fail(value.takeError());
+
+  ReproducerInfo info;
+  std::uint64_t seed = 0;
+  llvm::json::Path::Root root;
+  llvm::json::ObjectMapper mapper(*value, root);
+  if (!mapper || !mapper.map("seed", seed) ||
+      !mapper.map("arguments", info.arguments))
+    return fail(root.getError());
+
+  info.seed = static_cast<std::uint32_t>(seed);
+  return info;
+}

tools/hls-fuzzer/Reproducer.h

@@ -0,0 +1,42 @@
+#ifndef DYNAMATIC_HLS_FUZZER_REPRODUCER
+#define DYNAMATIC_HLS_FUZZER_REPRODUCER
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
+
+#include <cstdint>
+#include <filesystem>
+#include <optional>
+#include <string>
+#include <vector>
+
+namespace dynamatic {
+
+/// Name of the file dropped into every program's directory before generation
+/// and read back by '--reproduce' to replay the exact same program.
+inline constexpr llvm::StringLiteral REPRODUCER_FILE_NAME = "reproducer.json";
+
+/// Everything needed to regenerate and re-verify a single program.
+struct ReproducerInfo {
+  /// Seed the program's randomness source was created with.
+  std::uint32_t seed = 0;
+  /// The target options the program was generated with, as command-line
+  /// arguments that can be parsed again to reconstruct them. Storing the raw
+  /// arguments rather than their parsed meaning keeps the reproducer in sync
+  /// with the option definitions automatically.
+  std::vector<std::string> arguments;
+};
+
+/// Writes 'info' to 'file' as JSON. Returns an error describing any failure to
+/// do so rather than reporting it, leaving that decision to the caller.
+llvm::Error writeReproducer(const std::filesystem::path &file,
+                            const ReproducerInfo &info);
+
+/// Reads a reproducer written by 'writeReproducer' back from 'file'. Returns
+/// 'std::nullopt' if it could not be read, reporting the reason on the console.
+std::optional<ReproducerInfo>
+readReproducer(const std::filesystem::path &file);
+
+} // namespace dynamatic
+
+#endif