diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index edd67b5375..8ba18830ec 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -10,6 +10,7 @@ set(DYNAMATIC_TEST_DEPENDS split-file dynamatic-opt export-rtl + hls-fuzzer hls-fuzzer-check-bitwidth translate-llvm-to-std source-rewriter diff --git a/test/lit.cfg.py b/test/lit.cfg.py index 2adde39ef4..ca83976a0f 100644 --- a/test/lit.cfg.py +++ b/test/lit.cfg.py @@ -43,7 +43,7 @@ tool_dirs = [config.dynamatic_tools_dir, config.mlir_tools_dir, config.llvm_tools_dir] -tools = ["dynamatic-opt", "hls-fuzzer-check-bitwidth", +tools = ["dynamatic-opt", "hls-fuzzer", "hls-fuzzer-check-bitwidth", ToolSubst("%source-rewriter", command=f"cp %s %t.c && {config.dynamatic_tools_dir}/source-rewriter %t.c --"), ToolSubst("%export-vhdl", diff --git a/test/tools/hls-fuzzer/lit.local.cfg b/test/tools/hls-fuzzer/lit.local.cfg new file mode 100644 index 0000000000..2b9ca78edf --- /dev/null +++ b/test/tools/hls-fuzzer/lit.local.cfg @@ -0,0 +1 @@ +config.suffixes = [".mlir", ".test"] diff --git a/test/tools/hls-fuzzer/reproduce.test b/test/tools/hls-fuzzer/reproduce.test new file mode 100644 index 0000000000..5f2139cc00 --- /dev/null +++ b/test/tools/hls-fuzzer/reproduce.test @@ -0,0 +1,8 @@ +// RUN: rm -rf %t && mkdir -p %t +// RUN: hls-fuzzer --target random-c --single-program %t --no-verify +// RUN: mv %t/test.c %t/expected.c + +// Reproducing reads the seed and target options back from 'reproducer.json' and +// regenerates the program, which must match the one generated above. +// RUN: hls-fuzzer --reproduce %t --no-verify +// RUN: diff %t/test.c %t/expected.c diff --git a/tools/hls-fuzzer/CMakeLists.txt b/tools/hls-fuzzer/CMakeLists.txt index d8748362b4..85a6f82b37 100644 --- a/tools/hls-fuzzer/CMakeLists.txt +++ b/tools/hls-fuzzer/CMakeLists.txt @@ -12,6 +12,7 @@ add_llvm_library(DynamaticHLSFuzzer AST.cpp BasicCGenerator.cpp Randomly.cpp + Reproducer.cpp TypeSystem.cpp PARTIAL_SOURCES_INTENDED ) diff --git a/tools/hls-fuzzer/Options.h b/tools/hls-fuzzer/Options.h index 4bbfe7c937..a664600f1a 100644 --- a/tools/hls-fuzzer/Options.h +++ b/tools/hls-fuzzer/Options.h @@ -16,8 +16,18 @@ struct Options { // Path of this executable. std::string executablePath; std::string dynamaticExecutablePath; + // Arguments passed for options in the target options group, rendered so they + // can be parsed again. Stored in the reproducer of every program so it can be + // regenerated with the exact same options. See + // 'OptionsParser::getTargetArguments'. + std::vector targetArguments; OracleKind kind = OracleKind::Functional; + // Whether to skip verifying the generated program, as requested via the + // internal '--no-verify' option. Used by tests to exercise generation and + // reproduction without the heavy verification flow. + bool noVerify = false; + // Controls statistics reporting as requested via the '--statistics' option. // - If empty (nullopt), statistics collection is disabled. // - If present but the list is empty, all statistics are reported. diff --git a/tools/hls-fuzzer/OptionsParser.cpp b/tools/hls-fuzzer/OptionsParser.cpp index 4b0cac3d77..dc3eb3a1a6 100644 --- a/tools/hls-fuzzer/OptionsParser.cpp +++ b/tools/hls-fuzzer/OptionsParser.cpp @@ -2,6 +2,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Option/Arg.h" #include "llvm/Option/Option.h" #include "llvm/Support/raw_ostream.h" @@ -83,10 +84,40 @@ dynamatic::OptionsParser::getSingleProgramDirectory() const { return args.getLastArgValue(OPT_single_program).str(); } +std::optional +dynamatic::OptionsParser::getReproduceDirectory() const { + if (!args.hasArg(OPT_reproduce)) + return std::nullopt; + + return args.getLastArgValue(OPT_reproduce).str(); +} + std::string dynamatic::OptionsParser::getTargetName() const { return args.getLastArgValue(OPT_target).str(); } +std::vector +dynamatic::OptionsParser::getTargetArguments() const { + // Whether 'option' belongs to the target options group, directly or through + // one of its nested groups (e.g. the oracle options). + auto inTargetGroup = [](llvm::opt::Option option) { + for (llvm::opt::Option group = option.getGroup(); group.isValid(); + group = group.getGroup()) { + if (group.getID() == OPT_grp_target) + return true; + } + return false; + }; + + llvm::opt::ArgStringList rendered; + for (const llvm::opt::Arg *arg : args) { + if (inTargetGroup(arg->getOption())) + arg->render(args, rendered); + } + + return std::vector(rendered.begin(), rendered.end()); +} + std::optional> dynamatic::OptionsParser::getStatistics() const { if (!args.hasArg(OPT_statistics) && !args.hasArg(OPT_statistics_flag)) @@ -112,6 +143,8 @@ dynamatic::Options dynamatic::OptionsParser::apply(Options defaults) { if (arguments.size() == 1) defaults.dynamaticExecutablePath = getPositionalArguments()[0]; + defaults.targetArguments = getTargetArguments(); + defaults.noVerify = args.hasArg(OPT_no_verify); defaults.kind = args.hasFlag(OPT_functional, OPT_non_functional, defaults.kind == OracleKind::Functional) ? OracleKind::Functional diff --git a/tools/hls-fuzzer/OptionsParser.h b/tools/hls-fuzzer/OptionsParser.h index cf15245460..96b6faa3ac 100644 --- a/tools/hls-fuzzer/OptionsParser.h +++ b/tools/hls-fuzzer/OptionsParser.h @@ -40,9 +40,22 @@ class OptionsParser : llvm::opt::GenericOptTable { /// fuzz rather than work on the single program a fuzzer asked it for. std::optional getSingleProgramDirectory() const; + /// Returns the directory holding the 'reproducer.json' whose program should + /// be regenerated and verified in this process, as requested via + /// '--reproduce'. Returns 'std::nullopt' if the option was not specified, + /// i.e. this process should fuzz rather than reproduce a single program. + std::optional getReproduceDirectory() const; + /// Returns the name of the target fuzzer. std::string getTargetName() const; + /// Returns the arguments that were passed for options in the target options + /// group, rendered so that feeding them back to a new parser reconstructs the + /// exact same options. This is what a reproducer stores, which keeps it in + /// sync with the option definitions: any option added to the group is picked + /// up here automatically. + std::vector getTargetArguments() const; + /// Returns the statistics selection requested on the command line. /// Returns 'std::nullopt' if '--statistics' was not specified, an empty /// vector if it was specified without an explicit list (i.e. report all diff --git a/tools/hls-fuzzer/Opts.td b/tools/hls-fuzzer/Opts.td index 7e7a52c652..72ebb9b21f 100644 --- a/tools/hls-fuzzer/Opts.td +++ b/tools/hls-fuzzer/Opts.td @@ -2,7 +2,14 @@ include "llvm/Option/OptParser.td" class F : Flag<["-"], letter>, HelpText; -def target : Separate<["--"], "target">, MetaVarName<"">; +// Options that influence the program that is generated and how it is verified. +// Every option in this group (directly or through a nested group) is captured +// in a reproducer and replayed by '--reproduce', so adding a new option here is +// all that is needed for it to become part of reproducers. +def grp_target : OptionGroup<"Target options">; + +def target : Separate<["--"], "target">, MetaVarName<"">, + Group; def num_threads : JoinedOrSeparate<["-"], "j">, MetaVarName<"">, HelpText<"Number of threads to use">; @@ -23,6 +30,18 @@ def inplace : Flag<["--"], "inplace">, // the same in this process. def single_program : Separate<["--"], "single-program">, MetaVarName<"">, Flags<[HelpHidden]>; +// Internal option that generates a program but skips its verification. Only the +// generation is exercised, which lets tests check that a program can be +// regenerated from its reproducer without running the heavy verification flow. +// Hidden as it is not meant for normal use. Deliberately not a target option: +// it does not influence the generated program and is therefore not part of a +// reproducer. +def no_verify : Flag<["--"], "no-verify">, Flags<[HelpHidden]>; +def reproduce : Separate<["--"], "reproduce">, MetaVarName<"">, + HelpText<"Regenerate and verify, in this very process, the exact " + "program described by the 'reproducer.json' in . " + "Used to reproduce and debug a crash of the generation " + "or verification">; def help : Flag<["--"], "help">, HelpText<"displays this help text">; // Bare '--statistics' enables reporting of all statistics, while @@ -39,7 +58,7 @@ def json_output : Separate<["--"], "json-output">, "all requested statistics to as JSON instead " "of reporting them on the console">; -def grp_oracle : OptionGroup<"Oracle options">; +def grp_oracle : OptionGroup<"Oracle options">, Group; def functional : Flag<["--"], "functional">, HelpText<"Switch to functional testing">, diff --git a/tools/hls-fuzzer/Reproducer.cpp b/tools/hls-fuzzer/Reproducer.cpp new file mode 100644 index 0000000000..03e9b1886c --- /dev/null +++ b/tools/hls-fuzzer/Reproducer.cpp @@ -0,0 +1,55 @@ +#include "Reproducer.h" + +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/JSON.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +using namespace dynamatic; + +llvm::Error dynamatic::writeReproducer(const std::filesystem::path &file, + const ReproducerInfo &info) { + llvm::json::Array arguments; + for (const std::string &argument : info.arguments) + arguments.push_back(argument); + llvm::json::Value reproducer = llvm::json::Object{ + {"seed", info.seed}, + {"arguments", std::move(arguments)}, + }; + + return llvm::writeToOutput(file.string(), [&](llvm::raw_ostream &os) { + llvm::json::OStream(os, /*IndentSize=*/2).value(reproducer); + os << '\n'; + return llvm::Error::success(); + }); +} + +std::optional +dynamatic::readReproducer(const std::filesystem::path &file) { + auto fail = [&](llvm::Error error) -> std::optional { + llvm::errs() << "Failed to read '" << file.string() + << "': " << llvm::toString(std::move(error)) << '\n'; + return std::nullopt; + }; + + llvm::ErrorOr> buffer = + llvm::MemoryBuffer::getFile(file.string()); + if (std::error_code error = buffer.getError()) + return fail(llvm::errorCodeToError(error)); + + llvm::Expected value = + llvm::json::parse((*buffer)->getBuffer()); + if (!value) + return fail(value.takeError()); + + ReproducerInfo info; + std::uint64_t seed = 0; + llvm::json::Path::Root root; + llvm::json::ObjectMapper mapper(*value, root); + if (!mapper || !mapper.map("seed", seed) || + !mapper.map("arguments", info.arguments)) + return fail(root.getError()); + + info.seed = static_cast(seed); + return info; +} diff --git a/tools/hls-fuzzer/Reproducer.h b/tools/hls-fuzzer/Reproducer.h new file mode 100644 index 0000000000..2c3b955ed4 --- /dev/null +++ b/tools/hls-fuzzer/Reproducer.h @@ -0,0 +1,42 @@ +#ifndef DYNAMATIC_HLS_FUZZER_REPRODUCER +#define DYNAMATIC_HLS_FUZZER_REPRODUCER + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +#include +#include +#include +#include +#include + +namespace dynamatic { + +/// Name of the file dropped into every program's directory before generation +/// and read back by '--reproduce' to replay the exact same program. +inline constexpr llvm::StringLiteral REPRODUCER_FILE_NAME = "reproducer.json"; + +/// Everything needed to regenerate and re-verify a single program. +struct ReproducerInfo { + /// Seed the program's randomness source was created with. + std::uint32_t seed = 0; + /// The target options the program was generated with, as command-line + /// arguments that can be parsed again to reconstruct them. Storing the raw + /// arguments rather than their parsed meaning keeps the reproducer in sync + /// with the option definitions automatically. + std::vector arguments; +}; + +/// Writes 'info' to 'file' as JSON. Returns an error describing any failure to +/// do so rather than reporting it, leaving that decision to the caller. +llvm::Error writeReproducer(const std::filesystem::path &file, + const ReproducerInfo &info); + +/// Reads a reproducer written by 'writeReproducer' back from 'file'. Returns +/// 'std::nullopt' if it could not be read, reporting the reason on the console. +std::optional +readReproducer(const std::filesystem::path &file); + +} // namespace dynamatic + +#endif diff --git a/tools/hls-fuzzer/hls-fuzzer.cpp b/tools/hls-fuzzer/hls-fuzzer.cpp index 53ad836203..9be2903372 100644 --- a/tools/hls-fuzzer/hls-fuzzer.cpp +++ b/tools/hls-fuzzer/hls-fuzzer.cpp @@ -15,6 +15,7 @@ #include "Options.h" #include "OptionsParser.h" +#include "Reproducer.h" #include "TargetRegistry.h" #include "llvm/Support/Error.h" @@ -152,7 +153,7 @@ static void reportConsole(const Progress &progress) { // holding the newest of any two concurrently taken samples. static std::mutex progressMutex; static Progress totalProgress; -static const std::chrono::high_resolution_clock::time_point startTime = +static const std::chrono::high_resolution_clock::time_point START_TIME = std::chrono::high_resolution_clock::now(); /// Adds the progress 'made' to the progress of this process. May be called from @@ -167,7 +168,7 @@ static void addProgress(const Progress &made) { static void report(const dynamatic::Options &options) { std::scoped_lock lock{progressMutex}; totalProgress.duration = std::chrono::duration_cast( - std::chrono::high_resolution_clock::now() - startTime); + std::chrono::high_resolution_clock::now() - START_TIME); if (options.jsonOutput) reportJSON(*options.jsonOutput, totalProgress); @@ -175,21 +176,36 @@ static void report(const dynamatic::Options &options) { reportConsole(totalProgress); } -/// Generates a program into 'directory' and verifies it, returning what doing -/// so amounted to. +/// Generates a program into 'directory' using 'seed' as its randomness source +/// and verifies it, returning what doing so amounted to. /// /// 'directory' is wiped beforehand and serves as scratch space for the /// verification, leaving whatever it contains afterwards the reproducer of any -/// bug that was found. +/// bug that was found. A 'reproducer.json' recording 'seed' and the options is +/// dropped in before generation so that even a crash halfway through it leaves +/// enough behind for '--reproduce' to replay the program. static Progress runSingleProgram(const dynamatic::AbstractTarget &target, const dynamatic::Options &options, - const std::filesystem::path &directory) { + const std::filesystem::path &directory, + std::uint32_t seed) { std::unique_ptr worker = - target.createWorker(options, dynamatic::Randomly(std::random_device()())); + target.createWorker(options, dynamatic::Randomly(seed)); std::filesystem::remove_all(directory); std::filesystem::create_directories(directory); + // Record everything needed to replay this program before generating it, so + // that a crash halfway through generation still leaves a reproducer behind. + // A failure to do so only costs the ability to reproduce this one program and + // is therefore reported but not fatal. + dynamatic::ReproducerInfo info{seed, options.targetArguments}; + if (llvm::Error error = dynamatic::writeReproducer( + directory / dynamatic::REPRODUCER_FILE_NAME.str(), info)) { + std::scoped_lock lock{errorMutex}; + llvm::errs() << "Failed to write reproducer: " + << llvm::toString(std::move(error)) << '\n'; + } + const std::string functionName = "test"; std::filesystem::path sourceFile = directory / (functionName + ".c"); llvm::cantFail( @@ -200,8 +216,11 @@ static Progress runSingleProgram(const dynamatic::AbstractTarget &target, Progress progress; progress.numPrograms = 1; - progress.numBugs = - worker->verify(sourceFile) == dynamatic::AbstractWorker::Bug; + // '--no-verify' exercises only the generation, leaving the program unverified + // and therefore never counted as a bug. + if (!options.noVerify) + progress.numBugs = + worker->verify(sourceFile) == dynamatic::AbstractWorker::Bug; for (const dynamatic::Statistic &statistic : worker->getStatistics()) progress.merge(statistic); @@ -399,7 +418,7 @@ static Progress runProgramProcess( /// doing so amounted to, either in this process or in one of its own. static void work(const dynamatic::Options &options, const std::filesystem::path &directory, - const std::function& runProgram) { + const std::function &runProgram) { while (!quit) { if (hasProgramLimit && remainingPrograms.fetch_sub(1) <= 0) { quit = true; @@ -422,6 +441,69 @@ static void work(const dynamatic::Options &options, } } +/// Replays the program described by the reproducer in 'directory' using 'seed', +/// in this very process, so that a crash of the generation or verification can +/// be caught in a debugger. Returns a process exit code. +/// +/// An existing 'test.c' is never overwritten: the program is regenerated into +/// memory and, if a 'test.c' is already present but differs from it, an error +/// is reported and nothing is verified, since the reproducer no longer matches +/// what would be generated (e.g. because the fuzzer changed since it was +/// written). A matching 'test.c' is left in place and an absent one is written +/// out, after which the program is verified as usual. +static int reproduceProgram(const dynamatic::AbstractTarget &target, + const dynamatic::Options &options, + const std::filesystem::path &directory, + std::uint32_t seed) { + std::unique_ptr worker = + target.createWorker(options, dynamatic::Randomly(seed)); + + // Regenerate into memory first so it can be compared against an existing + // 'test.c' before anything on disk is touched. + std::string generated; + llvm::raw_string_ostream os(generated); + worker->generate(os, "test"); + os.flush(); + + std::filesystem::path sourceFile = directory / "test.c"; + if (std::filesystem::exists(sourceFile)) { + llvm::ErrorOr> existing = + llvm::MemoryBuffer::getFile(sourceFile.string()); + if (std::error_code error = existing.getError()) { + llvm::errs() << "Failed to read '" << sourceFile.string() + << "': " << error.message() << '\n'; + return -1; + } + + if ((*existing)->getBuffer() != generated) { + llvm::errs() << "Regenerated program does not match the existing '" + << sourceFile.string() + << "'; refusing to overwrite it. The reproducer no longer " + "describes this program, e.g. because the fuzzer changed " + "since it was written\n"; + return -1; + } + // Identical: leave the existing 'test.c' untouched. + } else if (llvm::Error error = llvm::writeToOutput( + sourceFile.string(), [&](llvm::raw_ostream &fileOs) { + fileOs << generated; + return llvm::Error::success(); + })) { + llvm::errs() << "Failed to write '" << sourceFile.string() + << "': " << llvm::toString(std::move(error)) << '\n'; + return -1; + } + + if (options.noVerify) + return 0; + + if (worker->verify(sourceFile) == dynamatic::AbstractWorker::Bug) + llvm::errs() << "Bug reproduced\n"; + else + llvm::errs() << "No bug found\n"; + return 0; +} + int main(int argc, char **argv) { auto signalHandler = +[](int) { interrupted = true; @@ -443,6 +525,55 @@ int main(int argc, char **argv) { dynamatic::TargetRegistry &instance = dynamatic::TargetRegistry::getInstance(); + dynamatic::Options defaults{}; +#pragma clang diagnostic ignored "-Wmain" + defaults.executablePath = llvm::sys::fs::getMainExecutable( + argv[0], reinterpret_cast(&main)); + defaults.dynamaticExecutablePath = + std::filesystem::path(defaults.executablePath).parent_path() / + "dynamatic"; + + auto options = optionsParser.apply(defaults); + + // Reproduce a single program from a previously written reproducer, in this + // very process rather than in one of its own, so that a crash of the + // generation or verification can be caught in a debugger. The target options + // are read back from the reproducer and parsed through the very same + // command-line logic used for a normal run, so nothing about how they map to + // options is duplicated here. + if (std::optional directory = + optionsParser.getReproduceDirectory()) { + std::optional info = + dynamatic::readReproducer(std::filesystem::path(*directory) / + dynamatic::REPRODUCER_FILE_NAME.str()); + if (!info) + return -1; + + llvm::SmallVector storage; + storage.emplace_back(defaults.executablePath); + llvm::append_range(storage, info->arguments); + llvm::SmallVector reproArgs; + for (std::string &arg : storage) + reproArgs.emplace_back(arg.data()); + + dynamatic::OptionsParser reproParser(reproArgs); + std::string targetName = reproParser.getTargetName(); + std::unique_ptr target = + instance.getTarget(targetName); + if (!target) { + llvm::errs() << "Unknown target '" << targetName << "' in reproducer\n"; + return -1; + } + + // Build the options off this invocation's own so that flags which are not + // part of the reproducer, such as '--no-verify' and the dynamatic path, + // still take effect. The reproducer only carries the target options, hence + // 'noVerify' must be restored from the actual invocation afterwards. + dynamatic::Options reproOptions = reproParser.apply(options); + reproOptions.noVerify = options.noVerify; + return reproduceProgram(*target, reproOptions, *directory, info->seed); + } + std::string targetName = optionsParser.getTargetName(); if (targetName.empty()) { llvm::errs() << "Missing '--target' argument\n"; @@ -459,22 +590,13 @@ int main(int argc, char **argv) { return -1; } - dynamatic::Options defaults{}; -#pragma clang diagnostic ignored "-Wmain" - defaults.executablePath = llvm::sys::fs::getMainExecutable( - argv[0], reinterpret_cast(&main)); - defaults.dynamaticExecutablePath = - std::filesystem::path(defaults.executablePath).parent_path() / - "dynamatic"; - - auto options = optionsParser.apply(defaults); - // Work on the single program a fuzzing process asked for and report what that // amounted to, which is all the process learns about a program of its own // that crashed halfway through. if (std::optional directory = optionsParser.getSingleProgramDirectory()) { - addProgress(runSingleProgram(*target, options, *directory)); + addProgress( + runSingleProgram(*target, options, *directory, std::random_device()())); report(options); return 0; } @@ -504,7 +626,8 @@ int main(int argc, char **argv) { std::function runProgram; if (inplace) { runProgram = [&target, &options, directory] { - return runSingleProgram(*target, options, directory); + return runSingleProgram(*target, options, directory, + std::random_device()()); }; } else { runProgram = [&options, args = llvm::ArrayRef(args), directory,