From b11d4d749a8b7e876800213275a0fc7797c8cb4b Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sat, 30 Apr 2022 23:37:58 -0700 Subject: [PATCH 01/51] Added custom_generator.fbs --- buildcc/schema/cmake/schema_generate.cmake | 2 ++ buildcc/schema/custom_generator.fbs | 26 ++++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 buildcc/schema/custom_generator.fbs diff --git a/buildcc/schema/cmake/schema_generate.cmake b/buildcc/schema/cmake/schema_generate.cmake index cbbe4b23..4ba97f55 100644 --- a/buildcc/schema/cmake/schema_generate.cmake +++ b/buildcc/schema/cmake/schema_generate.cmake @@ -3,11 +3,13 @@ set(SCHEMA_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/generated CACHE PATH "Generate set(FBS_FILES ${CMAKE_CURRENT_SOURCE_DIR}/path.fbs + ${CMAKE_CURRENT_SOURCE_DIR}/custom_generator.fbs ${CMAKE_CURRENT_SOURCE_DIR}/generator.fbs ${CMAKE_CURRENT_SOURCE_DIR}/target.fbs ) set(FBS_GEN_FILES ${SCHEMA_BUILD_DIR}/path_generated.h + ${SCHEMA_BUILD_DIR}/custom_generator_generated.h ${SCHEMA_BUILD_DIR}/generator_generated.h ${SCHEMA_BUILD_DIR}/target_generated.h ) diff --git a/buildcc/schema/custom_generator.fbs b/buildcc/schema/custom_generator.fbs new file mode 100644 index 00000000..33de0087 --- /dev/null +++ b/buildcc/schema/custom_generator.fbs @@ -0,0 +1,26 @@ +// Copyright 2021-2022 Niket Naidu. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +include "path.fbs"; + +namespace schema.internal; + +// Custom generator regenerate conditions +table CustomGenerator { + name:string (key); + recheck_path_dep:[Path]; + recheck_string_dep:[string]; +} + +root_type CustomGenerator; From 85aa66027734281ed0c5fb42b9c3fadb85249e30 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sat, 30 Apr 2022 23:50:22 -0700 Subject: [PATCH 02/51] Added custom_generator_serialization.h/.cpp --- buildcc/schema/cmake/schema.cmake | 2 + .../schema/custom_generator_serialization.h | 57 +++++++++++++++ .../src/custom_generator_serialization.cpp | 69 +++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 buildcc/schema/include/schema/custom_generator_serialization.h create mode 100644 buildcc/schema/src/custom_generator_serialization.cpp diff --git a/buildcc/schema/cmake/schema.cmake b/buildcc/schema/cmake/schema.cmake index b3c996f8..712c8f3a 100644 --- a/buildcc/schema/cmake/schema.cmake +++ b/buildcc/schema/cmake/schema.cmake @@ -1,6 +1,7 @@ # schema test if (${TESTING}) add_library(mock_schema STATIC + src/custom_generator_serialization.cpp src/generator_serialization.cpp src/target_serialization.cpp @@ -8,6 +9,7 @@ if (${TESTING}) include/schema/interface/serialization_interface.h include/schema/path.h + include/schema/custom_generator_serialization.h include/schema/generator_serialization.h include/schema/target_serialization.h ) diff --git a/buildcc/schema/include/schema/custom_generator_serialization.h b/buildcc/schema/include/schema/custom_generator_serialization.h new file mode 100644 index 00000000..adbb2292 --- /dev/null +++ b/buildcc/schema/include/schema/custom_generator_serialization.h @@ -0,0 +1,57 @@ +/* + * Copyright 2021-2022 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SCHEMA_GENERATOR_SERIALIZATION_H_ +#define SCHEMA_GENERATOR_SERIALIZATION_H_ + +#include +#include +#include + +#include "schema/path.h" + +#include "schema/interface/serialization_interface.h" + +namespace buildcc::internal { + +struct CustomGeneratorSchema { + std::string name; + path_unordered_set recheck_path_dep; + std::unordered_set recheck_string_dep; +}; + +class CustomGeneratorSerialization : public SerializationInterface { +public: + CustomGeneratorSerialization(const fs::path &serialized_file) + : SerializationInterface(serialized_file) {} + + void UpdateStore(const CustomGeneratorSchema &store) { store_ = store; } + const CustomGeneratorSchema &GetLoad() const { return load_; } + const CustomGeneratorSchema &GetStore() const { return store_; } + +private: + bool Verify(const std::string &serialized_data) override; + bool Load(const std::string &serialized_data) override; + bool Store(const fs::path &absolute_serialized_file) override; + +private: + CustomGeneratorSchema load_; + CustomGeneratorSchema store_; +}; + +} // namespace buildcc::internal + +#endif diff --git a/buildcc/schema/src/custom_generator_serialization.cpp b/buildcc/schema/src/custom_generator_serialization.cpp new file mode 100644 index 00000000..f9da5719 --- /dev/null +++ b/buildcc/schema/src/custom_generator_serialization.cpp @@ -0,0 +1,69 @@ +/* + * Copyright 2021-2022 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "schema/custom_generator_serialization.h" + +// Third party +#include "flatbuffers/flatbuffers.h" + +// Private +#include "schema/private/schema_util.h" + +// Schema generated +#include "custom_generator_generated.h" + +namespace buildcc::internal { + +// PRIVATE + +bool CustomGeneratorSerialization::Verify(const std::string &serialized_data) { + flatbuffers::Verifier verifier((const uint8_t *)serialized_data.c_str(), + serialized_data.length()); + return fbs::VerifyCustomGeneratorBuffer(verifier); +} + +bool CustomGeneratorSerialization::Load(const std::string &serialized_data) { + const auto *custom_generator = + fbs::GetCustomGenerator((const void *)serialized_data.c_str()); + if (custom_generator == nullptr) { + return false; + } + + extract_path(custom_generator->recheck_path_dep(), load_.recheck_path_dep); + extract(custom_generator->recheck_string_dep(), load_.recheck_string_dep); + return true; +} + +bool CustomGeneratorSerialization::Store( + const fs::path &absolute_serialized_file) { + flatbuffers::FlatBufferBuilder builder; + + auto fbs_recheck_path_dep = + internal::create_fbs_vector_path(builder, store_.recheck_path_dep); + auto fbs_recheck_string_dep = + internal::create_fbs_vector_string(builder, store_.recheck_string_dep); + + auto fbs_generator = fbs::CreateCustomGeneratorDirect( + builder, store_.name.c_str(), &fbs_recheck_path_dep, + &fbs_recheck_string_dep); + fbs::FinishCustomGeneratorBuffer(builder, fbs_generator); + + return env::save_file(path_as_string(absolute_serialized_file).c_str(), + (const char *)builder.GetBufferPointer(), + builder.GetSize(), true); +} + +} // namespace buildcc::internal From d0e60fce8c1b28b832fe380a0b030ac1714bc31e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 1 May 2022 00:06:58 -0700 Subject: [PATCH 03/51] Updated schema --- .../schema/include/schema/custom_generator_serialization.h | 6 +++--- buildcc/schema/src/custom_generator_serialization.cpp | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/buildcc/schema/include/schema/custom_generator_serialization.h b/buildcc/schema/include/schema/custom_generator_serialization.h index adbb2292..ade71d80 100644 --- a/buildcc/schema/include/schema/custom_generator_serialization.h +++ b/buildcc/schema/include/schema/custom_generator_serialization.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef SCHEMA_GENERATOR_SERIALIZATION_H_ -#define SCHEMA_GENERATOR_SERIALIZATION_H_ +#ifndef SCHEMA_CUSTOM_GENERATOR_SERIALIZATION_H_ +#define SCHEMA_CUSTOM_GENERATOR_SERIALIZATION_H_ #include #include @@ -29,7 +29,7 @@ namespace buildcc::internal { struct CustomGeneratorSchema { std::string name; - path_unordered_set recheck_path_dep; + path_unordered_set internal_recheck_path_dep; std::unordered_set recheck_string_dep; }; diff --git a/buildcc/schema/src/custom_generator_serialization.cpp b/buildcc/schema/src/custom_generator_serialization.cpp index f9da5719..b050df40 100644 --- a/buildcc/schema/src/custom_generator_serialization.cpp +++ b/buildcc/schema/src/custom_generator_serialization.cpp @@ -42,7 +42,8 @@ bool CustomGeneratorSerialization::Load(const std::string &serialized_data) { return false; } - extract_path(custom_generator->recheck_path_dep(), load_.recheck_path_dep); + extract_path(custom_generator->recheck_path_dep(), + load_.internal_recheck_path_dep); extract(custom_generator->recheck_string_dep(), load_.recheck_string_dep); return true; } @@ -51,8 +52,8 @@ bool CustomGeneratorSerialization::Store( const fs::path &absolute_serialized_file) { flatbuffers::FlatBufferBuilder builder; - auto fbs_recheck_path_dep = - internal::create_fbs_vector_path(builder, store_.recheck_path_dep); + auto fbs_recheck_path_dep = internal::create_fbs_vector_path( + builder, store_.internal_recheck_path_dep); auto fbs_recheck_string_dep = internal::create_fbs_vector_string(builder, store_.recheck_string_dep); From 27529deba33e880bcf32ee76404ca2ab388415f2 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 1 May 2022 02:45:57 -0700 Subject: [PATCH 04/51] Updated schema --- buildcc/schema/cmake/schema.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/buildcc/schema/cmake/schema.cmake b/buildcc/schema/cmake/schema.cmake index 712c8f3a..8dc2422e 100644 --- a/buildcc/schema/cmake/schema.cmake +++ b/buildcc/schema/cmake/schema.cmake @@ -36,6 +36,7 @@ if (${TESTING}) endif() set(SCHEMA_SRCS + src/custom_generator_serialization.cpp src/generator_serialization.cpp src/target_serialization.cpp From 6d7aab6f2b89733414d41a20d186febabf38728e Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 1 May 2022 20:53:08 -0700 Subject: [PATCH 05/51] Updated CustomGenerator schema --- buildcc/schema/custom_generator.fbs | 10 ++++- .../schema/custom_generator_serialization.h | 8 +++- .../src/custom_generator_serialization.cpp | 42 ++++++++++++++----- 3 files changed, 46 insertions(+), 14 deletions(-) diff --git a/buildcc/schema/custom_generator.fbs b/buildcc/schema/custom_generator.fbs index 33de0087..8c5ce146 100644 --- a/buildcc/schema/custom_generator.fbs +++ b/buildcc/schema/custom_generator.fbs @@ -16,11 +16,17 @@ include "path.fbs"; namespace schema.internal; +// Relational input and output files +table RelInputOutput { + id:string (key); + inputs:[Path]; + outputs:[string]; +} + // Custom generator regenerate conditions table CustomGenerator { name:string (key); - recheck_path_dep:[Path]; - recheck_string_dep:[string]; + rels:[RelInputOutput]; } root_type CustomGenerator; diff --git a/buildcc/schema/include/schema/custom_generator_serialization.h b/buildcc/schema/include/schema/custom_generator_serialization.h index ade71d80..0b440d3d 100644 --- a/buildcc/schema/include/schema/custom_generator_serialization.h +++ b/buildcc/schema/include/schema/custom_generator_serialization.h @@ -27,10 +27,14 @@ namespace buildcc::internal { +struct RelInputOutputSchema { + path_unordered_set internal_inputs; + fs_unordered_set outputs; +}; + struct CustomGeneratorSchema { std::string name; - path_unordered_set internal_recheck_path_dep; - std::unordered_set recheck_string_dep; + std::unordered_map internal_rels_map; }; class CustomGeneratorSerialization : public SerializationInterface { diff --git a/buildcc/schema/src/custom_generator_serialization.cpp b/buildcc/schema/src/custom_generator_serialization.cpp index b050df40..9b91b592 100644 --- a/buildcc/schema/src/custom_generator_serialization.cpp +++ b/buildcc/schema/src/custom_generator_serialization.cpp @@ -42,9 +42,24 @@ bool CustomGeneratorSerialization::Load(const std::string &serialized_data) { return false; } - extract_path(custom_generator->recheck_path_dep(), - load_.internal_recheck_path_dep); - extract(custom_generator->recheck_string_dep(), load_.recheck_string_dep); + const auto *fbs_rels = custom_generator->rels(); + if (fbs_rels == nullptr) { + return false; + } + + for (const auto *rel_io : *fbs_rels) { + if (rel_io == nullptr) { + return false; + } + RelInputOutputSchema schema; + if (rel_io->id()->c_str() == nullptr) { + return false; + } + extract_path(rel_io->inputs(), schema.internal_inputs); + extract(rel_io->outputs(), schema.outputs); + // load_.internal_rels.push_back(schema); + load_.internal_rels_map.emplace(rel_io->id()->c_str(), std::move(schema)); + } return true; } @@ -52,14 +67,21 @@ bool CustomGeneratorSerialization::Store( const fs::path &absolute_serialized_file) { flatbuffers::FlatBufferBuilder builder; - auto fbs_recheck_path_dep = internal::create_fbs_vector_path( - builder, store_.internal_recheck_path_dep); - auto fbs_recheck_string_dep = - internal::create_fbs_vector_string(builder, store_.recheck_string_dep); + std::vector> fbs_rels; + for (const auto &rel_miter : store_.internal_rels_map) { + const auto &id = rel_miter.first; + const auto &rel_io = rel_miter.second; + auto fbs_internal_inputs = + internal::create_fbs_vector_path(builder, rel_io.internal_inputs); + auto fbs_outputs = + internal::create_fbs_vector_string(builder, rel_io.outputs); + auto fbs_rel = fbs::CreateRelInputOutputDirect( + builder, id.c_str(), &fbs_internal_inputs, &fbs_outputs); + fbs_rels.push_back(fbs_rel); + } - auto fbs_generator = fbs::CreateCustomGeneratorDirect( - builder, store_.name.c_str(), &fbs_recheck_path_dep, - &fbs_recheck_string_dep); + auto fbs_generator = + fbs::CreateCustomGeneratorDirect(builder, store_.name.c_str(), &fbs_rels); fbs::FinishCustomGeneratorBuffer(builder, fbs_generator); return env::save_file(path_as_string(absolute_serialized_file).c_str(), From 357d3a70cc359e4001f133847de197d3f02efb22 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 1 May 2022 23:16:26 -0700 Subject: [PATCH 06/51] Updated schema --- .../schema/src/custom_generator_serialization.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/buildcc/schema/src/custom_generator_serialization.cpp b/buildcc/schema/src/custom_generator_serialization.cpp index 9b91b592..49eac4e3 100644 --- a/buildcc/schema/src/custom_generator_serialization.cpp +++ b/buildcc/schema/src/custom_generator_serialization.cpp @@ -38,26 +38,18 @@ bool CustomGeneratorSerialization::Verify(const std::string &serialized_data) { bool CustomGeneratorSerialization::Load(const std::string &serialized_data) { const auto *custom_generator = fbs::GetCustomGenerator((const void *)serialized_data.c_str()); - if (custom_generator == nullptr) { - return false; - } + // Verified, does not need a nullptr check const auto *fbs_rels = custom_generator->rels(); if (fbs_rels == nullptr) { return false; } + // rel_io->id is a required parameter, hence rel_io cannot be nullptr for (const auto *rel_io : *fbs_rels) { - if (rel_io == nullptr) { - return false; - } RelInputOutputSchema schema; - if (rel_io->id()->c_str() == nullptr) { - return false; - } extract_path(rel_io->inputs(), schema.internal_inputs); extract(rel_io->outputs(), schema.outputs); - // load_.internal_rels.push_back(schema); load_.internal_rels_map.emplace(rel_io->id()->c_str(), std::move(schema)); } return true; From 6a8b4cf047b5cf35f5127a95a012cd4ce502d677 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 1 May 2022 23:21:59 -0700 Subject: [PATCH 07/51] Added schema test for custom_generator_serialization --- buildcc/schema/cmake/schema.cmake | 9 +- buildcc/schema/test/.gitignore | 1 + buildcc/schema/test/dump/.gitkeep | 0 .../test_custom_generator_serialization.cpp | 83 +++++++++++++++++++ 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 buildcc/schema/test/.gitignore create mode 100644 buildcc/schema/test/dump/.gitkeep create mode 100644 buildcc/schema/test/test_custom_generator_serialization.cpp diff --git a/buildcc/schema/cmake/schema.cmake b/buildcc/schema/cmake/schema.cmake index 8dc2422e..01acf57f 100644 --- a/buildcc/schema/cmake/schema.cmake +++ b/buildcc/schema/cmake/schema.cmake @@ -32,7 +32,14 @@ if (${TESTING}) target_link_options(mock_schema PUBLIC ${TEST_LINK_FLAGS} ${BUILD_LINK_FLAGS}) # Tests - add_dependencies(mock_schema fbs_to_header) + add_executable(test_custom_generator_serialization + test/test_custom_generator_serialization.cpp + ) + target_link_libraries(test_custom_generator_serialization PRIVATE mock_schema) + + add_test(NAME test_custom_generator_serialization COMMAND test_custom_generator_serialization + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test + ) endif() set(SCHEMA_SRCS diff --git a/buildcc/schema/test/.gitignore b/buildcc/schema/test/.gitignore new file mode 100644 index 00000000..a8a0dcec --- /dev/null +++ b/buildcc/schema/test/.gitignore @@ -0,0 +1 @@ +*.bin diff --git a/buildcc/schema/test/dump/.gitkeep b/buildcc/schema/test/dump/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/buildcc/schema/test/test_custom_generator_serialization.cpp b/buildcc/schema/test/test_custom_generator_serialization.cpp new file mode 100644 index 00000000..1d69366c --- /dev/null +++ b/buildcc/schema/test/test_custom_generator_serialization.cpp @@ -0,0 +1,83 @@ +#include "schema/custom_generator_serialization.h" + +#include "schema/private/schema_util.h" + +#include "flatbuffers/flatbuffers.h" + +#include "custom_generator_generated.h" + +// NOTE, Make sure all these includes are AFTER the system and header includes +#include "CppUTest/CommandLineTestRunner.h" +#include "CppUTest/MemoryLeakDetectorNewMacros.h" +#include "CppUTest/TestHarness.h" +#include "CppUTest/Utest.h" +#include "CppUTestExt/MockSupport.h" + +// clang-format off +TEST_GROUP(CustomGeneratorSerializationTestGroup) +{ + void teardown() { + mock().clear(); + } +}; +// clang-format on + +TEST(CustomGeneratorSerializationTestGroup, FormatEmptyCheck) { + buildcc::internal::CustomGeneratorSerialization serialization( + "dump/FormatEmptyCheck.bin"); + + { + flatbuffers::FlatBufferBuilder builder; + // Entire std::vector is nullptr + auto fbs_generator = + schema::internal::CreateCustomGeneratorDirect(builder, "", nullptr); + schema::internal::FinishCustomGeneratorBuffer(builder, fbs_generator); + + CHECK_TRUE(buildcc::env::save_file( + serialization.GetSerializedFile().string().c_str(), + (const char *)builder.GetBufferPointer(), builder.GetSize(), true)); + + CHECK_FALSE(serialization.LoadFromFile()); + + fs::remove_all(serialization.GetSerializedFile()); + } + + { + flatbuffers::FlatBufferBuilder builder; + // RelInputOutput in nullptr + auto rel_io = schema::internal::CreateRelInputOutputDirect(builder, ""); + std::vector v{rel_io}; + auto fbs_generator = + schema::internal::CreateCustomGeneratorDirect(builder, "", &v); + schema::internal::FinishCustomGeneratorBuffer(builder, fbs_generator); + + CHECK_TRUE(buildcc::env::save_file( + serialization.GetSerializedFile().string().c_str(), + (const char *)builder.GetBufferPointer(), builder.GetSize(), true)); + + CHECK_TRUE(serialization.LoadFromFile()); + + fs::remove_all(serialization.GetSerializedFile()); + } +} + +TEST(CustomGeneratorSerializationTestGroup, EmptyFile_Failure) { + { + buildcc::internal::CustomGeneratorSerialization serialization( + "dump/empty_file.bin"); + CHECK_FALSE(serialization.LoadFromFile()); + } + + { + buildcc::internal::CustomGeneratorSerialization serialization( + "dump/empty_file.bin"); + char data[] = {0}; + buildcc::env::save_file(serialization.GetSerializedFile().string().c_str(), + (const char *)data, 1, true); + CHECK_FALSE(serialization.LoadFromFile()); + } +} + +int main(int ac, char **av) { + return CommandLineTestRunner::RunAllTests(ac, av); +} From c344210f19e753e7189f14b046f0875dbc3410f3 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 00:15:26 -0700 Subject: [PATCH 08/51] Updated builder_interface and test --- .../target/interface/builder_interface.h | 60 +++++++++++++++++ .../test/target/test_builder_interface.cpp | 64 +++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/buildcc/lib/target/include/target/interface/builder_interface.h b/buildcc/lib/target/include/target/interface/builder_interface.h index ae044274..22fbb7ba 100644 --- a/buildcc/lib/target/include/target/interface/builder_interface.h +++ b/buildcc/lib/target/include/target/interface/builder_interface.h @@ -27,6 +27,66 @@ namespace buildcc::internal { +enum PathState { + kNoChange, + kRemoved, + kAdded, + kUpdated, +}; + +template +inline bool CheckChanged(const T &previous, const T ¤t) { + bool changed = false; + if (previous != current) { + changed = true; + } + return changed; +} + +/** + * @brief + * + * @return PathState Returns first state found if `Removed`, `Added` or + * `Updated` + * If none of the above states are true then it returns `NoChange` + */ +inline PathState CheckPaths(const internal::path_unordered_set &previous_path, + const internal::path_unordered_set ¤t_path) { + PathState state{PathState::kNoChange}; + + // * Old path is removed + const bool removed = std::any_of( + previous_path.begin(), previous_path.end(), [&](const internal::Path &p) { + return current_path.find(p) == current_path.end(); + }); + if (removed) { + state = PathState::kRemoved; + } else { + (void)std::any_of(current_path.cbegin(), current_path.cend(), + [&](const internal::Path &p) -> bool { + bool dirty = false; + const auto find = previous_path.find(p); + const bool added_cond = (find == previous_path.end()); + if (added_cond) { + dirty = true; + state = kAdded; + } else { + const bool updated_cond = + (p.GetLastWriteTimestamp() > + find->GetLastWriteTimestamp()); + if (updated_cond) { + dirty = true; + state = kUpdated; + } else { + dirty = false; + } + } + return dirty; + }); + } + return state; +} + // TODO, 1. Consider updating Recheck* APIs - do not modify internal `dirty_` // flag // TODO, 2. Consider removing dependency on target/common/util.h diff --git a/buildcc/lib/target/test/target/test_builder_interface.cpp b/buildcc/lib/target/test/target/test_builder_interface.cpp index 9cc2d2ac..ba1d02e9 100644 --- a/buildcc/lib/target/test/target/test_builder_interface.cpp +++ b/buildcc/lib/target/test/target/test_builder_interface.cpp @@ -62,6 +62,70 @@ TEST(TestBuilderInterfaceGroup, IncorrectUpdateRecheckCb) { CHECK_THROWS(std::exception, tbi.RecheckPathIncorrectUpdateCb()); } +TEST(TestBuilderInterfaceGroup, CheckChanged) { + std::vector previous{"", "v1"}; + std::vector current{"", "v1"}; + CHECK_FALSE(buildcc::internal::CheckChanged(previous, current)); + + current.clear(); + current.push_back(""); + CHECK_TRUE(buildcc::internal::CheckChanged(previous, current)); +} + +TEST(TestBuilderInterfaceGroup, CheckPaths) { + { + auto pathstate = buildcc::internal::CheckPaths( + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x2345), + }, + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x2345), + }); + CHECK_TRUE(pathstate == buildcc::internal::PathState::kNoChange); + } + + { + auto pathstate = buildcc::internal::CheckPaths( + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x2345), + }, + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + }); + CHECK_TRUE(pathstate == buildcc::internal::PathState::kRemoved); + } + + { + auto pathstate = buildcc::internal::CheckPaths( + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x2345), + }, + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x2345), + buildcc::internal::Path::CreateNewPath("v2", 0x3456), + }); + CHECK_TRUE(pathstate == buildcc::internal::PathState::kAdded); + } + + { + auto pathstate = buildcc::internal::CheckPaths( + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x2345), + }, + { + buildcc::internal::Path::CreateNewPath("", 0x1234), + buildcc::internal::Path::CreateNewPath("v1", 0x3456), + }); + CHECK_TRUE(pathstate == buildcc::internal::PathState::kUpdated); + } +} + int main(int ac, char **av) { return CommandLineTestRunner::RunAllTests(ac, av); } From 4bb1dd59ac277a6d54e30c01781b0ea8a262ef0b Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 02:01:24 -0700 Subject: [PATCH 09/51] Added custom_generator --- .../lib/target/cmake/common_target_src.cmake | 1 + buildcc/lib/target/include/target/generator.h | 124 +++++++++- .../src/custom_generator/custom_generator.cpp | 219 ++++++++++++++++++ 3 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 buildcc/lib/target/src/custom_generator/custom_generator.cpp diff --git a/buildcc/lib/target/cmake/common_target_src.cmake b/buildcc/lib/target/cmake/common_target_src.cmake index 76564d86..d12a78b5 100644 --- a/buildcc/lib/target/cmake/common_target_src.cmake +++ b/buildcc/lib/target/cmake/common_target_src.cmake @@ -34,6 +34,7 @@ set(COMMON_TARGET_SRCS include/target/api/target_getter.h # Generator + src/custom_generator/custom_generator.cpp src/generator/generator.cpp include/target/generator.h diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 74f9612a..3918383c 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -32,6 +32,7 @@ #include "target/interface/builder_interface.h" +#include "schema/custom_generator_serialization.h" #include "schema/generator_serialization.h" #include "schema/path.h" @@ -39,6 +40,125 @@ namespace buildcc { +struct UserRelInputOutputSchema { + fs_unordered_set inputs; + fs_unordered_set outputs; +}; + +struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { + std::unordered_map rels_map; + + void ConvertToInternal() { + for (const auto &r_miter : rels_map) { + internal::RelInputOutputSchema internal_schema; + internal_schema.outputs = r_miter.second.outputs; + internal_schema.internal_inputs = internal::path_schema_convert( + r_miter.second.inputs, internal::Path::CreateExistingPath); + internal_rels_map.emplace(r_miter.first, std::move(internal_schema)); + } + } +}; + +class CustomGeneratorContext { +public: + CustomGeneratorContext(const env::Command &c) : command(c) {} + + void Success(const std::string &id) { + std::lock_guard m(success_schema_mutex_); + success_schema_.emplace(id, selected_user_schema.at(id)); + } + + void Failure() { env::set_task_state(env::TaskState::FAILURE); } + + const std::unordered_map & + GetSuccessSchema() { + return success_schema_; + } + +public: + const env::Command &command; + std::unordered_map + selected_user_schema; + +private: + std::mutex success_schema_mutex_; + std::unordered_map success_schema_; +}; + +// clang-format off +typedef std::function(tf::Subflow &, CustomGeneratorContext &)> RegenerateCb; +// clang-format on + +class CustomGenerator : public internal::BuilderInterface { +public: + CustomGenerator(const std::string &name, const TargetEnv &env, + bool parallel = false) + : name_(name), + env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), + serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)), + parallel_(parallel), ctx_(command_) { + Initialize(); + } + virtual ~CustomGenerator() = default; + CustomGenerator(const CustomGenerator &) = delete; + + /** + * @brief Add default arguments for input, output and command requirements + * + * @param arguments Key-Value pair for arguments + */ + void AddDefaultArguments( + const std::unordered_map &arguments); + + void AddRelInputOutput(const std::string &id, const fs_unordered_set &inputs, + const fs_unordered_set &outputs); + + // TODO, Rename this to GenerateCb + void AddRegenerateCb(const RegenerateCb ®enerate_cb); + + void Build() override; + + // Getters + const fs::path &GetBinaryPath() const { + return serialization_.GetSerializedFile(); + } + tf::Taskflow &GetTaskflow() { return tf_; } + +private: + void Initialize(); + + void GenerateTask(); + void Convert(); + void BuildGenerate(std::unordered_map + &gen_selected_map, + std::unordered_map + &dummy_gen_selected_map); + // Recheck states + void PathRemoved() {} + void PathAdded() {} + void PathUpdated() {} + + void StringChanged() {} + +private: + std::string name_; + TargetEnv env_; + internal::CustomGeneratorSerialization serialization_; + bool parallel_; + CustomGeneratorContext ctx_; + + // Serialization + UserCustomGeneratorSchema user_; + std::unordered_map + dummy_selected_user_schema_; + + // Internal + std::mutex task_state_mutex_; + env::Command command_; + tf::Taskflow tf_; + RegenerateCb regenerate_cb_; +}; + // TODO, Make this private struct UserGeneratorSchema : public internal::GeneratorSchema { fs_unordered_set inputs; @@ -54,8 +174,8 @@ class Generator : public internal::BuilderInterface { parallel_(parallel) { Initialize(); } - virtual ~Generator() {} - Generator(const Generator &generator) = delete; + virtual ~Generator() = default; + Generator(const Generator &) = delete; /** * @brief Add default arguments for input, output and command requirements diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp new file mode 100644 index 00000000..379d1061 --- /dev/null +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -0,0 +1,219 @@ +/* + * Copyright 2021-2022 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/generator.h" + +namespace { +constexpr const char *const kStartGeneratorTaskName = "Start Generator"; +constexpr const char *const kEndGeneratorTaskName = "End Generator"; + +constexpr const char *const kCommandTaskName = "Command"; +constexpr const char *const kGenerateTaskName = "Generate"; + +} // namespace + +namespace buildcc { + +void CustomGenerator::AddDefaultArguments( + const std::unordered_map &arguments) { + command_.AddDefaultArguments(arguments); +} + +void CustomGenerator::AddRelInputOutput(const std::string &id, + const fs_unordered_set &inputs, + const fs_unordered_set &outputs) { + env::assert_fatal(user_.rels_map.find(id) == user_.rels_map.end(), + fmt::format("Duplicate id {} detected", id)); + UserRelInputOutputSchema schema; + for (const auto &i : inputs) { + schema.inputs.emplace(command_.Construct(i)); + } + for (const auto &o : outputs) { + schema.outputs.emplace(command_.Construct(o)); + } + user_.rels_map.emplace(id, std::move(schema)); +} + +void CustomGenerator::AddRegenerateCb(const RegenerateCb ®enerate_cb) { + regenerate_cb_ = regenerate_cb; +} + +void CustomGenerator::Build() { + (void)serialization_.LoadFromFile(); + + GenerateTask(); +} + +// PRIVATE +void CustomGenerator::Initialize() { + // Checks + env::assert_fatal( + Project::IsInit(), + "Environment is not initialized. Use the buildcc::Project::Init API"); + + // + fs::create_directories(env_.GetTargetBuildDir()); + command_.AddDefaultArguments({ + {"gen_root_dir", path_as_string(env_.GetTargetRootDir())}, + {"gen_build_dir", path_as_string(env_.GetTargetBuildDir())}, + }); + + // + unique_id_ = name_; + tf_.name(name_); +} + +void CustomGenerator::GenerateTask() { + tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + + try { + Convert(); + BuildGenerate(ctx_.selected_user_schema, dummy_selected_user_schema_); + ASSERT_FATAL(regenerate_cb_, + "Supply your custom regenerate callback using the " + "CustomGenerator::AddRegenerateCb API"); + + auto task_map = regenerate_cb_(subflow, ctx_); + + // DONE, Graph Generation + + // Selected + for (const auto &selected_miter : ctx_.selected_user_schema) { + const auto &id = selected_miter.first; + env::assert_fatal( + task_map.find(id) != task_map.end(), + "Incorrect implementation of CustomGenerator::RegenerateCb. Please " + "make sure all the map ids have a Task associated with it."); + tf::Task gtask = task_map.at(id); + env::assert_fatal( + !gtask.empty(), + "Incorrect implementation of CustomGenerator::RegenerateCb. Task " + "returned is empty"); + gtask.name(id); + + const auto &inputs = selected_miter.second.inputs; + for (const auto &i : inputs) { + std::string name = + fmt::format("{}", i.lexically_relative(Project::GetRootDir())); + auto itask = subflow.placeholder().name(name); + itask.precede(gtask); + } + const auto &outputs = selected_miter.second.outputs; + for (const auto &o : outputs) { + std::string name = + fmt::format("{}", o.lexically_relative(Project::GetRootDir())); + tf::Task otask = subflow.placeholder().name(name); + otask.succeed(gtask); + } + } + + // TODO, Dummy Selected + // for (const auto &dummy_selected_miter : dummy_selected_user_schema_) { + // } + + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }); + generate_task.name(kGenerateTaskName); + + tf::Task end_task = tf_.emplace([this]() { + // Even if env::TaskState::FAILURE we still need to partially store the + // built files + if (dirty_) { + try { + UserCustomGeneratorSchema user_final_schema; + user_final_schema.rels_map.insert(dummy_selected_user_schema_.begin(), + dummy_selected_user_schema_.end()); + const auto &success_schema = ctx_.GetSuccessSchema(); + user_final_schema.rels_map.insert(success_schema.begin(), + success_schema.end()); + + user_final_schema.ConvertToInternal(); + serialization_.UpdateStore(user_final_schema); + env::assert_fatal(serialization_.StoreToFile(), + fmt::format("Store failed for {}", name_)); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + } + }); + end_task.name(kEndGeneratorTaskName); + + // Dependencies + generate_task.precede(end_task); +} + +void CustomGenerator::Convert() { user_.ConvertToInternal(); } + +void CustomGenerator::BuildGenerate( + std::unordered_map &gen_selected_map, + std::unordered_map + &dummy_gen_selected_map) { + if (!serialization_.IsLoaded()) { + gen_selected_map = user_.rels_map; + dirty_ = true; + } else { + // DONE, Conditionally select internal_rels depending on what has changed + const auto &curr_rels = user_.internal_rels_map; + const auto &prev_rels = serialization_.GetLoad().internal_rels_map; + + // TODO, MAP REMOVED condition Check if prev_rels exists in curr_rels + // If prev_rels does not exist in curr_rels, has been removed from existing + // build + // We need this condition to only set the dirty_ flag + + // DONE, MAP ADDED condition Check if curr_rels exists in prev_rels + // If curr_rels does not exist in prev_rels, has been added to existing + // build + + // DONE, MAP UPDATED condition, Check if curr_rels exists in prev_rels + // If curr_rels exists in prev_rels, but the + // * INPUT has been removed, added or changed + // * OUTPUT has been removed or added + for (const auto &curr_miter : curr_rels) { + const auto &id = curr_miter.first; + if (prev_rels.find(id) == prev_rels.end()) { + // MAP ADDED condition + gen_selected_map.emplace(curr_miter.first, + user_.rels_map.at(curr_miter.first)); + dirty_ = true; + } else { + // Check internal_inputs + auto path_state = CheckPaths(prev_rels.at(id).internal_inputs, + curr_miter.second.internal_inputs); + auto changed = + CheckChanged(prev_rels.at(id).outputs, curr_miter.second.outputs); + + if (path_state != internal::PathState::kNoChange || changed) { + // MAP UPDATED condition + gen_selected_map.emplace(curr_miter.first, + user_.rels_map.at(curr_miter.first)); + dirty_ = true; + } else { + // MAP NO CHANGE condition + dummy_gen_selected_map.emplace(curr_miter.first, + user_.rels_map.at(curr_miter.first)); + } + } + } + } +} + +} // namespace buildcc From 9ce04a6bdfee3be869ec533ec0541202efb6e9db Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 02:01:39 -0700 Subject: [PATCH 10/51] Added initial test custom generator --- buildcc/lib/target/mock/expect_generator.h | 2 + buildcc/lib/target/mock/generator/runner.cpp | 6 + buildcc/lib/target/test/target/CMakeLists.txt | 8 ++ .../test/target/test_custom_generator.cpp | 136 ++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 buildcc/lib/target/test/target/test_custom_generator.cpp diff --git a/buildcc/lib/target/mock/expect_generator.h b/buildcc/lib/target/mock/expect_generator.h index b41cd780..248ba079 100644 --- a/buildcc/lib/target/mock/expect_generator.h +++ b/buildcc/lib/target/mock/expect_generator.h @@ -5,6 +5,8 @@ namespace buildcc::m { +void CustomGeneratorRunner(CustomGenerator &custom_generator); + /** * @brief Runs the generator using Taskflow with 1 thread * CppUTest cannot mock with multiple threads diff --git a/buildcc/lib/target/mock/generator/runner.cpp b/buildcc/lib/target/mock/generator/runner.cpp index 3ca990e8..61fdfe2e 100644 --- a/buildcc/lib/target/mock/generator/runner.cpp +++ b/buildcc/lib/target/mock/generator/runner.cpp @@ -4,6 +4,12 @@ namespace buildcc::m { +void CustomGeneratorRunner(CustomGenerator &custom_generator) { + tf::Executor executor(1); + executor.run(custom_generator.GetTaskflow()); + executor.wait_for_all(); +} + void GeneratorRunner(Generator &generator) { tf::Executor executor(1); executor.run(generator.GetTaskflow()); diff --git a/buildcc/lib/target/test/target/CMakeLists.txt b/buildcc/lib/target/test/target/CMakeLists.txt index 44acb230..a77da279 100644 --- a/buildcc/lib/target/test/target/CMakeLists.txt +++ b/buildcc/lib/target/test/target/CMakeLists.txt @@ -42,11 +42,19 @@ target_link_libraries(test_target_state PRIVATE target_interface) add_test(NAME test_target_state COMMAND test_target_state) # Generator +add_executable(test_custom_generator + test_custom_generator.cpp +) +target_link_libraries(test_custom_generator PRIVATE target_interface) + add_executable(test_generator test_generator.cpp ) target_link_libraries(test_generator PRIVATE target_interface) +add_test(NAME test_custom_generator COMMAND test_custom_generator + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) add_test(NAME test_generator COMMAND test_generator WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp new file mode 100644 index 00000000..10fcf70d --- /dev/null +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -0,0 +1,136 @@ +#include "target/generator.h" + +#include "expect_command.h" +#include "expect_generator.h" + +// #include "test_target_util.h" +// #include "taskflow/taskflow.hpp" +// #include "env/util.h" + +// NOTE, Make sure all these includes are AFTER the system and header includes +#include "CppUTest/CommandLineTestRunner.h" +#include "CppUTest/MemoryLeakDetectorNewMacros.h" +#include "CppUTest/TestHarness.h" +#include "CppUTest/Utest.h" +#include "CppUTestExt/MockSupport.h" + +// clang-format off +TEST_GROUP(CustomGeneratorTestGroup) +{ + void teardown() { + mock().clear(); + } +}; +// clang-format on + +fs::path BUILD_DIR = fs::current_path() / "intermediate" / "custom_generator"; + +static std::unordered_map +BasicGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { + mock().actualCall("BasicGenerateCb"); + std::unordered_map uom; + for (const auto &miter : ctx.selected_user_schema) { + mock().actualCall(miter.first.c_str()); + auto task = subflow.placeholder(); + uom.emplace(miter.first, task); + } + return uom; +} + +TEST(CustomGeneratorTestGroup, Basic) { + buildcc::CustomGenerator cgen("custom_generator", ""); + cgen.AddRegenerateCb(BasicGenerateCb); + cgen.Build(); + + mock().expectOneCall("BasicGenerateCb"); + buildcc::m::CustomGeneratorRunner(cgen); +} + +TEST(CustomGeneratorTestGroup, Basic2) { + buildcc::CustomGenerator cgen("custom_generator2", ""); + cgen.AddRegenerateCb(BasicGenerateCb); + cgen.AddRelInputOutput("id1", {}, {}); + cgen.AddRelInputOutput("id2", {}, {}); + cgen.Build(); + + mock().expectOneCall("BasicGenerateCb"); + mock().expectOneCall("id1"); + mock().expectOneCall("id2"); + buildcc::m::CustomGeneratorRunner(cgen); +} + +static std::unordered_map +RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { + mock().actualCall("RealGenerateCb"); + std::unordered_map uom; + for (const auto &miter : ctx.selected_user_schema) { + auto task = subflow.emplace([&]() { + try { + bool executed = buildcc::env::Command::Execute(""); + buildcc::env::assert_fatal(executed, ""); + ctx.Success(miter.first); + mock().actualCall(fmt::format("{}:SUCCESS", miter.first).c_str()); + } catch (...) { + ctx.Failure(); + mock().actualCall(fmt::format("{}:FAILURE", miter.first).c_str()); + } + }); + uom.emplace(miter.first, task); + } + return uom; +} + +TEST(CustomGeneratorTestGroup, RealGenerate) { + constexpr const char *const kGenName = "real_generator"; + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddRegenerateCb(RealGenerateCb); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}); + cgen.AddRelInputOutput("id2", {}, {}); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("id1:SUCCESS"); + mock().expectOneCall("id2:SUCCESS"); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + fs::remove_all(cgen.GetBinaryPath()); + } + + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddRegenerateCb(RealGenerateCb); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}); + cgen.AddRelInputOutput("id2", {}, {}); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, false); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("id1:FAILURE"); + mock().expectOneCall("id2:SUCCESS"); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + + fs::remove_all(cgen.GetBinaryPath()); + } +} + +int main(int ac, char **av) { + fs::remove_all(BUILD_DIR); + buildcc::Project::Init(fs::current_path() / "data", BUILD_DIR); + return CommandLineTestRunner::RunAllTests(ac, av); +} From fbeef3bda6cd974ba0ce0dc8481980ed7b702ded Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 02:26:14 -0700 Subject: [PATCH 11/51] Updated custom generator API --- buildcc/lib/target/include/target/generator.h | 6 +++--- .../lib/target/src/custom_generator/custom_generator.cpp | 8 ++++---- buildcc/lib/target/test/target/test_custom_generator.cpp | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 3918383c..96af4213 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -86,7 +86,7 @@ class CustomGeneratorContext { }; // clang-format off -typedef std::function(tf::Subflow &, CustomGeneratorContext &)> RegenerateCb; +typedef std::function(tf::Subflow &, CustomGeneratorContext &)> GenerateCb; // clang-format on class CustomGenerator : public internal::BuilderInterface { @@ -114,7 +114,7 @@ class CustomGenerator : public internal::BuilderInterface { const fs_unordered_set &outputs); // TODO, Rename this to GenerateCb - void AddRegenerateCb(const RegenerateCb ®enerate_cb); + void AddGenerateCb(const GenerateCb ®enerate_cb); void Build() override; @@ -156,7 +156,7 @@ class CustomGenerator : public internal::BuilderInterface { std::mutex task_state_mutex_; env::Command command_; tf::Taskflow tf_; - RegenerateCb regenerate_cb_; + GenerateCb regenerate_cb_; }; // TODO, Make this private diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 379d1061..9f493cd2 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -47,7 +47,7 @@ void CustomGenerator::AddRelInputOutput(const std::string &id, user_.rels_map.emplace(id, std::move(schema)); } -void CustomGenerator::AddRegenerateCb(const RegenerateCb ®enerate_cb) { +void CustomGenerator::AddGenerateCb(const GenerateCb ®enerate_cb) { regenerate_cb_ = regenerate_cb; } @@ -87,7 +87,7 @@ void CustomGenerator::GenerateTask() { BuildGenerate(ctx_.selected_user_schema, dummy_selected_user_schema_); ASSERT_FATAL(regenerate_cb_, "Supply your custom regenerate callback using the " - "CustomGenerator::AddRegenerateCb API"); + "CustomGenerator::AddGenerateCb API"); auto task_map = regenerate_cb_(subflow, ctx_); @@ -98,12 +98,12 @@ void CustomGenerator::GenerateTask() { const auto &id = selected_miter.first; env::assert_fatal( task_map.find(id) != task_map.end(), - "Incorrect implementation of CustomGenerator::RegenerateCb. Please " + "Incorrect implementation of CustomGenerator::GenerateCb. Please " "make sure all the map ids have a Task associated with it."); tf::Task gtask = task_map.at(id); env::assert_fatal( !gtask.empty(), - "Incorrect implementation of CustomGenerator::RegenerateCb. Task " + "Incorrect implementation of CustomGenerator::GenerateCb. Task " "returned is empty"); gtask.name(id); diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 10fcf70d..db8e3b0c 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -39,7 +39,7 @@ BasicGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { TEST(CustomGeneratorTestGroup, Basic) { buildcc::CustomGenerator cgen("custom_generator", ""); - cgen.AddRegenerateCb(BasicGenerateCb); + cgen.AddGenerateCb(BasicGenerateCb); cgen.Build(); mock().expectOneCall("BasicGenerateCb"); @@ -48,7 +48,7 @@ TEST(CustomGeneratorTestGroup, Basic) { TEST(CustomGeneratorTestGroup, Basic2) { buildcc::CustomGenerator cgen("custom_generator2", ""); - cgen.AddRegenerateCb(BasicGenerateCb); + cgen.AddGenerateCb(BasicGenerateCb); cgen.AddRelInputOutput("id1", {}, {}); cgen.AddRelInputOutput("id2", {}, {}); cgen.Build(); @@ -84,7 +84,7 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { constexpr const char *const kGenName = "real_generator"; { buildcc::CustomGenerator cgen(kGenName, ""); - cgen.AddRegenerateCb(RealGenerateCb); + cgen.AddGenerateCb(RealGenerateCb); cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, {"{gen_build_dir}/dummy_main.o"}); cgen.AddRelInputOutput("id2", {}, {}); @@ -107,7 +107,7 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { { buildcc::CustomGenerator cgen(kGenName, ""); - cgen.AddRegenerateCb(RealGenerateCb); + cgen.AddGenerateCb(RealGenerateCb); cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, {"{gen_build_dir}/dummy_main.o"}); cgen.AddRelInputOutput("id2", {}, {}); From 01dbe583216319a0b1e15212401aee535a1aa2d2 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 04:02:35 -0700 Subject: [PATCH 12/51] Updated custom_generator and tests --- .../src/custom_generator/custom_generator.cpp | 70 ++++++------ .../test/target/test_custom_generator.cpp | 101 ++++++++++++++++-- 2 files changed, 128 insertions(+), 43 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 9f493cd2..80d44140 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -52,6 +52,9 @@ void CustomGenerator::AddGenerateCb(const GenerateCb ®enerate_cb) { } void CustomGenerator::Build() { + ASSERT_FATAL(regenerate_cb_, + "Supply your custom regenerate callback using the " + "CustomGenerator::AddGenerateCb API"); (void)serialization_.LoadFromFile(); GenerateTask(); @@ -85,44 +88,43 @@ void CustomGenerator::GenerateTask() { try { Convert(); BuildGenerate(ctx_.selected_user_schema, dummy_selected_user_schema_); - ASSERT_FATAL(regenerate_cb_, - "Supply your custom regenerate callback using the " - "CustomGenerator::AddGenerateCb API"); - auto task_map = regenerate_cb_(subflow, ctx_); - - // DONE, Graph Generation - - // Selected - for (const auto &selected_miter : ctx_.selected_user_schema) { - const auto &id = selected_miter.first; - env::assert_fatal( - task_map.find(id) != task_map.end(), - "Incorrect implementation of CustomGenerator::GenerateCb. Please " - "make sure all the map ids have a Task associated with it."); - tf::Task gtask = task_map.at(id); - env::assert_fatal( - !gtask.empty(), - "Incorrect implementation of CustomGenerator::GenerateCb. Task " - "returned is empty"); - gtask.name(id); - - const auto &inputs = selected_miter.second.inputs; - for (const auto &i : inputs) { - std::string name = - fmt::format("{}", i.lexically_relative(Project::GetRootDir())); - auto itask = subflow.placeholder().name(name); - itask.precede(gtask); - } - const auto &outputs = selected_miter.second.outputs; - for (const auto &o : outputs) { - std::string name = - fmt::format("{}", o.lexically_relative(Project::GetRootDir())); - tf::Task otask = subflow.placeholder().name(name); - otask.succeed(gtask); + if (!ctx_.selected_user_schema.empty()) { + auto task_map = regenerate_cb_(subflow, ctx_); + + // Selected + for (const auto &selected_miter : ctx_.selected_user_schema) { + const auto &id = selected_miter.first; + env::assert_fatal( + task_map.find(id) != task_map.end(), + "Incorrect implementation of CustomGenerator::GenerateCb. Please " + "make sure all the map ids have a Task associated with it."); + tf::Task gtask = task_map.at(id); + env::assert_fatal( + !gtask.empty(), + "Incorrect implementation of CustomGenerator::GenerateCb. Task " + "returned is empty"); + gtask.name(id); + + const auto &inputs = selected_miter.second.inputs; + for (const auto &i : inputs) { + std::string name = + fmt::format("{}", i.lexically_relative(Project::GetRootDir())); + auto itask = subflow.placeholder().name(name); + itask.precede(gtask); + } + const auto &outputs = selected_miter.second.outputs; + for (const auto &o : outputs) { + std::string name = + fmt::format("{}", o.lexically_relative(Project::GetRootDir())); + tf::Task otask = subflow.placeholder().name(name); + otask.succeed(gtask); + } } } + // DONE, Graph Generation + // TODO, Dummy Selected // for (const auto &dummy_selected_miter : dummy_selected_user_schema_) { // } diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index db8e3b0c..5c949844 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -18,6 +18,7 @@ TEST_GROUP(CustomGeneratorTestGroup) { void teardown() { + mock().checkExpectations(); mock().clear(); } }; @@ -40,15 +41,6 @@ BasicGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { TEST(CustomGeneratorTestGroup, Basic) { buildcc::CustomGenerator cgen("custom_generator", ""); cgen.AddGenerateCb(BasicGenerateCb); - cgen.Build(); - - mock().expectOneCall("BasicGenerateCb"); - buildcc::m::CustomGeneratorRunner(cgen); -} - -TEST(CustomGeneratorTestGroup, Basic2) { - buildcc::CustomGenerator cgen("custom_generator2", ""); - cgen.AddGenerateCb(BasicGenerateCb); cgen.AddRelInputOutput("id1", {}, {}); cgen.AddRelInputOutput("id2", {}, {}); cgen.Build(); @@ -59,6 +51,97 @@ TEST(CustomGeneratorTestGroup, Basic2) { buildcc::m::CustomGeneratorRunner(cgen); } +// tf::Task should not be empty +static std::unordered_map +BadGenerateCb_EmptyTask(tf::Subflow &subflow, + buildcc::CustomGeneratorContext &ctx) { + (void)subflow; + std::unordered_map uom; + for (const auto &miter : ctx.selected_user_schema) { + tf::Task task; + uom.emplace(miter.first, task); + } + return uom; +} + +// Does not provide existing ids, plus adds random id that does not exist +static std::unordered_map +BadGenerateCb_WrongId(tf::Subflow &subflow, + buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + std::unordered_map uom; + auto task = subflow.placeholder(); + uom.emplace("random_id_that_does_not_exist", task); + return uom; +} + +TEST(CustomGeneratorTestGroup, FailureCases) { + { + buildcc::CustomGenerator cgen("failure_no_cb", ""); + buildcc::GenerateCb cb; + cgen.AddGenerateCb(cb); + CHECK_THROWS(std::exception, cgen.Build()); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + + { + buildcc::CustomGenerator cgen("failure_empty_task_cb", ""); + cgen.AddGenerateCb(BadGenerateCb_EmptyTask); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}); + cgen.Build(); + buildcc::m::CustomGeneratorRunner(cgen); + + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + + { + buildcc::CustomGenerator cgen("failure_wrong_id_cb", ""); + cgen.AddGenerateCb(BadGenerateCb_WrongId); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}); + cgen.Build(); + buildcc::m::CustomGeneratorRunner(cgen); + + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + + { + buildcc::CustomGenerator cgen("failure_cannot_save", ""); + fs::create_directory( + cgen.GetBinaryPath()); // make a folder so that file cannot be saved + + cgen.AddGenerateCb(BasicGenerateCb); + cgen.AddRelInputOutput("id1", {}, {}); + cgen.AddRelInputOutput("id2", {}, {}); + cgen.Build(); + + mock().expectOneCall("BasicGenerateCb"); + mock().expectOneCall("id1"); + mock().expectOneCall("id2"); + buildcc::m::CustomGeneratorRunner(cgen); + + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + + { + buildcc::CustomGenerator cgen("gen_task_not_run_no_io", ""); + cgen.AddGenerateCb(BasicGenerateCb); + cgen.Build(); + buildcc::m::CustomGeneratorRunner(cgen); + } +} + static std::unordered_map RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { mock().actualCall("RealGenerateCb"); From ad9db48ce3f4f42973e15c5f4195f9d7116b4233 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 04:53:48 -0700 Subject: [PATCH 13/51] Updated custom_generator and tests --- buildcc/lib/target/include/target/generator.h | 15 ++++++--- .../src/custom_generator/custom_generator.cpp | 6 ++-- .../test/target/test_custom_generator.cpp | 31 ++++++++++++------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 96af4213..7a8c6689 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -61,11 +61,14 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { class CustomGeneratorContext { public: - CustomGeneratorContext(const env::Command &c) : command(c) {} + CustomGeneratorContext( + const env::Command &c, + const std::unordered_map &schema) + : command(c), selected_schema(schema) {} void Success(const std::string &id) { std::lock_guard m(success_schema_mutex_); - success_schema_.emplace(id, selected_user_schema.at(id)); + success_schema_.emplace(id, selected_schema.at(id)); } void Failure() { env::set_task_state(env::TaskState::FAILURE); } @@ -77,8 +80,8 @@ class CustomGeneratorContext { public: const env::Command &command; - std::unordered_map - selected_user_schema; + const std::unordered_map + &selected_schema; private: std::mutex success_schema_mutex_; @@ -96,7 +99,7 @@ class CustomGenerator : public internal::BuilderInterface { : name_(name), env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)), - parallel_(parallel), ctx_(command_) { + parallel_(parallel), ctx_(command_, selected_user_schema_) { Initialize(); } virtual ~CustomGenerator() = default; @@ -149,6 +152,8 @@ class CustomGenerator : public internal::BuilderInterface { // Serialization UserCustomGeneratorSchema user_; + std::unordered_map + selected_user_schema_; std::unordered_map dummy_selected_user_schema_; diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 80d44140..b0a9e749 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -87,13 +87,13 @@ void CustomGenerator::GenerateTask() { try { Convert(); - BuildGenerate(ctx_.selected_user_schema, dummy_selected_user_schema_); + BuildGenerate(selected_user_schema_, dummy_selected_user_schema_); - if (!ctx_.selected_user_schema.empty()) { + if (dirty_) { auto task_map = regenerate_cb_(subflow, ctx_); // Selected - for (const auto &selected_miter : ctx_.selected_user_schema) { + for (const auto &selected_miter : ctx_.selected_schema) { const auto &id = selected_miter.first; env::assert_fatal( task_map.find(id) != task_map.end(), diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 5c949844..1e2d711f 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -30,7 +30,7 @@ static std::unordered_map BasicGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { mock().actualCall("BasicGenerateCb"); std::unordered_map uom; - for (const auto &miter : ctx.selected_user_schema) { + for (const auto &miter : ctx.selected_schema) { mock().actualCall(miter.first.c_str()); auto task = subflow.placeholder(); uom.emplace(miter.first, task); @@ -39,10 +39,10 @@ BasicGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { } TEST(CustomGeneratorTestGroup, Basic) { - buildcc::CustomGenerator cgen("custom_generator", ""); + buildcc::CustomGenerator cgen("basic", ""); cgen.AddGenerateCb(BasicGenerateCb); - cgen.AddRelInputOutput("id1", {}, {}); - cgen.AddRelInputOutput("id2", {}, {}); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.c"}, {}); + cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}); cgen.Build(); mock().expectOneCall("BasicGenerateCb"); @@ -57,7 +57,7 @@ BadGenerateCb_EmptyTask(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { (void)subflow; std::unordered_map uom; - for (const auto &miter : ctx.selected_user_schema) { + for (const auto &miter : ctx.selected_schema) { tf::Task task; uom.emplace(miter.first, task); } @@ -138,15 +138,19 @@ TEST(CustomGeneratorTestGroup, FailureCases) { buildcc::CustomGenerator cgen("gen_task_not_run_no_io", ""); cgen.AddGenerateCb(BasicGenerateCb); cgen.Build(); + + mock().expectOneCall("BasicGenerateCb"); buildcc::m::CustomGeneratorRunner(cgen); } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } static std::unordered_map RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { mock().actualCall("RealGenerateCb"); std::unordered_map uom; - for (const auto &miter : ctx.selected_user_schema) { + for (const auto &miter : ctx.selected_schema) { auto task = subflow.emplace([&]() { try { bool executed = buildcc::env::Command::Execute(""); @@ -168,9 +172,8 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { { buildcc::CustomGenerator cgen(kGenName, ""); cgen.AddGenerateCb(RealGenerateCb); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, - {"{gen_build_dir}/dummy_main.o"}); - cgen.AddRelInputOutput("id2", {}, {}); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}); + cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.c"}, {}); cgen.Build(); mock().expectOneCall("RealGenerateCb"); @@ -191,9 +194,8 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { { buildcc::CustomGenerator cgen(kGenName, ""); cgen.AddGenerateCb(RealGenerateCb); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, - {"{gen_build_dir}/dummy_main.o"}); - cgen.AddRelInputOutput("id2", {}, {}); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}); + cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.c"}, {}); cgen.Build(); mock().expectOneCall("RealGenerateCb"); @@ -203,6 +205,9 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { mock().expectOneCall("id2:SUCCESS"); buildcc::m::CustomGeneratorRunner(cgen); + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + buildcc::internal::CustomGeneratorSerialization serialization( cgen.GetBinaryPath()); CHECK_TRUE(serialization.LoadFromFile()); @@ -210,6 +215,8 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { fs::remove_all(cgen.GetBinaryPath()); } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } int main(int ac, char **av) { From edefce79e6fd6e0ac56e1967c3807c84f272256c Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 04:58:46 -0700 Subject: [PATCH 14/51] Minor update to test_custom_generator.cpp --- buildcc/lib/target/test/target/test_custom_generator.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 1e2d711f..f886ffd4 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -172,8 +172,10 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { { buildcc::CustomGenerator cgen(kGenName, ""); cgen.AddGenerateCb(RealGenerateCb); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}); - cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.c"}, {}); + cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}); + cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}); cgen.Build(); mock().expectOneCall("RealGenerateCb"); From 6df772bb8179e5ab5513c1a81f8f896ee8622b6a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 2 May 2022 05:02:35 -0700 Subject: [PATCH 15/51] Updated custom_generator.cpp --- buildcc/lib/target/src/custom_generator/custom_generator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index b0a9e749..66c86eb1 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -39,10 +39,10 @@ void CustomGenerator::AddRelInputOutput(const std::string &id, fmt::format("Duplicate id {} detected", id)); UserRelInputOutputSchema schema; for (const auto &i : inputs) { - schema.inputs.emplace(command_.Construct(i)); + schema.inputs.emplace(command_.Construct(path_as_string(i))); } for (const auto &o : outputs) { - schema.outputs.emplace(command_.Construct(o)); + schema.outputs.emplace(command_.Construct(path_as_string(o))); } user_.rels_map.emplace(id, std::move(schema)); } From 9670090cce6780c89e2c35a4f2b61e0a73df5d0b Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 4 May 2022 19:50:04 -0700 Subject: [PATCH 16/51] Updated test_custom_generator.cpp --- .../lib/target/test/target/test_custom_generator.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index f886ffd4..ed8a9d87 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -20,6 +20,7 @@ TEST_GROUP(CustomGeneratorTestGroup) void teardown() { mock().checkExpectations(); mock().clear(); + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } }; // clang-format on @@ -144,6 +145,17 @@ TEST(CustomGeneratorTestGroup, FailureCases) { } buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + + { + buildcc::env::set_task_state(buildcc::env::TaskState::FAILURE); + + buildcc::CustomGenerator cgen("gen_task_state_failure", ""); + cgen.AddGenerateCb(BasicGenerateCb); + cgen.Build(); + buildcc::m::CustomGeneratorRunner(cgen); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } static std::unordered_map From 7a35ffdb251f873f299a5a7cbf7e9349d2cff0cb Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 01:59:18 -0700 Subject: [PATCH 17/51] Updated custom_generator --- buildcc/lib/target/include/target/generator.h | 88 ++++----- .../src/custom_generator/custom_generator.cpp | 178 +++++++++--------- 2 files changed, 128 insertions(+), 138 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 7a8c6689..e5904994 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -40,58 +40,43 @@ namespace buildcc { -struct UserRelInputOutputSchema { +class CustomGeneratorContext { +public: + CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, + const fs_unordered_set &o) + : command(c), input(i), output(o) {} + +public: + const env::Command &command; + const fs_unordered_set &input; + const fs_unordered_set &output; +}; + +// clang-format off +typedef std::function GenerateCb; + +typedef std::function &)> DependencyCb; +// clang-format on + +struct UserRelInputOutputSchema : internal::RelInputOutputSchema { fs_unordered_set inputs; - fs_unordered_set outputs; + GenerateCb generate_cb; }; struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { std::unordered_map rels_map; void ConvertToInternal() { - for (const auto &r_miter : rels_map) { - internal::RelInputOutputSchema internal_schema; - internal_schema.outputs = r_miter.second.outputs; - internal_schema.internal_inputs = internal::path_schema_convert( + for (auto &r_miter : rels_map) { + r_miter.second.internal_inputs = path_schema_convert( r_miter.second.inputs, internal::Path::CreateExistingPath); - internal_rels_map.emplace(r_miter.first, std::move(internal_schema)); + auto p = internal_rels_map.emplace(r_miter.first, r_miter.second); + env::assert_fatal(p.second, + fmt::format("Could not save {}", r_miter.first)); } } }; -class CustomGeneratorContext { -public: - CustomGeneratorContext( - const env::Command &c, - const std::unordered_map &schema) - : command(c), selected_schema(schema) {} - - void Success(const std::string &id) { - std::lock_guard m(success_schema_mutex_); - success_schema_.emplace(id, selected_schema.at(id)); - } - - void Failure() { env::set_task_state(env::TaskState::FAILURE); } - - const std::unordered_map & - GetSuccessSchema() { - return success_schema_; - } - -public: - const env::Command &command; - const std::unordered_map - &selected_schema; - -private: - std::mutex success_schema_mutex_; - std::unordered_map success_schema_; -}; - -// clang-format off -typedef std::function(tf::Subflow &, CustomGeneratorContext &)> GenerateCb; -// clang-format on - class CustomGenerator : public internal::BuilderInterface { public: CustomGenerator(const std::string &name, const TargetEnv &env, @@ -99,7 +84,7 @@ class CustomGenerator : public internal::BuilderInterface { : name_(name), env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)), - parallel_(parallel), ctx_(command_, selected_user_schema_) { + parallel_(parallel) { Initialize(); } virtual ~CustomGenerator() = default; @@ -113,11 +98,12 @@ class CustomGenerator : public internal::BuilderInterface { void AddDefaultArguments( const std::unordered_map &arguments); - void AddRelInputOutput(const std::string &id, const fs_unordered_set &inputs, - const fs_unordered_set &outputs); + void AddGenInfo(const std::string &id, const fs_unordered_set &inputs, + const fs_unordered_set &outputs, + const GenerateCb &generate_cb); - // TODO, Rename this to GenerateCb - void AddGenerateCb(const GenerateCb ®enerate_cb); + // Callbacks + void AddDependencyCb(const DependencyCb &dependency_cb); void Build() override; @@ -136,6 +122,10 @@ class CustomGenerator : public internal::BuilderInterface { &gen_selected_map, std::unordered_map &dummy_gen_selected_map); + + void AddSuccessSchema(const std::string &id, + const UserRelInputOutputSchema &schema); + // Recheck states void PathRemoved() {} void PathAdded() {} @@ -148,7 +138,6 @@ class CustomGenerator : public internal::BuilderInterface { TargetEnv env_; internal::CustomGeneratorSerialization serialization_; bool parallel_; - CustomGeneratorContext ctx_; // Serialization UserCustomGeneratorSchema user_; @@ -157,11 +146,16 @@ class CustomGenerator : public internal::BuilderInterface { std::unordered_map dummy_selected_user_schema_; + std::mutex success_schema_mutex_; + std::unordered_map success_schema_; + // Internal std::mutex task_state_mutex_; env::Command command_; tf::Taskflow tf_; - GenerateCb regenerate_cb_; + + // Callbacks + DependencyCb dependency_cb_; }; // TODO, Make this private diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 66c86eb1..3e049100 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -32,11 +32,14 @@ void CustomGenerator::AddDefaultArguments( command_.AddDefaultArguments(arguments); } -void CustomGenerator::AddRelInputOutput(const std::string &id, - const fs_unordered_set &inputs, - const fs_unordered_set &outputs) { +void CustomGenerator::AddGenInfo(const std::string &id, + const fs_unordered_set &inputs, + const fs_unordered_set &outputs, + const GenerateCb &generate_cb) { env::assert_fatal(user_.rels_map.find(id) == user_.rels_map.end(), fmt::format("Duplicate id {} detected", id)); + ASSERT_FATAL(generate_cb, "Invalid callback provided"); + UserRelInputOutputSchema schema; for (const auto &i : inputs) { schema.inputs.emplace(command_.Construct(path_as_string(i))); @@ -44,17 +47,15 @@ void CustomGenerator::AddRelInputOutput(const std::string &id, for (const auto &o : outputs) { schema.outputs.emplace(command_.Construct(path_as_string(o))); } + schema.generate_cb = generate_cb; user_.rels_map.emplace(id, std::move(schema)); } -void CustomGenerator::AddGenerateCb(const GenerateCb ®enerate_cb) { - regenerate_cb_ = regenerate_cb; +void CustomGenerator::AddDependencyCb(const DependencyCb &dependency_cb) { + dependency_cb_ = dependency_cb; } void CustomGenerator::Build() { - ASSERT_FATAL(regenerate_cb_, - "Supply your custom regenerate callback using the " - "CustomGenerator::AddGenerateCb API"); (void)serialization_.LoadFromFile(); GenerateTask(); @@ -79,89 +80,6 @@ void CustomGenerator::Initialize() { tf_.name(name_); } -void CustomGenerator::GenerateTask() { - tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { - if (env::get_task_state() != env::TaskState::SUCCESS) { - return; - } - - try { - Convert(); - BuildGenerate(selected_user_schema_, dummy_selected_user_schema_); - - if (dirty_) { - auto task_map = regenerate_cb_(subflow, ctx_); - - // Selected - for (const auto &selected_miter : ctx_.selected_schema) { - const auto &id = selected_miter.first; - env::assert_fatal( - task_map.find(id) != task_map.end(), - "Incorrect implementation of CustomGenerator::GenerateCb. Please " - "make sure all the map ids have a Task associated with it."); - tf::Task gtask = task_map.at(id); - env::assert_fatal( - !gtask.empty(), - "Incorrect implementation of CustomGenerator::GenerateCb. Task " - "returned is empty"); - gtask.name(id); - - const auto &inputs = selected_miter.second.inputs; - for (const auto &i : inputs) { - std::string name = - fmt::format("{}", i.lexically_relative(Project::GetRootDir())); - auto itask = subflow.placeholder().name(name); - itask.precede(gtask); - } - const auto &outputs = selected_miter.second.outputs; - for (const auto &o : outputs) { - std::string name = - fmt::format("{}", o.lexically_relative(Project::GetRootDir())); - tf::Task otask = subflow.placeholder().name(name); - otask.succeed(gtask); - } - } - } - - // DONE, Graph Generation - - // TODO, Dummy Selected - // for (const auto &dummy_selected_miter : dummy_selected_user_schema_) { - // } - - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }); - generate_task.name(kGenerateTaskName); - - tf::Task end_task = tf_.emplace([this]() { - // Even if env::TaskState::FAILURE we still need to partially store the - // built files - if (dirty_) { - try { - UserCustomGeneratorSchema user_final_schema; - user_final_schema.rels_map.insert(dummy_selected_user_schema_.begin(), - dummy_selected_user_schema_.end()); - const auto &success_schema = ctx_.GetSuccessSchema(); - user_final_schema.rels_map.insert(success_schema.begin(), - success_schema.end()); - - user_final_schema.ConvertToInternal(); - serialization_.UpdateStore(user_final_schema); - env::assert_fatal(serialization_.StoreToFile(), - fmt::format("Store failed for {}", name_)); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - } - }); - end_task.name(kEndGeneratorTaskName); - - // Dependencies - generate_task.precede(end_task); -} - void CustomGenerator::Convert() { user_.ConvertToInternal(); } void CustomGenerator::BuildGenerate( @@ -218,4 +136,82 @@ void CustomGenerator::BuildGenerate( } } +void CustomGenerator::AddSuccessSchema(const std::string &id, + const UserRelInputOutputSchema &schema) { + std::lock_guard guard(success_schema_mutex_); + success_schema_.emplace(id, schema); +} + +void CustomGenerator::GenerateTask() { + tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { + if (env::get_task_state() != env::TaskState::SUCCESS) { + return; + } + + try { + Convert(); + BuildGenerate(selected_user_schema_, dummy_selected_user_schema_); + + if (dirty_) { + std::unordered_map task_map; + + for (const auto &selected_miter : selected_user_schema_) { + const auto &id = selected_miter.first; + const auto &info = selected_miter.second; + + tf::Task task = subflow.emplace([&]() { + try { + CustomGeneratorContext ctx(command_, info.inputs, info.outputs); + bool success = info.generate_cb(ctx); + env::assert_fatal(success, "Generate Cb failed for id {}"); + AddSuccessSchema(id, info); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }); + task.name(id); + task_map.emplace(id, task); + } + + if (dependency_cb_) { + dependency_cb_(task_map); + } + + // TODO, Create Selected graph + } + + // TODO, Create Dummy Selected graph + + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }); + generate_task.name(kGenerateTaskName); + + tf::Task end_task = tf_.emplace([this]() { + // Even if env::TaskState::FAILURE we still need to partially store the + // built files + if (dirty_) { + try { + UserCustomGeneratorSchema user_final_schema; + user_final_schema.rels_map.insert(dummy_selected_user_schema_.begin(), + dummy_selected_user_schema_.end()); + user_final_schema.rels_map.insert(success_schema_.begin(), + success_schema_.end()); + + user_final_schema.ConvertToInternal(); + serialization_.UpdateStore(user_final_schema); + env::assert_fatal(serialization_.StoreToFile(), + fmt::format("Store failed for {}", name_)); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + } + }); + end_task.name(kEndGeneratorTaskName); + + // Dependencies + generate_task.precede(end_task); +} + } // namespace buildcc From bad1658b7d200d2524837e7c4224621032076b06 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 02:03:12 -0700 Subject: [PATCH 18/51] Updated custom_generator basic unit tests --- .../test/target/test_custom_generator.cpp | 446 +++++++++++------- 1 file changed, 265 insertions(+), 181 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index ed8a9d87..c2aeefbf 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -27,212 +27,296 @@ TEST_GROUP(CustomGeneratorTestGroup) fs::path BUILD_DIR = fs::current_path() / "intermediate" / "custom_generator"; -static std::unordered_map -BasicGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { - mock().actualCall("BasicGenerateCb"); - std::unordered_map uom; - for (const auto &miter : ctx.selected_schema) { - mock().actualCall(miter.first.c_str()); - auto task = subflow.placeholder(); - uom.emplace(miter.first, task); - } - return uom; +static bool BasicGenerateCb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + return mock().actualCall("BasicGenerateCb").returnBoolValue(); } TEST(CustomGeneratorTestGroup, Basic) { buildcc::CustomGenerator cgen("basic", ""); - cgen.AddGenerateCb(BasicGenerateCb); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.c"}, {}); - cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, {}, BasicGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + BasicGenerateCb); cgen.Build(); - mock().expectOneCall("BasicGenerateCb"); - mock().expectOneCall("id1"); - mock().expectOneCall("id2"); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); buildcc::m::CustomGeneratorRunner(cgen); -} - -// tf::Task should not be empty -static std::unordered_map -BadGenerateCb_EmptyTask(tf::Subflow &subflow, - buildcc::CustomGeneratorContext &ctx) { - (void)subflow; - std::unordered_map uom; - for (const auto &miter : ctx.selected_schema) { - tf::Task task; - uom.emplace(miter.first, task); - } - return uom; -} - -// Does not provide existing ids, plus adds random id that does not exist -static std::unordered_map -BadGenerateCb_WrongId(tf::Subflow &subflow, - buildcc::CustomGeneratorContext &ctx) { - (void)ctx; - std::unordered_map uom; - auto task = subflow.placeholder(); - uom.emplace("random_id_that_does_not_exist", task); - return uom; -} - -TEST(CustomGeneratorTestGroup, FailureCases) { - { - buildcc::CustomGenerator cgen("failure_no_cb", ""); - buildcc::GenerateCb cb; - cgen.AddGenerateCb(cb); - CHECK_THROWS(std::exception, cgen.Build()); - } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); - - { - buildcc::CustomGenerator cgen("failure_empty_task_cb", ""); - cgen.AddGenerateCb(BadGenerateCb_EmptyTask); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, - {"{gen_build_dir}/dummy_main.o"}); - cgen.Build(); - buildcc::m::CustomGeneratorRunner(cgen); - - CHECK_TRUE(buildcc::env::get_task_state() == - buildcc::env::TaskState::FAILURE); - } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); - - { - buildcc::CustomGenerator cgen("failure_wrong_id_cb", ""); - cgen.AddGenerateCb(BadGenerateCb_WrongId); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, - {"{gen_build_dir}/dummy_main.o"}); - cgen.Build(); - buildcc::m::CustomGeneratorRunner(cgen); - - CHECK_TRUE(buildcc::env::get_task_state() == - buildcc::env::TaskState::FAILURE); - } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + // Serialization check { - buildcc::CustomGenerator cgen("failure_cannot_save", ""); - fs::create_directory( - cgen.GetBinaryPath()); // make a folder so that file cannot be saved - - cgen.AddGenerateCb(BasicGenerateCb); - cgen.AddRelInputOutput("id1", {}, {}); - cgen.AddRelInputOutput("id2", {}, {}); - cgen.Build(); - - mock().expectOneCall("BasicGenerateCb"); - mock().expectOneCall("id1"); - mock().expectOneCall("id2"); - buildcc::m::CustomGeneratorRunner(cgen); - - CHECK_TRUE(buildcc::env::get_task_state() == - buildcc::env::TaskState::FAILURE); - } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); - - { - buildcc::CustomGenerator cgen("gen_task_not_run_no_io", ""); - cgen.AddGenerateCb(BasicGenerateCb); - cgen.Build(); - - mock().expectOneCall("BasicGenerateCb"); - buildcc::m::CustomGeneratorRunner(cgen); - } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); - - { - buildcc::env::set_task_state(buildcc::env::TaskState::FAILURE); + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); - buildcc::CustomGenerator cgen("gen_task_state_failure", ""); - cgen.AddGenerateCb(BasicGenerateCb); - cgen.Build(); - buildcc::m::CustomGeneratorRunner(cgen); + const auto &internal_map = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(internal_map.size(), 2); + const auto &id1_info = internal_map.at("id1"); + CHECK_EQUAL(id1_info.internal_inputs.size(), 1); + CHECK_EQUAL(id1_info.outputs.size(), 0); } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } -static std::unordered_map -RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { - mock().actualCall("RealGenerateCb"); - std::unordered_map uom; - for (const auto &miter : ctx.selected_schema) { - auto task = subflow.emplace([&]() { - try { - bool executed = buildcc::env::Command::Execute(""); - buildcc::env::assert_fatal(executed, ""); - ctx.Success(miter.first); - mock().actualCall(fmt::format("{}:SUCCESS", miter.first).c_str()); - } catch (...) { - ctx.Failure(); - mock().actualCall(fmt::format("{}:FAILURE", miter.first).c_str()); - } - }); - uom.emplace(miter.first, task); - } - return uom; -} +TEST(CustomGeneratorTestGroup, Basic_Failure) { + buildcc::CustomGenerator cgen("basic_failure", ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, {}, BasicGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, + BasicGenerateCb); + cgen.Build(); -TEST(CustomGeneratorTestGroup, RealGenerate) { - constexpr const char *const kGenName = "real_generator"; - { - buildcc::CustomGenerator cgen(kGenName, ""); - cgen.AddGenerateCb(RealGenerateCb); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, - {"{gen_build_dir}/dummy_main.o"}); - cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.c"}, - {"{gen_build_dir}/dummy_main.o"}); - cgen.Build(); - - mock().expectOneCall("RealGenerateCb"); - buildcc::env::m::CommandExpect_Execute(1, true); - buildcc::env::m::CommandExpect_Execute(1, true); - mock().expectOneCall("id1:SUCCESS"); - mock().expectOneCall("id2:SUCCESS"); - buildcc::m::CustomGeneratorRunner(cgen); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + mock().expectOneCall("BasicGenerateCb").andReturnValue(false); + buildcc::m::CustomGeneratorRunner(cgen); - buildcc::internal::CustomGeneratorSerialization serialization( - cgen.GetBinaryPath()); - CHECK_TRUE(serialization.LoadFromFile()); - CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + // Load + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); - fs::remove_all(cgen.GetBinaryPath()); - } + const auto &id1_info = serialization.GetLoad().internal_rels_map.at("id1"); + CHECK_EQUAL(id1_info.internal_inputs.size(), 1); + CHECK_EQUAL(id1_info.outputs.size(), 0); - { - buildcc::CustomGenerator cgen(kGenName, ""); - cgen.AddGenerateCb(RealGenerateCb); - cgen.AddRelInputOutput("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}); - cgen.AddRelInputOutput("id2", {"{gen_root_dir}/dummy_main.c"}, {}); - cgen.Build(); - - mock().expectOneCall("RealGenerateCb"); - buildcc::env::m::CommandExpect_Execute(1, false); - buildcc::env::m::CommandExpect_Execute(1, true); - mock().expectOneCall("id1:FAILURE"); - mock().expectOneCall("id2:SUCCESS"); - buildcc::m::CustomGeneratorRunner(cgen); - - CHECK_TRUE(buildcc::env::get_task_state() == - buildcc::env::TaskState::FAILURE); + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); + // Serialization check + { buildcc::internal::CustomGeneratorSerialization serialization( cgen.GetBinaryPath()); CHECK_TRUE(serialization.LoadFromFile()); - CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); - fs::remove_all(cgen.GetBinaryPath()); + const auto &internal_map = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(internal_map.size(), 1); } - - buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } +// // tf::Task should not be empty +// static std::unordered_map +// BadGenerateCb_EmptyTask(tf::Subflow &subflow, +// buildcc::CustomGeneratorContext &ctx) { +// (void)subflow; +// std::unordered_map uom; +// for (const auto &miter : ctx.selected_schema) { +// tf::Task task; +// uom.emplace(miter.first, task); +// } +// return uom; +// } + +// // Does not provide existing ids, plus adds random id that does not exist +// static std::unordered_map +// BadGenerateCb_WrongId(tf::Subflow &subflow, +// buildcc::CustomGeneratorContext &ctx) { +// (void)ctx; +// std::unordered_map uom; +// auto task = subflow.placeholder(); +// uom.emplace("random_id_that_does_not_exist", task); +// return uom; +// } + +// TEST(CustomGeneratorTestGroup, FailureCases) { +// { +// buildcc::CustomGenerator cgen("failure_no_cb", ""); +// buildcc::GenerateCb cb; +// cgen.AddGenerateCb(cb); +// CHECK_THROWS(std::exception, cgen.Build()); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + +// { +// buildcc::CustomGenerator cgen("failure_empty_task_cb", ""); +// cgen.AddGenerateCb(BadGenerateCb_EmptyTask); +// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.Build(); +// buildcc::m::CustomGeneratorRunner(cgen); + +// CHECK_TRUE(buildcc::env::get_task_state() == +// buildcc::env::TaskState::FAILURE); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + +// { +// buildcc::CustomGenerator cgen("failure_wrong_id_cb", ""); +// cgen.AddGenerateCb(BadGenerateCb_WrongId); +// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.Build(); +// buildcc::m::CustomGeneratorRunner(cgen); + +// CHECK_TRUE(buildcc::env::get_task_state() == +// buildcc::env::TaskState::FAILURE); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + +// { +// buildcc::CustomGenerator cgen("failure_cannot_save", ""); +// fs::create_directory( +// cgen.GetBinaryPath()); // make a folder so that file cannot be saved + +// cgen.AddGenerateCb(BasicGenerateCb); +// cgen.AddGenInfo("id1", {}, {}); +// cgen.AddGenInfo("id2", {}, {}); +// cgen.Build(); + +// mock().expectOneCall("BasicGenerateCb"); +// mock().expectOneCall("id1"); +// mock().expectOneCall("id2"); +// buildcc::m::CustomGeneratorRunner(cgen); + +// CHECK_TRUE(buildcc::env::get_task_state() == +// buildcc::env::TaskState::FAILURE); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + +// { +// buildcc::CustomGenerator cgen("gen_task_not_run_no_io", ""); +// cgen.AddGenerateCb(BasicGenerateCb); +// cgen.Build(); + +// mock().expectOneCall("BasicGenerateCb"); +// buildcc::m::CustomGeneratorRunner(cgen); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + +// { +// buildcc::env::set_task_state(buildcc::env::TaskState::FAILURE); + +// buildcc::CustomGenerator cgen("gen_task_state_failure", ""); +// cgen.AddGenerateCb(BasicGenerateCb); +// cgen.Build(); +// buildcc::m::CustomGeneratorRunner(cgen); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); +// } + +// static std::unordered_map +// RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { +// mock().actualCall("RealGenerateCb"); +// std::unordered_map uom; +// for (const auto &miter : ctx.selected_schema) { +// auto task = subflow.emplace([&]() { +// try { +// bool executed = buildcc::env::Command::Execute(""); +// buildcc::env::assert_fatal(executed, ""); +// ctx.Success(miter.first); +// mock().actualCall(fmt::format("{}:SUCCESS", miter.first).c_str()); +// } catch (...) { +// ctx.Failure(); +// mock().actualCall(fmt::format("{}:FAILURE", miter.first).c_str()); +// } +// }); +// uom.emplace(miter.first, task); +// } +// return uom; +// } + +// TEST(CustomGeneratorTestGroup, RealGenerate) { +// constexpr const char *const kGenName = "real_generator"; +// { +// buildcc::CustomGenerator cgen(kGenName, ""); +// cgen.AddGenerateCb(RealGenerateCb); +// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.Build(); + +// mock().expectOneCall("RealGenerateCb"); +// buildcc::env::m::CommandExpect_Execute(1, true); +// buildcc::env::m::CommandExpect_Execute(1, true); +// mock().expectOneCall("id1:SUCCESS"); +// mock().expectOneCall("id2:SUCCESS"); +// buildcc::m::CustomGeneratorRunner(cgen); + +// buildcc::internal::CustomGeneratorSerialization serialization( +// cgen.GetBinaryPath()); +// CHECK_TRUE(serialization.LoadFromFile()); +// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + +// fs::remove_all(cgen.GetBinaryPath()); +// } + +// { +// buildcc::CustomGenerator cgen(kGenName, ""); +// cgen.AddGenerateCb(RealGenerateCb); +// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}); +// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, {}); +// cgen.Build(); + +// mock().expectOneCall("RealGenerateCb"); +// buildcc::env::m::CommandExpect_Execute(1, false); +// buildcc::env::m::CommandExpect_Execute(1, true); +// mock().expectOneCall("id1:FAILURE"); +// mock().expectOneCall("id2:SUCCESS"); +// buildcc::m::CustomGeneratorRunner(cgen); + +// CHECK_TRUE(buildcc::env::get_task_state() == +// buildcc::env::TaskState::FAILURE); + +// buildcc::internal::CustomGeneratorSerialization serialization( +// cgen.GetBinaryPath()); +// CHECK_TRUE(serialization.LoadFromFile()); +// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + +// fs::remove_all(cgen.GetBinaryPath()); +// } + +// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); +// } + +// TEST(CustomGeneratorTestGroup, RealGenerate_RebuildCondition) { +// constexpr const char *const kGenName = "real_generator_rebuild"; +// { +// buildcc::CustomGenerator cgen(kGenName, ""); +// cgen.AddGenerateCb(RealGenerateCb); +// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.Build(); + +// mock().expectOneCall("RealGenerateCb"); +// buildcc::env::m::CommandExpect_Execute(1, true); +// buildcc::env::m::CommandExpect_Execute(1, true); +// mock().expectOneCall("id1:SUCCESS"); +// mock().expectOneCall("id2:SUCCESS"); +// buildcc::m::CustomGeneratorRunner(cgen); + +// buildcc::internal::CustomGeneratorSerialization serialization( +// cgen.GetBinaryPath()); +// CHECK_TRUE(serialization.LoadFromFile()); +// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); +// } + +// // Remove Rel IO condition +// { +// buildcc::CustomGenerator cgen(kGenName, ""); +// cgen.AddGenerateCb(RealGenerateCb); +// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, +// {"{gen_build_dir}/dummy_main.o"}); +// cgen.Build(); + +// mock().expectOneCall("RealGenerateCb"); +// buildcc::env::m::CommandExpect_Execute(1, true); +// buildcc::env::m::CommandExpect_Execute(1, true); +// mock().expectOneCall("id1:SUCCESS"); +// mock().expectOneCall("id2:SUCCESS"); +// buildcc::m::CustomGeneratorRunner(cgen); + +// buildcc::internal::CustomGeneratorSerialization serialization( +// cgen.GetBinaryPath()); +// CHECK_TRUE(serialization.LoadFromFile()); +// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); +// } +// } + int main(int ac, char **av) { fs::remove_all(BUILD_DIR); buildcc::Project::Init(fs::current_path() / "data", BUILD_DIR); From c865e1455cc05fdbc4327afb10526017efc12c07 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 02:13:17 -0700 Subject: [PATCH 19/51] Updated failure tests for custom_generator --- .../test/target/test_custom_generator.cpp | 130 +++++------------- 1 file changed, 38 insertions(+), 92 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index c2aeefbf..728aa854 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -90,111 +90,57 @@ TEST(CustomGeneratorTestGroup, Basic_Failure) { } } -// // tf::Task should not be empty -// static std::unordered_map -// BadGenerateCb_EmptyTask(tf::Subflow &subflow, -// buildcc::CustomGeneratorContext &ctx) { -// (void)subflow; -// std::unordered_map uom; -// for (const auto &miter : ctx.selected_schema) { -// tf::Task task; -// uom.emplace(miter.first, task); -// } -// return uom; -// } - -// // Does not provide existing ids, plus adds random id that does not exist -// static std::unordered_map -// BadGenerateCb_WrongId(tf::Subflow &subflow, -// buildcc::CustomGeneratorContext &ctx) { -// (void)ctx; -// std::unordered_map uom; -// auto task = subflow.placeholder(); -// uom.emplace("random_id_that_does_not_exist", task); -// return uom; -// } - -// TEST(CustomGeneratorTestGroup, FailureCases) { -// { -// buildcc::CustomGenerator cgen("failure_no_cb", ""); -// buildcc::GenerateCb cb; -// cgen.AddGenerateCb(cb); -// CHECK_THROWS(std::exception, cgen.Build()); -// } - -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); - -// { -// buildcc::CustomGenerator cgen("failure_empty_task_cb", ""); -// cgen.AddGenerateCb(BadGenerateCb_EmptyTask); -// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.Build(); -// buildcc::m::CustomGeneratorRunner(cgen); - -// CHECK_TRUE(buildcc::env::get_task_state() == -// buildcc::env::TaskState::FAILURE); -// } - -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); - -// { -// buildcc::CustomGenerator cgen("failure_wrong_id_cb", ""); -// cgen.AddGenerateCb(BadGenerateCb_WrongId); -// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.Build(); -// buildcc::m::CustomGeneratorRunner(cgen); +TEST(CustomGeneratorTestGroup, FailureCases) { + { + buildcc::CustomGenerator cgen("failure_no_cb", ""); + buildcc::GenerateCb cb; + CHECK_THROWS(std::exception, cgen.AddGenInfo("id1", {}, {}, cb)); + } -// CHECK_TRUE(buildcc::env::get_task_state() == -// buildcc::env::TaskState::FAILURE); -// } + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + { + buildcc::CustomGenerator cgen("failure_cannot_save", ""); + fs::create_directory( + cgen.GetBinaryPath()); // make a folder so that file cannot be saved -// { -// buildcc::CustomGenerator cgen("failure_cannot_save", ""); -// fs::create_directory( -// cgen.GetBinaryPath()); // make a folder so that file cannot be saved + cgen.AddGenInfo("id1", {}, {}, BasicGenerateCb); + cgen.AddGenInfo("id2", {}, {}, BasicGenerateCb); + cgen.Build(); -// cgen.AddGenerateCb(BasicGenerateCb); -// cgen.AddGenInfo("id1", {}, {}); -// cgen.AddGenInfo("id2", {}, {}); -// cgen.Build(); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + buildcc::m::CustomGeneratorRunner(cgen); -// mock().expectOneCall("BasicGenerateCb"); -// mock().expectOneCall("id1"); -// mock().expectOneCall("id2"); -// buildcc::m::CustomGeneratorRunner(cgen); + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + } -// CHECK_TRUE(buildcc::env::get_task_state() == -// buildcc::env::TaskState::FAILURE); -// } + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + { + buildcc::CustomGenerator cgen("gen_task_not_run_no_io", ""); + cgen.Build(); -// { -// buildcc::CustomGenerator cgen("gen_task_not_run_no_io", ""); -// cgen.AddGenerateCb(BasicGenerateCb); -// cgen.Build(); + buildcc::m::CustomGeneratorRunner(cgen); + } -// mock().expectOneCall("BasicGenerateCb"); -// buildcc::m::CustomGeneratorRunner(cgen); -// } + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + { + buildcc::env::set_task_state(buildcc::env::TaskState::FAILURE); -// { -// buildcc::env::set_task_state(buildcc::env::TaskState::FAILURE); + buildcc::CustomGenerator cgen("gen_task_state_failure", ""); + cgen.AddGenInfo("id1", {}, {}, BasicGenerateCb); + cgen.Build(); + buildcc::m::CustomGeneratorRunner(cgen); -// buildcc::CustomGenerator cgen("gen_task_state_failure", ""); -// cgen.AddGenerateCb(BasicGenerateCb); -// cgen.Build(); -// buildcc::m::CustomGeneratorRunner(cgen); -// } + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + } -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); -// } + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); +} // static std::unordered_map // RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { From 1aa5f7e1101df389253bf4136464dfa35322a007 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 02:16:04 -0700 Subject: [PATCH 20/51] Updated test_custom_generator --- buildcc/lib/target/test/target/test_custom_generator.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 728aa854..ef05ae82 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -34,7 +34,8 @@ static bool BasicGenerateCb(const buildcc::CustomGeneratorContext &ctx) { TEST(CustomGeneratorTestGroup, Basic) { buildcc::CustomGenerator cgen("basic", ""); - cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, {}, BasicGenerateCb); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, BasicGenerateCb); cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.cpp"}, {}, BasicGenerateCb); cgen.Build(); @@ -53,7 +54,11 @@ TEST(CustomGeneratorTestGroup, Basic) { CHECK_EQUAL(internal_map.size(), 2); const auto &id1_info = internal_map.at("id1"); CHECK_EQUAL(id1_info.internal_inputs.size(), 1); - CHECK_EQUAL(id1_info.outputs.size(), 0); + CHECK_EQUAL(id1_info.outputs.size(), 1); + + const auto &id2_info = internal_map.at("id2"); + CHECK_EQUAL(id2_info.internal_inputs.size(), 1); + CHECK_EQUAL(id2_info.outputs.size(), 0); } } From b9a6598900c8cde471a6a9b515d828a9abd91a0a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 02:24:49 -0700 Subject: [PATCH 21/51] Update custom_generator fix for potential dependency bug condition --- .../src/custom_generator/custom_generator.cpp | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 3e049100..a303c1fb 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -152,36 +152,44 @@ void CustomGenerator::GenerateTask() { Convert(); BuildGenerate(selected_user_schema_, dummy_selected_user_schema_); - if (dirty_) { - std::unordered_map task_map; - - for (const auto &selected_miter : selected_user_schema_) { - const auto &id = selected_miter.first; - const auto &info = selected_miter.second; - - tf::Task task = subflow.emplace([&]() { - try { - CustomGeneratorContext ctx(command_, info.inputs, info.outputs); - bool success = info.generate_cb(ctx); - env::assert_fatal(success, "Generate Cb failed for id {}"); - AddSuccessSchema(id, info); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }); - task.name(id); - task_map.emplace(id, task); - } - - if (dependency_cb_) { - dependency_cb_(task_map); - } + std::unordered_map task_map; + + // Create task for selected schema + for (const auto &selected_miter : selected_user_schema_) { + const auto &id = selected_miter.first; + const auto &info = selected_miter.second; + tf::Task task = + subflow + .emplace([&]() { + try { + CustomGeneratorContext ctx(command_, info.inputs, + info.outputs); + bool success = info.generate_cb(ctx); + env::assert_fatal(success, "Generate Cb failed for id {}"); + AddSuccessSchema(id, info); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }) + .name(id); + task_map.emplace(id, task); + } - // TODO, Create Selected graph + // Create placeholder task for dummy/not selected schema + for (const auto &dummy_selected_miter : dummy_selected_user_schema_) { + const auto &id = dummy_selected_miter.first; + tf::Task task = subflow.placeholder().name(id); + task_map.emplace(id, task); } - // TODO, Create Dummy Selected graph + // Dependencies between ids + if (dependency_cb_) { + dependency_cb_(task_map); + } + // TODO, Create Selected graph, See Target task + // TODO, Create Dummy Selected graph, See Target task + // We need to map (Inputs -> FN (id) -> Outputs) } catch (...) { env::set_task_state(env::TaskState::FAILURE); } From 90f19c0f4cae1e04b3f305c7f8ed88b5728d06ef Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:06:11 -0700 Subject: [PATCH 22/51] Updated custom_generator.cpp --- buildcc/lib/target/include/target/generator.h | 10 ++--- .../src/custom_generator/custom_generator.cpp | 45 +++++++++---------- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index e5904994..6f6d727c 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -44,12 +44,12 @@ class CustomGeneratorContext { public: CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, const fs_unordered_set &o) - : command(c), input(i), output(o) {} + : command(c), inputs(i), outputs(o) {} public: const env::Command &command; - const fs_unordered_set &input; - const fs_unordered_set &output; + const fs_unordered_set &inputs; + const fs_unordered_set &outputs; }; // clang-format off @@ -141,10 +141,6 @@ class CustomGenerator : public internal::BuilderInterface { // Serialization UserCustomGeneratorSchema user_; - std::unordered_map - selected_user_schema_; - std::unordered_map - dummy_selected_user_schema_; std::mutex success_schema_mutex_; std::unordered_map success_schema_; diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index a303c1fb..04c23c1e 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -149,13 +149,18 @@ void CustomGenerator::GenerateTask() { } try { + std::unordered_map + selected_user_schema; + std::unordered_map + dummy_selected_user_schema; Convert(); - BuildGenerate(selected_user_schema_, dummy_selected_user_schema_); + BuildGenerate(selected_user_schema, dummy_selected_user_schema); - std::unordered_map task_map; + // TODO, Additional dummy recheck + std::unordered_map task_map; // Create task for selected schema - for (const auto &selected_miter : selected_user_schema_) { + for (const auto &selected_miter : selected_user_schema) { const auto &id = selected_miter.first; const auto &info = selected_miter.second; tf::Task task = @@ -176,7 +181,7 @@ void CustomGenerator::GenerateTask() { } // Create placeholder task for dummy/not selected schema - for (const auto &dummy_selected_miter : dummy_selected_user_schema_) { + for (const auto &dummy_selected_miter : dummy_selected_user_schema) { const auto &id = dummy_selected_miter.first; tf::Task task = subflow.placeholder().name(id); task_map.emplace(id, task); @@ -187,23 +192,14 @@ void CustomGenerator::GenerateTask() { dependency_cb_(task_map); } - // TODO, Create Selected graph, See Target task - // TODO, Create Dummy Selected graph, See Target task - // We need to map (Inputs -> FN (id) -> Outputs) - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }); - generate_task.name(kGenerateTaskName); + // NOTE, Do not call detach otherwise this will fail + subflow.join(); - tf::Task end_task = tf_.emplace([this]() { - // Even if env::TaskState::FAILURE we still need to partially store the - // built files - if (dirty_) { - try { + // Store dummy_selected and successfully run schema + if (dirty_) { UserCustomGeneratorSchema user_final_schema; - user_final_schema.rels_map.insert(dummy_selected_user_schema_.begin(), - dummy_selected_user_schema_.end()); + user_final_schema.rels_map.insert(dummy_selected_user_schema.begin(), + dummy_selected_user_schema.end()); user_final_schema.rels_map.insert(success_schema_.begin(), success_schema_.end()); @@ -211,15 +207,14 @@ void CustomGenerator::GenerateTask() { serialization_.UpdateStore(user_final_schema); env::assert_fatal(serialization_.StoreToFile(), fmt::format("Store failed for {}", name_)); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); } + + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); } }); - end_task.name(kEndGeneratorTaskName); - - // Dependencies - generate_task.precede(end_task); + // TODO, Instead of "Generate" name the task of user's choice + generate_task.name(kGenerateTaskName); } } // namespace buildcc From 22c3978a7023b21e5342485e8a726792630f3989 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:07:07 -0700 Subject: [PATCH 23/51] Update command.cpp with try catch for exception handling --- buildcc/lib/env/src/command.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/env/src/command.cpp b/buildcc/lib/env/src/command.cpp index 6620c37e..2962aa24 100644 --- a/buildcc/lib/env/src/command.cpp +++ b/buildcc/lib/env/src/command.cpp @@ -61,7 +61,14 @@ std::string Command::Construct( }); // Construct your command - return fmt::vformat(pattern, store); + std::string ret; + try { + ret = fmt::vformat(pattern, store); + } catch (const std::exception &e) { + env::assert_fatal( + fmt::format("Construct command failed: {}", e.what())); + } + return ret; } } // namespace buildcc::env From c72d783b42747a4024946b4683f095163c0825d4 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:08:52 -0700 Subject: [PATCH 24/51] Added Map Removed condition --- .../target/src/custom_generator/custom_generator.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 04c23c1e..fda29f6f 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -94,10 +94,18 @@ void CustomGenerator::BuildGenerate( const auto &curr_rels = user_.internal_rels_map; const auto &prev_rels = serialization_.GetLoad().internal_rels_map; - // TODO, MAP REMOVED condition Check if prev_rels exists in curr_rels + // DONE, MAP REMOVED condition Check if prev_rels exists in curr_rels // If prev_rels does not exist in curr_rels, has been removed from existing // build // We need this condition to only set the dirty_ flag + for (const auto &prev_miter : prev_rels) { + const auto &id = prev_miter.first; + if (curr_rels.find(id) == curr_rels.end()) { + // MAP REMOVED condition + dirty_; + break; + } + } // DONE, MAP ADDED condition Check if curr_rels exists in prev_rels // If curr_rels does not exist in prev_rels, has been added to existing From d744d8e626366b7350af2e3215fcd284740ef438 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:28:14 -0700 Subject: [PATCH 25/51] Shifted header from generator.h to custom_generator.h --- .../target/include/target/custom_generator.h | 158 ++++++++++++++++++ buildcc/lib/target/include/target/generator.h | 117 ------------- .../src/custom_generator/custom_generator.cpp | 4 +- 3 files changed, 159 insertions(+), 120 deletions(-) create mode 100644 buildcc/lib/target/include/target/custom_generator.h diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h new file mode 100644 index 00000000..49a089d1 --- /dev/null +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -0,0 +1,158 @@ +/* + * Copyright 2021-2022 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef TARGET_CUSTOM_GENERATOR_H_ +#define TARGET_CUSTOM_GENERATOR_H_ + +#include +#include +#include +#include + +#include "taskflow/taskflow.hpp" + +#include "env/command.h" +#include "env/task_state.h" + +#include "target/interface/builder_interface.h" + +#include "schema/custom_generator_serialization.h" +#include "schema/path.h" + +#include "target/common/target_env.h" + +namespace buildcc { + +// TODO, Shift to a different file +class CustomGeneratorContext { +public: + CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, + const fs_unordered_set &o) + : command(c), inputs(i), outputs(o) {} + +public: + const env::Command &command; + const fs_unordered_set &inputs; + const fs_unordered_set &outputs; +}; + +// clang-format off +typedef std::function GenerateCb; + +typedef std::function &)> DependencyCb; +// clang-format on + +struct UserRelInputOutputSchema : internal::RelInputOutputSchema { + fs_unordered_set inputs; + GenerateCb generate_cb; +}; + +struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { + std::unordered_map rels_map; + + void ConvertToInternal() { + for (auto &r_miter : rels_map) { + r_miter.second.internal_inputs = path_schema_convert( + r_miter.second.inputs, internal::Path::CreateExistingPath); + auto p = internal_rels_map.emplace(r_miter.first, r_miter.second); + env::assert_fatal(p.second, + fmt::format("Could not save {}", r_miter.first)); + } + } +}; + +// TODO, Add comments to functions +class CustomGenerator : public internal::BuilderInterface { +public: + CustomGenerator(const std::string &name, const TargetEnv &env, + bool parallel = false) + : name_(name), + env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), + serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)), + parallel_(parallel) { + Initialize(); + } + virtual ~CustomGenerator() = default; + CustomGenerator(const CustomGenerator &) = delete; + + /** + * @brief Add default arguments for input, output and command requirements + * + * @param arguments Key-Value pair for arguments + */ + void AddDefaultArguments( + const std::unordered_map &arguments); + + void AddGenInfo(const std::string &id, const fs_unordered_set &inputs, + const fs_unordered_set &outputs, + const GenerateCb &generate_cb); + + // Callbacks + void AddDependencyCb(const DependencyCb &dependency_cb); + + void Build() override; + + // Getters + const fs::path &GetBinaryPath() const { + return serialization_.GetSerializedFile(); + } + tf::Taskflow &GetTaskflow() { return tf_; } + +private: + void Initialize(); + + void GenerateTask(); + void Convert(); + void BuildGenerate(std::unordered_map + &gen_selected_map, + std::unordered_map + &dummy_gen_selected_map); + + void AddSuccessSchema(const std::string &id, + const UserRelInputOutputSchema &schema); + + // Recheck states + // TODO, + void PathRemoved() {} + void PathAdded() {} + void PathUpdated() {} + + void StringChanged() {} + +private: + std::string name_; + TargetEnv env_; + internal::CustomGeneratorSerialization serialization_; + bool parallel_; + + // Serialization + UserCustomGeneratorSchema user_; + + std::mutex success_schema_mutex_; + std::unordered_map success_schema_; + + // Internal + std::mutex task_state_mutex_; + env::Command command_; + tf::Taskflow tf_; + + // Callbacks + DependencyCb dependency_cb_; +}; + +} // namespace buildcc + +#endif diff --git a/buildcc/lib/target/include/target/generator.h b/buildcc/lib/target/include/target/generator.h index 6f6d727c..3784e991 100644 --- a/buildcc/lib/target/include/target/generator.h +++ b/buildcc/lib/target/include/target/generator.h @@ -17,7 +17,6 @@ #ifndef TARGET_GENERATOR_H_ #define TARGET_GENERATOR_H_ -#include #include #include #include @@ -25,14 +24,12 @@ #include "taskflow/taskflow.hpp" -#include "env/env.h" #include "env/task_state.h" #include "env/command.h" #include "target/interface/builder_interface.h" -#include "schema/custom_generator_serialization.h" #include "schema/generator_serialization.h" #include "schema/path.h" @@ -40,120 +37,6 @@ namespace buildcc { -class CustomGeneratorContext { -public: - CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, - const fs_unordered_set &o) - : command(c), inputs(i), outputs(o) {} - -public: - const env::Command &command; - const fs_unordered_set &inputs; - const fs_unordered_set &outputs; -}; - -// clang-format off -typedef std::function GenerateCb; - -typedef std::function &)> DependencyCb; -// clang-format on - -struct UserRelInputOutputSchema : internal::RelInputOutputSchema { - fs_unordered_set inputs; - GenerateCb generate_cb; -}; - -struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { - std::unordered_map rels_map; - - void ConvertToInternal() { - for (auto &r_miter : rels_map) { - r_miter.second.internal_inputs = path_schema_convert( - r_miter.second.inputs, internal::Path::CreateExistingPath); - auto p = internal_rels_map.emplace(r_miter.first, r_miter.second); - env::assert_fatal(p.second, - fmt::format("Could not save {}", r_miter.first)); - } - } -}; - -class CustomGenerator : public internal::BuilderInterface { -public: - CustomGenerator(const std::string &name, const TargetEnv &env, - bool parallel = false) - : name_(name), - env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), - serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)), - parallel_(parallel) { - Initialize(); - } - virtual ~CustomGenerator() = default; - CustomGenerator(const CustomGenerator &) = delete; - - /** - * @brief Add default arguments for input, output and command requirements - * - * @param arguments Key-Value pair for arguments - */ - void AddDefaultArguments( - const std::unordered_map &arguments); - - void AddGenInfo(const std::string &id, const fs_unordered_set &inputs, - const fs_unordered_set &outputs, - const GenerateCb &generate_cb); - - // Callbacks - void AddDependencyCb(const DependencyCb &dependency_cb); - - void Build() override; - - // Getters - const fs::path &GetBinaryPath() const { - return serialization_.GetSerializedFile(); - } - tf::Taskflow &GetTaskflow() { return tf_; } - -private: - void Initialize(); - - void GenerateTask(); - void Convert(); - void BuildGenerate(std::unordered_map - &gen_selected_map, - std::unordered_map - &dummy_gen_selected_map); - - void AddSuccessSchema(const std::string &id, - const UserRelInputOutputSchema &schema); - - // Recheck states - void PathRemoved() {} - void PathAdded() {} - void PathUpdated() {} - - void StringChanged() {} - -private: - std::string name_; - TargetEnv env_; - internal::CustomGeneratorSerialization serialization_; - bool parallel_; - - // Serialization - UserCustomGeneratorSchema user_; - - std::mutex success_schema_mutex_; - std::unordered_map success_schema_; - - // Internal - std::mutex task_state_mutex_; - env::Command command_; - tf::Taskflow tf_; - - // Callbacks - DependencyCb dependency_cb_; -}; - // TODO, Make this private struct UserGeneratorSchema : public internal::GeneratorSchema { fs_unordered_set inputs; diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index fda29f6f..b5c1f4f0 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -14,7 +14,7 @@ * limitations under the License. */ -#include "target/generator.h" +#include "target/custom_generator.h" namespace { constexpr const char *const kStartGeneratorTaskName = "Start Generator"; @@ -164,8 +164,6 @@ void CustomGenerator::GenerateTask() { Convert(); BuildGenerate(selected_user_schema, dummy_selected_user_schema); - // TODO, Additional dummy recheck - std::unordered_map task_map; // Create task for selected schema for (const auto &selected_miter : selected_user_schema) { From e6b619dfc661e9d74cc97b8dc2cbe8e0f405c6ec Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:39:44 -0700 Subject: [PATCH 26/51] Added recheck_states.cpp --- .../lib/target/cmake/common_target_src.cmake | 1 + buildcc/lib/target/cmake/target.cmake | 2 ++ .../src/custom_generator/custom_generator.cpp | 5 +++- .../src/custom_generator/recheck_states.cpp | 25 +++++++++++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 buildcc/lib/target/src/custom_generator/recheck_states.cpp diff --git a/buildcc/lib/target/cmake/common_target_src.cmake b/buildcc/lib/target/cmake/common_target_src.cmake index d12a78b5..31ab0a03 100644 --- a/buildcc/lib/target/cmake/common_target_src.cmake +++ b/buildcc/lib/target/cmake/common_target_src.cmake @@ -35,6 +35,7 @@ set(COMMON_TARGET_SRCS # Generator src/custom_generator/custom_generator.cpp + include/target/custom_generator.h src/generator/generator.cpp include/target/generator.h diff --git a/buildcc/lib/target/cmake/target.cmake b/buildcc/lib/target/cmake/target.cmake index 98f16479..931b1c46 100644 --- a/buildcc/lib/target/cmake/target.cmake +++ b/buildcc/lib/target/cmake/target.cmake @@ -1,6 +1,8 @@ set(TARGET_SRCS ${COMMON_TARGET_SRCS} + src/custom_generator/recheck_states.cpp + src/generator/task.cpp src/generator/recheck_states.cpp diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index b5c1f4f0..f8e1f654 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -102,7 +102,8 @@ void CustomGenerator::BuildGenerate( const auto &id = prev_miter.first; if (curr_rels.find(id) == curr_rels.end()) { // MAP REMOVED condition - dirty_; + IdRemoved(); + dirty_ = true; break; } } @@ -119,6 +120,7 @@ void CustomGenerator::BuildGenerate( const auto &id = curr_miter.first; if (prev_rels.find(id) == prev_rels.end()) { // MAP ADDED condition + IdAdded(); gen_selected_map.emplace(curr_miter.first, user_.rels_map.at(curr_miter.first)); dirty_ = true; @@ -131,6 +133,7 @@ void CustomGenerator::BuildGenerate( if (path_state != internal::PathState::kNoChange || changed) { // MAP UPDATED condition + IdUpdated(); gen_selected_map.emplace(curr_miter.first, user_.rels_map.at(curr_miter.first)); dirty_ = true; diff --git a/buildcc/lib/target/src/custom_generator/recheck_states.cpp b/buildcc/lib/target/src/custom_generator/recheck_states.cpp new file mode 100644 index 00000000..0ccc632c --- /dev/null +++ b/buildcc/lib/target/src/custom_generator/recheck_states.cpp @@ -0,0 +1,25 @@ +/* + * Copyright 2021-2022 Niket Naidu. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "target/custom_generator.h" + +namespace buildcc { + +void CustomGenerator::IdRemoved() {} +void CustomGenerator::IdAdded() {} +void CustomGenerator::IdUpdated() {} + +} // namespace buildcc From e6c01f5bbb00b486bce3b62ae8edf2ad110931cf Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:40:42 -0700 Subject: [PATCH 27/51] Added recheck_states mocks --- buildcc/lib/target/cmake/mock_target.cmake | 3 ++ .../mock/custom_generator/recheck_states.cpp | 40 +++++++++++++++++++ .../target/mock/custom_generator/runner.cpp | 13 ++++++ .../lib/target/mock/expect_custom_generator.h | 23 +++++++++++ buildcc/lib/target/mock/expect_generator.h | 2 - buildcc/lib/target/mock/generator/runner.cpp | 6 --- 6 files changed, 79 insertions(+), 8 deletions(-) create mode 100644 buildcc/lib/target/mock/custom_generator/recheck_states.cpp create mode 100644 buildcc/lib/target/mock/custom_generator/runner.cpp create mode 100644 buildcc/lib/target/mock/expect_custom_generator.h diff --git a/buildcc/lib/target/cmake/mock_target.cmake b/buildcc/lib/target/cmake/mock_target.cmake index 2d648938..cf2b6bde 100644 --- a/buildcc/lib/target/cmake/mock_target.cmake +++ b/buildcc/lib/target/cmake/mock_target.cmake @@ -1,5 +1,8 @@ add_library(mock_target STATIC ${COMMON_TARGET_SRCS} + # Custom Generator mocks + mock/custom_generator/runner.cpp + mock/custom_generator/recheck_states.cpp # Generator mocks src/generator/task.cpp diff --git a/buildcc/lib/target/mock/custom_generator/recheck_states.cpp b/buildcc/lib/target/mock/custom_generator/recheck_states.cpp new file mode 100644 index 00000000..b9a48780 --- /dev/null +++ b/buildcc/lib/target/mock/custom_generator/recheck_states.cpp @@ -0,0 +1,40 @@ +#include "target/custom_generator.h" + +#include "expect_custom_generator.h" + +#include "CppUTestExt/MockSupport.h" + +namespace buildcc { + +static constexpr const char *const ID_REMOVED_FUNCTION = + "CustomGenerator::IdRemoved"; +static constexpr const char *const ID_ADDED_FUNCTION = + "CustomGenerator::IdAdded"; +static constexpr const char *const ID_UPDATED_FUNCTION = + "CustomGenerator::IdUpdated"; + +void CustomGenerator::IdRemoved() { + mock().actualCall(ID_REMOVED_FUNCTION).onObject(this); +} +void CustomGenerator::IdAdded() { + mock().actualCall(ID_ADDED_FUNCTION).onObject(this); +} +void CustomGenerator::IdUpdated() { + mock().actualCall(ID_UPDATED_FUNCTION).onObject(this); +} + +namespace m { + +void GeneratorExpect_IdRemoved(unsigned int calls, CustomGenerator *generator) { + mock().expectNCalls(calls, ID_REMOVED_FUNCTION).onObject(generator); +} +void GeneratorExpect_IdAdded(unsigned int calls, CustomGenerator *generator) { + mock().expectNCalls(calls, ID_ADDED_FUNCTION).onObject(generator); +} +void GeneratorExpect_IdUpdated(unsigned int calls, CustomGenerator *generator) { + mock().expectNCalls(calls, ID_UPDATED_FUNCTION).onObject(generator); +} + +} // namespace m + +} // namespace buildcc diff --git a/buildcc/lib/target/mock/custom_generator/runner.cpp b/buildcc/lib/target/mock/custom_generator/runner.cpp new file mode 100644 index 00000000..8014c851 --- /dev/null +++ b/buildcc/lib/target/mock/custom_generator/runner.cpp @@ -0,0 +1,13 @@ +#include "target/custom_generator.h" + +#include "expect_custom_generator.h" + +namespace buildcc::m { + +void CustomGeneratorRunner(CustomGenerator &custom_generator) { + tf::Executor executor(1); + executor.run(custom_generator.GetTaskflow()); + executor.wait_for_all(); +} + +} // namespace buildcc::m diff --git a/buildcc/lib/target/mock/expect_custom_generator.h b/buildcc/lib/target/mock/expect_custom_generator.h new file mode 100644 index 00000000..f54ed17c --- /dev/null +++ b/buildcc/lib/target/mock/expect_custom_generator.h @@ -0,0 +1,23 @@ +#ifndef TARGET_MOCK_EXPECT_CUSTOM_GENERATOR_H_ +#define TARGET_MOCK_EXPECT_CUSTOM_GENERATOR_H_ + +#include "target/custom_generator.h" + +namespace buildcc::m { + +/** + * @brief Runs the generator using Taskflow with 1 thread + * CppUTest cannot mock with multiple threads + */ +void CustomGeneratorRunner(CustomGenerator &custom_generator); + +void CustomGeneratorExpect_IdRemoved(unsigned int calls, + CustomGenerator *generator); +void CustomGeneratorExpect_IdAdded(unsigned int calls, + CustomGenerator *generator); +void CustomGeneratorExpect_IdUpdated(unsigned int calls, + CustomGenerator *generator); + +} // namespace buildcc::m + +#endif diff --git a/buildcc/lib/target/mock/expect_generator.h b/buildcc/lib/target/mock/expect_generator.h index 248ba079..b41cd780 100644 --- a/buildcc/lib/target/mock/expect_generator.h +++ b/buildcc/lib/target/mock/expect_generator.h @@ -5,8 +5,6 @@ namespace buildcc::m { -void CustomGeneratorRunner(CustomGenerator &custom_generator); - /** * @brief Runs the generator using Taskflow with 1 thread * CppUTest cannot mock with multiple threads diff --git a/buildcc/lib/target/mock/generator/runner.cpp b/buildcc/lib/target/mock/generator/runner.cpp index 61fdfe2e..3ca990e8 100644 --- a/buildcc/lib/target/mock/generator/runner.cpp +++ b/buildcc/lib/target/mock/generator/runner.cpp @@ -4,12 +4,6 @@ namespace buildcc::m { -void CustomGeneratorRunner(CustomGenerator &custom_generator) { - tf::Executor executor(1); - executor.run(custom_generator.GetTaskflow()); - executor.wait_for_all(); -} - void GeneratorRunner(Generator &generator) { tf::Executor executor(1); executor.run(generator.GetTaskflow()); From 31a0bbb5a06ea7903d2b84c7e22eb5d731be18a6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:41:05 -0700 Subject: [PATCH 28/51] Adedd Id states --- buildcc/lib/target/include/target/custom_generator.h | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 49a089d1..1d8e7724 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -125,12 +125,9 @@ class CustomGenerator : public internal::BuilderInterface { const UserRelInputOutputSchema &schema); // Recheck states - // TODO, - void PathRemoved() {} - void PathAdded() {} - void PathUpdated() {} - - void StringChanged() {} + void IdRemoved(); + void IdAdded(); + void IdUpdated(); private: std::string name_; From cb812410092ef378af748e943b57856973997335 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:41:13 -0700 Subject: [PATCH 29/51] Updated buildcc.h --- buildcc/buildcc.h | 1 + 1 file changed, 1 insertion(+) diff --git a/buildcc/buildcc.h b/buildcc/buildcc.h index 8f08c5db..e73bbffe 100644 --- a/buildcc/buildcc.h +++ b/buildcc/buildcc.h @@ -31,6 +31,7 @@ // Base #include "toolchain/toolchain.h" +#include "target/custom_generator.h" #include "target/generator.h" #include "target/target_info.h" #include "target/target.h" From 4d2890b7d701eb5b9cbf4cdbcbb72c63d0ee15c6 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:41:36 -0700 Subject: [PATCH 30/51] Updated unit tests --- buildcc/lib/target/test/target/test_custom_generator.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index ef05ae82..40d431b4 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -1,7 +1,7 @@ -#include "target/generator.h" +#include "target/custom_generator.h" #include "expect_command.h" -#include "expect_generator.h" +#include "expect_custom_generator.h" // #include "test_target_util.h" // #include "taskflow/taskflow.hpp" From 7bf8d32e5d49468770677b8bca47f8447431de3a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Fri, 6 May 2022 23:57:02 -0700 Subject: [PATCH 31/51] Updated test_custom_generator.cpp --- .../target/include/target/custom_generator.h | 1 + .../test/target/test_custom_generator.cpp | 176 ++++++------------ 2 files changed, 57 insertions(+), 120 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 1d8e7724..cdb9bc7d 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -37,6 +37,7 @@ namespace buildcc { // TODO, Shift to a different file +// TODO, Check if we need the "id" here as well class CustomGeneratorContext { public: CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i, diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 40d431b4..1feb25fa 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -147,126 +147,62 @@ TEST(CustomGeneratorTestGroup, FailureCases) { buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } -// static std::unordered_map -// RealGenerateCb(tf::Subflow &subflow, buildcc::CustomGeneratorContext &ctx) { -// mock().actualCall("RealGenerateCb"); -// std::unordered_map uom; -// for (const auto &miter : ctx.selected_schema) { -// auto task = subflow.emplace([&]() { -// try { -// bool executed = buildcc::env::Command::Execute(""); -// buildcc::env::assert_fatal(executed, ""); -// ctx.Success(miter.first); -// mock().actualCall(fmt::format("{}:SUCCESS", miter.first).c_str()); -// } catch (...) { -// ctx.Failure(); -// mock().actualCall(fmt::format("{}:FAILURE", miter.first).c_str()); -// } -// }); -// uom.emplace(miter.first, task); -// } -// return uom; -// } - -// TEST(CustomGeneratorTestGroup, RealGenerate) { -// constexpr const char *const kGenName = "real_generator"; -// { -// buildcc::CustomGenerator cgen(kGenName, ""); -// cgen.AddGenerateCb(RealGenerateCb); -// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.Build(); - -// mock().expectOneCall("RealGenerateCb"); -// buildcc::env::m::CommandExpect_Execute(1, true); -// buildcc::env::m::CommandExpect_Execute(1, true); -// mock().expectOneCall("id1:SUCCESS"); -// mock().expectOneCall("id2:SUCCESS"); -// buildcc::m::CustomGeneratorRunner(cgen); - -// buildcc::internal::CustomGeneratorSerialization serialization( -// cgen.GetBinaryPath()); -// CHECK_TRUE(serialization.LoadFromFile()); -// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); - -// fs::remove_all(cgen.GetBinaryPath()); -// } - -// { -// buildcc::CustomGenerator cgen(kGenName, ""); -// cgen.AddGenerateCb(RealGenerateCb); -// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}); -// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, {}); -// cgen.Build(); - -// mock().expectOneCall("RealGenerateCb"); -// buildcc::env::m::CommandExpect_Execute(1, false); -// buildcc::env::m::CommandExpect_Execute(1, true); -// mock().expectOneCall("id1:FAILURE"); -// mock().expectOneCall("id2:SUCCESS"); -// buildcc::m::CustomGeneratorRunner(cgen); - -// CHECK_TRUE(buildcc::env::get_task_state() == -// buildcc::env::TaskState::FAILURE); - -// buildcc::internal::CustomGeneratorSerialization serialization( -// cgen.GetBinaryPath()); -// CHECK_TRUE(serialization.LoadFromFile()); -// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); - -// fs::remove_all(cgen.GetBinaryPath()); -// } - -// buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); -// } - -// TEST(CustomGeneratorTestGroup, RealGenerate_RebuildCondition) { -// constexpr const char *const kGenName = "real_generator_rebuild"; -// { -// buildcc::CustomGenerator cgen(kGenName, ""); -// cgen.AddGenerateCb(RealGenerateCb); -// cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.Build(); - -// mock().expectOneCall("RealGenerateCb"); -// buildcc::env::m::CommandExpect_Execute(1, true); -// buildcc::env::m::CommandExpect_Execute(1, true); -// mock().expectOneCall("id1:SUCCESS"); -// mock().expectOneCall("id2:SUCCESS"); -// buildcc::m::CustomGeneratorRunner(cgen); - -// buildcc::internal::CustomGeneratorSerialization serialization( -// cgen.GetBinaryPath()); -// CHECK_TRUE(serialization.LoadFromFile()); -// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); -// } - -// // Remove Rel IO condition -// { -// buildcc::CustomGenerator cgen(kGenName, ""); -// cgen.AddGenerateCb(RealGenerateCb); -// cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, -// {"{gen_build_dir}/dummy_main.o"}); -// cgen.Build(); - -// mock().expectOneCall("RealGenerateCb"); -// buildcc::env::m::CommandExpect_Execute(1, true); -// buildcc::env::m::CommandExpect_Execute(1, true); -// mock().expectOneCall("id1:SUCCESS"); -// mock().expectOneCall("id2:SUCCESS"); -// buildcc::m::CustomGeneratorRunner(cgen); - -// buildcc::internal::CustomGeneratorSerialization serialization( -// cgen.GetBinaryPath()); -// CHECK_TRUE(serialization.LoadFromFile()); -// CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); -// } -// } +static bool RealGenerateCb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + mock().actualCall("RealGenerateCb"); + return buildcc::env::Command::Execute(""); +} + +TEST(CustomGeneratorTestGroup, RealGenerate) { + constexpr const char *const kGenName = "real_generator"; + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + fs::remove_all(cgen.GetBinaryPath()); + } + + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, {}, + RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, {}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, false); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + CHECK_TRUE(buildcc::env::get_task_state() == + buildcc::env::TaskState::FAILURE); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + + fs::remove_all(cgen.GetBinaryPath()); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); +} int main(int ac, char **av) { fs::remove_all(BUILD_DIR); From 1e836e70686fa94ca428775106ae4ff074856230 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sat, 7 May 2022 00:14:29 -0700 Subject: [PATCH 32/51] Added comments --- .../target/include/target/custom_generator.h | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index cdb9bc7d..f1053baf 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -75,7 +75,6 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { } }; -// TODO, Add comments to functions class CustomGenerator : public internal::BuilderInterface { public: CustomGenerator(const std::string &name, const TargetEnv &env, @@ -97,11 +96,34 @@ class CustomGenerator : public internal::BuilderInterface { void AddDefaultArguments( const std::unordered_map &arguments); + /** + * @brief Single Generator task for inputs->generate_cb->outputs + * + * @param id Unique id associated with Generator task + * @param inputs File inputs + * @param outputs File outputs + * @param generate_cb User-defined generate callback to build outputs from the + * provided inputs + */ void AddGenInfo(const std::string &id, const fs_unordered_set &inputs, const fs_unordered_set &outputs, const GenerateCb &generate_cb); // Callbacks + /** + * @brief Setup dependencies between Tasks using their `id` + * For example: `task_map["id1"].precede(task_map["id2"])` + * + * IMPORTANT: Successor tasks will not automatically run if dependent task is + * run. + * The Dependency callback only sets precedence (order in which your tasks + * should run) + * Default behaviour when dependency callback is not supplied: All task `id`s + * run in parallel. + * + * @param dependency_cb Unordered map of `id` and `task` + * The map can be safely mutated. + */ void AddDependencyCb(const DependencyCb &dependency_cb); void Build() override; From 21d4ab03ee29b5f70ed8d5da720ada5ff4d5fc47 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 20:45:32 -0700 Subject: [PATCH 33/51] Updated custom_generator to handle subflow dependencies --- .../target/include/target/custom_generator.h | 2 - .../src/custom_generator/custom_generator.cpp | 65 ++++++++++--------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index f1053baf..0c19e2d0 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -138,7 +138,6 @@ class CustomGenerator : public internal::BuilderInterface { void Initialize(); void GenerateTask(); - void Convert(); void BuildGenerate(std::unordered_map &gen_selected_map, std::unordered_map @@ -165,7 +164,6 @@ class CustomGenerator : public internal::BuilderInterface { std::unordered_map success_schema_; // Internal - std::mutex task_state_mutex_; env::Command command_; tf::Taskflow tf_; diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index f8e1f654..bd713289 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -80,8 +80,6 @@ void CustomGenerator::Initialize() { tf_.name(name_); } -void CustomGenerator::Convert() { user_.ConvertToInternal(); } - void CustomGenerator::BuildGenerate( std::unordered_map &gen_selected_map, std::unordered_map @@ -91,8 +89,8 @@ void CustomGenerator::BuildGenerate( dirty_ = true; } else { // DONE, Conditionally select internal_rels depending on what has changed - const auto &curr_rels = user_.internal_rels_map; const auto &prev_rels = serialization_.GetLoad().internal_rels_map; + const auto &curr_rels = user_.rels_map; // DONE, MAP REMOVED condition Check if prev_rels exists in curr_rels // If prev_rels does not exist in curr_rels, has been removed from existing @@ -111,37 +109,18 @@ void CustomGenerator::BuildGenerate( // DONE, MAP ADDED condition Check if curr_rels exists in prev_rels // If curr_rels does not exist in prev_rels, has been added to existing // build - - // DONE, MAP UPDATED condition, Check if curr_rels exists in prev_rels - // If curr_rels exists in prev_rels, but the - // * INPUT has been removed, added or changed - // * OUTPUT has been removed or added for (const auto &curr_miter : curr_rels) { const auto &id = curr_miter.first; if (prev_rels.find(id) == prev_rels.end()) { // MAP ADDED condition IdAdded(); - gen_selected_map.emplace(curr_miter.first, - user_.rels_map.at(curr_miter.first)); + gen_selected_map.emplace(curr_miter.first, curr_miter.second); dirty_ = true; } else { - // Check internal_inputs - auto path_state = CheckPaths(prev_rels.at(id).internal_inputs, - curr_miter.second.internal_inputs); - auto changed = - CheckChanged(prev_rels.at(id).outputs, curr_miter.second.outputs); - - if (path_state != internal::PathState::kNoChange || changed) { - // MAP UPDATED condition - IdUpdated(); - gen_selected_map.emplace(curr_miter.first, - user_.rels_map.at(curr_miter.first)); - dirty_ = true; - } else { - // MAP NO CHANGE condition - dummy_gen_selected_map.emplace(curr_miter.first, - user_.rels_map.at(curr_miter.first)); - } + // MAP UPDATED condition (*checked later) + // This is because tasks can have dependencies amongst each other we can + // compute task level rebuilds later + dummy_gen_selected_map.emplace(curr_miter.first, curr_miter.second); } } } @@ -164,7 +143,6 @@ void CustomGenerator::GenerateTask() { selected_user_schema; std::unordered_map dummy_selected_user_schema; - Convert(); BuildGenerate(selected_user_schema, dummy_selected_user_schema); std::unordered_map task_map; @@ -189,10 +167,35 @@ void CustomGenerator::GenerateTask() { task_map.emplace(id, task); } - // Create placeholder task for dummy/not selected schema for (const auto &dummy_selected_miter : dummy_selected_user_schema) { const auto &id = dummy_selected_miter.first; - tf::Task task = subflow.placeholder().name(id); + const auto &info = dummy_selected_miter.second; + serialization_.GetLoad().internal_rels_map.at(id); + + tf::Task task = subflow.emplace([&]() { + try { + auto curr_internal_inputs = internal::path_schema_convert( + info.inputs, internal::Path::CreateExistingPath); + auto previous_info = + serialization_.GetLoad().internal_rels_map.at(id); + bool rebuild = + internal::CheckPaths(previous_info.internal_inputs, + curr_internal_inputs) != + internal::PathState::kNoChange || + internal::CheckChanged(previous_info.outputs, info.outputs); + if (!rebuild) { + return; + } + + // TODO, Duplicated code from above + CustomGeneratorContext ctx(command_, info.inputs, info.outputs); + bool success = info.generate_cb(ctx); + env::assert_fatal(success, "Generate Cb failed for id {}"); + AddSuccessSchema(id, info); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }); task_map.emplace(id, task); } @@ -207,8 +210,6 @@ void CustomGenerator::GenerateTask() { // Store dummy_selected and successfully run schema if (dirty_) { UserCustomGeneratorSchema user_final_schema; - user_final_schema.rels_map.insert(dummy_selected_user_schema.begin(), - dummy_selected_user_schema.end()); user_final_schema.rels_map.insert(success_schema_.begin(), success_schema_.end()); From e364c9b3149e4aacdf9dd82b8af57c964347c712 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 21:32:57 -0700 Subject: [PATCH 34/51] Updated custom_generator --- .../target/include/target/custom_generator.h | 5 +- .../src/custom_generator/custom_generator.cpp | 82 +++++++++---------- 2 files changed, 41 insertions(+), 46 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 0c19e2d0..e028b44a 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -137,15 +137,14 @@ class CustomGenerator : public internal::BuilderInterface { private: void Initialize(); + template void TaskRunner(const std::string &id); + void GenerateTask(); void BuildGenerate(std::unordered_map &gen_selected_map, std::unordered_map &dummy_gen_selected_map); - void AddSuccessSchema(const std::string &id, - const UserRelInputOutputSchema &schema); - // Recheck states void IdRemoved(); void IdAdded(); diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index bd713289..0d196481 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -126,12 +126,6 @@ void CustomGenerator::BuildGenerate( } } -void CustomGenerator::AddSuccessSchema(const std::string &id, - const UserRelInputOutputSchema &schema) { - std::lock_guard guard(success_schema_mutex_); - success_schema_.emplace(id, schema); -} - void CustomGenerator::GenerateTask() { tf::Task generate_task = tf_.emplace([&](tf::Subflow &subflow) { if (env::get_task_state() != env::TaskState::SUCCESS) { @@ -149,49 +143,26 @@ void CustomGenerator::GenerateTask() { // Create task for selected schema for (const auto &selected_miter : selected_user_schema) { const auto &id = selected_miter.first; - const auto &info = selected_miter.second; - tf::Task task = - subflow - .emplace([&]() { - try { - CustomGeneratorContext ctx(command_, info.inputs, - info.outputs); - bool success = info.generate_cb(ctx); - env::assert_fatal(success, "Generate Cb failed for id {}"); - AddSuccessSchema(id, info); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }) - .name(id); + tf::Task task = subflow + .emplace([&]() { + try { + TaskRunner(id); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }) + .name(id); task_map.emplace(id, task); } - for (const auto &dummy_selected_miter : dummy_selected_user_schema) { + for (auto &dummy_selected_miter : dummy_selected_user_schema) { const auto &id = dummy_selected_miter.first; - const auto &info = dummy_selected_miter.second; - serialization_.GetLoad().internal_rels_map.at(id); - + auto ¤t_info = dummy_selected_miter.second; tf::Task task = subflow.emplace([&]() { try { - auto curr_internal_inputs = internal::path_schema_convert( - info.inputs, internal::Path::CreateExistingPath); - auto previous_info = - serialization_.GetLoad().internal_rels_map.at(id); - bool rebuild = - internal::CheckPaths(previous_info.internal_inputs, - curr_internal_inputs) != - internal::PathState::kNoChange || - internal::CheckChanged(previous_info.outputs, info.outputs); - if (!rebuild) { - return; - } - - // TODO, Duplicated code from above - CustomGeneratorContext ctx(command_, info.inputs, info.outputs); - bool success = info.generate_cb(ctx); - env::assert_fatal(success, "Generate Cb failed for id {}"); - AddSuccessSchema(id, info); + current_info.internal_inputs = internal::path_schema_convert( + current_info.inputs, internal::Path::CreateExistingPath); + TaskRunner(id); } catch (...) { env::set_task_state(env::TaskState::FAILURE); } @@ -227,4 +198,29 @@ void CustomGenerator::GenerateTask() { generate_task.name(kGenerateTaskName); } +template void CustomGenerator::TaskRunner(const std::string &id) { + const auto ¤t_info = user_.rels_map.at(id); + if constexpr (!run) { + const auto &previous_info = + serialization_.GetLoad().internal_rels_map.at(id); + bool rerun = + internal::CheckPaths(previous_info.internal_inputs, + current_info.internal_inputs) != + internal::PathState::kNoChange || + internal::CheckChanged(previous_info.outputs, current_info.outputs); + + // run && rerun == false, skip running the task + if (!rerun) { + return; + } + } + + buildcc::CustomGeneratorContext ctx(command_, current_info.inputs, + current_info.outputs); + bool success = current_info.generate_cb(ctx); + env::assert_fatal(success, "Generate Cb failed for id {}"); + std::lock_guard guard(success_schema_mutex_); + success_schema_.emplace(id, current_info); +} + } // namespace buildcc From f3c65ee9b2660f64f1a2547c3dde32d615997d81 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 21:38:45 -0700 Subject: [PATCH 35/51] Added dependency basic check --- .../test/target/test_custom_generator.cpp | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 1feb25fa..36dc57c9 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -147,6 +147,48 @@ TEST(CustomGeneratorTestGroup, FailureCases) { buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } +static bool Dep1Cb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + mock().actualCall("Dep1Cb"); + return buildcc::env::Command::Execute(""); +} + +static bool Dep2Cb(const buildcc::CustomGeneratorContext &ctx) { + (void)ctx; + mock().actualCall("Dep2Cb"); + return buildcc::env::Command::Execute(""); +} + +static void DependencyCb(std::unordered_map &task_map) { + task_map["id1"].precede(task_map["id2"]); +} + +TEST(CustomGeneratorTestGroup, AddDependency_BasicCheck) { + constexpr const char *const kGenName = "add_dependency"; + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, Dep2Cb); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, Dep1Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + mock().expectOneCall("Dep1Cb"); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("Dep2Cb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + fs::remove_all(cgen.GetBinaryPath()); + } +} + static bool RealGenerateCb(const buildcc::CustomGeneratorContext &ctx) { (void)ctx; mock().actualCall("RealGenerateCb"); From 2224dd5b4a032637851f1f0cf53ada3812f6d059 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 21:45:36 -0700 Subject: [PATCH 36/51] Added dependency_file test --- .../test/target/test_custom_generator.cpp | 39 ++++++++++++++++++- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 36dc57c9..7b5f951a 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -164,7 +164,7 @@ static void DependencyCb(std::unordered_map &task_map) { } TEST(CustomGeneratorTestGroup, AddDependency_BasicCheck) { - constexpr const char *const kGenName = "add_dependency"; + constexpr const char *const kGenName = "add_dependency_basic_check"; { buildcc::CustomGenerator cgen(kGenName, ""); cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, @@ -184,8 +184,43 @@ TEST(CustomGeneratorTestGroup, AddDependency_BasicCheck) { cgen.GetBinaryPath()); CHECK_TRUE(serialization.LoadFromFile()); CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + } +} - fs::remove_all(cgen.GetBinaryPath()); +static bool FileDep1Cb(const buildcc::CustomGeneratorContext &ctx) { + mock().actualCall("FileDep1Cb"); + for (const auto &o : ctx.outputs) { + CHECK_TRUE(buildcc::env::save_file(o.string().c_str(), "", false)); + } + return true; +} + +static bool FileDep2Cb(const buildcc::CustomGeneratorContext &ctx) { + mock().actualCall("FileDep2Cb"); + for (const auto &i : ctx.inputs) { + CHECK_TRUE(fs::exists(i)); + } + return true; +} + +TEST(CustomGeneratorTestGroup, AddDependency_FileDep) { + constexpr const char *const kGenName = "add_dependency_file_dep"; + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, FileDep1Cb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.o"}, {}, FileDep2Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + mock().expectOneCall("FileDep1Cb"); + mock().expectOneCall("FileDep2Cb"); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); } } From affa14e8738b2cba25e561355834abba3d539d1d Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 23:01:20 -0700 Subject: [PATCH 37/51] Updated custom_generator.cpp TaskRunner --- .../src/custom_generator/custom_generator.cpp | 53 ++++++++++--------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 0d196481..1809b036 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -158,15 +158,19 @@ void CustomGenerator::GenerateTask() { for (auto &dummy_selected_miter : dummy_selected_user_schema) { const auto &id = dummy_selected_miter.first; auto ¤t_info = dummy_selected_miter.second; - tf::Task task = subflow.emplace([&]() { - try { - current_info.internal_inputs = internal::path_schema_convert( - current_info.inputs, internal::Path::CreateExistingPath); - TaskRunner(id); - } catch (...) { - env::set_task_state(env::TaskState::FAILURE); - } - }); + tf::Task task = subflow + .emplace([&]() { + try { + user_.rels_map.at(id).internal_inputs = + internal::path_schema_convert( + current_info.inputs, + internal::Path::CreateExistingPath); + TaskRunner(id); + } catch (...) { + env::set_task_state(env::TaskState::FAILURE); + } + }) + .name(id); task_map.emplace(id, task); } @@ -184,6 +188,7 @@ void CustomGenerator::GenerateTask() { user_final_schema.rels_map.insert(success_schema_.begin(), success_schema_.end()); + // TODO, Update this user_final_schema.ConvertToInternal(); serialization_.UpdateStore(user_final_schema); env::assert_fatal(serialization_.StoreToFile(), @@ -200,25 +205,25 @@ void CustomGenerator::GenerateTask() { template void CustomGenerator::TaskRunner(const std::string &id) { const auto ¤t_info = user_.rels_map.at(id); - if constexpr (!run) { + bool rerun = false; + if constexpr (run) { + rerun = true; + } else { const auto &previous_info = serialization_.GetLoad().internal_rels_map.at(id); - bool rerun = - internal::CheckPaths(previous_info.internal_inputs, - current_info.internal_inputs) != - internal::PathState::kNoChange || - internal::CheckChanged(previous_info.outputs, current_info.outputs); - - // run && rerun == false, skip running the task - if (!rerun) { - return; - } + rerun = internal::CheckPaths(previous_info.internal_inputs, + current_info.internal_inputs) != + internal::PathState::kNoChange || + internal::CheckChanged(previous_info.outputs, current_info.outputs); + } + + if (rerun) { + buildcc::CustomGeneratorContext ctx(command_, current_info.inputs, + current_info.outputs); + bool success = current_info.generate_cb(ctx); + env::assert_fatal(success, "Generate Cb failed for id {}"); } - buildcc::CustomGeneratorContext ctx(command_, current_info.inputs, - current_info.outputs); - bool success = current_info.generate_cb(ctx); - env::assert_fatal(success, "Generate Cb failed for id {}"); std::lock_guard guard(success_schema_mutex_); success_schema_.emplace(id, current_info); } From 7f92800c1c2e16b7512f8348531b151593607411 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 23:01:40 -0700 Subject: [PATCH 38/51] Fix recheck_states --- .../lib/target/mock/custom_generator/recheck_states.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/mock/custom_generator/recheck_states.cpp b/buildcc/lib/target/mock/custom_generator/recheck_states.cpp index b9a48780..c8c6471c 100644 --- a/buildcc/lib/target/mock/custom_generator/recheck_states.cpp +++ b/buildcc/lib/target/mock/custom_generator/recheck_states.cpp @@ -25,13 +25,16 @@ void CustomGenerator::IdUpdated() { namespace m { -void GeneratorExpect_IdRemoved(unsigned int calls, CustomGenerator *generator) { +void CustomGeneratorExpect_IdRemoved(unsigned int calls, + CustomGenerator *generator) { mock().expectNCalls(calls, ID_REMOVED_FUNCTION).onObject(generator); } -void GeneratorExpect_IdAdded(unsigned int calls, CustomGenerator *generator) { +void CustomGeneratorExpect_IdAdded(unsigned int calls, + CustomGenerator *generator) { mock().expectNCalls(calls, ID_ADDED_FUNCTION).onObject(generator); } -void GeneratorExpect_IdUpdated(unsigned int calls, CustomGenerator *generator) { +void CustomGeneratorExpect_IdUpdated(unsigned int calls, + CustomGenerator *generator) { mock().expectNCalls(calls, ID_UPDATED_FUNCTION).onObject(generator); } From 9a40fb4c83cf7d6f826bf6b02a522f9a35c85747 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Sun, 8 May 2022 23:15:09 -0700 Subject: [PATCH 39/51] Updated with map remove condition --- .../test/target/test_custom_generator.cpp | 127 +++++++++++++++++- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 7b5f951a..94ef945a 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -160,7 +160,7 @@ static bool Dep2Cb(const buildcc::CustomGeneratorContext &ctx) { } static void DependencyCb(std::unordered_map &task_map) { - task_map["id1"].precede(task_map["id2"]); + task_map.at("id1").precede(task_map.at("id2")); } TEST(CustomGeneratorTestGroup, AddDependency_BasicCheck) { @@ -230,8 +230,8 @@ static bool RealGenerateCb(const buildcc::CustomGeneratorContext &ctx) { return buildcc::env::Command::Execute(""); } -TEST(CustomGeneratorTestGroup, RealGenerate) { - constexpr const char *const kGenName = "real_generator"; +TEST(CustomGeneratorTestGroup, RealGenerate_Basic) { + constexpr const char *const kGenName = "real_generator_basic"; { buildcc::CustomGenerator cgen(kGenName, ""); cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, @@ -281,6 +281,127 @@ TEST(CustomGeneratorTestGroup, RealGenerate) { buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); } +TEST(CustomGeneratorTestGroup, RealGenerate_RemoveAndAdd) { + constexpr const char *const kGenName = "real_generator_remove_and_add"; + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + } + + // Same, no change + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + } + + // Map Removed + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + + cgen.Build(); + + buildcc::m::CustomGeneratorExpect_IdRemoved(1, &cgen); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + } + + // // Map Added Failure + // { + // buildcc::CustomGenerator cgen(kGenName, ""); + // cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + // cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + // cgen.Build(); + + // buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); + // mock().expectOneCall("RealGenerateCb"); + // buildcc::env::m::CommandExpect_Execute(1, false); + // buildcc::m::CustomGeneratorRunner(cgen); + + // buildcc::internal::CustomGeneratorSerialization serialization( + // cgen.GetBinaryPath()); + // CHECK_TRUE(serialization.LoadFromFile()); + // CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + // auto imap = serialization.GetLoad().internal_rels_map; + // CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + // CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + // } + + // // Map Added Success + // { + // buildcc::CustomGenerator cgen(kGenName, ""); + // cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + // cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + // cgen.Build(); + + // buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); + // mock().expectOneCall("RealGenerateCb"); + // buildcc::env::m::CommandExpect_Execute(1, true); + // buildcc::m::CustomGeneratorRunner(cgen); + + // buildcc::internal::CustomGeneratorSerialization serialization( + // cgen.GetBinaryPath()); + // CHECK_TRUE(serialization.LoadFromFile()); + // CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + // auto imap = serialization.GetLoad().internal_rels_map; + // CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + // CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + + // CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + // CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + // } +} + int main(int ac, char **av) { fs::remove_all(BUILD_DIR); buildcc::Project::Init(fs::current_path() / "data", BUILD_DIR); From fdc4a4583ebeba64f0f56e315b524f7b09091989 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 9 May 2022 00:00:37 -0700 Subject: [PATCH 40/51] Updated Map Add condition --- .../test/target/test_custom_generator.cpp | 100 +++++++++--------- 1 file changed, 52 insertions(+), 48 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 94ef945a..357080f4 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -352,54 +352,58 @@ TEST(CustomGeneratorTestGroup, RealGenerate_RemoveAndAdd) { CHECK_EQUAL(imap.at("id1").outputs.size(), 1); } - // // Map Added Failure - // { - // buildcc::CustomGenerator cgen(kGenName, ""); - // cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, - // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); - // cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, - // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); - // cgen.Build(); - - // buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); - // mock().expectOneCall("RealGenerateCb"); - // buildcc::env::m::CommandExpect_Execute(1, false); - // buildcc::m::CustomGeneratorRunner(cgen); - - // buildcc::internal::CustomGeneratorSerialization serialization( - // cgen.GetBinaryPath()); - // CHECK_TRUE(serialization.LoadFromFile()); - // CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); - // auto imap = serialization.GetLoad().internal_rels_map; - // CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - // CHECK_EQUAL(imap.at("id1").outputs.size(), 1); - // } - - // // Map Added Success - // { - // buildcc::CustomGenerator cgen(kGenName, ""); - // cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, - // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); - // cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, - // {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); - // cgen.Build(); - - // buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); - // mock().expectOneCall("RealGenerateCb"); - // buildcc::env::m::CommandExpect_Execute(1, true); - // buildcc::m::CustomGeneratorRunner(cgen); - - // buildcc::internal::CustomGeneratorSerialization serialization( - // cgen.GetBinaryPath()); - // CHECK_TRUE(serialization.LoadFromFile()); - // CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); - // auto imap = serialization.GetLoad().internal_rels_map; - // CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); - // CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); - - // CHECK_EQUAL(imap.at("id1").outputs.size(), 1); - // CHECK_EQUAL(imap.at("id2").outputs.size(), 1); - // } + // Map Added Failure + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, false); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_THROWS(std::out_of_range, imap.at("id2")); + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + + // Map Added Success + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_root_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_root_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + } } int main(int ac, char **av) { From 9c27129628e86bd78edbb598f7ce1da27dd6e778 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 9 May 2022 00:01:21 -0700 Subject: [PATCH 41/51] Updated custom_generator --- buildcc/lib/target/src/custom_generator/custom_generator.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 1809b036..f89ae1dc 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -221,7 +221,7 @@ template void CustomGenerator::TaskRunner(const std::string &id) { buildcc::CustomGeneratorContext ctx(command_, current_info.inputs, current_info.outputs); bool success = current_info.generate_cb(ctx); - env::assert_fatal(success, "Generate Cb failed for id {}"); + env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id)); } std::lock_guard guard(success_schema_mutex_); From ba36b0f57e91e3b71bf92e616eb830a34c6cf916 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 9 May 2022 02:19:59 -0700 Subject: [PATCH 42/51] Updated unit tests for update example --- .../target/include/target/custom_generator.h | 2 + .../src/custom_generator/custom_generator.cpp | 2 +- .../test/target/test_custom_generator.cpp | 147 ++++++++++++++++++ 3 files changed, 150 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index e028b44a..6c54891f 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -132,6 +132,8 @@ class CustomGenerator : public internal::BuilderInterface { const fs::path &GetBinaryPath() const { return serialization_.GetSerializedFile(); } + const fs::path &GetRootDir() const { return env_.GetTargetRootDir(); } + const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); } tf::Taskflow &GetTaskflow() { return tf_; } private: diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index f89ae1dc..67e99004 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -188,7 +188,6 @@ void CustomGenerator::GenerateTask() { user_final_schema.rels_map.insert(success_schema_.begin(), success_schema_.end()); - // TODO, Update this user_final_schema.ConvertToInternal(); serialization_.UpdateStore(user_final_schema); env::assert_fatal(serialization_.StoreToFile(), @@ -218,6 +217,7 @@ template void CustomGenerator::TaskRunner(const std::string &id) { } if (rerun) { + dirty_ = true; buildcc::CustomGeneratorContext ctx(command_, current_info.inputs, current_info.outputs); bool success = current_info.generate_cb(ctx); diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 357080f4..47e64864 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -2,6 +2,7 @@ #include "expect_command.h" #include "expect_custom_generator.h" +#include "test_target_util.h" // #include "test_target_util.h" // #include "taskflow/taskflow.hpp" @@ -350,6 +351,8 @@ TEST(CustomGeneratorTestGroup, RealGenerate_RemoveAndAdd) { auto imap = serialization.GetLoad().internal_rels_map; CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + + CHECK_THROWS(std::out_of_range, imap.at("id2")); } // Map Added Failure @@ -406,6 +409,150 @@ TEST(CustomGeneratorTestGroup, RealGenerate_RemoveAndAdd) { } } +TEST(CustomGeneratorTestGroup, RealGenerate_Update_Failure) { + constexpr const char *const kGenName = "real_generator_update_failure"; + + { + buildcc::CustomGenerator cgen(kGenName, ""); + buildcc::env::save_file( + (cgen.GetBuildDir() / "dummy_main.c").string().c_str(), "", false); + buildcc::env::save_file( + (cgen.GetBuildDir() / "dummy_main.cpp").string().c_str(), "", false); + + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/other_dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + } + + buildcc::m::blocking_sleep(1); + + // Updated Input file Failure + UT_PRINT("Updated Input file: Failure\r\n"); + { + buildcc::CustomGenerator cgen(kGenName, ""); + buildcc::env::save_file( + (cgen.GetBuildDir() / "dummy_main.cpp").string().c_str(), "", false); + + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/other_dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, false); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 1); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); +} + +TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { + constexpr const char *const kGenName = "real_generator_update_success"; + + { + buildcc::CustomGenerator cgen(kGenName, ""); + buildcc::env::save_file( + (cgen.GetBuildDir() / "dummy_main.c").string().c_str(), "", false); + buildcc::env::save_file( + (cgen.GetBuildDir() / "dummy_main.cpp").string().c_str(), "", false); + + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/other_dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + } + + buildcc::m::blocking_sleep(1); + + // Updated Input file Success + UT_PRINT("Updated Input file: Success\r\n"); + { + buildcc::CustomGenerator cgen(kGenName, ""); + buildcc::env::save_file( + (cgen.GetBuildDir() / "dummy_main.cpp").string().c_str(), "", false); + + std::uint64_t last_write_timestamp = static_cast( + fs::last_write_time(cgen.GetBuildDir() / "dummy_main.cpp") + .time_since_epoch() + .count()); + + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/other_dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + + CHECK_EQUAL( + last_write_timestamp, + imap.at("id2").internal_inputs.begin()->GetLastWriteTimestamp()); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); + } +} + int main(int ac, char **av) { fs::remove_all(BUILD_DIR); buildcc::Project::Init(fs::current_path() / "data", BUILD_DIR); From e3554f9da6852fc2b012a6a65f7ce2859ad548db Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 9 May 2022 02:23:11 -0700 Subject: [PATCH 43/51] Updated test_custom_generator for output changed condition --- .../test/target/test_custom_generator.cpp | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 47e64864..02f55b84 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -551,6 +551,35 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); } + + // Updated Output file Success + UT_PRINT("Updated Output file: Success\r\n"); + { + buildcc::CustomGenerator cgen(kGenName, ""); + + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, RealGenerateCb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.cpp"}, + {"{gen_build_dir}/rename_dummy_main.o"}, RealGenerateCb); + cgen.Build(); + + mock().expectOneCall("RealGenerateCb"); + buildcc::env::m::CommandExpect_Execute(1, true); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + auto imap = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(imap.at("id1").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id1").outputs.size(), 1); + + CHECK_EQUAL(imap.at("id2").internal_inputs.size(), 1); + CHECK_EQUAL(imap.at("id2").outputs.size(), 1); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); + } } int main(int ac, char **av) { From e0abbdbb5f73fc951662b2cd143acef308f98a73 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 9 May 2022 03:05:58 -0700 Subject: [PATCH 44/51] Updated custom_generator with try/catch for dependency callback Updated unit tests for dependency callback --- .../src/custom_generator/custom_generator.cpp | 7 +- .../test/target/test_custom_generator.cpp | 132 ++++++++++++++++++ 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 67e99004..57134155 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -176,7 +176,12 @@ void CustomGenerator::GenerateTask() { // Dependencies between ids if (dependency_cb_) { - dependency_cb_(task_map); + try { + dependency_cb_(task_map); + } catch (...) { + env::log_critical(__FUNCTION__, "Dependency callback failed"); + env::set_task_state(env::TaskState::FAILURE); + } } // NOTE, Do not call detach otherwise this will fail diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 02f55b84..5a4e46a4 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -225,6 +225,136 @@ TEST(CustomGeneratorTestGroup, AddDependency_FileDep) { } } +TEST(CustomGeneratorTestGroup, AddDependency_FileDep_WithRebuild) { + constexpr const char *const kGenName = "add_dependency_file_dep_with_rebuild"; + + fs::path kInputFile = + (BUILD_DIR / kGenName / "dummy_main.c").make_preferred(); + UT_PRINT(kInputFile.string().c_str()); + fs::create_directories(BUILD_DIR / kGenName); + CHECK_TRUE(buildcc::env::save_file(kInputFile.string().c_str(), "", false)); + + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, FileDep1Cb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.o"}, {}, FileDep2Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + mock().expectOneCall("FileDep1Cb"); + mock().expectOneCall("FileDep2Cb"); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); + } + + // Same, no change + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, FileDep1Cb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.o"}, {}, FileDep2Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); + } + + // reset + fs::remove_all(BUILD_DIR / kGenName / "dummy_main.o"); + + // Remove id1, should cause id2 to fail + // NOTE, dirty_ == false is not made true when id2 is run, however id removed + // sets dirty_ == true + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.o"}, {}, FileDep2Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + buildcc::m::CustomGeneratorExpect_IdRemoved(1, &cgen); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 0); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); + } + + // reset + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + fs::remove_all(BUILD_DIR / kGenName / "dummy_main.o"); + + // Added + { + buildcc::CustomGenerator cgen(kGenName, ""); + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, FileDep1Cb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.o"}, {}, FileDep2Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); + buildcc::m::CustomGeneratorExpect_IdAdded(1, &cgen); + mock().expectOneCall("FileDep1Cb"); + mock().expectOneCall("FileDep2Cb"); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); + } + + // reset + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); + fs::remove_all(BUILD_DIR / kGenName / "dummy_main.o"); + + buildcc::m::blocking_sleep(1); + buildcc::env::save_file(kInputFile.string().c_str(), "", false); + + // Update id1:dummy_main.c -> updated dummy_main.o -> should rerun id2 as well + { + buildcc::CustomGenerator cgen(kGenName, ""); + + cgen.AddGenInfo("id1", {"{gen_build_dir}/dummy_main.c"}, + {"{gen_build_dir}/dummy_main.o"}, FileDep1Cb); + cgen.AddGenInfo("id2", {"{gen_build_dir}/dummy_main.o"}, {}, FileDep2Cb); + cgen.AddDependencyCb(DependencyCb); + cgen.Build(); + + mock().expectOneCall("FileDep1Cb"); + mock().expectOneCall("FileDep2Cb"); + buildcc::m::CustomGeneratorRunner(cgen); + + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + CHECK_EQUAL(serialization.GetLoad().internal_rels_map.size(), 2); + + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::SUCCESS); + } + + buildcc::env::set_task_state(buildcc::env::TaskState::SUCCESS); +} + static bool RealGenerateCb(const buildcc::CustomGeneratorContext &ctx) { (void)ctx; mock().actualCall("RealGenerateCb"); @@ -582,6 +712,8 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { } } +// TODO, Add default arguments + int main(int ac, char **av) { fs::remove_all(BUILD_DIR); buildcc::Project::Init(fs::current_path() / "data", BUILD_DIR); From 8cadce6386cfe199aaca81ae42849419af1d1a95 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Mon, 9 May 2022 03:22:08 -0700 Subject: [PATCH 45/51] Removed parallel from custom_generator --- buildcc/lib/target/include/target/custom_generator.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/buildcc/lib/target/include/target/custom_generator.h b/buildcc/lib/target/include/target/custom_generator.h index 6c54891f..8c10ee28 100644 --- a/buildcc/lib/target/include/target/custom_generator.h +++ b/buildcc/lib/target/include/target/custom_generator.h @@ -77,12 +77,10 @@ struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema { class CustomGenerator : public internal::BuilderInterface { public: - CustomGenerator(const std::string &name, const TargetEnv &env, - bool parallel = false) + CustomGenerator(const std::string &name, const TargetEnv &env) : name_(name), env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name), - serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)), - parallel_(parallel) { + serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)) { Initialize(); } virtual ~CustomGenerator() = default; @@ -156,7 +154,6 @@ class CustomGenerator : public internal::BuilderInterface { std::string name_; TargetEnv env_; internal::CustomGeneratorSerialization serialization_; - bool parallel_; // Serialization UserCustomGeneratorSchema user_; From 31e00e420585d1f4f00b617367dea168f5897720 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 11 May 2022 00:32:15 -0700 Subject: [PATCH 46/51] Updated tests for default argument usage --- .../test/target/test_custom_generator.cpp | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 5a4e46a4..0932e2a7 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -96,6 +96,40 @@ TEST(CustomGeneratorTestGroup, Basic_Failure) { } } +TEST(CustomGeneratorTestGroup, DefaultArgumentUsage) { + buildcc::CustomGenerator cgen("default_argument_usage", ""); + cgen.AddDefaultArguments({ + {"dummy_main_c", "{gen_root_dir}/dummy_main.c"}, + {"dummy_main_o", "{gen_build_dir}/dummy_main.o"}, + {"dummy_main_cpp", "{gen_root_dir}/dummy_main.cpp"}, + }); + cgen.AddGenInfo("id1", {"{dummy_main_c}"}, {"{dummy_main_o}"}, + BasicGenerateCb); + cgen.AddGenInfo("id2", {"{dummy_main_cpp}"}, {}, BasicGenerateCb); + cgen.Build(); + + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + mock().expectOneCall("BasicGenerateCb").andReturnValue(true); + buildcc::m::CustomGeneratorRunner(cgen); + + // Serialization check + { + buildcc::internal::CustomGeneratorSerialization serialization( + cgen.GetBinaryPath()); + CHECK_TRUE(serialization.LoadFromFile()); + + const auto &internal_map = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(internal_map.size(), 2); + const auto &id1_info = internal_map.at("id1"); + CHECK_EQUAL(id1_info.internal_inputs.size(), 1); + CHECK_EQUAL(id1_info.outputs.size(), 1); + + const auto &id2_info = internal_map.at("id2"); + CHECK_EQUAL(id2_info.internal_inputs.size(), 1); + CHECK_EQUAL(id2_info.outputs.size(), 0); + } +} + TEST(CustomGeneratorTestGroup, FailureCases) { { buildcc::CustomGenerator cgen("failure_no_cb", ""); @@ -712,8 +746,6 @@ TEST(CustomGeneratorTestGroup, RealGenerate_Update_Success) { } } -// TODO, Add default arguments - int main(int ac, char **av) { fs::remove_all(BUILD_DIR); buildcc::Project::Init(fs::current_path() / "data", BUILD_DIR); From 1c4cfa10c593ee0910a83e17fad18caf28068ef3 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 11 May 2022 00:32:27 -0700 Subject: [PATCH 47/51] Updated AddDefaultArgument usage --- buildcc/lib/target/src/custom_generator/custom_generator.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/buildcc/lib/target/src/custom_generator/custom_generator.cpp b/buildcc/lib/target/src/custom_generator/custom_generator.cpp index 57134155..8d980c3f 100644 --- a/buildcc/lib/target/src/custom_generator/custom_generator.cpp +++ b/buildcc/lib/target/src/custom_generator/custom_generator.cpp @@ -29,7 +29,10 @@ namespace buildcc { void CustomGenerator::AddDefaultArguments( const std::unordered_map &arguments) { - command_.AddDefaultArguments(arguments); + for (const auto &arg_iter : arguments) { + command_.AddDefaultArgument(arg_iter.first, + command_.Construct(arg_iter.second)); + } } void CustomGenerator::AddGenInfo(const std::string &id, From 692e2e4f6629c09e3d061fc955a0856de613ad96 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 11 May 2022 00:35:25 -0700 Subject: [PATCH 48/51] Minor update to CMakeLists.txt for toolchains --- buildcc/toolchains/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildcc/toolchains/CMakeLists.txt b/buildcc/toolchains/CMakeLists.txt index 4bb1a8c2..b1380fcd 100644 --- a/buildcc/toolchains/CMakeLists.txt +++ b/buildcc/toolchains/CMakeLists.txt @@ -37,7 +37,7 @@ if (${TESTING}) ${TEST_LINK_LIBS} ) - add_executable(test_toolchain_specialized test/test_toolchain_specialized) + add_executable(test_toolchain_specialized test/test_toolchain_specialized.cpp) target_link_libraries(test_toolchain_specialized PRIVATE mock_toolchain_specialized) add_test(NAME test_toolchain_specialized COMMAND test_toolchain_specialized) From c669365a7f97de9b89e26cea8ee2b76780acdf8a Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 11 May 2022 02:01:20 -0700 Subject: [PATCH 49/51] Updated bootstrap --- bootstrap/src/build_buildcc.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap/src/build_buildcc.cpp b/bootstrap/src/build_buildcc.cpp index e655a418..cae0ad44 100644 --- a/bootstrap/src/build_buildcc.cpp +++ b/bootstrap/src/build_buildcc.cpp @@ -20,10 +20,13 @@ namespace buildcc { void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe) { generator.AddInput("{gen_root_dir}/path.fbs", "path_fbs"); + generator.AddInput("{gen_root_dir}/custom_generator.fbs", + "custom_generator_fbs"); generator.AddInput("{gen_root_dir}/generator.fbs", "generator_fbs"); generator.AddInput("{gen_root_dir}/target.fbs", "target_fbs"); generator.AddOutput("{gen_build_dir}/path_generated.h"); + generator.AddOutput("{gen_build_dir}/custom_generator_generated.h"); generator.AddOutput("{gen_build_dir}/generator_generated.h"); generator.AddOutput("{gen_build_dir}/target_generated.h"); @@ -33,7 +36,7 @@ void schema_gen_cb(BaseGenerator &generator, const BaseTarget &flatc_exe) { // generator.AddCommand("{flatc_compiler} --help"); generator.AddCommand( "{flatc_compiler} -o {gen_build_dir} -I {gen_root_dir} --gen-object-api " - "--cpp {path_fbs} {generator_fbs} {target_fbs}"); + "--cpp {path_fbs} {custom_generator_fbs} {generator_fbs} {target_fbs}"); generator.Build(); } @@ -71,6 +74,7 @@ void buildcc_cb(BaseTarget &target, const BaseGenerator &schema_gen, // TARGET target.GlobSources("lib/target/src/common"); + target.GlobSources("lib/target/src/custom_generator"); target.GlobSources("lib/target/src/generator"); target.GlobSources("lib/target/src/api"); target.GlobSources("lib/target/src/target_info"); From 225eabd30844003dd491d76b44ba5e398990550d Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 11 May 2022 02:27:56 -0700 Subject: [PATCH 50/51] Removed plugin test --- buildcc/plugins/CMakeLists.txt | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/buildcc/plugins/CMakeLists.txt b/buildcc/plugins/CMakeLists.txt index 0518831b..42e4700d 100644 --- a/buildcc/plugins/CMakeLists.txt +++ b/buildcc/plugins/CMakeLists.txt @@ -21,16 +21,17 @@ target_link_libraries(mock_plugins PUBLIC ) # Tests -add_executable(test_buildcc_find - test/test_buildcc_find.cpp -) -target_link_libraries(test_buildcc_find PRIVATE - mock_plugins -) +# Removed test_buildcc_find till it is not complete +# add_executable(test_buildcc_find +# test/test_buildcc_find.cpp +# ) +# target_link_libraries(test_buildcc_find PRIVATE +# mock_plugins +# ) -add_test(NAME test_buildcc_find COMMAND test_buildcc_find - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test -) +# add_test(NAME test_buildcc_find COMMAND test_buildcc_find +# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test +# ) endif() From 4031c397e21c97c58bb8c247ae54f6df869cbe15 Mon Sep 17 00:00:00 2001 From: Niket Naidu Date: Wed, 11 May 2022 03:15:25 -0700 Subject: [PATCH 51/51] Update test_custom_generator.cpp --- .../test/target/test_custom_generator.cpp | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/buildcc/lib/target/test/target/test_custom_generator.cpp b/buildcc/lib/target/test/target/test_custom_generator.cpp index 0932e2a7..27def93c 100644 --- a/buildcc/lib/target/test/target/test_custom_generator.cpp +++ b/buildcc/lib/target/test/target/test_custom_generator.cpp @@ -74,26 +74,15 @@ TEST(CustomGeneratorTestGroup, Basic_Failure) { mock().expectOneCall("BasicGenerateCb").andReturnValue(false); buildcc::m::CustomGeneratorRunner(cgen); + CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); + // Load buildcc::internal::CustomGeneratorSerialization serialization( cgen.GetBinaryPath()); CHECK_TRUE(serialization.LoadFromFile()); - const auto &id1_info = serialization.GetLoad().internal_rels_map.at("id1"); - CHECK_EQUAL(id1_info.internal_inputs.size(), 1); - CHECK_EQUAL(id1_info.outputs.size(), 0); - - CHECK(buildcc::env::get_task_state() == buildcc::env::TaskState::FAILURE); - - // Serialization check - { - buildcc::internal::CustomGeneratorSerialization serialization( - cgen.GetBinaryPath()); - CHECK_TRUE(serialization.LoadFromFile()); - - const auto &internal_map = serialization.GetLoad().internal_rels_map; - CHECK_EQUAL(internal_map.size(), 1); - } + const auto &internal_map = serialization.GetLoad().internal_rels_map; + CHECK_EQUAL(internal_map.size(), 1); } TEST(CustomGeneratorTestGroup, DefaultArgumentUsage) {