Skip to content

Commit d261a33

Browse files
coder137coder137
authored
[Enhancement] Path Serialization (#228)
1 parent 59f43d5 commit d261a33

File tree

8 files changed

+437
-254
lines changed

8 files changed

+437
-254
lines changed

‎buildcc/lib/target/include/target/custom_generator.h

Lines changed: 9 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -39,38 +39,34 @@ namespace buildcc {
3939

4040
struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
4141
struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo {
42-
fs_unordered_set inputs; // TODO, Remove
43-
GenerateCb generate_cb;
44-
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
45-
4642
void ConvertToInternal() {
47-
internal_inputs = internal::path_schema_convert(
48-
inputs, internal::Path::CreateExistingPath);
43+
inputs.ComputeHashForAll();
4944
userblob = blob_handler != nullptr ? blob_handler->GetSerializedData()
5045
: std::vector<uint8_t>();
5146
}
52-
};
5347

54-
using UserIdPair = std::pair<const IdKey, UserIdInfo>;
55-
std::unordered_map<IdKey, UserIdInfo> ids;
48+
GenerateCb generate_cb;
49+
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
50+
};
5651

5752
void ConvertToInternal() {
5853
for (auto &[id_key, id_info] : ids) {
59-
id_info.internal_inputs = path_schema_convert(
60-
id_info.inputs, internal::Path::CreateExistingPath);
54+
id_info.ConvertToInternal();
6155
auto [_, success] = internal_ids.try_emplace(id_key, id_info);
6256
env::assert_fatal(success, fmt::format("Could not save {}", id_key));
6357
}
6458
}
59+
60+
std::unordered_map<IdKey, UserIdInfo> ids;
6561
};
6662

6763
class CustomGenerator : public internal::BuilderInterface {
6864
public:
6965
CustomGenerator(const std::string &name, const TargetEnv &env)
7066
: name_(name),
7167
env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name),
72-
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)),
73-
comparator_(serialization_.GetLoad(), user_) {
68+
serialization_(env_.GetTargetBuildDir() /
69+
fmt::format("{}.json", name)) {
7470
Initialize();
7571
}
7672
virtual ~CustomGenerator() = default;
@@ -115,100 +111,9 @@ class CustomGenerator : public internal::BuilderInterface {
115111
const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); }
116112
const std::string &Get(const std::string &file_identifier) const;
117113

118-
private:
119-
struct Comparator {
120-
Comparator(const internal::CustomGeneratorSchema &loaded,
121-
const UserCustomGeneratorSchema &us)
122-
: loaded_schema_(loaded), current_schema_(us) {}
123-
124-
enum class State {
125-
kRemoved,
126-
kAdded,
127-
kCheckLater,
128-
};
129-
130-
void AddAllIds() {
131-
const auto &curr_ids = current_schema_.ids;
132-
for (const auto &[id, _] : curr_ids) {
133-
id_state_info_.at(State::kAdded).insert(id);
134-
}
135-
}
136-
137-
void CompareIds() {
138-
const auto &prev_ids = loaded_schema_.internal_ids;
139-
const auto &curr_ids = current_schema_.ids;
140-
141-
for (const auto &[prev_id, _] : prev_ids) {
142-
if (curr_ids.find(prev_id) == curr_ids.end()) {
143-
// Id Removed condition, previous id is not present in the current run
144-
id_state_info_.at(State::kRemoved).insert(prev_id);
145-
}
146-
}
147-
148-
for (const auto &[curr_id, _] : curr_ids) {
149-
if (prev_ids.find(curr_id) == prev_ids.end()) {
150-
// Id Added condition
151-
id_state_info_.at(State::kAdded).insert(curr_id);
152-
} else {
153-
// Id Check Later condition
154-
id_state_info_.at(State::kCheckLater).insert(curr_id);
155-
}
156-
}
157-
}
158-
159-
bool IsChanged(const std::string &id) const {
160-
const auto &previous_id_info = loaded_schema_.internal_ids.at(id);
161-
const auto &current_id_info = current_schema_.ids.at(id);
162-
163-
bool changed = internal::CheckPaths(previous_id_info.internal_inputs,
164-
current_id_info.internal_inputs) !=
165-
internal::PathState::kNoChange;
166-
changed = changed || internal::CheckChanged(previous_id_info.outputs,
167-
current_id_info.outputs);
168-
if (!changed && current_id_info.blob_handler != nullptr) {
169-
// We only check blob handler if not changed by inputs/outputs
170-
// Checking blob_handler could be expensive so this optimization is made
171-
// to run only when changed == false
172-
changed = current_id_info.blob_handler->CheckChanged(
173-
previous_id_info.userblob, current_id_info.userblob);
174-
}
175-
return changed;
176-
}
177-
178-
const std::unordered_set<std::string> &GetRemovedIds() const {
179-
return id_state_info_.at(State::kRemoved);
180-
}
181-
182-
const std::unordered_set<std::string> &GetAddedIds() const {
183-
return id_state_info_.at(State::kAdded);
184-
}
185-
186-
const std::unordered_set<std::string> &GetCheckLaterIds() const {
187-
return id_state_info_.at(State::kCheckLater);
188-
}
189-
190-
bool IsIdAdded(const std::string &id) const {
191-
return id_state_info_.at(State::kAdded).count(id) == 1;
192-
}
193-
194-
private:
195-
const internal::CustomGeneratorSchema &loaded_schema_;
196-
const UserCustomGeneratorSchema &current_schema_;
197-
std::unordered_map<State, std::unordered_set<std::string>> id_state_info_{
198-
{State::kRemoved, std::unordered_set<std::string>()},
199-
{State::kAdded, std::unordered_set<std::string>()},
200-
{State::kCheckLater, std::unordered_set<std::string>()},
201-
};
202-
};
203-
204114
private:
205115
void Initialize();
206-
207-
tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id);
208-
void TaskRunner(const std::string &id);
209-
210116
void GenerateTask();
211-
void BuildGenerate();
212117

213118
// Recheck states
214119
void IdRemoved();
@@ -227,13 +132,6 @@ class CustomGenerator : public internal::BuilderInterface {
227132
// Serialization
228133
UserCustomGeneratorSchema user_;
229134

230-
// Comparator
231-
Comparator comparator_;
232-
233-
std::mutex success_schema_mutex_;
234-
std::unordered_map<std::string, UserCustomGeneratorSchema::UserIdInfo>
235-
success_schema_;
236-
237135
// Internal
238136
env::Command command_;
239137
};

‎buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323

2424
namespace buildcc {
2525

26+
/**
27+
* @brief Abstract class for serializing additional data for which rebuilds
28+
* might be triggered i.e data that is not input/output files
29+
* TODO, Add examples here
30+
*
31+
*/
2632
class CustomBlobHandler {
2733
public:
2834
CustomBlobHandler() = default;
@@ -54,6 +60,48 @@ class CustomBlobHandler {
5460
virtual std::vector<uint8_t> Serialize() const = 0;
5561
};
5662

63+
/**
64+
* @brief Typed Custom Blob handler which automatically performs Serialization
65+
* and Deserialization as long as it is JSON serializable
66+
*
67+
* NOTE: Type data is stored as a reference (to avoid copying large amount of
68+
* data) when constructing TypedCustomBlobHandler
69+
*
70+
* @tparam Type should be JSON serializable (see nlohmann::json compatible
71+
* objects)
72+
*/
73+
template <typename Type>
74+
class TypedCustomBlobHandler : public CustomBlobHandler {
75+
public:
76+
explicit TypedCustomBlobHandler(const Type &data) : data_(data) {}
77+
78+
// serialized_data has already been verified
79+
static Type Deserialize(const std::vector<uint8_t> &serialized_data) {
80+
json j = json::from_msgpack(serialized_data, true, false);
81+
Type deserialized;
82+
j.get_to(deserialized);
83+
return deserialized;
84+
}
85+
86+
private:
87+
const Type &data_;
88+
89+
bool Verify(const std::vector<uint8_t> &serialized_data) const override {
90+
json j = json::from_msgpack(serialized_data, true, false);
91+
return !j.is_discarded();
92+
}
93+
94+
bool IsEqual(const std::vector<uint8_t> &previous,
95+
const std::vector<uint8_t> &current) const override {
96+
return Deserialize(previous) == Deserialize(current);
97+
}
98+
99+
std::vector<uint8_t> Serialize() const override {
100+
json j = data_;
101+
return json::to_msgpack(j);
102+
}
103+
};
104+
57105
} // namespace buildcc
58106

59107
#endif

‎buildcc/lib/target/include/target/custom_generator/custom_generator_context.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,20 @@ namespace buildcc {
2525

2626
class CustomGeneratorContext {
2727
public:
28-
CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i,
29-
const fs_unordered_set &o,
28+
CustomGeneratorContext(const env::Command &c,
29+
const std::unordered_set<std::string> &i,
30+
const std::unordered_set<std::string> &o,
3031
const std::vector<uint8_t> &ub)
3132
: command(c), inputs(i), outputs(o), userblob(ub) {}
3233

3334
const env::Command &command;
34-
const fs_unordered_set &inputs;
35-
const fs_unordered_set &outputs;
35+
const std::unordered_set<std::string> &inputs;
36+
const std::unordered_set<std::string> &outputs;
3637
const std::vector<uint8_t> &userblob;
3738
};
3839

3940
// clang-format off
40-
using GenerateCb = std::function<bool (CustomGeneratorContext &)>;
41+
using GenerateCb = std::function<bool (const CustomGeneratorContext &)>;
4142
// clang-format on
4243

4344
} // namespace buildcc

0 commit comments

Comments
 (0)