Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitd261a33

Browse files
authored
[Enhancement] Path Serialization (#228)
1 parent59f43d5 commitd261a33

File tree

8 files changed

+437
-254
lines changed

8 files changed

+437
-254
lines changed

‎buildcc/lib/target/include/target/custom_generator.h‎

Lines changed: 9 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -39,38 +39,34 @@ namespace buildcc {
3939

4040
structUserCustomGeneratorSchema :publicinternal::CustomGeneratorSchema {
4141
structUserIdInfo : internal::CustomGeneratorSchema::IdInfo {
42-
fs_unordered_set inputs;// TODO, Remove
43-
GenerateCb generate_cb;
44-
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
45-
4642
voidConvertToInternal() {
47-
internal_inputs =internal::path_schema_convert(
48-
inputs, internal::Path::CreateExistingPath);
43+
inputs.ComputeHashForAll();
4944
userblob = blob_handler !=nullptr ? blob_handler->GetSerializedData()
5045
: std::vector<uint8_t>();
5146
}
52-
};
5347

54-
using UserIdPair = std::pair<const IdKey, UserIdInfo>;
55-
std::unordered_map<IdKey, UserIdInfo> ids;
48+
GenerateCb generate_cb;
49+
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
50+
};
5651

5752
voidConvertToInternal() {
5853
for (auto &[id_key, id_info] : ids) {
59-
id_info.internal_inputs =path_schema_convert(
60-
id_info.inputs, internal::Path::CreateExistingPath);
54+
id_info.ConvertToInternal();
6155
auto [_, success] = internal_ids.try_emplace(id_key, id_info);
6256
env::assert_fatal(success,fmt::format("Could not save {}", id_key));
6357
}
6458
}
59+
60+
std::unordered_map<IdKey, UserIdInfo> ids;
6561
};
6662

6763
classCustomGenerator :publicinternal::BuilderInterface {
6864
public:
6965
CustomGenerator(const std::string &name,const TargetEnv &env)
7066
: name_(name),
7167
env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name),
72-
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)),
73-
comparator_(serialization_.GetLoad(), user_) {
68+
serialization_(env_.GetTargetBuildDir() /
69+
fmt::format("{}.json", name)) {
7470
Initialize();
7571
}
7672
virtual~CustomGenerator() =default;
@@ -115,100 +111,9 @@ class CustomGenerator : public internal::BuilderInterface {
115111
const fs::path &GetBuildDir()const {return env_.GetTargetBuildDir(); }
116112
const std::string &Get(const std::string &file_identifier)const;
117113

118-
private:
119-
structComparator {
120-
Comparator(const internal::CustomGeneratorSchema &loaded,
121-
const UserCustomGeneratorSchema &us)
122-
: loaded_schema_(loaded), current_schema_(us) {}
123-
124-
enumclassState {
125-
kRemoved,
126-
kAdded,
127-
kCheckLater,
128-
};
129-
130-
voidAddAllIds() {
131-
constauto &curr_ids = current_schema_.ids;
132-
for (constauto &[id, _] : curr_ids) {
133-
id_state_info_.at(State::kAdded).insert(id);
134-
}
135-
}
136-
137-
voidCompareIds() {
138-
constauto &prev_ids = loaded_schema_.internal_ids;
139-
constauto &curr_ids = current_schema_.ids;
140-
141-
for (constauto &[prev_id, _] : prev_ids) {
142-
if (curr_ids.find(prev_id) == curr_ids.end()) {
143-
// Id Removed condition, previous id is not present in the current run
144-
id_state_info_.at(State::kRemoved).insert(prev_id);
145-
}
146-
}
147-
148-
for (constauto &[curr_id, _] : curr_ids) {
149-
if (prev_ids.find(curr_id) == prev_ids.end()) {
150-
// Id Added condition
151-
id_state_info_.at(State::kAdded).insert(curr_id);
152-
}else {
153-
// Id Check Later condition
154-
id_state_info_.at(State::kCheckLater).insert(curr_id);
155-
}
156-
}
157-
}
158-
159-
boolIsChanged(const std::string &id)const {
160-
constauto &previous_id_info = loaded_schema_.internal_ids.at(id);
161-
constauto &current_id_info = current_schema_.ids.at(id);
162-
163-
bool changed =internal::CheckPaths(previous_id_info.internal_inputs,
164-
current_id_info.internal_inputs) !=
165-
internal::PathState::kNoChange;
166-
changed = changed ||internal::CheckChanged(previous_id_info.outputs,
167-
current_id_info.outputs);
168-
if (!changed && current_id_info.blob_handler !=nullptr) {
169-
// We only check blob handler if not changed by inputs/outputs
170-
// Checking blob_handler could be expensive so this optimization is made
171-
// to run only when changed == false
172-
changed = current_id_info.blob_handler->CheckChanged(
173-
previous_id_info.userblob, current_id_info.userblob);
174-
}
175-
return changed;
176-
}
177-
178-
const std::unordered_set<std::string> &GetRemovedIds()const {
179-
return id_state_info_.at(State::kRemoved);
180-
}
181-
182-
const std::unordered_set<std::string> &GetAddedIds()const {
183-
return id_state_info_.at(State::kAdded);
184-
}
185-
186-
const std::unordered_set<std::string> &GetCheckLaterIds()const {
187-
return id_state_info_.at(State::kCheckLater);
188-
}
189-
190-
boolIsIdAdded(const std::string &id)const {
191-
return id_state_info_.at(State::kAdded).count(id) ==1;
192-
}
193-
194-
private:
195-
const internal::CustomGeneratorSchema &loaded_schema_;
196-
const UserCustomGeneratorSchema &current_schema_;
197-
std::unordered_map<State, std::unordered_set<std::string>> id_state_info_{
198-
{State::kRemoved, std::unordered_set<std::string>()},
199-
{State::kAdded, std::unordered_set<std::string>()},
200-
{State::kCheckLater, std::unordered_set<std::string>()},
201-
};
202-
};
203-
204114
private:
205115
voidInitialize();
206-
207-
tf::TaskCreateTaskRunner(tf::Subflow &subflow,const std::string &id);
208-
voidTaskRunner(const std::string &id);
209-
210116
voidGenerateTask();
211-
voidBuildGenerate();
212117

213118
// Recheck states
214119
voidIdRemoved();
@@ -227,13 +132,6 @@ class CustomGenerator : public internal::BuilderInterface {
227132
// Serialization
228133
UserCustomGeneratorSchema user_;
229134

230-
// Comparator
231-
Comparator comparator_;
232-
233-
std::mutex success_schema_mutex_;
234-
std::unordered_map<std::string, UserCustomGeneratorSchema::UserIdInfo>
235-
success_schema_;
236-
237135
// Internal
238136
env::Command command_;
239137
};

‎buildcc/lib/target/include/target/custom_generator/custom_blob_handler.h‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,12 @@
2323

2424
namespacebuildcc {
2525

26+
/**
27+
* @brief Abstract class for serializing additional data for which rebuilds
28+
* might be triggered i.e data that is not input/output files
29+
* TODO, Add examples here
30+
*
31+
*/
2632
classCustomBlobHandler {
2733
public:
2834
CustomBlobHandler() =default;
@@ -54,6 +60,48 @@ class CustomBlobHandler {
5460
virtual std::vector<uint8_t>Serialize()const = 0;
5561
};
5662

63+
/**
64+
* @brief Typed Custom Blob handler which automatically performs Serialization
65+
* and Deserialization as long as it is JSON serializable
66+
*
67+
* NOTE: Type data is stored as a reference (to avoid copying large amount of
68+
* data) when constructing TypedCustomBlobHandler
69+
*
70+
* @tparam Type should be JSON serializable (see nlohmann::json compatible
71+
* objects)
72+
*/
73+
template<typename Type>
74+
classTypedCustomBlobHandler :publicCustomBlobHandler {
75+
public:
76+
explicitTypedCustomBlobHandler(const Type &data) : data_(data) {}
77+
78+
// serialized_data has already been verified
79+
static TypeDeserialize(const std::vector<uint8_t> &serialized_data) {
80+
json j =json::from_msgpack(serialized_data,true,false);
81+
Type deserialized;
82+
j.get_to(deserialized);
83+
return deserialized;
84+
}
85+
86+
private:
87+
const Type &data_;
88+
89+
boolVerify(const std::vector<uint8_t> &serialized_data)constoverride {
90+
json j =json::from_msgpack(serialized_data,true,false);
91+
return !j.is_discarded();
92+
}
93+
94+
boolIsEqual(const std::vector<uint8_t> &previous,
95+
const std::vector<uint8_t> &current)constoverride {
96+
returnDeserialize(previous) ==Deserialize(current);
97+
}
98+
99+
std::vector<uint8_t>Serialize()constoverride {
100+
json j = data_;
101+
returnjson::to_msgpack(j);
102+
}
103+
};
104+
57105
}// namespace buildcc
58106

59107
#endif

‎buildcc/lib/target/include/target/custom_generator/custom_generator_context.h‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,20 @@ namespace buildcc {
2525

2626
classCustomGeneratorContext {
2727
public:
28-
CustomGeneratorContext(const env::Command &c,const fs_unordered_set &i,
29-
const fs_unordered_set &o,
28+
CustomGeneratorContext(const env::Command &c,
29+
const std::unordered_set<std::string> &i,
30+
const std::unordered_set<std::string> &o,
3031
const std::vector<uint8_t> &ub)
3132
: command(c), inputs(i), outputs(o), userblob(ub) {}
3233

3334
const env::Command &command;
34-
constfs_unordered_set &inputs;
35-
constfs_unordered_set &outputs;
35+
conststd::unordered_set<std::string> &inputs;
36+
conststd::unordered_set<std::string> &outputs;
3637
const std::vector<uint8_t> &userblob;
3738
};
3839

3940
// clang-format off
40-
using GenerateCb = std::function<bool (CustomGeneratorContext &)>;
41+
using GenerateCb = std::function<bool (constCustomGeneratorContext &)>;
4142
// clang-format on
4243

4344
}// namespace buildcc

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp