Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit59f43d5

Browse files
authored
[Functionality] Reduce Custom generator scope to a Single stage generator (#227)
1 parent57a26a5 commit59f43d5

File tree

11 files changed

+337
-923
lines changed

11 files changed

+337
-923
lines changed

‎buildcc/lib/target/cmake/common_target_src.cmake‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ set(COMMON_TARGET_SRCS
2727
include/target/api/target_getter.h
2828

2929
# Generator
30+
include/target/custom_generator/custom_generator_context.h
31+
include/target/custom_generator/custom_blob_handler.h
32+
3033
src/custom_generator/custom_generator.cpp
3134
include/target/custom_generator.h
3235
src/generator/file_generator.cpp

‎buildcc/lib/target/include/target/custom_generator.h‎

Lines changed: 118 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -30,69 +30,28 @@
3030
#include"schema/custom_generator_serialization.h"
3131
#include"schema/path.h"
3232

33+
#include"custom_generator/custom_blob_handler.h"
34+
#include"custom_generator/custom_generator_context.h"
35+
3336
#include"target/common/target_env.h"
3437

3538
namespacebuildcc {
3639

37-
// TODO, Shift to a different file
38-
// TODO, Check if we need the "id" here as well
39-
classCustomGeneratorContext {
40-
public:
41-
CustomGeneratorContext(const env::Command &c,const fs_unordered_set &i,
42-
const fs_unordered_set &o,
43-
const std::vector<uint8_t> &ub)
44-
: command(c), inputs(i), outputs(o), userblob(ub) {}
45-
46-
const env::Command &command;
47-
const fs_unordered_set &inputs;
48-
const fs_unordered_set &outputs;
49-
const std::vector<uint8_t> &userblob;
50-
};
51-
52-
// clang-format off
53-
using GenerateCb = std::function<bool (CustomGeneratorContext &)>;
54-
55-
using DependencyCb = std::function<void (std::unordered_map<std::string, tf::Task> &&)>;
56-
// clang-format on
57-
58-
classCustomBlobHandler {
59-
public:
60-
CustomBlobHandler() =default;
61-
virtual~CustomBlobHandler() =default;
62-
63-
boolCheckChanged(const std::vector<uint8_t> &previous,
64-
const std::vector<uint8_t> &current)const {
65-
env::assert_fatal(
66-
Verify(previous),
67-
"Stored blob is corrupted or User verification is incorrect");
68-
env::assert_fatal(
69-
Verify(current),
70-
"Current blob is corrupted or User verification is incorrect");
71-
return !IsEqual(previous, current);
40+
structUserCustomGeneratorSchema :publicinternal::CustomGeneratorSchema {
41+
structUserIdInfo : internal::CustomGeneratorSchema::IdInfo {
42+
fs_unordered_set inputs;// TODO, Remove
43+
GenerateCb generate_cb;
44+
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
45+
46+
voidConvertToInternal() {
47+
internal_inputs =internal::path_schema_convert(
48+
inputs, internal::Path::CreateExistingPath);
49+
userblob = blob_handler !=nullptr ? blob_handler->GetSerializedData()
50+
: std::vector<uint8_t>();
51+
}
7252
};
7353

74-
std::vector<uint8_t>GetSerializedData()const {
75-
auto serialized_data =Serialize();
76-
env::assert_fatal(
77-
Verify(serialized_data),
78-
"Serialized data is corrupted or Serialize function is incorrect");
79-
return serialized_data;
80-
}
81-
82-
private:
83-
virtualboolVerify(const std::vector<uint8_t> &serialized_data)const = 0;
84-
virtualboolIsEqual(const std::vector<uint8_t> &previous,
85-
const std::vector<uint8_t> &current)const = 0;
86-
virtual std::vector<uint8_t>Serialize()const = 0;
87-
};
88-
89-
structUserIdInfo : internal::CustomGeneratorSchema::IdInfo {
90-
fs_unordered_set inputs;
91-
GenerateCb generate_cb;
92-
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
93-
};
94-
95-
structUserCustomGeneratorSchema :publicinternal::CustomGeneratorSchema {
54+
using UserIdPair = std::pair<const IdKey, UserIdInfo>;
9655
std::unordered_map<IdKey, UserIdInfo> ids;
9756

9857
voidConvertToInternal() {
@@ -110,7 +69,8 @@ class CustomGenerator : public internal::BuilderInterface {
11069
CustomGenerator(const std::string &name,const TargetEnv &env)
11170
: name_(name),
11271
env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name),
113-
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)) {
72+
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)),
73+
comparator_(serialization_.GetLoad(), user_) {
11474
Initialize();
11575
}
11676
virtual~CustomGenerator() =default;
@@ -137,33 +97,12 @@ class CustomGenerator : public internal::BuilderInterface {
13797
* @param generate_cb User-defined generate callback to build outputs from the
13898
* provided inputs
13999
*/
140-
voidAddIdInfo(const std::string &id,
141-
const std::unordered_set<std::string> &inputs,
142-
const std::unordered_set<std::string> &outputs,
143-
const GenerateCb &generate_cb,
144-
std::shared_ptr<CustomBlobHandler> blob_handler =nullptr);
145-
146-
// TODO, Doc
147-
voidAddGroupInfo(const std::string &group_id,
148-
std::initializer_list<std::string> ids,
149-
const DependencyCb &dependency_cb = DependencyCb());
150-
151-
// Callbacks
152-
/**
153-
* @brief Setup dependencies between Tasks using their `id`
154-
* For example: `task_map["id1"].precede(task_map["id2"])`
155-
*
156-
* IMPORTANT: Successor tasks will not automatically run if dependent task is
157-
* run.
158-
* The Dependency callback only sets precedence (order in which your tasks
159-
* should run)
160-
* Default behaviour when dependency callback is not supplied: All task `id`s
161-
* run in parallel.
162-
*
163-
* @param dependency_cb Unordered map of `id` and `task`
164-
* The map can be safely mutated.
165-
*/
166-
voidAddDependencyCb(const DependencyCb &dependency_cb);
100+
void
101+
AddIdInfo(const std::string &id,
102+
const std::unordered_set<std::string> &inputs,
103+
const std::unordered_set<std::string> &outputs,
104+
const GenerateCb &generate_cb,
105+
const std::shared_ptr<CustomBlobHandler> &blob_handler =nullptr);
167106

168107
voidBuild()override;
169108

@@ -176,19 +115,100 @@ class CustomGenerator : public internal::BuilderInterface {
176115
const fs::path &GetBuildDir()const {return env_.GetTargetBuildDir(); }
177116
const std::string &Get(const std::string &file_identifier)const;
178117

118+
private:
119+
structComparator {
120+
Comparator(const internal::CustomGeneratorSchema &loaded,
121+
const UserCustomGeneratorSchema &us)
122+
: loaded_schema_(loaded), current_schema_(us) {}
123+
124+
enumclassState {
125+
kRemoved,
126+
kAdded,
127+
kCheckLater,
128+
};
129+
130+
voidAddAllIds() {
131+
constauto &curr_ids = current_schema_.ids;
132+
for (constauto &[id, _] : curr_ids) {
133+
id_state_info_.at(State::kAdded).insert(id);
134+
}
135+
}
136+
137+
voidCompareIds() {
138+
constauto &prev_ids = loaded_schema_.internal_ids;
139+
constauto &curr_ids = current_schema_.ids;
140+
141+
for (constauto &[prev_id, _] : prev_ids) {
142+
if (curr_ids.find(prev_id) == curr_ids.end()) {
143+
// Id Removed condition, previous id is not present in the current run
144+
id_state_info_.at(State::kRemoved).insert(prev_id);
145+
}
146+
}
147+
148+
for (constauto &[curr_id, _] : curr_ids) {
149+
if (prev_ids.find(curr_id) == prev_ids.end()) {
150+
// Id Added condition
151+
id_state_info_.at(State::kAdded).insert(curr_id);
152+
}else {
153+
// Id Check Later condition
154+
id_state_info_.at(State::kCheckLater).insert(curr_id);
155+
}
156+
}
157+
}
158+
159+
boolIsChanged(const std::string &id)const {
160+
constauto &previous_id_info = loaded_schema_.internal_ids.at(id);
161+
constauto &current_id_info = current_schema_.ids.at(id);
162+
163+
bool changed =internal::CheckPaths(previous_id_info.internal_inputs,
164+
current_id_info.internal_inputs) !=
165+
internal::PathState::kNoChange;
166+
changed = changed ||internal::CheckChanged(previous_id_info.outputs,
167+
current_id_info.outputs);
168+
if (!changed && current_id_info.blob_handler !=nullptr) {
169+
// We only check blob handler if not changed by inputs/outputs
170+
// Checking blob_handler could be expensive so this optimization is made
171+
// to run only when changed == false
172+
changed = current_id_info.blob_handler->CheckChanged(
173+
previous_id_info.userblob, current_id_info.userblob);
174+
}
175+
return changed;
176+
}
177+
178+
const std::unordered_set<std::string> &GetRemovedIds()const {
179+
return id_state_info_.at(State::kRemoved);
180+
}
181+
182+
const std::unordered_set<std::string> &GetAddedIds()const {
183+
return id_state_info_.at(State::kAdded);
184+
}
185+
186+
const std::unordered_set<std::string> &GetCheckLaterIds()const {
187+
return id_state_info_.at(State::kCheckLater);
188+
}
189+
190+
boolIsIdAdded(const std::string &id)const {
191+
return id_state_info_.at(State::kAdded).count(id) ==1;
192+
}
193+
194+
private:
195+
const internal::CustomGeneratorSchema &loaded_schema_;
196+
const UserCustomGeneratorSchema &current_schema_;
197+
std::unordered_map<State, std::unordered_set<std::string>> id_state_info_{
198+
{State::kRemoved, std::unordered_set<std::string>()},
199+
{State::kAdded, std::unordered_set<std::string>()},
200+
{State::kCheckLater, std::unordered_set<std::string>()},
201+
};
202+
};
203+
179204
private:
180205
voidInitialize();
181206

182-
voidTaskRunner(bool run,const std::string &id);
183-
tf::TaskCreateTaskRunner(tf::Subflow &subflow,bool build,
184-
const std::string &id);
207+
tf::TaskCreateTaskRunner(tf::Subflow &subflow,const std::string &id);
208+
voidTaskRunner(const std::string &id);
185209

186210
voidGenerateTask();
187-
voidBuildGenerate(std::unordered_set<std::string> &gen_selected_ids,
188-
std::unordered_set<std::string> &dummy_gen_selected_ids);
189-
190-
voidInvokeDependencyCb(std::unordered_map<std::string, tf::Task>
191-
&&registered_tasks)constnoexcept;
211+
voidBuildGenerate();
192212

193213
// Recheck states
194214
voidIdRemoved();
@@ -199,47 +219,23 @@ class CustomGenerator : public internal::BuilderInterface {
199219
const env::Command &ConstCommand()const {return command_; }
200220
env::Command &RefCommand() {return command_; }
201221

202-
private:
203-
structGroupMetadata {
204-
std::vector<std::string> ids;
205-
DependencyCb dependency_cb;
206-
207-
voidInvokeDependencyCb(const std::string &group_id,
208-
std::unordered_map<std::string, tf::Task>
209-
&&registered_tasks)constnoexcept {
210-
if (!dependency_cb) {
211-
return;
212-
}
213-
try {
214-
dependency_cb(std::move(registered_tasks));
215-
}catch (...) {
216-
env::log_critical(
217-
__FUNCTION__,
218-
fmt::format("Dependency callback failed for group id {}",
219-
group_id));
220-
env::set_task_state(env::TaskState::FAILURE);
221-
}
222-
}
223-
};
224-
225222
private:
226223
std::string name_;
227224
TargetEnv env_;
228225
internal::CustomGeneratorSerialization serialization_;
229226

230227
// Serialization
231228
UserCustomGeneratorSchema user_;
232-
std::unordered_map<std::string, GroupMetadata> grouped_ids_;
233-
std::unordered_set<std::string> ungrouped_ids_;
229+
230+
// Comparator
231+
Comparator comparator_;
234232

235233
std::mutex success_schema_mutex_;
236-
std::unordered_map<std::string, UserIdInfo> success_schema_;
234+
std::unordered_map<std::string, UserCustomGeneratorSchema::UserIdInfo>
235+
success_schema_;
237236

238237
// Internal
239238
env::Command command_;
240-
241-
// Callbacks
242-
DependencyCb dependency_cb_;
243239
};
244240

245241
}// namespace buildcc
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2021-2022 Niket Naidu. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef TARGET_CUSTOM_GENERATOR_CUSTOM_BLOB_HANDLER_H_
18+
#defineTARGET_CUSTOM_GENERATOR_CUSTOM_BLOB_HANDLER_H_
19+
20+
#include<vector>
21+
22+
#include"env/assert_fatal.h"
23+
24+
namespacebuildcc {
25+
26+
classCustomBlobHandler {
27+
public:
28+
CustomBlobHandler() =default;
29+
virtual~CustomBlobHandler() =default;
30+
31+
boolCheckChanged(const std::vector<uint8_t> &previous,
32+
const std::vector<uint8_t> &current)const {
33+
env::assert_fatal(
34+
Verify(previous),
35+
"Stored blob is corrupted or User verification is incorrect");
36+
env::assert_fatal(
37+
Verify(current),
38+
"Current blob is corrupted or User verification is incorrect");
39+
return !IsEqual(previous, current);
40+
};
41+
42+
std::vector<uint8_t>GetSerializedData()const {
43+
auto serialized_data =Serialize();
44+
env::assert_fatal(
45+
Verify(serialized_data),
46+
"Serialized data is corrupted or Serialize function is incorrect");
47+
return serialized_data;
48+
}
49+
50+
private:
51+
virtualboolVerify(const std::vector<uint8_t> &serialized_data)const = 0;
52+
virtualboolIsEqual(const std::vector<uint8_t> &previous,
53+
const std::vector<uint8_t> &current)const = 0;
54+
virtual std::vector<uint8_t>Serialize()const = 0;
55+
};
56+
57+
}// namespace buildcc
58+
59+
#endif

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp