Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

[Serialization] Custom Generator unit tests#224

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
coder137 merged 12 commits intomainfromcustom_generator_schema_unit_tests
Nov 4, 2022
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
Show all changes
12 commits
Select commitHold shift + click to select a range
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 16 additions & 16 deletionsbuildcc/lib/target/include/target/custom_generator.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -86,21 +86,21 @@ class CustomBlobHandler {
virtual std::vector<uint8_t> Serialize() const = 0;
};

structUserGenInfo : internal::CustomGeneratorSchema::IdInfo {
structUserIdInfo : internal::CustomGeneratorSchema::IdInfo {
fs_unordered_set inputs;
GenerateCb generate_cb;
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
};

struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
std::unordered_map<std::string, UserGenInfo> gen_info_map;
std::unordered_map<IdKey, UserIdInfo> ids;

void ConvertToInternal() {
for (auto &[id, gen_info] :gen_info_map) {
gen_info.internal_inputs = path_schema_convert(
gen_info.inputs, internal::Path::CreateExistingPath);
auto [_, success] = internal_ids.try_emplace(id, gen_info);
env::assert_fatal(success, fmt::format("Could not save {}",id));
for (auto &[id_key, id_info] :ids) {
id_info.internal_inputs = path_schema_convert(
id_info.inputs, internal::Path::CreateExistingPath);
auto [_, success] = internal_ids.try_emplace(id_key, id_info);
env::assert_fatal(success, fmt::format("Could not save {}",id_key));
}
}
};
Expand DownExpand Up@@ -137,16 +137,16 @@ class CustomGenerator : public internal::BuilderInterface {
* @param generate_cb User-defined generate callback to build outputs from the
* provided inputs
*/
voidAddGenInfo(const std::string &id,
const std::unordered_set<std::string> &inputs,
const std::unordered_set<std::string> &outputs,
const GenerateCb &generate_cb,
std::shared_ptr<CustomBlobHandler> blob_handler = nullptr);
voidAddIdInfo(const std::string &id,
const std::unordered_set<std::string> &inputs,
const std::unordered_set<std::string> &outputs,
const GenerateCb &generate_cb,
std::shared_ptr<CustomBlobHandler> blob_handler = nullptr);

// TODO, Doc
voidAddGroup(const std::string &group_id,
std::initializer_list<std::string> ids,
const DependencyCb &dependency_cb = DependencyCb());
voidAddGroupInfo(const std::string &group_id,
std::initializer_list<std::string> ids,
const DependencyCb &dependency_cb = DependencyCb());

// Callbacks
/**
Expand DownExpand Up@@ -233,7 +233,7 @@ class CustomGenerator : public internal::BuilderInterface {
std::unordered_set<std::string> ungrouped_ids_;

std::mutex success_schema_mutex_;
std::unordered_map<std::string,UserGenInfo> success_schema_;
std::unordered_map<std::string,UserIdInfo> success_schema_;

// Internal
env::Command command_;
Expand Down
4 changes: 2 additions & 2 deletionsbuildcc/lib/target/include/target/file_generator.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -69,8 +69,8 @@ class FileGenerator : public CustomGenerator {
// Restrict access to certain custom generator APIs
private:
using CustomGenerator::AddDependencyCb;
using CustomGenerator::AddGenInfo;
using CustomGenerator::AddGroup;
using CustomGenerator::AddGroupInfo;
using CustomGenerator::AddIdInfo;
using CustomGenerator::Build;

private:
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -74,8 +74,8 @@ inline PathState CheckPaths(const internal::path_unordered_set &previous_path,
state = PathState::kAdded;
} else {
const bool updated_cond =
(p.GetLastWriteTimestamp() >
find->GetLastWriteTimestamp());
(p.last_write_timestamp >
find->last_write_timestamp);
if (updated_cond) {
dirty = true;
state = PathState::kUpdated;
Expand Down
4 changes: 2 additions & 2 deletionsbuildcc/lib/target/include/target/template_generator.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -43,8 +43,8 @@ class TemplateGenerator : public CustomGenerator {
// Restrict access to certain custom generator APIs
private:
using CustomGenerator::AddDependencyCb;
using CustomGenerator::AddGenInfo;
using CustomGenerator::AddGroup;
using CustomGenerator::AddGroupInfo;
using CustomGenerator::AddIdInfo;
using CustomGenerator::Build;

private:
Expand Down
92 changes: 47 additions & 45 deletionsbuildcc/lib/target/src/custom_generator/custom_generator.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -51,16 +51,16 @@ CustomGenerator::Get(const std::string &file_identifier) const {
return command_.GetDefaultValueByKey(file_identifier);
}

void CustomGenerator::AddGenInfo(
void CustomGenerator::AddIdInfo(
const std::string &id, const std::unordered_set<std::string> &inputs,
const std::unordered_set<std::string> &outputs,
const GenerateCb &generate_cb,
std::shared_ptr<CustomBlobHandler> blob_handler) {
env::assert_fatal(user_.gen_info_map.find(id) == user_.gen_info_map.end(),
env::assert_fatal(user_.ids.find(id) == user_.ids.end(),
fmt::format("Duplicate id {} detected", id));
ASSERT_FATAL(generate_cb, "Invalid callback provided");

UserGenInfo schema;
UserIdInfo schema;
for (const auto &i : inputs) {
fs::path input = string_as_path(command_.Construct(i));
schema.inputs.emplace(std::move(input));
Expand All@@ -71,17 +71,17 @@ void CustomGenerator::AddGenInfo(
}
schema.generate_cb = generate_cb;
schema.blob_handler = std::move(blob_handler);
user_.gen_info_map.try_emplace(id, std::move(schema));
user_.ids.try_emplace(id, std::move(schema));
ungrouped_ids_.emplace(id);
}

void CustomGenerator::AddGroup(const std::string &group_id,
std::initializer_list<std::string> ids,
const DependencyCb &dependency_cb) {
void CustomGenerator::AddGroupInfo(const std::string &group_id,
std::initializer_list<std::string> ids,
const DependencyCb &dependency_cb) {
// Verify that the ids exist
// Remove those ids from ungrouped_ids
for (const auto &id : ids) {
env::assert_fatal(user_.gen_info_map.find(id) != user_.gen_info_map.end(),
env::assert_fatal(user_.ids.find(id) != user_.ids.end(),
fmt::format("Id '{}' is not found", id));
ungrouped_ids_.erase(id);
}
Expand DownExpand Up@@ -131,34 +131,34 @@ void CustomGenerator::BuildGenerate(
std::unordered_set<std::string> &gen_selected_ids,
std::unordered_set<std::string> &dummy_gen_selected_ids) {
if (!serialization_.IsLoaded()) {
std::for_each(
user_.gen_info_map.begin(), user_.gen_info_map.end(),
[&](const auto &iter) { gen_selected_ids.insert(iter.first);});
std::for_each(user_.ids.begin(), user_.ids.end(), [&](const auto &iter) {
gen_selected_ids.insert(iter.first);
});
dirty_ = true;
} else {
// DONE, Conditionally select internal_ids depending on what has
// changed
const auto &prev_gen_info_map = serialization_.GetLoad().internal_ids;
const auto &curr_gen_info_map = user_.gen_info_map;
const auto &prev_ids = serialization_.GetLoad().internal_ids;
const auto &curr_ids = user_.ids;

// DONE, MAP REMOVED condition Check ifprev_gen_info_map exists in
//curr_gen_info_map Ifprev_gen_info_map does not exist in
//curr_gen_info_map, has been removed from existing build We need this
// DONE, MAP REMOVED condition Check ifprev_ids exists in
//curr_ids Ifprev_ids does not exist in
//curr_ids, has been removed from existing build We need this
// condition to only set the dirty_ flag
for (const auto &[id, _] :prev_gen_info_map) {
if (curr_gen_info_map.find(id) ==curr_gen_info_map.end()) {
for (const auto &[id, _] :prev_ids) {
if (curr_ids.find(id) ==curr_ids.end()) {
// MAP REMOVED condition
IdRemoved();
dirty_ = true;
break;
}
}

// DONE, MAP ADDED condition Check ifcurr_gen_info_map exists in
//prev_gen_info_map Ifcurr_gen_info_map does not exist in
//prev_gen_info_map, has been added to existing build
for (const auto &[id, _] :curr_gen_info_map) {
if (prev_gen_info_map.find(id) ==prev_gen_info_map.end()) {
// DONE, MAP ADDED condition Check ifcurr_ids exists in
//prev_ids Ifcurr_ids does not exist in
//prev_ids, has been added to existing build
for (const auto &[id, _] :curr_ids) {
if (prev_ids.find(id) ==prev_ids.end()) {
// MAP ADDED condition
IdAdded();
gen_selected_ids.insert(id);
Expand DownExpand Up@@ -232,8 +232,8 @@ void CustomGenerator::GenerateTask() {
// Store dummy_selected and successfully run schema
if (dirty_) {
UserCustomGeneratorSchema user_final_schema;
user_final_schema.gen_info_map.insert(success_schema_.begin(),
success_schema_.end());
user_final_schema.ids.insert(success_schema_.begin(),
success_schema_.end());

user_final_schema.ConvertToInternal();
serialization_.UpdateStore(user_final_schema);
Expand DownExpand Up@@ -278,42 +278,44 @@ tf::Task CustomGenerator::CreateTaskRunner(tf::Subflow &subflow, bool build,

void CustomGenerator::TaskRunner(bool run, const std::string &id) {
// Convert
auto &current_gen_info = user_.gen_info_map.at(id);
current_gen_info.internal_inputs = internal::path_schema_convert(
current_gen_info.inputs, internal::Path::CreateExistingPath);
current_gen_info.userblob =
current_gen_info.blob_handler != nullptr
? current_gen_info.blob_handler->GetSerializedData()
: std::vector<uint8_t>();
{
auto &curr_id_info = user_.ids.at(id);
curr_id_info.internal_inputs = internal::path_schema_convert(
curr_id_info.inputs, internal::Path::CreateExistingPath);
curr_id_info.userblob = curr_id_info.blob_handler != nullptr
? curr_id_info.blob_handler->GetSerializedData()
: std::vector<uint8_t>();
}

// Run
const auto &current_info = user_.gen_info_map.at(id);
const auto &current_id_info = user_.ids.at(id);
bool rerun = false;
if (run) {
rerun = true;
} else {
const auto &previous_info = serialization_.GetLoad().internal_ids.at(id);
rerun = internal::CheckPaths(previous_info.internal_inputs,
current_info.internal_inputs) !=
internal::PathState::kNoChange ||
internal::CheckChanged(previous_info.outputs, current_info.outputs);
if (!rerun && current_info.blob_handler != nullptr) {
rerun = current_info.blob_handler->CheckChanged(previous_info.userblob,
current_info.userblob);
rerun =
internal::CheckPaths(previous_info.internal_inputs,
current_id_info.internal_inputs) !=
internal::PathState::kNoChange ||
internal::CheckChanged(previous_info.outputs, current_id_info.outputs);
if (!rerun && current_id_info.blob_handler != nullptr) {
rerun = current_id_info.blob_handler->CheckChanged(
previous_info.userblob, current_id_info.userblob);
}
}

if (rerun) {
dirty_ = true;
buildcc::CustomGeneratorContext ctx(command_,current_info.inputs,
current_info.outputs,
current_info.userblob);
bool success =current_info.generate_cb(ctx);
buildcc::CustomGeneratorContext ctx(command_,current_id_info.inputs,
current_id_info.outputs,
current_id_info.userblob);
bool success =current_id_info.generate_cb(ctx);
env::assert_fatal(success, fmt::format("Generate Cb failed for id {}", id));
}

std::scoped_lock<std::mutex> guard(success_schema_mutex_);
success_schema_.try_emplace(id,current_info);
success_schema_.try_emplace(id,current_id_info);
}

} // namespace buildcc
4 changes: 2 additions & 2 deletionsbuildcc/lib/target/src/generator/file_generator.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -111,8 +111,8 @@ void FileGenerator::AddCommand(
}

void FileGenerator::Build() {
AddGenInfo("Generate", inputs_, outputs_, FileGeneratorGenerateCb,
std::make_shared<FileGeneratorBlobHandler>(commands_));
AddIdInfo("Generate", inputs_, outputs_, FileGeneratorGenerateCb,
std::make_shared<FileGeneratorBlobHandler>(commands_));
this->CustomGenerator::Build();
}

Expand Down
4 changes: 2 additions & 2 deletionsbuildcc/lib/target/src/generator/template_generator.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -66,8 +66,8 @@ void TemplateGenerator::Build() {
std::string name = string_as_path(ParsePattern(info.input_pattern))
.lexically_relative(Project::GetRootDir())
.string();
AddGenInfo(name, {info.input_pattern}, {info.output_pattern},
template_generate_cb);
AddIdInfo(name, {info.input_pattern}, {info.output_pattern},
template_generate_cb);
}
this->CustomGenerator::Build();
}
Expand Down
3 changes: 1 addition & 2 deletionsbuildcc/lib/target/src/target/friend/compile_object.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -254,8 +254,7 @@ void CompileObject::RecompileSources(
target_.SourceAdded();
} else {
// *2 Current file is updated
if (current_file.GetLastWriteTimestamp() >
iter->GetLastWriteTimestamp()) {
if (current_file.last_write_timestamp > iter->last_write_timestamp) {
source_files.push_back(current_file);
target_.dirty_ = true;
target_.SourceUpdated();
Expand Down
8 changes: 4 additions & 4 deletionsbuildcc/lib/target/src/target/tasks.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -94,12 +94,12 @@ void CompileObject::Task() {

for (const auto &s : selected_source_files) {
std::string name = fmt::format(
"{}", s.GetPathname().lexically_relative(Project::GetRootDir()));
"{}", s.pathname.lexically_relative(Project::GetRootDir()));
(void)subflow
.emplace([this, s]() {
try {
bool success = env::Command::Execute(
GetObjectData(s.GetPathname()).command);
bool success =
env::Command::Execute(GetObjectData(s.pathname).command);
env::assert_fatal(success, "Could not compile source");
target_.serialization_.AddSource(s);
} catch (...) {
Expand All@@ -112,7 +112,7 @@ void CompileObject::Task() {
// For graph generation
for (const auto &ds : selected_dummy_source_files) {
std::string name = fmt::format(
"{}", ds.GetPathname().lexically_relative(Project::GetRootDir()));
"{}", ds.pathname.lexically_relative(Project::GetRootDir()));
(void)subflow.placeholder().name(name);
}
} catch (...) {
Expand Down
36 changes: 17 additions & 19 deletionsbuildcc/lib/target/test/path/test_path.cpp
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -26,11 +26,11 @@ static const auto current_file_path =
TEST(PathTestGroup, Path_ExistingPathStaticConstructor) {
auto existing_path =
buildcc::internal::Path::CreateExistingPath(current_file_path);
STRCMP_EQUAL(existing_path.GetPathname().string().c_str(),
STRCMP_EQUAL(existing_path.pathname.string().c_str(),
current_file_path.string().c_str());
// * NOTE, Last write timestamp changes whenever we resave or re-download
// This would not work well with Git
// UNSIGNED_LONGLONGS_EQUAL(existing_path.GetLastWriteTimestamp(),
// UNSIGNED_LONGLONGS_EQUAL(existing_path.last_write_timestamp,
// 13623997187709551616ULL);
}

Expand All@@ -41,19 +41,18 @@ TEST(PathTestGroup, Path_ExistingPathStaticConstructor_ThrowFileException) {

TEST(PathTestGroup, PathConstructor_NewPathStaticConstructor) {
buildcc::internal::Path p =
buildcc::internal::Path::CreateNewPath("random_path_main.cpp", 12345ULL);
STRCMP_EQUAL(p.GetPathname().string().c_str(), "random_path_main.cpp");
UNSIGNED_LONGLONGS_EQUAL(p.GetLastWriteTimestamp(), 12345ULL);
buildcc::internal::Path("random_path_main.cpp", 12345ULL);
STRCMP_EQUAL(p.pathname.string().c_str(), "random_path_main.cpp");
UNSIGNED_LONGLONGS_EQUAL(p.last_write_timestamp, 12345ULL);
}

TEST(PathTestGroup, Path_EqualityOperator) {
buildcc::internal::Path p =
buildcc::internal::Path::CreateExistingPath(current_file_path);
STRCMP_EQUAL(p.GetPathname().string().c_str(),
current_file_path.string().c_str());
STRCMP_EQUAL(p.pathname.string().c_str(), current_file_path.string().c_str());

buildcc::internal::Path newp =
buildcc::internal::Path::CreateNewPath(current_file_path, 12345ULL);
buildcc::internal::Path(current_file_path, 12345ULL);

// NOTE, Equality does not match the last_write_timestamp
// ONLY matches the string
Expand All@@ -71,25 +70,24 @@ TEST(PathTestGroup, Path_UnorderedSet) {
.insert(buildcc::internal::Path::CreateExistingPath(
current_file_path))
.second);
CHECK_FALSE(unique_paths
.insert(buildcc::internal::Path::CreateNewPath(
current_file_path, 12345ULL))
.second);
CHECK_TRUE(unique_paths
.insert(buildcc::internal::Path::CreateNewPath(
"random_path_main.cpp", 98765ULL))
.second);
CHECK_FALSE(
unique_paths.insert(buildcc::internal::Path(current_file_path, 12345ULL))
.second);
CHECK_TRUE(
unique_paths
.insert(buildcc::internal::Path("random_path_main.cpp", 98765ULL))
.second);

// Check finds
// * NOTE, Only matches pathname
CHECK_FALSE(unique_paths.find(buildcc::internal::Path::CreateExistingPath(
current_file_path)) == unique_paths.end());

CHECK_FALSE(unique_paths.find(buildcc::internal::Path::CreateNewPath(
CHECK_FALSE(unique_paths.find(buildcc::internal::Path(
current_file_path, 1111ULL)) == unique_paths.end());
CHECK_FALSE(unique_paths.find(buildcc::internal::Path::CreateNewPath(
CHECK_FALSE(unique_paths.find(buildcc::internal::Path(
"random_path_main.cpp", 12345ULL)) == unique_paths.end());
CHECK_TRUE(unique_paths.find(buildcc::internal::Path::CreateNewPath(
CHECK_TRUE(unique_paths.find(buildcc::internal::Path(
"incorrect_path_main.cpp", 0000ULL)) == unique_paths.end());
}

Expand Down
Loading
[8]ページ先頭
©2009-2025 Movatter.jp