Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

[Functionality] Reduce Custom generator scope to a Single stage generator#227

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
coder137 merged 19 commits intomainfromcustom_generator_improvements
Nov 24, 2022
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
Show all changes
19 commits
Select commitHold shift + click to select a range
346925c
Added Comparator for custom_generator
coder137Nov 7, 2022
3b1d63e
Updated CreateTaskRunner and TaskRunner APIs
coder137Nov 7, 2022
2395186
Added GroupRunner
coder137Nov 10, 2022
83fab19
Removed dependency_cb
coder137Nov 19, 2022
da0ef69
Update gcovr.cmake
coder137Nov 19, 2022
9352f01
Update test_custom_generator.cpp
coder137Nov 19, 2022
33a1111
Removed groups from Custom Generator
coder137Nov 20, 2022
240881e
Removed ungrouped ids from custom generator
coder137Nov 21, 2022
19ce132
Updated comparator variable names
coder137Nov 21, 2022
950d840
Added IsChanged function to Comparator
coder137Nov 21, 2022
fa20529
Added a ConvertToInternal function for UserCustomGeneratorSchema
coder137Nov 21, 2022
6d03060
Removed groups from custom_generator_schema
coder137Nov 23, 2022
2bfff72
Shifted classes to seperate include files
coder137Nov 24, 2022
0156aec
Update custom_generator.cpp
coder137Nov 24, 2022
a0316aa
Pass blob_handler shared_ptr by ref
coder137Nov 24, 2022
fdc7849
Update custom_generator.h
coder137Nov 24, 2022
6a0e493
Update cppcheck.cmake
coder137Nov 24, 2022
a4a52b2
Updated tests for custom_generator
coder137Nov 24, 2022
03061bd
Updated custom_generator blob_handler implementation
coder137Nov 24, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletionsbuildcc/lib/target/cmake/common_target_src.cmake
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -27,6 +27,9 @@ set(COMMON_TARGET_SRCS
include/target/api/target_getter.h

# Generator
include/target/custom_generator/custom_generator_context.h
include/target/custom_generator/custom_blob_handler.h

src/custom_generator/custom_generator.cpp
include/target/custom_generator.h
src/generator/file_generator.cpp
Expand Down
240 changes: 118 additions & 122 deletionsbuildcc/lib/target/include/target/custom_generator.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -30,69 +30,28 @@
#include "schema/custom_generator_serialization.h"
#include "schema/path.h"

#include "custom_generator/custom_blob_handler.h"
#include "custom_generator/custom_generator_context.h"

#include "target/common/target_env.h"

namespace buildcc {

// TODO, Shift to a different file
// TODO, Check if we need the "id" here as well
class CustomGeneratorContext {
public:
CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i,
const fs_unordered_set &o,
const std::vector<uint8_t> &ub)
: command(c), inputs(i), outputs(o), userblob(ub) {}

const env::Command &command;
const fs_unordered_set &inputs;
const fs_unordered_set &outputs;
const std::vector<uint8_t> &userblob;
};

// clang-format off
using GenerateCb = std::function<bool (CustomGeneratorContext &)>;

using DependencyCb = std::function<void (std::unordered_map<std::string, tf::Task> &&)>;
// clang-format on

class CustomBlobHandler {
public:
CustomBlobHandler() = default;
virtual ~CustomBlobHandler() = default;

bool CheckChanged(const std::vector<uint8_t> &previous,
const std::vector<uint8_t> &current) const {
env::assert_fatal(
Verify(previous),
"Stored blob is corrupted or User verification is incorrect");
env::assert_fatal(
Verify(current),
"Current blob is corrupted or User verification is incorrect");
return !IsEqual(previous, current);
struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo {
fs_unordered_set inputs; // TODO, Remove
GenerateCb generate_cb;
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};

void ConvertToInternal() {
internal_inputs = internal::path_schema_convert(
inputs, internal::Path::CreateExistingPath);
userblob = blob_handler != nullptr ? blob_handler->GetSerializedData()
: std::vector<uint8_t>();
}
};

std::vector<uint8_t> GetSerializedData() const {
auto serialized_data = Serialize();
env::assert_fatal(
Verify(serialized_data),
"Serialized data is corrupted or Serialize function is incorrect");
return serialized_data;
}

private:
virtual bool Verify(const std::vector<uint8_t> &serialized_data) const = 0;
virtual bool IsEqual(const std::vector<uint8_t> &previous,
const std::vector<uint8_t> &current) const = 0;
virtual std::vector<uint8_t> Serialize() const = 0;
};

struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo {
fs_unordered_set inputs;
GenerateCb generate_cb;
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
};

struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
using UserIdPair = std::pair<const IdKey, UserIdInfo>;
std::unordered_map<IdKey, UserIdInfo> ids;

void ConvertToInternal() {
Expand All@@ -110,7 +69,8 @@ class CustomGenerator : public internal::BuilderInterface {
CustomGenerator(const std::string &name, const TargetEnv &env)
: name_(name),
env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name),
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.bin", name)) {
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)),
comparator_(serialization_.GetLoad(), user_) {
Initialize();
}
virtual ~CustomGenerator() = default;
Expand All@@ -137,33 +97,12 @@ class CustomGenerator : public internal::BuilderInterface {
* @param generate_cb User-defined generate callback to build outputs from the
* provided inputs
*/
void AddIdInfo(const std::string &id,
const std::unordered_set<std::string> &inputs,
const std::unordered_set<std::string> &outputs,
const GenerateCb &generate_cb,
std::shared_ptr<CustomBlobHandler> blob_handler = nullptr);

// TODO, Doc
void AddGroupInfo(const std::string &group_id,
std::initializer_list<std::string> ids,
const DependencyCb &dependency_cb = DependencyCb());

// Callbacks
/**
* @brief Setup dependencies between Tasks using their `id`
* For example: `task_map["id1"].precede(task_map["id2"])`
*
* IMPORTANT: Successor tasks will not automatically run if dependent task is
* run.
* The Dependency callback only sets precedence (order in which your tasks
* should run)
* Default behaviour when dependency callback is not supplied: All task `id`s
* run in parallel.
*
* @param dependency_cb Unordered map of `id` and `task`
* The map can be safely mutated.
*/
void AddDependencyCb(const DependencyCb &dependency_cb);
void
AddIdInfo(const std::string &id,
const std::unordered_set<std::string> &inputs,
const std::unordered_set<std::string> &outputs,
const GenerateCb &generate_cb,
const std::shared_ptr<CustomBlobHandler> &blob_handler = nullptr);

void Build() override;

Expand All@@ -176,19 +115,100 @@ class CustomGenerator : public internal::BuilderInterface {
const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); }
const std::string &Get(const std::string &file_identifier) const;

private:
struct Comparator {
Comparator(const internal::CustomGeneratorSchema &loaded,
const UserCustomGeneratorSchema &us)
: loaded_schema_(loaded), current_schema_(us) {}

enum class State {
kRemoved,
kAdded,
kCheckLater,
};

void AddAllIds() {
const auto &curr_ids = current_schema_.ids;
for (const auto &[id, _] : curr_ids) {
id_state_info_.at(State::kAdded).insert(id);
}
}

void CompareIds() {
const auto &prev_ids = loaded_schema_.internal_ids;
const auto &curr_ids = current_schema_.ids;

for (const auto &[prev_id, _] : prev_ids) {
if (curr_ids.find(prev_id) == curr_ids.end()) {
// Id Removed condition, previous id is not present in the current run
id_state_info_.at(State::kRemoved).insert(prev_id);
}
}

for (const auto &[curr_id, _] : curr_ids) {
if (prev_ids.find(curr_id) == prev_ids.end()) {
// Id Added condition
id_state_info_.at(State::kAdded).insert(curr_id);
} else {
// Id Check Later condition
id_state_info_.at(State::kCheckLater).insert(curr_id);
}
}
}

bool IsChanged(const std::string &id) const {
const auto &previous_id_info = loaded_schema_.internal_ids.at(id);
const auto &current_id_info = current_schema_.ids.at(id);

bool changed = internal::CheckPaths(previous_id_info.internal_inputs,
current_id_info.internal_inputs) !=
internal::PathState::kNoChange;
changed = changed || internal::CheckChanged(previous_id_info.outputs,
current_id_info.outputs);
if (!changed && current_id_info.blob_handler != nullptr) {
// We only check blob handler if not changed by inputs/outputs
// Checking blob_handler could be expensive so this optimization is made
// to run only when changed == false
changed = current_id_info.blob_handler->CheckChanged(
previous_id_info.userblob, current_id_info.userblob);
}
return changed;
}

const std::unordered_set<std::string> &GetRemovedIds() const {
return id_state_info_.at(State::kRemoved);
}

const std::unordered_set<std::string> &GetAddedIds() const {
return id_state_info_.at(State::kAdded);
}

const std::unordered_set<std::string> &GetCheckLaterIds() const {
return id_state_info_.at(State::kCheckLater);
}

bool IsIdAdded(const std::string &id) const {
return id_state_info_.at(State::kAdded).count(id) == 1;
}

private:
const internal::CustomGeneratorSchema &loaded_schema_;
const UserCustomGeneratorSchema &current_schema_;
std::unordered_map<State, std::unordered_set<std::string>> id_state_info_{
{State::kRemoved, std::unordered_set<std::string>()},
{State::kAdded, std::unordered_set<std::string>()},
{State::kCheckLater, std::unordered_set<std::string>()},
};
};

private:
void Initialize();

void TaskRunner(bool run, const std::string &id);
tf::Task CreateTaskRunner(tf::Subflow &subflow, bool build,
const std::string &id);
tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id);
void TaskRunner(const std::string &id);

void GenerateTask();
void BuildGenerate(std::unordered_set<std::string> &gen_selected_ids,
std::unordered_set<std::string> &dummy_gen_selected_ids);

void InvokeDependencyCb(std::unordered_map<std::string, tf::Task>
&&registered_tasks) const noexcept;
void BuildGenerate();

// Recheck states
void IdRemoved();
Expand All@@ -199,47 +219,23 @@ class CustomGenerator : public internal::BuilderInterface {
const env::Command &ConstCommand() const { return command_; }
env::Command &RefCommand() { return command_; }

private:
struct GroupMetadata {
std::vector<std::string> ids;
DependencyCb dependency_cb;

void InvokeDependencyCb(const std::string &group_id,
std::unordered_map<std::string, tf::Task>
&&registered_tasks) const noexcept {
if (!dependency_cb) {
return;
}
try {
dependency_cb(std::move(registered_tasks));
} catch (...) {
env::log_critical(
__FUNCTION__,
fmt::format("Dependency callback failed for group id {}",
group_id));
env::set_task_state(env::TaskState::FAILURE);
}
}
};

private:
std::string name_;
TargetEnv env_;
internal::CustomGeneratorSerialization serialization_;

// Serialization
UserCustomGeneratorSchema user_;
std::unordered_map<std::string, GroupMetadata> grouped_ids_;
std::unordered_set<std::string> ungrouped_ids_;

// Comparator
Comparator comparator_;

std::mutex success_schema_mutex_;
std::unordered_map<std::string, UserIdInfo> success_schema_;
std::unordered_map<std::string, UserCustomGeneratorSchema::UserIdInfo>
success_schema_;

// Internal
env::Command command_;

// Callbacks
DependencyCb dependency_cb_;
};

} // namespace buildcc
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
/*
* Copyright 2021-2022 Niket Naidu. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

#ifndef TARGET_CUSTOM_GENERATOR_CUSTOM_BLOB_HANDLER_H_
#define TARGET_CUSTOM_GENERATOR_CUSTOM_BLOB_HANDLER_H_

#include <vector>

#include "env/assert_fatal.h"

namespace buildcc {

class CustomBlobHandler {
public:
CustomBlobHandler() = default;
virtual ~CustomBlobHandler() = default;

bool CheckChanged(const std::vector<uint8_t> &previous,
const std::vector<uint8_t> &current) const {
env::assert_fatal(
Verify(previous),
"Stored blob is corrupted or User verification is incorrect");
env::assert_fatal(
Verify(current),
"Current blob is corrupted or User verification is incorrect");
return !IsEqual(previous, current);
};

std::vector<uint8_t> GetSerializedData() const {
auto serialized_data = Serialize();
env::assert_fatal(
Verify(serialized_data),
"Serialized data is corrupted or Serialize function is incorrect");
return serialized_data;
}

private:
virtual bool Verify(const std::vector<uint8_t> &serialized_data) const = 0;
virtual bool IsEqual(const std::vector<uint8_t> &previous,
const std::vector<uint8_t> &current) const = 0;
virtual std::vector<uint8_t> Serialize() const = 0;
};

} // namespace buildcc

#endif
Loading
[8]ページ先頭
©2009-2025 Movatter.jp