Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

[Enhancement] Path Serialization#228

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Merged
coder137 merged 12 commits intomainfrompath_serialization_improvement
Dec 2, 2022
Merged
Show file tree
Hide file tree
Changes fromall commits
Commits
Show all changes
12 commits
Select commitHold shift + click to select a range
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 9 additions & 111 deletionsbuildcc/lib/target/include/target/custom_generator.h
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -39,38 +39,34 @@ namespace buildcc {

struct UserCustomGeneratorSchema : public internal::CustomGeneratorSchema {
struct UserIdInfo : internal::CustomGeneratorSchema::IdInfo {
fs_unordered_set inputs; // TODO, Remove
GenerateCb generate_cb;
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};

void ConvertToInternal() {
internal_inputs = internal::path_schema_convert(
inputs, internal::Path::CreateExistingPath);
inputs.ComputeHashForAll();
userblob = blob_handler != nullptr ? blob_handler->GetSerializedData()
: std::vector<uint8_t>();
}
};

using UserIdPair = std::pair<const IdKey, UserIdInfo>;
std::unordered_map<IdKey, UserIdInfo> ids;
GenerateCb generate_cb;
std::shared_ptr<CustomBlobHandler> blob_handler{nullptr};
};

void ConvertToInternal() {
for (auto &[id_key, id_info] : ids) {
id_info.internal_inputs = path_schema_convert(
id_info.inputs, internal::Path::CreateExistingPath);
id_info.ConvertToInternal();
auto [_, success] = internal_ids.try_emplace(id_key, id_info);
env::assert_fatal(success, fmt::format("Could not save {}", id_key));
}
}

std::unordered_map<IdKey, UserIdInfo> ids;
};

class CustomGenerator : public internal::BuilderInterface {
public:
CustomGenerator(const std::string &name, const TargetEnv &env)
: name_(name),
env_(env.GetTargetRootDir(), env.GetTargetBuildDir() / name),
serialization_(env_.GetTargetBuildDir() / fmt::format("{}.json", name)),
comparator_(serialization_.GetLoad(), user_) {
serialization_(env_.GetTargetBuildDir() /
fmt::format("{}.json", name)) {
Initialize();
}
virtual ~CustomGenerator() = default;
Expand DownExpand Up@@ -115,100 +111,9 @@ class CustomGenerator : public internal::BuilderInterface {
const fs::path &GetBuildDir() const { return env_.GetTargetBuildDir(); }
const std::string &Get(const std::string &file_identifier) const;

private:
struct Comparator {
Comparator(const internal::CustomGeneratorSchema &loaded,
const UserCustomGeneratorSchema &us)
: loaded_schema_(loaded), current_schema_(us) {}

enum class State {
kRemoved,
kAdded,
kCheckLater,
};

void AddAllIds() {
const auto &curr_ids = current_schema_.ids;
for (const auto &[id, _] : curr_ids) {
id_state_info_.at(State::kAdded).insert(id);
}
}

void CompareIds() {
const auto &prev_ids = loaded_schema_.internal_ids;
const auto &curr_ids = current_schema_.ids;

for (const auto &[prev_id, _] : prev_ids) {
if (curr_ids.find(prev_id) == curr_ids.end()) {
// Id Removed condition, previous id is not present in the current run
id_state_info_.at(State::kRemoved).insert(prev_id);
}
}

for (const auto &[curr_id, _] : curr_ids) {
if (prev_ids.find(curr_id) == prev_ids.end()) {
// Id Added condition
id_state_info_.at(State::kAdded).insert(curr_id);
} else {
// Id Check Later condition
id_state_info_.at(State::kCheckLater).insert(curr_id);
}
}
}

bool IsChanged(const std::string &id) const {
const auto &previous_id_info = loaded_schema_.internal_ids.at(id);
const auto &current_id_info = current_schema_.ids.at(id);

bool changed = internal::CheckPaths(previous_id_info.internal_inputs,
current_id_info.internal_inputs) !=
internal::PathState::kNoChange;
changed = changed || internal::CheckChanged(previous_id_info.outputs,
current_id_info.outputs);
if (!changed && current_id_info.blob_handler != nullptr) {
// We only check blob handler if not changed by inputs/outputs
// Checking blob_handler could be expensive so this optimization is made
// to run only when changed == false
changed = current_id_info.blob_handler->CheckChanged(
previous_id_info.userblob, current_id_info.userblob);
}
return changed;
}

const std::unordered_set<std::string> &GetRemovedIds() const {
return id_state_info_.at(State::kRemoved);
}

const std::unordered_set<std::string> &GetAddedIds() const {
return id_state_info_.at(State::kAdded);
}

const std::unordered_set<std::string> &GetCheckLaterIds() const {
return id_state_info_.at(State::kCheckLater);
}

bool IsIdAdded(const std::string &id) const {
return id_state_info_.at(State::kAdded).count(id) == 1;
}

private:
const internal::CustomGeneratorSchema &loaded_schema_;
const UserCustomGeneratorSchema &current_schema_;
std::unordered_map<State, std::unordered_set<std::string>> id_state_info_{
{State::kRemoved, std::unordered_set<std::string>()},
{State::kAdded, std::unordered_set<std::string>()},
{State::kCheckLater, std::unordered_set<std::string>()},
};
};

private:
void Initialize();

tf::Task CreateTaskRunner(tf::Subflow &subflow, const std::string &id);
void TaskRunner(const std::string &id);

void GenerateTask();
void BuildGenerate();

// Recheck states
void IdRemoved();
Expand All@@ -227,13 +132,6 @@ class CustomGenerator : public internal::BuilderInterface {
// Serialization
UserCustomGeneratorSchema user_;

// Comparator
Comparator comparator_;

std::mutex success_schema_mutex_;
std::unordered_map<std::string, UserCustomGeneratorSchema::UserIdInfo>
success_schema_;

// Internal
env::Command command_;
};
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -23,6 +23,12 @@

namespace buildcc {

/**
* @brief Abstract class for serializing additional data for which rebuilds
* might be triggered i.e data that is not input/output files
* TODO, Add examples here
*
*/
class CustomBlobHandler {
public:
CustomBlobHandler() = default;
Expand DownExpand Up@@ -54,6 +60,48 @@ class CustomBlobHandler {
virtual std::vector<uint8_t> Serialize() const = 0;
};

/**
* @brief Typed Custom Blob handler which automatically performs Serialization
* and Deserialization as long as it is JSON serializable
*
* NOTE: Type data is stored as a reference (to avoid copying large amount of
* data) when constructing TypedCustomBlobHandler
*
* @tparam Type should be JSON serializable (see nlohmann::json compatible
* objects)
*/
template <typename Type>
class TypedCustomBlobHandler : public CustomBlobHandler {
public:
explicit TypedCustomBlobHandler(const Type &data) : data_(data) {}

// serialized_data has already been verified
static Type Deserialize(const std::vector<uint8_t> &serialized_data) {
json j = json::from_msgpack(serialized_data, true, false);
Type deserialized;
j.get_to(deserialized);
return deserialized;
}

private:
const Type &data_;

bool Verify(const std::vector<uint8_t> &serialized_data) const override {
json j = json::from_msgpack(serialized_data, true, false);
return !j.is_discarded();
}

bool IsEqual(const std::vector<uint8_t> &previous,
const std::vector<uint8_t> &current) const override {
return Deserialize(previous) == Deserialize(current);
}

std::vector<uint8_t> Serialize() const override {
json j = data_;
return json::to_msgpack(j);
}
};

} // namespace buildcc

#endif
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -25,19 +25,20 @@ namespace buildcc {

class CustomGeneratorContext {
public:
CustomGeneratorContext(const env::Command &c, const fs_unordered_set &i,
const fs_unordered_set &o,
CustomGeneratorContext(const env::Command &c,
const std::unordered_set<std::string> &i,
const std::unordered_set<std::string> &o,
const std::vector<uint8_t> &ub)
: command(c), inputs(i), outputs(o), userblob(ub) {}

const env::Command &command;
constfs_unordered_set &inputs;
constfs_unordered_set &outputs;
conststd::unordered_set<std::string> &inputs;
conststd::unordered_set<std::string> &outputs;
const std::vector<uint8_t> &userblob;
};

// clang-format off
using GenerateCb = std::function<bool (CustomGeneratorContext &)>;
using GenerateCb = std::function<bool (constCustomGeneratorContext &)>;
// clang-format on

} // namespace buildcc
Expand Down
Loading
[8]ページ先頭
©2009-2025 Movatter.jp