179 emg() : _tables{make_tables_()} {}
184 emg& operator=(
emg&&) noexcept = default;
194 static emg inst{auto_probe_tag{}};
218 if (_model.profiles.empty())
throw std::runtime_error(
"No email data loaded. Call load() first.");
219 const core::rng call{call_seed};
224 const auto it = _model.by_region.find(upper_(*request.
country));
225 if (it == _model.by_region.end())
227 throw std::invalid_argument(
"get_email: unknown country '" + *request.
country +
"'");
233 auto cr = call.split(
"country");
234 ci = request.
weighted ? _model.all_population.pick(cr) : _model.all_uniform.pick(cr);
236 return make_email_(request, _model.countries[ci], call_seed, call);
245 const core::batch_options& options)
264 const core::batch_options& options,
265 std::uint64_t batch_seed)
const
267 return core::run_batch(options, batch_seed,
268 [&](std::uint64_t item_seed) {
return get_email(request, item_seed); });
278 template <
typename Sink>
280 std::uint64_t batch_seed, Sink&& sink)
const
282 core::for_each_batch(options, batch_seed,
283 [&](std::uint64_t item_seed) {
return get_email(request, item_seed); },
284 std::forward<Sink>(sink));
293 template <
typename Sink>
296 for_each_email(request, options, draw_seed_(), std::forward<Sink>(sink));
303 [[nodiscard]]
bool has_data()
const {
return !_model.countries.empty(); }
307 [[nodiscard]] std::size_t
country_count()
const {
return _model.countries.size(); }
312 [[nodiscard]]
const std::vector<std::string>&
countries()
const {
return _model.codes; }
319 return _model.by_region.contains(upper_(country));
331 void load(
const std::filesystem::path& dir)
334 if (!std::filesystem::is_directory(dir, ec) ||
335 !std::filesystem::is_regular_file(dir / file_name_(email_table::countries), ec))
339 apply_(read_directory_(dir), core::layer_mode::replace,
true);
352 [[nodiscard]]
bool load(dataset tier)
354 const std::string tier_dir{core::tier_name(tier)};
355 const auto marker = tier_dir +
"/" + std::string{file_name_(email_table::countries)};
356 const auto root = core::find_resources(library_name_, marker);
358 for (
const auto t : all_tables_)
360 docs.at(index_(t)) = core::load_resource(tier_dir +
"/" + std::string{file_name_(t)},
361 embedded_files_(), root);
363 if (!docs.at(index_(email_table::countries)))
return false;
364 apply_(std::move(docs), core::layer_mode::replace,
true);
379 void extend(
const std::filesystem::path& path, core::layer_mode mode = core::layer_mode::override)
382 if (std::filesystem::is_directory(path, ec))
384 apply_(read_directory_(path), mode,
false);
387 for (
const auto t : all_tables_)
389 if (path.filename() == file_name_(t))
391 (void)
extend(t, core::tsv_document::load(path), mode);
395 throw std::invalid_argument(
"extend: expected one of the email tables (countries.tsv, fold.tsv, "
396 "particles.tsv, profiles.tsv, email_styles.tsv, providers.tsv, words.tsv), got " +
397 path.filename().string());
412 core::layer_mode mode = core::layer_mode::override)
415 docs.at(index_(which)).emplace(std::move(doc));
416 apply_(std::move(docs), mode,
false);
417 return _tables.at(index_(which)).layers().back();
429 core::layer_mode mode = core::layer_mode::override)
431 return extend(which, core::tsv_document::parse(text, std::move(source)), mode);
439 return _tables.at(index_(which)).layers();
449 for (
const auto& t : _tables)
451 any = any || !t.layers().empty();
452 h = core::hash_combine(h, t.data_hash());
462 static constexpr std::string_view library_name_ =
"email-generator";
463 static constexpr std::size_t table_count_ = 7;
464 static constexpr std::array<table, table_count_> all_tables_{
465 table::countries, table::fold, table::particles, table::profiles,
466 table::email_styles, table::providers, table::words};
468 using documents = std::array<std::optional<core::tsv_document>, table_count_>;
469 using tables = std::array<core::layered_table, table_count_>;
471 static constexpr std::size_t index_(table t) {
return static_cast<std::size_t
>(t); }
473 static constexpr std::string_view file_name_(table t)
477 case table::countries:
return "countries.tsv";
478 case table::fold:
return "fold.tsv";
479 case table::particles:
return "particles.tsv";
480 case table::profiles:
return "profiles.tsv";
481 case table::email_styles:
return "email_styles.tsv";
482 case table::providers:
return "providers.tsv";
483 case table::words:
return "words.tsv";
488 static core::table_schema schema_(table t)
492 case table::countries:
493 return {.key = {
"region"}, .required = {}, .optional = {
"population"}};
494 case table::fold:
return {.key = {
"from"}, .required = {
"to"}, .optional = {}};
495 case table::particles:
return {.key = {
"particle"}, .required = {}, .optional = {}};
496 case table::profiles:
return {.key = {
"region"}, .required = {
"profile"}, .optional = {}};
497 case table::email_styles:
return {.key = {
"profile",
"pattern"}, .required = {
"weight"}, .optional = {}};
498 case table::providers:
return {.key = {
"domain",
"country"}, .required = {
"kind",
"weight"}, .optional = {}};
499 case table::words:
return {.key = {
"list",
"word"}, .required = {}, .optional = {}};
504 static tables make_tables_()
506 return {core::layered_table{schema_(table::countries)},
507 core::layered_table{schema_(table::fold)}, core::layered_table{schema_(table::particles)},
508 core::layered_table{schema_(table::profiles)}, core::layered_table{schema_(table::email_styles)},
509 core::layered_table{schema_(table::providers)}, core::layered_table{schema_(table::words)}};
514 static constexpr std::size_t none_ =
static_cast<std::size_t
>(-1);
516 enum class tok : std::uint8_t { lit, f, l, fi, li, n, y2, y4, r, r2, adj, animal, jp };
526 std::vector<token> tokens;
528 bool needs_given{
false};
529 bool needs_family{
false};
530 bool needs_nickname{
false};
531 bool needs_year{
false};
532 std::array<bool, 3> needs_words{};
545 double population{1.0};
546 std::size_t profile{none_};
548 core::weighted_table<std::size_t> provider_weights;
554 std::vector<std::string> codes;
555 std::unordered_map<std::string, std::size_t> by_region;
556 std::vector<std::vector<style_data>>
profiles;
557 std::unordered_map<std::string, std::size_t> profile_index;
558 std::unordered_map<std::uint32_t, std::string>
fold;
559 std::unordered_set<std::string>
particles;
560 std::array<std::vector<std::string>, 3>
words;
561 core::weighted_table<std::size_t> all_population;
562 core::weighted_table<std::size_t> all_uniform;
567 core::rng _engine{core::rng::from_entropy()};
569 struct auto_probe_tag {};
571 explicit emg(auto_probe_tag ) : _tables{make_tables_()}
573 if (!
load(dataset::lite)) (void)
load(dataset::full);
578 static std::span<const core::embedded_file> embedded_files_()
580#ifdef DASMIG_EMAIL_EMBED
581 return embedded::emailgen::files;
587 std::uint64_t draw_seed_() {
return _engine(); }
589 static std::string upper_(std::string_view s)
594 if (c >=
'a' && c <=
'z') c =
static_cast<char>(c -
'a' +
'A');
602 static char fold_ascii_(
char c,
bool keep_marks)
604 if (c >=
'A' && c <=
'Z')
return static_cast<char>(c -
'A' +
'a');
605 if ((c >=
'a' && c <=
'z') || (c >=
'0' && c <=
'9'))
return c;
606 if (keep_marks)
return is_mark_(c) ? c :
'\0';
612 [[nodiscard]] std::string fold_(std::string_view text,
bool keep_marks)
const
615 for (std::size_t i = 0; i < text.size();)
617 std::size_t used = 1;
618 const auto cp = decode_(text, i, used);
620 if (cp != 0 && cp < 0x80)
622 const char c = fold_ascii_(
static_cast<char>(cp), keep_marks);
623 if (c !=
'\0') out += c;
626 const auto f = cp == 0 ? _model.fold.end() : _model.fold.find(cp);
627 if (f != _model.fold.end())
631 else if (!keep_marks)
639 static std::vector<std::string> split_words_(
const std::string& text)
641 std::vector<std::string> out;
643 for (
const char c : text)
647 if (!cur.empty()) out.push_back(std::move(cur));
655 if (!cur.empty()) out.push_back(std::move(cur));
660 [[nodiscard]] std::string name_part_(std::string_view text,
bool family)
const
662 auto parts = split_words_(fold_(text,
false));
665 std::erase_if(parts, [&](
const std::string& p) {
return _model.particles.contains(p); });
667 return parts.empty() ? std::string{} : parts.front();
670 static bool is_mark_(
char c) {
return c ==
'.' || c ==
'_' || c ==
'-'; }
673 static std::string tidy_local_(std::string_view text)
676 for (
const char c : text)
678 if (is_mark_(c) && (out.empty() || is_mark_(out.back())))
continue;
680 if (out.size() == 64)
break;
682 while (!out.empty() && is_mark_(out.back())) out.pop_back();
686 static void append_digits_(std::string& out, core::rng& r,
int count)
688 for (
int i = 0; i < count; ++i) out += static_cast<char>(
'0' + core::uniform_int(r, 0, 9));
700 [[nodiscard]] name_inputs read_names_(
const email_request& request)
const
703 n.given = name_part_(request.given,
false);
704 n.family = name_part_(request.family,
true);
705 n.nick = tidy_local_(fold_(request.nickname,
true));
706 n.year_ok = request.birth_year && *request.birth_year >= 1000 && *request.birth_year <= 9999;
711 [[nodiscard]]
bool style_fits_(
const style_data& st,
const name_inputs& n)
const
713 const bool names = (!st.needs_given || !n.given.empty()) && (!st.needs_family || !n.family.empty()) &&
714 (!st.needs_nickname || !n.nick.empty()) && (!st.needs_year || n.year_ok);
716 for (std::size_t list = 0; list < st.needs_words.size(); ++list)
718 words =
words && (!st.needs_words.at(list) || !_model.words.at(list).empty());
720 return names &&
words && st.weight > 0.0;
724 [[nodiscard]]
const style_data& pick_style_(
const std::vector<style_data>& styles,
const name_inputs& n,
725 bool weighted,
const core::rng& call,
726 const std::string& region)
const
728 const auto share = [&](
const style_data& st) {
return weighted ? st.weight : 1.0; };
730 const style_data* last =
nullptr;
731 for (
const auto& st : styles)
733 if (!style_fits_(st, n))
continue;
737 if (last ==
nullptr)
throw std::runtime_error(
"get_email: no style fits the request in the profile of " + region);
738 auto sr = call.split(
"style");
739 double x = core::uniform01(sr) * total;
740 for (
const auto& st : styles)
742 if (!style_fits_(st, n))
continue;
744 if (x < 0.0)
return st;
750 [[nodiscard]] std::string render_local_(
const style_data& style,
const name_inputs& n,
const email_request& request,
751 const core::rng& call)
const
753 auto lr = call.split(
"local");
754 auto nr = call.split(
"number");
755 const auto initial = [](
const std::string& text) {
return text.substr(0, 1); };
757 for (
const auto& t : style.tokens)
761 case tok::lit: local += t.text;
break;
762 case tok::f: local += n.given;
break;
763 case tok::l: local += n.family;
break;
764 case tok::fi: local += initial(n.given);
break;
765 case tok::li: local += initial(n.family);
break;
766 case tok::n: local += n.nick;
break;
767 case tok::y2: local += std::format(
"{:02}", *request.birth_year % 100);
break;
768 case tok::y4: local += std::format(
"{:04}", *request.birth_year);
break;
769 case tok::r: append_digits_(local, nr, core::uniform_int(nr, 1, 3));
break;
770 case tok::r2: append_digits_(local, nr, 2);
break;
771 case tok::adj: local += pick_word_(0, lr);
break;
772 case tok::animal: local += pick_word_(1, lr);
break;
773 case tok::jp: local += pick_word_(2, lr);
break;
776 local = tidy_local_(local);
777 return local.empty() ?
"user" : local;
781 static void pick_domain_(email& v,
const email_request& request,
const country_data& country,
const core::rng& call)
783 auto dr = call.split(
"domain");
786 static constexpr std::array<std::string_view, 3> reserved{
"example.com",
"example.net",
"example.org"};
787 v.domain = std::string{reserved.at(core::uniform_int<std::size_t>(dr, 0, reserved.size() - 1))};
791 if (country.providers.empty() || (request.weighted && country.provider_weights.empty()))
793 throw std::runtime_error(
"get_email: no provider domain for " + country.region);
795 const auto& p = country.providers[request.weighted ? country.provider_weights.pick(dr)
796 : core::uniform_int<std::size_t>(dr, 0, country.providers.size() - 1)];
801 [[nodiscard]] email make_email_(
const email_request& request,
const country_data& country, std::uint64_t call_seed,
802 const core::rng& call)
const
804 const std::size_t pi = country.profile != none_ ? country.profile : default_profile_();
805 if (pi == none_ || _model.profiles[pi].empty())
throw std::runtime_error(
"No email styles loaded for " + country.region);
806 const auto names = read_names_(request);
807 const auto& style = pick_style_(_model.profiles[pi], names, request.weighted, call, country.region);
810 v._signature = {.seed = call_seed, .data =
data_hash()};
811 v.country_code = country.region;
812 v.local = render_local_(style, names, request, call);
813 pick_domain_(v, request, country, call);
817 [[nodiscard]] std::size_t default_profile_()
const
819 const auto it = _model.profile_index.find(
"default");
820 return it == _model.profile_index.end() ? none_ : it->second;
823 const std::string& pick_word_(std::size_t list, core::rng& r)
const
825 const auto& w = _model.words.at(list);
826 return w[core::uniform_int<std::size_t>(r, 0, w.size() - 1)];
829 static documents read_directory_(
const std::filesystem::path& dir)
832 for (
const auto t : all_tables_)
834 const auto file = dir / file_name_(t);
836 if (std::filesystem::is_regular_file(file, ec))
838 docs.at(index_(t)) = core::tsv_document::load(file);
846 void apply_(documents docs, core::layer_mode mode,
bool reset)
848 tables next = _tables;
849 for (
const auto t : all_tables_)
851 auto& doc = docs.at(index_(t));
852 auto& tbl = next.at(index_(t));
855 if (reset) tbl.clear();
858 (void)tbl.extend(std::move(*doc), mode, [t](
const core::tsv_row& row) { validate_(t, row); });
860 auto built = build_model_(next);
861 _tables = std::move(next);
862 _model = std::move(built);
867 static void require_region_(
const core::tsv_row& row)
869 if (row[
"region"].size() != 2)
throw row.error(
"region",
"must be an ISO 3166-1 alpha-2 code");
872 static void require_chars_(
const core::tsv_row& row, std::string_view column,
bool dots, std::string_view message)
874 for (
const char c : row[column])
876 const bool ok = (c >=
'a' && c <=
'z') || (c >=
'0' && c <=
'9') || (dots && (c ==
'.' || c ==
'-'));
877 if (!ok)
throw row.error(column, std::string{message});
881 static void validate_provider_(
const core::tsv_row& row)
883 const auto domain = row[
"domain"];
884 if (!domain.contains(
'.') || domain.front() ==
'.' || domain.back() ==
'.')
886 throw row.error(
"domain",
"must be a domain name, got '" + std::string{domain} +
"'");
888 require_chars_(row,
"domain",
true,
"must be lowercase letters, digits, dots and hyphens");
889 if (row[
"country"] !=
"*" && row[
"country"].size() != 2)
891 throw row.error(
"country",
"must be '*' or an ISO 3166-1 alpha-2 code");
893 if (row[
"kind"] !=
"free" && row[
"kind"] !=
"isp")
throw row.error(
"kind",
"must be 'free' or 'isp'");
894 if (!(row.get<
double>(
"weight") >= 0.0))
throw row.error(
"weight",
"must not be negative");
897 static void validate_(table t,
const core::tsv_row& row)
901 case table::countries:
902 require_region_(row);
903 if (row.get_or<
double>(
"population", 1.0) < 0.0)
throw row.error(
"population",
"must not be negative");
906 if (decode_one_(row[
"from"]) == 0)
throw row.error(
"from",
"must be one character");
907 for (
const char c : row[
"to"])
909 if (c <
'a' || c >
'z')
throw row.error(
"to",
"must be lowercase ASCII letters");
912 case table::particles:
913 if (row[
"particle"].empty())
throw row.error(
"particle",
"must not be empty");
915 case table::profiles:
916 require_region_(row);
917 if (row[
"profile"].empty())
throw row.error(
"profile",
"must not be empty");
919 case table::email_styles:
920 if (row[
"profile"].empty())
throw row.error(
"profile",
"must not be empty");
921 (void)parse_style_(row);
923 case table::providers:
924 validate_provider_(row);
927 if (word_list_(row) == none_)
throw row.error(
"list",
"must be 'adj', 'animal' or 'jp'");
928 if (row[
"word"].empty())
throw row.error(
"word",
"must not be empty");
929 require_chars_(row,
"word",
false,
"must be lowercase letters and digits");
934 static std::size_t word_list_(
const core::tsv_row& row)
936 const auto l = row[
"list"];
937 if (l ==
"adj")
return 0;
938 if (l ==
"animal")
return 1;
939 if (l ==
"jp")
return 2;
944 static std::uint32_t decode_one_(std::string_view text)
946 if (text.empty())
return 0;
947 std::size_t used = 0;
948 const auto cp = decode_(text, 0, used);
949 return used == text.size() ? cp : 0;
953 static std::uint32_t decode_(std::string_view text, std::size_t i, std::size_t& used)
955 const auto c =
static_cast<unsigned char>(text[i]);
957 if (c < 0x80)
return c;
959 std::uint32_t cp = 0;
960 if ((c & 0xE0U) == 0xC0U)
965 else if ((c & 0xF0U) == 0xE0U)
970 else if ((c & 0xF8U) == 0xF0U)
979 if (i + n > text.size())
return 0;
980 for (std::size_t k = 1; k < n; ++k)
982 const auto b =
static_cast<unsigned char>(text[i + k]);
983 if ((b & 0xC0U) != 0x80U)
return 0;
984 cp = (cp << 6U) | (b & 0x3FU);
991 static bool note_part_(std::string_view name, style_data& st, tok& kind)
993 using part = std::pair<std::string_view, tok>;
994 static constexpr std::array<part, 12> parts{{{
"f", tok::f}, {
"l", tok::l}, {
"fi", tok::fi}, {
"li", tok::li},
995 {
"n", tok::n}, {
"y2", tok::y2}, {
"y4", tok::y4}, {
"r", tok::r},
996 {
"r2", tok::r2}, {
"adj", tok::adj}, {
"animal", tok::animal},
998 const auto it = std::ranges::find(parts, name, &part::first);
999 if (it == parts.end())
return false;
1004 case tok::fi: st.needs_given =
true;
break;
1006 case tok::li: st.needs_family =
true;
break;
1007 case tok::n: st.needs_nickname =
true;
break;
1009 case tok::y4: st.needs_year =
true;
break;
1010 case tok::adj: st.needs_words[0] =
true;
break;
1011 case tok::animal: st.needs_words[1] =
true;
break;
1012 case tok::jp: st.needs_words[2] =
true;
break;
1015 case tok::r2:
break;
1020 static style_data parse_style_(
const core::tsv_row& row)
1023 const auto text = row[
"pattern"];
1024 st.weight = row.get<
double>(
"weight");
1025 if (!(st.weight >= 0.0))
throw row.error(
"weight",
"must not be negative");
1027 const auto flush = [&] {
1028 if (!lit.empty()) st.tokens.push_back({.kind = tok::lit, .text = std::move(lit)});
1031 for (std::size_t i = 0; i < text.size(); ++i)
1033 const char c = text[i];
1036 const bool literal = (c >=
'a' && c <=
'z') || (c >=
'0' && c <=
'9') || c ==
'.' || c ==
'_' || c ==
'-';
1037 if (!literal)
throw row.error(
"pattern", std::string{
"unexpected '"} + c +
"' in '" + std::string{text} +
"'");
1041 const auto close = text.find(
'}', i);
1042 if (close == std::string_view::npos)
throw row.error(
"pattern",
"unclosed '{'");
1043 const auto name = text.substr(i + 1, close - i - 1);
1046 if (!note_part_(name, st, kind))
throw row.error(
"pattern",
"unknown part '{" + std::string{name} +
"}'");
1047 st.tokens.push_back({.kind = kind, .text = {}});
1051 if (st.tokens.empty())
throw row.error(
"pattern",
"must not be empty");
1055 static void read_countries_(model& m,
const tables& tbls)
1057 for (
const auto& row : tbls.at(index_(table::
countries)).rows())
1060 c.region = upper_(row[
"region"]);
1061 c.population = std::max(row.get_or<
double>(
"population", 1.0), 1.0);
1062 m.by_region.emplace(c.region, m.countries.size());
1063 m.codes.push_back(c.region);
1064 m.countries.push_back(std::move(c));
1066 for (std::size_t i = 0; i < m.countries.size(); ++i)
1068 m.all_population.add(i, m.countries[i].population);
1069 m.all_uniform.add(i, 1.0);
1073 static void read_text_tables_(model& m,
const tables& tbls)
1075 for (
const auto& row : tbls.at(index_(table::
fold)).rows())
1077 m.fold[decode_one_(row[
"from"])] = std::string{row[
"to"]};
1079 for (
const auto& row : tbls.at(index_(table::
particles)).rows()) m.
particles.insert(std::string{row[
"particle"]});
1080 for (
const auto& row : tbls.at(index_(table::
words)).rows())
1082 m.words.at(word_list_(row)).emplace_back(row[
"word"]);
1086 static void read_styles_(model& m,
const tables& tbls)
1088 for (
const auto& row : tbls.at(index_(table::
email_styles)).rows())
1090 const std::string name{row[
"profile"]};
1091 auto it = m.profile_index.find(name);
1092 if (it == m.profile_index.end())
1094 it = m.profile_index.emplace(name, m.profiles.size()).first;
1095 m.profiles.emplace_back();
1097 m.profiles[it->second].push_back(parse_style_(row));
1099 for (
const auto& row : tbls.at(index_(table::
profiles)).rows())
1101 const auto it = m.by_region.find(upper_(row[
"region"]));
1102 if (it == m.by_region.end())
1104 throw row.error(
"region",
"no country '" + std::string{row[
"region"]} +
"' in countries.tsv");
1106 const auto pi = m.profile_index.find(std::string{row[
"profile"]});
1107 if (pi == m.profile_index.end())
1109 throw row.error(
"profile",
"no style for profile '" + std::string{row[
"profile"]} +
"' in email_styles.tsv");
1111 m.countries[it->second].profile = pi->second;
1117 static void read_providers_(model& m,
const tables& tbls)
1119 std::unordered_map<std::string, std::vector<provider_data>> by_country;
1120 for (
const auto& row : tbls.at(index_(table::
providers)).rows())
1122 by_country[upper_(row[
"country"])].push_back(
1123 {.domain = std::string{row[
"domain"]}, .kind = std::string{row[
"kind"]}, .weight = row.get<
double>(
"weight")});
1125 const auto global = by_country.find(
"*");
1128 const auto own = by_country.find(c.region);
1129 if (own != by_country.end()) c.providers = own->second;
1130 if (global != by_country.end())
1132 for (
const auto& g : global->second)
1134 const bool replaced = own != by_country.end() &&
1135 std::ranges::any_of(own->second, [&](
const provider_data& d) { return d.domain == g.domain; });
1136 if (!replaced) c.providers.push_back(g);
1139 std::erase_if(c.providers, [](
const provider_data& d) { return !(d.weight > 0.0); });
1140 for (std::size_t k = 0; k < c.providers.size(); ++k) c.provider_weights.add(k, c.providers[k].weight);
1144 static model build_model_(
const tables& tbls)
1147 read_countries_(m, tbls);
1148 read_text_tables_(m, tbls);
1149 read_styles_(m, tbls);
1150 read_providers_(m, tbls);
1154 void require_data_()
const
1156 if (_model.countries.empty())
1158 throw std::runtime_error(
"No email data loaded. Call load() first.");