diff --git a/.gitattributes b/.gitattributes index 8e366e85..03cca427 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,3 +6,8 @@ model/quantized/tokenizer_config.json filter= diff= merge= text model/quantized/special_tokens_map.json filter= diff= merge= text model/quantized/added_tokens.json filter= diff= merge= text *.onnx filter=lfs diff=lfs merge=lfs -text + +# PII generator word pools are embedded via //go:embed and parsed line by line. +# Pin them to LF so the bytes embedded into the binary are identical regardless +# of the build platform's checkout settings. +src/backend/pii/generators/data/*.txt text eol=lf diff --git a/src/backend/pii/generators/data/cities.txt b/src/backend/pii/generators/data/cities.txt new file mode 100644 index 00000000..a3b4da53 --- /dev/null +++ b/src/backend/pii/generators/data/cities.txt @@ -0,0 +1,74 @@ +# Dummy city pool for CityGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +# US cities +Springfield +Riverside +Greenville +Fairview +Madison +Georgetown +Salem +Arlington +Franklin +Clinton +Bristol +Chester +Dayton +Kingston +Newport +Oakland +Plymouth +Burlington +Manchester +Lexington +Milton +Ashland +Clayton +Auburn +Fairfield +Marion +Oxford +Dover + +# Canadian cities +Toronto +Vancouver +Calgary +Ottawa +Edmonton +Winnipeg +Halifax +Victoria +Regina +Saskatoon +Hamilton +Kitchener +London +Windsor +Burnaby +Guelph + +# UK cities +Birmingham +Edinburgh +Liverpool +Leeds +Sheffield +Newcastle +Nottingham +Southampton +Portsmouth +Oxford +Cambridge +York +Bath +Brighton +Cardiff +Belfast +Glasgow +Aberdeen +Dundee +Swansea +Norwich +Exeter diff --git a/src/backend/pii/generators/data/company_prefixes.txt b/src/backend/pii/generators/data/company_prefixes.txt new file mode 100644 index 00000000..7ea6c8b9 --- /dev/null +++ b/src/backend/pii/generators/data/company_prefixes.txt @@ -0,0 +1,47 @@ +# Dummy company-name prefix pool for CompanyNameGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +Acme +Global +United +Pacific +Atlantic +Northern +Summit +Horizon +Apex +Vanguard +Pinnacle +Premier +Elite +Prime +Sterling +Meridian +Coastal +Central +National +Continental +Metro +Allied +Dynamic +Synergy +Fusion +Vertex +Quantum +Nova +Titan +Omega +Pioneer +Frontier +Legacy +Heritage +Keystone +Benchmark +Catalyst +Spectrum +Nexus +Compass +Evergreen +Cornerstone +Beacon +Cascade diff --git a/src/backend/pii/generators/data/company_suffixes.txt b/src/backend/pii/generators/data/company_suffixes.txt new file mode 100644 index 00000000..91bdb8c6 --- /dev/null +++ b/src/backend/pii/generators/data/company_suffixes.txt @@ -0,0 +1,39 @@ +# Dummy company-name suffix pool for CompanyNameGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +# US/International +Inc +LLC +Corp +Industries +Solutions +Group +Holdings +Partners +Enterprises +Co +Technologies +Systems +Services +Consulting +Associates +International +Worldwide +Labs +Ventures + +# UK +Ltd +PLC +Limited + +# German +GmbH +AG + +# French/Spanish +SA +SL + +# Australian +Pty Ltd diff --git a/src/backend/pii/generators/data/countries.txt b/src/backend/pii/generators/data/countries.txt new file mode 100644 index 00000000..6f1f7810 --- /dev/null +++ b/src/backend/pii/generators/data/countries.txt @@ -0,0 +1,78 @@ +# Dummy country pool for CountryGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +# North America +United States +Canada +Mexico + +# Europe +United Kingdom +Germany +France +Italy +Spain +Netherlands +Belgium +Switzerland +Austria +Sweden +Norway +Denmark +Finland +Ireland +Poland +Portugal +Greece +Czech Republic +Hungary +Romania +Iceland +Croatia + +# Asia +Japan +China +South Korea +India +Singapore +Thailand +Vietnam +Indonesia +Malaysia +Philippines +Taiwan +Hong Kong +Nepal +Sri Lanka + +# Oceania +Australia +New Zealand + +# South America +Brazil +Argentina +Chile +Colombia +Peru +Uruguay +Ecuador + +# Africa +South Africa +Nigeria +Kenya +Egypt +Morocco +Ghana +Tanzania +Senegal + +# Middle East +United Arab Emirates +Israel +Saudi Arabia +Turkey +Jordan +Qatar diff --git a/src/backend/pii/generators/data/email_firstnames.txt b/src/backend/pii/generators/data/email_firstnames.txt new file mode 100644 index 00000000..1342860f --- /dev/null +++ b/src/backend/pii/generators/data/email_firstnames.txt @@ -0,0 +1,78 @@ +# Dummy email local-part first names for EmailGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. +# Must stay lowercase single ASCII words (no spaces, apostrophes, or accents): +# the local part is formatted as first.last and is checked against [a-z]+. + +# Western +jane +john +alex +sam +taylor +casey +jordan +riley +michael +sarah +david +emily +james +emma +robert +olivia +joseph +laura +morgan +avery + +# Asian +wei +mei +hiroshi +yuki +jin +min +raj +priya +ravi +deepa + +# African +amara +kofi +zara +kwame +nia +jelani +thabo +lerato + +# Middle Eastern +yusuf +fatima +omar +layla +ali +nadia +samir +rania + +# Latin American +carlos +maria +diego +sofia +miguel +lucia +javier +daniela + +# Eastern European +dmitri +anna +ivan +katya +alexei +elena +boris +zofia diff --git a/src/backend/pii/generators/data/email_surnames.txt b/src/backend/pii/generators/data/email_surnames.txt new file mode 100644 index 00000000..87e3a3e3 --- /dev/null +++ b/src/backend/pii/generators/data/email_surnames.txt @@ -0,0 +1,80 @@ +# Dummy email local-part surnames for EmailGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. +# Must stay lowercase single ASCII words (no spaces, apostrophes, or accents): +# the local part is formatted as first.last and is checked against [a-z]+. + +# Western +doe +smith +johnson +brown +davis +wilson +moore +taylor +anderson +thomas +jackson +white +harris +martin +thompson +clark +lewis +walker +young + +# Asian +chen +wang +kim +nguyen +tanaka +yamamoto +patel +singh +kumar +sharma + +# African +okonkwo +diallo +mensah +osei +abebe +adeyemi +owusu + +# Middle Eastern +mohammed +ahmed +hassan +khan +ali +ibrahim +malik + +# Latin American +garcia +rodriguez +martinez +lopez +gonzalez +hernandez +perez +torres + +# Eastern European +ivanov +petrov +kowalski +novak +horvat +popov + +# Celtic +obrien +murphy +kelly +sullivan +walsh diff --git a/src/backend/pii/generators/data/firstnames.txt b/src/backend/pii/generators/data/firstnames.txt new file mode 100644 index 00000000..56e0c305 --- /dev/null +++ b/src/backend/pii/generators/data/firstnames.txt @@ -0,0 +1,145 @@ +# Dummy first-name pool for FirstNameGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. +# Grouped by rough region only for readability; order does not affect output. + +# Western +John +Jane +Michael +Sarah +David +Emily +James +Emma +Robert +Olivia +William +Elizabeth +Richard +Jennifer +Thomas +Jessica +Charles +Amanda +Christopher +Ashley +Daniel +Stephanie +Matthew +Nicole +Anthony +Melissa +Joseph +Margaret +Andrew +Laura +Benjamin +Hannah +Samuel +Grace +Nathan +Rebecca + +# Asian +Wei +Mei +Hiroshi +Yuki +Jin +Min +Raj +Priya +Kenji +Sakura +Chen +Li +Aiko +Takeshi +Ananya +Arjun +Haruto +Yui +Ravi +Deepa +Jae +Soo + +# African +Amara +Kofi +Zara +Kwame +Nia +Jelani +Amina +Chioma +Oluwaseun +Aisha +Tariq +Imani +Sekou +Adaeze +Kwabena +Ama +Thabo +Lerato + +# Middle Eastern +Yusuf +Fatima +Omar +Layla +Ali +Nadia +Hassan +Mariam +Khalid +Zahra +Ahmed +Leila +Ibrahim +Yasmin +Samir +Rania +Karim +Salma + +# Latin American +Carlos +Maria +Diego +Sofia +Miguel +Lucia +Alejandro +Valentina +Fernando +Camila +Ricardo +Isabella +Andres +Gabriela +Javier +Daniela +Mateo +Ximena + +# Eastern European +Dmitri +Anna +Ivan +Katya +Alexei +Elena +Nikolai +Olga +Sergei +Natasha +Vladimir +Irina +Mikhail +Tatiana +Boris +Zofia +Pavel +Ludmila diff --git a/src/backend/pii/generators/data/states.txt b/src/backend/pii/generators/data/states.txt new file mode 100644 index 00000000..5193a319 --- /dev/null +++ b/src/backend/pii/generators/data/states.txt @@ -0,0 +1,90 @@ +# Dummy state / region pool for StateGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +# US States +Alabama +Alaska +Arizona +Arkansas +California +Colorado +Connecticut +Delaware +Florida +Georgia +Hawaii +Idaho +Illinois +Indiana +Iowa +Kansas +Kentucky +Louisiana +Maine +Maryland +Massachusetts +Michigan +Minnesota +Mississippi +Missouri +Montana +Nebraska +Nevada +New Hampshire +New Jersey +New Mexico +New York +North Carolina +North Dakota +Ohio +Oklahoma +Oregon +Pennsylvania +Rhode Island +South Carolina +South Dakota +Tennessee +Texas +Utah +Vermont +Virginia +Washington +West Virginia +Wisconsin +Wyoming + +# Canadian Provinces +Ontario +Quebec +British Columbia +Alberta +Manitoba +Saskatchewan +Nova Scotia +New Brunswick +Newfoundland and Labrador +Prince Edward Island + +# UK Regions/Nations +England +Scotland +Wales +Northern Ireland + +# UK Counties +Yorkshire +Kent +Essex +Hampshire +Surrey +Lancashire +Devon +Cornwall +Oxfordshire +Cambridgeshire +Berkshire +Somerset +Dorset +Wiltshire +Norfolk +Suffolk diff --git a/src/backend/pii/generators/data/streets.txt b/src/backend/pii/generators/data/streets.txt new file mode 100644 index 00000000..aab35587 --- /dev/null +++ b/src/backend/pii/generators/data/streets.txt @@ -0,0 +1,54 @@ +# Dummy street-name pool for StreetGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +# US style streets +Main St +Oak Ave +Maple Dr +Park Blvd +Elm Street +Pine Road +Cedar Lane +Washington St +Broadway +Market St +Church St +Mill Road +School Lane +Lake Ave +River Road +Highland Ave +Forest Dr +Valley Road +Sunset Blvd +Spring St +Garden Way +Lincoln Ave +Jefferson St +Franklin Blvd +Madison Ave +Monroe Dr +Adams St +Willow Way +Birch Lane +Hillcrest Dr + +# UK/Canada style streets +High Street +Station Road +Church Lane +Victoria Road +Queens Road +King Street +Manor Road +Park Lane +The Crescent +Green Lane +Mill Lane +New Road +Chapel Street +West End +North Terrace +Kingsway +Meadow Lane +Orchard Close diff --git a/src/backend/pii/generators/data/surnames.txt b/src/backend/pii/generators/data/surnames.txt new file mode 100644 index 00000000..0394a478 --- /dev/null +++ b/src/backend/pii/generators/data/surnames.txt @@ -0,0 +1,156 @@ +# Dummy surname pool for SurnameGenerator. +# One value per line. Blank lines and lines starting with '#' are ignored. + +# Western +Smith +Johnson +Williams +Brown +Jones +Garcia +Miller +Davis +Martinez +Wilson +Anderson +Taylor +Thomas +Moore +Jackson +Martin +Lee +Thompson +White +Harris +Clark +Lewis +Robinson +Walker +Hall +Young +King +Wright +Hill +Scott +Green +Baker +Adams +Nelson +Carter +Mitchell + +# Asian +Chen +Wang +Li +Zhang +Liu +Kim +Park +Choi +Nguyen +Tran +Tanaka +Yamamoto +Suzuki +Watanabe +Patel +Singh +Sharma +Kumar +Yang +Huang +Sato +Ito + +# African +Okonkwo +Diallo +Mensah +Osei +Abebe +Adeyemi +Nkosi +Mbeki +Kamara +Toure +Dlamini +Ndlovu +Owusu +Achebe +Baldé +Mwangi + +# Middle Eastern +Mohammed +Ahmed +Hassan +Khan +Ali +Ibrahim +Hussein +Malik +Nazari +Hosseini +Rahman +Begum +Saleh +Karimi +Farah +Aziz + +# Latin American +Rodriguez +Lopez +Gonzalez +Hernandez +Perez +Sanchez +Ramirez +Torres +Flores +Rivera +Gomez +Diaz +Reyes +Morales +Cruz +Ortiz +Castillo +Vargas + +# Eastern European +Ivanov +Petrov +Kowalski +Novak +Horvat +Popov +Volkov +Kozlov +Nowak +Kovalenko +Bondarenko +Shevchenko +Sokolov +Wojcik +Kucera +Marek + +# Celtic +O'Brien +Murphy +Kelly +Sullivan +O'Connor +Walsh +Ryan +Byrne +MacDonald +Campbell +Stewart +Murray +Fraser +MacLeod +Doyle +Gallagher diff --git a/src/backend/pii/generators/pii_generators.go b/src/backend/pii/generators/pii_generators.go index 08ab79e1..5154d729 100644 --- a/src/backend/pii/generators/pii_generators.go +++ b/src/backend/pii/generators/pii_generators.go @@ -29,43 +29,11 @@ func pickExcluding(rng *rand.Rand, choices []string, original string) string { // EmailGenerator generates dummy email addresses func EmailGenerator(rng *rand.Rand, original string) string { - firstNames := []string{ - // Western names - "jane", "john", "alex", "sam", "taylor", "casey", "jordan", "riley", - "michael", "sarah", "david", "emily", "james", "emma", "robert", "olivia", - // Asian names - "wei", "mei", "hiroshi", "yuki", "jin", "min", "raj", "priya", - // African names - "amara", "kofi", "zara", "kwame", "nia", "jelani", - // Middle Eastern names - "yusuf", "fatima", "omar", "layla", "ali", "nadia", - // Latin American names - "carlos", "maria", "diego", "sofia", "miguel", "lucia", - // Eastern European names - "dmitri", "anna", "ivan", "katya", "alexei", "elena", - } - lastNames := []string{ - // Western surnames - "doe", "smith", "johnson", "brown", "davis", "wilson", "moore", "taylor", - "anderson", "thomas", "jackson", "white", "harris", "martin", "thompson", - // Asian surnames - "chen", "wang", "kim", "nguyen", "tanaka", "yamamoto", "patel", "singh", - // African surnames - "okonkwo", "diallo", "mensah", "osei", "abebe", - // Middle Eastern surnames - "mohammed", "ahmed", "hassan", "khan", "ali", - // Latin American surnames - "garcia", "rodriguez", "martinez", "lopez", "gonzalez", "hernandez", - // Eastern European surnames - "ivanov", "petrov", "kowalski", "novak", "horvat", - // Celtic surnames - "obrien", "murphy", "kelly", "sullivan", - } // RFC 2606 / RFC 6761 reserved domains only domains := []string{"example.com", "example.org", "example.net", "test.com", "test.org", "test.net", "invalid.com", "invalid.org"} - firstName := firstNames[rng.Intn(len(firstNames))] - lastName := lastNames[rng.Intn(len(lastNames))] + firstName := emailFirstNames[rng.Intn(len(emailFirstNames))] + lastName := emailSurnames[rng.Intn(len(emailSurnames))] domain := domains[rng.Intn(len(domains))] return fmt.Sprintf("%s.%s@%s", firstName, lastName, domain) @@ -181,18 +149,7 @@ func DateOfBirthGenerator(rng *rand.Rand, original string) string { // StreetGenerator generates dummy street addresses func StreetGenerator(rng *rand.Rand, original string) string { - streetNames := []string{ - // US style streets - "Main St", "Oak Ave", "Maple Dr", "Park Blvd", "Elm Street", "Pine Road", "Cedar Lane", "Washington St", - "Broadway", "Market St", "Church St", "Mill Road", "School Lane", "Lake Ave", "River Road", - "Highland Ave", "Forest Dr", "Valley Road", "Sunset Blvd", "Spring St", "Garden Way", - "Lincoln Ave", "Jefferson St", "Franklin Blvd", "Madison Ave", "Monroe Dr", - // UK/Canada style streets - "High Street", "Station Road", "Church Lane", "Victoria Road", "Queens Road", - "King Street", "Manor Road", "Park Lane", "The Crescent", "Green Lane", - "Mill Lane", "New Road", "Chapel Street", "West End", "North Terrace", - } - return pickExcluding(rng, streetNames, original) + return pickExcluding(rng, streets, original) } // ZipCodeGenerator generates dummy zip codes @@ -211,19 +168,6 @@ func ZipCodeGenerator(rng *rand.Rand, original string) string { // CityGenerator generates dummy city names func CityGenerator(rng *rand.Rand, original string) string { - cities := []string{ - // US cities - "Springfield", "Riverside", "Greenville", "Fairview", "Madison", "Georgetown", "Salem", "Arlington", - "Franklin", "Clinton", "Bristol", "Chester", "Dayton", "Kingston", "Newport", "Oakland", - "Plymouth", "Burlington", "Manchester", "Lexington", "Milton", "Ashland", "Clayton", - // Canadian cities - "Toronto", "Vancouver", "Calgary", "Ottawa", "Edmonton", "Winnipeg", "Halifax", - "Victoria", "Regina", "Saskatoon", "Hamilton", "Kitchener", "London", "Windsor", - // UK cities - "Birmingham", "Edinburgh", "Liverpool", "Leeds", "Sheffield", "Newcastle", - "Nottingham", "Southampton", "Portsmouth", "Oxford", "Cambridge", "York", "Bath", - "Brighton", "Cardiff", "Belfast", "Glasgow", "Aberdeen", "Dundee", "Swansea", - } return pickExcluding(rng, cities, original) } @@ -242,56 +186,11 @@ func BuildingNumGenerator(rng *rand.Rand, original string) string { // FirstNameGenerator generates dummy first names func FirstNameGenerator(rng *rand.Rand, original string) string { - names := []string{ - // Western names - "John", "Jane", "Michael", "Sarah", "David", "Emily", "James", "Emma", "Robert", "Olivia", - "William", "Elizabeth", "Richard", "Jennifer", "Thomas", "Jessica", "Charles", "Amanda", - "Christopher", "Ashley", "Daniel", "Stephanie", "Matthew", "Nicole", "Anthony", "Melissa", - // Asian names - "Wei", "Mei", "Hiroshi", "Yuki", "Jin", "Min", "Raj", "Priya", - "Kenji", "Sakura", "Chen", "Li", "Aiko", "Takeshi", "Ananya", "Arjun", - // African names - "Amara", "Kofi", "Zara", "Kwame", "Nia", "Jelani", "Amina", "Chioma", - "Oluwaseun", "Aisha", "Tariq", "Imani", "Sekou", "Adaeze", - // Middle Eastern names - "Yusuf", "Fatima", "Omar", "Layla", "Ali", "Nadia", "Hassan", "Mariam", - "Khalid", "Zahra", "Ahmed", "Leila", "Ibrahim", "Yasmin", - // Latin American names - "Carlos", "Maria", "Diego", "Sofia", "Miguel", "Lucia", "Alejandro", "Valentina", - "Fernando", "Camila", "Ricardo", "Isabella", "Andres", "Gabriela", - // Eastern European names - "Dmitri", "Anna", "Ivan", "Katya", "Alexei", "Elena", "Nikolai", "Olga", - "Sergei", "Natasha", "Vladimir", "Irina", "Mikhail", "Tatiana", - } - return pickExcluding(rng, names, original) + return pickExcluding(rng, firstNames, original) } // SurnameGenerator generates dummy last names func SurnameGenerator(rng *rand.Rand, original string) string { - surnames := []string{ - // Western surnames - "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Martinez", "Wilson", - "Anderson", "Taylor", "Thomas", "Moore", "Jackson", "Martin", "Lee", "Thompson", "White", "Harris", - "Clark", "Lewis", "Robinson", "Walker", "Hall", "Young", "King", "Wright", "Hill", "Scott", - // Asian surnames - "Chen", "Wang", "Li", "Zhang", "Liu", "Kim", "Park", "Choi", "Nguyen", "Tran", - "Tanaka", "Yamamoto", "Suzuki", "Watanabe", "Patel", "Singh", "Sharma", "Kumar", - // African surnames - "Okonkwo", "Diallo", "Mensah", "Osei", "Abebe", "Adeyemi", "Nkosi", "Mbeki", - "Kamara", "Toure", "Dlamini", "Ndlovu", - // Middle Eastern surnames - "Mohammed", "Ahmed", "Hassan", "Khan", "Ali", "Ibrahim", "Hussein", "Malik", - "Nazari", "Hosseini", "Rahman", "Begum", - // Latin American surnames - "Rodriguez", "Lopez", "Gonzalez", "Hernandez", "Perez", "Sanchez", "Ramirez", "Torres", - "Flores", "Rivera", "Gomez", "Diaz", "Reyes", "Morales", "Cruz", "Ortiz", - // Eastern European surnames - "Ivanov", "Petrov", "Kowalski", "Novak", "Horvat", "Popov", "Volkov", "Kozlov", - "Nowak", "Kovalenko", "Bondarenko", "Shevchenko", - // Celtic surnames - "O'Brien", "Murphy", "Kelly", "Sullivan", "O'Connor", "Walsh", "Ryan", "Byrne", - "MacDonald", "Campbell", "Stewart", "Murray", "Fraser", "MacLeod", - } return pickExcluding(rng, surnames, original) } @@ -342,76 +241,19 @@ func UrlGenerator(rng *rand.Rand, original string) string { // CompanyNameGenerator generates dummy company names func CompanyNameGenerator(rng *rand.Rand, original string) string { - prefixes := []string{ - "Acme", "Global", "United", "Pacific", "Atlantic", "Northern", "Summit", "Horizon", "Apex", "Vanguard", - "Pinnacle", "Premier", "Elite", "Prime", "Sterling", "Meridian", "Coastal", "Central", "National", "Continental", - "Metro", "Allied", "Dynamic", "Synergy", "Fusion", "Vertex", "Quantum", "Nova", "Titan", "Omega", - "Pioneer", "Frontier", "Legacy", "Heritage", "Keystone", "Benchmark", "Catalyst", "Spectrum", "Nexus", "Compass", - } - suffixes := []string{ - // US/International - "Inc", "LLC", "Corp", "Industries", "Solutions", "Group", "Holdings", "Partners", "Enterprises", "Co", - "Technologies", "Systems", "Services", "Consulting", "Associates", "International", "Worldwide", - // UK - "Ltd", "PLC", "Limited", - // German - "GmbH", "AG", - // French/Spanish - "SA", "SL", - // Australian - "Pty Ltd", - } - - prefix := prefixes[rng.Intn(len(prefixes))] - suffix := suffixes[rng.Intn(len(suffixes))] + prefix := companyPrefixes[rng.Intn(len(companyPrefixes))] + suffix := companySuffixes[rng.Intn(len(companySuffixes))] return fmt.Sprintf("%s %s", prefix, suffix) } // StateGenerator generates dummy state names func StateGenerator(rng *rand.Rand, original string) string { - states := []string{ - // US States - "Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", "Delaware", - "Florida", "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", - "Kentucky", "Louisiana", "Maine", "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", - "Missouri", "Montana", "Nebraska", "Nevada", "New Hampshire", "New Jersey", "New Mexico", "New York", - "North Carolina", "North Dakota", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", - "South Dakota", "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "Washington", "West Virginia", - "Wisconsin", "Wyoming", - // Canadian Provinces - "Ontario", "Quebec", "British Columbia", "Alberta", "Manitoba", "Saskatchewan", - "Nova Scotia", "New Brunswick", "Newfoundland and Labrador", "Prince Edward Island", - // UK Regions/Nations - "England", "Scotland", "Wales", "Northern Ireland", - // UK Counties - "Yorkshire", "Kent", "Essex", "Hampshire", "Surrey", "Lancashire", "Devon", "Cornwall", - "Oxfordshire", "Cambridgeshire", "Berkshire", "Somerset", "Dorset", "Wiltshire", "Norfolk", "Suffolk", - } return pickExcluding(rng, states, original) } // CountryGenerator generates dummy country names func CountryGenerator(rng *rand.Rand, original string) string { - countries := []string{ - // North America - "United States", "Canada", "Mexico", - // Europe - "United Kingdom", "Germany", "France", "Italy", "Spain", "Netherlands", "Belgium", - "Switzerland", "Austria", "Sweden", "Norway", "Denmark", "Finland", "Ireland", - "Poland", "Portugal", "Greece", "Czech Republic", "Hungary", "Romania", - // Asia - "Japan", "China", "South Korea", "India", "Singapore", "Thailand", "Vietnam", - "Indonesia", "Malaysia", "Philippines", "Taiwan", "Hong Kong", - // Oceania - "Australia", "New Zealand", - // South America - "Brazil", "Argentina", "Chile", "Colombia", "Peru", - // Africa - "South Africa", "Nigeria", "Kenya", "Egypt", "Morocco", "Ghana", - // Middle East - "United Arab Emirates", "Israel", "Saudi Arabia", "Turkey", - } return pickExcluding(rng, countries, original) } diff --git a/src/backend/pii/generators/pools.go b/src/backend/pii/generators/pools.go new file mode 100644 index 00000000..edade87f --- /dev/null +++ b/src/backend/pii/generators/pools.go @@ -0,0 +1,69 @@ +package pii + +import ( + "bufio" + "embed" + "fmt" + "io" + "strings" +) + +// poolData holds the newline-delimited word pools the generators draw from. +// Keeping them as embedded data files rather than inline slices lets the pools +// grow (and gain locale-aware variants later) without touching Go code, while +// the binary stays self-contained. +// +//go:embed data/*.txt +var poolData embed.FS + +// parsePool reads a newline-delimited pool from r. Surrounding whitespace is +// trimmed, and blank lines and comment lines (those beginning with '#') are +// skipped so the data files can carry headers and section labels. +func parsePool(r io.Reader) []string { + var entries []string + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + entries = append(entries, line) + } + if err := scanner.Err(); err != nil { + panic(fmt.Sprintf("pii generators: reading pool: %v", err)) + } + return entries +} + +// loadPool loads a pool file from the embedded data/ directory. It panics on a +// missing, unreadable, or empty file: the files are embedded at compile time, +// so any of those is a build or packaging error that should fail loudly at +// startup rather than leave a generator with an empty pool. +func loadPool(name string) []string { + f, err := poolData.Open("data/" + name) + if err != nil { + panic(fmt.Sprintf("pii generators: cannot open embedded pool %q: %v", name, err)) + } + defer f.Close() + + entries := parsePool(f) + if len(entries) == 0 { + panic(fmt.Sprintf("pii generators: embedded pool %q is empty", name)) + } + return entries +} + +// Word pools loaded once at package initialization from the embedded data +// files. The generators reference these directly. +var ( + emailFirstNames = loadPool("email_firstnames.txt") + emailSurnames = loadPool("email_surnames.txt") + firstNames = loadPool("firstnames.txt") + surnames = loadPool("surnames.txt") + cities = loadPool("cities.txt") + streets = loadPool("streets.txt") + states = loadPool("states.txt") + countries = loadPool("countries.txt") + companyPrefixes = loadPool("company_prefixes.txt") + companySuffixes = loadPool("company_suffixes.txt") +) diff --git a/src/backend/pii/generators/pools_test.go b/src/backend/pii/generators/pools_test.go new file mode 100644 index 00000000..6ca7ce20 --- /dev/null +++ b/src/backend/pii/generators/pools_test.go @@ -0,0 +1,81 @@ +package pii + +import ( + "regexp" + "strings" + "testing" +) + +func TestParsePool(t *testing.T) { + input := strings.Join([]string{ + "# a comment", + "", + " Alpha ", + "Beta", + " # indented comment", + "\tGamma\t", + " ", + "Delta", + }, "\n") + + got := parsePool(strings.NewReader(input)) + want := []string{"Alpha", "Beta", "Gamma", "Delta"} + + if len(got) != len(want) { + t.Fatalf("parsePool returned %d entries (%v), want %d (%v)", len(got), got, len(want), want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("parsePool entry %d = %q, want %q", i, got[i], want[i]) + } + } +} + +// loadedPools is every pool the generators draw from, so the checks below cover +// them uniformly. +var loadedPools = map[string][]string{ + "email_firstnames": emailFirstNames, + "email_surnames": emailSurnames, + "firstnames": firstNames, + "surnames": surnames, + "cities": cities, + "streets": streets, + "states": states, + "countries": countries, + "company_prefixes": companyPrefixes, + "company_suffixes": companySuffixes, +} + +// Every embedded pool must load with real entries and no parser leakage +// (blank lines or comment markers surviving into the data). +func TestLoadedPoolsAreCleanAndNonEmpty(t *testing.T) { + for name, pool := range loadedPools { + if len(pool) == 0 { + t.Errorf("pool %q loaded empty", name) + } + for _, entry := range pool { + if entry == "" { + t.Errorf("pool %q contains an empty entry", name) + } + if strings.HasPrefix(entry, "#") { + t.Errorf("pool %q contains an unstripped comment: %q", name, entry) + } + if entry != strings.TrimSpace(entry) { + t.Errorf("pool %q entry has surrounding whitespace: %q", name, entry) + } + } + } +} + +// EmailGenerator formats the local part as first.last and callers assert it +// matches [a-z]+\.[a-z]+, so the email pools must stay lowercase ASCII words. +func TestEmailPoolsAreLowercaseTokens(t *testing.T) { + tokenPattern := regexp.MustCompile(`^[a-z]+$`) + for _, name := range []string{"email_firstnames", "email_surnames"} { + for _, entry := range loadedPools[name] { + if !tokenPattern.MatchString(entry) { + t.Errorf("pool %q entry %q is not a lowercase ASCII token", name, entry) + } + } + } +}