99
1010use std:: collections:: BTreeMap ;
1111use std:: fs;
12- use std:: io:: { BufReader , Write } ;
12+ use std:: io:: Write ;
13+ use std:: path:: PathBuf ;
1314
1415use anyhow:: Context ;
1516use md5:: { Digest , Md5 } ;
1617use serde:: { Deserialize , Serialize } ;
1718
18- /// The path of a protobuf file and its [`md5`] hash.
19+ /// The path of an object definition file and its [`md5`] hash.
1920///
2021/// We store a hash of all the files to make sure they don't accidentally change, which would
2122/// invalidate our snapshotted types, and could silently introduce bugs.
2223#[ derive( Debug , Clone , Deserialize , Serialize ) ]
23- struct ProtoHash {
24+ struct ObjectsHash {
2425 name : String ,
2526 md5 : String ,
2627}
2728
28- const PROTO_DIRECTORY : & str = "protos" ;
29- const PROTO_HASHES : & str = "protos/hashes.json" ;
29+ const OBJECTS_HASHES : & str = "objects_hashes.json" ;
3030
3131fn main ( ) -> anyhow:: Result < ( ) > {
32- println ! ( "cargo:rerun-if-changed={PROTO_DIRECTORY}" ) ;
32+ let crate_root = PathBuf :: from ( env ! ( "CARGO_MANIFEST_DIR" ) ) ;
3333
3434 // Read in the persisted hashes from disk.
35- let hashes = fs :: File :: open ( PROTO_HASHES ) . context ( "opening proto hashes" ) ? ;
36- let reader = BufReader :: new ( & hashes ) ;
37- let hashes: Vec < ProtoHash > = serde_json:: from_reader ( reader ) ?;
35+ let hashes_path = crate_root . join ( OBJECTS_HASHES ) ;
36+ let hashes_json = fs :: read_to_string ( & hashes_path ) ? ;
37+ let hashes: Vec < ObjectsHash > = serde_json:: from_str ( & hashes_json ) ?;
3838 let mut persisted: BTreeMap < String , String > =
3939 hashes. into_iter ( ) . map ( |e| ( e. name , e. md5 ) ) . collect ( ) ;
4040
41- // Discover all of the protobuf files on disk.
42- let protos: BTreeMap < String , String > = fs:: read_dir ( PROTO_DIRECTORY ) ?
41+ // Discover all of the object definition files on disk.
42+ let src_dir = crate_root. join ( "src" ) ;
43+ let objects: BTreeMap < String , String > = fs:: read_dir ( src_dir) ?
4344 // If we fail to read one file, fail everything.
4445 . collect :: < Result < Vec < _ > , _ > > ( ) ?
4546 . into_iter ( )
46- // Filter to only files with the .proto extension .
47+ // Filter to only files with the of the form `objects*.rs` .
4748 . filter ( |entry| {
48- entry
49- . path ( )
50- . extension ( )
51- . map ( |e| e. to_string_lossy ( ) . contains ( "proto" ) )
52- . unwrap_or ( false )
49+ let name = entry. file_name ( ) ;
50+ let s = name. to_string_lossy ( ) ;
51+ s. starts_with ( "objects" ) && s. ends_with ( ".rs" )
5352 } )
5453 . map ( |file| {
5554 let path = file. path ( ) ;
@@ -71,34 +70,34 @@ fn main() -> anyhow::Result<()> {
7170 } )
7271 . collect ( ) ;
7372
74- // After validating our hashes we'll re-write the file if any new protos
73+ // After validating our hashes we'll re-write the file if any new object definitions
7574 // have been added.
76- let mut to_persist: Vec < ProtoHash > = Vec :: new ( ) ;
75+ let mut to_persist: Vec < ObjectsHash > = Vec :: new ( ) ;
7776 let mut any_new = false ;
7877
7978 // Check the persisted hashes against what we just read in from disk.
80- for ( name, hash) in protos {
79+ for ( name, hash) in objects {
8180 match persisted. remove ( & name) {
8281 // Hashes have changed!
8382 Some ( og_hash) if hash != og_hash => {
8483 anyhow:: bail!( error_message( og_hash, hash, name) ) ;
8584 }
86- // Found a proto file on disk that we didn't have persisted, we'll just persist it.
85+ // Found an objects file on disk that we didn't have persisted, we'll just persist it.
8786 None => {
88- to_persist. push ( ProtoHash { name, md5 : hash } ) ;
87+ to_persist. push ( ObjectsHash { name, md5 : hash } ) ;
8988 any_new = true ;
9089 }
9190 // We match!
92- Some ( _) => to_persist. push ( ProtoHash { name, md5 : hash } ) ,
91+ Some ( _) => to_persist. push ( ObjectsHash { name, md5 : hash } ) ,
9392 }
9493 }
9594
96- // Check if there are any proto files we should have had hashes for, but didn't exist.
95+ // Check if there are any objects files we should have had hashes for, but didn't exist.
9796 if !persisted. is_empty ( ) {
9897 anyhow:: bail!( "Have persisted hashes, but no files on disk? {persisted:#?}" ) ;
9998 }
10099
101- // Write the hashes back out to disk if and only if there are new protos . We
100+ // Write the hashes back out to disk if and only if there are new object definitions . We
102101 // don't do this unconditionally or we'll get stuck in a rebuild loop:
103102 // executing this build script will change the mtime on the hashes file,
104103 // which will force the next compile to rebuild the crate, even if nothing
@@ -107,103 +106,23 @@ fn main() -> anyhow::Result<()> {
107106 let mut file = fs:: File :: options ( )
108107 . write ( true )
109108 . truncate ( true )
110- . open ( PROTO_HASHES )
109+ . open ( hashes_path )
111110 . context ( "opening hashes file to write" ) ?;
112111 serde_json:: to_writer_pretty ( & mut file, & to_persist) . context ( "persisting hashes" ) ?;
113112 write ! ( & mut file, "\n " ) . context ( "writing newline" ) ?;
114113 }
115114
116- // Generate protos!
117- let paths: Vec < _ > = to_persist
118- . iter ( )
119- . map ( |entry| format ! ( "protos/{}" , entry. name) )
120- . collect ( ) ;
121-
122- const ATTR : & str = "#[derive(Eq, PartialOrd, Ord, ::serde::Serialize, ::serde::Deserialize)]" ;
123- const ARBITRARY_ATTR : & str = "#[derive(::proptest_derive::Arbitrary)]" ;
124-
125- // 'as' is okay here because we're using it to define the type of the empty slice, which is
126- // necessary since the method takes the slice as a generic arg.
127- #[ allow( clippy:: as_conversions) ]
128- // DO NOT change how JSON serialization works for these objects. The catalog relies on the JSON
129- // serialization of these objects remaining stable for a specific objects_vX version. If you
130- // want to change the JSON serialization format then follow these steps:
131- //
132- // 1. Create a new version of the `objects.proto` file.
133- // 2. Update the path of .proto files given to this compile block so that it is only the
134- // previous .proto files.
135- // 3. Add a new `prost_build::Config::new()...compile_protos(...)` block that only compiles
136- // the new and all future .proto files with the changed JSON serialization.
137- //
138- // Once we delete all the `.proto` that use the old JSON serialization, then we can delete
139- // the compile block for them as well.
140- prost_build:: Config :: new ( )
141- . protoc_executable ( mz_build_tools:: protoc ( ) )
142- . btree_map ( [ "." ] )
143- . bytes ( [ "." ] )
144- . message_attribute ( "." , ATTR )
145- // Note(parkmycar): This is annoying, but we need to manually specify each oneof so we can
146- // get them to implement Eq, PartialEq, and Ord. If you define a new oneof you should add
147- // it here.
148- . enum_attribute ( "CatalogItem.value" , ATTR )
149- . enum_attribute ( "ClusterConfig.variant" , ATTR )
150- . enum_attribute ( "GlobalId.value" , ATTR )
151- . enum_attribute ( "CatalogItemId.value" , ATTR )
152- . enum_attribute ( "ClusterId.value" , ATTR )
153- . enum_attribute ( "DatabaseId.value" , ATTR )
154- . enum_attribute ( "SchemaId.value" , ATTR )
155- . enum_attribute ( "ReplicaId.value" , ATTR )
156- . enum_attribute ( "RoleId.value" , ATTR )
157- . enum_attribute ( "NetworkPolicyId.value" , ATTR )
158- . enum_attribute ( "NetworkPolicyRule.action" , ATTR )
159- . enum_attribute ( "NetworkPolicyRule.direction" , ATTR )
160- . enum_attribute ( "ReplicaConfig.location" , ATTR )
161- . enum_attribute ( "AuditLogEventV1.details" , ATTR )
162- . enum_attribute ( "AuditLogKey.event" , ATTR )
163- . enum_attribute ( "StorageUsageKey.usage" , ATTR )
164- . enum_attribute ( "ResolvedDatabaseSpecifier.value" , ATTR )
165- . enum_attribute ( "CommentKey.object" , ATTR )
166- . enum_attribute ( "CommentKey.sub_component" , ATTR )
167- . enum_attribute ( "ResolvedDatabaseSpecifier.spec" , ATTR )
168- . enum_attribute ( "SchemaSpecifier.spec" , ATTR )
169- . enum_attribute ( "RoleVars.Entry.val" , ATTR )
170- . enum_attribute ( "StateUpdateKind.kind" , ATTR )
171- . enum_attribute ( "ClusterScheduleOptionValue.value" , ATTR )
172- . enum_attribute ( "ClusterSchedule.value" , ATTR )
173- . enum_attribute ( "CreateOrDropClusterReplicaReasonV1.reason" , ATTR )
174- . enum_attribute ( "RefreshDecisionWithReasonV1.decision" , ATTR )
175- . enum_attribute ( "RefreshDecisionWithReasonV2.decision" , ATTR )
176- // Serialize/deserialize the top-level enum in the persist-backed
177- // catalog as "internally tagged"[^1] to set up persist pushdown
178- // statistics for success.
179- //
180- // [^1]: https://serde.rs/enum-representations.html#internally-tagged
181- . enum_attribute ( "StateUpdateKind.kind" , "#[serde(tag = \" kind\" )]" )
182- // We derive Arbitrary for all protobuf types for wire compatibility testing.
183- . message_attribute ( "." , ARBITRARY_ATTR )
184- . enum_attribute ( "." , ARBITRARY_ATTR )
185- . compile_protos (
186- & paths,
187- & [ /*
188- This is purposefully empty, and we should never
189- add any includes because we don't want to allow
190- our protos to have dependencies. This allows us
191- to ensure our snapshots can't silently change.
192- */
193- ] as & [ & str ] ,
194- ) ?;
195-
196115 Ok ( ( ) )
197116}
198117
199118/// A (hopefully) helpful error message that describes what to do when the hashes differ.
200119fn error_message ( og_hash : String , hash : String , filename : String ) -> String {
201- let title = "Hashes changed for the persisted protobuf files!" ;
120+ let title = "Hashes changed for the persisted object definition files!" ;
202121 let body1 = format ! (
203- "If you changed '{filename}' without first making a snapshot, then you need to copy '{filename}' and rename it with a suffix like '_vX.proto '."
122+ "If you changed '{filename}' without first making a snapshot, then you need to copy '{filename}' and rename it with a suffix like '_vX.rs '."
204123 ) ;
205124 let body2 = format ! (
206- "Otherwise you can update the hash for '{filename}' in '{PROTO_HASHES }' to be '{hash}'."
125+ "Otherwise you can update the hash for '{filename}' in '{OBJECTS_HASHES }' to be '{hash}'."
207126 ) ;
208127 let hashes = format ! ( "persisted_hash({og_hash}) != current_hash({hash})\n File: {filename}" ) ;
209128
0 commit comments