diff --git a/Cargo.lock b/Cargo.lock index 8ab97090d..8320b07f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -405,6 +405,21 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "async-tar" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e9933aa2da420042c67e2ea83eec765919347d9742592744dc97cc42ef20c5d" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.7.0", + "tokio", + "tokio-stream", + "xattr", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -2124,6 +2139,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "docs_rs_import_release" +version = "0.6.0" +dependencies = [ + "anyhow", + "async-tar", + "clap", + "docs_rs_cargo_metadata", + "docs_rs_context", + "docs_rs_database", + "docs_rs_logging", + "docs_rs_registry_api", + "docs_rs_repository_stats", + "docs_rs_rustdoc_json", + "docs_rs_storage", + "docs_rs_types", + "docs_rs_utils", + "docsrs-metadata", + "futures-util", + "regex", + "reqwest 0.13.1", + "serde", + "sqlx", + "tempfile", + "tokio", + "tracing", + "walkdir", + "zip", +] + [[package]] name = "docs_rs_integration_tests" version = "0.1.0" @@ -6152,12 +6197,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls 0.26.4", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -8193,6 +8240,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.85" diff --git a/Cargo.toml b/Cargo.toml index 32c66b928..026c1de6e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ postcard = { version = "1.1.3", default-features = false, features = ["use-std"] pretty_assertions = "1.4.0" rand = "0.9" regex = "1" -reqwest = { version = "0.13", features = ["json", "gzip"] } +reqwest = { version = "0.13", features = ["json", "gzip", "stream"] } sentry = { version = "0.46.0", features = ["panic", "tracing", "tower-http", "anyhow", "backtrace"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" @@ -64,3 +64,4 @@ toml = "0.9.2" tracing = "0.1.37" url = { version = "2.1.1", features = ["serde"] } walkdir = "2" +zip = { version = "7.0.0", default-features = false, features = ["bzip2"] } diff --git a/README.md b/README.md index cf101f39c..78c7a92d0 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,8 @@ cargo run --bin docs_rs_admin -- database migrate cargo run --bin docs_rs_builder -- build update-toolchain # Build a sample crate to make sure it works cargo run --bin docs_rs_builder -- build crate regex 1.3.1 +# if you don't want to run the builder, you can import a release from docs.rs itself +cargo run -p docs_rs_import_release -- regex latest # This starts the web server but does not build any crates. # It does not automatically run the migrations, so you need to do that manually (see above). cargo run --bin docs_rs_web diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-0a932ffd17414950513a2c8aca2ccd5e29780e00b105fbf79d2de83a11d33ddd.json b/crates/bin/docs_rs_import_release/.sqlx/query-0a932ffd17414950513a2c8aca2ccd5e29780e00b105fbf79d2de83a11d33ddd.json new file mode 100644 index 000000000..4c95721d1 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-0a932ffd17414950513a2c8aca2ccd5e29780e00b105fbf79d2de83a11d33ddd.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM owner_rels\n WHERE\n cid = $1 AND\n NOT (oid = ANY($2))", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "0a932ffd17414950513a2c8aca2ccd5e29780e00b105fbf79d2de83a11d33ddd" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-120c268463e7890553e639ac37e667f3fcdc3f5ceaab3c229b71dbb799c0cddc.json b/crates/bin/docs_rs_import_release/.sqlx/query-120c268463e7890553e639ac37e667f3fcdc3f5ceaab3c229b71dbb799c0cddc.json new file mode 100644 index 000000000..fa8db5d2f --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-120c268463e7890553e639ac37e667f3fcdc3f5ceaab3c229b71dbb799c0cddc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO compression_rels (release, algorithm)\n VALUES ($1, $2)\n ON CONFLICT DO NOTHING;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "120c268463e7890553e639ac37e667f3fcdc3f5ceaab3c229b71dbb799c0cddc" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-12c203c76454f3b597186769c28550affce7342fc6a79de7c3b3da048232e3ec.json b/crates/bin/docs_rs_import_release/.sqlx/query-12c203c76454f3b597186769c28550affce7342fc6a79de7c3b3da048232e3ec.json new file mode 100644 index 000000000..fd127620f --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-12c203c76454f3b597186769c28550affce7342fc6a79de7c3b3da048232e3ec.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO crate_priorities (pattern, priority) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "12c203c76454f3b597186769c28550affce7342fc6a79de7c3b3da048232e3ec" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-1a0487c0559ce9e4283e9ede644546283bb6261466bfb7ed23c49df2d042e3c5.json b/crates/bin/docs_rs_import_release/.sqlx/query-1a0487c0559ce9e4283e9ede644546283bb6261466bfb7ed23c49df2d042e3c5.json new file mode 100644 index 000000000..ba5ca8af5 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-1a0487c0559ce9e4283e9ede644546283bb6261466bfb7ed23c49df2d042e3c5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT host_id\n FROM repositories\n WHERE\n host = $1 AND\n updated_at < NOW() - INTERVAL '1 day'\n ORDER BY updated_at\n ;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "host_id", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1a0487c0559ce9e4283e9ede644546283bb6261466bfb7ed23c49df2d042e3c5" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json b/crates/bin/docs_rs_import_release/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json new file mode 100644 index 000000000..e1bcf20ab --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE crates\n SET latest_version_id = $2\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "1e660947261dfa1a5d1745d1732df59e0cf67ef1906da818086d063e6a0e21c6" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-2056327f710b8e427e21bf427a24edaffcf58f68237f572e1c14801dc41fba3a.json b/crates/bin/docs_rs_import_release/.sqlx/query-2056327f710b8e427e21bf427a24edaffcf58f68237f572e1c14801dc41fba3a.json new file mode 100644 index 000000000..6d4f685e9 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-2056327f710b8e427e21bf427a24edaffcf58f68237f572e1c14801dc41fba3a.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE builds\n SET\n rustc_version = $1,\n docsrs_version = $2,\n build_status = $3,\n build_server = $4,\n errors = $5,\n documentation_size = $6,\n rustc_nightly_date = $7,\n build_finished = NOW()\n WHERE\n id = $8\n RETURNING rid as \"rid: ReleaseId\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rid: ReleaseId", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + }, + "Text", + "Text", + "Int8", + "Date", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2056327f710b8e427e21bf427a24edaffcf58f68237f572e1c14801dc41fba3a" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-2c1f393f0c38c013c99629b5f0d3611c5153730091dd5488e2ef21b495942b93.json b/crates/bin/docs_rs_import_release/.sqlx/query-2c1f393f0c38c013c99629b5f0d3611c5153730091dd5488e2ef21b495942b93.json new file mode 100644 index 000000000..b76453c01 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-2c1f393f0c38c013c99629b5f0d3611c5153730091dd5488e2ef21b495942b93.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO crates (name)\n VALUES ($1)\n ON CONFLICT (name) DO UPDATE\n SET -- this `SET` is needed so the id is always returned.\n name = EXCLUDED.name\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "2c1f393f0c38c013c99629b5f0d3611c5153730091dd5488e2ef21b495942b93" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-38ca6c4bca81a0762e6e2db08a05ef0ba191d37199165d2745c64a4036e20790.json b/crates/bin/docs_rs_import_release/.sqlx/query-38ca6c4bca81a0762e6e2db08a05ef0ba191d37199165d2745c64a4036e20790.json new file mode 100644 index 000000000..c7752a36e --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-38ca6c4bca81a0762e6e2db08a05ef0ba191d37199165d2745c64a4036e20790.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT slug FROM keywords WHERE slug = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "slug", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "38ca6c4bca81a0762e6e2db08a05ef0ba191d37199165d2745c64a4036e20790" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-3aac87e4968ad4d83991de6b4ec164c44f00495b3a9a2e9b5ae460697a19278f.json b/crates/bin/docs_rs_import_release/.sqlx/query-3aac87e4968ad4d83991de6b4ec164c44f00495b3a9a2e9b5ae460697a19278f.json new file mode 100644 index 000000000..ab68991d0 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-3aac87e4968ad4d83991de6b4ec164c44f00495b3a9a2e9b5ae460697a19278f.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO keyword_rels (rid, kid)\n SELECT $1 as rid, id as kid\n FROM keywords\n WHERE slug = ANY($2)\n ON CONFLICT DO NOTHING;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "3aac87e4968ad4d83991de6b4ec164c44f00495b3a9a2e9b5ae460697a19278f" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-4115addabb6f03cd0de31ac2ddeafd6797b25b99a7112cb68a93a4fac5b07a23.json b/crates/bin/docs_rs_import_release/.sqlx/query-4115addabb6f03cd0de31ac2ddeafd6797b25b99a7112cb68a93a4fac5b07a23.json new file mode 100644 index 000000000..aed78ccde --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-4115addabb6f03cd0de31ac2ddeafd6797b25b99a7112cb68a93a4fac5b07a23.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id as \"id: CrateId\" FROM crates WHERE crates.name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: CrateId", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4115addabb6f03cd0de31ac2ddeafd6797b25b99a7112cb68a93a4fac5b07a23" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-412ad6a09abd2798dfc1c0b39a52d5b08f9f29f07b033251b9ce9ae5df354ceb.json b/crates/bin/docs_rs_import_release/.sqlx/query-412ad6a09abd2798dfc1c0b39a52d5b08f9f29f07b033251b9ce9ae5df354ceb.json new file mode 100644 index 000000000..e1d0c0a19 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-412ad6a09abd2798dfc1c0b39a52d5b08f9f29f07b033251b9ce9ae5df354ceb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id\n FROM queue\n WHERE\n name = $1 AND\n version = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "412ad6a09abd2798dfc1c0b39a52d5b08f9f29f07b033251b9ce9ae5df354ceb" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json b/crates/bin/docs_rs_import_release/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json new file mode 100644 index 000000000..ca0a44f4b --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n releases.id as \"id: ReleaseId\",\n releases.version as \"version: Version\",\n release_build_status.build_status as \"build_status!: BuildStatus\",\n releases.yanked,\n releases.is_library,\n releases.rustdoc_status,\n releases.release_time,\n releases.target_name,\n releases.default_target,\n releases.doc_targets\n FROM releases\n INNER JOIN release_build_status ON releases.id = release_build_status.rid\n WHERE\n releases.crate_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: ReleaseId", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "build_status!: BuildStatus", + "type_info": { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "yanked", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "is_library", + "type_info": "Bool" + }, + { + "ordinal": 5, + "name": "rustdoc_status", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "release_time", + "type_info": "Timestamptz" + }, + { + "ordinal": 7, + "name": "target_name", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "default_target", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "doc_targets", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + false, + true, + true, + true, + true, + true, + true, + true + ] + }, + "hash": "4894c7d8c4e354dca1d952362b2e0cb25441e8e65b273e01ed86d2d3ecebfe84" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-494e6b594aba915ba2582eca4797b13012576b2800c257ae3b0d3b7706c69a2c.json b/crates/bin/docs_rs_import_release/.sqlx/query-494e6b594aba915ba2582eca4797b13012576b2800c257ae3b0d3b7706c69a2c.json new file mode 100644 index 000000000..cf141db65 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-494e6b594aba915ba2582eca4797b13012576b2800c257ae3b0d3b7706c69a2c.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO doc_coverage (\n release_id, total_items, documented_items,\n total_items_needing_examples, items_with_examples\n )\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (release_id) DO UPDATE\n SET\n total_items = $2,\n documented_items = $3,\n total_items_needing_examples = $4,\n items_with_examples = $5\n RETURNING release_id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "release_id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int4", + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "494e6b594aba915ba2582eca4797b13012576b2800c257ae3b0d3b7706c69a2c" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json b/crates/bin/docs_rs_import_release/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json new file mode 100644 index 000000000..530c4d879 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO config (name, value)\n VALUES ($1, $2)\n ON CONFLICT (name) DO UPDATE SET value = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Json" + ] + }, + "nullable": [] + }, + "hash": "4a6887c2d436121cb2ba6a9c5069455b8f222d929672dc1ff810fa49c2940e2c" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-4b13fe2c8df2b8b8bf019344313b2bc6442482a604cf90fb6106154f8e69a1c2.json b/crates/bin/docs_rs_import_release/.sqlx/query-4b13fe2c8df2b8b8bf019344313b2bc6442482a604cf90fb6106154f8e69a1c2.json new file mode 100644 index 000000000..6d306761e --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-4b13fe2c8df2b8b8bf019344313b2bc6442482a604cf90fb6106154f8e69a1c2.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE\n FROM queue\n WHERE\n name = $1 AND\n version = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4b13fe2c8df2b8b8bf019344313b2bc6442482a604cf90fb6106154f8e69a1c2" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json b/crates/bin/docs_rs_import_release/.sqlx/query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json new file mode 100644 index 000000000..dd15d3dde --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO releases (crate_id, version, archive_storage)\n VALUES ($1, $2, TRUE)\n ON CONFLICT (crate_id, version) DO UPDATE\n SET -- this `SET` is needed so the id is always returned.\n version = EXCLUDED.version\n RETURNING id as \"id: ReleaseId\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: ReleaseId", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4b4bfd97b03f632357a68c84f281c286fa0ec399ed98db0dda1ee81e6a6e7f3a" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb.json b/crates/bin/docs_rs_import_release/.sqlx/query-4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb.json new file mode 100644 index 000000000..a95b272dd --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM crates WHERE crates.name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4f81678f0d680c4be7215ef0667729e8518063e0ee4ecad5aa7e559e88b8e1cb" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-58c1056e05fa7efdd691ad1c568bc00ec0f7c56584bdbe9d88963cc2f7ed7133.json b/crates/bin/docs_rs_import_release/.sqlx/query-58c1056e05fa7efdd691ad1c568bc00ec0f7c56584bdbe9d88963cc2f7ed7133.json new file mode 100644 index 000000000..cd88562f2 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-58c1056e05fa7efdd691ad1c568bc00ec0f7c56584bdbe9d88963cc2f7ed7133.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO keywords (name, slug) VALUES ($1, $2)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "58c1056e05fa7efdd691ad1c568bc00ec0f7c56584bdbe9d88963cc2f7ed7133" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json b/crates/bin/docs_rs_import_release/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json new file mode 100644 index 000000000..264e19fd2 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM config WHERE name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5ad9cd6cd9d444d258f7486fda178d4dd071cf43cb0ea950574af8e2f37b4a21" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05.json b/crates/bin/docs_rs_import_release/.sqlx/query-5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05.json new file mode 100644 index 000000000..cc79bd27a --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE releases SET repository_id = $1 WHERE id = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "5b401b6a191f7364be11110c23228933120dc7c39d0ef436ececc8bee9695c05" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-5cf9b185b54dde447b6ba1458178c4fad4c75e72c6a7b7735d17835b9b746ac3.json b/crates/bin/docs_rs_import_release/.sqlx/query-5cf9b185b54dde447b6ba1458178c4fad4c75e72c6a7b7735d17835b9b746ac3.json new file mode 100644 index 000000000..92b428002 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-5cf9b185b54dde447b6ba1458178c4fad4c75e72c6a7b7735d17835b9b746ac3.json @@ -0,0 +1,61 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE releases\n SET release_time = $2,\n dependencies = $3,\n target_name = $4,\n yanked = $5,\n rustdoc_status = $6,\n test_status = $7,\n license = $8,\n repository_url = $9,\n homepage_url = $10,\n description = $11,\n description_long = $12,\n readme = $13,\n keywords = $14,\n have_examples = $15,\n downloads = $16,\n files = $17,\n doc_targets = $18,\n is_library = $19,\n documentation_url = $20,\n default_target = $21,\n features = $22,\n repository_id = $23,\n archive_storage = $24,\n source_size = $25\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Timestamptz", + "Json", + "Varchar", + "Bool", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Json", + "Bool", + "Int4", + "Json", + "Json", + "Bool", + "Varchar", + "Varchar", + { + "Custom": { + "name": "feature[]", + "kind": { + "Array": { + "Custom": { + "name": "feature", + "kind": { + "Composite": [ + [ + "name", + "Text" + ], + [ + "subfeatures", + "TextArray" + ] + ] + } + } + } + } + } + }, + "Int4", + "Bool", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "5cf9b185b54dde447b6ba1458178c4fad4c75e72c6a7b7735d17835b9b746ac3" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae.json b/crates/bin/docs_rs_import_release/.sqlx/query-688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae.json new file mode 100644 index 000000000..dcc82bf13 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO blacklisted_crates (crate_name) VALUES ($1);", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "688c28ae99e5511238a6cbe1c53e3abc3924b4ecffc216e1556a53dda936ceae" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965.json b/crates/bin/docs_rs_import_release/.sqlx/query-718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965.json new file mode 100644 index 000000000..032419e27 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965.json @@ -0,0 +1,29 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO repositories (\n host, host_id, name, description, last_commit, stars, forks, issues, updated_at\n ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW())\n ON CONFLICT (host, host_id) DO\n UPDATE SET\n name = $3,\n description = $4,\n last_commit = $5,\n stars = $6,\n forks = $7,\n issues = $8,\n updated_at = NOW()\n RETURNING id;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Timestamptz", + "Int4", + "Int4", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "718576e299a41495b28c843737921e3493a61c0629a9d9a5d04066d443663965" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3.json b/crates/bin/docs_rs_import_release/.sqlx/query-73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3.json new file mode 100644 index 000000000..6f21daee0 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO sandbox_overrides (\n crate_name, max_memory_bytes, max_targets, timeout_seconds\n )\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (crate_name) DO UPDATE\n SET\n max_memory_bytes = $2,\n max_targets = $3,\n timeout_seconds = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Int8", + "Int4", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "73ff86cdb5b9d0ab312493690d4108803ce04531d497d6dd8d67ad05a844eab3" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json b/crates/bin/docs_rs_import_release/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json new file mode 100644 index 000000000..55b209a3f --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO queue (name, version, priority, registry)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT (name, version) DO UPDATE\n SET priority = EXCLUDED.priority,\n registry = EXCLUDED.registry,\n attempt = 0,\n last_attempt = NULL\n ;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Int4", + "Text" + ] + }, + "nullable": [] + }, + "hash": "743604e86c489f7f330adf83d66c810678cd8bbee215532ce26f2c4e76e54a67" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-7544bfef94310c33c8d34c30bc24e91b4aa2f198c1efd0f9ef562c0b85f4dddb.json b/crates/bin/docs_rs_import_release/.sqlx/query-7544bfef94310c33c8d34c30bc24e91b4aa2f198c1efd0f9ef562c0b85f4dddb.json new file mode 100644 index 000000000..1928a3804 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-7544bfef94310c33c8d34c30bc24e91b4aa2f198c1efd0f9ef562c0b85f4dddb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO release_build_status(rid, last_build_time, build_status)\n SELECT\n summary.id,\n summary.last_build_time,\n CASE\n WHEN summary.success_count > 0 THEN 'success'::build_status\n WHEN summary.failure_count > 0 THEN 'failure'::build_status\n ELSE 'in_progress'::build_status\n END as build_status\n\n FROM (\n SELECT\n r.id,\n MAX(b.build_finished) as last_build_time,\n SUM(CASE WHEN b.build_status = 'success' THEN 1 ELSE 0 END) as success_count,\n SUM(CASE WHEN b.build_status = 'failure' THEN 1 ELSE 0 END) as failure_count\n FROM\n releases as r\n LEFT OUTER JOIN builds AS b on b.rid = r.id\n WHERE\n r.id = $1\n GROUP BY r.id\n ) as summary\n\n ON CONFLICT (rid) DO UPDATE\n SET\n last_build_time = EXCLUDED.last_build_time,\n build_status=EXCLUDED.build_status", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "7544bfef94310c33c8d34c30bc24e91b4aa2f198c1efd0f9ef562c0b85f4dddb" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-7fda06d8c2a6cac88f9bf06b3480f07d801c711693e19ee40991f069b2393fec.json b/crates/bin/docs_rs_import_release/.sqlx/query-7fda06d8c2a6cac88f9bf06b3480f07d801c711693e19ee40991f069b2393fec.json new file mode 100644 index 000000000..60bddfbca --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-7fda06d8c2a6cac88f9bf06b3480f07d801c711693e19ee40991f069b2393fec.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT crate_id as \"crate_id: CrateId\"\n FROM releases\n WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_id: CrateId", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7fda06d8c2a6cac88f9bf06b3480f07d801c711693e19ee40991f069b2393fec" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6.json b/crates/bin/docs_rs_import_release/.sqlx/query-864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6.json new file mode 100644 index 000000000..cc5d0f370 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM blacklisted_crates WHERE crate_name = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "864a659b317f451949bf793899a7ea5b78ebf8a962fbb12b88eb8f1d807fbab6" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json b/crates/bin/docs_rs_import_release/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json new file mode 100644 index 000000000..8aedc7865 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO owners (login, avatar, kind)\n VALUES ($1, $2, $3)\n ON CONFLICT (login) DO UPDATE\n SET\n avatar = EXCLUDED.avatar,\n kind = EXCLUDED.kind\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + { + "Custom": { + "name": "owner_kind", + "kind": { + "Enum": [ + "user", + "team" + ] + } + } + } + ] + }, + "nullable": [ + false + ] + }, + "hash": "87952bd450ed2c13b99bd502a73a84edd7d17e6171523ebbd57f1d9dd7c9b46c" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-8f1900a52809215672eb6c5ca684082c77a81874c88cab453681eaa660a13ae0.json b/crates/bin/docs_rs_import_release/.sqlx/query-8f1900a52809215672eb6c5ca684082c77a81874c88cab453681eaa660a13ae0.json new file mode 100644 index 000000000..9af771395 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-8f1900a52809215672eb6c5ca684082c77a81874c88cab453681eaa660a13ae0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pattern, priority FROM crate_priorities", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pattern", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "priority", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + }, + "hash": "8f1900a52809215672eb6c5ca684082c77a81874c88cab453681eaa660a13ae0" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-90ff19a8b5452159a09930f450d614fd5f516c362bb2c195bcbea917775b9b54.json b/crates/bin/docs_rs_import_release/.sqlx/query-90ff19a8b5452159a09930f450d614fd5f516c362bb2c195bcbea917775b9b54.json new file mode 100644 index 000000000..7e7ce93f0 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-90ff19a8b5452159a09930f450d614fd5f516c362bb2c195bcbea917775b9b54.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pattern, priority FROM crate_priorities WHERE $1 LIKE pattern LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pattern", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "priority", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "90ff19a8b5452159a09930f450d614fd5f516c362bb2c195bcbea917775b9b54" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-91398f5f994e0440669ac251f7e5adae4cba983d1479a6b7ccaa1bf6bda7721f.json b/crates/bin/docs_rs_import_release/.sqlx/query-91398f5f994e0440669ac251f7e5adae4cba983d1479a6b7ccaa1bf6bda7721f.json new file mode 100644 index 000000000..f25ade8d2 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-91398f5f994e0440669ac251f7e5adae4cba983d1479a6b7ccaa1bf6bda7721f.json @@ -0,0 +1,50 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n name as \"name: KrateName\",\n version as \"version: Version\",\n priority,\n registry,\n attempt\n FROM queue\n ORDER BY priority ASC, attempt ASC, id ASC", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name: KrateName", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "registry", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attempt", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + true, + false + ] + }, + "hash": "91398f5f994e0440669ac251f7e5adae4cba983d1479a6b7ccaa1bf6bda7721f" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-95bc487e596e04e5e1cfd20903d8bdb54917c2ea91a0ee9c1dad7d5290ca2e62.json b/crates/bin/docs_rs_import_release/.sqlx/query-95bc487e596e04e5e1cfd20903d8bdb54917c2ea91a0ee9c1dad7d5290ca2e62.json new file mode 100644 index 000000000..312fd50c3 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-95bc487e596e04e5e1cfd20903d8bdb54917c2ea91a0ee9c1dad7d5290ca2e62.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO owner_rels (cid, oid)\n SELECT $1,oid\n FROM UNNEST($2::int[]) as oid\n ON CONFLICT (cid,oid)\n DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "95bc487e596e04e5e1cfd20903d8bdb54917c2ea91a0ee9c1dad7d5290ca2e62" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323.json b/crates/bin/docs_rs_import_release/.sqlx/query-97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323.json new file mode 100644 index 000000000..33a64a917 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT releases.id, crates.name, releases.version, releases.repository_url\n FROM releases\n INNER JOIN crates ON (crates.id = releases.crate_id)\n WHERE repository_id IS NULL AND repository_url LIKE $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "version", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "repository_url", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + true + ] + }, + "hash": "97a9b51028cbf8e585e120382efdff87417d99c179c695ded1bdb6cd584a7323" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-9e7595c7b9b336b24241c133870b99e1ee70e750849956d268ef1cb6df4f53d4.json b/crates/bin/docs_rs_import_release/.sqlx/query-9e7595c7b9b336b24241c133870b99e1ee70e750849956d268ef1cb6df4f53d4.json new file mode 100644 index 000000000..fff5127ec --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-9e7595c7b9b336b24241c133870b99e1ee70e750849956d268ef1cb6df4f53d4.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET\n attempt = attempt + 1,\n last_attempt = NOW()\n WHERE id = $1\n RETURNING attempt;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "attempt", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9e7595c7b9b336b24241c133870b99e1ee70e750849956d268ef1cb6df4f53d4" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538.json b/crates/bin/docs_rs_import_release/.sqlx/query-a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538.json new file mode 100644 index 000000000..87fe73890 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT * FROM sandbox_overrides WHERE crate_name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_name", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "max_memory_bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "max_targets", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + true, + true, + true + ] + }, + "hash": "a3920d6701d1a80f23562ee83682d82ff35a52eeaa93ed45a97adc5e559d3538" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-a708e47c863614354e9a756da43b47d6071608366fd01f0d67db0991eb588b68.json b/crates/bin/docs_rs_import_release/.sqlx/query-a708e47c863614354e9a756da43b47d6071608366fd01f0d67db0991eb588b68.json new file mode 100644 index 000000000..9439ea53a --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-a708e47c863614354e9a756da43b47d6071608366fd01f0d67db0991eb588b68.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO builds(rid, build_status, build_server, build_started)\n VALUES ($1, $2, $3, NOW())\n RETURNING id as \"id: BuildId\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id: BuildId", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4", + { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a708e47c863614354e9a756da43b47d6071608366fd01f0d67db0991eb588b68" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-ada0debe54392d7ffc05d6dfa2ad33760cc3361550f19a26c179ff53a7a33064.json b/crates/bin/docs_rs_import_release/.sqlx/query-ada0debe54392d7ffc05d6dfa2ad33760cc3361550f19a26c179ff53a7a33064.json new file mode 100644 index 000000000..0ebe18d32 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-ada0debe54392d7ffc05d6dfa2ad33760cc3361550f19a26c179ff53a7a33064.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n priority,\n COUNT(*) as \"count!\"\n FROM queue\n GROUP BY priority", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null + ] + }, + "hash": "ada0debe54392d7ffc05d6dfa2ad33760cc3361550f19a26c179ff53a7a33064" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-c8328ef704887faa486f9caebba0ba39a115b8e278ac4c6bb6b67f2dafefcfbb.json b/crates/bin/docs_rs_import_release/.sqlx/query-c8328ef704887faa486f9caebba0ba39a115b8e278ac4c6bb6b67f2dafefcfbb.json new file mode 100644 index 000000000..b76920724 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-c8328ef704887faa486f9caebba0ba39a115b8e278ac4c6bb6b67f2dafefcfbb.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM queue WHERE id = $1;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [] + }, + "hash": "c8328ef704887faa486f9caebba0ba39a115b8e278ac4c6bb6b67f2dafefcfbb" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-cd568b56b5d3e43427218845ee19c4e9d598f61cf18eab47b36205b1aa1be301.json b/crates/bin/docs_rs_import_release/.sqlx/query-cd568b56b5d3e43427218845ee19c4e9d598f61cf18eab47b36205b1aa1be301.json new file mode 100644 index 000000000..5dc057abf --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-cd568b56b5d3e43427218845ee19c4e9d598f61cf18eab47b36205b1aa1be301.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM crate_priorities WHERE pattern = $1 RETURNING priority", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "priority", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "cd568b56b5d3e43427218845ee19c4e9d598f61cf18eab47b36205b1aa1be301" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2.json b/crates/bin/docs_rs_import_release/.sqlx/query-ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2.json new file mode 100644 index 000000000..4200b5330 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM repositories WHERE name = $1 AND host = $2 LIMIT 1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ce3c4ceec1fee051b9c9716d7e3dc94b213dc5449083dc109c1d935164e647e2" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa.json b/crates/bin/docs_rs_import_release/.sqlx/query-cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa.json new file mode 100644 index 000000000..6bfe26455 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM repositories WHERE host_id = $1 AND host = $2;", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "cf83cc24df4c285fe67808f1e1a2b56f850014b975a7506597a4ae3c5c6851aa" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-d484acb977a659075364b39e6f021e18cb969b2d735c5e7bb08e8eb4e34f8418.json b/crates/bin/docs_rs_import_release/.sqlx/query-d484acb977a659075364b39e6f021e18cb969b2d735c5e7bb08e8eb4e34f8418.json new file mode 100644 index 000000000..a5bdecf4a --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-d484acb977a659075364b39e6f021e18cb969b2d735c5e7bb08e8eb4e34f8418.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n crate_name as \"crate_name: KrateName\"\n FROM blacklisted_crates\n ORDER BY crate_name asc;\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_name: KrateName", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "d484acb977a659075364b39e6f021e18cb969b2d735c5e7bb08e8eb4e34f8418" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-d675aff9079a87e41891c589eaffa2497eba27fcee660f3d92241eaeda02ac0f.json b/crates/bin/docs_rs_import_release/.sqlx/query-d675aff9079a87e41891c589eaffa2497eba27fcee660f3d92241eaeda02ac0f.json new file mode 100644 index 000000000..154247910 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-d675aff9079a87e41891c589eaffa2497eba27fcee660f3d92241eaeda02ac0f.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE builds\n SET\n build_status = $1,\n errors = $2\n WHERE id = $3\n RETURNING rid as \"rid: ReleaseId\" ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "rid: ReleaseId", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + { + "Custom": { + "name": "build_status", + "kind": { + "Enum": [ + "in_progress", + "success", + "failure" + ] + } + } + }, + "Text", + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "d675aff9079a87e41891c589eaffa2497eba27fcee660f3d92241eaeda02ac0f" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579.json b/crates/bin/docs_rs_import_release/.sqlx/query-de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579.json new file mode 100644 index 000000000..945d619af --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM sandbox_overrides WHERE crate_name = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "de4ba149a561c4bb467bcea081bdedff233398cddf4996734a64536b6a8c6579" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-e4c989a6678cff1c7187dd737f97d0548e814d98b6e161de1c39457d25c10e3b.json b/crates/bin/docs_rs_import_release/.sqlx/query-e4c989a6678cff1c7187dd737f97d0548e814d98b6e161de1c39457d25c10e3b.json new file mode 100644 index 000000000..150257361 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-e4c989a6678cff1c7187dd737f97d0548e814d98b6e161de1c39457d25c10e3b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT 1 FROM blacklisted_crates WHERE crate_name = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "e4c989a6678cff1c7187dd737f97d0548e814d98b6e161de1c39457d25c10e3b" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-e550304c4f4f1b7aba42b7eb0eb7c1fb7732b174f49c76a145ae86fad2556671.json b/crates/bin/docs_rs_import_release/.sqlx/query-e550304c4f4f1b7aba42b7eb0eb7c1fb7732b174f49c76a145ae86fad2556671.json new file mode 100644 index 000000000..530ae1ffe --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-e550304c4f4f1b7aba42b7eb0eb7c1fb7732b174f49c76a145ae86fad2556671.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET priority = GREATEST(priority, $1)\n WHERE\n name = $2\n AND version != $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "e550304c4f4f1b7aba42b7eb0eb7c1fb7732b174f49c76a145ae86fad2556671" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-ea8226cc06934df7dd79e1aab9dc427a5e21c76af88b408a911b6d3b82cdd9d2.json b/crates/bin/docs_rs_import_release/.sqlx/query-ea8226cc06934df7dd79e1aab9dc427a5e21c76af88b408a911b6d3b82cdd9d2.json new file mode 100644 index 000000000..57870465c --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-ea8226cc06934df7dd79e1aab9dc427a5e21c76af88b408a911b6d3b82cdd9d2.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n id,\n name as \"name: KrateName\",\n version as \"version: Version\",\n priority,\n registry,\n attempt\n FROM queue\n WHERE\n last_attempt IS NULL OR last_attempt < NOW() - make_interval(secs => $1)\n ORDER BY priority ASC, attempt ASC, id ASC\n LIMIT 1\n FOR UPDATE SKIP LOCKED", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name: KrateName", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "version: Version", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "priority", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "registry", + "type_info": "Text" + }, + { + "ordinal": 5, + "name": "attempt", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Float8" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false + ] + }, + "hash": "ea8226cc06934df7dd79e1aab9dc427a5e21c76af88b408a911b6d3b82cdd9d2" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-f8b389df3451e4b5e6539e9260ba6340edf69c7dba22e667aedd510e868b0f00.json b/crates/bin/docs_rs_import_release/.sqlx/query-f8b389df3451e4b5e6539e9260ba6340edf69c7dba22e667aedd510e868b0f00.json new file mode 100644 index 000000000..937d9c012 --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-f8b389df3451e4b5e6539e9260ba6340edf69c7dba22e667aedd510e868b0f00.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE\n FROM queue\n WHERE name = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "f8b389df3451e4b5e6539e9260ba6340edf69c7dba22e667aedd510e868b0f00" +} diff --git a/crates/bin/docs_rs_import_release/.sqlx/query-fe36361977b1d8857f1bbd4d1c30ef29516987d053d35a9afc0a09fee31cd1c8.json b/crates/bin/docs_rs_import_release/.sqlx/query-fe36361977b1d8857f1bbd4d1c30ef29516987d053d35a9afc0a09fee31cd1c8.json new file mode 100644 index 000000000..4223b877d --- /dev/null +++ b/crates/bin/docs_rs_import_release/.sqlx/query-fe36361977b1d8857f1bbd4d1c30ef29516987d053d35a9afc0a09fee31cd1c8.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n crate_name as \"crate_name: KrateName\",\n max_memory_bytes,\n timeout_seconds,\n max_targets\n FROM sandbox_overrides\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "crate_name: KrateName", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "max_memory_bytes", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "timeout_seconds", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "max_targets", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + true, + true, + true + ] + }, + "hash": "fe36361977b1d8857f1bbd4d1c30ef29516987d053d35a9afc0a09fee31cd1c8" +} diff --git a/crates/bin/docs_rs_import_release/Cargo.toml b/crates/bin/docs_rs_import_release/Cargo.toml new file mode 100644 index 000000000..53feb0fc7 --- /dev/null +++ b/crates/bin/docs_rs_import_release/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "docs_rs_import_release" +version.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +edition.workspace = true +description = "Import a successfully built release from docs.rs into a test deployment" + +[dependencies] +anyhow = { workspace = true } +async-tar = { version = "0.6.0", default-features = false, features = ["runtime-tokio", "xattr"] } +clap = { workspace = true } +docs_rs_cargo_metadata = { path = "../../lib/docs_rs_cargo_metadata" } +docs_rs_context = { path = "../../lib/docs_rs_context" } +docs_rs_database = { path = "../../lib/docs_rs_database" } +docs_rs_logging = { path = "../../lib/docs_rs_logging" } +docs_rs_registry_api = { path = "../../lib/docs_rs_registry_api" } +docs_rs_repository_stats = { path = "../../lib/docs_rs_repository_stats" } +docs_rs_rustdoc_json = { path = "../../lib/docs_rs_rustdoc_json" } +docs_rs_storage = { path = "../../lib/docs_rs_storage" } +docs_rs_types = { path = "../../lib/docs_rs_types" } +docs_rs_utils = { path = "../../lib/docs_rs_utils" } +docsrs-metadata = { path = "../../lib/metadata" } +futures-util = { workspace = true } +regex = { workspace = true } +reqwest = { workspace = true } +serde = { workspace = true } +sqlx = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +walkdir = { workspace = true } +zip = { workspace = true } diff --git a/crates/bin/docs_rs_import_release/src/common.rs b/crates/bin/docs_rs_import_release/src/common.rs new file mode 100644 index 000000000..f37e5b454 --- /dev/null +++ b/crates/bin/docs_rs_import_release/src/common.rs @@ -0,0 +1,50 @@ +use anyhow::Result; +use docs_rs_utils::{APP_USER_AGENT, spawn_blocking}; +use futures_util::StreamExt as _; +use std::{fmt, sync::LazyLock}; +use tokio::{ + fs, + io::{AsyncSeekExt as _, AsyncWriteExt as _}, +}; +use tracing::debug; + +pub(crate) const DOCS_RS: &str = "https://docs.rs"; +pub(crate) static CLIENT: LazyLock = LazyLock::new(|| { + reqwest::Client::builder() + .user_agent(APP_USER_AGENT) + .build() + .expect("can't create request client & connection pool") +}); + +pub(crate) async fn download(url: impl reqwest::IntoUrl + fmt::Debug) -> Result> { + debug!("downloading..."); + + Ok(CLIENT + .get(url) + .send() + .await? + .error_for_status()? + .bytes() + .await? + .to_vec()) +} + +pub(crate) async fn download_to_temp_file(url: impl reqwest::IntoUrl) -> Result { + debug!("downloading to temp file.."); + + let response = CLIENT.get(url).send().await?.error_for_status()?; + + // NOTE: even after being convert to a `tokio::fs::File`, this kind of temporary file + // will be cleaned up by the OS, when the last handle is closed. + let mut file = fs::File::from_std(spawn_blocking(|| Ok(tempfile::tempfile()?)).await?); + + let mut stream = response.bytes_stream(); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + file.write_all(&chunk).await?; + } + + file.sync_all().await?; + file.seek(std::io::SeekFrom::Start(0)).await?; + Ok(file) +} diff --git a/crates/bin/docs_rs_import_release/src/crates_io.rs b/crates/bin/docs_rs_import_release/src/crates_io.rs new file mode 100644 index 000000000..f1b2ba2ab --- /dev/null +++ b/crates/bin/docs_rs_import_release/src/crates_io.rs @@ -0,0 +1,55 @@ +use crate::common::download_to_temp_file; +use anyhow::{Result, bail}; +use async_tar::Archive; +use docs_rs_storage::compression::wrap_reader_for_decompression; +use docs_rs_types::{CompressionAlgorithm, KrateName, Version}; +use docs_rs_utils::spawn_blocking; +use std::path::{Path, PathBuf}; +use tokio::io; +use tracing::debug; + +#[derive(Debug)] +pub(crate) struct SourceDir { + _temp_dir: tempfile::TempDir, + pub(crate) source_path: PathBuf, +} + +impl AsRef for SourceDir { + fn as_ref(&self) -> &Path { + &self.source_path + } +} + +pub(crate) async fn download_and_extract_source( + name: &KrateName, + version: &Version, +) -> Result { + debug!("downloading source"); + let crate_archive = download_to_temp_file(format!( + "https://static.crates.io/crates/{name}/{name}-{version}.crate" + )) + .await?; + + let temp_dir = spawn_blocking(|| Ok(tempfile::tempdir()?)).await?; + + debug!("unpacking source archive"); + { + let mut file = io::BufReader::new(crate_archive); + let mut decompressed = wrap_reader_for_decompression(&mut file, CompressionAlgorithm::Gzip); + let archive = Archive::new(&mut decompressed); + archive.unpack(&temp_dir).await?; + } + + let source_path = temp_dir.path().join(format!("{name}-{version}")); + if !source_path.is_dir() { + bail!( + "broken crate archive, missing source directory {:?}", + source_path + ); + }; + + Ok(SourceDir { + source_path, + _temp_dir: temp_dir, + }) +} diff --git a/crates/bin/docs_rs_import_release/src/import.rs b/crates/bin/docs_rs_import_release/src/import.rs new file mode 100644 index 000000000..dbf90a921 --- /dev/null +++ b/crates/bin/docs_rs_import_release/src/import.rs @@ -0,0 +1,249 @@ +use crate::{ + common::{DOCS_RS, download, download_to_temp_file}, + crates_io::download_and_extract_source, + rustdoc::{download_static_files, find_static_paths, find_successful_build_targets}, + rustdoc_status::fetch_rustdoc_status, +}; +use anyhow::{Result, bail}; +use docs_rs_cargo_metadata::CargoMetadata; +use docs_rs_database::releases::{ + finish_build, finish_release, initialize_build, initialize_crate, initialize_release, + update_build_with_error, +}; +use docs_rs_registry_api::RegistryApi; +use docs_rs_repository_stats::RepositoryStatsUpdater; +use docs_rs_rustdoc_json::{ + RUSTDOC_JSON_COMPRESSION_ALGORITHMS, RustdocJsonFormatVersion, + read_format_version_from_rustdoc_json, +}; +use docs_rs_storage::{AsyncStorage, file_list_to_json, rustdoc_archive_path, source_archive_path}; +use docs_rs_storage::{compress, decompress, rustdoc_json_path}; +use docs_rs_types::{BuildId, BuildStatus, CrateId, KrateName, ReleaseId, ReqVersion, Version}; +use docs_rs_utils::{BUILD_VERSION, spawn_blocking}; +use docsrs_metadata::Metadata; +use std::collections::HashSet; +use tracing::{info, instrument}; + +const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; + +/// import an existing crate release build from docs.rs into the +/// local database & storage. +/// +/// CAVEATS: +/// * is currently only tested for newer releases, since there are some hacks in place. +/// * to find the needed rustdoc-static files, we have to scan all the HTML files for certain paths. +/// For bigger releases this might take some time. +/// * we assume when the normal target build is successfull, we also have a valid rustdoc json file, +/// and we'll ignore any rustdoc JSON files related to failed targets. +/// * build logs are fake, but are created. +/// +/// SECURITY: +/// we execute `cargo metadata` on the downloaded source code, so +/// this function MUST NOT be used with untrusted crate names/versions. +pub(crate) async fn import_test_release( + conn: &mut sqlx::PgConnection, + storage: &AsyncStorage, + registry_api: &RegistryApi, + repository_stats: &RepositoryStatsUpdater, + name: &KrateName, + version: &ReqVersion, +) -> Result<()> { + let status = fetch_rustdoc_status(name, version).await?; + if !status.doc_status { + bail!("No rustdoc available for {name} {version}"); + } + let version = status.version; + + let crate_id = initialize_crate(&mut *conn, name).await?; + let release_id = initialize_release(&mut *conn, crate_id, &version).await?; + let build_id = initialize_build(&mut *conn, release_id).await?; + + let result = import_test_release_inner( + &mut *conn, + storage, + registry_api, + repository_stats, + name, + &version, + crate_id, + release_id, + build_id, + ) + .await; + + if let Err(err) = &result { + update_build_with_error(&mut *conn, build_id, Some(&format!("{err:?}"))).await?; + } + + result +} + +#[allow(clippy::too_many_arguments)] +#[instrument(skip_all, fields(name=%name, version=%version))] +async fn import_test_release_inner( + conn: &mut sqlx::PgConnection, + storage: &AsyncStorage, + registry_api: &RegistryApi, + repository_stats: &RepositoryStatsUpdater, + name: &KrateName, + version: &Version, + crate_id: CrateId, + release_id: ReleaseId, + build_id: BuildId, +) -> Result<()> { + info!("download & inspect source from crates.io..."); + let source_dir = download_and_extract_source(name, version).await?; + + let cargo_metadata = spawn_blocking({ + let source_dir = source_dir.source_path.clone(); + move || CargoMetadata::load_from_host_path(&source_dir) + }) + .await?; + let docsrs_metadata = spawn_blocking({ + let source_dir = source_dir.source_path.clone(); + move || Ok(Metadata::from_crate_root(&source_dir)?) + }) + .await?; + + let mut algs = HashSet::new(); + let (source_files_list, source_size) = { + info!("writing source files to storage..."); + let (files_list, new_alg) = storage + .store_all_in_archive(&source_archive_path(name, version), &source_dir) + .await?; + + algs.insert(new_alg); + let source_size: u64 = files_list.iter().map(|info| info.size).sum(); + (files_list, source_size) + }; + + let registry_data = registry_api.get_release_data(name, version).await?; + + let rustdoc_dir = { + info!("download & extract rustdoc archive..."); + let rustdoc_archive = + download_to_temp_file(format!("{DOCS_RS}/crate/{name}/{version}/download")) + .await? + .into_std() + .await; + + spawn_blocking(|| { + let mut zip = zip::ZipArchive::new(rustdoc_archive)?; + + let temp_dir = tempfile::tempdir()?; + zip.extract(&temp_dir)?; + Ok(temp_dir) + }) + .await? + }; + + info!("find successfull build targets..."); + let (default_target, all_targets) = { + let build_targets = docsrs_metadata.targets_for_host(true, DEFAULT_TARGET); + ( + build_targets.default_target, + find_successful_build_targets( + &rustdoc_dir, + build_targets.default_target, + build_targets.other_targets, + ) + .await?, + ) + }; + + info!("uploading fake build logs"); + for build_target in &all_targets { + storage + .store_one( + format!("build-logs/{build_id}/{build_target}.txt"), + format!("fake build output\nbuild target: {}", build_target), + ) + .await?; + } + + info!("finding used rustdoc static files in HTML..."); + { + let static_files = find_static_paths(&rustdoc_dir).await?; + download_static_files(storage, static_files.iter().map(AsRef::as_ref)).await?; + } + + info!("writing rustdoc files to storage..."); + let (rustdoc_file_list, new_alg) = storage + .store_all_in_archive(&rustdoc_archive_path(name, version), &rustdoc_dir) + .await?; + let documentation_size: u64 = rustdoc_file_list.iter().map(|info| info.size).sum(); + algs.insert(new_alg); + + info!("loading repository stats..."); + let repository_id = repository_stats + .load_repository(cargo_metadata.root()) + .await?; + + for target in &all_targets { + info!("copying rustdoc json for target {target}..."); + + let json_compression = RUSTDOC_JSON_COMPRESSION_ALGORITHMS[0]; + let rustdoc_json = decompress( + &*download(format!( + "{DOCS_RS}/crate/{name}/{version}/{target}/json.{}", + json_compression.file_extension() + )) + .await?, + json_compression, + usize::MAX, + )?; + if rustdoc_json.is_empty() || rustdoc_json[0] != b'{' { + bail!("invalid rustdoc json for {name} {version} {target}"); + } + + let format_version = spawn_blocking({ + let rustdoc_json = rustdoc_json.clone(); + move || read_format_version_from_rustdoc_json(&*rustdoc_json) + }) + .await?; + + for alg in RUSTDOC_JSON_COMPRESSION_ALGORITHMS { + let compressed_json = compress(&*rustdoc_json, *alg)?; + + for format_version in [format_version, RustdocJsonFormatVersion::Latest] { + let path = rustdoc_json_path(name, version, target, format_version, Some(*alg)); + storage + .store_one_uncompressed(&path, compressed_json.clone()) + .await?; + } + } + } + + info!("finish release & build"); + finish_release( + &mut *conn, + crate_id, + release_id, + cargo_metadata.root(), + &source_dir, + default_target, + file_list_to_json(source_files_list), + all_targets, + ®istry_data, + true, + false, // FIXME: real has_examples? + algs, + repository_id, + true, + source_size, + ) + .await?; + + finish_build( + &mut *conn, + build_id, + "rustc 1.95.0-nightly (873d4682c 2026-01-25)", + BUILD_VERSION, + BuildStatus::Success, + Some(documentation_size), + None, + ) + .await?; + + Ok(()) +} diff --git a/crates/bin/docs_rs_import_release/src/main.rs b/crates/bin/docs_rs_import_release/src/main.rs new file mode 100644 index 000000000..f81601950 --- /dev/null +++ b/crates/bin/docs_rs_import_release/src/main.rs @@ -0,0 +1,64 @@ +pub(crate) mod common; +pub(crate) mod crates_io; +mod import; +mod rustdoc; +pub(crate) mod rustdoc_status; + +use anyhow::{Context as _, Result}; +use clap::Parser; +use docs_rs_context::Context; +use docs_rs_types::{KrateName, ReqVersion}; + +#[tokio::main] +async fn main() -> Result<()> { + let _guard = docs_rs_logging::init().context("error initializing logging")?; + + if let Err(err) = CommandLine::parse().handle_args().await { + eprintln!("error importing release: {err:?}"); + drop(_guard); + std::process::exit(1); + } + + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq, Parser)] +#[command( + about = env!("CARGO_PKG_DESCRIPTION"), + version = docs_rs_utils::BUILD_VERSION, + rename_all = "kebab-case", +)] +struct CommandLine { + #[arg(name = "CRATE")] + name: KrateName, + + #[arg(name = "CRATE_VERSION", default_value_t)] + version: ReqVersion, +} + +impl CommandLine { + async fn handle_args(self) -> Result<()> { + let ctx = Context::builder() + .with_runtime() + .await? + .with_meter_provider()? + .with_pool() + .await? + .with_storage() + .await? + .with_registry_api()? + .with_repository_stats()? + .build()?; + + let mut conn = ctx.pool()?.get_async().await?; + import::import_test_release( + &mut conn, + ctx.storage()?, + ctx.registry_api()?, + ctx.repository_stats()?, + &self.name, + &self.version, + ) + .await + } +} diff --git a/crates/bin/docs_rs_import_release/src/rustdoc.rs b/crates/bin/docs_rs_import_release/src/rustdoc.rs new file mode 100644 index 000000000..4054fbb6a --- /dev/null +++ b/crates/bin/docs_rs_import_release/src/rustdoc.rs @@ -0,0 +1,168 @@ +use crate::common::{DOCS_RS, download}; +use anyhow::{Result, bail}; +use docs_rs_storage::AsyncStorage; +use docs_rs_utils::spawn_blocking; +use futures_util::{StreamExt as _, TryStreamExt as _, stream}; +use regex::Regex; +use std::{collections::HashSet, fmt, path::Path, sync::LazyLock}; +use tokio::{ + fs, + io::{self, AsyncBufReadExt as _}, + process::Command, +}; +use tracing::debug; +use walkdir::WalkDir; + +const KNOWN_STATIC_PATHS: &[&str] = &[ + "/-/rustdoc.static/FiraSans-Italic-81dc35de.woff2", + "/-/rustdoc.static/FiraSans-Medium-e1aa3f0a.woff2", + "/-/rustdoc.static/FiraSans-MediumItalic-ccf7e434.woff2", + "/-/rustdoc.static/FiraSans-Regular-0fe48ade.woff2", + "/-/rustdoc.static/SourceCodePro-Regular-8badfe75.ttf.woff2", + "/-/rustdoc.static/SourceCodePro-Semibold-aa29a496.ttf.woff2", + "/-/rustdoc.static/SourceSerif4-Regular-6b053e98.ttf.woff2", +]; + +pub(crate) async fn find_static_paths( + root_dir: impl AsRef + fmt::Debug, +) -> Result> { + static RE: LazyLock = LazyLock::new(|| { + Regex::new(&format!( + r#""({}[^"]+)""#, + docs_rs_utils::RUSTDOC_STATIC_PATH.replace(".", "\\.") + )) + .unwrap() + }); + + debug!("finding HTML files..."); + let root_dir = root_dir.as_ref(); + let html_files = spawn_blocking({ + let root_dir = root_dir.to_path_buf(); + move || { + let mut files = Vec::new(); + for entry in WalkDir::new(&root_dir).follow_links(false).into_iter() { + let entry = entry?; + let path = entry.path(); + if entry.file_type().is_file() + && path + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| ext.eq_ignore_ascii_case("html")) + { + files.push(path.to_path_buf()); + } + } + Ok(files) + } + }) + .await?; + + debug!( + count = html_files.len(), + "finding static URLs in HTML files..." + ); + const MAX_RUSTDOC_STATIC_FILE_COUNT: usize = 64; + let mut urls = stream::iter(html_files) + .map(|path| async move { + let reader = io::BufReader::new(fs::File::open(&path).await?); + let mut lines = reader.lines(); + + let mut matches = HashSet::with_capacity(MAX_RUSTDOC_STATIC_FILE_COUNT); + while let Some(line) = lines.next_line().await? { + matches.extend( + RE.captures_iter(&line) + .filter_map(|cap| cap.get(1).map(|m| m.as_str().to_string())), + ); + } + + Ok::<_, anyhow::Error>(matches) + }) + .buffer_unordered(16) + .try_fold( + HashSet::with_capacity(MAX_RUSTDOC_STATIC_FILE_COUNT), + |mut urls, matches| async move { + urls.extend(matches); + Ok(urls) + }, + ) + .await?; + + // these files aren't referenced directly in the HTML code, but their imports + // are generated through JS. + // Since these are statically known and barely change, I can just add them here. + urls.remove("/-/rustdoc.static/${f}"); + urls.extend(KNOWN_STATIC_PATHS.iter().map(ToString::to_string)); + + Ok(urls) +} + +pub(crate) async fn download_static_files( + storage: &AsyncStorage, + paths: impl IntoIterator, +) -> Result<()> { + for path in paths { + let key = format!( + "{}{}", + docs_rs_utils::RUSTDOC_STATIC_STORAGE_PREFIX, + path.trim_start_matches(docs_rs_utils::RUSTDOC_STATIC_PATH) + ); + + if storage.exists(&key).await? { + debug!("static file already exists in storage: {}", &key); + continue; + } + + storage + .store_one(key, download(format!("{DOCS_RS}{path}")).await?) + .await?; + } + + Ok(()) +} + +pub(crate) async fn find_successful_build_targets( + rustdoc_dir: impl AsRef + fmt::Debug, + default_target: &str, + other_targets: impl IntoIterator, +) -> Result> { + let rustdoc_dir = rustdoc_dir.as_ref(); + + let mut potential_other_targets: HashSet = + other_targets.into_iter().map(ToString::to_string).collect(); + potential_other_targets.extend(fetch_target_list().await?.into_iter()); + potential_other_targets.remove(default_target); + + let mut targets: Vec = vec![default_target.into()]; + for t in potential_other_targets { + if rustdoc_dir.join(&t).is_dir() { + // non-default targets lead to a subdirectory in rustdoc + targets.push(t); + } + } + + Ok(targets) +} + +async fn fetch_target_list() -> Result> { + let output = Command::new("rustc") + .arg("--print") + .arg("target-list") + .output() + .await?; + + if !output.status.success() { + bail!( + "`rustc --print target-list` failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + + let stdout = String::from_utf8(output.stdout)?; + + Ok(stdout + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .map(str::to_owned) + .collect()) +} diff --git a/crates/bin/docs_rs_import_release/src/rustdoc_status.rs b/crates/bin/docs_rs_import_release/src/rustdoc_status.rs new file mode 100644 index 000000000..2a7a1c7f1 --- /dev/null +++ b/crates/bin/docs_rs_import_release/src/rustdoc_status.rs @@ -0,0 +1,25 @@ +use crate::common::{CLIENT, DOCS_RS}; +use anyhow::Result; +use docs_rs_types::{KrateName, ReqVersion, Version}; +use serde::Deserialize; +use tracing::debug; + +#[derive(Debug, Deserialize)] +pub(crate) struct RustdocStatus { + pub(crate) doc_status: bool, + pub(crate) version: Version, +} + +pub(crate) async fn fetch_rustdoc_status( + name: &KrateName, + version: &ReqVersion, +) -> Result { + debug!("fetching rustdoc status..."); + Ok(CLIENT + .get(format!("{DOCS_RS}/crate/{name}/{version}/status.json")) + .send() + .await? + .error_for_status()? + .json() + .await?) +} diff --git a/crates/lib/docs_rs_cargo_metadata/src/metadata.rs b/crates/lib/docs_rs_cargo_metadata/src/metadata.rs index e9b2b3c52..9655288c8 100644 --- a/crates/lib/docs_rs_cargo_metadata/src/metadata.rs +++ b/crates/lib/docs_rs_cargo_metadata/src/metadata.rs @@ -1,25 +1,25 @@ use anyhow::{Context, Result}; use docs_rs_types::{Version, VersionReq}; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use std::{collections::BTreeMap, path::Path, process, str}; pub struct CargoMetadata { root: Package, } impl CargoMetadata { - #[cfg(feature = "testing")] - pub fn load_from_host_path(source_dir: &std::path::Path) -> Result { - let res = std::process::Command::new("cargo") - .args(["metadata", "--format-version", "1", "--offline"]) + pub fn load_from_host_path(source_dir: impl AsRef) -> Result { + let source_dir = source_dir.as_ref(); + let res = process::Command::new("cargo") + .args(["metadata", "--format-version", "1"]) .current_dir(source_dir) .output()?; let status = res.status; if !status.success() { - let stderr = std::str::from_utf8(&res.stderr).unwrap_or(""); + let stderr = str::from_utf8(&res.stderr).unwrap_or(""); anyhow::bail!("error returned by `cargo metadata`: {status}\n{stderr}") } - Self::load_from_metadata(std::str::from_utf8(&res.stdout)?) + Self::load_from_metadata(str::from_utf8(&res.stdout)?) } pub fn load_from_metadata(metadata: &str) -> Result { diff --git a/crates/lib/docs_rs_database/src/releases.rs b/crates/lib/docs_rs_database/src/releases.rs index 8de5e3ac0..0fd23d182 100644 --- a/crates/lib/docs_rs_database/src/releases.rs +++ b/crates/lib/docs_rs_database/src/releases.rs @@ -11,7 +11,7 @@ use serde_json::Value; use slug::slugify; use std::{ collections::{HashMap, HashSet}, - fs, + fmt, fs, io::{BufRead, BufReader}, path::Path, }; @@ -30,7 +30,7 @@ pub async fn finish_release( crate_id: CrateId, release_id: ReleaseId, metadata_pkg: &MetadataPackage, - source_dir: &Path, + source_dir: impl AsRef + fmt::Debug, default_target: &str, source_files: Value, doc_targets: Vec, @@ -42,6 +42,7 @@ pub async fn finish_release( archive_storage: bool, source_size: u64, ) -> Result<()> { + let source_dir = source_dir.as_ref(); debug!("updating release data"); let dependencies: ReleaseDependencyList = metadata_pkg .dependencies diff --git a/crates/lib/docs_rs_storage/Cargo.toml b/crates/lib/docs_rs_storage/Cargo.toml index a5e056b1f..bcab326ba 100644 --- a/crates/lib/docs_rs_storage/Cargo.toml +++ b/crates/lib/docs_rs_storage/Cargo.toml @@ -47,7 +47,7 @@ thiserror = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } walkdir = { workspace = true } -zip = { version = "7.0.0", default-features = false, features = ["bzip2"] } +zip = { workspace = true } zstd = "0.13.0" [dev-dependencies] diff --git a/crates/lib/docs_rs_utils/src/lib.rs b/crates/lib/docs_rs_utils/src/lib.rs index bafc3fd56..37429eebb 100644 --- a/crates/lib/docs_rs_utils/src/lib.rs +++ b/crates/lib/docs_rs_utils/src/lib.rs @@ -39,6 +39,9 @@ pub const APP_USER_AGENT: &str = concat!( /// `s3://rust-docs-rs//rustdoc-static/something.css` pub const RUSTDOC_STATIC_STORAGE_PREFIX: &str = "/rustdoc-static/"; +/// Where we want to serve the rustdoc static files stored in the storage prefix above +pub const RUSTDOC_STATIC_PATH: &str = "/-/rustdoc.static/"; + /// a wrapper around tokio's `spawn_blocking` that /// enables us to write nicer code when the closure /// returns an `anyhow::Result`. diff --git a/crates/lib/metadata/lib.rs b/crates/lib/metadata/lib.rs index 32e0525a5..bdc29c2cb 100644 --- a/crates/lib/metadata/lib.rs +++ b/crates/lib/metadata/lib.rs @@ -206,11 +206,20 @@ impl Metadata { /// /// All of the above is ignored for proc-macros, which are always only compiled for the host. pub fn targets(&self, include_default_targets: bool) -> BuildTargets<'_> { + self.targets_for_host(include_default_targets, HOST_TARGET) + } + + /// Return the targets that should be built, given a different simulated HOST_TARGET. + pub fn targets_for_host( + &self, + include_default_targets: bool, + host_target: &'static str, + ) -> BuildTargets<'_> { // Proc macros can only be compiled for the host, so just completely ignore any configured targets. // It would be nice to warn about this somehow ... if self.proc_macro { return BuildTargets { - default_target: HOST_TARGET, + default_target: host_target, other_targets: HashSet::default(), }; } @@ -224,7 +233,7 @@ impl Metadata { .as_ref() .and_then(|targets| targets.first().map(String::as_str)) }) - .unwrap_or(HOST_TARGET); + .unwrap_or(host_target); let crate_targets = self .targets diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index e165a4e00..95f51b393 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -210,4 +210,3 @@ COPY --from=build /artifacts/docs_rs_admin /usr/local/bin/ COPY --from=build /artifacts/cratesfyi /usr/local/bin ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/docs_rs_admin", "--"] -