Sglang inferencing in distributed system

cvs/input/config_file/inference/sglang/mi30x_sglang_distributed.json

@@ -0,0 +1,188 @@
+{
+    "config":
+    {
+        "container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260603",
+        "_container_image": "rocm/sgl-dev:v0.5.12.post1-rocm720-mi30x-20260601",
+        "container_name": "sglang_container",
+        "_example_nnodes": "4",
+        "nnodes": "2",
+        "hf_token_file": "/home/{user-id}/.hf_token",
+        "shm_size": "128G",
+	"_log_dir_comments": "Provide some common file system that is accessible from any node",
+        "log_dir": "/home/{user-id}/LOGS/sglang",
+	"log_level": "info",
+	"nic_type": "thor2",
+	"_example_nccl_ib_hca_list": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7",
+        "nccl_ib_hca_list": "<changeme>",
+        "_example_nccl_ib_hca": "rdma0,rdma1,rdma2,rdma3,rdma4,rdma5,rdma6,rdma7",
+        "nccl_ib_hca": "<changeme>",
+	"_example_nccl_socket_ifname": "eno0",
+        "nccl_socket_ifname": "<changeme>",
+	"_example_gloo_socket_ifname": "eno0",
+        "gloo_socket_ifname": "<changeme>",
+	"_example_gloo_tcp_ifname": "eno0",
+	"gloo_tcp_ifname": "<changeme>",
+	"nccl_ib_gid_index": "3",
+	"nccl_debug": "ERROR",
+	"prefill_node_list": ["<changeme>", "<changeme>"],
+        "decode_node_list": ["<changeme>", "<changeme>"],
+        "proxy_router_node": "<changeme>",
+        "benchmark_serv_node": "<changeme>",
+	"prefill_serv_port": "30001",
+	"decode_serv_port": "30002",
+	"proxy_router_port": "8000",
+	"_prefill_coordinator_addr": "This is the master address for co-ordination among all prefill nodes",
+	"prefill_coordinator_addr": "<changeme>",
+	"_decode_coordinator_addr": "This is the master address for co-ordination among all decode nodes",
+	"decode_coordinator_addr": "<changeme>",
+	"prefill_coordinator_port": "40001",
+	"decode_coordinator_port": "40002",
+	"proxy_router_serv_port": "8000",
+        "container_config":
+        {
+	  "device_list": [ "/dev/dri", "/dev/kfd", "/dev/infiniband/rdma_cm" ],
+	  "volume_dict":
+	  {
+	      "/home/{user-id}": "/home/{user-id}",
+	      "/mnt/dtni/models": "/root/models",
+	      "/dev/infiniband": "/dev/infiniband",
+	      "/usr/local/lib/libbnxt_re-rdmav34.so": "/usr/lib/x86_64-linux-gnu/libibverbs/libbnxt_re-rdmav34.so.host",
+	      "/lib/libibverbs.d": "/lib/libibverbs.d"
+          },
+		"env_dict":
+		{
+		}
+        }
+
+    },
+    "benchmark_params":
+    {
+        "llama-70b":
+             {
+		 "backend": "sglang",
+		 "max_concurrency": "25",
+		 "model": "meta-llama/Llama-3.1-70B-Instruct",
+		 "prefill_policy": "cache_aware",
+		 "decode_policy": "cache_aware",
+		 "tensor_parallelism": "8",
+		 "memory_fraction": "0.85",
+		 "tokenizer_mode": "auto",
+		 "inference_poll_iterations": "16",
+		 "inference_tests":
+		  {
+		      "gsm8k":
+	              {
+			  "backend": "sglang",
+			  "num_questions": "1000",
+			  "max_concurrency": "25",
+                          "expected_results":
+		          {
+			       "auto":
+			           {
+			               "tokens_per_sec": "350"
+				   }
+			  }
+	              },
+		      "bench_serv_random":
+		      {
+			  "backend": "sglang",
+			  "data_set_name": "random",
+			  "num_prompts": "100",
+			  "input_length": "1024",
+			  "output_length": "1024",
+			  "random_range_ratio": "0.5",
+                          "expected_results":
+		          {
+			       "auto":
+			           {
+                   		       "output_throughput_per_sec": "1000",
+                                       "mean_ttft_ms": "60000",
+                                       "mean_tpot_ms": "150"
+				   }
+			  }
+		      },
+		      "bench_serv_generated_shared_prefix":
+		      {
+			  "backend": "sglang",
+			  "gsp_num_groups": "1",
+			  "gsp_prompts_per_group": "16",
+			  "gsp_system_prompt_len": "0",
+			  "gsp_question_len": "1024",
+			  "gsp_output_len": "1024",
+                          "expected_results":
+		          {
+			       "auto":
+			           {
+				   }
+			  }
+		      }
+
+		  }
+             },
+        "deepseek-r1":
+             {
+		 "backend": "sglang",
+		 "max_concurrency": "64",
+		 "_comments_model": "If the model is local, specify the full path of the model",
+		 "model": "/root/models/DeepSeek-R1-0528",
+		 "prefill_policy": "cache_aware",
+		 "decode_policy": "cache_aware",
+		 "tensor_parallelism": "16",
+		 "memory_fraction": "0.85",
+		 "tokenizer_mode": "auto",
+		 "inference_poll_iterations": "16",
+		 "inference_tests":
+		  {
+		      "gsm8k":
+	              {
+			  "backend": "sglang",
+			  "num_questions": "1000",
+			  "max_concurrency": "100",
+                          "expected_results":
+		          {
+			       "auto":
+			           {
+			               "tokens_per_sec": "700"
+				   }
+			  }
+	              },
+		      "bench_serv_random":
+		      {
+			  "backend": "sglang",
+			  "data_set_name": "random",
+			  "num_prompts": "100",
+			  "input_length": "1024",
+			  "output_length": "1024",
+			  "random_range_ratio": "0.5",
+                          "expected_results":
+		          {
+			       "auto":
+			           {
+                   		       "output_throughput_per_sec": "1400",
+                                       "mean_ttft_ms": "60000",
+                                       "mean_tpot_ms": "110"
+				   }
+			  }
+		      },
+		      "bench_serv_generated_shared_prefix":
+		      {
+			  "backend": "sglang",
+			  "gsp_num_groups": "1",
+			  "gsp_prompts_per_group": "16",
+			  "gsp_system_prompt_len": "0",
+			  "gsp_question_len": "1024",
+			  "gsp_output_len": "1024",
+                          "expected_results":
+		          {
+			       "auto":
+			           {
+				   }
+			  }
+		      }
+
+		  }
+             }
+
+        }
+
+}