[WIP]feat: support llpkg for building

internal/build/build.go

@@ -40,8 +40,10 @@ import (
 	"github.com/goplus/llgo/cl"
 	"github.com/goplus/llgo/internal/crosscompile"
 	"github.com/goplus/llgo/internal/env"
+	"github.com/goplus/llgo/internal/llpkg"
 	"github.com/goplus/llgo/internal/mockable"
 	"github.com/goplus/llgo/internal/packages"
+	"github.com/goplus/llgo/internal/taskqueue"
 	"github.com/goplus/llgo/internal/typepatch"
 	"github.com/goplus/llgo/ssa/abi"
 	"github.com/goplus/llgo/xtool/clang"
@@ -434,6 +436,14 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs
 				// need to be linked with external library
 				// format: ';' separated alternative link methods. e.g.
 				//   link: $LLGO_LIB_PYTHON; $(pkg-config --libs python3-embed); -lpython3
+				if llpkg.IsGithubHosted(aPkg.PkgPath) {
+					moduleDir, err := llpkg.LLGoModuleDirOf(aPkg.Module.Path, aPkg.Module.Version)
+					if err == nil {
+						pkgPCPath := filepath.Join(moduleDir, "lib", "pkgconfig")
+						// TODO(MeteorsLiu): support Windows
+						os.Setenv("PKG_CONFIG_PATH", pkgPCPath+":"+os.Getenv("PKG_CONFIG_PATH"))
+					}
+				}
 				altParts := strings.Split(param, ";")
 				expdArgs := make([]string, 0, len(altParts))
 				for _, param := range altParts {
@@ -659,6 +669,11 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose
 	// Add common linker arguments based on target OS and architecture
 	if IsDbgSymsEnabled() {
 		buildArgs = append(buildArgs, "-gdwarf-4")
+		if runtime.GOOS == "darwin" {
+			buildArgs = append(buildArgs, "-Wl,-t", "-Wl,-map,symbol.map")
+		} else {
+			buildArgs = append(buildArgs, "-Wl,--Map=symbol.map")
+		}
 	}
 
 	buildArgs = append(buildArgs, ctx.crossCompile.LDFLAGS...)
@@ -926,6 +941,10 @@ type Package = *aPackage
 func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aPackage, errs []*packages.Package) {
 	prog := ctx.progSSA
 	built := ctx.built
+
+	taskQueue := taskqueue.NewTaskQueue(runtime.GOMAXPROCS(0))
+	defer taskQueue.Close()
+
 	packages.Visit(initial, nil, func(p *packages.Package) {
 		if p.Types != nil && !p.IllTyped {
 			pkgPath := p.PkgPath
@@ -940,10 +959,49 @@ func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aP
 				}
 			}
 			all = append(all, &aPackage{p, ssaPkg, altPkg, nil, nil, nil})
+
+			if llpkg.IsGithubHosted(pkgPath) {
+				moduleDir, err := llpkg.LLGoModuleDirOf(p.Module.Path, p.Module.Version)
+				if err != nil {
+					return
+				}
+				// if this llpkg is Github hosted, check if we need to fetch it
+				if llpkg.IsInstalled(moduleDir) {
+					if verbose {
+						fmt.Printf("skip installing llpkg binary to %s\n", moduleDir)
+					}
+					return
+				}
+
+				cfg, err := llpkg.ParseConfigFile(filepath.Join(p.Dir, "llpkg.cfg"))
+				if err != nil {
+					return
+				}
+				cfg.Upstream.Package.SetModuleVersion(p.Module.Version)
+
+				taskQueue.Push(func() {
+					// check again
+					if !llpkg.IsInstalled(moduleDir) {
+						if verbose {
+							fmt.Printf("Installing llpkg binary %s to %s\n", cfg.Upstream.Package.Name, moduleDir)
+						}
+						err := llpkg.InstallBinary(cfg, moduleDir)
+						if err != nil {
+							fmt.Fprintf(os.Stderr, "failed to installing llpkg binary: %v\n", err)
+						}
+						return
+					}
+					if verbose {
+						fmt.Printf("skip installing llpkg binary to %s\n", moduleDir)
+					}
+				})
+			}
 		} else {
 			errs = append(errs, p)
 		}
 	})
+
+	taskQueue.Wait()
 	return
 }
 

internal/llpkg/config.go

@@ -0,0 +1,22 @@
+package llpkg
+
+import "github.com/goplus/llgo/internal/llpkg/installer"
+
+// LLPkgConfig represents the configuration structure parsed from llpkg.cfg files.
+type LLPkgConfig struct {
+	Upstream UpstreamConfig `json:"upstream"`
+}
+
+// UpstreamConfig defines the upstream configuration containing installer settings and package metadata.
+type UpstreamConfig struct {
+	Installer InstallerConfig   `json:"installer"`
+	Package   installer.Package `json:"package"`
+}
+
+// InstallerConfig specifies the installer type and its configuration options.
+// "name" field must match supported installers (e.g., "conan").
+// "config" holds installer-specific parameters (optional).
+type InstallerConfig struct {
+	Name   string            `json:"name"`
+	Config map[string]string `json:"config,omitempty"`
+}

internal/llpkg/config_test.go

@@ -0,0 +1,116 @@
+package llpkg
+
+import (
+	"testing"
+
+	"github.com/goplus/llgo/internal/llpkg/installer"
+)
+
+func TestLLPkgConfig(t *testing.T) {
+	config := LLPkgConfig{
+		Upstream: UpstreamConfig{
+			Installer: InstallerConfig{
+				Name: "conan",
+				Config: map[string]string{
+					"profile": "default",
+				},
+			},
+			Package: installer.Package{
+				Name:    "libxslt",
+				Version: "v1.0.3",
+			},
+		},
+	}
+
+	if config.Upstream.Installer.Name != "conan" {
+		t.Errorf("expected installer name 'conan', got %s", config.Upstream.Installer.Name)
+	}
+
+	if config.Upstream.Package.Name != "libxslt" {
+		t.Errorf("expected package name 'libxslt', got %s", config.Upstream.Package.Name)
+	}
+
+	if config.Upstream.Package.Version != "v1.0.3" {
+		t.Errorf("expected package version 'v1.0.3', got %s", config.Upstream.Package.Version)
+	}
+
+	if config.Upstream.Installer.Config["profile"] != "default" {
+		t.Errorf("expected config profile 'default', got %s", config.Upstream.Installer.Config["profile"])
+	}
+}
+
+func TestUpstreamConfig(t *testing.T) {
+	upstream := UpstreamConfig{
+		Installer: InstallerConfig{
+			Name:   "ghrelease",
+			Config: nil,
+		},
+		Package: installer.Package{
+			Name:    "testpkg",
+			Version: "v2.0.0",
+		},
+	}
+
+	if upstream.Installer.Name != "ghrelease" {
+		t.Errorf("expected installer name 'ghrelease', got %s", upstream.Installer.Name)
+	}
+
+	if upstream.Installer.Config != nil {
+		t.Errorf("expected nil config, got %v", upstream.Installer.Config)
+	}
+
+	if upstream.Package.Name != "testpkg" {
+		t.Errorf("expected package name 'testpkg', got %s", upstream.Package.Name)
+	}
+
+	if upstream.Package.Version != "v2.0.0" {
+		t.Errorf("expected package version 'v2.0.0', got %s", upstream.Package.Version)
+	}
+
+	upstream.Package.SetModuleVersion("v1.0.0")
+
+	if upstream.Package.ModuleVersion() != "v1.0.0" {
+		t.Errorf("expected package version 'v1.0.0', got %s", upstream.Package.ModuleVersion())
+	}
+}
+
+func TestInstallerConfig(t *testing.T) {
+	config := InstallerConfig{
+		Name: "conan",
+		Config: map[string]string{
+			"profile":  "default",
+			"settings": "os=Linux",
+		},
+	}
+
+	if config.Name != "conan" {
+		t.Errorf("expected name 'conan', got %s", config.Name)
+	}
+
+	if len(config.Config) != 2 {
+		t.Errorf("expected 2 config items, got %d", len(config.Config))
+	}
+
+	if config.Config["profile"] != "default" {
+		t.Errorf("expected profile 'default', got %s", config.Config["profile"])
+	}
+
+	if config.Config["settings"] != "os=Linux" {
+		t.Errorf("expected settings 'os=Linux', got %s", config.Config["settings"])
+	}
+}
+
+func TestInstallerConfig_EmptyConfig(t *testing.T) {
+	config := InstallerConfig{
+		Name:   "simple",
+		Config: map[string]string{},
+	}
+
+	if config.Name != "simple" {
+		t.Errorf("expected name 'simple', got %s", config.Name)
+	}
+
+	if len(config.Config) != 0 {
+		t.Errorf("expected empty config, got %v", config.Config)
+	}
+}

internal/llpkg/installer/fetch.go

@@ -0,0 +1,100 @@
+package installer
+
+import (
+	"archive/zip"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// wrapDownloadError wraps an error with a descriptive message for download failures.
+// It returns nil if the input error is nil, otherwise returns a wrapped error with context.
+func wrapDownloadError(err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("failed to download llpkg binary files: %w", err)
+}
+
+// Unzip extracts a zip file to the specified output directory.
+// It creates the output directory if it doesn't exist, handles nested directories,
+// and recursively unzips any nested zip files found within the archive.
+func Unzip(zipFilePath, outputDir string) error {
+	// make sure path exists
+	if err := os.MkdirAll(outputDir, 0700); err != nil {
+		return err
+	}
+	r, err := zip.OpenReader(zipFilePath)
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+	decompress := func(file *zip.File) error {
+		path := filepath.Join(outputDir, file.Name)
+
+		if file.FileInfo().IsDir() {
+			return os.MkdirAll(path, 0700)
+		}
+
+		fs, err := file.Open()
+		if err != nil {
+			return err
+		}
+		defer fs.Close()
+
+		w, err := os.Create(path)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(w, fs); err != nil {
+			w.Close()
+			return err
+		}
+		if err := w.Close(); err != nil {
+			return err
+		}
+		// if this is a nested zip, unzip it.
+		if strings.HasSuffix(path, ".zip") {
+			if err := Unzip(path, outputDir); err != nil {
+				return err
+			}
+			os.Remove(path)
+		}
+		return nil
+	}
+
+	for _, file := range r.File {
+		if err = decompress(file); err != nil {
+			break
+		}
+	}
+	return err
+}
+
+// DownloadFile downloads a file from the given URL and saves it to a temporary file.
+// It returns the path to the downloaded temporary file.
+// The caller is responsible for cleaning up the temporary file when no longer needed.
+func DownloadFile(url string) (fileName string, err error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", wrapDownloadError(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("failed to download llpkg binary files: status %s", resp.Status)
+	}
+
+	tempFile, err := os.CreateTemp("", "download*.temp")
+	if err != nil {
+		return "", wrapDownloadError(err)
+	}
+	defer tempFile.Close()
+
+	_, err = io.Copy(tempFile, resp.Body)
+
+	return tempFile.Name(), wrapDownloadError(err)
+}