diff --git a/internal/build/build.go b/internal/build/build.go index 88a994004..356e2f356 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -40,8 +40,10 @@ import ( "github.com/goplus/llgo/cl" "github.com/goplus/llgo/internal/crosscompile" "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/llpkg" "github.com/goplus/llgo/internal/mockable" "github.com/goplus/llgo/internal/packages" + "github.com/goplus/llgo/internal/taskqueue" "github.com/goplus/llgo/internal/typepatch" "github.com/goplus/llgo/ssa/abi" "github.com/goplus/llgo/xtool/clang" @@ -434,6 +436,14 @@ func buildAllPkgs(ctx *context, initial []*packages.Package, verbose bool) (pkgs // need to be linked with external library // format: ';' separated alternative link methods. e.g. // link: $LLGO_LIB_PYTHON; $(pkg-config --libs python3-embed); -lpython3 + if llpkg.IsGithubHosted(aPkg.PkgPath) { + moduleDir, err := llpkg.LLGoModuleDirOf(aPkg.Module.Path, aPkg.Module.Version) + if err == nil { + pkgPCPath := filepath.Join(moduleDir, "lib", "pkgconfig") + // TODO(MeteorsLiu): support Windows + os.Setenv("PKG_CONFIG_PATH", pkgPCPath+":"+os.Getenv("PKG_CONFIG_PATH")) + } + } altParts := strings.Split(param, ";") expdArgs := make([]string, 0, len(altParts)) for _, param := range altParts { @@ -659,6 +669,11 @@ func linkObjFiles(ctx *context, app string, objFiles, linkArgs []string, verbose // Add common linker arguments based on target OS and architecture if IsDbgSymsEnabled() { buildArgs = append(buildArgs, "-gdwarf-4") + if runtime.GOOS == "darwin" { + buildArgs = append(buildArgs, "-Wl,-t", "-Wl,-map,symbol.map") + } else { + buildArgs = append(buildArgs, "-Wl,--Map=symbol.map") + } } buildArgs = append(buildArgs, ctx.crossCompile.LDFLAGS...) @@ -926,6 +941,10 @@ type Package = *aPackage func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aPackage, errs []*packages.Package) { prog := ctx.progSSA built := ctx.built + + taskQueue := taskqueue.NewTaskQueue(runtime.GOMAXPROCS(0)) + defer taskQueue.Close() + packages.Visit(initial, nil, func(p *packages.Package) { if p.Types != nil && !p.IllTyped { pkgPath := p.PkgPath @@ -940,10 +959,49 @@ func allPkgs(ctx *context, initial []*packages.Package, verbose bool) (all []*aP } } all = append(all, &aPackage{p, ssaPkg, altPkg, nil, nil, nil}) + + if llpkg.IsGithubHosted(pkgPath) { + moduleDir, err := llpkg.LLGoModuleDirOf(p.Module.Path, p.Module.Version) + if err != nil { + return + } + // if this llpkg is Github hosted, check if we need to fetch it + if llpkg.IsInstalled(moduleDir) { + if verbose { + fmt.Printf("skip installing llpkg binary to %s\n", moduleDir) + } + return + } + + cfg, err := llpkg.ParseConfigFile(filepath.Join(p.Dir, "llpkg.cfg")) + if err != nil { + return + } + cfg.Upstream.Package.SetModuleVersion(p.Module.Version) + + taskQueue.Push(func() { + // check again + if !llpkg.IsInstalled(moduleDir) { + if verbose { + fmt.Printf("Installing llpkg binary %s to %s\n", cfg.Upstream.Package.Name, moduleDir) + } + err := llpkg.InstallBinary(cfg, moduleDir) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to installing llpkg binary: %v\n", err) + } + return + } + if verbose { + fmt.Printf("skip installing llpkg binary to %s\n", moduleDir) + } + }) + } } else { errs = append(errs, p) } }) + + taskQueue.Wait() return } diff --git a/internal/llpkg/config.go b/internal/llpkg/config.go new file mode 100644 index 000000000..509b0e1c1 --- /dev/null +++ b/internal/llpkg/config.go @@ -0,0 +1,22 @@ +package llpkg + +import "github.com/goplus/llgo/internal/llpkg/installer" + +// LLPkgConfig represents the configuration structure parsed from llpkg.cfg files. +type LLPkgConfig struct { + Upstream UpstreamConfig `json:"upstream"` +} + +// UpstreamConfig defines the upstream configuration containing installer settings and package metadata. +type UpstreamConfig struct { + Installer InstallerConfig `json:"installer"` + Package installer.Package `json:"package"` +} + +// InstallerConfig specifies the installer type and its configuration options. +// "name" field must match supported installers (e.g., "conan"). +// "config" holds installer-specific parameters (optional). +type InstallerConfig struct { + Name string `json:"name"` + Config map[string]string `json:"config,omitempty"` +} diff --git a/internal/llpkg/config_test.go b/internal/llpkg/config_test.go new file mode 100644 index 000000000..70fb0eba1 --- /dev/null +++ b/internal/llpkg/config_test.go @@ -0,0 +1,116 @@ +package llpkg + +import ( + "testing" + + "github.com/goplus/llgo/internal/llpkg/installer" +) + +func TestLLPkgConfig(t *testing.T) { + config := LLPkgConfig{ + Upstream: UpstreamConfig{ + Installer: InstallerConfig{ + Name: "conan", + Config: map[string]string{ + "profile": "default", + }, + }, + Package: installer.Package{ + Name: "libxslt", + Version: "v1.0.3", + }, + }, + } + + if config.Upstream.Installer.Name != "conan" { + t.Errorf("expected installer name 'conan', got %s", config.Upstream.Installer.Name) + } + + if config.Upstream.Package.Name != "libxslt" { + t.Errorf("expected package name 'libxslt', got %s", config.Upstream.Package.Name) + } + + if config.Upstream.Package.Version != "v1.0.3" { + t.Errorf("expected package version 'v1.0.3', got %s", config.Upstream.Package.Version) + } + + if config.Upstream.Installer.Config["profile"] != "default" { + t.Errorf("expected config profile 'default', got %s", config.Upstream.Installer.Config["profile"]) + } +} + +func TestUpstreamConfig(t *testing.T) { + upstream := UpstreamConfig{ + Installer: InstallerConfig{ + Name: "ghrelease", + Config: nil, + }, + Package: installer.Package{ + Name: "testpkg", + Version: "v2.0.0", + }, + } + + if upstream.Installer.Name != "ghrelease" { + t.Errorf("expected installer name 'ghrelease', got %s", upstream.Installer.Name) + } + + if upstream.Installer.Config != nil { + t.Errorf("expected nil config, got %v", upstream.Installer.Config) + } + + if upstream.Package.Name != "testpkg" { + t.Errorf("expected package name 'testpkg', got %s", upstream.Package.Name) + } + + if upstream.Package.Version != "v2.0.0" { + t.Errorf("expected package version 'v2.0.0', got %s", upstream.Package.Version) + } + + upstream.Package.SetModuleVersion("v1.0.0") + + if upstream.Package.ModuleVersion() != "v1.0.0" { + t.Errorf("expected package version 'v1.0.0', got %s", upstream.Package.ModuleVersion()) + } +} + +func TestInstallerConfig(t *testing.T) { + config := InstallerConfig{ + Name: "conan", + Config: map[string]string{ + "profile": "default", + "settings": "os=Linux", + }, + } + + if config.Name != "conan" { + t.Errorf("expected name 'conan', got %s", config.Name) + } + + if len(config.Config) != 2 { + t.Errorf("expected 2 config items, got %d", len(config.Config)) + } + + if config.Config["profile"] != "default" { + t.Errorf("expected profile 'default', got %s", config.Config["profile"]) + } + + if config.Config["settings"] != "os=Linux" { + t.Errorf("expected settings 'os=Linux', got %s", config.Config["settings"]) + } +} + +func TestInstallerConfig_EmptyConfig(t *testing.T) { + config := InstallerConfig{ + Name: "simple", + Config: map[string]string{}, + } + + if config.Name != "simple" { + t.Errorf("expected name 'simple', got %s", config.Name) + } + + if len(config.Config) != 0 { + t.Errorf("expected empty config, got %v", config.Config) + } +} diff --git a/internal/llpkg/installer/fetch.go b/internal/llpkg/installer/fetch.go new file mode 100644 index 000000000..dad2dff9b --- /dev/null +++ b/internal/llpkg/installer/fetch.go @@ -0,0 +1,100 @@ +package installer + +import ( + "archive/zip" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" +) + +// wrapDownloadError wraps an error with a descriptive message for download failures. +// It returns nil if the input error is nil, otherwise returns a wrapped error with context. +func wrapDownloadError(err error) error { + if err == nil { + return nil + } + return fmt.Errorf("failed to download llpkg binary files: %w", err) +} + +// Unzip extracts a zip file to the specified output directory. +// It creates the output directory if it doesn't exist, handles nested directories, +// and recursively unzips any nested zip files found within the archive. +func Unzip(zipFilePath, outputDir string) error { + // make sure path exists + if err := os.MkdirAll(outputDir, 0700); err != nil { + return err + } + r, err := zip.OpenReader(zipFilePath) + if err != nil { + return err + } + defer r.Close() + decompress := func(file *zip.File) error { + path := filepath.Join(outputDir, file.Name) + + if file.FileInfo().IsDir() { + return os.MkdirAll(path, 0700) + } + + fs, err := file.Open() + if err != nil { + return err + } + defer fs.Close() + + w, err := os.Create(path) + if err != nil { + return err + } + if _, err := io.Copy(w, fs); err != nil { + w.Close() + return err + } + if err := w.Close(); err != nil { + return err + } + // if this is a nested zip, unzip it. + if strings.HasSuffix(path, ".zip") { + if err := Unzip(path, outputDir); err != nil { + return err + } + os.Remove(path) + } + return nil + } + + for _, file := range r.File { + if err = decompress(file); err != nil { + break + } + } + return err +} + +// DownloadFile downloads a file from the given URL and saves it to a temporary file. +// It returns the path to the downloaded temporary file. +// The caller is responsible for cleaning up the temporary file when no longer needed. +func DownloadFile(url string) (fileName string, err error) { + resp, err := http.Get(url) + if err != nil { + return "", wrapDownloadError(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to download llpkg binary files: status %s", resp.Status) + } + + tempFile, err := os.CreateTemp("", "download*.temp") + if err != nil { + return "", wrapDownloadError(err) + } + defer tempFile.Close() + + _, err = io.Copy(tempFile, resp.Body) + + return tempFile.Name(), wrapDownloadError(err) +} diff --git a/internal/llpkg/installer/fetch_test.go b/internal/llpkg/installer/fetch_test.go new file mode 100644 index 000000000..af85b3de6 --- /dev/null +++ b/internal/llpkg/installer/fetch_test.go @@ -0,0 +1,505 @@ +package installer + +import ( + "archive/zip" + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestUnzip(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + zipPath := filepath.Join(tempDir, "test.zip") + outputDir := filepath.Join(tempDir, "output") + + // Create test zip file with helper function + if err := createTestZip(zipPath, map[string]string{ + "test.txt": "test content", + "testdir/": "", // directory entry + "testdir/subfile.txt": "sub content", + }); err != nil { + t.Fatalf("failed to create test zip: %v", err) + } + + // Test unzip operation + err = Unzip(zipPath, outputDir) + if err != nil { + t.Fatalf("Unzip failed: %v", err) + } + + // Validate extracted files + validateFile(t, filepath.Join(outputDir, "test.txt"), "test content") + validateFile(t, filepath.Join(outputDir, "testdir", "subfile.txt"), "sub content") + validateDir(t, filepath.Join(outputDir, "testdir")) +} + +func TestUnzip_NestedZip(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_nested_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // First create a nested zip file + nestedZipPath := filepath.Join(tempDir, "nested.zip") + if err := createTestZip(nestedZipPath, map[string]string{ + "nested_file.txt": "nested content", + }); err != nil { + t.Fatalf("failed to create nested zip: %v", err) + } + + // Read the nested zip content + nestedContent, err := os.ReadFile(nestedZipPath) + if err != nil { + t.Fatalf("failed to read nested zip: %v", err) + } + + // Create main zip with nested zip + mainZipPath := filepath.Join(tempDir, "main.zip") + if err := createTestZipWithBinary(mainZipPath, map[string]interface{}{ + "main.txt": "main content", + "nested.zip": nestedContent, + }); err != nil { + t.Fatalf("failed to create main zip: %v", err) + } + + outputDir := filepath.Join(tempDir, "output") + + // Test unzip operation + err = Unzip(mainZipPath, outputDir) + if err != nil { + t.Fatalf("Unzip failed: %v", err) + } + + // Validate main file + validateFile(t, filepath.Join(outputDir, "main.txt"), "main content") + + // Validate nested files were extracted + validateFile(t, filepath.Join(outputDir, "nested_file.txt"), "nested content") + + // The nested.zip should be removed after extraction + if _, err := os.Stat(filepath.Join(outputDir, "nested.zip")); !os.IsNotExist(err) { + t.Error("nested.zip should be removed after extraction") + } +} + +func TestUnzip_InvalidZip(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_invalid_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + invalidZipPath := filepath.Join(tempDir, "invalid.zip") + err = os.WriteFile(invalidZipPath, []byte("not a zip file"), 0644) + if err != nil { + t.Fatal(err) + } + + outputDir := filepath.Join(tempDir, "output") + err = Unzip(invalidZipPath, outputDir) + if err == nil { + t.Error("expected error for invalid zip file") + } +} + +func TestUnzip_OutputDirCreationFailure(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_mkdir_fail_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + zipPath := filepath.Join(tempDir, "test.zip") + if err := createTestZip(zipPath, map[string]string{ + "test.txt": "test content", + }); err != nil { + t.Fatalf("failed to create test zip: %v", err) + } + + // Create a file with the same name as the output directory to cause mkdir failure + outputDir := filepath.Join(tempDir, "output") + if err := os.WriteFile(outputDir, []byte("blocking file"), 0644); err != nil { + t.Fatal(err) + } + + err = Unzip(zipPath, outputDir) + if err == nil { + t.Error("expected error when output directory creation fails") + } +} + +func TestUnzip_FileCreationFailure(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_file_create_fail_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + zipPath := filepath.Join(tempDir, "test.zip") + if err := createTestZip(zipPath, map[string]string{ + "test.txt": "test content", + }); err != nil { + t.Fatalf("failed to create test zip: %v", err) + } + + outputDir := filepath.Join(tempDir, "output") + + // Create the output directory first + if err := os.MkdirAll(outputDir, 0700); err != nil { + t.Fatal(err) + } + + // Create a directory with the same name as the file to be extracted + conflictPath := filepath.Join(outputDir, "test.txt") + if err := os.MkdirAll(conflictPath, 0700); err != nil { + t.Fatal(err) + } + + err = Unzip(zipPath, outputDir) + if err == nil { + t.Error("expected error when file creation fails due to directory conflict") + } +} + +func TestUnzip_DirectoryMkdirError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_dir_mkdir_error_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + zipPath := filepath.Join(tempDir, "test.zip") + if err := createTestZip(zipPath, map[string]string{ + "testdir/": "", + "testdir/file.txt": "content", + }); err != nil { + t.Fatalf("failed to create test zip: %v", err) + } + + outputDir := filepath.Join(tempDir, "output") + + // Create the output directory first + if err := os.MkdirAll(outputDir, 0700); err != nil { + t.Fatal(err) + } + + // Create a file with the same name as the directory to be created + conflictPath := filepath.Join(outputDir, "testdir") + if err := os.WriteFile(conflictPath, []byte("blocking file"), 0644); err != nil { + t.Fatal(err) + } + + err = Unzip(zipPath, outputDir) + if err == nil { + t.Error("expected error when directory creation fails due to file conflict") + } +} + +func TestUnzip_NestedZipError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_nested_error_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Create a corrupted nested zip + corruptedZipContent := []byte("corrupted zip content") + + // Create main zip with corrupted nested zip + mainZipPath := filepath.Join(tempDir, "main.zip") + if err := createTestZipWithBinary(mainZipPath, map[string]interface{}{ + "main.txt": "main content", + "nested.zip": corruptedZipContent, + }); err != nil { + t.Fatalf("failed to create main zip: %v", err) + } + + outputDir := filepath.Join(tempDir, "output") + + // Test unzip operation - should fail when trying to unzip the corrupted nested zip + err = Unzip(mainZipPath, outputDir) + if err == nil { + t.Error("expected error when nested zip is corrupted") + } +} + +func TestUnzip_FileOpenError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "unzip_file_open_error_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Create a zip file that will cause file.Open() to fail + zipPath := filepath.Join(tempDir, "corrupted_entry.zip") + + // Create a zip file manually with a corrupted entry + zipFile, err := os.Create(zipPath) + if err != nil { + t.Fatal(err) + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + + // Create a normal file first + normalWriter, err := zipWriter.Create("normal.txt") + if err != nil { + t.Fatal(err) + } + normalWriter.Write([]byte("normal content")) + + // Create a file entry that will be corrupted + corruptedWriter, err := zipWriter.Create("corrupted.txt") + if err != nil { + t.Fatal(err) + } + corruptedWriter.Write([]byte("this will be corrupted")) + + zipWriter.Close() + + // Now corrupt the zip file by truncating it in the middle + // This should cause file.Open() to fail for some entries + file, err := os.OpenFile(zipPath, os.O_WRONLY, 0644) + if err != nil { + t.Fatal(err) + } + stat, err := file.Stat() + if err != nil { + t.Fatal(err) + } + // Truncate to about 80% of original size to corrupt some entries + file.Truncate(stat.Size() * 8 / 10) + file.Close() + + outputDir := filepath.Join(tempDir, "output") + err = Unzip(zipPath, outputDir) + if err == nil { + t.Error("expected error when file.Open() fails due to corruption") + } +} + +func TestDownloadFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "download_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + t.Run("download real file", func(t *testing.T) { + url := "https://github.com/goplus/llpkg/releases/download/libxslt%2Fv1.0.3/libxslt_darwin_amd64.zip" + + absFilename, err := DownloadFile(url) + if err != nil { + t.Skipf("skipping download test due to network error: %v", err) + } + + // DownloadFile returns a temporary file name, not the final filename + if absFilename == "" { + t.Error("expected non-empty filename") + } + + if _, err := os.Stat(absFilename); os.IsNotExist(err) { + t.Error("downloaded file does not exist") + } + + fileInfo, err := os.Stat(absFilename) + if err != nil { + t.Fatal(err) + } + if fileInfo.Size() == 0 { + t.Error("downloaded file is empty") + } + + // Verify it's a temporary file + if !strings.HasSuffix(absFilename, ".temp") { + t.Errorf("expected temporary file name, got %s", absFilename) + } + }) + + t.Run("invalid url", func(t *testing.T) { + invalidURL := "invalid-url" + + _, err := DownloadFile(invalidURL) + if err == nil { + t.Error("expected error for invalid URL") + } + if !strings.Contains(err.Error(), "failed to download llpkg binary files") { + t.Errorf("expected wrapped error message, got %s", err.Error()) + } + }) + + t.Run("http status not ok", func(t *testing.T) { + // Use a URL that returns 404 + notFoundURL := "https://httpbin.org/status/404" + + _, err := DownloadFile(notFoundURL) + if err == nil { + t.Error("expected error for 404 status") + } + if !strings.Contains(err.Error(), "failed to download llpkg binary files: status") { + t.Errorf("expected status error message, got %s", err.Error()) + } + }) +} + +func TestDownloadFile_CreateTempFailure(t *testing.T) { + tempDir, err := os.MkdirTemp("", "download_createtemp_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Set TMPDIR to a non-existent directory to cause CreateTemp to fail + originalTmpDir := os.Getenv("TMPDIR") + defer os.Setenv("TMPDIR", originalTmpDir) + + nonExistentTmpDir := filepath.Join(tempDir, "nonexistent") + os.Setenv("TMPDIR", nonExistentTmpDir) + + url := "https://httpbin.org/get" + + _, err = DownloadFile(url) + if err == nil { + t.Error("expected error when CreateTemp fails") + } + if !strings.Contains(err.Error(), "failed to download llpkg binary files") { + t.Errorf("expected wrapped error message, got %s", err.Error()) + } +} + +func TestWrapDownloadError(t *testing.T) { + t.Run("nil error", func(t *testing.T) { + result := wrapDownloadError(nil) + if result != nil { + t.Errorf("expected nil, got %v", result) + } + }) + + t.Run("non-nil error", func(t *testing.T) { + originalErr := fmt.Errorf("original error") + result := wrapDownloadError(originalErr) + if result == nil { + t.Error("expected non-nil error") + } + if !strings.Contains(result.Error(), "failed to download llpkg binary files") { + t.Errorf("expected wrapped error message, got %s", result.Error()) + } + if !strings.Contains(result.Error(), "original error") { + t.Errorf("expected original error in wrapped message, got %s", result.Error()) + } + }) +} + +// Helper functions + +func createTestZip(zipPath string, files map[string]string) error { + zipFile, err := os.Create(zipPath) + if err != nil { + return err + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + // Sort entries to ensure directories come before files + // This prevents the issue where files are created before their parent directories + var entries []string + for name := range files { + entries = append(entries, name) + } + + // Sort with custom logic: directories (ending with /) first, then files + // Within each category, sort alphabetically + for i := 0; i < len(entries); i++ { + for j := i + 1; j < len(entries); j++ { + // If entries[i] is a file and entries[j] is a directory, swap + if !strings.HasSuffix(entries[i], "/") && strings.HasSuffix(entries[j], "/") { + entries[i], entries[j] = entries[j], entries[i] + } else if strings.HasSuffix(entries[i], "/") == strings.HasSuffix(entries[j], "/") { + // Both are files or both are directories, sort alphabetically + if entries[i] > entries[j] { + entries[i], entries[j] = entries[j], entries[i] + } + } + } + } + + for _, name := range entries { + content := files[name] + fileWriter, err := zipWriter.Create(name) + if err != nil { + return err + } + if content == "" { + continue + } + _, err = fileWriter.Write([]byte(content)) + if err != nil { + return err + } + } + + return nil +} + +func createTestZipWithBinary(zipPath string, files map[string]interface{}) error { + zipFile, err := os.Create(zipPath) + if err != nil { + return err + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + for name, content := range files { + fileWriter, err := zipWriter.Create(name) + if err != nil { + return err + } + + switch v := content.(type) { + case string: + _, err = fileWriter.Write([]byte(v)) + case []byte: + _, err = fileWriter.Write(v) + default: + return err + } + + if err != nil { + return err + } + } + + return nil +} + +func validateFile(t *testing.T, filePath, expectedContent string) { + t.Helper() + content, err := os.ReadFile(filePath) + if err != nil { + t.Errorf("failed to read file %s: %v", filePath, err) + return + } + if string(content) != expectedContent { + t.Errorf("file %s: expected '%s', got '%s'", filePath, expectedContent, string(content)) + } +} + +func validateDir(t *testing.T, dirPath string) { + t.Helper() + if _, err := os.Stat(dirPath); os.IsNotExist(err) { + t.Errorf("directory %s was not created", dirPath) + } +} diff --git a/internal/llpkg/installer/ghrelease/ghrelease.go b/internal/llpkg/installer/ghrelease/ghrelease.go new file mode 100644 index 000000000..8de24e295 --- /dev/null +++ b/internal/llpkg/installer/ghrelease/ghrelease.go @@ -0,0 +1,52 @@ +package ghrelease + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/goplus/llgo/internal/llpkg/installer" + "github.com/goplus/llgo/internal/llpkg/installer/pcgen" +) + +type ghReleasesInstaller struct { + owner, repo string +} + +// New creates a new GitHub releases installer for the specified owner and repository. +// It returns an installer that can download and install packages from GitHub releases. +func New(owner, repo string) installer.Installer { + return &ghReleasesInstaller{owner: owner, repo: repo} +} + +// assertUrl returns the URL for the specified package. +// The URL is constructed based on the package name, version, and the installer configuration. +func (c *ghReleasesInstaller) assertUrl(pkg installer.Package) string { + // NOTE(MeteorsLiu): release binary url requires mapped version, aka module version for llpkg here. + releaseName := fmt.Sprintf("%s/%s", pkg.Name, pkg.ModuleVersion()) + fileName := fmt.Sprintf("%s_%s.zip", pkg.Name, runtime.GOOS+"_"+runtime.GOARCH) + return fmt.Sprintf("https://github.com/%s/%s/releases/download/%s/%s", c.owner, c.repo, releaseName, fileName) +} + +// Install downloads and installs a package from GitHub releases to the specified output directory. +// It downloads the package zip file, extracts it, and generates pkg-config files from templates. +// The installation process includes cleanup of temporary files and proper error handling. +func (c *ghReleasesInstaller) Install(pkg installer.Package, outputDir string) error { + absOutputDir, err := filepath.Abs(outputDir) + if err != nil { + return err + } + zipFilePath, err := installer.DownloadFile(c.assertUrl(pkg)) + if err != nil { + return err + } + defer os.Remove(zipFilePath) + + err = installer.Unzip(zipFilePath, absOutputDir) + if err != nil { + return fmt.Errorf("failed to unzip llpkg: %w", err) + } + // generate actual pc files from pc.tmpl files + return pcgen.GeneratePC(filepath.Join(outputDir, "lib", "pkgconfig"), absOutputDir) +} diff --git a/internal/llpkg/installer/ghrelease/ghrelease_test.go b/internal/llpkg/installer/ghrelease/ghrelease_test.go new file mode 100644 index 000000000..25f1fbdf5 --- /dev/null +++ b/internal/llpkg/installer/ghrelease/ghrelease_test.go @@ -0,0 +1,122 @@ +package ghrelease + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/goplus/llgo/internal/llpkg/installer" +) + +func TestNew(t *testing.T) { + owner := "goplus" + repo := "llpkg" + + inst := New(owner, repo) + + if inst == nil { + t.Fatal("New() returned nil") + } + + ghInst, ok := inst.(*ghReleasesInstaller) + if !ok { + t.Fatal("New() did not return *ghReleasesInstaller") + } + + if ghInst.owner != owner { + t.Errorf("expected owner %s, got %s", owner, ghInst.owner) + } + + if ghInst.repo != repo { + t.Errorf("expected repo %s, got %s", repo, ghInst.repo) + } +} + +func TestAssertUrl(t *testing.T) { + inst := &ghReleasesInstaller{ + owner: "goplus", + repo: "llpkg", + } + + pkg := installer.Package{ + Name: "libxslt", + Version: "1.7.18", + } + pkg.SetModuleVersion("v1.0.3") + + result := inst.assertUrl(pkg) + expected := "https://github.com/goplus/llpkg/releases/download/libxslt/v1.0.3/libxslt_" + runtime.GOOS + "_" + runtime.GOARCH + ".zip" + + if result != expected { + t.Errorf("expected %s, got %s", expected, result) + } +} + +func TestInstall(t *testing.T) { + // Create temporary directory for test + tempDir, err := os.MkdirTemp("", "ghrelease_install_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Create installer with mock data + inst := &ghReleasesInstaller{ + owner: "goplus", + repo: "llpkg", + } + + // Create package with mock data: libxslt/v1.0.3 + pkg := installer.Package{ + Name: "libxslt", + Version: "1.7.18", // This is the actual version + } + pkg.SetModuleVersion("v1.0.3") // This is the module version used in URL + + outputDir := filepath.Join(tempDir, "output") + + // Test the Install method + // Note: This test will attempt to download from the actual GitHub release + // If the network is unavailable or the release doesn't exist, the test will be skipped + err = inst.Install(pkg, outputDir) + if err != nil { + // Skip test if it's a network-related error + if strings.Contains(err.Error(), "failed to download") || + strings.Contains(err.Error(), "no such host") || + strings.Contains(err.Error(), "connection") { + t.Skipf("skipping install test due to network error: %v", err) + } + t.Fatalf("Install failed: %v", err) + } + + // Verify that the output directory was created + if _, err := os.Stat(outputDir); os.IsNotExist(err) { + t.Error("output directory was not created") + } + + // Verify that lib/pkgconfig directory exists (this is where PC files should be) + pkgConfigDir := filepath.Join(outputDir, "lib", "pkgconfig") + if _, err := os.Stat(pkgConfigDir); os.IsNotExist(err) { + t.Error("lib/pkgconfig directory was not created") + } + + // Check if any .pc files were generated + pcFiles, err := filepath.Glob(filepath.Join(pkgConfigDir, "*.pc")) + if err != nil { + t.Errorf("failed to check for .pc files: %v", err) + } + if len(pcFiles) == 0 { + t.Error("no .pc files were generated") + } + + // Verify that .pc.tmpl files were removed (they should be cleaned up) + tmplFiles, err := filepath.Glob(filepath.Join(pkgConfigDir, "*.pc.tmpl")) + if err != nil { + t.Errorf("failed to check for .pc.tmpl files: %v", err) + } + if len(tmplFiles) > 0 { + t.Error("template files were not cleaned up") + } +} diff --git a/internal/llpkg/installer/installer.go b/internal/llpkg/installer/installer.go new file mode 100644 index 000000000..3a2037f6e --- /dev/null +++ b/internal/llpkg/installer/installer.go @@ -0,0 +1,30 @@ +package installer + +// Installer represents a package installer that can download, install, and locate binaries from a remote repository. +// It provides methods to install packages to specific directories and search for installed package information. +type Installer interface { + Install(pkg Package, outputDir string) error +} + +// Package defines the metadata required to identify and install a software library. +// The Name and Version fields provide precise identification of the library. +type Package struct { + Name string + Version string + // mapped version, aka module version, in llpkg design, we call it mapped version + // this field isn't a part of design, aims to help installer like ghrelease get the mapped version quickly + // so we cannot export it + mappedVersion string +} + +// ModuleVersion returns the mapped version of the package. +// This is used internally to retrieve the module version that corresponds to the package version. +func (p *Package) ModuleVersion() string { + return p.mappedVersion +} + +// SetModuleVersion sets the mapped version for the package. +// This method is used internally to store the module version that maps to the package version. +func (p *Package) SetModuleVersion(ver string) { + p.mappedVersion = ver +} diff --git a/internal/llpkg/installer/installer_test.go b/internal/llpkg/installer/installer_test.go new file mode 100644 index 000000000..59d3bbb38 --- /dev/null +++ b/internal/llpkg/installer/installer_test.go @@ -0,0 +1,24 @@ +package installer + +import "testing" + +func TestUpstreamConfig(t *testing.T) { + pkg := Package{ + Name: "testpkg", + Version: "v2.0.0", + } + + if pkg.Name != "testpkg" { + t.Errorf("expected package name 'testpkg', got %s", pkg.Name) + } + + if pkg.Version != "v2.0.0" { + t.Errorf("expected package version 'v2.0.0', got %s", pkg.Version) + } + + pkg.SetModuleVersion("v1.0.0") + + if pkg.ModuleVersion() != "v1.0.0" { + t.Errorf("expected package version 'v1.0.0', got %s", pkg.ModuleVersion()) + } +} diff --git a/internal/llpkg/installer/pcgen/pcgen.go b/internal/llpkg/installer/pcgen/pcgen.go new file mode 100644 index 000000000..2ee5e6e05 --- /dev/null +++ b/internal/llpkg/installer/pcgen/pcgen.go @@ -0,0 +1,55 @@ +package pcgen + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" +) + +// GeneratePC generates pkg-config (.pc) files from template files (.pc.tmpl) in the specified directory. +// It processes all .pc.tmpl files found in pkgConfigPath, replaces template variables with actual values, +// and creates corresponding .pc files. The template files are removed after successful generation. +// The absOutputDir parameter is used as the "Prefix" value in template substitution. +func GeneratePC(pkgConfigPath, absOutputDir string) error { + pcTmpls, err := filepath.Glob(filepath.Join(pkgConfigPath, "*.pc.tmpl")) + if err != nil { + return err + } + if len(pcTmpls) == 0 { + return fmt.Errorf("failed to generate pc files for llpkg: pc files not found") + } + + for _, pcTmpl := range pcTmpls { + tmplContent, err := os.ReadFile(pcTmpl) + if err != nil { + return err + } + tmplName := filepath.Base(pcTmpl) + tmpl, err := template.New(tmplName).Parse(string(tmplContent)) + if err != nil { + return err + } + + pcFilePath := filepath.Join(pkgConfigPath, strings.TrimSuffix(tmplName, ".tmpl")) + var buf bytes.Buffer + // The Prefix field specifies the absolute path to the output directory, + // which is used to replace placeholders in the .pc template files. + if err := tmpl.Execute(&buf, map[string]any{ + "Prefix": absOutputDir, + }); err != nil { + return err + } + if err := os.WriteFile(pcFilePath, buf.Bytes(), 0644); err != nil { + return err + } + // remove .pc.tmpl file + err = os.Remove(filepath.Join(pkgConfigPath, tmplName)) + if err != nil { + return fmt.Errorf("failed to remove template file: %w", err) + } + } + return nil +} diff --git a/internal/llpkg/installer/pcgen/pcgen_test.go b/internal/llpkg/installer/pcgen/pcgen_test.go new file mode 100644 index 000000000..12d29ff9d --- /dev/null +++ b/internal/llpkg/installer/pcgen/pcgen_test.go @@ -0,0 +1,342 @@ +package pcgen + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGeneratePC(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + tmplContent := `prefix={{.Prefix}} +libdir=${prefix}/lib +includedir=${prefix}/include +bindir=${prefix}/bin + +Name: libexslt +Description: Conan component: libexslt +Version: 1.1.42 +Libs: -L"${libdir}" -lexslt +Cflags: -I"${includedir}" +Requires: libxslt` + + tmplPath := filepath.Join(pkgConfigPath, "libexslt.pc.tmpl") + err = os.WriteFile(tmplPath, []byte(tmplContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err != nil { + t.Fatal(err) + } + + pcPath := filepath.Join(pkgConfigPath, "libexslt.pc") + content, err := os.ReadFile(pcPath) + if err != nil { + t.Fatal(err) + } + + expectedContent := `prefix=/usr/local +libdir=${prefix}/lib +includedir=${prefix}/include +bindir=${prefix}/bin + +Name: libexslt +Description: Conan component: libexslt +Version: 1.1.42 +Libs: -L"${libdir}" -lexslt +Cflags: -I"${includedir}" +Requires: libxslt` + + if string(content) != expectedContent { + t.Errorf("expected:\n%s\ngot:\n%s", expectedContent, string(content)) + } + + if _, err := os.Stat(tmplPath); !os.IsNotExist(err) { + t.Error("template file should have been removed") + } +} + +func TestGeneratePC_NoTemplates(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_no_templates_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err == nil { + t.Error("expected error when no template files found") + } + if !strings.Contains(err.Error(), "pc files not found") { + t.Errorf("expected 'pc files not found' error, got %s", err.Error()) + } +} + +func TestGeneratePC_InvalidTemplate(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_invalid_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + invalidTmplContent := `prefix={{.Prefix +Name: invalid` + + tmplPath := filepath.Join(pkgConfigPath, "invalid.pc.tmpl") + err = os.WriteFile(tmplPath, []byte(invalidTmplContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err == nil { + t.Error("expected error for invalid template") + } +} + +func TestGeneratePC_ReadFileError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_readfile_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + // Create a template file with no read permissions + tmplPath := filepath.Join(pkgConfigPath, "test.pc.tmpl") + err = os.WriteFile(tmplPath, []byte("test content"), 0000) // No permissions + if err != nil { + t.Fatal(err) + } + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err == nil { + t.Error("expected error when template file cannot be read") + } +} + +func TestGeneratePC_WriteFileError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_writefile_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + // Create a valid template + tmplContent := `prefix={{.Prefix}}` + tmplPath := filepath.Join(pkgConfigPath, "test.pc.tmpl") + err = os.WriteFile(tmplPath, []byte(tmplContent), 0644) + if err != nil { + t.Fatal(err) + } + + // Create a directory with the same name as the output file to cause write error + pcPath := filepath.Join(pkgConfigPath, "test.pc") + err = os.MkdirAll(pcPath, 0755) + if err != nil { + t.Fatal(err) + } + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err == nil { + t.Error("expected error when output file cannot be written") + } +} + +func TestGeneratePC_MultipleTemplates(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_multiple_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + // Create multiple template files + templates := map[string]string{ + "lib1.pc.tmpl": `prefix={{.Prefix}} +Name: lib1 +Version: 1.0.0`, + "lib2.pc.tmpl": `prefix={{.Prefix}} +Name: lib2 +Version: 2.0.0`, + } + + for filename, content := range templates { + tmplPath := filepath.Join(pkgConfigPath, filename) + err = os.WriteFile(tmplPath, []byte(content), 0644) + if err != nil { + t.Fatal(err) + } + } + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err != nil { + t.Fatal(err) + } + + // Check that both PC files were created and template files were removed + for filename := range templates { + pcFilename := strings.TrimSuffix(filename, ".tmpl") + pcPath := filepath.Join(pkgConfigPath, pcFilename) + + // Check PC file exists + if _, err := os.Stat(pcPath); os.IsNotExist(err) { + t.Errorf("PC file %s should exist", pcFilename) + } + + // Check template file was removed + tmplPath := filepath.Join(pkgConfigPath, filename) + if _, err := os.Stat(tmplPath); !os.IsNotExist(err) { + t.Errorf("template file %s should have been removed", filename) + } + } +} + +func TestGeneratePC_RemoveTemplateError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_remove_error_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + // Create a valid template + tmplContent := `prefix={{.Prefix}}` + tmplPath := filepath.Join(pkgConfigPath, "test.pc.tmpl") + err = os.WriteFile(tmplPath, []byte(tmplContent), 0644) + if err != nil { + t.Fatal(err) + } + + // Create the PC file first, then make it read-only to prevent removal + pcPath := filepath.Join(pkgConfigPath, "test.pc") + err = os.WriteFile(pcPath, []byte("dummy"), 0644) + if err != nil { + t.Fatal(err) + } + + // Make the template file read-only to prevent removal + err = os.Chmod(tmplPath, 0444) + if err != nil { + t.Fatal(err) + } + defer os.Chmod(tmplPath, 0644) // Restore permissions for cleanup + + // Make the directory read-only to prevent file removal + err = os.Chmod(pkgConfigPath, 0555) + if err != nil { + t.Fatal(err) + } + defer os.Chmod(pkgConfigPath, 0755) // Restore permissions for cleanup + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err == nil { + t.Error("expected error when template file cannot be removed") + } + if !strings.Contains(err.Error(), "failed to remove template file") { + t.Errorf("expected 'failed to remove template file' error, got %s", err.Error()) + } +} + +func TestGeneratePC_TemplateExecuteError(t *testing.T) { + tempDir, err := os.MkdirTemp("", "pcgen_execute_error_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + pkgConfigPath := filepath.Join(tempDir, "pkgconfig") + err = os.MkdirAll(pkgConfigPath, 0755) + if err != nil { + t.Fatal(err) + } + + absOutputDir := "/usr/local" + + // Create a template that will cause execution error by calling a function that doesn't exist + tmplContent := `prefix={{.Prefix}} +{{.Prefix | nonExistentFunction}}` + tmplPath := filepath.Join(pkgConfigPath, "test.pc.tmpl") + err = os.WriteFile(tmplPath, []byte(tmplContent), 0644) + if err != nil { + t.Fatal(err) + } + + err = GeneratePC(pkgConfigPath, absOutputDir) + if err == nil { + t.Error("expected error when template execution fails") + } +} + +func TestGeneratePC_GlobError(t *testing.T) { + // Test with an invalid glob pattern that would cause filepath.Glob to return an error + // This is difficult to trigger in practice, but we can try with a malformed pattern + invalidPath := "[\x00" + absOutputDir := "/usr/local" + + err := GeneratePC(invalidPath, absOutputDir) + if err == nil { + t.Error("expected error for invalid glob pattern") + } +} diff --git a/internal/llpkg/llpkg.go b/internal/llpkg/llpkg.go new file mode 100644 index 000000000..42d46205e --- /dev/null +++ b/internal/llpkg/llpkg.go @@ -0,0 +1,67 @@ +package llpkg + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/goplus/llgo/internal/env" + "github.com/goplus/llgo/internal/llpkg/installer/ghrelease" + "golang.org/x/mod/module" +) + +var _defaultInstaller = ghrelease.New("goplus", "llpkg") + +// IsGithubHosted checks if the given module path is hosted on the goplus/llpkg GitHub repository. +// It returns true if the module path starts with "github.com/goplus/llpkg", false otherwise. +func IsGithubHosted(modulePath string) bool { + return strings.HasPrefix(modulePath, "github.com/goplus/llpkg") +} + +// IsInstalled checks if a module is already installed by verifying the existence of the "lib" directory. +// It returns true if the lib directory exists in the specified module directory, false otherwise. +func IsInstalled(moduleDir string) bool { + _, err := os.Stat(filepath.Join(moduleDir, "lib")) + return !os.IsNotExist(err) +} + +// LLGoModuleDirOf constructs and creates the cache directory path for a given module and version. +// It escapes the module path according to Go module conventions, handles special characters for unix-like systems, +// and ensures the directory exists with proper permissions (0700). +// Returns the full directory path and any error encountered during directory creation. +func LLGoModuleDirOf(modulePath, moduleVersion string) (string, error) { + escapedPath, err := module.EscapePath(modulePath) + if err != nil { + return "", err + } + // NOTE(MeteorsLiu): In unix-like system, -L cannot recognize the path with ! + escapedPath = strings.ReplaceAll(escapedPath, "!", `\!`) + + dir := filepath.Join(env.LLGoCacheDir(), escapedPath+"@"+moduleVersion) + + err = os.MkdirAll(dir, 0700) + + return dir, err +} + +// InstallBinary installs a binary package using the default GitHub release installer. +// It takes an LLPkgConfig containing package information and an output directory path. +// Returns any error encountered during the installation process. +func InstallBinary(llpkgConfig LLPkgConfig, outputDir string) error { + return _defaultInstaller.Install(llpkgConfig.Upstream.Package, outputDir) +} + +// ParseConfigFile reads and parses an llpkg configuration file from the specified file path. +// It opens the JSON file, decodes it into an LLPkgConfig struct, and handles file closure automatically. +// Returns the parsed configuration and any error encountered during file operations or JSON decoding. +func ParseConfigFile(fileName string) (configFile LLPkgConfig, err error) { + llpkgConfigFile, err := os.Open(fileName) + if err != nil { + return + } + defer llpkgConfigFile.Close() + + err = json.NewDecoder(llpkgConfigFile).Decode(&configFile) + return +} diff --git a/internal/llpkg/llpkg_test.go b/internal/llpkg/llpkg_test.go new file mode 100644 index 000000000..5839b7359 --- /dev/null +++ b/internal/llpkg/llpkg_test.go @@ -0,0 +1,221 @@ +package llpkg + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/goplus/llgo/internal/llpkg/installer" +) + +func TestIsGithubHosted(t *testing.T) { + tests := []struct { + name string + modulePath string + expected bool + }{ + { + name: "github hosted module", + modulePath: "github.com/goplus/llpkg/libxslt", + expected: true, + }, + { + name: "github hosted root", + modulePath: "github.com/goplus/llpkg", + expected: true, + }, + { + name: "other github module", + modulePath: "github.com/other/repo", + expected: false, + }, + { + name: "non-github module", + modulePath: "example.com/module", + expected: false, + }, + { + name: "empty path", + modulePath: "", + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := IsGithubHosted(tt.modulePath) + if result != tt.expected { + t.Errorf("expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestCanSkipFetch(t *testing.T) { + tempDir, err := os.MkdirTemp("", "can_skip_fetch_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + t.Run("pkgconfig exists", func(t *testing.T) { + moduleDir := filepath.Join(tempDir, "with_pkgconfig") + pkgConfigDir := filepath.Join(moduleDir, "lib", "pkgconfig") + err := os.MkdirAll(pkgConfigDir, 0755) + if err != nil { + t.Fatal(err) + } + + result := IsInstalled(moduleDir) + if !result { + t.Error("expected true when pkgconfig directory exists") + } + }) + + t.Run("pkgconfig does not exist", func(t *testing.T) { + moduleDir := filepath.Join(tempDir, "without_pkgconfig") + err := os.MkdirAll(moduleDir, 0755) + if err != nil { + t.Fatal(err) + } + + result := IsInstalled(moduleDir) + if result { + t.Error("expected false when pkgconfig directory does not exist") + } + }) + + t.Run("module dir does not exist", func(t *testing.T) { + nonexistentDir := filepath.Join(tempDir, "nonexistent") + + result := IsInstalled(nonexistentDir) + if result { + t.Error("expected false when module directory does not exist") + } + }) +} + +func TestModuleDirOf(t *testing.T) { + tests := []struct { + name string + modulePath string + moduleVersion string + expectError bool + expectSuffix string + }{ + { + name: "normal module", + modulePath: "github.com/goplus/llpkg/libxslt", + moduleVersion: "v1.0.3", + expectError: false, + expectSuffix: "github.com/goplus/llpkg/libxslt@v1.0.3", + }, + { + name: "module with special chars", + modulePath: "example.com/moduleTest", + moduleVersion: "v2.0.0", + expectError: false, + expectSuffix: `example.com/module\!test@v2.0.0`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := LLGoModuleDirOf(tt.modulePath, tt.moduleVersion) + + if tt.expectError { + if err == nil { + t.Error("expected error but got none") + } + return + } + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if !strings.HasSuffix(result, tt.expectSuffix) { + t.Errorf("expected result to end with %s, got %s", tt.expectSuffix, result) + } + }) + } +} + +func TestParseConfigFile(t *testing.T) { + tempDir, err := os.MkdirTemp("", "parse_config_test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + t.Run("valid config file", func(t *testing.T) { + config := LLPkgConfig{ + Upstream: UpstreamConfig{ + Installer: InstallerConfig{ + Name: "conan", + Config: map[string]string{ + "profile": "default", + }, + }, + Package: installer.Package{ + Name: "libxslt", + Version: "v1.0.3", + }, + }, + } + + configFile := filepath.Join(tempDir, "valid_config.json") + file, err := os.Create(configFile) + if err != nil { + t.Fatal(err) + } + + encoder := json.NewEncoder(file) + err = encoder.Encode(config) + file.Close() + if err != nil { + t.Fatal(err) + } + + result, err := ParseConfigFile(configFile) + if err != nil { + t.Fatal(err) + } + + if result.Upstream.Installer.Name != "conan" { + t.Errorf("expected installer name 'conan', got %s", result.Upstream.Installer.Name) + } + + if result.Upstream.Package.Name != "libxslt" { + t.Errorf("expected package name 'libxslt', got %s", result.Upstream.Package.Name) + } + + if result.Upstream.Package.Version != "v1.0.3" { + t.Errorf("expected package version 'v1.0.3', got %s", result.Upstream.Package.Version) + } + }) + + t.Run("nonexistent file", func(t *testing.T) { + nonexistentFile := filepath.Join(tempDir, "nonexistent.json") + + _, err := ParseConfigFile(nonexistentFile) + if err == nil { + t.Error("expected error for nonexistent file") + } + }) + + t.Run("invalid json", func(t *testing.T) { + invalidFile := filepath.Join(tempDir, "invalid.json") + err := os.WriteFile(invalidFile, []byte("invalid json content"), 0644) + if err != nil { + t.Fatal(err) + } + + _, err = ParseConfigFile(invalidFile) + if err == nil { + t.Error("expected error for invalid JSON") + } + }) +} diff --git a/internal/taskqueue/taskqueue.go b/internal/taskqueue/taskqueue.go new file mode 100644 index 000000000..0714885c0 --- /dev/null +++ b/internal/taskqueue/taskqueue.go @@ -0,0 +1,64 @@ +package taskqueue + +import "sync" + +// TaskQueue represents a concurrent task queue that manages task execution +// with a semaphore-based synchronization mechanism. It allows multiple +// goroutines to process tasks concurrently while providing synchronization +// through a WaitGroup. +type TaskQueue struct { + sema sync.WaitGroup // Semaphore for tracking pending tasks + taskQueue chan func() // Channel for queuing tasks to be executed +} + +// NewTaskQueue creates a new TaskQueue with n worker goroutines. +// Each worker goroutine will continuously process tasks from the queue. +// The parameter n specifies both the buffer size of the task channel +// and the number of worker goroutines to spawn. +func NewTaskQueue(n int) *TaskQueue { + sq := &TaskQueue{taskQueue: make(chan func(), n)} + + // Start n worker goroutines to process tasks + for i := 0; i < n; i++ { + go sq.run() + } + + return sq +} + +// Push adds a new task to the queue for execution. +// The task will be executed by one of the available worker goroutines. +// This method increments the semaphore counter before queuing the task. +func (sq *TaskQueue) Push(task func()) { + sq.sema.Add(1) + sq.taskQueue <- task +} + +// Wait blocks until all currently queued tasks have been completed. +// This method waits for the semaphore counter to reach zero, +// indicating that all pushed tasks have finished execution. +func (sq *TaskQueue) Wait() { + sq.sema.Wait() +} + +// run is the main worker loop executed by each worker goroutine. +// It continuously receives tasks from the task channel and executes them. +// When a task is completed, it decrements the semaphore counter. +// The loop terminates when it receives a nil task or when the channel is closed. +func (sq *TaskQueue) run() { + for task := range sq.taskQueue { + if task == nil { + return + } + task() + sq.sema.Done() + } +} + +// Close shuts down the TaskQueue by closing the task channel. +// This will cause all worker goroutines to terminate once they finish +// processing their current tasks. Returns nil as no error can occur. +func (sq *TaskQueue) Close() error { + close(sq.taskQueue) + return nil +} diff --git a/internal/taskqueue/taskqueue_test.go b/internal/taskqueue/taskqueue_test.go new file mode 100644 index 000000000..aacc81983 --- /dev/null +++ b/internal/taskqueue/taskqueue_test.go @@ -0,0 +1,132 @@ +package taskqueue + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestNewTaskQueue(t *testing.T) { + tq := NewTaskQueue(3) + if tq == nil { + t.Fatal("NewTaskQueue returned nil") + } + if tq.taskQueue == nil { + t.Fatal("taskQueue channel is nil") + } + + tq.Close() +} + +func TestTaskQueueBasicExecution(t *testing.T) { + tq := NewTaskQueue(2) + defer tq.Close() + + var executed int32 + + tq.Push(func() { + atomic.AddInt32(&executed, 1) + }) + + tq.Wait() + + if atomic.LoadInt32(&executed) != 1 { + t.Errorf("Expected 1 task executed, got %d", executed) + } +} + +func TestTaskQueueMultipleTasks(t *testing.T) { + tq := NewTaskQueue(3) + defer tq.Close() + + var executed int32 + taskCount := 10 + + for i := 0; i < taskCount; i++ { + tq.Push(func() { + atomic.AddInt32(&executed, 1) + }) + } + + tq.Wait() + + if atomic.LoadInt32(&executed) != int32(taskCount) { + t.Errorf("Expected %d tasks executed, got %d", taskCount, executed) + } +} + +func TestTaskQueueConcurrentExecution(t *testing.T) { + workerCount := 5 + tq := NewTaskQueue(workerCount) + defer tq.Close() + + var started int32 + var finished int32 + taskCount := 20 + + for i := 0; i < taskCount; i++ { + tq.Push(func() { + atomic.AddInt32(&started, 1) + time.Sleep(10 * time.Millisecond) + atomic.AddInt32(&finished, 1) + }) + } + + tq.Wait() + + if atomic.LoadInt32(&started) != int32(taskCount) { + t.Errorf("Expected %d tasks started, got %d", taskCount, started) + } + if atomic.LoadInt32(&finished) != int32(taskCount) { + t.Errorf("Expected %d tasks finished, got %d", taskCount, finished) + } +} + +func TestTaskQueueWaitFunctionality(t *testing.T) { + tq := NewTaskQueue(2) + defer tq.Close() + + var taskCompleted bool + var mu sync.Mutex + + tq.Push(func() { + time.Sleep(50 * time.Millisecond) + mu.Lock() + taskCompleted = true + mu.Unlock() + }) + + tq.Wait() + + mu.Lock() + completed := taskCompleted + mu.Unlock() + + if !completed { + t.Error("Wait() returned before task completion") + } +} + +func TestTaskQueueClose(t *testing.T) { + tq := NewTaskQueue(2) + + var executed int32 + + for i := 0; i < 5; i++ { + tq.Push(func() { + atomic.AddInt32(&executed, 1) + }) + } + + tq.Wait() + + err := tq.Close() + if err != nil { + t.Errorf("Close() returned error: %v", err) + } + + if atomic.LoadInt32(&executed) != 5 { + t.Errorf("Expected 5 tasks executed before close, got %d", executed) + } +}