diff --git a/cmd/warpforge/catalog.go b/cmd/warpforge/catalog.go index 3fb36a81..525bb78c 100644 --- a/cmd/warpforge/catalog.go +++ b/cmd/warpforge/catalog.go @@ -3,6 +3,7 @@ package main import ( "bytes" "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -13,10 +14,12 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" "github.com/urfave/cli/v2" + "go.opentelemetry.io/otel" + + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/pkg/logging" "github.com/warpfork/warpforge/pkg/plotexec" "github.com/warpfork/warpforge/wfapi" - "go.opentelemetry.io/otel" ) const defaultCatalogUrl = "https://github.com/warpsys/mincatalog.git" @@ -108,9 +111,10 @@ func cmdCatalogInit(c *cli.Context) error { return fmt.Errorf("no catalog name provided") } catalogName := c.Args().First() + fsys := os.DirFS("/") // open the workspace set and get the catalog path - wsSet, err := openWorkspaceSet() + wsSet, err := openWorkspaceSet(fsys) if err != nil { return err } @@ -145,8 +149,10 @@ func cmdCatalogAdd(c *cli.Context) error { catalogRefStr := c.Args().Get(1) url := c.Args().Get(2) + fsys := os.DirFS("/") + // open the workspace set - wsSet, err := openWorkspaceSet() + wsSet, err := openWorkspaceSet(fsys) if err != nil { return err } @@ -255,7 +261,9 @@ func cmdCatalogAdd(c *cli.Context) error { } func cmdCatalogLs(c *cli.Context) error { - wsSet, err := openWorkspaceSet() + fsys := os.DirFS("/") + + wsSet, err := openWorkspaceSet(fsys) if err != nil { return err } @@ -311,7 +319,9 @@ func gatherCatalogRefs(plot wfapi.Plot) []wfapi.CatalogRef { } func cmdCatalogBundle(c *cli.Context) error { - wsSet, err := openWorkspaceSet() + fsys := os.DirFS("/") + + wsSet, err := openWorkspaceSet(fsys) if err != nil { return err } @@ -320,8 +330,9 @@ func cmdCatalogBundle(c *cli.Context) error { if err != nil { return fmt.Errorf("failed to get pwd: %s", err) } + pwd = pwd[1:] // Drop leading slash, for use with fs package. - plot, err := plotFromFile(filepath.Join(pwd, PLOT_FILE_NAME)) + plot, err := dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot)) if err != nil { return err } @@ -330,14 +341,14 @@ func cmdCatalogBundle(c *cli.Context) error { catalogPath := filepath.Join(pwd, ".warpforge", "catalog") // create a catalog if it does not exist - if _, err = os.Stat(catalogPath); os.IsNotExist(err) { - err = os.MkdirAll(catalogPath, 0755) + if _, err = fs.Stat(fsys, catalogPath); os.IsNotExist(err) { + err = os.MkdirAll("/"+catalogPath, 0755) if err != nil { return fmt.Errorf("failed to create catalog directory: %s", err) } // we need to reopen the workspace set after creating the directory - wsSet, err = openWorkspaceSet() + wsSet, err = openWorkspaceSet(fsys) if err != nil { return err } @@ -397,7 +408,9 @@ func installDefaultRemoteCatalog(c *cli.Context, path string) error { } func cmdCatalogUpdate(c *cli.Context) error { - wss, err := openWorkspaceSet() + fsys := os.DirFS("/") + + wss, err := openWorkspaceSet(fsys) if err != nil { return fmt.Errorf("failed to open workspace set: %s", err) } @@ -480,8 +493,10 @@ func cmdCatalogRelease(c *cli.Context) error { } catalogName := c.String("name") + fsys := os.DirFS("/") + // open the workspace set - wsSet, err := openWorkspaceSet() + wsSet, err := openWorkspaceSet(fsys) if err != nil { return err } @@ -499,7 +514,7 @@ func cmdCatalogRelease(c *cli.Context) error { } // get the module, release, and item values (in format `module:release:item`) - module, err := moduleFromFile("module.wf") + module, err := dab.ModuleFromFile(fsys, "module.wf") if err != nil { return err } @@ -507,7 +522,7 @@ func cmdCatalogRelease(c *cli.Context) error { releaseName := c.Args().Get(0) fmt.Printf("building replay for module = %q, release = %q, executing plot...\n", module.Name, releaseName) - plot, err := plotFromFile(PLOT_FILE_NAME) + plot, err := dab.PlotFromFile(fsys, dab.MagicFilename_Plot) if err != nil { return err } @@ -561,6 +576,8 @@ func cmdIngestGitTags(c *cli.Context) error { url := c.Args().Get(1) itemName := c.Args().Get(2) + fsys := os.DirFS("/") + // open the remote and list all references remote := git.NewRemote(memory.NewStorage(), &config.RemoteConfig{ Name: "origin", @@ -573,7 +590,7 @@ func cmdIngestGitTags(c *cli.Context) error { // open the workspace set and catalog catalogName := c.String("name") - wsSet, err := openWorkspaceSet() + wsSet, err := openWorkspaceSet(fsys) if err != nil { return err } diff --git a/cmd/warpforge/check.go b/cmd/warpforge/check.go index a58e5deb..7dbd9ad4 100644 --- a/cmd/warpforge/check.go +++ b/cmd/warpforge/check.go @@ -7,6 +7,7 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/json" "github.com/urfave/cli/v2" + "github.com/warpfork/warpforge/pkg/plotexec" "github.com/warpfork/warpforge/wfapi" ) diff --git a/cmd/warpforge/ferk.go b/cmd/warpforge/ferk.go index e8bc9fc2..f8a53f71 100644 --- a/cmd/warpforge/ferk.go +++ b/cmd/warpforge/ferk.go @@ -8,10 +8,12 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/json" "github.com/urfave/cli/v2" + "go.opentelemetry.io/otel" + + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/pkg/logging" "github.com/warpfork/warpforge/pkg/plotexec" "github.com/warpfork/warpforge/wfapi" - "go.opentelemetry.io/otel" ) var ferkCmdDef = cli.Command{ @@ -89,7 +91,9 @@ func cmdFerk(c *cli.Context) error { ctx, span := tr.Start(ctx, c.Command.FullName()) defer span.End() - wss, err := openWorkspaceSet() + fsys := os.DirFS("/") + + wss, err := openWorkspaceSet(fsys) if err != nil { return err } @@ -97,7 +101,7 @@ func cmdFerk(c *cli.Context) error { plot := wfapi.Plot{} if c.String("plot") != "" { // plot was provided, load from file - plot, err = plotFromFile(c.String("plot")) + plot, err = dab.PlotFromFile(fsys, c.String("plot")) if err != nil { return fmt.Errorf("error loading plot from file %q: %s", c.String("plot"), err) } diff --git a/cmd/warpforge/main.go b/cmd/warpforge/main.go index 4713019a..40952194 100644 --- a/cmd/warpforge/main.go +++ b/cmd/warpforge/main.go @@ -59,6 +59,13 @@ func makeApp(stdin io.Reader, stdout, stderr io.Writer) *cli.App { &statusCmdDef, &quickstartCmdDef, &ferkCmdDef, + &cli.Command{ + Name: "workspace", + Usage: "Grouping for subcommands that inspect or affect a whole workspace.", + Subcommands: []*cli.Command{ + &cmdDefWorkspaceInspect, + }, + }, } return app } diff --git a/cmd/warpforge/main_test.go b/cmd/warpforge/main_test.go index defea2e0..e3d2b50f 100644 --- a/cmd/warpforge/main_test.go +++ b/cmd/warpforge/main_test.go @@ -11,6 +11,7 @@ import ( qt "github.com/frankban/quicktest" "github.com/warpfork/go-testmark" "github.com/warpfork/go-testmark/testexec" + "github.com/warpfork/warpforge/pkg/workspace" ) diff --git a/cmd/warpforge/quickstart.go b/cmd/warpforge/quickstart.go index 933a789f..7ef3ad5b 100644 --- a/cmd/warpforge/quickstart.go +++ b/cmd/warpforge/quickstart.go @@ -7,6 +7,8 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/json" "github.com/urfave/cli/v2" + + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/wfapi" ) @@ -59,13 +61,13 @@ func cmdQuickstart(c *cli.Context) error { return fmt.Errorf("no module name provided") } - _, err := os.Stat(MODULE_FILE_NAME) + _, err := os.Stat(dab.MagicFilename_Module) if !os.IsNotExist(err) { - return fmt.Errorf("%s file already exists", MODULE_FILE_NAME) + return fmt.Errorf("%s file already exists", dab.MagicFilename_Module) } - _, err = os.Stat(PLOT_FILE_NAME) + _, err = os.Stat(dab.MagicFilename_Plot) if !os.IsNotExist(err) { - return fmt.Errorf("%s file already exists", PLOT_FILE_NAME) + return fmt.Errorf("%s file already exists", dab.MagicFilename_Plot) } moduleName := c.Args().First() @@ -79,7 +81,7 @@ func cmdQuickstart(c *cli.Context) error { if err != nil { return fmt.Errorf("failed to serialize module") } - err = os.WriteFile(MODULE_FILE_NAME, moduleSerial, 0644) + err = os.WriteFile(dab.MagicFilename_Module, moduleSerial, 0644) if err != nil { return fmt.Errorf("failed to write module.json file: %s", err) } @@ -94,17 +96,17 @@ func cmdQuickstart(c *cli.Context) error { return fmt.Errorf("failed to serialize plot") } - err = os.WriteFile(PLOT_FILE_NAME, plotSerial, 0644) + err = os.WriteFile(dab.MagicFilename_Plot, plotSerial, 0644) if err != nil { - return fmt.Errorf("failed to write %s: %s", PLOT_FILE_NAME, err) + return fmt.Errorf("failed to write %s: %s", dab.MagicFilename_Plot, err) } if !c.Bool("quiet") { - fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", MODULE_FILE_NAME, PLOT_FILE_NAME, moduleName) + fmt.Fprintf(c.App.Writer, "Successfully created %s and %s for module %q.\n", dab.MagicFilename_Module, dab.MagicFilename_Plot, moduleName) fmt.Fprintf(c.App.Writer, "Ensure your catalogs are up to date by running `%s catalog update.`.\n", os.Args[0]) fmt.Fprintf(c.App.Writer, "You can check status of this module with `%s status`.\n", os.Args[0]) fmt.Fprintf(c.App.Writer, "You can run this module with `%s run`.\n", os.Args[0]) - fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", PLOT_FILE_NAME) + fmt.Fprintf(c.App.Writer, "Once you've run the Hello World example, edit the 'script' section of %s to customize what happens.\n", dab.MagicFilename_Plot) } return nil diff --git a/cmd/warpforge/run.go b/cmd/warpforge/run.go index 881625fe..a19cb21e 100644 --- a/cmd/warpforge/run.go +++ b/cmd/warpforge/run.go @@ -3,19 +3,21 @@ package main import ( "context" "fmt" - "io/ioutil" + "io/fs" "os" "path/filepath" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/codec/json" "github.com/urfave/cli/v2" + "go.opentelemetry.io/otel" + + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/pkg/formulaexec" "github.com/warpfork/warpforge/pkg/logging" "github.com/warpfork/warpforge/pkg/plotexec" "github.com/warpfork/warpforge/pkg/workspace" "github.com/warpfork/warpforge/wfapi" - "go.opentelemetry.io/otel" ) var runCmdDef = cli.Command{ @@ -36,16 +38,16 @@ var runCmdDef = cli.Command{ }, } -func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) { +func execModule(ctx context.Context, fsys fs.FS, config wfapi.PlotExecConfig, fileName string) (wfapi.PlotResults, error) { result := wfapi.PlotResults{} // parse the module, even though it is not currently used - _, err := moduleFromFile(fileName) + _, err := dab.ModuleFromFile(fsys, fileName) if err != nil { return result, err } - plot, err := plotFromFile(filepath.Join(filepath.Dir(fileName), PLOT_FILE_NAME)) + plot, err := dab.PlotFromFile(fsys, filepath.Join(filepath.Dir(fileName), dab.MagicFilename_Plot)) if err != nil { return result, err } @@ -55,7 +57,7 @@ func execModule(ctx context.Context, config wfapi.PlotExecConfig, fileName strin return result, err } - wss, err := openWorkspaceSet() + wss, err := openWorkspaceSet(fsys) if err != nil { return result, err } @@ -97,13 +99,16 @@ func cmdRun(c *cli.Context) error { }, } + fsys := os.DirFS("/") + if !c.Args().Present() { // execute the module in the current directory pwd, err := os.Getwd() if err != nil { return fmt.Errorf("could not get current directory") } - _, err = execModule(ctx, config, filepath.Join(pwd, MODULE_FILE_NAME)) + pwd = pwd[1:] // Drop leading slash, for use with fs package. + _, err = execModule(ctx, fsys, config, filepath.Join(pwd, dab.MagicFilename_Module)) if err != nil { return err } @@ -114,11 +119,11 @@ func cmdRun(c *cli.Context) error { if err != nil { return err } - if filepath.Base(path) == MODULE_FILE_NAME { + if filepath.Base(path) == dab.MagicFilename_Module { if c.Bool("verbose") { logger.Debug("executing %q", path) } - _, err = execModule(ctx, config, path) + _, err = execModule(ctx, fsys, config, path) if err != nil { return err } @@ -134,13 +139,13 @@ func cmdRun(c *cli.Context) error { } if info.IsDir() { // directory provided, execute module if it exists - _, err := execModule(ctx, config, filepath.Join(fileName, "module.wf")) + _, err := execModule(ctx, fsys, config, filepath.Join(fileName, "module.wf")) if err != nil { return err } } else { // formula or module file provided - f, err := ioutil.ReadFile(fileName) + f, err := fs.ReadFile(fsys, fileName) if err != nil { return err } @@ -169,7 +174,7 @@ func cmdRun(c *cli.Context) error { return err } case "module": - _, err := execModule(ctx, config, fileName) + _, err := execModule(ctx, fsys, config, fileName) if err != nil { return err } diff --git a/cmd/warpforge/status.go b/cmd/warpforge/status.go index 07cb2061..c44933d5 100644 --- a/cmd/warpforge/status.go +++ b/cmd/warpforge/status.go @@ -3,13 +3,17 @@ package main import ( "bytes" "fmt" + "io/fs" "os" "os/exec" "path/filepath" "github.com/fatih/color" "github.com/urfave/cli/v2" + + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/pkg/formulaexec" + "github.com/warpfork/warpforge/pkg/plotexec" "github.com/warpfork/warpforge/wfapi" ) @@ -25,10 +29,12 @@ func cmdStatus(c *cli.Context) error { fmtWarning := color.New(color.FgHiRed, color.Bold) verbose := c.Bool("verbose") + fsys := os.DirFS("/") pwd, err := os.Getwd() if err != nil { return fmt.Errorf("could not get current directory") } + pwd = pwd[1:] // Drop leading slash, for use with fs package. // display version if verbose { @@ -85,12 +91,16 @@ func cmdStatus(c *cli.Context) error { // check if pwd is a module, read module and set flag isModule := false var module wfapi.Module - if _, err := os.Stat(filepath.Join(pwd, MODULE_FILE_NAME)); err == nil { + if _, err := fs.Stat(fsys, filepath.Join(pwd, dab.MagicFilename_Module)); err == nil { isModule = true - module, err = moduleFromFile(filepath.Join(pwd, MODULE_FILE_NAME)) + module, err = dab.ModuleFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Module)) if err != nil { return fmt.Errorf("failed to open module file: %s", err) } + } else if os.IsNotExist(err) { + // fine; it can just not exist. + } else { + return err } if isModule { @@ -102,11 +112,11 @@ func cmdStatus(c *cli.Context) error { // display module and plot info var plot wfapi.Plot hasPlot := false - _, err = os.Stat(filepath.Join(pwd, PLOT_FILE_NAME)) + _, err = fs.Stat(fsys, filepath.Join(pwd, dab.MagicFilename_Plot)) if isModule && err == nil { // module.wf and plot.wf exists, read the plot hasPlot = true - plot, err = plotFromFile(filepath.Join(pwd, PLOT_FILE_NAME)) + plot, err = dab.PlotFromFile(fsys, filepath.Join(pwd, dab.MagicFilename_Plot)) if err != nil { return fmt.Errorf("failed to open plot file: %s", err) } @@ -118,42 +128,27 @@ func cmdStatus(c *cli.Context) error { len(plot.Steps.Keys), len(plot.Outputs.Keys)) - // check for missing catalog refs - wss, err := openWorkspaceSet() + // Load up workspaces; we'll need to read them to check for missing catalog refs. + wss, err := openWorkspaceSet(fsys) if err != nil { return fmt.Errorf("failed to open workspace: %s", err) } - catalogRefCount := 0 - resolvedCatalogRefCount := 0 - ingestCount := 0 - mountCount := 0 - for _, input := range plot.Inputs.Values { - if input.Basis().Mount != nil { - mountCount++ - } else if input.Basis().Ingest != nil { - ingestCount++ - } else if input.Basis().CatalogRef != nil { - catalogRefCount++ - ware, _, err := wss.GetCatalogWare(*input.PlotInputSimple.CatalogRef) - if err != nil { - return fmt.Errorf("failed to lookup catalog ref: %s", err) - } - if ware == nil { - fmt.Fprintf(c.App.Writer, "\tMissing catalog item: %q.\n", input.Basis().CatalogRef.String()) - } else if err == nil { - resolvedCatalogRefCount++ - } - } + + // Compute stats on the plot and report on them (especially any problematic ones). + plotStats, err := plotexec.ComputeStats(plot, wss) + + fmt.Fprintf(c.App.Writer, "\tPlot contains %d catalog inputs. %d/%d catalog inputs resolved successfully.\n", plotStats.InputsUsingCatalog, plotStats.ResolvableCatalogInputs, plotStats.InputsUsingCatalog) + if plotStats.ResolvableCatalogInputs < plotStats.InputsUsingCatalog { + fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d unresolved catalog inputs!\n", (plotStats.InputsUsingCatalog - plotStats.ResolvableCatalogInputs)) } - fmt.Fprintf(c.App.Writer, "\tPlot contains %d catalog inputs. %d/%d catalog inputs resolved successfully.\n", catalogRefCount, resolvedCatalogRefCount, catalogRefCount) - if resolvedCatalogRefCount < catalogRefCount { - fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d unresolved catalog inputs!\n", (catalogRefCount - resolvedCatalogRefCount)) + for k, _ := range plotStats.UnresolvedCatalogInputs { + fmt.Fprintf(c.App.Writer, "\tMissing catalog item: %q.\n", k.String()) } - if ingestCount > 0 { - fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d ingest inputs and is not hermetic!\n", ingestCount) + if plotStats.InputsUsingIngest > 0 { + fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d ingest inputs and is not hermetic!\n", plotStats.InputsUsingIngest) } - if mountCount > 0 { - fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d mount inputs and is not hermetic!\n", mountCount) + if plotStats.InputsUsingMount > 0 { + fmt.Fprintf(c.App.Writer, "\tWarning: plot contains %d mount inputs and is not hermetic!\n", plotStats.InputsUsingMount) } } else if isModule { @@ -163,26 +158,26 @@ func cmdStatus(c *cli.Context) error { // display workspace info fmt.Fprintf(c.App.Writer, "\nWorkspace:\n") - wss, err := openWorkspaceSet() + wss, err := openWorkspaceSet(fsys) if err != nil { return fmt.Errorf("failed to open workspace set: %s", err) } // handle special case for pwd - fmt.Fprintf(c.App.Writer, "\t%s (pwd", pwd) + fmt.Fprintf(c.App.Writer, "\t/%s (pwd", pwd) if isModule { fmt.Fprintf(c.App.Writer, ", module") } // check if it's a workspace - if _, err := os.Stat(filepath.Join(pwd, ".warpforge")); !os.IsNotExist(err) { + if _, err := fs.Stat(fsys, filepath.Join(pwd, ".warpforge")); !os.IsNotExist(err) { fmt.Fprintf(c.App.Writer, ", workspace") } // check if it's a root workspace - if _, err := os.Stat(filepath.Join(pwd, ".warpforge/root")); !os.IsNotExist(err) { + if _, err := fs.Stat(fsys, filepath.Join(pwd, ".warpforge/root")); !os.IsNotExist(err) { fmt.Fprintf(c.App.Writer, ", root workspace") } // check if it's a git repo - if _, err := os.Stat(filepath.Join(pwd, ".git")); !os.IsNotExist(err) { + if _, err := fs.Stat(fsys, filepath.Join(pwd, ".git")); !os.IsNotExist(err) { fmt.Fprintf(c.App.Writer, ", git repo") } @@ -193,7 +188,7 @@ func cmdStatus(c *cli.Context) error { fs, subPath := ws.Path() path := fmt.Sprintf("%s%s", fs, subPath) - if path == pwd { + if path == "/"+pwd { // we handle pwd earlier, ignore continue } diff --git a/cmd/warpforge/util.go b/cmd/warpforge/util.go index 9da219fd..68be19f6 100644 --- a/cmd/warpforge/util.go +++ b/cmd/warpforge/util.go @@ -2,22 +2,14 @@ package main import ( "fmt" - "io/ioutil" + "io/fs" "os" "path/filepath" "strings" - "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/codec/json" "github.com/warpfork/warpforge/pkg/workspace" - "github.com/warpfork/warpforge/wfapi" ) -// special file names for plot and module files -// these are json files with special formatting for detection -const PLOT_FILE_NAME = "plot.wf" -const MODULE_FILE_NAME = "module.wf" - // Returns the file type, which is the file name without extension // e.g., formula.wf -> formula, module.wf -> module, etc... func getFileType(name string) (string, error) { @@ -49,53 +41,15 @@ func binPath(bin string) (string, error) { // stack: a workspace stack starting at the current working directory, // root workspace: the first marked root workspace in the stack, or the home workspace if none are marked, // home workspace: the workspace at the user's homedir -func openWorkspaceSet() (workspace.WorkspaceSet, error) { - pwd, err := os.Getwd() +func openWorkspaceSet(fsys fs.FS) (workspace.WorkspaceSet, error) { + pwd, err := os.Getwd() // FIXME why are you doing this again? you almost certainly already did it moments ago. if err != nil { return workspace.WorkspaceSet{}, fmt.Errorf("failed to get working directory: %s", err) } - wss, err := workspace.OpenWorkspaceSet(os.DirFS("/"), "", pwd[1:]) + wss, err := workspace.OpenWorkspaceSet(fsys, "", pwd[1:]) if err != nil { return workspace.WorkspaceSet{}, fmt.Errorf("failed to open workspace: %s", err) } return wss, nil } - -// takes a path to a plot file, returns a plot -func plotFromFile(filename string) (wfapi.Plot, error) { - f, err := ioutil.ReadFile(filename) - if err != nil { - return wfapi.Plot{}, err - } - - plotCapsule := wfapi.PlotCapsule{} - _, err = ipld.Unmarshal(f, json.Decode, &plotCapsule, wfapi.TypeSystem.TypeByName("PlotCapsule")) - if err != nil { - return wfapi.Plot{}, err - } - if plotCapsule.Plot == nil { - return wfapi.Plot{}, fmt.Errorf("no v1 Plot in PlotCapsule") - } - - return *plotCapsule.Plot, nil -} - -// takes a path to a module file, returns a module -func moduleFromFile(filename string) (wfapi.Module, error) { - f, err := ioutil.ReadFile(filename) - if err != nil { - return wfapi.Module{}, err - } - - moduleCapsule := wfapi.ModuleCapsule{} - _, err = ipld.Unmarshal(f, json.Decode, &moduleCapsule, wfapi.TypeSystem.TypeByName("ModuleCapsule")) - if err != nil { - return wfapi.Module{}, err - } - if moduleCapsule.Module == nil { - return wfapi.Module{}, fmt.Errorf("no v1 Module in ModuleCapsule") - } - - return *moduleCapsule.Module, nil -} diff --git a/cmd/warpforge/watch.go b/cmd/warpforge/watch.go index f90ceb67..94e3b75d 100644 --- a/cmd/warpforge/watch.go +++ b/cmd/warpforge/watch.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "path/filepath" "time" @@ -9,9 +10,11 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" "github.com/urfave/cli/v2" + "go.opentelemetry.io/otel" + + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/pkg/logging" "github.com/warpfork/warpforge/wfapi" - "go.opentelemetry.io/otel" ) var watchCmdDef = cli.Command{ @@ -38,10 +41,12 @@ func cmdWatch(c *cli.Context) error { defer span.End() path := c.Args().First() + fsys := os.DirFS("/") // TODO: currently we read the module/plot from the provided path. // instead, we should read it from the git cache dir - plot, err := plotFromFile(filepath.Join(path, PLOT_FILE_NAME)) + // FIXME: though it's rare, this can be considerably divergent + plot, err := dab.PlotFromFile(fsys, filepath.Join(path, dab.MagicFilename_Plot)) if err != nil { return err } @@ -94,7 +99,9 @@ func cmdWatch(c *cli.Context) error { if ingestCache[path] != hash { fmt.Println("path", path, "changed, new hash", hash) ingestCache[path] = hash - _, err := execModule(ctx, config, filepath.Join(c.Args().First(), MODULE_FILE_NAME)) + // FIXME: this is also reading off the working tree filesystem instead of out of the git index, which is wrong + // Perhaps ideally we'd like to give this thing a whole fsys that just keeps reading out of the git index. + _, err := execModule(ctx, fsys, config, filepath.Join(c.Args().First(), dab.MagicFilename_Module)) if err != nil { fmt.Printf("exec failed: %s\n", err) } diff --git a/cmd/warpforge/winspect.go b/cmd/warpforge/winspect.go new file mode 100644 index 00000000..a4f392a5 --- /dev/null +++ b/cmd/warpforge/winspect.go @@ -0,0 +1,205 @@ +package main + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/urfave/cli/v2" + + "github.com/warpfork/warpforge/pkg/dab" + "github.com/warpfork/warpforge/pkg/plotexec" +) + +var cmdDefWorkspaceInspect = cli.Command{ + Name: "inspect", + Usage: "Inspect and report upon the situation of the current workspace (how many modules are there, have we got a cached evaluation of them, etc).", + Action: cmdFnWorkspaceInspect, + // Aliases: []string{"winspect"}, // doesn't put them at the top level. Womp. + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "gohard", + Usage: "whether to spend effort checking the health of modules found; if false, just list them.", + Value: true, + }, + }, +} + +func cmdFnWorkspaceInspect(c *cli.Context) error { + fsys := os.DirFS("/") + + // First, find the workspace. + wss, err := openWorkspaceSet(fsys) + if err != nil { + return fmt.Errorf("failed to open workspace set: %s", err) + } + + // Briefly report on the nearest workspace. + // (We could talk about the grandparents too, but 'wf status' already does that; here we want to focus more on contents than parentage.) + wsFs, wsPath := wss.Stack[0].Path() + fmt.Fprintf(c.App.Writer, "Workspace: %s%s\n", wsFs, wsPath) + + // Search for modules within the workspace. + return fs.WalkDir(wsFs, wsPath, func(path string, d fs.DirEntry, err error) error { + // fmt.Fprintf(c.App.Writer, "hi: %s%s\n", wsFs, path) + + if err != nil { + return err + } + + // Don't ever look into warpforge guts directories. + if d.Name() == dab.MagicFilename_Workspace { + return fs.SkipDir + } + + // If this is a dir (beyond the root): look see if it contains a workspace marker. + // If it does, we might not want to report on it. + // TODO: a bool flag for this. + if d.IsDir() && len(path) > len(wsPath) { + _, e2 := fs.Stat(wsFs, filepath.Join(path, dab.MagicFilename_Workspace)) + if e2 == nil || os.IsNotExist(e2) { + // carry on + } else { + return fs.SkipDir + } + } + + // Peek for module file. + if d.Name() == dab.MagicFilename_Module { + modPathWithinWs := path[len(wsPath)+1 : len(path)-len(dab.MagicFilename_Module)] // leave the trailing slash on. For disambig in case we support multiple module files per dir someday. + mod, err := dab.ModuleFromFile(wsFs, path) + modName := mod.Name + if err != nil { + modName = "!!Unknown!!" + } + + // 0 = idk; 1 = yes; 2 = no. (0 generally doesn't get rendered.) + everythingParses := 0 + importsResolve := 0 + noticeIngestUsage := 0 + noticeMountUsage := 0 + havePacksCached := 0 // maybe should have a variant for "or we have a replay we're hopeful about"? + haveRunrecord := 0 + haveHappyExit := 0 + if c.Bool("gohard") { + if err != nil { + everythingParses = 2 + goto _checksDone + } + plot, err := dab.PlotFromFile(wsFs, filepath.Join(filepath.Dir(path), dab.MagicFilename_Plot)) + if err != nil { + everythingParses = 2 + goto _checksDone + } + everythingParses = 1 + plotStats, err := plotexec.ComputeStats(plot, wss) + if err != nil { + return err // if it's hardcore catalog errors, rather than just unresolvables, I'm out + } + if plotStats.ResolvableCatalogInputs == plotStats.InputsUsingCatalog { + importsResolve = 1 + } else { + importsResolve = 2 + } + if plotStats.InputsUsingIngest > 0 { + noticeIngestUsage = 1 + } else { + noticeIngestUsage = 2 + } + if plotStats.InputsUsingMount > 0 { + noticeMountUsage = 1 + } else { + noticeMountUsage = 2 + } + // TODO: havePacksCached is not supported right now :( + // TODO: haveRunrecord needs to both do resolve, and go peek at memos, and yet (obviously) not actually run. + // TODO: haveHappyExit needs the above. + } + _checksDone: + + // Tell me about it. + // FUTURE: perhaps a workspace configuration option for defaults for these padding sizes. + fmt.Fprintf(c.App.Writer, "Module found: %-40q -- at path %-26q", modName, modPathWithinWs) + if c.Bool("gohard") { + fmt.Fprintf(c.App.Writer, " -- %v %v %v %v %v %v %v", + glyphCheckOrKlaxon(everythingParses), + glyphCheckOrX(importsResolve), + glyphCautionary(noticeIngestUsage), + glyphCautionary(noticeMountUsage), + glyphCheckOrYellow(havePacksCached), + glyphCheckOrNada(haveRunrecord), + glyphCheckOrKlaxon(haveHappyExit), + ) + } + fmt.Fprintf(c.App.Writer, "\n") + } + + return nil + }) +} + +func glyphCheckOrX(state int) string { + switch state { + case 0: + return " " + case 1: + return "✔" + case 2: + return "✘" + default: + panic("unreachable") + } +} + +func glyphCheckOrKlaxon(state int) string { + switch state { + case 0: + return " " + case 1: + return "✔" + case 2: + return "!" + default: + panic("unreachable") + } +} + +func glyphCheckOrYellow(state int) string { + switch state { + case 0: + return " " + case 1: + return "✔" + case 2: + return "å" + default: + panic("unreachable") + } +} + +func glyphCheckOrNada(state int) string { + switch state { + case 0: + return " " + case 1: + return "✔" + case 2: + return "_" + default: + panic("unreachable") + } +} + +func glyphCautionary(state int) string { + switch state { + case 0: + return " " + case 1: + return "⚠" + case 2: + return "_" + default: + panic("unreachable") + } +} diff --git a/pkg/dab/doc.go b/pkg/dab/doc.go new file mode 100644 index 00000000..72c95632 --- /dev/null +++ b/pkg/dab/doc.go @@ -0,0 +1,23 @@ +/* + Package dab -- short for Data Access Broker -- contains functions that help save and load data, + mostly to a local filesystem (but sometimes to a blind content-addressed objectstore, as well). + + Most dab functions return objects from the wfapi package. + Some return a dab type, in which case that object is to help manage further access -- + but eventually you should still reach wfapi data types. + + Functions that deal with the filesystem may expect to be dealing with either + a workspace filesystem (e.g., conmingled with other user files), + or a catalog filesystem projection (a somewhat stricter situation). + Sometimes these are the same. + The function name should provide a hint about which situations it handles. + + Sometimes, search features are provided for workspace filesystems, + since there is no other index of those contents aside from the filesystem itself. + + Most of these functions return the "latest" version of their relevant API type. + At the moment, that's not saying much, because we haven't grown in such a way + that we support major varations of API object reversions -- but in the future, + this means these functions may do "migrational" transforms to the data on the fly. +*/ +package dab diff --git a/pkg/dab/module.go b/pkg/dab/module.go new file mode 100644 index 00000000..3aee51d9 --- /dev/null +++ b/pkg/dab/module.go @@ -0,0 +1,78 @@ +package dab + +import ( + "fmt" + "io/fs" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/json" + + "github.com/warpfork/warpforge/wfapi" +) + +const ( + MagicFilename_Module = "module.wf" + MagicFilename_Plot = "plot.wf" +) + +// ModuleFromFile loads a wfapi.Module from filesystem path. +// +// In typical usage, the filename parameter will have the suffix of MagicFilename_Module. +// +// Errors: +// +// - warpforge-error-io -- for errors reading from fsys. +// - warpforge-error-serialization -- for errors from try to parse the data as a Module. +// - warpforge-error-datatoonew -- if encountering unknown data from a newer version of warpforge! +// +func ModuleFromFile(fsys fs.FS, filename string) (wfapi.Module, error) { + situation := "loading a module" + + f, err := fs.ReadFile(fsys, filename) + if err != nil { + return wfapi.Module{}, wfapi.ErrorIo(situation, &filename, err) + } + + moduleCapsule := wfapi.ModuleCapsule{} + _, err = ipld.Unmarshal(f, json.Decode, &moduleCapsule, wfapi.TypeSystem.TypeByName("ModuleCapsule")) + if err != nil { + return wfapi.Module{}, wfapi.ErrorSerialization(situation, err) + } + if moduleCapsule.Module == nil { + // ... this isn't really reachable. + return wfapi.Module{}, wfapi.ErrorDataTooNew(situation, fmt.Errorf("no v1 Module in ModuleCapsule")) + } + + return *moduleCapsule.Module, nil +} + +// PlotFromFile loads a wfapi.Plot from filesystem path. +// +// In typical usage, the filename parameter will have the suffix of MagicFilename_Plot. +// +// Errors: +// +// - warpforge-error-io -- for errors reading from fsys. +// - warpforge-error-serialization -- for errors from try to parse the data as a Plot. +// - warpforge-error-datatoonew -- if encountering unknown data from a newer version of warpforge! +// +func PlotFromFile(fsys fs.FS, filename string) (wfapi.Plot, error) { + situation := "loading a plot" + + f, err := fs.ReadFile(fsys, filename) + if err != nil { + return wfapi.Plot{}, wfapi.ErrorIo(situation, &filename, err) + } + + plotCapsule := wfapi.PlotCapsule{} + _, err = ipld.Unmarshal(f, json.Decode, &plotCapsule, wfapi.TypeSystem.TypeByName("PlotCapsule")) + if err != nil { + return wfapi.Plot{}, wfapi.ErrorSerialization(situation, err) + } + if plotCapsule.Plot == nil { + // ... this isn't really reachable. + return wfapi.Plot{}, wfapi.ErrorDataTooNew(situation, fmt.Errorf("no v1 Plot in PlotCapsule")) + } + + return *plotCapsule.Plot, nil +} diff --git a/pkg/dab/workspace.go b/pkg/dab/workspace.go new file mode 100644 index 00000000..d80267ef --- /dev/null +++ b/pkg/dab/workspace.go @@ -0,0 +1,5 @@ +package dab + +const ( + MagicFilename_Workspace = ".warpforge" +) diff --git a/pkg/plotexec/plot_stats.go b/pkg/plotexec/plot_stats.go new file mode 100644 index 00000000..5c54580a --- /dev/null +++ b/pkg/plotexec/plot_stats.go @@ -0,0 +1,97 @@ +package plotexec + +import ( + "github.com/warpfork/warpforge/pkg/workspace" + "github.com/warpfork/warpforge/wfapi" +) + +// Might not match the package name -- funcs in this file certainly don't exec anything. + +type PlotStats struct { + InputsUsingCatalog int + InputsUsingIngest int + InputsUsingMount int + ResolvableCatalogInputs int + ResolvedCatalogInputs map[wfapi.CatalogRef]wfapi.WareID // might as well remember it if we already did all that work. + UnresolvedCatalogInputs map[wfapi.CatalogRef]struct{} +} + +// ComputeStats counts up how many times a plot uses various features, +// and also checks for reference resolvablity. +// +// Any errors arising from this process have to do with failure to load +// catalog info, and causes immediate abort which will result in +// incomplete counts for all features. +// +// Errors: +// +// - warpforge-error-catalog-invalid -- like it says on the tin. +// - warpforge-error-catalog-parse -- like it says on the tin. +// - warpforge-error-io -- for IO errors while reading catalogs. +// +func ComputeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) (PlotStats, error) { + v := PlotStats{ + ResolvedCatalogInputs: make(map[wfapi.CatalogRef]wfapi.WareID), + UnresolvedCatalogInputs: make(map[wfapi.CatalogRef]struct{}), + } + return v, v.computeStats(plot, wsSet) +} + +func (v *PlotStats) computeStats(plot wfapi.Plot, wsSet workspace.WorkspaceSet) error { + for _, input := range plot.Inputs.Values { + if err := v.accountForInput(input, wsSet); err != nil { + return err + } + } + for _, step := range plot.Steps.Values { + switch { + case step.Plot != nil: + if err := v.computeStats(*step.Plot, wsSet); err != nil { + return err + } + case step.Protoformula != nil: + for _, input := range step.Protoformula.Inputs.Values { + if err := v.accountForInput(input, wsSet); err != nil { + return err + } + } + default: + panic("unreachable") + } + } + return nil +} + +func (v *PlotStats) accountForInput(input wfapi.PlotInput, wsSet workspace.WorkspaceSet) error { + inputBasis := input.Basis() // unwrap if it's a complex filtered thing. + switch { + // This switch should be exhaustive on the possible members of PlotInputSimple. + case inputBasis.WareID != nil: + return nil // not interesting :) + case inputBasis.Mount != nil: + v.InputsUsingMount++ + return nil + case inputBasis.Literal != nil: + return nil // not interesting :) + case inputBasis.Pipe != nil: + return nil // not interesting :) + case inputBasis.CatalogRef != nil: + v.InputsUsingCatalog++ + ware, _, err := wsSet.GetCatalogWare(*inputBasis.CatalogRef) + if err != nil { + return err // These mean catalog read failed entirely, so we're in deep water. + } + if ware == nil { + v.UnresolvedCatalogInputs[*inputBasis.CatalogRef] = struct{}{} + } else { + v.ResolvableCatalogInputs++ + v.ResolvedCatalogInputs[*inputBasis.CatalogRef] = *ware + } + return nil + case inputBasis.Ingest != nil: + v.InputsUsingIngest++ + return nil + default: + panic("unreachable") + } +} diff --git a/pkg/workspace/fsdetect.go b/pkg/workspace/fsdetect.go index 531cbf0d..17c767d2 100644 --- a/pkg/workspace/fsdetect.go +++ b/pkg/workspace/fsdetect.go @@ -6,11 +6,12 @@ import ( "os" "path/filepath" + "github.com/warpfork/warpforge/pkg/dab" "github.com/warpfork/warpforge/wfapi" ) const ( - magicWorkspaceDirname = ".warpforge" + magicWorkspaceDirname = dab.MagicFilename_Workspace ) var homedir string diff --git a/pkg/workspace/workspace_set.go b/pkg/workspace/workspace_set.go index 00bf7021..64576873 100644 --- a/pkg/workspace/workspace_set.go +++ b/pkg/workspace/workspace_set.go @@ -14,7 +14,7 @@ import ( type WorkspaceSet struct { Home *Workspace Root *Workspace - Stack []*Workspace + Stack []*Workspace // the 0'th index is the closest workspace; the next is its parent, and so on. } // Opens a full WorkspaceSet diff --git a/wfapi/error.go b/wfapi/error.go index 958fa2f2..11145824 100644 --- a/wfapi/error.go +++ b/wfapi/error.go @@ -175,6 +175,24 @@ func ErrorSerialization(context string, cause error) Error { } } +// ErrorDataTooNew is returned when some data was (partially) deserialized, +// but only enough that we could recognize it as being a newer version of message +// than this application supports. +// +// Errors: +// +// - warpforge-error-datatoonew -- if some data is too new to parse completely. +func ErrorDataTooNew(context string, cause error) Error { + return &ErrorVal{ + CodeString: "warpforge-error-datatoonew", + Message: fmt.Sprintf("while %s, encountered data from an unknown version: %s", context, cause), + Details: [][2]string{ + {"context", context}, + }, + Cause: wrapErr(cause), + } +} + // ErrorWareUnpack is returned when the unpacking of a ware fails // // Errors: