fix: improve import speed

docs/src/content/docs/reference/configuration.md

@@ -145,11 +145,6 @@ If the environment variable is defined without **and** with the suffix at the sa
 
 - Description: A unix timestamp. If an imported listen has a timestamp before this, it will be discarded.
 
-##### KOITO_FETCH_IMAGES_DURING_IMPORT
-
-- Default: `false`
-- Description: When true, images will be downloaded and cached during imports.
-
 ##### KOITO_CORS_ALLOWED_ORIGINS
 
 - Default: No CORS policy

engine/engine.go

@@ -236,23 +236,33 @@ func Run(
 
 	l.Info().Msg("Engine: Beginning startup tasks...")
 
-	l.Debug().Msg("Engine: Checking import configuration")
-	if !cfg.SkipImport() {
-		go func() {
+	go func() {
+		l.Info().Msg("Engine: Checking image cache migration status")
+		if err := catalog.MigrateImageCache(ctx, store); err != nil {
+			l.Err(err).Msg("Engine: Failed to migrate image cache")
+		}
+		l.Info().Msg("Engine: Pruning orphaned images")
+		if err := catalog.PruneOrphanedImages(ctx, store); err != nil {
+			l.Err(err).Msg("Engine: Failed to prune orphaned images")
+		}
+
+		l.Debug().Msg("Engine: Checking import configuration")
+		if !cfg.SkipImport() {
 			RunImporter(l, store, mbzC)
-		}()
-	}
+		}
+
+		l.Info().Msg("Engine: Attempting to fetch missing artist images")
+		if err := catalog.FetchMissingArtistImages(ctx, store); err != nil {
+			l.Err(err).Msg("Engine: Failed to fetch missing artist images")
+		}
+		l.Info().Msg("Engine: Attempting to fetch missing album images")
+		if err := catalog.FetchMissingAlbumImages(ctx, store); err != nil {
+			l.Err(err).Msg("Engine: Failed to fetch missing album images")
+		}
+	}()
 
-	l.Info().Msg("Engine: Pruning orphaned images")
-	go catalog.PruneOrphanedImages(logger.NewContext(l), store)
-	l.Info().Msg("Engine: Checking image cache migration status")
-	go catalog.MigrateImageCache(logger.NewContext(l), store)
 	l.Info().Msg("Engine: Running duration backfill task")
 	go catalog.BackfillTrackDurationsFromMusicBrainz(ctx, store, mbzC)
-	l.Info().Msg("Engine: Attempting to fetch missing artist images")
-	go catalog.FetchMissingArtistImages(ctx, store)
-	l.Info().Msg("Engine: Attempting to fetch missing album images")
-	go catalog.FetchMissingAlbumImages(ctx, store)
 
 	l.Info().Msg("Engine: Initialization finished")
 	quit := make(chan os.Signal, 1)

internal/catalog/associate_album.go

@@ -23,7 +23,7 @@ type AssociateAlbumOpts struct {
 	ReleaseName       string
 	TrackName         string // required
 	Mbzc              mbz.MusicBrainzCaller
-	SkipCacheImage    bool
+	SkipImageLookup   bool
 }
 
 func AssociateAlbum(ctx context.Context, d db.AlbumStore, opts AssociateAlbumOpts) (*models.Album, error) {
@@ -121,29 +121,27 @@ func createOrUpdateAlbumWithMbzReleaseID(ctx context.Context, d db.AlbumStore, o
 			}
 		}
 
-		l.Debug().Msg("Searching for album images...")
 		var imgid uuid.UUID
-		imgUrl, err := images.GetAlbumImage(ctx, images.AlbumImageOpts{
-			Artists:      utils.UniqueIgnoringCase(slices.Concat(utils.FlattenMbzArtistCreditNames(release.ArtistCredit), utils.FlattenArtistNames(opts.Artists))),
-			Album:        release.Title,
-			ReleaseMbzID: &opts.ReleaseMbzID,
-		})
-
-		if err == nil && imgUrl != "" {
-			imgid = uuid.New()
-			if !opts.SkipCacheImage {
+		var imgUrl string
+		if !opts.SkipImageLookup {
+			l.Debug().Msg("Searching for album images...")
+			var imgErr error
+			imgUrl, imgErr = images.GetAlbumImage(ctx, images.AlbumImageOpts{
+				Artists:      utils.UniqueIgnoringCase(slices.Concat(utils.FlattenMbzArtistCreditNames(release.ArtistCredit), utils.FlattenArtistNames(opts.Artists))),
+				Album:        release.Title,
+				ReleaseMbzID: &opts.ReleaseMbzID,
+			})
+			if imgErr == nil && imgUrl != "" {
+				imgid = uuid.New()
 				l.Debug().Msg("Downloading album image from source...")
-				err = imagecache.DownloadImage(imgid, imgUrl)
-				if err != nil {
-					l.Err(err).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image")
+				if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil {
+					l.Err(dlErr).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image")
 				}
+			} else if imgErr != nil {
+				l.Debug().Msgf("createOrUpdateAlbumWithMbzReleaseID: failed to get album images for %s: %s", release.Title, imgErr.Error())
 			}
 		}
 
-		if err != nil {
-			l.Debug().Msgf("createOrUpdateAlbumWithMbzReleaseID: failed to get album images for %s: %s", release.Title, err.Error())
-		}
-
 		album, err = d.SaveAlbum(ctx, db.SaveAlbumOpts{
 			Title:          release.Title,
 			MusicBrainzID:  opts.ReleaseMbzID,
@@ -210,24 +208,24 @@ func matchAlbumByTitle(ctx context.Context, d db.AlbumStore, opts AssociateAlbum
 		return nil, fmt.Errorf("matchAlbumByTitle: %w", err)
 	} else {
 		var imgid uuid.UUID
-		imgUrl, err := images.GetAlbumImage(ctx, images.AlbumImageOpts{
-			Artists:      utils.FlattenArtistNames(opts.Artists),
-			Album:        opts.ReleaseName,
-			ReleaseMbzID: &opts.ReleaseMbzID,
-		})
-		if err == nil && imgUrl != "" {
-			imgid = uuid.New()
-			if !opts.SkipCacheImage {
+		var imgUrl string
+		if !opts.SkipImageLookup {
+			var imgErr error
+			imgUrl, imgErr = images.GetAlbumImage(ctx, images.AlbumImageOpts{
+				Artists:      utils.FlattenArtistNames(opts.Artists),
+				Album:        opts.ReleaseName,
+				ReleaseMbzID: &opts.ReleaseMbzID,
+			})
+			if imgErr == nil && imgUrl != "" {
+				imgid = uuid.New()
 				l.Debug().Msg("Downloading album image from source...")
-				err = imagecache.DownloadImage(imgid, imgUrl)
-				if err != nil {
-					l.Err(err).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image")
+				if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil {
+					l.Err(dlErr).Msg("matchAlbumByTitle: failed to cache image")
 				}
+			} else if imgErr != nil {
+				l.Debug().AnErr("error", imgErr).Msgf("matchAlbumByTitle: failed to get album images for %s", opts.ReleaseName)
 			}
 		}
-		if err != nil {
-			l.Debug().AnErr("error", err).Msgf("matchAlbumByTitle: failed to get album images for %s", opts.ReleaseName)
-		}
 
 		a, err = d.SaveAlbum(ctx, db.SaveAlbumOpts{
 			Title:         releaseName,

internal/catalog/associate_artists.go

@@ -25,7 +25,7 @@ type AssociateArtistsOpts struct {
 	TrackTitle    string
 	Mbzc          mbz.MusicBrainzCaller
 
-	SkipCacheImage bool
+	SkipImageLookup bool
 }
 
 func AssociateArtists(ctx context.Context, d db.ArtistStore, opts AssociateArtistsOpts) ([]*models.Artist, error) {
@@ -127,20 +127,21 @@ func matchArtistsByMBIDMappings(ctx context.Context, d db.ArtistStore, opts Asso
 			l.Warn().AnErr("error", err).Msg("matchArtistsByMBIDMappings: MusicBrainz unreachable, creating new artist with provided MusicBrainz ID mapping")
 
 			var imgid uuid.UUID
-			imgUrl, imgErr := images.GetArtistImage(ctx, images.ArtistImageOpts{
-				Aliases: []string{a.Artist},
-			})
-			if imgErr == nil && imgUrl != "" {
-				imgid = uuid.New()
-				if !opts.SkipCacheImage {
+			var imgUrl string
+			if !opts.SkipImageLookup {
+				var imgErr error
+				imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{
+					Aliases: []string{a.Artist},
+				})
+				if imgErr == nil && imgUrl != "" {
+					imgid = uuid.New()
 					l.Debug().Msg("Downloading artist image from source...")
-					err = imagecache.DownloadImage(imgid, imgUrl)
-					if err != nil {
-						l.Err(err).Msg("Failed to cache image")
+					if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil {
+						l.Err(dlErr).Msg("Failed to cache image")
 					}
+				} else {
+					l.Err(imgErr).Msgf("matchArtistsByMBIDMappings: Failed to get artist image for artist '%s'", a.Artist)
 				}
-			} else {
-				l.Err(imgErr).Msgf("matchArtistsByMBIDMappings: Failed to get artist image for artist '%s'", a.Artist)
 			}
 
 			artist, err = d.SaveArtist(ctx, db.SaveArtistOpts{
@@ -240,20 +241,21 @@ func resolveAliasOrCreateArtist(ctx context.Context, mbzID uuid.UUID, names []st
 	}
 
 	var imgid uuid.UUID
-	imgUrl, err := images.GetArtistImage(ctx, images.ArtistImageOpts{
-		Aliases: aliases,
-	})
-	if err == nil && imgUrl != "" {
-		imgid = uuid.New()
-		if !opts.SkipCacheImage {
+	var imgUrl string
+	if !opts.SkipImageLookup {
+		var imgErr error
+		imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{
+			Aliases: aliases,
+		})
+		if imgErr == nil && imgUrl != "" {
+			imgid = uuid.New()
 			l.Debug().Msg("Downloading artist image from source...")
-			err = imagecache.DownloadImage(imgid, imgUrl)
-			if err != nil {
-				l.Err(err).Msg("Failed to cache image")
+			if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil {
+				l.Err(dlErr).Msg("Failed to cache image")
 			}
+		} else if imgErr != nil {
+			l.Warn().AnErr("error", imgErr).Msg("Failed to get artist image from ImageSrc")
 		}
-	} else if err != nil {
-		l.Warn().AnErr("error", err).Msg("Failed to get artist image from ImageSrc")
 	}
 
 	u, err := d.SaveArtist(ctx, db.SaveArtistOpts{
@@ -289,20 +291,21 @@ func matchArtistsByNames(ctx context.Context, names []string, existing []*models
 		}
 		if errors.Is(err, db.ErrNotFound) {
 			var imgid uuid.UUID
-			imgUrl, err := images.GetArtistImage(ctx, images.ArtistImageOpts{
-				Aliases: []string{name},
-			})
-			if err == nil && imgUrl != "" {
-				imgid = uuid.New()
-				if !opts.SkipCacheImage {
+			var imgUrl string
+			if !opts.SkipImageLookup {
+				var imgErr error
+				imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{
+					Aliases: []string{name},
+				})
+				if imgErr == nil && imgUrl != "" {
+					imgid = uuid.New()
 					l.Debug().Msg("Downloading artist image from source...")
-					err = imagecache.DownloadImage(imgid, imgUrl)
-					if err != nil {
-						l.Err(err).Msg("Failed to cache image")
+					if dlErr := imagecache.DownloadImage(imgid, imgUrl); dlErr != nil {
+						l.Err(dlErr).Msg("Failed to cache image")
 					}
+				} else if imgErr != nil {
+					l.Debug().AnErr("error", imgErr).Msgf("Failed to get artist images for %s", name)
 				}
-			} else if err != nil {
-				l.Debug().AnErr("error", err).Msgf("Failed to get artist images for %s", name)
 			}
 			a, err = d.SaveArtist(ctx, db.SaveArtistOpts{Name: name, Image: imgid, ImageSrc: imgUrl})
 			if err != nil {

internal/catalog/catalog.go

@@ -42,8 +42,8 @@ type SubmitListenOpts struct {
 	// artist, release, release group, and track in DB
 	SkipSaveListen bool
 
-	// When true, skips caching the images and only stores the image url in the db
-	SkipCacheImage bool
+	// When true, skips requests for album and artist images
+	SkipImageLookup bool
 
 	MbzCaller          mbz.MusicBrainzCaller
 	ArtistNames        []string
@@ -88,13 +88,13 @@ func SubmitListen(ctx context.Context, store submitListenStore, opts SubmitListe
 		ctx,
 		store,
 		AssociateArtistsOpts{
-			ArtistMbzIDs:   opts.ArtistMbzIDs,
-			ArtistNames:    opts.ArtistNames,
-			ArtistName:     opts.Artist,
-			ArtistMbidMap:  opts.ArtistMbidMappings,
-			Mbzc:           opts.MbzCaller,
-			TrackTitle:     opts.TrackTitle,
-			SkipCacheImage: opts.SkipCacheImage,
+			ArtistMbzIDs:    opts.ArtistMbzIDs,
+			ArtistNames:     opts.ArtistNames,
+			ArtistName:      opts.Artist,
+			ArtistMbidMap:   opts.ArtistMbidMappings,
+			Mbzc:            opts.MbzCaller,
+			TrackTitle:      opts.TrackTitle,
+			SkipImageLookup: opts.SkipImageLookup,
 		})
 	if err != nil {
 		l.Err(err).Msg("Failed to associate artists to listen")
@@ -116,7 +116,7 @@ func SubmitListen(ctx context.Context, store submitListenStore, opts SubmitListe
 		TrackName:         opts.TrackTitle,
 		Mbzc:              opts.MbzCaller,
 		Artists:           artists,
-		SkipCacheImage:    opts.SkipCacheImage,
+		SkipImageLookup:   opts.SkipImageLookup,
 	})
 	if err != nil {
 		l.Error().Err(err).Msg("Failed to associate release group to listen")

internal/cfg/cfg.go

@@ -57,38 +57,37 @@ type config struct {
 	listenPort int
 	configDir  string
 	// baseUrl              string
-	sqliteEnabled          bool
-	databaseUrl            string
-	musicBrainzUrl         string
-	musicBrainzRateLimit   int
-	logLevel               int
-	structuredLogging      bool
-	lbzRelayEnabled        bool
-	lbzRelayUrl            string
-	lbzRelayToken          string
-	defaultPw              string
-	defaultUsername        string
-	defaultTheme           string
-	disableDeezer          bool
-	disableCAA             bool
-	disableMusicBrainz     bool
-	subsonicUrl            string
-	subsonicParams         string
-	lastfmApiKey           string
-	subsonicEnabled        bool
-	skipImport             bool
-	fetchImageDuringImport bool
-	allowedHosts           []string
-	allowAllHosts          bool
-	allowedOrigins         []string
-	disableRateLimit       bool
-	importThrottleMs       int
-	userAgent              string
-	importBefore           time.Time
-	importAfter            time.Time
-	artistSeparators       []*regexp.Regexp
-	loginGate              bool
-	forceTZ                *time.Location
+	sqliteEnabled        bool
+	databaseUrl          string
+	musicBrainzUrl       string
+	musicBrainzRateLimit int
+	logLevel             int
+	structuredLogging    bool
+	lbzRelayEnabled      bool
+	lbzRelayUrl          string
+	lbzRelayToken        string
+	defaultPw            string
+	defaultUsername      string
+	defaultTheme         string
+	disableDeezer        bool
+	disableCAA           bool
+	disableMusicBrainz   bool
+	subsonicUrl          string
+	subsonicParams       string
+	lastfmApiKey         string
+	subsonicEnabled      bool
+	skipImport           bool
+	allowedHosts         []string
+	allowAllHosts        bool
+	allowedOrigins       []string
+	disableRateLimit     bool
+	importThrottleMs     int
+	userAgent            string
+	importBefore         time.Time
+	importAfter          time.Time
+	artistSeparators     []*regexp.Regexp
+	loginGate            bool
+	forceTZ              *time.Location
 }
 
 var (
@@ -163,7 +162,10 @@ func loadConfig(getenv func(string) string, version string) (*config, error) {
 	cfg.disableRateLimit = parseBool(getenv(DISABLE_RATE_LIMIT_ENV))
 
 	cfg.structuredLogging = parseBool(getenv(ENABLE_STRUCTURED_LOGGING_ENV))
-	cfg.fetchImageDuringImport = parseBool(getenv(FETCH_IMAGES_DURING_IMPORT_ENV))
+	fetchImageDuringImport := getenv(FETCH_IMAGES_DURING_IMPORT_ENV)
+	if fetchImageDuringImport != "" {
+		return nil, fmt.Errorf(`loadConfig: %s has no effect. Images are always fetched after import`, FETCH_IMAGES_DURING_IMPORT_ENV)
+	}
 
 	cfg.disableDeezer = parseBool(getenv(DISABLE_DEEZER_ENV))
 	cfg.disableCAA = parseBool(getenv(DISABLE_COVER_ART_ARCHIVE_ENV))

internal/cfg/getters.go

@@ -181,12 +181,6 @@ func ImportWindow() (time.Time, time.Time) {
 	return globalConfig.importBefore, globalConfig.importAfter
 }
 
-func FetchImagesDuringImport() bool {
-	lock.RLock()
-	defer lock.RUnlock()
-	return globalConfig.fetchImageDuringImport
-}
-
 func ArtistSeparators() []*regexp.Regexp {
 	lock.RLock()
 	defer lock.RUnlock()