diff --git a/simpletuner/simpletuner_sdk/server/services/cloud/job_submission.py b/simpletuner/simpletuner_sdk/server/services/cloud/job_submission.py
index 421770f54..b6e776444 100644
--- a/simpletuner/simpletuner_sdk/server/services/cloud/job_submission.py
+++ b/simpletuner/simpletuner_sdk/server/services/cloud/job_submission.py
@@ -49,6 +49,16 @@ class SubmissionContext:
reservation_id: Optional[str] = None
quota_warnings: List[str] = field(default_factory=list)
+ @property
+ def tracker_metadata(self) -> Dict[str, str]:
+ metadata = {}
+ project = self.config.get("--tracker_project_name") or self.config.get("tracker_project_name")
+ if project:
+ metadata["tracker_project_name"] = project
+ if self.tracker_run_name:
+ metadata["tracker_run_name"] = self.tracker_run_name
+ return metadata
+
@dataclass
class SubmissionResult:
@@ -206,8 +216,7 @@ async def submit(
unified_job.output_url = f"https://huggingface.co/{hub_model_id}"
if snapshot_metadata:
unified_job.metadata["snapshot"] = snapshot_metadata
- if ctx.tracker_run_name:
- unified_job.metadata["tracker_run_name"] = ctx.tracker_run_name
+ unified_job.metadata.update(ctx.tracker_metadata)
if ctx.hardware_profile:
unified_job.metadata["hardware_profile"] = ctx.hardware_profile
@@ -296,8 +305,7 @@ async def _create_upload_job(
metadata: Dict[str, Any] = {"upload_id": job_id}
if snapshot_metadata:
metadata["snapshot"] = snapshot_metadata
- if ctx.tracker_run_name:
- metadata["tracker_run_name"] = ctx.tracker_run_name
+ metadata.update(ctx.tracker_metadata)
if ctx.hardware_profile:
metadata["hardware_profile"] = ctx.hardware_profile
@@ -327,8 +335,7 @@ async def _update_upload_job_from_provider(
metadata.setdefault("prediction_id", cloud_job.job_id)
if snapshot_metadata:
metadata["snapshot"] = snapshot_metadata
- if ctx.tracker_run_name:
- metadata["tracker_run_name"] = ctx.tracker_run_name
+ metadata.update(ctx.tracker_metadata)
if ctx.upload_id:
metadata["upload_id"] = ctx.upload_id
if ctx.hardware_profile:
diff --git a/simpletuner/static/js/modules/cloud/index.js b/simpletuner/static/js/modules/cloud/index.js
index 772c1e683..0781cc259 100644
--- a/simpletuner/static/js/modules/cloud/index.js
+++ b/simpletuner/static/js/modules/cloud/index.js
@@ -612,7 +612,7 @@ if (!window.cloudDashboardComponent) {
filtered = filtered.filter(j => {
const jobId = (j.job_id || '').toLowerCase();
const configName = (j.config_name || '').toLowerCase();
- const trackerName = (j.metadata?.tracker_run_name || '').toLowerCase();
+ const trackerName = this.jobDisplayName(j).toLowerCase();
const status = (j.status || '').toLowerCase();
const provider = (j.provider || '').toLowerCase();
return jobId.includes(query) || configName.includes(query) ||
diff --git a/simpletuner/static/js/modules/cloud/jobs.js b/simpletuner/static/js/modules/cloud/jobs.js
index 9a1e09206..3ce34fe34 100644
--- a/simpletuner/static/js/modules/cloud/jobs.js
+++ b/simpletuner/static/js/modules/cloud/jobs.js
@@ -5,6 +5,25 @@
*/
window.cloudJobMethods = {
+ async continueJob(job) {
+ const trainer = Alpine.store('trainer');
+ if (trainer.continuingJob || job.job_type !== 'local' || job.status !== 'failed') return;
+ trainer.continuingJob = job.job_id;
+ try {
+ const switched = await trainer.switchEnvironment(job.metadata?.env_name || job.config_name);
+ if (!switched) return;
+ trainer.switchTab('basic');
+ await htmx.ajax('GET', '/web/trainer/tabs/basic', { target: '#tab-content', swap: 'innerHTML' });
+ await Alpine.nextTick();
+ document.getElementById('runBtn').click();
+ } catch (error) {
+ console.error('Failed to continue job:', error);
+ window.showToast('Failed to continue job', 'error');
+ } finally {
+ trainer.continuingJob = null;
+ }
+ },
+
async loadJobs(syncActive = false) {
if (!syncActive) {
this.jobsLoading = true;
diff --git a/simpletuner/static/js/modules/cloud/utilities.js b/simpletuner/static/js/modules/cloud/utilities.js
index 9f189e42c..75e710d40 100644
--- a/simpletuner/static/js/modules/cloud/utilities.js
+++ b/simpletuner/static/js/modules/cloud/utilities.js
@@ -46,7 +46,12 @@ window.cloudUtilityMethods = {
},
jobDisplayName(job) {
- return job.metadata?.tracker_run_name || job.config_name || job.job_id.substring(0, 8);
+ const metadata = job.metadata || {};
+ const config = metadata.runtime_config || {};
+ const project = metadata.tracker_project_name || config['--tracker_project_name'] || config.tracker_project_name;
+ const run = metadata.tracker_run_name || config['--tracker_run_name'] || config.tracker_run_name ||
+ metadata.run_name || job.config_name || job.job_id.substring(0, 8);
+ return project ? `${project} / ${run}` : run;
},
formatDuration(seconds) {
@@ -204,7 +209,7 @@ window.cloudComputedProperties = {
filtered = filtered.filter(j => {
const jobId = (j.job_id || '').toLowerCase();
const configName = (j.config_name || '').toLowerCase();
- const trackerName = (j.metadata?.tracker_run_name || '').toLowerCase();
+ const trackerName = window.cloudUtilityMethods.jobDisplayName(j).toLowerCase();
const status = (j.status || '').toLowerCase();
const provider = (j.provider || '').toLowerCase();
diff --git a/simpletuner/templates/partials/cloud_job_card.html b/simpletuner/templates/partials/cloud_job_card.html
index 15a296d47..b6d8925b3 100644
--- a/simpletuner/templates/partials/cloud_job_card.html
+++ b/simpletuner/templates/partials/cloud_job_card.html
@@ -20,7 +20,7 @@
+ :title="jobDisplayName(job)" x-text="jobDisplayName(job)">
@@ -74,6 +74,14 @@
+
+
+
diff --git a/simpletuner/templates/trainer_htmx.html b/simpletuner/templates/trainer_htmx.html
index d6be7ed7e..d2e2c476b 100644
--- a/simpletuner/templates/trainer_htmx.html
+++ b/simpletuner/templates/trainer_htmx.html
@@ -780,6 +780,8 @@
this.environmentsLoading = false;
}
},
+ continuingJob: null,
+
async switchEnvironment(configName) {
try {
const sanitizedName = sanitizeConfigName(configName);
@@ -819,15 +821,19 @@
}
}));
- await this.fetchActiveEnvironmentConfig();
+ if (!await this.fetchActiveEnvironmentConfig()) {
+ throw new Error('Failed to load activated configuration');
+ }
await this.loadDatasetsAfterEnvironmentChange();
window.showToast(`Switched to ${configName} configuration`, 'success');
// Refresh current tab content instead of reloading
await this.refreshAfterEnvironmentSwitch();
+ return true;
} catch (error) {
console.error('Error switching config:', error);
window.showToast('Failed to switch configuration', 'error');
+ return false;
}
},
async refreshAfterEnvironmentSwitch() {
diff --git a/tests/js/cloud_utilities.test.js b/tests/js/cloud_utilities.test.js
index 26340426c..5d2027c4b 100644
--- a/tests/js/cloud_utilities.test.js
+++ b/tests/js/cloud_utilities.test.js
@@ -466,3 +466,23 @@ describe('cloudComputedProperties', () => {
});
});
});
+
+describe('job project identity', () => {
+ test.each([
+ [{ tracker_project_name: 'portraits', tracker_run_name: 'run' }, 'portraits / run'],
+ [{ runtime_config: { '--tracker_project_name': 'portraits', '--tracker_run_name': 'run' } }, 'portraits / run'],
+ [{ runtime_config: { tracker_project_name: 'portraits', tracker_run_name: 'run' } }, 'portraits / run'],
+ [{ run_name: 'legacy-run' }, 'legacy-run'],
+ ])('uses persisted tracker identity %j', (metadata, expected) => {
+ expect(window.cloudUtilityMethods.jobDisplayName({ job_id: '123456789', config_name: 'config', metadata })).toBe(expected);
+ });
+});
+
+test('project name search distinguishes jobs with the same run name', () => {
+ const jobs = ['portraits', 'landscapes'].map(project => ({
+ job_id: project, config_name: 'config', status: 'failed',
+ metadata: { runtime_config: { tracker_project_name: project, tracker_run_name: 'run' } },
+ }));
+ const getter = Object.getOwnPropertyDescriptor(window.cloudComputedProperties, 'filteredJobs').get;
+ expect(getter.call({ jobs, jobSearchQuery: 'LANDSCAPES' }).map(job => job.job_id)).toEqual(['landscapes']);
+});
diff --git a/tests/test_cloud_job_tracker_metadata.py b/tests/test_cloud_job_tracker_metadata.py
new file mode 100644
index 000000000..54afd6c3a
--- /dev/null
+++ b/tests/test_cloud_job_tracker_metadata.py
@@ -0,0 +1,38 @@
+"""Tracker identity survives each cloud job persistence path."""
+
+import unittest
+from unittest.mock import AsyncMock
+
+from simpletuner.simpletuner_sdk.server.services.cloud.job_submission import JobSubmissionService, SubmissionContext
+
+
+class CloudJobTrackerMetadataTestCase(unittest.IsolatedAsyncioTestCase):
+ async def test_upload_job_records_project_and_run(self):
+ for prefix in ("", "--"):
+ with self.subTest(prefix=prefix):
+ store = AsyncMock()
+ service = JobSubmissionService(store)
+ context = SubmissionContext(
+ config={f"{prefix}tracker_project_name": "portraits"},
+ dataloader_config=[],
+ tracker_run_name="shared-run",
+ )
+ await service._create_upload_job(context, "upload-1", {})
+ job = store.add_job.call_args.args[0]
+ self.assertEqual(job.metadata["tracker_project_name"], "portraits")
+ self.assertEqual(job.metadata["tracker_run_name"], "shared-run")
+
+ async def test_provider_update_preserves_project_identity(self):
+ from simpletuner.simpletuner_sdk.server.services.cloud.base import CloudJobInfo, CloudJobStatus
+
+ store = AsyncMock()
+ service = JobSubmissionService(store)
+ context = SubmissionContext(
+ config={"tracker_project_name": "portraits"}, dataloader_config=[], tracker_run_name="shared-run"
+ )
+ job = CloudJobInfo(job_id="provider-1", provider="replicate", status=CloudJobStatus.RUNNING, created_at="now")
+ await service._update_upload_job_from_provider("upload-1", job, context, {}, None, None, None)
+ metadata = store.update_job.call_args.args[1]["metadata"]
+ self.assertEqual(metadata["tracker_project_name"], "portraits")
+ self.assertEqual(metadata["tracker_run_name"], "shared-run")
+ self.assertEqual(metadata["prediction_id"], "provider-1")
diff --git a/tests/test_webui_e2e.py b/tests/test_webui_e2e.py
index 5b7219172..404e5b328 100644
--- a/tests/test_webui_e2e.py
+++ b/tests/test_webui_e2e.py
@@ -3915,5 +3915,109 @@ def _config_created(_driver):
self.for_each_browser("test_save_as_preserves_config_and_repoints_identity", scenario)
+class FailedLocalJobContinueTestCase(_TrainerPageMixin, WebUITestCase):
+ MAX_BROWSERS = 1
+
+ def _continue_scenario(self, driver, *, missing_config=False, config_load_failure=False):
+ driver.get(f"{self.base_url}/web/trainer#cloud")
+ self.dismiss_onboarding(driver)
+ self._show_configured_cloud_dashboard(driver)
+ driver.execute_script(
+ """
+ window.__continueRequests = [];
+ window.__configLoadFailures = 0;
+ if (arguments[1]) {
+ const originalFetch = window.fetch;
+ window.fetch = (url, options) => {
+ if (new URL(url, window.location.origin).pathname === '/api/configs/test-config') {
+ window.__configLoadFailures++;
+ return Promise.resolve(new Response('{}', {status: 500}));
+ }
+ return originalFetch(url, options);
+ };
+ }
+ document.body.addEventListener('htmx:beforeRequest', event => {
+ if (event.detail.requestConfig.path === '/api/training/start') {
+ window.__continueRequests.push({
+ environment: Alpine.store('trainer').activeEnvironment,
+ parameters: {...event.detail.requestConfig.parameters},
+ });
+ event.preventDefault();
+ }
+ });
+ const comp = Alpine.$data(document.querySelector('#cloud-tab-content'));
+ comp.stopPolling();
+ comp.jobs = [{job_id: 'failed-job', job_type: 'local', provider: 'local',
+ status: 'failed', config_name: arguments[0], created_at: new Date().toISOString(),
+ metadata: {env_name: arguments[0], runtime_config: {
+ '--tracker_project_name': 'portraits', '--tracker_run_name': 'shared-run'
+ }}}];
+ comp.selectedJob = null;
+ window.__selectedJobs = 0;
+ comp.selectJob = () => { window.__selectedJobs++; };
+ """,
+ "missing-config" if missing_config else "test-config",
+ config_load_failure,
+ )
+ button = WebDriverWait(driver, 15).until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".job-continue-btn")))
+ name = driver.find_element(By.CSS_SELECTOR, ".job-name")
+ self.assertEqual(name.text, "portraits / shared-run")
+ self.assertEqual(name.get_attribute("title"), "portraits / shared-run")
+ driver.execute_script("Alpine.$data(document.querySelector('#cloud-tab-content')).jobSearchQuery = 'PORTRAITS';")
+ self.assertEqual(len(driver.find_elements(By.CSS_SELECTOR, ".job-continue-btn")), 1)
+ self.dismiss_onboarding(driver)
+ driver.execute_script("arguments[0].scrollIntoView({block: 'center'});", button)
+ if missing_config:
+ button.send_keys(Keys.ENTER)
+ else:
+ button.click()
+ if config_load_failure:
+ WebDriverWait(driver, 15).until(
+ lambda d: d.execute_script(
+ "return window.__configLoadFailures > 0 && Alpine.store('trainer').continuingJob === null;"
+ )
+ )
+ self.assertEqual(driver.execute_script("return window.__continueRequests"), [])
+ self.assertEqual(driver.execute_script("return Alpine.store('trainer').activeEnvironment"), "test-config")
+ self.assertIsNone(driver.execute_script("return Alpine.store('trainer').activeEnvironmentConfig"))
+ self.assertFalse(driver.find_element(By.CSS_SELECTOR, ".job-continue-btn").get_property("disabled"))
+ elif missing_config:
+ WebDriverWait(driver, 15).until(
+ lambda d: not d.find_element(By.CSS_SELECTOR, ".job-continue-btn").get_property("disabled")
+ )
+ self.assertEqual(driver.execute_script("return window.__continueRequests"), [])
+ self.assertEqual(driver.execute_script("return Alpine.store('trainer').activeEnvironment"), "default")
+ else:
+ WebDriverWait(driver, 20).until(lambda d: d.execute_script("return window.__continueRequests.length === 1"))
+ request = driver.execute_script("return window.__continueRequests[0]")
+ self.assertEqual(request["environment"], "test-config")
+ self.assertEqual(request["parameters"]["--resume_from_checkpoint"], "checkpoint-100")
+ self.assertEqual(driver.execute_script("return Alpine.store('trainer').activeTab"), "basic")
+ self.assertEqual(driver.execute_script("return window.__selectedJobs"), 0)
+
+ def test_continue_selects_failed_job_config_and_uses_run_flow(self):
+ self.with_sample_environment()
+ config_path = self.config_dir / "test-config" / "config.json"
+ config = json.loads(config_path.read_text())
+ config["--resume_from_checkpoint"] = "checkpoint-100"
+ config_path.write_text(json.dumps(config))
+ self.seed_defaults(active_config="default")
+ self.for_each_browser("continue_failed_local_job", lambda driver, _: self._continue_scenario(driver))
+
+ def test_continue_does_not_run_when_config_activation_fails(self):
+ self.seed_defaults(active_config="default")
+ self.for_each_browser(
+ "continue_missing_config", lambda driver, _: self._continue_scenario(driver, missing_config=True)
+ )
+
+ def test_continue_does_not_run_when_activated_config_cannot_be_loaded(self):
+ self.with_sample_environment()
+ self.seed_defaults(active_config="default")
+ self.for_each_browser(
+ "continue_config_load_failure",
+ lambda driver, _: self._continue_scenario(driver, config_load_failure=True),
+ )
+
+
if __name__ == "__main__":
unittest.main()