Replace expanded sequence arrays with compact descriptors

src/components/Shell.tsx

@@ -41,6 +41,7 @@ import {
   resolveParticipantConditions,
 } from '../utils/handleConditionLogic';
 import { StartupErrorScreen } from './StartupErrorScreen';
+import { createCompactSequenceDescriptor } from '../utils/sequenceDescriptor';
 
 type StartupStorageStatus = Pick<StorageEngine, 'getEngine' | 'isConnected'>;
 
@@ -305,12 +306,13 @@ export function Shell({ globalConfig }: { globalConfig: GlobalConfig }) {
 
         await storageEngine.saveConfig(activeConfig);
 
-        const sequenceArray = await storageEngine.getSequenceArray();
+        const sequenceArtifactHash = await activeHashPromise;
+        const sequenceArtifact = await storageEngine.getSequenceArtifact(sequenceArtifactHash);
 
-        if (!sequenceArray) {
-          const generatedSequenceArray = await generateSequenceArray(activeConfig);
-
-          await storageEngine.setSequenceArray(generatedSequenceArray);
+        if (!sequenceArtifact) {
+          await storageEngine.setSequenceDescriptor(
+            createCompactSequenceDescriptor(sequenceArtifactHash, activeConfig),
+          );
         }
 
         // Get or generate participant session

src/components/tests/Shell.spec.tsx

@@ -297,7 +297,7 @@ describe('Shell', () => {
     mockStorageEngine = {
       initializeStudyDb: vi.fn().mockResolvedValue(undefined),
       saveConfig: vi.fn().mockResolvedValue(undefined),
-      getSequenceArray: vi.fn().mockResolvedValue(['seq1']), // non-null → no setSequenceArray
+      getSequenceArtifact: vi.fn().mockResolvedValue(['seq1']),
       getModes: vi.fn().mockResolvedValue({ developmentModeEnabled: false, dataSharingEnabled: false, dataCollectionEnabled: true }),
       initializeParticipantSession: vi.fn().mockResolvedValue(baseSession),
       getParticipantCompletionStatus: vi.fn().mockResolvedValue(false),
@@ -312,14 +312,14 @@ describe('Shell', () => {
     await waitFor(() => expect(vi.mocked(studyStoreCreator)).toHaveBeenCalled(), { timeout: 3000 });
   });
 
-  test('calls setSequenceArray when getSequenceArray returns null', async () => {
+  test('publishes a compact descriptor when no sequence artifact exists', async () => {
     vi.mocked(getStudyConfig).mockResolvedValue(mockActiveConfig);
 
     mockStorageEngine = {
       initializeStudyDb: vi.fn().mockResolvedValue(undefined),
       saveConfig: vi.fn().mockResolvedValue(undefined),
-      getSequenceArray: vi.fn().mockResolvedValue(null), // null → calls setSequenceArray
-      setSequenceArray: vi.fn().mockResolvedValue(undefined),
+      getSequenceArtifact: vi.fn().mockResolvedValue(null),
+      setSequenceDescriptor: vi.fn().mockResolvedValue(undefined),
       getModes: vi.fn().mockResolvedValue({ developmentModeEnabled: false, dataSharingEnabled: false, dataCollectionEnabled: true }),
       initializeParticipantSession: vi.fn().mockResolvedValue(baseSession),
       getParticipantCompletionStatus: vi.fn().mockResolvedValue(false),
@@ -330,7 +330,13 @@ describe('Shell', () => {
     };
 
     render(<Shell globalConfig={globalConfig} />);
-    await waitFor(() => expect(mockStorageEngine!.setSequenceArray).toHaveBeenCalled(), { timeout: 3000 });
+    await waitFor(() => expect(mockStorageEngine!.setSequenceDescriptor).toHaveBeenCalledWith(
+      expect.objectContaining({
+        format: 'revisit-compact-sequence',
+        version: 1,
+        numSequences: 1000,
+      }),
+    ), { timeout: 3000 });
   });
 
   test('covers study condition update path', async () => {
@@ -340,7 +346,7 @@ describe('Shell', () => {
     mockStorageEngine = {
       initializeStudyDb: vi.fn().mockResolvedValue(undefined),
       saveConfig: vi.fn().mockResolvedValue(undefined),
-      getSequenceArray: vi.fn().mockResolvedValue(['seq1']),
+      getSequenceArtifact: vi.fn().mockResolvedValue(['seq1']),
       getModes: vi.fn().mockResolvedValue({ developmentModeEnabled: true, dataSharingEnabled: true, dataCollectionEnabled: true }),
       initializeParticipantSession: vi.fn().mockResolvedValue({
         ...baseSession,
@@ -366,7 +372,7 @@ describe('Shell', () => {
     mockStorageEngine = {
       initializeStudyDb: vi.fn().mockResolvedValue(undefined),
       saveConfig: vi.fn().mockResolvedValue(undefined),
-      getSequenceArray: vi.fn().mockResolvedValue(['seq1']),
+      getSequenceArtifact: vi.fn().mockResolvedValue(['seq1']),
       getModes: vi.fn().mockResolvedValue({ developmentModeEnabled: false, dataSharingEnabled: false, dataCollectionEnabled: true }),
       initializeParticipantSession: vi.fn().mockResolvedValue({
         ...baseSession,

src/storage/engines/types.ts

@@ -19,6 +19,13 @@ import {
   SnapshotParticipantCounts,
   calculateSnapshotParticipantCounts,
 } from './utils/snapshotParticipantCounts';
+import {
+  CompactSequenceDescriptor,
+  SequenceArtifact,
+  isCompactSequenceDescriptor,
+  parseCompactSequenceDescriptor,
+  resolveCompactSequence,
+} from '../../utils/sequenceDescriptor';
 
 export interface StoredUser {
   email: string | null,
@@ -90,7 +97,7 @@ const defaultStageColor = '#F05A30';
 export type StorageObjectType = 'sequenceArray' | 'participantData' | 'config' | string;
 export type StorageObject<T extends StorageObjectType> =
   T extends 'sequenceArray'
-  ? Sequence[]
+  ? SequenceArtifact
   : T extends 'participantData'
   ? ParticipantData
   : T extends 'config'
@@ -203,6 +210,8 @@ export abstract class StorageEngine {
 
   private assetUploadActivityVersion = 0;
 
+  private activeConfigHash: string | null = null;
+
   constructor(engine: typeof this.engine, testing: boolean) {
     this.engine = engine;
     this.testing = testing;
@@ -769,6 +778,7 @@ export abstract class StorageEngine {
     const currentConfigHash = await this._getCurrentConfigHash();
     // Hash the provided config
     const configHash = await hash(JSON.stringify(config));
+    this.activeConfigHash = configHash;
 
     // Skip saving config if the active config is already saved in storage
     if (currentConfigHash === configHash) {
@@ -786,17 +796,6 @@ export abstract class StorageEngine {
     );
     await this._cacheStorageObject(`configs/${configHash}`, 'config');
 
-    // Clear sequence array if the config has changed.
-    // Keep currentParticipantId so existing participant sessions can continue
-    // against their original participantConfigHash.
-    if (currentConfigHash && currentConfigHash !== configHash) {
-      try {
-        await this._deleteFromStorage('', 'sequenceArray');
-      } catch {
-        // pass, if this happens, we didn't have a sequence array yet
-      }
-    }
-
     if (currentConfigHash !== configHash) {
       await this._setCurrentConfigHash(configHash);
     }
@@ -867,7 +866,11 @@ export abstract class StorageEngine {
   // This function is one of the most critical functions in the storage engine.
   // It uses the notion of sequence intents and assignments to determine the current sequence for the participant.
   // It handles rejected participants and allows for reusing a rejected participant's sequence.
-  protected async _getSequence(conditions?: string[], bootstrapData?: ModesAndStageData) {
+  protected async _getSequence(
+    config: StudyConfig,
+    conditions?: string[],
+    bootstrapData?: ModesAndStageData,
+  ) {
     if (!this.currentParticipantId) {
       throw new Error('Participant not initialized');
     }
@@ -927,35 +930,55 @@ export abstract class StorageEngine {
     // Query all the intents to get a sequence and find our position in the queue
     sequenceAssignments = await this.getAllSequenceAssignments(this.studyId);
 
-    // Get the latin square
-    const sequenceArray = await this.getSequenceArray();
-    if (!sequenceArray) {
-      throw new Error('Latin square not initialized');
+    const configHash = await hash(JSON.stringify(config));
+    const sequenceArtifact = await this.getSequenceArtifact(configHash);
+    if (!sequenceArtifact) {
+      throw new Error('Study sequence is not initialized');
     }
 
-    // Get the current row
-    const intentIndex = sequenceAssignments.filter((assignment) => !assignment.rejected).findIndex(
-      (assignment) => assignment.participantId === this.currentParticipantId,
-    ) % sequenceArray.length;
-    if (sequenceArray.length === 0) {
-      throw new Error('Something really bad happened with sequence assignment');
+    let sequenceCount: number;
+    if (Array.isArray(sequenceArtifact)) {
+      sequenceCount = sequenceArtifact.length;
+    } else {
+      const descriptor = parseCompactSequenceDescriptor(sequenceArtifact);
+      if (descriptor.configHash !== configHash) {
+        throw new Error(
+          'The stored sequence descriptor does not match this study config. '
+          + 'Republish the study sequence.',
+        );
+      }
+      sequenceCount = descriptor.numSequences;
     }
-    // If index = -1, we probably have data collection disabled. Give a random assignment.
-    if (intentIndex === -1) {
-      return {
-        currentRow: sequenceArray[Math.floor(Math.random() * sequenceArray.length)],
-        creationIndex: 1,
-      };
+
+    if (sequenceCount === 0) {
+      throw new Error('Study sequence is empty');
+    }
+
+    let sequenceIndex = sequenceAssignments.filter((assignment) => !assignment.rejected).findIndex(
+      (assignment) => assignment.participantId === this.currentParticipantId,
+    );
+    const isUnassigned = sequenceIndex === -1;
+    if (isUnassigned) {
+      sequenceIndex = Math.floor(Math.random() * sequenceCount);
+    } else {
+      sequenceIndex %= sequenceCount;
     }
-    const currentRow = sequenceArray[intentIndex];
+
+    const currentRow = Array.isArray(sequenceArtifact)
+      ? sequenceArtifact[sequenceIndex]
+      : resolveCompactSequence(config, sequenceArtifact, sequenceIndex);
 
     if (!currentRow) {
-      throw new Error('Latin square is empty');
+      throw new Error('Study sequence is empty');
     }
 
     const creationSorted = sequenceAssignments.sort((a, b) => a.createdTime - b.createdTime);
 
-    const creationIndex = creationSorted.findIndex((assignment) => assignment.participantId === this.currentParticipantId) + 1;
+    const creationIndex = isUnassigned
+      ? 1
+      : creationSorted.findIndex(
+        (assignment) => assignment.participantId === this.currentParticipantId,
+      ) + 1;
 
     return { currentRow, creationIndex };
   }
@@ -1004,7 +1027,11 @@ export abstract class StorageEngine {
     const participantConfigHash = await hash(JSON.stringify(config));
     const parsedConditions = parseConditionParam(searchParams.condition);
     const conditions = parsedConditions.length > 0 ? parsedConditions : undefined;
-    const { currentRow, creationIndex } = await this._getSequence(conditions, { modes, stageData });
+    const { currentRow, creationIndex } = await this._getSequence(
+      config,
+      conditions,
+      { modes, stageData },
+    );
     this.participantData = {
       participantId: this.currentParticipantId,
       participantConfigHash,
@@ -1764,16 +1791,58 @@ export abstract class StorageEngine {
     });
   }
 
-  // Gets the sequence array from the storage engine.
-  async getSequenceArray() {
+  // Gets the versioned sequence artifact from the storage engine.
+  async getSequenceArtifact(configHash?: string) {
     await this.verifyStudyDatabase();
 
-    const sequenceArrayDocData = await this._getFromStorage(
-      '',
+    const resolvedConfigHash = configHash ?? this.activeConfigHash;
+    let sequenceArtifact = await this._getFromStorage(
+      resolvedConfigHash ? `sequenceArrays/${resolvedConfigHash}` : '',
       'sequenceArray',
     );
+    if (
+      resolvedConfigHash
+      && (
+        sequenceArtifact === null
+        || sequenceArtifact === undefined
+        || (
+          typeof sequenceArtifact === 'object'
+          && !Array.isArray(sequenceArtifact)
+          && Object.keys(sequenceArtifact).length === 0
+        )
+      )
+    ) {
+      const currentConfigHash = await this._getCurrentConfigHash();
+      if (currentConfigHash === null || currentConfigHash === resolvedConfigHash) {
+        sequenceArtifact = await this._getFromStorage('', 'sequenceArray');
+      }
+    }
 
-    return Array.isArray(sequenceArrayDocData) ? sequenceArrayDocData : null;
+    if (
+      sequenceArtifact === null
+      || sequenceArtifact === undefined
+      || (
+        typeof sequenceArtifact === 'object'
+        && !Array.isArray(sequenceArtifact)
+        && Object.keys(sequenceArtifact).length === 0
+      )
+    ) {
+      return null;
+    }
+    if (Array.isArray(sequenceArtifact)) {
+      return sequenceArtifact;
+    }
+    if (isCompactSequenceDescriptor(sequenceArtifact)) {
+      return parseCompactSequenceDescriptor(sequenceArtifact);
+    }
+
+    throw new Error('The stored sequence descriptor is corrupt. Republish the study sequence.');
+  }
+
+  // Gets a legacy expanded sequence array, if one is stored.
+  async getSequenceArray(configHash?: string) {
+    const sequenceArtifact = await this.getSequenceArtifact(configHash);
+    return Array.isArray(sequenceArtifact) ? sequenceArtifact : null;
   }
 
   // Sets the sequence array in the storage engine.
@@ -1783,6 +1852,19 @@ export abstract class StorageEngine {
     await this._pushToStorage('', 'sequenceArray', latinSquare);
   }
 
+  // Sets a compact versioned sequence descriptor in the storage engine.
+  async setSequenceDescriptor(descriptor: CompactSequenceDescriptor) {
+    await this.verifyStudyDatabase();
+
+    const parsedDescriptor = parseCompactSequenceDescriptor(descriptor);
+    this.activeConfigHash = parsedDescriptor.configHash;
+    await this._pushToStorage(
+      `sequenceArrays/${parsedDescriptor.configHash}`,
+      'sequenceArray',
+      parsedDescriptor,
+    );
+  }
+
   protected async __testingReset() {
     this.clearPendingParticipantDataWriteTimer();
     this.pendingParticipantDataWrite = undefined;
@@ -1794,6 +1876,7 @@ export abstract class StorageEngine {
     this.failedAssetRetryOperations.clear();
     this.pendingProgressDataWrite = undefined;
     this.assetUploadActivityVersion = 0;
+    this.activeConfigHash = null;
     this.participantData = undefined;
     if (this.studyId) {
       await this.clearCurrentParticipantId();